diff options
Diffstat (limited to 'build/android')
| -rw-r--r-- | build/android/pylib/base_test_runner.py | 22 | ||||
| -rw-r--r-- | build/android/pylib/base_test_sharder.py | 9 | ||||
| -rw-r--r-- | build/android/pylib/chrome_test_server_spawner.py | 6 | ||||
| -rw-r--r-- | build/android/pylib/cmd_helper.py | 19 | ||||
| -rw-r--r-- | build/android/pylib/forwarder.py | 244 | ||||
| -rw-r--r-- | build/android/pylib/ports.py | 28 | ||||
| -rw-r--r-- | build/android/pylib/run_java_tests.py | 6 | ||||
| -rwxr-xr-x | build/android/run_tests.py | 2 |
8 files changed, 164 insertions, 172 deletions
diff --git a/build/android/pylib/base_test_runner.py b/build/android/pylib/base_test_runner.py index 619bc6e..5355633 100644 --- a/build/android/pylib/base_test_runner.py +++ b/build/android/pylib/base_test_runner.py @@ -82,7 +82,7 @@ class BaseTestRunner(object): def SetUp(self): """Called before tests run.""" - pass + Forwarder.KillDevice(self.adb) def HasTests(self): """Whether the test suite has tests to run.""" @@ -129,6 +129,13 @@ class BaseTestRunner(object): self.StartForwarderForHttpServer() return (self._forwarder_device_port, self._http_server.port) + def _CreateAndRunForwarder( + self, adb, port_pairs, tool, host_name, build_type): + """Creates and run a forwarder.""" + forwarder = Forwarder(adb, build_type) + forwarder.Run(port_pairs, tool, host_name) + return forwarder + def StartForwarder(self, port_pairs): """Starts TCP traffic forwarding for the given |port_pairs|. @@ -137,7 +144,7 @@ class BaseTestRunner(object): """ if self._forwarder: self._forwarder.Close() - self._forwarder = Forwarder( + self._forwarder = self._CreateAndRunForwarder( self.adb, port_pairs, self.tool, '127.0.0.1', self.build_type) def StartForwarderForHttpServer(self): @@ -162,14 +169,7 @@ class BaseTestRunner(object): # Forwarders should be killed before the actual servers they're forwarding # to as they are clients potentially with open connections and to allow for # proper hand-shake/shutdown. - if self._forwarder or self._spawner_forwarder: - # Kill all forwarders on the device and then kill the process on the host - # (if it exists) - self.adb.KillAll('device_forwarder') - if self._forwarder: - self._forwarder.Close() - if self._spawner_forwarder: - self._spawner_forwarder.Close() + Forwarder.KillDevice(self.adb) if self._http_server: self._http_server.ShutdownHttpServer() if self._spawning_server: @@ -204,7 +204,7 @@ class BaseTestRunner(object): logging.error(';'.join(error_msgs)) raise Exception('Can not start the test spawner server.') self._PushTestServerPortInfoToDevice() - self._spawner_forwarder = Forwarder( + self._spawner_forwarder = self._CreateAndRunForwarder( self.adb, [(self.test_server_spawner_port, self.test_server_spawner_port)], self.tool, '127.0.0.1', self.build_type) diff --git a/build/android/pylib/base_test_sharder.py b/build/android/pylib/base_test_sharder.py index b8d03c83..fa05abec 100644 --- a/build/android/pylib/base_test_sharder.py +++ b/build/android/pylib/base_test_sharder.py @@ -8,6 +8,7 @@ import logging import multiprocessing from android_commands import errors +from forwarder import Forwarder from test_result import TestResults @@ -24,6 +25,7 @@ def _ShardedTestRunnable(test): except SystemExit: return TestResults() + def SetTestsContainer(tests_container): """Sets tests container. @@ -42,12 +44,13 @@ class BaseTestSharder(object): # See more in SetTestsContainer. tests_container = None - def __init__(self, attached_devices): + def __init__(self, attached_devices, build_type='Debug'): self.attached_devices = attached_devices # Worst case scenario: a device will drop offline per run, so we need # to retry until we're out of devices. self.retries = len(self.attached_devices) self.tests = [] + self.build_type = build_type def CreateShardedTestRunner(self, device, index): """Factory function to create a suite-specific test runner. @@ -63,11 +66,11 @@ class BaseTestSharder(object): def SetupSharding(self, tests): """Called before starting the shards.""" - pass + Forwarder.KillHost(self.build_type) def OnTestsCompleted(self, test_runners, test_results): """Notifies that we completed the tests.""" - pass + Forwarder.KillHost(self.build_type) def RunShardedTests(self): """Runs the tests in all connected devices. diff --git a/build/android/pylib/chrome_test_server_spawner.py b/build/android/pylib/chrome_test_server_spawner.py index 512a609..8206ca0 100644 --- a/build/android/pylib/chrome_test_server_spawner.py +++ b/build/android/pylib/chrome_test_server_spawner.py @@ -217,9 +217,9 @@ class TestServerThread(threading.Thread): else: self.is_ready = _CheckPortStatus(self.host_port, True) if self.is_ready: - self._test_server_forwarder = Forwarder( - self.adb, [(0, self.host_port)], self.tool, '127.0.0.1', - self.build_type) + self._test_server_forwarder = Forwarder(self.adb, self.build_type) + self._test_server_forwarder.Run( + [(0, self.host_port)], self.tool, '127.0.0.1') # Check whether the forwarder is ready on the device. self.is_ready = False device_port = self._test_server_forwarder.DevicePortForHostPort( diff --git a/build/android/pylib/cmd_helper.py b/build/android/pylib/cmd_helper.py index 8b50130..46b6981 100644 --- a/build/android/pylib/cmd_helper.py +++ b/build/android/pylib/cmd_helper.py @@ -40,11 +40,28 @@ def GetCmdOutput(args, cwd=None, shell=False): Captures and returns the command's stdout. Prints the command's stderr to logger (which defaults to stdout). """ + (_, output) = GetCmdStatusAndOutput(args, cwd, shell) + return output + +def GetCmdStatusAndOutput(args, cwd=None, shell=False): + """Executes a subprocess and returns its exit code and output. + + Args: + args: A string or a sequence of program arguments. The program to execute is + the string or the first item in the args sequence. + cwd: If not None, the subprocess's current directory will be changed to + |cwd| before it's executed. + shell: Whether to execute args as a shell command. + + Returns: + The tuple (exit code, output). + """ logging.info(str(args) + ' ' + (cwd or '')) p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell) stdout, stderr = p.communicate() + exit_code = p.returncode if stderr: logging.critical(stderr) logging.info(stdout[:4096]) # Truncate output longer than 4k. - return stdout + return (exit_code, stdout) diff --git a/build/android/pylib/forwarder.py b/build/android/pylib/forwarder.py index bc41db3..a0dc5f2 100644 --- a/build/android/pylib/forwarder.py +++ b/build/android/pylib/forwarder.py @@ -15,6 +15,11 @@ import ports from pylib import pexpect + +def _MakeBinaryPath(build_type, binary_name): + return os.path.join(constants.CHROME_DIR, 'out', build_type, binary_name) + + class Forwarder(object): """Class to manage port forwards from the device to the host.""" @@ -24,13 +29,27 @@ class Forwarder(object): _DEVICE_ADB_CONTROL_PORT = 'chrome_device_forwarder' _TIMEOUT_SECS = 30 - def __init__(self, adb, port_pairs, tool, host_name, build_type): + def __init__(self, adb, build_type): """Forwards TCP ports on the device back to the host. Works like adb forward, but in reverse. Args: adb: Instance of AndroidCommands for talking to the device. + build_type: 'Release' or 'Debug'. + """ + assert build_type in ('Release', 'Debug') + self._adb = adb + self._host_to_device_port_map = dict() + self._device_process = None + self._host_forwarder_path = _MakeBinaryPath(build_type, 'host_forwarder') + self._device_forwarder_path = _MakeBinaryPath( + build_type, 'device_forwarder') + + def Run(self, port_pairs, tool, host_name): + """Runs the forwarder. + + Args: port_pairs: A list of tuples (device_port, host_port) to forward. Note that you can specify 0 as a device_port, in which case a port will by dynamically assigned on the device. You can @@ -40,159 +59,104 @@ class Forwarder(object): forwarder (see valgrind_tools.py). host_name: Address to forward to, must be addressable from the host machine. Usually use loopback '127.0.0.1'. - build_type: 'Release' or 'Debug'. Raises: Exception on failure to forward the port. """ - self._adb = adb - self._host_to_device_port_map = dict() - self._host_process = None - self._device_process = None - self._adb_forward_process = None - - self._host_adb_control_port = ports.AllocateTestServerPort() - if not self._host_adb_control_port: + host_adb_control_port = ports.AllocateTestServerPort() + if not host_adb_control_port: raise Exception('Failed to allocate a TCP port in the host machine.') - adb.PushIfNeeded( - os.path.join(constants.CHROME_DIR, 'out', build_type, - 'device_forwarder'), - Forwarder._DEVICE_FORWARDER_PATH) - self._host_forwarder_path = os.path.join(constants.CHROME_DIR, - 'out', - build_type, - 'host_forwarder') - forward_string = ['%d:%d:%s' % - (device, host, host_name) for device, host in port_pairs] - logging.info('Forwarding ports: %s', forward_string) - timeout_sec = 5 - host_pattern = 'host_forwarder.*' + ' '.join(forward_string) - # TODO(felipeg): Rather than using a blocking kill() here, the device - # forwarder could try to bind the Unix Domain Socket until it succeeds or - # while it fails because the socket is already bound (with appropriate - # timeout handling obviously). - self._KillHostForwarderBlocking(host_pattern, timeout_sec) - self._KillDeviceForwarderBlocking(timeout_sec) - self._adb_forward_process = pexpect.spawn( - 'adb', ['-s', - adb._adb.GetSerialNumber(), - 'forward', - 'tcp:%s' % self._host_adb_control_port, - 'localabstract:%s' % Forwarder._DEVICE_ADB_CONTROL_PORT]) - self._device_process = pexpect.spawn( - 'adb', ['-s', - adb._adb.GetSerialNumber(), - 'shell', - '%s %s -D --adb_sock=%s' % ( - tool.GetUtilWrapper(), - Forwarder._DEVICE_FORWARDER_PATH, - Forwarder._DEVICE_ADB_CONTROL_PORT)]) - - device_success_re = re.compile('Starting Device Forwarder.') - device_failure_re = re.compile('.*:ERROR:(.*)') - index = self._device_process.expect([device_success_re, - device_failure_re, - pexpect.EOF, - pexpect.TIMEOUT], - Forwarder._TIMEOUT_SECS) - if index == 1: - # Failure - error_msg = str(self._device_process.match.group(1)) - logging.error(self._device_process.before) - self._CloseProcess() - raise Exception('Failed to start Device Forwarder with Error: %s' % - error_msg) - elif index == 2: - logging.error(self._device_process.before) - self._CloseProcess() - raise Exception('Unexpected EOF while trying to start Device Forwarder.') - elif index == 3: - logging.error(self._device_process.before) - self._CloseProcess() - raise Exception('Timeout while trying start Device Forwarder') - - self._host_process = pexpect.spawn(self._host_forwarder_path, - ['--adb_port=%s' % ( - self._host_adb_control_port)] + - forward_string) - - # Read the output of the command to determine which device ports where - # forwarded to which host ports (necessary if - host_success_re = re.compile('Forwarding device port (\d+) to host (\d+):') - host_failure_re = re.compile('Couldn\'t start forwarder server for port ' - 'spec: (\d+):(\d+)') - for pair in port_pairs: - index = self._host_process.expect([host_success_re, - host_failure_re, - pexpect.EOF, - pexpect.TIMEOUT], - Forwarder._TIMEOUT_SECS) - if index == 0: - # Success - device_port = int(self._host_process.match.group(1)) - host_port = int(self._host_process.match.group(2)) - self._host_to_device_port_map[host_port] = device_port - logging.info("Forwarding device port: %d to host port: %d." % - (device_port, host_port)) - elif index == 1: - # Failure - device_port = int(self._host_process.match.group(1)) - host_port = int(self._host_process.match.group(2)) - self._CloseProcess() - raise Exception('Failed to forward port %d to %d' % (device_port, - host_port)) + self._adb.PushIfNeeded(self._device_forwarder_path, + Forwarder._DEVICE_FORWARDER_PATH) + redirection_commands = [ + '%d:%d:%d:%s' % (host_adb_control_port, device, host, + host_name) for device, host in port_pairs] + logging.info('Command format: <ADB port>:<Device port>' + + '[:<Forward to port>:<Forward to address>]') + logging.info('Forwarding using commands: %s', redirection_commands) + if cmd_helper.RunCmd( + ['adb', '-s', self._adb._adb.GetSerialNumber(), 'forward', + 'tcp:%s' % host_adb_control_port, + 'localabstract:%s' % Forwarder._DEVICE_ADB_CONTROL_PORT]) != 0: + raise Exception('Error while running adb forward.') + + if not self._adb.ExtractPid('device_forwarder'): + # TODO(pliard): Get rid of pexpect here and make device_forwarder a daemon + # with a blocking CLI process that exits with a proper exit code and not + # while the daemon is still setting up. This would be similar to how + # host_forwarder works. + self._device_process = pexpect.spawn( + 'adb', ['-s', + self._adb._adb.GetSerialNumber(), + 'shell', + '%s %s -D --adb_sock=%s' % ( + tool.GetUtilWrapper(), + Forwarder._DEVICE_FORWARDER_PATH, + Forwarder._DEVICE_ADB_CONTROL_PORT)]) + device_success_re = re.compile('Starting Device Forwarder.') + device_failure_re = re.compile('.*:ERROR:(.*)') + index = self._device_process.expect([device_success_re, + device_failure_re, + pexpect.EOF, + pexpect.TIMEOUT], + Forwarder._TIMEOUT_SECS) + if index == 1: + error_msg = str(self._device_process.match.group(1)) + logging.error(self._device_process.before) + self._device_process.close() + raise Exception('Failed to start Device Forwarder with Error: %s' % + error_msg) elif index == 2: - logging.error(self._host_process.before) - self._CloseProcess() - raise Exception('Unexpected EOF while trying to forward ports %s' % - port_pairs) + logging.error(self._device_process.before) + self._device_process.close() + raise Exception( + 'Unexpected EOF while trying to start Device Forwarder.') elif index == 3: - logging.error(self._host_process.before) - self._CloseProcess() - raise Exception('Timeout while trying to forward ports %s' % port_pairs) - - def _KillHostForwarderBlocking(self, host_pattern, timeout_sec): - """Kills any existing host forwarders using the provided pattern. - - Note that this waits until the process terminates. - """ - cmd_helper.RunCmd(['pkill', '-f', host_pattern]) - elapsed = 0 - wait_period = 0.1 - while not cmd_helper.RunCmd(['pgrep', '-f', host_pattern]) and ( - elapsed < timeout_sec): - time.sleep(wait_period) - elapsed += wait_period - if elapsed >= timeout_sec: - raise Exception('Timed out while killing ' + host_pattern) - - def _KillDeviceForwarderBlocking(self, timeout_sec): - """Kills any existing device forwarders. - - Note that this waits until the process terminates. - """ - processes_killed = self._adb.KillAllBlocking( - 'device_forwarder', timeout_sec) + logging.error(self._device_process.before) + self._device_process.close() + raise Exception('Timeout while trying start Device Forwarder') + + for redirection_command in redirection_commands: + (exit_code, output) = cmd_helper.GetCmdStatusAndOutput( + [self._host_forwarder_path, redirection_command]) + if exit_code != 0: + raise Exception('%s exited with %d: %s' % (self._host_forwarder_path, + exit_code, output)) + tokens = output.split(':') + if len(tokens) != 2: + raise Exception('Unexpected host forwarder output "%s", ' + + 'expected "device_port:host_port"' % output) + device_port = int(tokens[0]) + host_port = int(tokens[1]) + self._host_to_device_port_map[host_port] = device_port + logging.info('Forwarding device port: %d to host port: %d.', device_port, + host_port) + + @staticmethod + def KillHost(build_type): + host_forwarder_path = _MakeBinaryPath(build_type, 'host_forwarder') + (exit_code, output) = cmd_helper.GetCmdStatusAndOutput( + [host_forwarder_path, 'kill-server']) + if exit_code != 0: + raise Exception('%s exited with %d: %s' % (host_forwarder_path, + exit_code, output)) + + @staticmethod + def KillDevice(adb): + logging.info('Killing device_forwarder.') + timeout_sec = 5 + processes_killed = adb.KillAllBlocking('device_forwarder', timeout_sec) if not processes_killed: - pids = self._adb.ExtractPid('device_forwarder') + pids = adb.ExtractPid('device_forwarder') if pids: raise Exception('Timed out while killing device_forwarder') - def _CloseProcess(self): - if self._host_process: - self._host_process.close() - if self._device_process: - self._device_process.close() - if self._adb_forward_process: - self._adb_forward_process.close() - self._host_process = None - self._device_process = None - self._adb_forward_process = None - def DevicePortForHostPort(self, host_port): """Get the device port that corresponds to a given host port.""" return self._host_to_device_port_map.get(host_port) def Close(self): """Terminate the forwarder process.""" - self._CloseProcess() + if self._device_process: + self._device_process.close() + self._device_process = None diff --git a/build/android/pylib/ports.py b/build/android/pylib/ports.py index 06b9f58..74c84c1 100644 --- a/build/android/pylib/ports.py +++ b/build/android/pylib/ports.py @@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Functions that deals with local and device ports.""" +"""Functions that deal with local and device ports.""" import contextlib import fcntl @@ -17,12 +17,13 @@ import cmd_helper import constants -#The following two methods are used to allocate the port source for various -# types of test servers. Because some net relates tests can be run on shards -# at same time, it's important to have a mechanism to allocate the port process -# safe. In here, we implement the safe port allocation by leveraging flock. +# The following two methods are used to allocate the port source for various +# types of test servers. Because some net-related tests can be run on shards at +# same time, it's important to have a mechanism to allocate the port +# process-safe. In here, we implement the safe port allocation by leveraging +# flock. def ResetTestServerPortAllocation(): - """Reset the port allocation to start from TEST_SERVER_PORT_FIRST. + """Resets the port allocation to start from TEST_SERVER_PORT_FIRST. Returns: Returns True if reset successes. Otherwise returns False. @@ -39,7 +40,7 @@ def ResetTestServerPortAllocation(): def AllocateTestServerPort(): - """Allocate a port incrementally. + """Allocates a port incrementally. Returns: Returns a valid port which should be in between TEST_SERVER_PORT_FIRST and @@ -90,10 +91,12 @@ def IsHostPortUsed(host_port): Returns: True if the port on host is already used, otherwise returns False. """ - port_info = '(127\.0\.0\.1)|(localhost)\:%d' % host_port - # TODO(jnd): Find a better way to filter the port. + port_info = '(\*)|(127\.0\.0\.1)|(localhost):%d' % host_port + # TODO(jnd): Find a better way to filter the port. Note that connecting to the + # socket and closing it would leave it in the TIME_WAIT state. Setting + # SO_LINGER on it and then closing it makes the Python HTTP server crash. re_port = re.compile(port_info, re.MULTILINE) - if re_port.findall(cmd_helper.GetCmdOutput(['lsof', '-nPi:%d' % host_port])): + if re_port.search(cmd_helper.GetCmdOutput(['lsof', '-nPi:%d' % host_port])): return True return False @@ -115,6 +118,11 @@ def IsDevicePortUsed(adb, device_port, state=''): for single_connect in netstat_results: # Column 3 is the local address which we want to check with. connect_results = single_connect.split() + if connect_results[0] != 'tcp': + continue + if len(connect_results) < 6: + raise Exception('Unexpected format while parsing netstat line: ' + + single_connect) is_state_match = connect_results[5] == state if state else True if connect_results[3] == base_url and is_state_match: return True diff --git a/build/android/pylib/run_java_tests.py b/build/android/pylib/run_java_tests.py index 62947fa..4e61e2d 100644 --- a/build/android/pylib/run_java_tests.py +++ b/build/android/pylib/run_java_tests.py @@ -268,8 +268,8 @@ class TestRunner(BaseTestRunner): # We need to remember which ports the HTTP server is using, since the # forwarder will stomp on them otherwise. port_pairs.append(http_server_ports) - self.forwarder = Forwarder( - self.adb, port_pairs, self.tool, '127.0.0.1', self.build_type) + self.forwarder = Forwarder(self.adb, self.build_type) + self.forwarder.Run(port_pairs, self.tool, '127.0.0.1') self.CopyTestFilesOnce() self.flags.AddFlags(['--enable-test-intents']) @@ -490,7 +490,7 @@ class TestSharder(BaseTestSharder): """Responsible for sharding the tests on the connected devices.""" def __init__(self, attached_devices, options, tests, apks): - BaseTestSharder.__init__(self, attached_devices) + BaseTestSharder.__init__(self, attached_devices, options.build_type) self.options = options self.tests = tests self.apks = apks diff --git a/build/android/run_tests.py b/build/android/run_tests.py index 67cfbbf..b4a8bf1 100755 --- a/build/android/run_tests.py +++ b/build/android/run_tests.py @@ -196,7 +196,7 @@ class TestSharder(BaseTestSharder): test_arguments, timeout, rebaseline, performance_test, cleanup_test_files, tool, log_dump_name, fast_and_loose, build_type): - BaseTestSharder.__init__(self, attached_devices) + BaseTestSharder.__init__(self, attached_devices, build_type) self.test_suite = test_suite self.test_suite_basename = os.path.basename(test_suite) self.gtest_filter = gtest_filter or '' |