summaryrefslogtreecommitdiffstats
path: root/build/android/pylib
diff options
context:
space:
mode:
Diffstat (limited to 'build/android/pylib')
-rw-r--r--build/android/pylib/base_test_runner.py22
-rw-r--r--build/android/pylib/base_test_sharder.py9
-rw-r--r--build/android/pylib/chrome_test_server_spawner.py6
-rw-r--r--build/android/pylib/cmd_helper.py19
-rw-r--r--build/android/pylib/forwarder.py244
-rw-r--r--build/android/pylib/ports.py28
-rw-r--r--build/android/pylib/run_java_tests.py6
7 files changed, 163 insertions, 171 deletions
diff --git a/build/android/pylib/base_test_runner.py b/build/android/pylib/base_test_runner.py
index 619bc6e..5355633 100644
--- a/build/android/pylib/base_test_runner.py
+++ b/build/android/pylib/base_test_runner.py
@@ -82,7 +82,7 @@ class BaseTestRunner(object):
def SetUp(self):
"""Called before tests run."""
- pass
+ Forwarder.KillDevice(self.adb)
def HasTests(self):
"""Whether the test suite has tests to run."""
@@ -129,6 +129,13 @@ class BaseTestRunner(object):
self.StartForwarderForHttpServer()
return (self._forwarder_device_port, self._http_server.port)
+ def _CreateAndRunForwarder(
+ self, adb, port_pairs, tool, host_name, build_type):
+ """Creates and run a forwarder."""
+ forwarder = Forwarder(adb, build_type)
+ forwarder.Run(port_pairs, tool, host_name)
+ return forwarder
+
def StartForwarder(self, port_pairs):
"""Starts TCP traffic forwarding for the given |port_pairs|.
@@ -137,7 +144,7 @@ class BaseTestRunner(object):
"""
if self._forwarder:
self._forwarder.Close()
- self._forwarder = Forwarder(
+ self._forwarder = self._CreateAndRunForwarder(
self.adb, port_pairs, self.tool, '127.0.0.1', self.build_type)
def StartForwarderForHttpServer(self):
@@ -162,14 +169,7 @@ class BaseTestRunner(object):
# Forwarders should be killed before the actual servers they're forwarding
# to as they are clients potentially with open connections and to allow for
# proper hand-shake/shutdown.
- if self._forwarder or self._spawner_forwarder:
- # Kill all forwarders on the device and then kill the process on the host
- # (if it exists)
- self.adb.KillAll('device_forwarder')
- if self._forwarder:
- self._forwarder.Close()
- if self._spawner_forwarder:
- self._spawner_forwarder.Close()
+ Forwarder.KillDevice(self.adb)
if self._http_server:
self._http_server.ShutdownHttpServer()
if self._spawning_server:
@@ -204,7 +204,7 @@ class BaseTestRunner(object):
logging.error(';'.join(error_msgs))
raise Exception('Can not start the test spawner server.')
self._PushTestServerPortInfoToDevice()
- self._spawner_forwarder = Forwarder(
+ self._spawner_forwarder = self._CreateAndRunForwarder(
self.adb,
[(self.test_server_spawner_port, self.test_server_spawner_port)],
self.tool, '127.0.0.1', self.build_type)
diff --git a/build/android/pylib/base_test_sharder.py b/build/android/pylib/base_test_sharder.py
index b8d03c83..fa05abec 100644
--- a/build/android/pylib/base_test_sharder.py
+++ b/build/android/pylib/base_test_sharder.py
@@ -8,6 +8,7 @@ import logging
import multiprocessing
from android_commands import errors
+from forwarder import Forwarder
from test_result import TestResults
@@ -24,6 +25,7 @@ def _ShardedTestRunnable(test):
except SystemExit:
return TestResults()
+
def SetTestsContainer(tests_container):
"""Sets tests container.
@@ -42,12 +44,13 @@ class BaseTestSharder(object):
# See more in SetTestsContainer.
tests_container = None
- def __init__(self, attached_devices):
+ def __init__(self, attached_devices, build_type='Debug'):
self.attached_devices = attached_devices
# Worst case scenario: a device will drop offline per run, so we need
# to retry until we're out of devices.
self.retries = len(self.attached_devices)
self.tests = []
+ self.build_type = build_type
def CreateShardedTestRunner(self, device, index):
"""Factory function to create a suite-specific test runner.
@@ -63,11 +66,11 @@ class BaseTestSharder(object):
def SetupSharding(self, tests):
"""Called before starting the shards."""
- pass
+ Forwarder.KillHost(self.build_type)
def OnTestsCompleted(self, test_runners, test_results):
"""Notifies that we completed the tests."""
- pass
+ Forwarder.KillHost(self.build_type)
def RunShardedTests(self):
"""Runs the tests in all connected devices.
diff --git a/build/android/pylib/chrome_test_server_spawner.py b/build/android/pylib/chrome_test_server_spawner.py
index 512a609..8206ca0 100644
--- a/build/android/pylib/chrome_test_server_spawner.py
+++ b/build/android/pylib/chrome_test_server_spawner.py
@@ -217,9 +217,9 @@ class TestServerThread(threading.Thread):
else:
self.is_ready = _CheckPortStatus(self.host_port, True)
if self.is_ready:
- self._test_server_forwarder = Forwarder(
- self.adb, [(0, self.host_port)], self.tool, '127.0.0.1',
- self.build_type)
+ self._test_server_forwarder = Forwarder(self.adb, self.build_type)
+ self._test_server_forwarder.Run(
+ [(0, self.host_port)], self.tool, '127.0.0.1')
# Check whether the forwarder is ready on the device.
self.is_ready = False
device_port = self._test_server_forwarder.DevicePortForHostPort(
diff --git a/build/android/pylib/cmd_helper.py b/build/android/pylib/cmd_helper.py
index 8b50130..46b6981 100644
--- a/build/android/pylib/cmd_helper.py
+++ b/build/android/pylib/cmd_helper.py
@@ -40,11 +40,28 @@ def GetCmdOutput(args, cwd=None, shell=False):
Captures and returns the command's stdout.
Prints the command's stderr to logger (which defaults to stdout).
"""
+ (_, output) = GetCmdStatusAndOutput(args, cwd, shell)
+ return output
+
+def GetCmdStatusAndOutput(args, cwd=None, shell=False):
+ """Executes a subprocess and returns its exit code and output.
+
+ Args:
+ args: A string or a sequence of program arguments. The program to execute is
+ the string or the first item in the args sequence.
+ cwd: If not None, the subprocess's current directory will be changed to
+ |cwd| before it's executed.
+ shell: Whether to execute args as a shell command.
+
+ Returns:
+ The tuple (exit code, output).
+ """
logging.info(str(args) + ' ' + (cwd or ''))
p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=shell)
stdout, stderr = p.communicate()
+ exit_code = p.returncode
if stderr:
logging.critical(stderr)
logging.info(stdout[:4096]) # Truncate output longer than 4k.
- return stdout
+ return (exit_code, stdout)
diff --git a/build/android/pylib/forwarder.py b/build/android/pylib/forwarder.py
index bc41db3..a0dc5f2 100644
--- a/build/android/pylib/forwarder.py
+++ b/build/android/pylib/forwarder.py
@@ -15,6 +15,11 @@ import ports
from pylib import pexpect
+
+def _MakeBinaryPath(build_type, binary_name):
+ return os.path.join(constants.CHROME_DIR, 'out', build_type, binary_name)
+
+
class Forwarder(object):
"""Class to manage port forwards from the device to the host."""
@@ -24,13 +29,27 @@ class Forwarder(object):
_DEVICE_ADB_CONTROL_PORT = 'chrome_device_forwarder'
_TIMEOUT_SECS = 30
- def __init__(self, adb, port_pairs, tool, host_name, build_type):
+ def __init__(self, adb, build_type):
"""Forwards TCP ports on the device back to the host.
Works like adb forward, but in reverse.
Args:
adb: Instance of AndroidCommands for talking to the device.
+ build_type: 'Release' or 'Debug'.
+ """
+ assert build_type in ('Release', 'Debug')
+ self._adb = adb
+ self._host_to_device_port_map = dict()
+ self._device_process = None
+ self._host_forwarder_path = _MakeBinaryPath(build_type, 'host_forwarder')
+ self._device_forwarder_path = _MakeBinaryPath(
+ build_type, 'device_forwarder')
+
+ def Run(self, port_pairs, tool, host_name):
+ """Runs the forwarder.
+
+ Args:
port_pairs: A list of tuples (device_port, host_port) to forward. Note
that you can specify 0 as a device_port, in which case a
port will by dynamically assigned on the device. You can
@@ -40,159 +59,104 @@ class Forwarder(object):
forwarder (see valgrind_tools.py).
host_name: Address to forward to, must be addressable from the
host machine. Usually use loopback '127.0.0.1'.
- build_type: 'Release' or 'Debug'.
Raises:
Exception on failure to forward the port.
"""
- self._adb = adb
- self._host_to_device_port_map = dict()
- self._host_process = None
- self._device_process = None
- self._adb_forward_process = None
-
- self._host_adb_control_port = ports.AllocateTestServerPort()
- if not self._host_adb_control_port:
+ host_adb_control_port = ports.AllocateTestServerPort()
+ if not host_adb_control_port:
raise Exception('Failed to allocate a TCP port in the host machine.')
- adb.PushIfNeeded(
- os.path.join(constants.CHROME_DIR, 'out', build_type,
- 'device_forwarder'),
- Forwarder._DEVICE_FORWARDER_PATH)
- self._host_forwarder_path = os.path.join(constants.CHROME_DIR,
- 'out',
- build_type,
- 'host_forwarder')
- forward_string = ['%d:%d:%s' %
- (device, host, host_name) for device, host in port_pairs]
- logging.info('Forwarding ports: %s', forward_string)
- timeout_sec = 5
- host_pattern = 'host_forwarder.*' + ' '.join(forward_string)
- # TODO(felipeg): Rather than using a blocking kill() here, the device
- # forwarder could try to bind the Unix Domain Socket until it succeeds or
- # while it fails because the socket is already bound (with appropriate
- # timeout handling obviously).
- self._KillHostForwarderBlocking(host_pattern, timeout_sec)
- self._KillDeviceForwarderBlocking(timeout_sec)
- self._adb_forward_process = pexpect.spawn(
- 'adb', ['-s',
- adb._adb.GetSerialNumber(),
- 'forward',
- 'tcp:%s' % self._host_adb_control_port,
- 'localabstract:%s' % Forwarder._DEVICE_ADB_CONTROL_PORT])
- self._device_process = pexpect.spawn(
- 'adb', ['-s',
- adb._adb.GetSerialNumber(),
- 'shell',
- '%s %s -D --adb_sock=%s' % (
- tool.GetUtilWrapper(),
- Forwarder._DEVICE_FORWARDER_PATH,
- Forwarder._DEVICE_ADB_CONTROL_PORT)])
-
- device_success_re = re.compile('Starting Device Forwarder.')
- device_failure_re = re.compile('.*:ERROR:(.*)')
- index = self._device_process.expect([device_success_re,
- device_failure_re,
- pexpect.EOF,
- pexpect.TIMEOUT],
- Forwarder._TIMEOUT_SECS)
- if index == 1:
- # Failure
- error_msg = str(self._device_process.match.group(1))
- logging.error(self._device_process.before)
- self._CloseProcess()
- raise Exception('Failed to start Device Forwarder with Error: %s' %
- error_msg)
- elif index == 2:
- logging.error(self._device_process.before)
- self._CloseProcess()
- raise Exception('Unexpected EOF while trying to start Device Forwarder.')
- elif index == 3:
- logging.error(self._device_process.before)
- self._CloseProcess()
- raise Exception('Timeout while trying start Device Forwarder')
-
- self._host_process = pexpect.spawn(self._host_forwarder_path,
- ['--adb_port=%s' % (
- self._host_adb_control_port)] +
- forward_string)
-
- # Read the output of the command to determine which device ports where
- # forwarded to which host ports (necessary if
- host_success_re = re.compile('Forwarding device port (\d+) to host (\d+):')
- host_failure_re = re.compile('Couldn\'t start forwarder server for port '
- 'spec: (\d+):(\d+)')
- for pair in port_pairs:
- index = self._host_process.expect([host_success_re,
- host_failure_re,
- pexpect.EOF,
- pexpect.TIMEOUT],
- Forwarder._TIMEOUT_SECS)
- if index == 0:
- # Success
- device_port = int(self._host_process.match.group(1))
- host_port = int(self._host_process.match.group(2))
- self._host_to_device_port_map[host_port] = device_port
- logging.info("Forwarding device port: %d to host port: %d." %
- (device_port, host_port))
- elif index == 1:
- # Failure
- device_port = int(self._host_process.match.group(1))
- host_port = int(self._host_process.match.group(2))
- self._CloseProcess()
- raise Exception('Failed to forward port %d to %d' % (device_port,
- host_port))
+ self._adb.PushIfNeeded(self._device_forwarder_path,
+ Forwarder._DEVICE_FORWARDER_PATH)
+ redirection_commands = [
+ '%d:%d:%d:%s' % (host_adb_control_port, device, host,
+ host_name) for device, host in port_pairs]
+ logging.info('Command format: <ADB port>:<Device port>' +
+ '[:<Forward to port>:<Forward to address>]')
+ logging.info('Forwarding using commands: %s', redirection_commands)
+ if cmd_helper.RunCmd(
+ ['adb', '-s', self._adb._adb.GetSerialNumber(), 'forward',
+ 'tcp:%s' % host_adb_control_port,
+ 'localabstract:%s' % Forwarder._DEVICE_ADB_CONTROL_PORT]) != 0:
+ raise Exception('Error while running adb forward.')
+
+ if not self._adb.ExtractPid('device_forwarder'):
+ # TODO(pliard): Get rid of pexpect here and make device_forwarder a daemon
+ # with a blocking CLI process that exits with a proper exit code and not
+ # while the daemon is still setting up. This would be similar to how
+ # host_forwarder works.
+ self._device_process = pexpect.spawn(
+ 'adb', ['-s',
+ self._adb._adb.GetSerialNumber(),
+ 'shell',
+ '%s %s -D --adb_sock=%s' % (
+ tool.GetUtilWrapper(),
+ Forwarder._DEVICE_FORWARDER_PATH,
+ Forwarder._DEVICE_ADB_CONTROL_PORT)])
+ device_success_re = re.compile('Starting Device Forwarder.')
+ device_failure_re = re.compile('.*:ERROR:(.*)')
+ index = self._device_process.expect([device_success_re,
+ device_failure_re,
+ pexpect.EOF,
+ pexpect.TIMEOUT],
+ Forwarder._TIMEOUT_SECS)
+ if index == 1:
+ error_msg = str(self._device_process.match.group(1))
+ logging.error(self._device_process.before)
+ self._device_process.close()
+ raise Exception('Failed to start Device Forwarder with Error: %s' %
+ error_msg)
elif index == 2:
- logging.error(self._host_process.before)
- self._CloseProcess()
- raise Exception('Unexpected EOF while trying to forward ports %s' %
- port_pairs)
+ logging.error(self._device_process.before)
+ self._device_process.close()
+ raise Exception(
+ 'Unexpected EOF while trying to start Device Forwarder.')
elif index == 3:
- logging.error(self._host_process.before)
- self._CloseProcess()
- raise Exception('Timeout while trying to forward ports %s' % port_pairs)
-
- def _KillHostForwarderBlocking(self, host_pattern, timeout_sec):
- """Kills any existing host forwarders using the provided pattern.
-
- Note that this waits until the process terminates.
- """
- cmd_helper.RunCmd(['pkill', '-f', host_pattern])
- elapsed = 0
- wait_period = 0.1
- while not cmd_helper.RunCmd(['pgrep', '-f', host_pattern]) and (
- elapsed < timeout_sec):
- time.sleep(wait_period)
- elapsed += wait_period
- if elapsed >= timeout_sec:
- raise Exception('Timed out while killing ' + host_pattern)
-
- def _KillDeviceForwarderBlocking(self, timeout_sec):
- """Kills any existing device forwarders.
-
- Note that this waits until the process terminates.
- """
- processes_killed = self._adb.KillAllBlocking(
- 'device_forwarder', timeout_sec)
+ logging.error(self._device_process.before)
+ self._device_process.close()
+ raise Exception('Timeout while trying start Device Forwarder')
+
+ for redirection_command in redirection_commands:
+ (exit_code, output) = cmd_helper.GetCmdStatusAndOutput(
+ [self._host_forwarder_path, redirection_command])
+ if exit_code != 0:
+ raise Exception('%s exited with %d: %s' % (self._host_forwarder_path,
+ exit_code, output))
+ tokens = output.split(':')
+ if len(tokens) != 2:
+ raise Exception('Unexpected host forwarder output "%s", ' +
+ 'expected "device_port:host_port"' % output)
+ device_port = int(tokens[0])
+ host_port = int(tokens[1])
+ self._host_to_device_port_map[host_port] = device_port
+ logging.info('Forwarding device port: %d to host port: %d.', device_port,
+ host_port)
+
+ @staticmethod
+ def KillHost(build_type):
+ host_forwarder_path = _MakeBinaryPath(build_type, 'host_forwarder')
+ (exit_code, output) = cmd_helper.GetCmdStatusAndOutput(
+ [host_forwarder_path, 'kill-server'])
+ if exit_code != 0:
+ raise Exception('%s exited with %d: %s' % (host_forwarder_path,
+ exit_code, output))
+
+ @staticmethod
+ def KillDevice(adb):
+ logging.info('Killing device_forwarder.')
+ timeout_sec = 5
+ processes_killed = adb.KillAllBlocking('device_forwarder', timeout_sec)
if not processes_killed:
- pids = self._adb.ExtractPid('device_forwarder')
+ pids = adb.ExtractPid('device_forwarder')
if pids:
raise Exception('Timed out while killing device_forwarder')
- def _CloseProcess(self):
- if self._host_process:
- self._host_process.close()
- if self._device_process:
- self._device_process.close()
- if self._adb_forward_process:
- self._adb_forward_process.close()
- self._host_process = None
- self._device_process = None
- self._adb_forward_process = None
-
def DevicePortForHostPort(self, host_port):
"""Get the device port that corresponds to a given host port."""
return self._host_to_device_port_map.get(host_port)
def Close(self):
"""Terminate the forwarder process."""
- self._CloseProcess()
+ if self._device_process:
+ self._device_process.close()
+ self._device_process = None
diff --git a/build/android/pylib/ports.py b/build/android/pylib/ports.py
index 06b9f58..74c84c1 100644
--- a/build/android/pylib/ports.py
+++ b/build/android/pylib/ports.py
@@ -2,7 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-"""Functions that deals with local and device ports."""
+"""Functions that deal with local and device ports."""
import contextlib
import fcntl
@@ -17,12 +17,13 @@ import cmd_helper
import constants
-#The following two methods are used to allocate the port source for various
-# types of test servers. Because some net relates tests can be run on shards
-# at same time, it's important to have a mechanism to allocate the port process
-# safe. In here, we implement the safe port allocation by leveraging flock.
+# The following two methods are used to allocate the port source for various
+# types of test servers. Because some net-related tests can be run on shards at
+# same time, it's important to have a mechanism to allocate the port
+# process-safe. In here, we implement the safe port allocation by leveraging
+# flock.
def ResetTestServerPortAllocation():
- """Reset the port allocation to start from TEST_SERVER_PORT_FIRST.
+ """Resets the port allocation to start from TEST_SERVER_PORT_FIRST.
Returns:
Returns True if reset successes. Otherwise returns False.
@@ -39,7 +40,7 @@ def ResetTestServerPortAllocation():
def AllocateTestServerPort():
- """Allocate a port incrementally.
+ """Allocates a port incrementally.
Returns:
Returns a valid port which should be in between TEST_SERVER_PORT_FIRST and
@@ -90,10 +91,12 @@ def IsHostPortUsed(host_port):
Returns:
True if the port on host is already used, otherwise returns False.
"""
- port_info = '(127\.0\.0\.1)|(localhost)\:%d' % host_port
- # TODO(jnd): Find a better way to filter the port.
+ port_info = '(\*)|(127\.0\.0\.1)|(localhost):%d' % host_port
+ # TODO(jnd): Find a better way to filter the port. Note that connecting to the
+ # socket and closing it would leave it in the TIME_WAIT state. Setting
+ # SO_LINGER on it and then closing it makes the Python HTTP server crash.
re_port = re.compile(port_info, re.MULTILINE)
- if re_port.findall(cmd_helper.GetCmdOutput(['lsof', '-nPi:%d' % host_port])):
+ if re_port.search(cmd_helper.GetCmdOutput(['lsof', '-nPi:%d' % host_port])):
return True
return False
@@ -115,6 +118,11 @@ def IsDevicePortUsed(adb, device_port, state=''):
for single_connect in netstat_results:
# Column 3 is the local address which we want to check with.
connect_results = single_connect.split()
+ if connect_results[0] != 'tcp':
+ continue
+ if len(connect_results) < 6:
+ raise Exception('Unexpected format while parsing netstat line: ' +
+ single_connect)
is_state_match = connect_results[5] == state if state else True
if connect_results[3] == base_url and is_state_match:
return True
diff --git a/build/android/pylib/run_java_tests.py b/build/android/pylib/run_java_tests.py
index 62947fa..4e61e2d 100644
--- a/build/android/pylib/run_java_tests.py
+++ b/build/android/pylib/run_java_tests.py
@@ -268,8 +268,8 @@ class TestRunner(BaseTestRunner):
# We need to remember which ports the HTTP server is using, since the
# forwarder will stomp on them otherwise.
port_pairs.append(http_server_ports)
- self.forwarder = Forwarder(
- self.adb, port_pairs, self.tool, '127.0.0.1', self.build_type)
+ self.forwarder = Forwarder(self.adb, self.build_type)
+ self.forwarder.Run(port_pairs, self.tool, '127.0.0.1')
self.CopyTestFilesOnce()
self.flags.AddFlags(['--enable-test-intents'])
@@ -490,7 +490,7 @@ class TestSharder(BaseTestSharder):
"""Responsible for sharding the tests on the connected devices."""
def __init__(self, attached_devices, options, tests, apks):
- BaseTestSharder.__init__(self, attached_devices)
+ BaseTestSharder.__init__(self, attached_devices, options.build_type)
self.options = options
self.tests = tests
self.apks = apks
generated by cgit v1.1 at 2025-03-05 04:33:14 +0000