diff options
Diffstat (limited to 'build')
23 files changed, 255 insertions, 166 deletions
diff --git a/build/android/adb_install_apk.py b/build/android/adb_install_apk.py index 50faea7..3000b81 100755 --- a/build/android/adb_install_apk.py +++ b/build/android/adb_install_apk.py @@ -53,6 +53,7 @@ def main(): 'Default is env var BUILDTYPE or Debug.') parser.add_argument('-d', '--device', dest='device', help='Target device for apk to install on.') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') parser.add_argument('-v', '--verbose', action='count', help='Enable verbose logging.') @@ -82,7 +83,13 @@ def main(): and helper.GetSplitName()): splits.append(f) - devices = device_utils.DeviceUtils.HealthyDevices() + if args.blacklist_file: + blacklist = device_blacklist.Blacklist(args.blacklist_file) + else: + # TODO(jbudorick): Remove this once the bots are converted. + blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON) + + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) if args.device: devices = [d for d in devices if d == args.device] @@ -99,12 +106,14 @@ def main(): device.Install(apk, reinstall=args.keep_data) except device_errors.CommandFailedError: logging.exception('Failed to install %s', args.apk_name) - device_blacklist.ExtendBlacklist([str(device)]) - logging.warning('Blacklisting %s', str(device)) + if blacklist: + blacklist.Extend([str(device)]) + logging.warning('Blacklisting %s', str(device)) except device_errors.CommandTimeoutError: logging.exception('Timed out while installing %s', args.apk_name) - device_blacklist.ExtendBlacklist([str(device)]) - logging.warning('Blacklisting %s', str(device)) + if blacklist: + blacklist.Extend([str(device)]) + logging.warning('Blacklisting %s', str(device)) device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_install) diff --git a/build/android/adb_reverse_forwarder.py b/build/android/adb_reverse_forwarder.py index 3ce5359..a3bfaf0 100755 --- a/build/android/adb_reverse_forwarder.py +++ b/build/android/adb_reverse_forwarder.py @@ -19,6 +19,7 @@ import time from pylib import constants from pylib import forwarder from pylib.device import adb_wrapper +from pylib.device import device_blacklist from pylib.device import device_errors from pylib.device import device_utils from pylib.utils import run_tests_helper @@ -36,6 +37,7 @@ def main(argv): help='Verbose level (multiple times for more)') parser.add_option('--device', help='Serial number of device we should use.') + parser.add_option('--blacklist-file', help='Device blacklist JSON file.') parser.add_option('--debug', action='store_const', const='Debug', dest='build_type', default='Release', help='Use Debug build of host tools instead of Release.') @@ -54,7 +56,12 @@ def main(argv): parser.error('Bad port number') sys.exit(1) - devices = device_utils.DeviceUtils.HealthyDevices() + if options.blacklist_file: + blacklist = device_blacklist.Blacklist(options.blacklist_file) + else: + blacklist = None + + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) if options.device: device = next((d for d in devices if d == options.device), None) diff --git a/build/android/buildbot/bb_device_status_check.py b/build/android/buildbot/bb_device_status_check.py index 52261ac..65fc5fe 100755 --- a/build/android/buildbot/bb_device_status_check.py +++ b/build/android/buildbot/bb_device_status_check.py @@ -5,9 +5,9 @@ # found in the LICENSE file. """A class to keep track of devices across builds and report state.""" +import argparse import json import logging -import optparse import os import psutil import re @@ -41,7 +41,7 @@ from pylib.utils import timeout_retry _RE_DEVICE_ID = re.compile('Device ID = (\d+)') -def DeviceInfo(device, options): +def DeviceInfo(device, args): """Gathers info on a device via various adb calls. Args: @@ -96,7 +96,7 @@ def DeviceInfo(device, options): dev_good = False if not battery.GetCharging(): battery.SetCharging(True) - if not options.no_provisioning_check: + if not args.no_provisioning_check: setup_wizard_disabled = ( device.GetProp('ro.setupwizard.mode') == 'DISABLED') if not setup_wizard_disabled and device.build_type != 'user': @@ -114,16 +114,16 @@ def DeviceInfo(device, options): return (build_product, build_id, battery_level, errors, dev_good, json_data) -def CheckForMissingDevices(options, devices): +def CheckForMissingDevices(args, devices): """Uses file of previous online devices to detect broken phones. Args: - options: out_dir parameter of options argument is used as the base + args: out_dir parameter of args argument is used as the base directory to load and update the cache file. devices: A list of DeviceUtils instance for the currently visible and online attached devices. """ - out_dir = os.path.abspath(options.out_dir) + out_dir = os.path.abspath(args.out_dir) device_serials = set(d.adb.GetDeviceSerial() for d in devices) # last_devices denotes all known devices prior to this run @@ -224,9 +224,9 @@ def KillAllAdb(): pass -def RecoverDevices(args): +def RecoverDevices(blacklist, output_directory): # Remove the last build's "bad devices" before checking device statuses. - device_blacklist.ResetBlacklist() + blacklist.Reset() previous_devices = set(a.GetDeviceSerial() for a in adb_wrapper.AdbWrapper.Devices()) @@ -236,7 +236,7 @@ def RecoverDevices(args): try: expected_devices = set(device_list.GetPersistentDeviceList( - os.path.join(args.out_dir, device_list.LAST_DEVICES_FILENAME))) + os.path.join(output_directory, device_list.LAST_DEVICES_FILENAME))) except IOError: expected_devices = set() @@ -249,15 +249,15 @@ def RecoverDevices(args): except device_errors.CommandFailedError: logging.exception('Failure while waiting for %s. Adding to blacklist.', str(device)) - device_blacklist.ExtendBlacklist([str(device)]) + blacklist.Extend([str(device)]) except device_errors.CommandTimeoutError: logging.exception('Timed out while waiting for %s. Adding to blacklist.', str(device)) - device_blacklist.ExtendBlacklist([str(device)]) + blacklist.Extend([str(device)]) device_utils.DeviceUtils.parallel(all_devices).pMap(blacklisting_recovery) - devices = device_utils.DeviceUtils.HealthyDevices() + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) device_serials = set(d.adb.GetDeviceSerial() for d in devices) missing_devices = expected_devices.difference(device_serials) @@ -275,35 +275,41 @@ def RecoverDevices(args): def main(): - parser = optparse.OptionParser() - parser.add_option('', '--out-dir', - help='Directory where the device path is stored', - default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) - parser.add_option('--no-provisioning-check', action='store_true', - help='Will not check if devices are provisioned properly.') - parser.add_option('--device-status-dashboard', action='store_true', - help='Output device status data for dashboard.') - parser.add_option('--restart-usb', action='store_true', - help='DEPRECATED. ' - 'This script now always tries to reset USB.') - parser.add_option('--json-output', - help='Output JSON information into a specified file.') - parser.add_option('-v', '--verbose', action='count', default=1, - help='Log more information.') - - options, args = parser.parse_args() - if args: - parser.error('Unknown options %s' % args) - - run_tests_helper.SetLogLevel(options.verbose) - - devices = RecoverDevices(options) + parser = argparse.ArgumentParser() + parser.add_argument('--out-dir', + help='Directory where the device path is stored', + default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) + parser.add_argument('--no-provisioning-check', action='store_true', + help='Will not check if devices are provisioned ' + 'properly.') + parser.add_argument('--device-status-dashboard', action='store_true', + help='Output device status data for dashboard.') + parser.add_argument('--restart-usb', action='store_true', + help='DEPRECATED. ' + 'This script now always tries to reset USB.') + parser.add_argument('--json-output', + help='Output JSON information into a specified file.') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') + parser.add_argument('-v', '--verbose', action='count', default=1, + help='Log more information.') + + args = parser.parse_args() + + run_tests_helper.SetLogLevel(args.verbose) + + if args.blacklist_file: + blacklist = device_blacklist.Blacklist(args.blacklist_file) + else: + # TODO(jbudorick): Remove this once bots pass the blacklist file. + blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON) + + devices = RecoverDevices(blacklist, args.out_dir) types, builds, batteries, errors, devices_ok, json_data = ( [], [], [], [], [], []) if devices: types, builds, batteries, errors, devices_ok, json_data = ( - zip(*[DeviceInfo(dev, options) for dev in devices])) + zip(*[DeviceInfo(dev, args) for dev in devices])) # Write device info to file for buildbot info display. if os.path.exists('/home/chrome-bot'): @@ -316,7 +322,7 @@ def main(): except Exception: pass - err_msg = CheckForMissingDevices(options, devices) or [] + err_msg = CheckForMissingDevices(args, devices) or [] unique_types = list(set(types)) unique_builds = list(set(builds)) @@ -350,7 +356,7 @@ def main(): subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name) SendEmail(from_address, to_addresses, [], subject, '\n'.join(err_msg)) - if options.device_status_dashboard: + if args.device_status_dashboard: offline_devices = [ device_utils.DeviceUtils(a) for a in adb_wrapper.AdbWrapper.Devices(is_ready=False) @@ -366,15 +372,15 @@ def main(): [battery], '%', 'unimportant') - if options.json_output: - with open(options.json_output, 'wb') as f: + if args.json_output: + with open(args.json_output, 'wb') as f: f.write(json.dumps(json_data, indent=4)) num_failed_devs = 0 for device_ok, device in zip(devices_ok, devices): if not device_ok: logging.warning('Blacklisting %s', str(device)) - device_blacklist.ExtendBlacklist([str(device)]) + blacklist.Extend([str(device)]) num_failed_devs += 1 if num_failed_devs == len(devices): diff --git a/build/android/buildbot/bb_device_steps.py b/build/android/buildbot/bb_device_steps.py index fbbf23be..ed7453b 100755 --- a/build/android/buildbot/bb_device_steps.py +++ b/build/android/buildbot/bb_device_steps.py @@ -213,7 +213,7 @@ def RunChromeProxyTests(options): """ InstallApk(options, INSTRUMENTATION_TESTS['ChromeShell'], False) args = ['--browser', 'android-chrome-shell'] - devices = device_utils.DeviceUtils.HealthyDevices() + devices = device_utils.DeviceUtils.HealthyDevices(blacklist=None) if devices: args = args + ['--device', devices[0].adb.GetDeviceSerial()] bb_annotations.PrintNamedStep('chrome_proxy') @@ -232,7 +232,7 @@ def RunTelemetryTests(options, step_name, run_tests_path): """ InstallApk(options, INSTRUMENTATION_TESTS['ChromeShell'], False) args = ['--browser', 'android-chrome-shell'] - devices = device_utils.DeviceUtils.HealthyDevices() + devices = device_utils.DeviceUtils.HealthyDevices(blacklist=None) if devices: args = args + ['--device', 'android'] bb_annotations.PrintNamedStep(step_name) diff --git a/build/android/enable_asserts.py b/build/android/enable_asserts.py index 8fb7dca..b33e212 100755 --- a/build/android/enable_asserts.py +++ b/build/android/enable_asserts.py @@ -9,12 +9,15 @@ import argparse import sys +from pylib.device import device_blacklist from pylib.device import device_utils def main(): parser = argparse.ArgumentParser() + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') + set_asserts_group = parser.add_mutually_exclusive_group(required=True) set_asserts_group.add_argument( '--enable_asserts', dest='set_asserts', action='store_true', @@ -25,9 +28,15 @@ def main(): args = parser.parse_args() + if args.blacklist_file: + blacklist = device_blacklist.Blacklist(args.blacklist_file) + else: + blacklist = None + # TODO(jbudorick): Accept optional serial number and run only for the # specified device when present. - devices = device_utils.DeviceUtils.parallel() + devices = device_utils.DeviceUtils.parallel( + device_utils.DeviceUtils.HealthyDevices(blacklist)) def set_java_asserts_and_restart(device): if device.SetJavaAsserts(args.set_asserts): diff --git a/build/android/host_heartbeat.py b/build/android/host_heartbeat.py index 6a7cdd1..d5898ab 100755 --- a/build/android/host_heartbeat.py +++ b/build/android/host_heartbeat.py @@ -19,7 +19,7 @@ PULSE_PERIOD = 20 def main(): while True: try: - devices = device_utils.DeviceUtils.HealthyDevices() + devices = device_utils.DeviceUtils.HealthyDevices(blacklist=None) for d in devices: d.RunShellCommand(['touch', '/sdcard/host_heartbeat'], check_return=True) diff --git a/build/android/provision_devices.py b/build/android/provision_devices.py index cde01ad..14b4885 100755 --- a/build/android/provision_devices.py +++ b/build/android/provision_devices.py @@ -48,27 +48,33 @@ class _PHASES(object): ALL = [WIPE, PROPERTIES, FINISH] -def ProvisionDevices(options): - devices = device_utils.DeviceUtils.HealthyDevices() - if options.device: - devices = [d for d in devices if d == options.device] +def ProvisionDevices(args): + if args.blacklist_file: + blacklist = device_blacklist.Blacklist(args.blacklist_file) + else: + # TODO(jbudorick): Remove once the bots have switched over. + blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON) + + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) + if args.device: + devices = [d for d in devices if d == args.device] if not devices: - raise device_errors.DeviceUnreachableError(options.device) + raise device_errors.DeviceUnreachableError(args.device) parallel_devices = device_utils.DeviceUtils.parallel(devices) - parallel_devices.pMap(ProvisionDevice, options) - if options.auto_reconnect: + parallel_devices.pMap(ProvisionDevice, blacklist, args) + if args.auto_reconnect: _LaunchHostHeartbeat() - blacklist = device_blacklist.ReadBlacklist() - if options.output_device_blacklist: - with open(options.output_device_blacklist, 'w') as f: - json.dump(blacklist, f) - if all(d in blacklist for d in devices): + blacklisted_devices = blacklist.Read() + if args.output_device_blacklist: + with open(args.output_device_blacklist, 'w') as f: + json.dump(blacklisted_devices, f) + if all(d in blacklisted_devices for d in devices): raise device_errors.NoDevicesError return 0 -def ProvisionDevice(device, options): +def ProvisionDevice(device, blacklist, options): if options.reboot_timeout: reboot_timeout = options.reboot_timeout elif (device.build_version_sdk >= @@ -104,12 +110,12 @@ def ProvisionDevice(device, options): except device_errors.CommandTimeoutError: logging.exception('Timed out waiting for device %s. Adding to blacklist.', str(device)) - device_blacklist.ExtendBlacklist([str(device)]) + blacklist.Extend([str(device)]) except device_errors.CommandFailedError: logging.exception('Failed to provision device %s. Adding to blacklist.', str(device)) - device_blacklist.ExtendBlacklist([str(device)]) + blacklist.Extend([str(device)]) def WipeDevice(device, options): @@ -311,6 +317,7 @@ def main(): parser.add_argument('-d', '--device', metavar='SERIAL', help='the serial number of the device to be provisioned' ' (the default is to provision all devices attached)') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') parser.add_argument('--phase', action='append', choices=_PHASES.ALL, dest='phases', help='Phases of provisioning to run. ' diff --git a/build/android/pylib/device/device_blacklist.py b/build/android/pylib/device/device_blacklist.py index a141d62..0724654 100644 --- a/build/android/pylib/device/device_blacklist.py +++ b/build/android/pylib/device/device_blacklist.py @@ -7,55 +7,76 @@ import os import threading from pylib import constants -_BLACKLIST_JSON = os.path.join( + +# TODO(jbudorick): Remove this once the blacklist is optional. +BLACKLIST_JSON = os.path.join( constants.DIR_SOURCE_ROOT, os.environ.get('CHROMIUM_OUT_DIR', 'out'), 'bad_devices.json') -# Note that this only protects against concurrent accesses to the blacklist -# within a process. -_blacklist_lock = threading.RLock() +class Blacklist(object): -def ReadBlacklist(): - """Reads the blacklist from the _BLACKLIST_JSON file. + def __init__(self, path): + self._blacklist_lock = threading.RLock() + self._path = path - Returns: - A list containing bad devices. - """ - with _blacklist_lock: - if not os.path.exists(_BLACKLIST_JSON): - return [] + def Read(self): + """Reads the blacklist from the blacklist file. - with open(_BLACKLIST_JSON, 'r') as f: - return json.load(f) + Returns: + A list containing bad devices. + """ + with self._blacklist_lock: + if not os.path.exists(self._path): + return [] + with open(self._path, 'r') as f: + return json.load(f) -def WriteBlacklist(blacklist): - """Writes the provided blacklist to the _BLACKLIST_JSON file. + def Write(self, blacklist): + """Writes the provided blacklist to the blacklist file. - Args: - blacklist: list of bad devices to write to the _BLACKLIST_JSON file. - """ - with _blacklist_lock: - with open(_BLACKLIST_JSON, 'w') as f: - json.dump(list(set(blacklist)), f) + Args: + blacklist: list of bad devices to write to the blacklist file. + """ + with self._blacklist_lock: + with open(self._path, 'w') as f: + json.dump(list(set(blacklist)), f) + def Extend(self, devices): + """Adds devices to blacklist file. -def ExtendBlacklist(devices): - """Adds devices to _BLACKLIST_JSON file. + Args: + devices: list of bad devices to be added to the blacklist file. + """ + with self._blacklist_lock: + blacklist = ReadBlacklist() + blacklist.extend(devices) + WriteBlacklist(blacklist) - Args: - devices: list of bad devices to be added to the _BLACKLIST_JSON file. - """ - with _blacklist_lock: - blacklist = ReadBlacklist() - blacklist.extend(devices) - WriteBlacklist(blacklist) + def Reset(self): + """Erases the blacklist file if it exists.""" + with self._blacklist_lock: + if os.path.exists(self._path): + os.remove(self._path) + + +def ReadBlacklist(): + # TODO(jbudorick): Phase out once all clients have migrated. + return Blacklist(BLACKLIST_JSON).Read() + + +def WriteBlacklist(blacklist): + # TODO(jbudorick): Phase out once all clients have migrated. + Blacklist(BLACKLIST_JSON).Write(blacklist) + + +def ExtendBlacklist(devices): + # TODO(jbudorick): Phase out once all clients have migrated. + Blacklist(BLACKLIST_JSON).Extend(devices) def ResetBlacklist(): - """Erases the _BLACKLIST_JSON file if it exists.""" - with _blacklist_lock: - if os.path.exists(_BLACKLIST_JSON): - os.remove(_BLACKLIST_JSON) + # TODO(jbudorick): Phase out once all clients have migrated. + Blacklist(BLACKLIST_JSON).Reset() diff --git a/build/android/pylib/device/device_utils.py b/build/android/pylib/device/device_utils.py index bc1f2ab..522bf33 100644 --- a/build/android/pylib/device/device_utils.py +++ b/build/android/pylib/device/device_utils.py @@ -1843,7 +1843,7 @@ class DeviceUtils(object): } @classmethod - def parallel(cls, devices=None, async=False): + def parallel(cls, devices, async=False): """Creates a Parallelizer to operate over the provided list of devices. If |devices| is either |None| or an empty list, the Parallelizer will @@ -1860,9 +1860,7 @@ class DeviceUtils(object): A Parallelizer operating over |devices|. """ if not devices: - devices = cls.HealthyDevices() - if not devices: - raise device_errors.NoDevicesError() + raise device_errors.NoDevicesError() devices = [d if isinstance(d, cls) else cls(d) for d in devices] if async: @@ -1871,10 +1869,14 @@ class DeviceUtils(object): return parallelizer.SyncParallelizer(devices) @classmethod - def HealthyDevices(cls): - blacklist = device_blacklist.ReadBlacklist() + def HealthyDevices(cls, blacklist=None): + if not blacklist: + # TODO(jbudorick): Remove once clients pass in the blacklist. + blacklist = device_blacklist.Blacklist(device_blacklist.BLACKLIST_JSON) + + blacklisted_devices = blacklist.Read() def blacklisted(adb): - if adb.GetDeviceSerial() in blacklist: + if adb.GetDeviceSerial() in blacklisted_devices: logging.warning('Device %s is blacklisted.', adb.GetDeviceSerial()) return True return False diff --git a/build/android/pylib/device/device_utils_test.py b/build/android/pylib/device/device_utils_test.py index 580794d..47ae9be 100755 --- a/build/android/pylib/device/device_utils_test.py +++ b/build/android/pylib/device/device_utils_test.py @@ -23,6 +23,7 @@ from pylib import cmd_helper from pylib import constants from pylib import device_signal from pylib.device import adb_wrapper +from pylib.device import device_blacklist from pylib.device import device_errors from pylib.device import device_utils from pylib.device import intent @@ -1869,25 +1870,6 @@ class DeviceUtilsClientCache(DeviceUtilsTest): self.assertEqual(client_cache_two, {}) -class DeviceUtilsParallelTest(mock_calls.TestCase): - - def testParallel_default(self): - test_serials = ['0123456789abcdef', 'fedcba9876543210'] - with self.assertCall( - mock.call.pylib.device.device_utils.DeviceUtils.HealthyDevices(), - [device_utils.DeviceUtils(s) for s in test_serials]): - parallel_devices = device_utils.DeviceUtils.parallel() - for serial, device in zip(test_serials, parallel_devices.pGet(None)): - self.assertTrue(isinstance(device, device_utils.DeviceUtils)) - self.assertEquals(serial, device.adb.GetDeviceSerial()) - - def testParallel_noDevices(self): - with self.assertCall( - mock.call.pylib.device.device_utils.DeviceUtils.HealthyDevices(), []): - with self.assertRaises(device_errors.NoDevicesError): - device_utils.DeviceUtils.parallel() - - class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): def _createAdbWrapperMock(self, serial, is_ready=True): @@ -1895,25 +1877,25 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): adb.is_ready = is_ready return adb - def testHealthyDevices_default(self): + def testHealthyDevices_emptyBlacklist(self): test_serials = ['0123456789abcdef', 'fedcba9876543210'] with self.assertCalls( - (mock.call.pylib.device.device_blacklist.ReadBlacklist(), []), (mock.call.pylib.device.adb_wrapper.AdbWrapper.Devices(), [self._createAdbWrapperMock(s) for s in test_serials])): - devices = device_utils.DeviceUtils.HealthyDevices() + blacklist = mock.NonCallableMock(**{'Read.return_value': []}) + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) for serial, device in zip(test_serials, devices): self.assertTrue(isinstance(device, device_utils.DeviceUtils)) self.assertEquals(serial, device.adb.GetDeviceSerial()) - def testHealthyDevices_blacklisted(self): + def testHealthyDevices_blacklist(self): test_serials = ['0123456789abcdef', 'fedcba9876543210'] with self.assertCalls( - (mock.call.pylib.device.device_blacklist.ReadBlacklist(), - ['fedcba9876543210']), (mock.call.pylib.device.adb_wrapper.AdbWrapper.Devices(), [self._createAdbWrapperMock(s) for s in test_serials])): - devices = device_utils.DeviceUtils.HealthyDevices() + blacklist = mock.NonCallableMock( + **{'Read.return_value': ['fedcba9876543210']}) + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) self.assertEquals(1, len(devices)) self.assertTrue(isinstance(devices[0], device_utils.DeviceUtils)) self.assertEquals('0123456789abcdef', devices[0].adb.GetDeviceSerial()) diff --git a/build/android/pylib/host_driven/test_case.py b/build/android/pylib/host_driven/test_case.py index 6ff4c5f..817fce6 100644 --- a/build/android/pylib/host_driven/test_case.py +++ b/build/android/pylib/host_driven/test_case.py @@ -155,7 +155,8 @@ class HostDrivenTestCase(object): start_ms = int(time.time()) * 1000 done = False for test_filter in test_filters: - tests = test_pkg.GetAllMatchingTests(None, None, test_filter) + tests = test_pkg.GetAllMatchingTests( + None, None, test_filter, [self.device]) # Filters should always result in >= 1 test. if len(tests) == 0: raise Exception('Java test filter "%s" returned no tests.' diff --git a/build/android/pylib/instrumentation/setup.py b/build/android/pylib/instrumentation/setup.py index 7a0501e..698b609 100644 --- a/build/android/pylib/instrumentation/setup.py +++ b/build/android/pylib/instrumentation/setup.py @@ -82,7 +82,8 @@ def Setup(test_options, devices): tests = test_pkg.GetAllMatchingTests( test_options.annotations, test_options.exclude_annotations, - test_options.test_filter) + test_options.test_filter, + devices) if not tests: logging.error('No instrumentation tests to run with current args.') diff --git a/build/android/pylib/instrumentation/test_jar.py b/build/android/pylib/instrumentation/test_jar.py index 7ad8997..179e6b1 100644 --- a/build/android/pylib/instrumentation/test_jar.py +++ b/build/android/pylib/instrumentation/test_jar.py @@ -170,7 +170,7 @@ class TestJar(object): attached_min_sdk_level >= required_min_sdk_level) def GetAllMatchingTests(self, annotation_filter_list, - exclude_annotation_list, test_filter): + exclude_annotation_list, test_filter, devices): """Get a list of tests matching any of the annotations and the filter. Args: @@ -180,6 +180,7 @@ class TestJar(object): exclude_annotation_list: List of test annotations. A test must not have any of these annotations. test_filter: Filter used for partial matching on the test method names. + devices: The set of devices against which tests will be run. Returns: List of all matching tests. @@ -218,7 +219,7 @@ class TestJar(object): # Filter out any tests with SDK level requirements that don't match the set # of attached devices. - devices = device_utils.DeviceUtils.parallel() + devices = device_utils.DeviceUtils.parallel(devices) min_sdk_version = min(devices.build_version_sdk.pGet(None)) tests = [t for t in tests if self._IsTestValidForSdkRange(t, min_sdk_version)] diff --git a/build/android/pylib/local/device/local_device_environment.py b/build/android/pylib/local/device/local_device_environment.py index 04f9ab7..4a0d567 100644 --- a/build/android/pylib/local/device/local_device_environment.py +++ b/build/android/pylib/local/device/local_device_environment.py @@ -4,6 +4,7 @@ from pylib.base import environment from pylib.device import adb_wrapper +from pylib.device import device_blacklist from pylib.device import device_errors from pylib.device import device_utils from pylib.utils import parallelizer @@ -13,6 +14,8 @@ class LocalDeviceEnvironment(environment.Environment): def __init__(self, args, _error_func): super(LocalDeviceEnvironment, self).__init__() + self._blacklist = device_blacklist.Blacklist( + args.blacklist_file or device_blacklist.BLACKLIST_JSON) self._device_serial = args.test_device self._devices = [] self._max_tries = 1 + args.num_retries @@ -20,7 +23,8 @@ class LocalDeviceEnvironment(environment.Environment): #override def SetUp(self): - available_devices = device_utils.DeviceUtils.HealthyDevices() + available_devices = device_utils.DeviceUtils.HealthyDevices( + self._blacklist) if not available_devices: raise device_errors.NoDevicesError if self._device_serial: diff --git a/build/android/pylib/perf/perf_control_unittest.py b/build/android/pylib/perf/perf_control_unittest.py index 69b8b46..33088d3 100644 --- a/build/android/pylib/perf/perf_control_unittest.py +++ b/build/android/pylib/perf/perf_control_unittest.py @@ -17,7 +17,7 @@ class TestPerfControl(unittest.TestCase): if not os.getenv('BUILDTYPE'): os.environ['BUILDTYPE'] = 'Debug' - devices = device_utils.DeviceUtils.HealthyDevices() + devices = device_utils.DeviceUtils.HealthyDevices(blacklist=None) self.assertGreater(len(devices), 0, 'No device attached!') self._device = devices[0] diff --git a/build/android/pylib/perf/setup.py b/build/android/pylib/perf/setup.py index 8e1fc28..8635aca 100644 --- a/build/android/pylib/perf/setup.py +++ b/build/android/pylib/perf/setup.py @@ -18,7 +18,7 @@ from pylib.perf import test_runner from pylib.utils import test_environment -def _GetAllDevices(): +def _GetAllDevices(active_devices): devices_path = os.path.join(os.environ.get('CHROMIUM_OUT_DIR', 'out'), device_list.LAST_DEVICES_FILENAME) try: @@ -26,7 +26,7 @@ def _GetAllDevices(): for s in device_list.GetPersistentDeviceList(devices_path)] except IOError as e: logging.error('Unable to find %s [%s]', devices_path, e) - devices = device_utils.DeviceUtils.HealthyDevices() + devices = active_devices return sorted(devices) @@ -56,7 +56,7 @@ def _GetStepsDict(test_options): return steps -def Setup(test_options): +def Setup(test_options, active_devices): """Create and return the test runner factory and tests. Args: @@ -72,10 +72,10 @@ def Setup(test_options): os.makedirs(constants.PERF_OUTPUT_DIR) # Before running the tests, kill any leftover server. - test_environment.CleanupLeftoverProcesses() + test_environment.CleanupLeftoverProcesses(active_devices) # We want to keep device affinity, so return all devices ever seen. - all_devices = _GetAllDevices() + all_devices = _GetAllDevices(active_devices) steps_dict = _GetStepsDict(test_options) sorted_step_names = sorted(steps_dict['steps'].keys()) diff --git a/build/android/pylib/uiautomator/setup.py b/build/android/pylib/uiautomator/setup.py index bd8ffc7..1b8746b4 100644 --- a/build/android/pylib/uiautomator/setup.py +++ b/build/android/pylib/uiautomator/setup.py @@ -10,11 +10,12 @@ from pylib.uiautomator import test_package from pylib.uiautomator import test_runner -def Setup(test_options): +def Setup(test_options, devices): """Runs uiautomator tests on connected device(s). Args: test_options: A UIAutomatorOptions object. + devices: The list of that tests will run on. Returns: A tuple of (TestRunnerFactory, tests). @@ -23,7 +24,8 @@ def Setup(test_options): test_options.uiautomator_info_jar) tests = test_pkg.GetAllMatchingTests(test_options.annotations, test_options.exclude_annotations, - test_options.test_filter) + test_options.test_filter, + devices) if not tests: logging.error('No uiautomator tests to run with current args.') diff --git a/build/android/pylib/utils/emulator.py b/build/android/pylib/utils/emulator.py index cc07e61..c614d0d 100644 --- a/build/android/pylib/utils/emulator.py +++ b/build/android/pylib/utils/emulator.py @@ -18,6 +18,7 @@ import time from pylib import cmd_helper from pylib import constants from pylib import pexpect +from pylib.device import adb_wrapper from pylib.device import device_errors from pylib.device import device_utils from pylib.utils import time_profile @@ -89,16 +90,16 @@ def _KillAllEmulators(): running but a device slot is taken. A little bot trouble and we're out of room forever. """ - emulators = [d for d in device_utils.DeviceUtils.HealthyDevices() - if d.adb.is_emulator] + emulators = [device_utils.DeviceUtils(a) + for a in adb_wrapper.AdbWrapper.Devices() + if a.is_emulator] if not emulators: return for e in emulators: e.adb.Emu(['kill']) logging.info('Emulator killing is async; give a few seconds for all to die.') for _ in range(5): - if not any(d.adb.is_emulator for d - in device_utils.DeviceUtils.HealthyDevices()): + if not any(a.is_emulator for a in adb_wrapper.AdbWrapper.Devices()): return time.sleep(1) @@ -142,8 +143,9 @@ class PortPool(object): def _GetAvailablePort(): """Returns an available TCP port for the console.""" used_ports = [] - emulators = [d for d in device_utils.DeviceUtils.HealthyDevices() - if d.adb.is_emulator] + emulators = [device_utils.DeviceUtils(a) + for a in adb_wrapper.AdbWrapper.Devices() + if a.is_emulator] for emulator in emulators: used_ports.append(emulator.adb.GetDeviceSerial().split('-')[1]) for port in PortPool.port_range(): diff --git a/build/android/pylib/utils/test_environment.py b/build/android/pylib/utils/test_environment.py index f55a4ac..3b31d46 100644 --- a/build/android/pylib/utils/test_environment.py +++ b/build/android/pylib/utils/test_environment.py @@ -30,8 +30,12 @@ def _KillWebServers(): logging.warning('Failed waiting for %s to die. %s', p.pid, e) -def CleanupLeftoverProcesses(): - """Clean up the test environment, restarting fresh adb and HTTP daemons.""" +def CleanupLeftoverProcesses(devices): + """Clean up the test environment, restarting fresh adb and HTTP daemons. + + Args: + devices: The devices to clean. + """ _KillWebServers() device_utils.RestartServer() @@ -43,5 +47,5 @@ def CleanupLeftoverProcesses(): logging.error(str(e)) d.WaitUntilFullyBooted() - device_utils.DeviceUtils.parallel().pMap(cleanup_device) + device_utils.DeviceUtils.parallel(devices).pMap(cleanup_device) diff --git a/build/android/screenshot.py b/build/android/screenshot.py index 097739f..b52b706 100755 --- a/build/android/screenshot.py +++ b/build/android/screenshot.py @@ -12,6 +12,7 @@ import os import sys from pylib import screenshot +from pylib.device import device_blacklist from pylib.device import device_errors from pylib.device import device_utils @@ -47,6 +48,7 @@ def main(): usage='screenshot.py [options] [filename]') parser.add_option('-d', '--device', metavar='ANDROID_DEVICE', help='Serial ' 'number of Android device to use.', default=None) + parser.add_option('--blacklist-file', help='Device blacklist JSON file.') parser.add_option('-f', '--file', help='Save result to file instead of ' 'generating a timestamped file name.', metavar='FILE') parser.add_option('-v', '--verbose', help='Verbose logging.', @@ -73,7 +75,12 @@ def main(): if options.verbose: logging.getLogger().setLevel(logging.DEBUG) - devices = device_utils.DeviceUtils.HealthyDevices() + if options.blacklist_file: + blacklist = device_blacklist.Blacklist(options.blacklist_file) + else: + blacklist = None + + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) if options.device: device = next((d for d in devices if d == options.device), None) if not device: diff --git a/build/android/test_runner.py b/build/android/test_runner.py index 7b49c9e..8bae397 100755 --- a/build/android/test_runner.py +++ b/build/android/test_runner.py @@ -24,6 +24,7 @@ from pylib.base import environment_factory from pylib.base import test_dispatcher from pylib.base import test_instance_factory from pylib.base import test_run_factory +from pylib.device import device_blacklist from pylib.device import device_errors from pylib.device import device_utils from pylib.gtest import gtest_config @@ -188,6 +189,7 @@ def AddDeviceOptions(parser): group.add_argument('-d', '--device', dest='test_device', help=('Target device for the test suite ' 'to run on.')) + group.add_argument('--blacklist-file', help='Device blacklist file.') def AddGTestOptions(parser): @@ -767,7 +769,7 @@ def _RunUIAutomatorTests(args, devices): """Subcommand of RunTestsCommands which runs uiautomator tests.""" uiautomator_options = ProcessUIAutomatorOptions(args) - runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) + runner_factory, tests = uiautomator_setup.Setup(uiautomator_options, devices) results, exit_code = test_dispatcher.RunTests( tests, runner_factory, devices, shard=True, test_timeout=None, @@ -823,7 +825,7 @@ def _RunMonkeyTests(args, devices): return exit_code -def _RunPerfTests(args): +def _RunPerfTests(args, active_devices): """Subcommand of RunTestsCommands which runs perf tests.""" perf_options = ProcessPerfTestOptions(args) @@ -837,7 +839,8 @@ def _RunPerfTests(args): return perf_test_runner.PrintTestOutput( perf_options.print_step, perf_options.output_chartjson_data) - runner_factory, tests, devices = perf_setup.Setup(perf_options) + runner_factory, tests, devices = perf_setup.Setup( + perf_options, active_devices) # shard=False means that each device will get the full list of tests # and then each one will decide their own affinity. @@ -882,7 +885,7 @@ def _RunPythonTests(args): sys.path = sys.path[1:] -def _GetAttachedDevices(test_device=None): +def _GetAttachedDevices(blacklist_file, test_device): """Get all attached devices. Args: @@ -891,7 +894,14 @@ def _GetAttachedDevices(test_device=None): Returns: A list of attached devices. """ - attached_devices = device_utils.DeviceUtils.HealthyDevices() + if not blacklist_file: + # TODO(jbudorick): Remove this once bots pass the blacklist file. + blacklist_file = device_blacklist.BLACKLIST_JSON + logging.warning('Using default device blacklist %s', + device_blacklist.BLACKLIST_JSON) + + blacklist = device_blacklist.Blacklist(blacklist_file) + attached_devices = device_utils.DeviceUtils.HealthyDevices(blacklist) if test_device: test_device = [d for d in attached_devices if d == test_device] if not test_device: @@ -930,7 +940,7 @@ def RunTestsCommand(args, parser): if command in constants.LOCAL_MACHINE_TESTS: devices = [] else: - devices = _GetAttachedDevices(args.test_device) + devices = _GetAttachedDevices(args.blacklist_file, args.test_device) forwarder.Forwarder.RemoveHostLog() if not ports.ResetTestServerPortAllocation(): @@ -951,7 +961,7 @@ def RunTestsCommand(args, parser): elif command == 'monkey': return _RunMonkeyTests(args, devices) elif command == 'perf': - return _RunPerfTests(args) + return _RunPerfTests(args, devices) elif command == 'python': return _RunPythonTests(args) else: diff --git a/build/android/tombstones.py b/build/android/tombstones.py index 2ff74d1..a3eb2c8 100755 --- a/build/android/tombstones.py +++ b/build/android/tombstones.py @@ -20,6 +20,7 @@ import sys import optparse from pylib.device import adb_wrapper +from pylib.device import device_blacklist from pylib.device import device_errors from pylib.device import device_utils from pylib.utils import run_tests_helper @@ -222,6 +223,7 @@ def main(): parser.add_option('--device', help='The serial number of the device. If not specified ' 'will use all devices.') + parser.add_option('--blacklist-file', help='Device blacklist JSON file.') parser.add_option('-a', '--all-tombstones', action='store_true', help="""Resolve symbols for all tombstones, rather than just the most recent""") @@ -235,10 +237,15 @@ def main(): 'crash stacks.') options, _ = parser.parse_args() + if options.blacklist_file: + blacklist = device_blacklist.Blacklist(options.blacklist_file) + else: + blacklist = None + if options.device: devices = [device_utils.DeviceUtils(options.device)] else: - devices = device_utils.DeviceUtils.HealthyDevices() + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) # This must be done serially because strptime can hit a race condition if # used for the first time in a multithreaded environment. diff --git a/build/android/update_verification.py b/build/android/update_verification.py index 05d083b..72d156d 100755 --- a/build/android/update_verification.py +++ b/build/android/update_verification.py @@ -31,6 +31,7 @@ import sys import time from pylib import constants +from pylib.device import device_blacklist from pylib.device import device_errors from pylib.device import device_utils from pylib.utils import apk_helper @@ -64,6 +65,7 @@ def main(): description="Script to do semi-automated upgrade testing.") parser.add_argument('-v', '--verbose', action='count', help='Print verbose log information.') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') command_parsers = parser.add_subparsers(dest='command') subparser = command_parsers.add_parser('create_app_data') @@ -88,7 +90,12 @@ def main(): args = parser.parse_args() run_tests_helper.SetLogLevel(args.verbose) - devices = device_utils.DeviceUtils.HealthyDevices() + if args.blacklist_file: + blacklist = device_blacklist.Blacklist(args.blacklist_file) + else: + blacklist = None + + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) if not devices: raise device_errors.NoDevicesError() device = devices[0] |