#!/usr/bin/env python # # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """A class to keep track of devices across builds and report state.""" import argparse import json import logging import os import psutil import re import signal import sys sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import devil_chromium from devil import devil_env from devil.android import battery_utils from devil.android import device_blacklist from devil.android import device_errors from devil.android import device_list from devil.android import device_utils from devil.android.sdk import adb_wrapper from devil.constants import exit_codes from devil.utils import lsusb from devil.utils import reset_usb from devil.utils import run_tests_helper from pylib.constants import host_paths _RE_DEVICE_ID = re.compile(r'Device ID = (\d+)') def KillAllAdb(): def GetAllAdb(): for p in psutil.process_iter(): try: if 'adb' in p.name: yield p except (psutil.NoSuchProcess, psutil.AccessDenied): pass for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]: for p in GetAllAdb(): try: logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name, ' '.join(p.cmdline)) p.send_signal(sig) except (psutil.NoSuchProcess, psutil.AccessDenied): pass for p in GetAllAdb(): try: logging.error('Unable to kill %d (%s [%s])', p.pid, p.name, ' '.join(p.cmdline)) except (psutil.NoSuchProcess, psutil.AccessDenied): pass def _IsBlacklisted(serial, blacklist): return blacklist and serial in blacklist.Read() def _BatteryStatus(device, blacklist): battery_info = {} try: battery = battery_utils.BatteryUtils(device) battery_info = battery.GetBatteryInfo(timeout=5) battery_level = int(battery_info.get('level', 100)) if battery_level < 15: logging.error('Critically low battery level (%d)', battery_level) battery = battery_utils.BatteryUtils(device) if not battery.GetCharging(): battery.SetCharging(True) if blacklist: blacklist.Extend([device.adb.GetDeviceSerial()], reason='low_battery') except device_errors.CommandFailedError: logging.exception('Failed to get battery information for %s', str(device)) return battery_info def _IMEISlice(device): imei_slice = '' try: for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'], check_return=True, timeout=5): m = _RE_DEVICE_ID.match(l) if m: imei_slice = m.group(1)[-6:] except device_errors.CommandFailedError: logging.exception('Failed to get IMEI slice for %s', str(device)) return imei_slice def DeviceStatus(devices, blacklist): """Generates status information for the given devices. Args: devices: The devices to generate status for. blacklist: The current device blacklist. Returns: A dict of the following form: { '': { 'serial': '', 'adb_status': str, 'usb_status': bool, 'blacklisted': bool, # only if the device is connected and not blacklisted 'type': ro.build.product, 'build': ro.build.id, 'build_detail': ro.build.fingerprint, 'battery': { ... }, 'imei_slice': str, 'wifi_ip': str, }, ... } """ adb_devices = { a[0].GetDeviceSerial(): a for a in adb_wrapper.AdbWrapper.Devices(desired_state=None, long_list=True) } usb_devices = set(lsusb.get_android_devices()) def blacklisting_device_status(device): serial = device.adb.GetDeviceSerial() adb_status = ( adb_devices[serial][1] if serial in adb_devices else 'missing') usb_status = bool(serial in usb_devices) device_status = { 'serial': serial, 'adb_status': adb_status, 'usb_status': usb_status, } if not _IsBlacklisted(serial, blacklist): if adb_status == 'device': try: build_product = device.build_product build_id = device.build_id build_fingerprint = device.GetProp('ro.build.fingerprint', cache=True) wifi_ip = device.GetProp('dhcp.wlan0.ipaddress') battery_info = _BatteryStatus(device, blacklist) imei_slice = _IMEISlice(device) if (device.product_name == 'mantaray' and battery_info.get('AC powered', None) != 'true'): logging.error('Mantaray device not connected to AC power.') device_status.update({ 'ro.build.product': build_product, 'ro.build.id': build_id, 'ro.build.fingerprint': build_fingerprint, 'battery': battery_info, 'imei_slice': imei_slice, 'wifi_ip': wifi_ip, # TODO(jbudorick): Remove these once no clients depend on them. 'type': build_product, 'build': build_id, 'build_detail': build_fingerprint, }) except device_errors.CommandFailedError: logging.exception('Failure while getting device status for %s.', str(device)) if blacklist: blacklist.Extend([serial], reason='status_check_failure') except device_errors.CommandTimeoutError: logging.exception('Timeout while getting device status for %s.', str(device)) if blacklist: blacklist.Extend([serial], reason='status_check_timeout') elif blacklist: blacklist.Extend([serial], reason=adb_status if usb_status else 'offline') device_status['blacklisted'] = _IsBlacklisted(serial, blacklist) return device_status parallel_devices = device_utils.DeviceUtils.parallel(devices) statuses = parallel_devices.pMap(blacklisting_device_status).pGet(None) return statuses def RecoverDevices(devices, blacklist): """Attempts to recover any inoperable devices in the provided list. Args: devices: The list of devices to attempt to recover. blacklist: The current device blacklist, which will be used then reset. Returns: Nothing. """ statuses = DeviceStatus(devices, blacklist) should_restart_usb = set( status['serial'] for status in statuses if (not status['usb_status'] or status['adb_status'] in ('offline', 'missing'))) should_restart_adb = should_restart_usb.union(set( status['serial'] for status in statuses if status['adb_status'] == 'unauthorized')) should_reboot_device = should_restart_adb.union(set( status['serial'] for status in statuses if status['blacklisted'])) logging.debug('Should restart USB for:') for d in should_restart_usb: logging.debug(' %s', d) logging.debug('Should restart ADB for:') for d in should_restart_adb: logging.debug(' %s', d) logging.debug('Should reboot:') for d in should_reboot_device: logging.debug(' %s', d) if blacklist: blacklist.Reset() if should_restart_adb: KillAllAdb() for serial in should_restart_usb: try: reset_usb.reset_android_usb(serial) except IOError: logging.exception('Unable to reset USB for %s.', serial) if blacklist: blacklist.Extend([serial], reason='usb_failure') except device_errors.DeviceUnreachableError: logging.exception('Unable to reset USB for %s.', serial) if blacklist: blacklist.Extend([serial], reason='offline') def blacklisting_recovery(device): if _IsBlacklisted(device.adb.GetDeviceSerial(), blacklist): logging.debug('%s is blacklisted, skipping recovery.', str(device)) return if str(device) in should_reboot_device: try: device.WaitUntilFullyBooted(retries=0) return except (device_errors.CommandTimeoutError, device_errors.CommandFailedError): logging.exception('Failure while waiting for %s. ' 'Attempting to recover.', str(device)) try: try: device.Reboot(block=False, timeout=5, retries=0) except device_errors.CommandTimeoutError: logging.warning('Timed out while attempting to reboot %s normally.' 'Attempting alternative reboot.', str(device)) # The device drops offline before we can grab the exit code, so # we don't check for status. device.adb.Root() device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None, timeout=5, retries=0) except device_errors.CommandFailedError: logging.exception('Failed to reboot %s.', str(device)) if blacklist: blacklist.Extend([device.adb.GetDeviceSerial()], reason='reboot_failure') except device_errors.CommandTimeoutError: logging.exception('Timed out while rebooting %s.', str(device)) if blacklist: blacklist.Extend([device.adb.GetDeviceSerial()], reason='reboot_timeout') try: device.WaitUntilFullyBooted(retries=0) except device_errors.CommandFailedError: logging.exception('Failure while waiting for %s.', str(device)) if blacklist: blacklist.Extend([device.adb.GetDeviceSerial()], reason='reboot_failure') except device_errors.CommandTimeoutError: logging.exception('Timed out while waiting for %s.', str(device)) if blacklist: blacklist.Extend([device.adb.GetDeviceSerial()], reason='reboot_timeout') device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery) def main(): parser = argparse.ArgumentParser() parser.add_argument('--out-dir', help='Directory where the device path is stored', default=os.path.join(host_paths.DIR_SOURCE_ROOT, 'out')) parser.add_argument('--restart-usb', action='store_true', help='DEPRECATED. ' 'This script now always tries to reset USB.') parser.add_argument('--json-output', help='Output JSON information into a specified file.') parser.add_argument('--adb-path', help='Absolute path to the adb binary to use.') parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') parser.add_argument('--known-devices-file', action='append', default=[], dest='known_devices_files', help='Path to known device lists.') parser.add_argument('-v', '--verbose', action='count', default=1, help='Log more information.') args = parser.parse_args() run_tests_helper.SetLogLevel(args.verbose) devil_custom_deps = None if args.adb_path: devil_custom_deps = { 'adb': { devil_env.GetPlatform(): [args.adb_path], }, } devil_chromium.Initialize(custom_deps=devil_custom_deps) blacklist = (device_blacklist.Blacklist(args.blacklist_file) if args.blacklist_file else None) last_devices_path = os.path.join( args.out_dir, device_list.LAST_DEVICES_FILENAME) args.known_devices_files.append(last_devices_path) expected_devices = set() try: for path in args.known_devices_files: if os.path.exists(path): expected_devices.update(device_list.GetPersistentDeviceList(path)) except IOError: logging.warning('Problem reading %s, skipping.', path) logging.info('Expected devices:') for device in expected_devices: logging.info(' %s', device) usb_devices = set(lsusb.get_android_devices()) devices = [device_utils.DeviceUtils(s) for s in expected_devices.union(usb_devices)] RecoverDevices(devices, blacklist) statuses = DeviceStatus(devices, blacklist) # Log the state of all devices. for status in statuses: logging.info(status['serial']) adb_status = status.get('adb_status') blacklisted = status.get('blacklisted') logging.info(' USB status: %s', 'online' if status.get('usb_status') else 'offline') logging.info(' ADB status: %s', adb_status) logging.info(' Blacklisted: %s', str(blacklisted)) if adb_status == 'device' and not blacklisted: logging.info(' Device type: %s', status.get('ro.build.product')) logging.info(' OS build: %s', status.get('ro.build.id')) logging.info(' OS build fingerprint: %s', status.get('ro.build.fingerprint')) logging.info(' Battery state:') for k, v in status.get('battery', {}).iteritems(): logging.info(' %s: %s', k, v) logging.info(' IMEI slice: %s', status.get('imei_slice')) logging.info(' WiFi IP: %s', status.get('wifi_ip')) # Update the last devices file(s). for path in args.known_devices_files: device_list.WritePersistentDeviceList( path, [status['serial'] for status in statuses]) # Write device info to file for buildbot info display. if os.path.exists('/home/chrome-bot'): with open('/home/chrome-bot/.adb_device_info', 'w') as f: for status in statuses: try: if status['adb_status'] == 'device': f.write('{serial} {adb_status} {build_product} {build_id} ' '{temperature:.1f}C {level}%\n'.format( serial=status['serial'], adb_status=status['adb_status'], build_product=status['type'], build_id=status['build'], temperature=float(status['battery']['temperature']) / 10, level=status['battery']['level'] )) elif status.get('usb_status', False): f.write('{serial} {adb_status}\n'.format( serial=status['serial'], adb_status=status['adb_status'] )) else: f.write('{serial} offline\n'.format( serial=status['serial'] )) except Exception: # pylint: disable=broad-except pass # Dump the device statuses to JSON. if args.json_output: with open(args.json_output, 'wb') as f: f.write(json.dumps(statuses, indent=4)) live_devices = [status['serial'] for status in statuses if (status['adb_status'] == 'device' and not _IsBlacklisted(status['serial'], blacklist))] # If all devices failed, or if there are no devices, it's an infra error. return 0 if live_devices else exit_codes.INFRA if __name__ == '__main__': sys.exit(main())