[Android] Revise recovery logic in device_status_check.

The current logic seems to be causing problems on the N6 perf bot:
http://build.chromium.org/p/chromium.perf/builders/Android%20Nexus6%20Perf

BUG=

Review URL: https://codereview.chromium.org/1299573002

Cr-Commit-Position: refs/heads/master@{#343602}

1 files changed, 48 insertions, 29 deletions

diff --git a/build/android/buildbot/bb_device_status_check.py b/build/android/buildbot/bb_device_status_check.py
index abfe78f..52261ac 100755
--- a/build/android/buildbot/bb_device_status_check.py
+++ b/build/android/buildbot/bb_device_status_check.py
@@ -224,47 +224,38 @@ def KillAllAdb():
       pass
 
 
-def main():
-  parser = optparse.OptionParser()
-  parser.add_option('', '--out-dir',
-                    help='Directory where the device path is stored',
-                    default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
-  parser.add_option('--no-provisioning-check', action='store_true',
-                    help='Will not check if devices are provisioned properly.')
-  parser.add_option('--device-status-dashboard', action='store_true',
-                    help='Output device status data for dashboard.')
-  parser.add_option('--restart-usb', action='store_true',
-                    help='DEPRECATED. '
-                         'This script now always tries to reset USB.')
-  parser.add_option('--json-output',
-                    help='Output JSON information into a specified file.')
-  parser.add_option('-v', '--verbose', action='count', default=1,
-                    help='Log more information.')
-
-  options, args = parser.parse_args()
-  if args:
-    parser.error('Unknown options %s' % args)
-
-  run_tests_helper.SetLogLevel(options.verbose)
-
+def RecoverDevices(args):
   # Remove the last build's "bad devices" before checking device statuses.
   device_blacklist.ResetBlacklist()
 
+  previous_devices = set(a.GetDeviceSerial()
+                         for a in adb_wrapper.AdbWrapper.Devices())
+
   KillAllAdb()
   reset_usb.reset_all_android_devices()
 
   try:
     expected_devices = set(device_list.GetPersistentDeviceList(
-        os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME)))
+        os.path.join(args.out_dir, device_list.LAST_DEVICES_FILENAME)))
   except IOError:
     expected_devices = set()
 
-  def all_devices_found():
-    devices = device_utils.DeviceUtils.HealthyDevices()
-    device_serials = set(d.adb.GetDeviceSerial() for d in devices)
-    return not bool(expected_devices.difference(device_serials))
+  all_devices = [device_utils.DeviceUtils(d)
+                 for d in previous_devices.union(expected_devices)]
 
-  timeout_retry.WaitFor(all_devices_found, wait_period=1, max_tries=5)
+  def blacklisting_recovery(device):
+    try:
+      device.WaitUntilFullyBooted()
+    except device_errors.CommandFailedError:
+      logging.exception('Failure while waiting for %s. Adding to blacklist.',
+                        str(device))
+      device_blacklist.ExtendBlacklist([str(device)])
+    except device_errors.CommandTimeoutError:
+      logging.exception('Timed out while waiting for %s. Adding to blacklist.',
+                        str(device))
+      device_blacklist.ExtendBlacklist([str(device)])
+
+  device_utils.DeviceUtils.parallel(all_devices).pMap(blacklisting_recovery)
 
   devices = device_utils.DeviceUtils.HealthyDevices()
   device_serials = set(d.adb.GetDeviceSerial() for d in devices)
@@ -280,6 +271,34 @@ def main():
     for d in sorted(device_serials):
       logging.warning('  %s', d)
 
+  return devices
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('', '--out-dir',
+                    help='Directory where the device path is stored',
+                    default=os.path.join(constants.DIR_SOURCE_ROOT, 'out'))
+  parser.add_option('--no-provisioning-check', action='store_true',
+                    help='Will not check if devices are provisioned properly.')
+  parser.add_option('--device-status-dashboard', action='store_true',
+                    help='Output device status data for dashboard.')
+  parser.add_option('--restart-usb', action='store_true',
+                    help='DEPRECATED. '
+                         'This script now always tries to reset USB.')
+  parser.add_option('--json-output',
+                    help='Output JSON information into a specified file.')
+  parser.add_option('-v', '--verbose', action='count', default=1,
+                    help='Log more information.')
+
+  options, args = parser.parse_args()
+  if args:
+    parser.error('Unknown options %s' % args)
+
+  run_tests_helper.SetLogLevel(options.verbose)
+
+  devices = RecoverDevices(options)
+
   types, builds, batteries, errors, devices_ok, json_data = (
       [], [], [], [], [], [])
   if devices: