Add option to restart usb on device status check before performing check.

Restart USB has been known to bring back offline devices. Add option to device
status check to restart usb ports before performing the check. This requires
restart_usb to be installed (see
https://code.google.com/p/chromium/issues/detail?id=305769). Prints warning if
the utility has not been installed on the host.

We will add this first to perf bots. Restarting usb has fixed device issues on these bots. If we find it works and does not have unseen consequences, we will deploy on all bots (i.e. make default of --restart-usb to True).

BUG=299891

Review URL: https://codereview.chromium.org/26747004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@229553 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 56 insertions, 3 deletions

diff --git a/build/android/buildbot/bb_device_status_check.py b/build/android/buildbot/bb_device_status_check.py
index a5ef54b3..e279b50 100755
--- a/build/android/buildbot/bb_device_status_check.py
+++ b/build/android/buildbot/bb_device_status_check.py
@@ -9,11 +9,13 @@ import logging
 import optparse
 import os
 import smtplib
+import subprocess
 import sys
 import re
 import urllib
 
 import bb_annotations
+import bb_utils
 
 sys.path.append(os.path.join(os.path.dirname(__file__),
                              os.pardir, os.pardir, 'util', 'lib',
@@ -203,6 +205,42 @@ def SendDeviceStatusAlert(msg):
     print 'Failed to send alert email. Error: %s' % e
 
 
+def RestartUsb():
+  if not os.path.isfile('/usr/bin/restart_usb'):
+    print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed '
+           'on host (see BUG=305769).')
+    return 1
+
+  lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE)
+  lsusb_output, _ = lsusb_proc.communicate()
+  if lsusb_proc.returncode:
+    print ('Error: Could not get list of USB ports (i.e. lsusb).')
+    return lsusb_proc.returncode
+
+  usb_devices = [re.findall('Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0]
+                 for lsusb_line in lsusb_output.strip().split('\n')]
+
+  failed_restart = False
+  # Walk USB devices from leaves up (i.e reverse sorted) restarting the
+  # connection. If a parent node (e.g. usb hub) is restarted before the
+  # devices connected to it, the (bus, dev) for the hub can change, making the
+  # output we have wrong. This way we restart the devices before the hub.
+  for (bus, dev) in reversed(sorted(usb_devices)):
+    # Can not restart root usb connections
+    if dev != '001':
+      return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev])
+      if return_code:
+        print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev)
+        failed_restart = True
+      else:
+        print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev)
+
+  if failed_restart:
+    return 1
+
+  return 0
+
+
 def main():
   parser = optparse.OptionParser()
   parser.add_option('', '--out-dir',
@@ -212,9 +250,17 @@ def main():
                     help='Will not check if devices are provisioned properly.')
   parser.add_option('--device-status-dashboard', action='store_true',
                     help='Output device status data for dashboard.')
+  parser.add_option('--restart-usb', action='store_true',
+                    help='Restart USB ports before running device check.')
   options, args = parser.parse_args()
   if args:
     parser.error('Unknown options %s' % args)
+
+  if options.restart_usb:
+    rc = RestartUsb()
+    if rc:
+      return 1
+
   devices = android_commands.GetAttachedDevices()
   # TODO(navabi): Test to make sure this fails and then fix call
   offline_devices = android_commands.GetAttachedDevices(hardware=False,
diff --git a/build/android/buildbot/bb_device_steps.py b/build/android/buildbot/bb_device_steps.py
index 7129f37..232d119 100755
--- a/build/android/buildbot/bb_device_steps.py
+++ b/build/android/buildbot/bb_device_steps.py
@@ -396,10 +396,12 @@ def ProvisionDevices(options):
   RunCmd(provision_cmd)
 
 
-def DeviceStatusCheck(_):
+def DeviceStatusCheck(options):
   bb_annotations.PrintNamedStep('device_status_check')
-  RunCmd(['build/android/buildbot/bb_device_status_check.py'],
-         halt_on_failure=True)
+  cmd = ['build/android/buildbot/bb_device_status_check.py']
+  if options.restart_usb:
+    cmd.append('--restart-usb')
+  RunCmd(cmd, halt_on_failure=True)
 
 
 def GetDeviceSetupStepCmds():
@@ -560,6 +562,8 @@ def GetDeviceStepsOptParser():
   parser.add_option('--coverage-bucket',
                     help=('Bucket name to store coverage results. Coverage is '
                           'only run if this is set.'))
+  parser.add_option('--restart-usb', action='store_true',
+                    help='Restart usb ports before device status check.')
   parser.add_option(
       '--flakiness-server',
       help=('The flakiness dashboard server to which the results should be '
diff --git a/build/android/buildbot/bb_utils.py b/build/android/buildbot/bb_utils.py
index f901103..3c16cc2 100644
--- a/build/android/buildbot/bb_utils.py
+++ b/build/android/buildbot/bb_utils.py
@@ -45,6 +45,9 @@ def SpawnCmd(command, stdout=None, cwd=CHROME_SRC):
       @staticmethod
       def wait():
         return 0
+      @staticmethod
+      def communicate():
+        return '', ''
     return MockPopen()
   return subprocess.Popen(command, cwd=cwd, stdout=stdout)