Retry tests on other bots if the device is unresponsive/offline

Currently this works for offline devices when trying to access files
on devices or emulators.
Use DeviceUnresponsiveError to indicate the device is offline.

BUG=153718
TESTS=

Review URL: https://chromiumcodereview.appspot.com/11232037

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@164013 0039d316-1c4b-4281-b951-d872f2087c98

6 files changed, 64 insertions, 18 deletions

diff --git a/build/android/pylib/android_commands.py b/build/android/pylib/android_commands.py
index ef53556..852a1ce 100644
--- a/build/android/pylib/android_commands.py
+++ b/build/android/pylib/android_commands.py
@@ -110,6 +110,9 @@ def GetAttachedDevices():
     devices.insert(0, preferred_device)
   return devices
 
+def IsDeviceAttached(device):
+  return device in GetAttachedDevices()
+
 def _GetFilesFromRecursiveLsOutput(path, ls_output, re_file, utc_offset=None):
   """Gets a list of files from `ls` command output.
 
@@ -199,6 +202,7 @@ class AndroidCommands(object):
     self._adb = adb_interface.AdbInterface()
     if device:
       self._adb.SetTargetSerial(device)
+    self._device = device
     self._logcat = None
     self.logcat_process = None
     self._pushed_files = []
@@ -1042,14 +1046,20 @@ class AndroidCommands(object):
       True if the file exists, False otherwise.
     """
     assert '"' not in file_name, 'file_name cannot contain double quotes'
-    status = self._adb.SendShellCommand(
-        '\'test -e "%s"; echo $?\'' % (file_name))
-    if 'test: not found' not in status:
+    try:
+      status = self._adb.SendShellCommand(
+          '\'test -e "%s"; echo $?\'' % (file_name))
+      if 'test: not found' not in status:
+        return int(status) == 0
+
+      status = self._adb.SendShellCommand(
+          '\'ls "%s" >/dev/null 2>&1; echo $?\'' % (file_name))
       return int(status) == 0
+    except ValueError:
+      if IsDeviceAttached(self._device):
+        raise errors.DeviceUnresponsiveError('Device may be offline.')
 
-    status = self._adb.SendShellCommand(
-        '\'ls "%s" >/dev/null 2>&1; echo $?\'' % (file_name))
-    return int(status) == 0
+      return False
 
 
 class NewLineNormalizer(object):
diff --git a/build/android/pylib/base_test_sharder.py b/build/android/pylib/base_test_sharder.py
index 48206c2..5306769 100644
--- a/build/android/pylib/base_test_sharder.py
+++ b/build/android/pylib/base_test_sharder.py
@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 
+import android_commands
 import logging
 import multiprocessing
 
@@ -96,8 +97,17 @@ class BaseTestSharder(object):
       # So use map_async instead.
       async_results = pool.map_async(_ShardedTestRunnable, test_runners)
       results_lists = async_results.get(999999)
+
       test_results = TestResults.FromTestResults(results_lists)
-      if retry == self.retries - 1:
+      # Re-check the attached devices for some devices may
+      # become offline
+      retry_devices = set(android_commands.GetAttachedDevices())
+      # Remove devices that had exceptions.
+      retry_devices -= TestResults.DeviceExceptions(results_lists)
+      # Retry on devices that didn't have any exception.
+      self.attached_devices = list(retry_devices)
+      if (retry == self.retries - 1 or
+          len(self.attached_devices) == 0):
         all_passed = final_results.ok + test_results.ok
         final_results = test_results
         final_results.ok = all_passed
diff --git a/build/android/pylib/single_test_runner.py b/build/android/pylib/single_test_runner.py
index a680c68..ae1aa69 100644
--- a/build/android/pylib/single_test_runner.py
+++ b/build/android/pylib/single_test_runner.py
@@ -8,13 +8,15 @@ import os
 import sys
 
 from base_test_runner import BaseTestRunner
+import android_commands
 import debug_info
 import constants
 import perf_tests_helper
 import run_tests_helper
+from android_commands import errors
 from test_package_apk import TestPackageApk
 from test_package_executable import TestPackageExecutable
-from test_result import TestResults
+from test_result import BaseTestResult, TestResults
 
 
 class SingleTestRunner(BaseTestRunner):
@@ -307,14 +309,28 @@ class SingleTestRunner(BaseTestRunner):
     Returns:
       A TestResults object.
     """
-    if self.test_package.rebaseline:
-      self.RebaselineTests()
-    else:
-      if not self._gtest_filter:
-        self._gtest_filter = ('-' + ':'.join(self.GetDisabledTests()) + ':' +
-                             ':'.join(['*.' + x + '*' for x in
-                                     self.test_package.GetDisabledPrefixes()]))
-      self.RunTestsWithFilter()
+    try:
+      if self.test_package.rebaseline:
+        self.RebaselineTests()
+      else:
+        if not self._gtest_filter:
+          self._gtest_filter = ('-' + ':'.join(self.GetDisabledTests()) + ':' +
+                               ':'.join(['*.' + x + '*' for x in
+                                 self.test_package.GetDisabledPrefixes()]))
+        self.RunTestsWithFilter()
+    except errors.DeviceUnresponsiveError as e:
+      # Make sure this device is not attached
+      if android_commands.IsDeviceAttached(self.device):
+        raise e
+
+      # Wrap the results
+      logging.warning(e)
+      failed_tests = []
+      for t in self._gtest_filter.split(':'):
+        failed_tests += [BaseTestResult(t, '')]
+      self.test_results = TestResults.FromRun(
+          failed=failed_tests, device_exception=self.device)
+
     return self.test_results
 
   def SetUp(self):
diff --git a/build/android/pylib/test_package.py b/build/android/pylib/test_package.py
index a47ed72..f2bea72 100644
--- a/build/android/pylib/test_package.py
+++ b/build/android/pylib/test_package.py
@@ -12,6 +12,7 @@ from perf_tests_helper import PrintPerfResult
 from pylib import pexpect
 from test_result import BaseTestResult, TestResults
 
+from android_commands import errors
 
 # TODO(bulach): TestPackage, TestPackageExecutable and
 # TestPackageApk are a work in progress related to making the native tests
@@ -179,6 +180,7 @@ class TestPackage(object):
         failed_tests += [BaseTestResult(full_test_name, p.before)]
     except pexpect.EOF:
       logging.error('Test terminated - EOF')
+      raise errors.DeviceUnresponsiveError('Device may be offline')
     except pexpect.TIMEOUT:
       logging.error('Test terminated after %d second timeout.',
                     self.timeout)
diff --git a/build/android/pylib/test_package_apk.py b/build/android/pylib/test_package_apk.py
index 42b9ade6..598f513 100644
--- a/build/android/pylib/test_package_apk.py
+++ b/build/android/pylib/test_package_apk.py
@@ -11,6 +11,7 @@ import time
 
 import android_commands
 import constants
+from android_commands import errors
 from test_package import TestPackage
 from pylib import pexpect
 
@@ -65,7 +66,8 @@ class TestPackageApk(TestPackage):
         break
       time.sleep(i)
     else:
-      raise Exception('Unable to find fifo on device %s ' % self._GetFifo())
+      raise errors.DeviceUnresponsiveError(
+          'Unable to find fifo on device %s ' % self._GetFifo())
     args = shlex.split(self.adb.Adb()._target_arg)
     args += ['shell', 'cat', self._GetFifo()]
     return pexpect.spawn('adb', args, timeout=timeout, logfile=logfile)
diff --git a/build/android/pylib/test_result.py b/build/android/pylib/test_result.py
index 31a546a..00d139c 100644
--- a/build/android/pylib/test_result.py
+++ b/build/android/pylib/test_result.py
@@ -54,16 +54,18 @@ class TestResults(object):
     self.unknown = []
     self.timed_out = False
     self.overall_fail = False
+    self.device_exception = None
 
   @staticmethod
   def FromRun(ok=None, failed=None, crashed=None, timed_out=False,
-              overall_fail=False):
+              overall_fail=False, device_exception=None):
     ret = TestResults()
     ret.ok = ok or []
     ret.failed = failed or []
     ret.crashed = crashed or []
     ret.timed_out = timed_out
     ret.overall_fail = overall_fail
+    ret.device_exception = device_exception
     return ret
 
   @staticmethod
@@ -109,6 +111,10 @@ class TestResults(object):
     results.failed.append(exc_result)
     return results
 
+  @staticmethod
+  def DeviceExceptions(results):
+    return set(filter(lambda t: t.device_exception, results))
+
   def _Log(self, sorted_list):
     for t in sorted_list:
       logging.critical(t.name)