Increase Android test robustness.

Use of the emulator is probably a lost cause, but maybe this can hold
us for now.

BUG=None
TEST=

Review URL: http://codereview.chromium.org/9185043

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@117458 0039d316-1c4b-4281-b951-d872f2087c98

6 files changed, 67 insertions, 16 deletions

diff --git a/build/android/emulator.py b/build/android/emulator.py
index 67adfb3..07f64bb 100755
--- a/build/android/emulator.py
+++ b/build/android/emulator.py
@@ -92,7 +92,14 @@ class Emulator(object):
   # Time to wait for a "wait for boot complete" (property set on device).
   _WAITFORBOOT_TIMEOUT = 300
 
-  def __init__(self):
+  def __init__(self, fast_and_loose=False):
+    """Init an Emulator.
+
+    Args:
+      fast_and_loose: Loosen up the rules for reliable running for speed.
+        Intended for quick testing or re-testing.
+
+    """
     try:
       android_sdk_root = os.environ['ANDROID_SDK_ROOT']
     except KeyError:
@@ -102,6 +109,7 @@ class Emulator(object):
     self.emulator = os.path.join(android_sdk_root, 'tools', 'emulator')
     self.popen = None
     self.device = None
+    self.fast_and_loose = fast_and_loose
 
   def _DeviceName(self):
     """Return our device name."""
@@ -114,7 +122,8 @@ class Emulator(object):
     If fails, an exception will be raised.
     """
     _KillAllEmulators()  # just to be sure
-    self._AggressiveImageCleanup()
+    if not self.fast_and_loose:
+      self._AggressiveImageCleanup()
     (self.device, port) = self._DeviceName()
     emulator_command = [
         self.emulator,
@@ -123,13 +132,16 @@ class Emulator(object):
         # The default /data size is 64M.
         # That's not enough for 4 unit test bundles and their data.
         '-partition-size', '256',
-        # ALWAYS wipe the data.  We've seen cases where an emulator
-        # gets 'stuck' if we don't do this (every thousand runs or
-        # so).
-        '-wipe-data',
         # Use a familiar name and port.
         '-avd', 'buildbot',
         '-port', str(port)]
+    if not self.fast_and_loose:
+      emulator_command.extend([
+          # Wipe the data.  We've seen cases where an emulator
+          # gets 'stuck' if we don't do this (every thousand runs or
+          # so).
+          '-wipe-data',
+          ])
     logging.info('Emulator launch command: %s', ' '.join(emulator_command))
     self.popen = subprocess.Popen(args=emulator_command,
                                   stderr=subprocess.STDOUT)
diff --git a/build/android/run_tests.py b/build/android/run_tests.py
index 6e3eccb..d864a2e 100755
--- a/build/android/run_tests.py
+++ b/build/android/run_tests.py
@@ -132,7 +132,7 @@ class Xvfb(object):
 
 def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
              timeout, performance_test, cleanup_test_files, tool,
-             log_dump_name):
+             log_dump_name, fast_and_loose=False):
   """Runs the tests.
 
   Args:
@@ -146,6 +146,8 @@ def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
     cleanup_test_files: Whether or not to cleanup test files on device.
     tool: Name of the Valgrind tool.
     log_dump_name: Name of log dump file.
+    fast_and_loose: should we go extra-fast but sacrifice stability
+      and/or correctness?  Intended for quick cycle testing; not for bots!
 
   Returns:
     A TestResults object.
@@ -172,7 +174,8 @@ def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
   for t in _TEST_SUITES:
     test = SingleTestRunner(device, t, gtest_filter, test_arguments,
                             timeout, rebaseline, performance_test,
-                            cleanup_test_files, tool, not not log_dump_name)
+                            cleanup_test_files, tool, not not log_dump_name,
+                            fast_and_loose=fast_and_loose)
     test.RunTests()
     results += [test.test_results]
     # Collect debug info.
@@ -190,6 +193,7 @@ def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
       log_dump_name, [d for d in debug_info_list if d])
   return TestResults.FromTestResults(results)
 
+
 def Dispatch(options):
   """Dispatches the tests, sharding if possible.
 
@@ -214,7 +218,7 @@ def Dispatch(options):
 
   if options.use_emulator:
     t = TimeProfile('Emulator launch')
-    buildbot_emulator = emulator.Emulator()
+    buildbot_emulator = emulator.Emulator(options.fast_and_loose)
     buildbot_emulator.Launch()
     t.Stop()
     attached_devices.append(buildbot_emulator.device)
@@ -230,13 +234,25 @@ def Dispatch(options):
                           options.rebaseline, options.timeout,
                           options.performance_test,
                           options.cleanup_test_files, options.tool,
-                          options.log_dump)
+                          options.log_dump,
+                          fast_and_loose=options.fast_and_loose)
   if buildbot_emulator:
     buildbot_emulator.Shutdown()
   if options.use_xvfb:
     xvfb.Stop()
 
-  return len(test_results.failed)
+  # Another chance if we timed out?  At this point It is safe(r) to
+  # run fast and loose since we just uploaded all the test data and
+  # binary.
+  if test_results.timed_out and options.repeat:
+    logging.critical('Timed out; repeating in fast_and_loose mode.')
+    options.fast_and_loose = True
+    options.repeat = options.repeat - 1
+    logging.critical('Repeats left: ' + str(options.repeat))
+    return Dispatch(options)
+  else:
+    return len(test_results.failed)
+
 
 def ListTestSuites():
   """Display a list of available test suites
@@ -256,7 +272,7 @@ def main(argv):
                            help='Rebaseline and update *testsuite_disabled',
                            action='store_true',
                            default=False)
-  option_parser.add_option('-f', dest='gtest_filter',
+  option_parser.add_option('-f', '--gtest_filter', dest='gtest_filter',
                            help='gtest filter')
   option_parser.add_option('-a', '--test_arguments', dest='test_arguments',
                            help='Additional arguments to pass to the test')
@@ -275,6 +291,17 @@ def main(argv):
   option_parser.add_option('-x', '--xvfb', dest='use_xvfb',
                            action='store_true', default=False,
                            help='Use Xvfb around tests (ignored if not Linux)')
+  option_parser.add_option('--fast', '--fast_and_loose', dest='fast_and_loose',
+                           action='store_true', default=False,
+                           help='Go faster (but be less stable), '
+                           'for quick testing.  Example: when tracking down '
+                           'tests that hang to add to the disabled list, '
+                           'there is no need to redeploy the test binary '
+                           'or data to the device again.  '
+                           'Don\'t use on bots by default!')
+  option_parser.add_option('--repeat', dest='repeat', type='int',
+                           default=2,
+                           help='Repeat count on test timeout')
   options, args = option_parser.parse_args(argv)
   if len(args) > 1:
     print 'Unknown argument:', args[1:]
diff --git a/build/android/single_test_runner.py b/build/android/single_test_runner.py
index 8ec9501..ed422c1 100644
--- a/build/android/single_test_runner.py
+++ b/build/android/single_test_runner.py
@@ -31,7 +31,8 @@ class SingleTestRunner(BaseTestRunner):
 
   def __init__(self, device, test_suite, gtest_filter, test_arguments, timeout,
                rebaseline, performance_test, cleanup_test_files, tool,
-               dump_debug_info=False):
+               dump_debug_info=False,
+               fast_and_loose=False):
     BaseTestRunner.__init__(self, device)
     self._running_on_emulator = self.device.startswith('emulator')
     self._gtest_filter = gtest_filter
@@ -42,6 +43,7 @@ class SingleTestRunner(BaseTestRunner):
            os.path.basename(test_suite), gtest_filter)
     else:
       self.dump_debug_info = None
+    self.fast_and_loose = fast_and_loose
 
     self.test_package = TestPackageExecutable(self.adb, device,
         test_suite, timeout, rebaseline, performance_test, cleanup_test_files,
@@ -205,7 +207,7 @@ class SingleTestRunner(BaseTestRunner):
     self.test_package.StripAndCopyExecutable()
     self.test_package.tool.CopyFiles()
     test_data = self.GetDataFilesForTestSuite()
-    if test_data:
+    if test_data and not self.fast_and_loose:
       if self.test_package.test_suite_basename == 'page_cycler_tests':
         # Since the test data for page cycler are huge (around 200M), we use
         # sdcard to store the data and create symbol links to map them to
diff --git a/build/android/test_package.py b/build/android/test_package.py
index 433e7f1..16b834c 100644
--- a/build/android/test_package.py
+++ b/build/android/test_package.py
@@ -128,6 +128,7 @@ class TestPackage(object):
     """
     ok_tests = []
     failed_tests = []
+    timed_out = False
     re_run = re.compile('\[ RUN      \] ?(.*)\r\n')
     re_fail = re.compile('\[  FAILED  \] ?(.*)\r\n')
     re_ok = re.compile('\[       OK \] ?(.*)\r\n')
@@ -152,6 +153,7 @@ class TestPackage(object):
         if found == 3:  # pexpect.TIMEOUT
           logging.error('Test terminated after %d second timeout.',
                         self.timeout)
+          timed_out = True
         break
     p.close()
     if not self.rebaseline and ready_to_continue:
@@ -163,4 +165,4 @@ class TestPackage(object):
                                         '\npexpect.after: %s'
                                         % (p.before,
                                            p.after))]
-    return TestResults.FromOkAndFailed(ok_tests, failed_tests)
+    return TestResults.FromOkAndFailed(ok_tests, failed_tests, timed_out)
diff --git a/build/android/test_package_executable.py b/build/android/test_package_executable.py
index badea4a..dae8408 100644
--- a/build/android/test_package_executable.py
+++ b/build/android/test_package_executable.py
@@ -43,6 +43,7 @@ class TestPackageExecutable(TestPackage):
 
   def _GetGTestReturnCode(self):
     ret = None
+    ret_code = 1  # Assume failure if we can't find it
     ret_code_file = tempfile.NamedTemporaryFile()
     try:
       if not self.adb.Adb().Pull(
@@ -105,6 +106,9 @@ class TestPackageExecutable(TestPackage):
     cmd_helper.RunCmd(['chmod', '+x', sh_script_file.name])
     self.adb.PushIfNeeded(sh_script_file.name,
                           '/data/local/chrome_test_runner.sh')
+    logging.info('Conents of the test runner script: ')
+    for line in open(sh_script_file.name).readlines():
+      logging.info('  ' + line.rstrip())
 
   def RunTestsAndListResults(self):
     """Runs all the tests and checks for failures.
diff --git a/build/android/test_result.py b/build/android/test_result.py
index eb468f3..0c56436 100644
--- a/build/android/test_result.py
+++ b/build/android/test_result.py
@@ -57,12 +57,14 @@ class TestResults(object):
     self.unknown = []
     self.disabled = []
     self.unexpected_pass = []
+    self.timed_out = False
 
   @staticmethod
-  def FromOkAndFailed(ok, failed):
+  def FromOkAndFailed(ok, failed, timed_out=False):
     ret = TestResults()
     ret.ok = ok
     ret.failed = failed
+    ret.timed_out = timed_out
     return ret
 
   @staticmethod
@@ -76,6 +78,8 @@ class TestResults(object):
       ret.unknown += t.unknown
       ret.disabled += t.disabled
       ret.unexpected_pass += t.unexpected_pass
+      if t.timed_out:
+        ret.timed_out = True
     return ret
 
   def _Log(self, sorted_list):