Run mojo apptests up to three attempts on failure.

Attempt to re-run apptests fixtures (or suites) on failure.
TODO: don't repeat the entire suite in non-isolated mode...
Start with 3 attempts; just print failures (no other reporting).

Abort running an apptest suite with 20+ fixture failures.
(this only works for apptests running with fixture isolation)
Abort the runner with 3+ apptest suites exhibiting failures.

BUG=493535
TEST=Infrequent flaky failures cause fewer headaches...
R=sky@chromium.org

Review URL: https://codereview.chromium.org/1200523002

Cr-Commit-Position: refs/heads/master@{#335708}

2 files changed, 26 insertions, 2 deletions

diff --git a/mojo/tools/apptest_runner.py b/mojo/tools/apptest_runner.py
index f216f38..6e0d2a3 100755
--- a/mojo/tools/apptest_runner.py
+++ b/mojo/tools/apptest_runner.py
@@ -55,6 +55,7 @@ def main():
 
   tests = []
   failed = []
+  failed_suites = 0
   for _ in range(args.repeat_count):
     for test_dict in test_list:
       test = test_dict["test"]
@@ -73,6 +74,13 @@ def main():
       result = test and not fail
       print "[  PASSED  ]" if result else "[  FAILED  ]",
       print test_name if args.verbose or not result else ""
+      # Abort when 3 apptest suites, or a tenth of all, have failed.
+      # base::TestLauncher does this for timeouts and unknown results.
+      failed_suites += 0 if result else 1
+      if failed_suites >= max(3, len(test_list) / 10):
+        print "Too many failing suites (%d), exiting now." % failed_suites
+        failed.append("Test runner aborted for excessive failures.")
+        break;
 
     if failed:
       break;
diff --git a/mojo/tools/mopy/gtest.py b/mojo/tools/mopy/gtest.py
index b4ffbcd..ed9848a 100644
--- a/mojo/tools/mopy/gtest.py
+++ b/mojo/tools/mopy/gtest.py
@@ -37,17 +37,33 @@ def run_apptest(config, shell, args, apptest, isolate):
   failed = []
   if not isolate:
     # TODO(msw): Parse fixture-granular successes and failures in this case.
-    if not _run_apptest(config, shell, args, apptest):
+    # TODO(msw): Retry fixtures that failed, not the entire apptest suite.
+    if not _run_apptest_with_retry(config, shell, args, apptest):
       failed.append(apptest)
   else:
     tests = _get_fixtures(config, shell, args, apptest)
     for fixture in tests:
       arguments = args + ["--gtest_filter=%s" % fixture]
-      if not _run_apptest(config, shell, arguments, apptest):
+      if not _run_apptest_with_retry(config, shell, arguments, apptest):
         failed.append(fixture)
+      # Abort when 20 fixtures, or a tenth of the apptest fixtures, have failed.
+      # base::TestLauncher does this for timeouts and unknown results.
+      if len(failed) >= max(20, len(tests) / 10):
+        print "Too many failing fixtures (%d), exiting now." % len(failed)
+        return (tests, failed + [apptest + " aborted for excessive failures."])
   return (tests, failed)
 
 
+# TODO(msw): Determine proper test retry counts; allow configuration.
+def _run_apptest_with_retry(config, shell, args, apptest, try_count=3):
+  """Runs an apptest, retrying on failure; returns True if any run passed."""
+  for try_number in range(try_count):
+    if _run_apptest(config, shell, args, apptest):
+      return True
+    print "Failed %s/%s test run attempts." % (try_number + 1, try_count)
+  return False
+
+
 def _run_apptest(config, shell, args, apptest):
   """Runs an apptest and checks the output for signs of gtest failure."""
   command = _build_command_line(config, args, apptest)