GTTF: Limit number of retried tests.

Since the retries are serial and without batching multiple tests into
the same process, they are quite slow. With lots of retries total test
run time can easily be in hours.

BUG=236893
R=sky@chromium.org

Review URL: https://codereview.chromium.org/115233007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@241243 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 71 insertions, 62 deletions

diff --git a/base/test/launcher/test_launcher.cc b/base/test/launcher/test_launcher.cc
index 329ee2d..c2c618d 100644
--- a/base/test/launcher/test_launcher.cc
+++ b/base/test/launcher/test_launcher.cc
@@ -519,68 +519,6 @@ void TestLauncher::OnTestFinished(const TestResult& result) {
   // We just printed a status line, reset the watchdog timer.
   watchdog_timer_.Reset();
 
-  if (test_finished_count_ == test_started_count_) {
-    if (!tests_to_retry_.empty() && retry_count_ < retry_limit_) {
-      retry_count_++;
-
-      std::vector<std::string> test_names(tests_to_retry_.begin(),
-                                          tests_to_retry_.end());
-
-      tests_to_retry_.clear();
-
-      size_t retry_started_count =
-          launcher_delegate_->RetryTests(this, test_names);
-      if (retry_started_count == 0) {
-        // Signal failure, but continue to run all requested test iterations.
-        // With the summary of all iterations at the end this is a good default.
-        run_result_ = false;
-
-        // The current iteration is done.
-        fprintf(stdout, "%" PRIuS " test%s run\n",
-                test_finished_count_,
-                test_finished_count_ > 1 ? "s" : "");
-        fflush(stdout);
-
-        results_tracker_.PrintSummaryOfCurrentIteration();
-
-        // Kick off the next iteration.
-        MessageLoop::current()->PostTask(
-            FROM_HERE,
-            Bind(&TestLauncher::RunTestIteration, Unretained(this)));
-      } else {
-        fprintf(stdout, "Retrying %" PRIuS " test%s (retry #%" PRIuS ")\n",
-                retry_started_count,
-                retry_started_count > 1 ? "s" : "",
-                retry_count_);
-        fflush(stdout);
-
-        test_started_count_ += retry_started_count;
-      }
-    } else {
-      // The current iteration is done.
-      fprintf(stdout, "%" PRIuS " test%s run\n",
-              test_finished_count_,
-              test_finished_count_ > 1 ? "s" : "");
-      fflush(stdout);
-
-      results_tracker_.PrintSummaryOfCurrentIteration();
-
-      // When we retry tests, success is determined by having nothing more
-      // to retry (everything eventually passed), as opposed to having
-      // no failures at all.
-      if (!tests_to_retry_.empty()) {
-        // Signal failure, but continue to run all requested test iterations.
-        // With the summary of all iterations at the end this is a good default.
-        run_result_ = false;
-      }
-
-      // Kick off the next iteration.
-      MessageLoop::current()->PostTask(
-          FROM_HERE,
-          Bind(&TestLauncher::RunTestIteration, Unretained(this)));
-    }
-  }
-
   // Do not waste time on timeouts. We include tests with unknown results here
   // because sometimes (e.g. hang in between unit tests) that's how a timeout
   // gets reported.
@@ -604,6 +542,50 @@ void TestLauncher::OnTestFinished(const TestResult& result) {
 
     exit(1);
   }
+
+  if (test_finished_count_ == test_started_count_) {
+    if (!tests_to_retry_.empty()) {
+      if (retry_count_ < retry_limit_ &&
+          tests_to_retry_.size() < broken_threshold) {
+        retry_count_++;
+
+        std::vector<std::string> test_names(tests_to_retry_.begin(),
+                                            tests_to_retry_.end());
+
+        tests_to_retry_.clear();
+
+        size_t retry_started_count =
+            launcher_delegate_->RetryTests(this, test_names);
+        if (retry_started_count == 0) {
+          // Signal failure, but continue to run all requested test iterations.
+          // With the summary of all iterations at the end this is a good
+          // default.
+          run_result_ = false;
+
+          OnTestIterationFinished();
+        } else {
+          fprintf(stdout, "Retrying %" PRIuS " test%s (retry #%" PRIuS ")\n",
+                  retry_started_count,
+                  retry_started_count > 1 ? "s" : "",
+                  retry_count_);
+          fflush(stdout);
+
+          test_started_count_ += retry_started_count;
+        }
+      } else {
+        fprintf(stdout,
+                "Too many failing tests (%" PRIuS "), skipping retries.\n",
+                tests_to_retry_.size());
+        fflush(stdout);
+
+        results_tracker_.AddGlobalTag("BROKEN_TEST_SKIPPED_RETRIES");
+
+        OnTestIterationFinished();
+      }
+    } else {
+      OnTestIterationFinished();
+    }
+  }
 }
 
 bool TestLauncher::Init() {
@@ -820,6 +802,30 @@ void TestLauncher::OnLaunchTestProcessFinished(
   callback.Run(exit_code, elapsed_time, was_timeout, output);
 }
 
+void TestLauncher::OnTestIterationFinished() {
+  // The current iteration is done.
+  fprintf(stdout, "%" PRIuS " test%s run\n",
+          test_finished_count_,
+          test_finished_count_ > 1 ? "s" : "");
+  fflush(stdout);
+
+  results_tracker_.PrintSummaryOfCurrentIteration();
+
+  // When we retry tests, success is determined by having nothing more
+  // to retry (everything eventually passed), as opposed to having
+  // no failures at all.
+  if (!tests_to_retry_.empty()) {
+    // Signal failure, but continue to run all requested test iterations.
+    // With the summary of all iterations at the end this is a good default.
+    run_result_ = false;
+  }
+
+  // Kick off the next iteration.
+  MessageLoop::current()->PostTask(
+      FROM_HERE,
+      Bind(&TestLauncher::RunTestIteration, Unretained(this)));
+}
+
 void TestLauncher::OnOutputTimeout() {
   DCHECK(thread_checker_.CalledOnValidThread());
 
diff --git a/base/test/launcher/test_launcher.h b/base/test/launcher/test_launcher.h
index 3907c75..94eafa8 100644
--- a/base/test/launcher/test_launcher.h
+++ b/base/test/launcher/test_launcher.h
@@ -129,6 +129,9 @@ class TestLauncher {
       bool was_timeout,
       const std::string& output);
 
+  // Called when a test iteration is finished.
+  void OnTestIterationFinished();
+
   // Called by the delay timer when no output was made for a while.
   void OnOutputTimeout();