[TTF] [GTTF] Implement the FAILS_ prefix for tests.

After this change, the meaning of the prefixes is as follows:

- DISABLED_: the test is not run unless --gtest_also_run_disabled_tests
  flag is used; should be used for tests that crash or exceed the time limit
- FLAKY_: for tests that behave intermittently, and don't crash nor time out
- FAILS_: for tests that fail consistently, but don't crash nor time out

The Testing Task Force aims to run as much tests as possible, to keep
the coverage high.

The Green Tree Task Force aims to fix the flaky tests because they are
unpredictable.

To satisfy both goals, we should differentiate between the two cases,
and that's why we have both the FLAKY_ and FAILS_ prefixes.

When a test crashes or exceeds time limit (even intermittently),
we have no choice other than disable it, and that's why we have DISABLED_
(and it's built-in the gtest framework anyway).

TEST=none
BUG=none

Review URL: http://codereview.chromium.org/2015013

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@47011 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 25 insertions, 13 deletions

diff --git a/base/test/test_suite.h b/base/test/test_suite.h
index 9aaf055..37045da 100644
--- a/base/test/test_suite.h
+++ b/base/test/test_suite.h
@@ -68,19 +68,25 @@ class TestSuite {
     CommandLine::Reset();
   }
 
-  // Returns true if a string starts with FLAKY_.
-  static bool IsFlaky(const char* name) {
-    return strncmp(name, "FLAKY_", 6) == 0;
+  // Returns true if the test is marked as flaky.
+  static bool IsMarkedFlaky(const testing::TestInfo& test) {
+    return strncmp(test.name(), "FLAKY_", 6) == 0;
   }
 
-  // Returns true if the test is marked as flaky.
-  static bool FlakyTest(const testing::TestInfo& test) {
-    return IsFlaky(test.name()) || IsFlaky(test.test_case_name());
+  // Returns true if the test is marked as failing.
+  static bool IsMarkedFailing(const testing::TestInfo& test) {
+    return strncmp(test.name(), "FAILS_", 6) == 0;
+  }
+
+  // Returns true if the test failure should be ignored.
+  static bool ShouldIgnoreFailure(const testing::TestInfo& test) {
+    return IsMarkedFlaky(test) || IsMarkedFailing(test);
   }
 
-  // Returns true if the test failed and is not marked as flaky.
-  static bool NonFlakyFailures(const testing::TestInfo& test) {
-    return test.should_run() && test.result()->Failed() && !FlakyTest(test);
+  // Returns true if the test failed and the failure shouldn't be ignored.
+  static bool NonIgnoredFailures(const testing::TestInfo& test) {
+    return test.should_run() && test.result()->Failed() &&
+        !ShouldIgnoreFailure(test);
   }
 
   // Returns the number of tests where the match function returns true.
@@ -124,18 +130,24 @@ class TestSuite {
     }
     int result = RUN_ALL_TESTS();
 
-    // Reset the result code if only flaky test failed.
-    if (result != 0 && GetTestCount(&TestSuite::NonFlakyFailures) == 0) {
+    // If there are failed tests, see if we should ignore the failures.
+    if (result != 0 && GetTestCount(&TestSuite::NonIgnoredFailures) == 0)
       result = 0;
-    }
 
     // Display the number of flaky tests.
-    int flaky_count = GetTestCount(&TestSuite::FlakyTest);
+    int flaky_count = GetTestCount(&TestSuite::IsMarkedFlaky);
     if (flaky_count) {
       printf("  YOU HAVE %d FLAKY %s\n\n", flaky_count,
              flaky_count == 1 ? "TEST" : "TESTS");
     }
 
+    // Display the number of tests with ignored failures (FAILS).
+    int failing_count = GetTestCount(&TestSuite::IsMarkedFailing);
+    if (failing_count) {
+      printf("  YOU HAVE %d %s with ignored failures (FAILS prefix)\n\n",
+             failing_count, failing_count == 1 ? "test" : "tests");
+    }
+
     // This MUST happen before Shutdown() since Shutdown() tears down
     // objects (such as NotificationService::current()) that Cocoa
     // objects use to remove themselves as observers.