Record out-of-proc-tests' FAILS/FLAKY failures in the XML output.

BUG=40473
TEST=Run browser_tests and see if failed FAILS_ or FLAKY_ tests are recorded in its XML output.  Also make sure that failures of FAILS/FLAKY tests don't make the browser_tests exit with non-zero return value.

Review URL: http://codereview.chromium.org/2431003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@48873 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 29 insertions, 9 deletions

diff --git a/base/test/test_suite.h b/base/test/test_suite.h
index 37045da..609a0e1 100644
--- a/base/test/test_suite.h
+++ b/base/test/test_suite.h
@@ -28,6 +28,10 @@
 #include <gtk/gtk.h>
 #endif
 
+// A command-line flag that makes a test failure always result in a non-zero
+// process exit code.
+const char kStrictFailureHandling[] = "strict_failure_handling";
+
 // Match function used by the GetTestCount method.
 typedef bool (*TestMatch)(const testing::TestInfo&);
 
@@ -80,6 +84,8 @@ class TestSuite {
 
   // Returns true if the test failure should be ignored.
   static bool ShouldIgnoreFailure(const testing::TestInfo& test) {
+    if (CommandLine::ForCurrentProcess()->HasSwitch(kStrictFailureHandling))
+      return false;
     return IsMarkedFlaky(test) || IsMarkedFailing(test);
   }
 
diff --git a/chrome/test/test_launcher/out_of_proc_test_runner.cc b/chrome/test/test_launcher/out_of_proc_test_runner.cc
index cc522da..c27b051 100644
--- a/chrome/test/test_launcher/out_of_proc_test_runner.cc
+++ b/chrome/test/test_launcher/out_of_proc_test_runner.cc
@@ -7,6 +7,7 @@
 #include "base/command_line.h"
 #include "base/logging.h"
 #include "base/process_util.h"
+#include "base/test/test_suite.h"
 
 #include "chrome/test/test_launcher/test_runner.h"
 #include "chrome/test/unit/chrome_test_suite.h"
@@ -61,6 +62,10 @@ class OutOfProcTestRunner : public tests::TestRunner {
     new_cmd_line.AppendSwitchWithValue("gtest_filter", test_name);
     new_cmd_line.AppendSwitch(kChildProcessFlag);
 
+    // Do not let the child ignore failures.  We need to propagate the
+    // failure status back to the parent.
+    new_cmd_line.AppendSwitch(kStrictFailureHandling);
+
     base::ProcessHandle process_handle;
     if (!base::LaunchApp(new_cmd_line, false, false, &process_handle))
       return false;
diff --git a/chrome/test/test_launcher/test_runner.cc b/chrome/test/test_launcher/test_runner.cc
index 33c7586..a13519a 100644
--- a/chrome/test/test_launcher/test_runner.cc
+++ b/chrome/test/test_launcher/test_runner.cc
@@ -14,6 +14,7 @@
 #include "base/process_util.h"
 #include "base/scoped_ptr.h"
 #include "base/string_util.h"
+#include "base/test/test_suite.h"
 #include "base/time.h"
 #include "net/base/escape.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ -45,7 +46,7 @@ class ResultsPrinter {
   void OnTestCaseStart(const char* name, int test_count) const;
   void OnTestCaseEnd() const;
   void OnTestEnd(const char* name, const char* case_name, bool run,
-                 bool failure, double elapsed_time) const;
+                 bool failed, bool failure_ignored, double elapsed_time) const;
  private:
   FILE* out_;
 };
@@ -110,19 +111,20 @@ void ResultsPrinter::OnTestCaseEnd() const {
 }
 
 void ResultsPrinter::OnTestEnd(const char* name, const char* case_name,
-                               bool run, bool failure,
+                               bool run, bool failed, bool failure_ignored,
                                double elapsed_time) const {
   if (!out_)
     return;
   fprintf(out_, "    <testcase name=\"%s\" status=\"%s\" time=\"%.3f\""
           " classname=\"%s\"",
           name, run ? "run" : "notrun", elapsed_time / 1000.0, case_name);
-  if (!failure) {
+  if (!failed) {
     fprintf(out_, " />\n");
     return;
   }
   fprintf(out_, ">\n");
-  fprintf(out_, "      <failure message=\"\" type=\"\"></failure>\n");
+  fprintf(out_, "      <failure message=\"\" type=\"\"%s></failure>\n",
+          failure_ignored ? " ignored=\"true\"" : "");
   fprintf(out_, "    </testcase>\n");
 }
 
@@ -202,6 +204,7 @@ bool RunTests(const TestRunnerFactory& test_runner_factory) {
       kGTestFilterFlag);
 
   int test_run_count = 0;
+  int ignored_failure_count = 0;
   std::vector<std::string> failed_tests;
 
   ResultsPrinter printer(*command_line);
@@ -215,7 +218,7 @@ bool RunTests(const TestRunnerFactory& test_runner_factory) {
       if (std::string(test_info->name()).find("DISABLED") == 0 &&
           !command_line->HasSwitch(kGTestRunDisabledTestsFlag)) {
         printer.OnTestEnd(test_info->name(), test_case->name(),
-                          false, false, 0);
+                          false, false, false, 0);
         continue;
       }
       std::string test_name = test_info->test_case_name();
@@ -224,7 +227,7 @@ bool RunTests(const TestRunnerFactory& test_runner_factory) {
       // Skip the test that doesn't match the filter string (if given).
       if (filter_string.size() && !MatchesFilter(test_name, filter_string)) {
         printer.OnTestEnd(test_info->name(), test_case->name(),
-                          false, false, 0);
+                          false, false, false, 0);
         continue;
       }
       base::Time start_time = base::Time::Now();
@@ -235,19 +238,25 @@ bool RunTests(const TestRunnerFactory& test_runner_factory) {
       ++test_run_count;
       if (!test_runner->RunTest(test_name.c_str())) {
         failed_tests.push_back(test_name);
+        bool ignore_failure = TestSuite::ShouldIgnoreFailure(*test_info);
         printer.OnTestEnd(test_info->name(), test_case->name(), true, true,
+                          ignore_failure,
                           (base::Time::Now() - start_time).InMillisecondsF());
+        if (ignore_failure)
+          ++ignored_failure_count;
       } else {
         printer.OnTestEnd(test_info->name(), test_case->name(), true, false,
+                          false,
                           (base::Time::Now() - start_time).InMillisecondsF());
       }
     }
   }
 
   printf("%d test%s run\n", test_run_count, test_run_count > 1 ? "s" : "");
-  printf("%d test%s failed\n", static_cast<int>(failed_tests.size()),
-                               failed_tests.size() > 1 ? "s" : "");
-  if (failed_tests.empty())
+  printf("%d test%s failed (%d ignored)\n",
+         static_cast<int>(failed_tests.size()),
+         failed_tests.size() > 1 ? "s" : "", ignored_failure_count);
+  if (failed_tests.size() - ignored_failure_count == 0)
     return true;
 
   printf("Failing tests:\n");