GTTF: Include info about disabled tests and platforms in JSON test summary.

This will make it possible to analyze which tests are disabled on specific
platforms, and whether the failures are really platform-specific.

Including list of all tests in the summary makes it possible to e.g. detect
when a test first appeared (or disappeared), which platforms a test is compiled
on, or whether some tests have been filtered out.

BUG=236893
R=sky@chromium.org

Review URL: https://codereview.chromium.org/93603008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@243232 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 121 insertions, 4 deletions

diff --git a/base/test/launcher/test_launcher.cc b/base/test/launcher/test_launcher.cc
index aec30b6..ad3a58c 100644
--- a/base/test/launcher/test_launcher.cc
+++ b/base/test/launcher/test_launcher.cc
@@ -24,6 +24,7 @@
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
+#include "base/strings/stringize_macros.h"
 #include "base/strings/stringprintf.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/test/launcher/test_results_tracker.h"
@@ -685,6 +686,68 @@ bool TestLauncher::Init() {
     return 1;
   }
 
+#if defined(NDEBUG)
+  results_tracker_.AddGlobalTag("MODE_RELEASE");
+#else
+  results_tracker_.AddGlobalTag("MODE_DEBUG");
+#endif
+
+  // Operating systems (sorted alphabetically).
+  // Note that they can deliberately overlap, e.g. OS_LINUX is a subset
+  // of OS_POSIX.
+#if defined(OS_ANDROID)
+  results_tracker_.AddGlobalTag("OS_ANDROID");
+#endif
+
+#if defined(OS_BSD)
+  results_tracker_.AddGlobalTag("OS_BSD");
+#endif
+
+#if defined(OS_FREEBSD)
+  results_tracker_.AddGlobalTag("OS_FREEBSD");
+#endif
+
+#if defined(OS_IOS)
+  results_tracker_.AddGlobalTag("OS_IOS");
+#endif
+
+#if defined(OS_LINUX)
+  results_tracker_.AddGlobalTag("OS_LINUX");
+#endif
+
+#if defined(OS_MACOSX)
+  results_tracker_.AddGlobalTag("OS_MACOSX");
+#endif
+
+#if defined(OS_NACL)
+  results_tracker_.AddGlobalTag("OS_NACL");
+#endif
+
+#if defined(OS_OPENBSD)
+  results_tracker_.AddGlobalTag("OS_OPENBSD");
+#endif
+
+#if defined(OS_POSIX)
+  results_tracker_.AddGlobalTag("OS_POSIX");
+#endif
+
+#if defined(OS_SOLARIS)
+  results_tracker_.AddGlobalTag("OS_SOLARIS");
+#endif
+
+#if defined(OS_WIN)
+  results_tracker_.AddGlobalTag("OS_WIN");
+#endif
+
+  // CPU-related tags.
+#if defined(ARCH_CPU_32_BITS)
+  results_tracker_.AddGlobalTag("CPU_32_BITS");
+#endif
+
+#if defined(ARCH_CPU_64_BITS)
+  results_tracker_.AddGlobalTag("CPU_64_BITS");
+#endif
+
   return true;
 }
 
@@ -703,11 +766,15 @@ void TestLauncher::RunTests() {
       test_name.append(".");
       test_name.append(test_info->name());
 
-      // Skip disabled tests.
+      results_tracker_.AddTest(test_name);
+
       const CommandLine* command_line = CommandLine::ForCurrentProcess();
-      if (test_name.find("DISABLED") != std::string::npos &&
-          !command_line->HasSwitch(kGTestRunDisabledTestsFlag)) {
-        continue;
+      if (test_name.find("DISABLED") != std::string::npos) {
+        results_tracker_.AddDisabledTest(test_name);
+
+        // Skip disabled tests unless explicitly requested.
+        if (!command_line->HasSwitch(kGTestRunDisabledTestsFlag))
+          continue;
       }
 
       std::string filtering_test_name =
diff --git a/base/test/launcher/test_results_tracker.cc b/base/test/launcher/test_results_tracker.cc
index 3e68093..9841e5c 100644
--- a/base/test/launcher/test_results_tracker.cc
+++ b/base/test/launcher/test_results_tracker.cc
@@ -12,6 +12,7 @@
 #include "base/json/json_file_value_serializer.h"
 #include "base/json/string_escape.h"
 #include "base/logging.h"
+#include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/test/launcher/test_launcher.h"
 #include "base/values.h"
@@ -47,6 +48,12 @@ void PrintTests(InputIterator first,
   fflush(stdout);
 }
 
+std::string TestNameWithoutDisabledPrefix(const std::string& test_name) {
+  std::string test_name_no_disabled(test_name);
+  ReplaceSubstringsAfterOffset(&test_name_no_disabled, 0, "DISABLED_", "");
+  return test_name_no_disabled;
+}
+
 }  // namespace
 
 TestResultsTracker::TestResultsTracker() : iteration_(-1), out_(NULL) {
@@ -155,6 +162,18 @@ void TestResultsTracker::OnTestIterationStarting() {
   per_iteration_data_.push_back(PerIterationData());
 }
 
+void TestResultsTracker::AddTest(const std::string& test_name) {
+  // Record disabled test names without DISABLED_ prefix so that they are easy
+  // to compare with regular test names, e.g. before or after disabling.
+  all_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
+}
+
+void TestResultsTracker::AddDisabledTest(const std::string& test_name) {
+  // Record disabled test names without DISABLED_ prefix so that they are easy
+  // to compare with regular test names, e.g. before or after disabling.
+  disabled_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
+}
+
 void TestResultsTracker::AddTestResult(const TestResult& result) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
@@ -252,6 +271,24 @@ bool TestResultsTracker::SaveSummaryAsJSON(const FilePath& path) const {
     global_tags->AppendString(*i);
   }
 
+  ListValue* all_tests = new ListValue;
+  summary_root->Set("all_tests", all_tests);
+
+  for (std::set<std::string>::const_iterator i = all_tests_.begin();
+       i != all_tests_.end();
+       ++i) {
+    all_tests->AppendString(*i);
+  }
+
+  ListValue* disabled_tests = new ListValue;
+  summary_root->Set("disabled_tests", disabled_tests);
+
+  for (std::set<std::string>::const_iterator i = disabled_tests_.begin();
+       i != disabled_tests_.end();
+       ++i) {
+    disabled_tests->AppendString(*i);
+  }
+
   ListValue* per_iteration_data = new ListValue;
   summary_root->Set("per_iteration_data", per_iteration_data);
 
diff --git a/base/test/launcher/test_results_tracker.h b/base/test/launcher/test_results_tracker.h
index e129599..dbf3d04 100644
--- a/base/test/launcher/test_results_tracker.h
+++ b/base/test/launcher/test_results_tracker.h
@@ -41,6 +41,13 @@ class TestResultsTracker {
   // Called when a test iteration is starting.
   void OnTestIterationStarting();
 
+  // Adds |test_name| to the set of discovered tests (this includes all tests
+  // present in the executable, not necessarily run).
+  void AddTest(const std::string& test_name);
+
+  // Adds |test_name| to the set of disabled tests.
+  void AddDisabledTest(const std::string& test_name);
+
   // Adds |result| to the stored test results.
   void AddTestResult(const TestResult& result);
 
@@ -81,6 +88,12 @@ class TestResultsTracker {
   // the entire test run.
   std::set<std::string> global_tags_;
 
+  // Set of all test names discovered in the current executable.
+  std::set<std::string> all_tests_;
+
+  // Set of all disabled tests in the current executable.
+  std::set<std::string> disabled_tests_;
+
   // Store test results for each iteration.
   std::vector<PerIterationData> per_iteration_data_;