GTTF: Move core parts of test_launcher down to base.

BUG=236893
R=jam@chromium.org

Review URL: https://codereview.chromium.org/16820008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@207629 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 432 insertions, 0 deletions

diff --git a/base/base.gyp b/base/base.gyp
index 381540a..d27b2b8 100644
--- a/base/base.gyp
+++ b/base/base.gyp
@@ -890,6 +890,8 @@
         'test/test_file_util_mac.cc',
         'test/test_file_util_posix.cc',
         'test/test_file_util_win.cc',
+        'test/test_launcher.cc',
+        'test/test_launcher.h',
         'test/test_listener_ios.h',
         'test/test_listener_ios.mm',
         'test/test_pending_task.cc',
diff --git a/base/test/test_launcher.cc b/base/test/test_launcher.cc
new file mode 100644
index 0000000..7dac374
--- /dev/null
+++ b/base/test/test_launcher.cc
@@ -0,0 +1,378 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/test/test_launcher.h"
+
+#include "base/at_exit.h"
+#include "base/command_line.h"
+#include "base/environment.h"
+#include "base/files/file_path.h"
+#include "base/file_util.h"
+#include "base/logging.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/test/test_timeouts.h"
+#include "base/time.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+#if defined(OS_MACOSX)
+#include "base/mac/scoped_nsautorelease_pool.h"
+#endif
+
+namespace base {
+
+// See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ .
+using ::operator<<;
+
+// The environment variable name for the total number of test shards.
+const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
+// The environment variable name for the test shard index.
+const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
+
+// The default output file for XML output.
+const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL(
+    "test_detail.xml");
+
+namespace {
+
+// Parses the environment variable var as an Int32.  If it is unset, returns
+// default_val.  If it is set, unsets it then converts it to Int32 before
+// returning it.  If unsetting or converting to an Int32 fails, print an
+// error and exit with failure.
+int32 Int32FromEnvOrDie(const char* const var, int32 default_val) {
+  scoped_ptr<Environment> env(Environment::Create());
+  std::string str_val;
+  int32 result;
+  if (!env->GetVar(var, &str_val))
+    return default_val;
+  if (!env->UnSetVar(var)) {
+    LOG(ERROR) << "Invalid environment: we could not unset " << var << ".\n";
+    exit(EXIT_FAILURE);
+  }
+  if (!StringToInt(str_val, &result)) {
+    LOG(ERROR) << "Invalid environment: " << var << " is not an integer.\n";
+    exit(EXIT_FAILURE);
+  }
+  return result;
+}
+
+// Checks whether sharding is enabled by examining the relevant
+// environment variable values.  If the variables are present,
+// but inconsistent (i.e., shard_index >= total_shards), prints
+// an error and exits.
+void InitSharding(int32* total_shards, int32* shard_index) {
+  *total_shards = Int32FromEnvOrDie(kTestTotalShards, 1);
+  *shard_index = Int32FromEnvOrDie(kTestShardIndex, 0);
+
+  if (*total_shards == -1 && *shard_index != -1) {
+    LOG(ERROR) << "Invalid environment variables: you have "
+               << kTestShardIndex << " = " << *shard_index
+               << ", but have left " << kTestTotalShards << " unset.\n";
+    exit(EXIT_FAILURE);
+  } else if (*total_shards != -1 && *shard_index == -1) {
+    LOG(ERROR) << "Invalid environment variables: you have "
+               << kTestTotalShards << " = " << *total_shards
+               << ", but have left " << kTestShardIndex << " unset.\n";
+    exit(EXIT_FAILURE);
+  } else if (*shard_index < 0 || *shard_index >= *total_shards) {
+    LOG(ERROR) << "Invalid environment variables: we require 0 <= "
+               << kTestShardIndex << " < " << kTestTotalShards
+               << ", but you have " << kTestShardIndex << "=" << *shard_index
+               << ", " << kTestTotalShards << "=" << *total_shards << ".\n";
+    exit(EXIT_FAILURE);
+  }
+}
+
+// Given the total number of shards, the shard index, and the test id, returns
+// true iff the test should be run on this shard.  The test id is some arbitrary
+// but unique non-negative integer assigned by this launcher to each test
+// method.  Assumes that 0 <= shard_index < total_shards, which is first
+// verified in ShouldShard().
+bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) {
+  return (test_id % total_shards) == shard_index;
+}
+
+// A helper class to output results.
+// Note: as currently XML is the only supported format by gtest, we don't
+// check output format (e.g. "xml:" prefix) here and output an XML file
+// unconditionally.
+// Note: we don't output per-test-case or total summary info like
+// total failed_test_count, disabled_test_count, elapsed_time and so on.
+// Only each test (testcase element in the XML) will have the correct
+// failed/disabled/elapsed_time information. Each test won't include
+// detailed failure messages either.
+class ResultsPrinter {
+ public:
+  explicit ResultsPrinter(const CommandLine& command_line);
+  ~ResultsPrinter();
+  void OnTestCaseStart(const char* name, int test_count) const;
+  void OnTestCaseEnd() const;
+
+  void OnTestEnd(const char* name, const char* case_name,
+                 bool success, double elapsed_time) const;
+ private:
+  FILE* out_;
+
+  DISALLOW_COPY_AND_ASSIGN(ResultsPrinter);
+};
+
+ResultsPrinter::ResultsPrinter(const CommandLine& command_line) : out_(NULL) {
+  if (!command_line.HasSwitch(kGTestOutputFlag))
+    return;
+  std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag);
+  size_t colon_pos = flag.find(':');
+  FilePath path;
+  if (colon_pos != std::string::npos) {
+    FilePath flag_path =
+        command_line.GetSwitchValuePath(kGTestOutputFlag);
+    FilePath::StringType path_string = flag_path.value();
+    path = FilePath(path_string.substr(colon_pos + 1));
+    // If the given path ends with '/', consider it is a directory.
+    // Note: This does NOT check that a directory (or file) actually exists
+    // (the behavior is same as what gtest does).
+    if (path.EndsWithSeparator()) {
+      FilePath executable = command_line.GetProgram().BaseName();
+      path = path.Append(executable.ReplaceExtension(
+          FilePath::StringType(FILE_PATH_LITERAL("xml"))));
+    }
+  }
+  if (path.value().empty())
+    path = FilePath(kDefaultOutputFile);
+  FilePath dir_name = path.DirName();
+  if (!file_util::DirectoryExists(dir_name)) {
+    LOG(WARNING) << "The output directory does not exist. "
+                 << "Creating the directory: " << dir_name.value();
+    // Create the directory if necessary (because the gtest does the same).
+    file_util::CreateDirectory(dir_name);
+  }
+  out_ = file_util::OpenFile(path, "w");
+  if (!out_) {
+    LOG(ERROR) << "Cannot open output file: "
+               << path.value() << ".";
+    return;
+  }
+  fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+  fprintf(out_, "<testsuites name=\"AllTests\" tests=\"\" failures=\"\""
+          " disabled=\"\" errors=\"\" time=\"\">\n");
+}
+
+ResultsPrinter::~ResultsPrinter() {
+  if (!out_)
+    return;
+  fprintf(out_, "</testsuites>\n");
+  fclose(out_);
+}
+
+void ResultsPrinter::OnTestCaseStart(const char* name, int test_count) const {
+  if (!out_)
+    return;
+  fprintf(out_, "  <testsuite name=\"%s\" tests=\"%d\" failures=\"\""
+          " disabled=\"\" errors=\"\" time=\"\">\n", name, test_count);
+}
+
+void ResultsPrinter::OnTestCaseEnd() const {
+  if (!out_)
+    return;
+  fprintf(out_, "  </testsuite>\n");
+}
+
+void ResultsPrinter::OnTestEnd(const char* name,
+                               const char* case_name,
+                               bool success,
+                               double elapsed_time) const {
+  if (!out_)
+    return;
+  fprintf(out_, "    <testcase name=\"%s\" status=\"run\" time=\"%.3f\""
+          " classname=\"%s\">\n",
+          name, elapsed_time / 1000.0, case_name);
+  if (!success)
+    fprintf(out_, "      <failure message=\"\" type=\"\"></failure>\n");
+  fprintf(out_, "</testcase>\n");
+}
+
+class TestCasePrinterHelper {
+ public:
+  TestCasePrinterHelper(const ResultsPrinter& printer,
+                        const char* name,
+                        int total_test_count)
+      : printer_(printer) {
+    printer_.OnTestCaseStart(name, total_test_count);
+  }
+  ~TestCasePrinterHelper() {
+    printer_.OnTestCaseEnd();
+  }
+ private:
+  const ResultsPrinter& printer_;
+
+  DISALLOW_COPY_AND_ASSIGN(TestCasePrinterHelper);
+};
+
+// For a basic pattern matching for gtest_filter options.  (Copied from
+// gtest.cc, see the comment below and http://crbug.com/44497)
+bool PatternMatchesString(const char* pattern, const char* str) {
+  switch (*pattern) {
+    case '\0':
+    case ':':  // Either ':' or '\0' marks the end of the pattern.
+      return *str == '\0';
+    case '?':  // Matches any single character.
+      return *str != '\0' && PatternMatchesString(pattern + 1, str + 1);
+    case '*':  // Matches any string (possibly empty) of characters.
+      return (*str != '\0' && PatternMatchesString(pattern, str + 1)) ||
+          PatternMatchesString(pattern + 1, str);
+    default:  // Non-special character.  Matches itself.
+      return *pattern == *str &&
+          PatternMatchesString(pattern + 1, str + 1);
+  }
+}
+
+// TODO(phajdan.jr): Avoid duplicating gtest code. (http://crbug.com/44497)
+// For basic pattern matching for gtest_filter options.  (Copied from
+// gtest.cc)
+bool MatchesFilter(const std::string& name, const std::string& filter) {
+  const char *cur_pattern = filter.c_str();
+  for (;;) {
+    if (PatternMatchesString(cur_pattern, name.c_str())) {
+      return true;
+    }
+
+    // Finds the next pattern in the filter.
+    cur_pattern = strchr(cur_pattern, ':');
+
+    // Returns if no more pattern can be found.
+    if (cur_pattern == NULL) {
+      return false;
+    }
+
+    // Skips the pattern separater (the ':' character).
+    cur_pattern++;
+  }
+}
+
+bool RunTests(TestLauncherDelegate* launcher_delegate,
+              int total_shards,
+              int shard_index) {
+  const CommandLine* command_line = CommandLine::ForCurrentProcess();
+
+  DCHECK(!command_line->HasSwitch(kGTestListTestsFlag));
+
+  testing::UnitTest* const unit_test = testing::UnitTest::GetInstance();
+
+  std::string filter = command_line->GetSwitchValueASCII(kGTestFilterFlag);
+
+  // Split --gtest_filter at '-', if there is one, to separate into
+  // positive filter and negative filter portions.
+  std::string positive_filter = filter;
+  std::string negative_filter;
+  size_t dash_pos = filter.find('-');
+  if (dash_pos != std::string::npos) {
+    positive_filter = filter.substr(0, dash_pos);  // Everything up to the dash.
+    negative_filter = filter.substr(dash_pos + 1); // Everything after the dash.
+  }
+
+  int num_runnable_tests = 0;
+  int test_run_count = 0;
+  std::vector<std::string> failed_tests;
+
+  ResultsPrinter printer(*command_line);
+  for (int i = 0; i < unit_test->total_test_case_count(); ++i) {
+    const testing::TestCase* test_case = unit_test->GetTestCase(i);
+    TestCasePrinterHelper helper(printer, test_case->name(),
+                                 test_case->total_test_count());
+    for (int j = 0; j < test_case->total_test_count(); ++j) {
+      const testing::TestInfo* test_info = test_case->GetTestInfo(j);
+      std::string test_name = test_info->test_case_name();
+      test_name.append(".");
+      test_name.append(test_info->name());
+
+      // Skip disabled tests.
+      if (test_name.find("DISABLED") != std::string::npos &&
+          !command_line->HasSwitch(kGTestRunDisabledTestsFlag)) {
+        continue;
+      }
+
+      // Skip the test that doesn't match the filter string (if given).
+      if ((!positive_filter.empty() &&
+           !MatchesFilter(test_name, positive_filter)) ||
+          MatchesFilter(test_name, negative_filter)) {
+        continue;
+      }
+
+      if (!launcher_delegate->ShouldRunTest(test_case, test_info))
+        continue;
+
+      bool should_run = ShouldRunTestOnShard(total_shards, shard_index,
+                                             num_runnable_tests);
+      num_runnable_tests += 1;
+      if (!should_run)
+        continue;
+
+      TimeTicks start_time = TimeTicks::Now();
+      ++test_run_count;
+      bool success = launcher_delegate->RunTest(test_case, test_info);
+      if (!success)
+        failed_tests.push_back(test_name);
+      printer.OnTestEnd(
+          test_info->name(), test_case->name(), success,
+          (TimeTicks::Now() - start_time).InMillisecondsF());
+    }
+  }
+
+  printf("%d test%s run\n", test_run_count, test_run_count > 1 ? "s" : "");
+  printf("%d test%s failed\n",
+         static_cast<int>(failed_tests.size()),
+         failed_tests.size() != 1 ? "s" : "");
+  if (failed_tests.empty())
+    return true;
+
+  printf("Failing tests:\n");
+  for (std::vector<std::string>::const_iterator iter = failed_tests.begin();
+       iter != failed_tests.end(); ++iter) {
+    printf("%s\n", iter->c_str());
+  }
+
+  return false;
+}
+
+}  // namespace
+
+const char kGTestFilterFlag[] = "gtest_filter";
+const char kGTestListTestsFlag[] = "gtest_list_tests";
+const char kGTestRepeatFlag[] = "gtest_repeat";
+const char kGTestRunDisabledTestsFlag[] = "gtest_also_run_disabled_tests";
+const char kGTestOutputFlag[] = "gtest_output";
+
+const char kHelpFlag[]   = "help";
+
+TestLauncherDelegate::~TestLauncherDelegate() {
+}
+
+int LaunchTests(TestLauncherDelegate* launcher_delegate,
+                int argc,
+                char** argv) {
+  const CommandLine* command_line = CommandLine::ForCurrentProcess();
+
+  int32 total_shards;
+  int32 shard_index;
+  InitSharding(&total_shards, &shard_index);
+
+  int cycles = 1;
+  if (command_line->HasSwitch(kGTestRepeatFlag))
+    StringToInt(command_line->GetSwitchValueASCII(kGTestRepeatFlag), &cycles);
+
+  int exit_code = 0;
+  while (cycles != 0) {
+    if (!RunTests(launcher_delegate, total_shards, shard_index)) {
+      exit_code = 1;
+      break;
+    }
+
+    // Special value "-1" means "repeat indefinitely".
+    if (cycles != -1)
+      cycles--;
+  }
+
+  return exit_code;
+}
+
+}  // namespace base
diff --git a/base/test/test_launcher.h b/base/test/test_launcher.h
new file mode 100644
index 0000000..720c74c
--- /dev/null
+++ b/base/test/test_launcher.h
@@ -0,0 +1,52 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BASE_TEST_TEST_LAUNCHER_H_
+#define BASE_TEST_TEST_LAUNCHER_H_
+
+#include "base/basictypes.h"
+#include "base/compiler_specific.h"
+
+namespace testing {
+class TestCase;
+class TestInfo;
+}
+
+namespace base {
+
+// Constants for GTest command-line flags.
+extern const char kGTestFilterFlag[];
+extern const char kGTestListTestsFlag[];
+extern const char kGTestRepeatFlag[];
+extern const char kGTestRunDisabledTestsFlag[];
+extern const char kGTestOutputFlag[];
+
+// Interface for use with LaunchTests that abstracts away exact details
+// which tests and how are run.
+class TestLauncherDelegate {
+ public:
+  // Called before a test is considered for running. If it returns false,
+  // the test is not run. If it returns true, the test will be run provided
+  // it is part of the current shard.
+  virtual bool ShouldRunTest(const testing::TestCase* test_case,
+                             const testing::TestInfo* test_info) = 0;
+
+  // Called to make the delegate run specified test. Should return true
+  // on success.
+  virtual bool RunTest(const testing::TestCase* test_case,
+                       const testing::TestInfo* test_info) = 0;
+
+ protected:
+  virtual ~TestLauncherDelegate();
+};
+
+// Launches GTest-based tests from the current executable
+// using |launcher_delegate|.
+int LaunchTests(TestLauncherDelegate* launcher_delegate,
+                int argc,
+                char** argv) WARN_UNUSED_RESULT;
+
+}  // namespace base
+
+#endif  // BASE_TEST_TEST_LAUNCHER_H_