Convert telemetry unittests to output in GTest format.

This also corrects an issue where failing tests were incorrectly reported as passing.

BUG=156398


Review URL: https://chromiumcodereview.appspot.com/12090102

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@180472 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 146 insertions, 15 deletions

diff --git a/tools/perf/run_tests b/tools/perf/run_tests
index 384a0ca..0ec9c71 100755
--- a/tools/perf/run_tests
+++ b/tools/perf/run_tests
@@ -12,17 +12,22 @@ import os
 import sys
 
 import perf_tools
+import telemetry.gtest_testrunner
 import telemetry.run_tests
 
 if __name__ == '__main__':
   top_level_dir = os.path.abspath(
     os.path.dirname(__file__))
+  runner = telemetry.gtest_testrunner.GTestTestRunner(
+      print_result_after_run=False)
   start_dir = 'perf_tools'
   ret = telemetry.run_tests.Main(
-    sys.argv[1:], start_dir, top_level_dir)
+    sys.argv[1:], start_dir, top_level_dir, runner)
 
   start_dir = 'page_sets'
-  ret = telemetry.run_tests.Main(
-    sys.argv[1:], start_dir, top_level_dir)
+  ret = ret + telemetry.run_tests.Main(
+    sys.argv[1:], start_dir, top_level_dir, runner)
+
+  runner.result.PrintSummary()
 
-  sys.exit(ret)
+  sys.exit(max(ret + runner.result.num_errors, 255))
diff --git a/tools/telemetry/run_tests b/tools/telemetry/run_tests
index 43a9ec2..3fbadde 100755
--- a/tools/telemetry/run_tests
+++ b/tools/telemetry/run_tests
@@ -5,12 +5,18 @@
 import os
 import sys
 
+import telemetry.gtest_testrunner
 import telemetry.run_tests
 
 if __name__ == '__main__':
   top_level_dir = os.path.abspath(
     os.path.dirname(__file__))
+  runner = telemetry.gtest_testrunner.GTestTestRunner(
+      print_result_after_run=False)
   start_dir = 'telemetry'
   ret = telemetry.run_tests.Main(
-    sys.argv[1:], start_dir, top_level_dir)
-  sys.exit(ret)
+    sys.argv[1:], start_dir, top_level_dir, runner)
+
+  runner.result.PrintSummary()
+
+  sys.exit(max(ret + runner.result.num_errors, 255))
diff --git a/tools/telemetry/telemetry/gtest_testrunner.py b/tools/telemetry/telemetry/gtest_testrunner.py
new file mode 100755
index 0000000..9626d2b
--- /dev/null
+++ b/tools/telemetry/telemetry/gtest_testrunner.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Implements a unittest TestRunner with GTest output.
+
+This output is ported from gtest.cc's PrettyUnitTestResultPrinter, but
+designed to be a drop-in replacement for unittest's TextTestRunner.
+"""
+
+import time
+import unittest
+
+
+class GTestTestResult(unittest.TestResult):
+  def __init__(self):
+    unittest.TestResult.__init__(self)
+    self.timestamp = None
+    self.num_successes = 0
+
+  def _GetMs(self):
+    return (time.time() - self.timestamp) * 1000
+
+  @property
+  def num_errors(self):
+    return len(self.errors) + len(self.failures)
+
+  @staticmethod
+  def _formatTestname(test):
+    chunks = test.id().split('.')[-2:]
+    return '.'.join(chunks)
+
+  def _emitFailure(self, test, err):
+    print self._exc_info_to_string(err, test)
+    test_name = GTestTestResult._formatTestname(test)
+    print '[  FAILED  ]', test_name, '(%0.f ms)' % self._GetMs()
+
+  def addError(self, test, err):
+    self._emitFailure(test, err)
+    super(GTestTestResult, self).addError(test, err)
+
+  def addFailure(self, test, err):
+    self._emitFailure(test, err)
+    super(GTestTestResult, self).addFailure(test, err)
+
+  def startTest(self, test):
+    print '[ RUN      ]', GTestTestResult._formatTestname(test)
+    self.timestamp = time.time()
+    super(GTestTestResult, self).startTest(test)
+
+  def addSuccess(self, test):
+    self.num_successes = self.num_successes + 1
+    test_name = GTestTestResult._formatTestname(test)
+    print '[       OK ]', test_name, '(%0.f ms)' % self._GetMs()
+
+  def PrintSummary(self):
+    unit = 'test' if self.num_successes == 1 else 'tests'
+    print '[  PASSED  ] %d %s.' % (self.num_successes, unit)
+    if self.errors or self.failures:
+      all_errors = self.errors[:]
+      all_errors.extend(self.failures)
+      unit = 'test' if len(all_errors) == 1 else 'tests'
+      print '[  FAILED  ] %d %s, listed below:' % (len(all_errors), unit)
+      for test, _ in all_errors:
+        print '[  FAILED  ] ', GTestTestResult._formatTestname(test)
+    if not self.wasSuccessful():
+      print
+      count = len(self.errors) + len(self.failures)
+      unit = 'TEST' if count == 1 else 'TESTS'
+      print '%d FAILED %s' % (count, unit)
+    print
+
+
+class GTestTestSuite(unittest.TestSuite):
+  def __call__(self, *args, **kwargs):
+    result = args[0]
+    timestamp = time.time()
+    unit = 'test' if len(self._tests) == 1 else 'tests'
+    if not any(isinstance(x, unittest.TestSuite) for x in self._tests):
+      print '[----------] %d %s' % (len(self._tests), unit)
+    for test in self._tests:
+      if result.shouldStop:
+        break
+      test(result)
+    endts = time.time()
+    ms = (endts - timestamp) * 1000
+    if not any(isinstance(x, unittest.TestSuite) for x in self._tests):
+      print '[----------] %d %s (%d ms total)' % (len(self._tests), unit, ms)
+      print
+    return result
+
+
+class GTestTestRunner(object):
+  def __init__(self, print_result_after_run=True, runner=None):
+    self.print_result_after_run = print_result_after_run
+    self.result = None
+    if runner:
+      self.result = runner.result
+
+  def run(self, test):
+    "Run the given test case or test suite."
+    if not self.result:
+      self.result = GTestTestResult()
+    test(self.result)
+    if self.print_result_after_run:
+      self.result.PrintSummary()
+    return self.result
diff --git a/tools/telemetry/telemetry/run_tests.py b/tools/telemetry/telemetry/run_tests.py
index 2c857df..e1c2128 100644
--- a/tools/telemetry/telemetry/run_tests.py
+++ b/tools/telemetry/telemetry/run_tests.py
@@ -7,15 +7,18 @@ import os
 import traceback
 import unittest
 
+from telemetry import gtest_testrunner
 from telemetry import browser_options
 from telemetry import options_for_unittests
 
+
 def RequiresBrowserOfType(*types):
   def wrap(func):
     func._requires_browser_types = types
     return func
   return wrap
 
+
 def Discover(start_dir, pattern = 'test*.py', top_level_dir = None):
   modules = []
   for dirpath, _, filenames in os.walk(start_dir):
@@ -43,6 +46,7 @@ def Discover(start_dir, pattern = 'test*.py', top_level_dir = None):
       modules.append(module)
 
   loader = unittest.defaultTestLoader
+  loader.suiteClass = gtest_testrunner.GTestTestSuite
   subsuites = []
   for module in modules:
     if hasattr(module, 'suite'):
@@ -51,10 +55,11 @@ def Discover(start_dir, pattern = 'test*.py', top_level_dir = None):
       new_suite = loader.loadTestsFromModule(module)
     if new_suite.countTestCases():
       subsuites.append(new_suite)
-  return unittest.TestSuite(subsuites)
+  return gtest_testrunner.GTestTestSuite(subsuites)
+
 
 def FilterSuite(suite, predicate):
-  new_suite = unittest.TestSuite()
+  new_suite = suite.__class__()
   for x in suite:
     if isinstance(x, unittest.TestSuite):
       subsuite = FilterSuite(x, predicate)
@@ -70,7 +75,11 @@ def FilterSuite(suite, predicate):
 
   return new_suite
 
-def DiscoverAndRunTests(dir_name, args, top_level_dir):
+
+def DiscoverAndRunTests(dir_name, args, top_level_dir, runner=None):
+  if not runner:
+    runner = gtest_testrunner.GTestTestRunner(inner=True)
+
   suite = Discover(dir_name, '*_unittest.py', top_level_dir)
 
   def IsTestSelected(test):
@@ -94,11 +103,11 @@ def DiscoverAndRunTests(dir_name, args, top_level_dir):
     return True
 
   filtered_suite = FilterSuite(suite, IsTestSelected)
-  runner = unittest.TextTestRunner(verbosity = 2)
   test_result = runner.run(filtered_suite)
-  return len(test_result.errors) + len(test_result.failures)
+  return test_result
+
 
-def Main(args, start_dir, top_level_dir):
+def Main(args, start_dir, top_level_dir, runner=None):
   """Unit test suite that collects all test cases for telemetry."""
   default_options = browser_options.BrowserOptions()
   default_options.browser_type = 'any'
@@ -124,14 +133,17 @@ def Main(args, start_dir, top_level_dir):
   options_for_unittests.Set(default_options,
                             browser_to_create.browser_type)
   olddir = os.getcwd()
-  num_errors = 0
   try:
     os.chdir(top_level_dir)
+    success = True
     for _ in range(
         default_options.run_test_repeat_count): # pylint: disable=E1101
-      num_errors += DiscoverAndRunTests(start_dir, args, top_level_dir)
+      success = success and DiscoverAndRunTests(start_dir, args, top_level_dir,
+                                                runner)
+    if success:
+      return 0
   finally:
     os.chdir(olddir)
     options_for_unittests.Set(None, None)
 
-  return min(num_errors, 255)
+  return 1