[telemetry] Make testMeasurementSmoke generate separate tests.

This way failures in one benchmark won't stop the rest of the tests, and it'll print separate GTest steps for each benchmark.
The test names follow the format BenchmarkSmokeTest.benchmark.name so you can still find BenchmarkSmokeTest with a search.

Also rename it to the more apt BenchmarkSmokeTest.

BUG=368401
TEST=tools/perf/run_tests benchmark

Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=270766

Review URL: https://codereview.chromium.org/287773002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@273712 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 87 insertions, 74 deletions

diff --git a/tools/perf/benchmarks/benchmark_unittest.py b/tools/perf/benchmarks/benchmark_unittest.py
new file mode 100644
index 0000000..f1e0eef
--- /dev/null
+++ b/tools/perf/benchmarks/benchmark_unittest.py
@@ -0,0 +1,86 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Run the first page of every benchmark that has a composable measurement.
+
+Ideally this test would be comprehensive, but the above serves as a
+kind of smoke test.
+"""
+
+import os
+import unittest
+
+from telemetry import test
+from telemetry.core import discover
+from telemetry.page import page_measurement
+from telemetry.unittest import gtest_testrunner
+from telemetry.unittest import options_for_unittests
+
+
+def SmokeTestGenerator(benchmark):
+  # In general you should @test.Disabled individual benchmarks that fail,
+  # instead of this entire smoke test suite.
+  # TODO(achuith): Multiple tests failing on CrOS. crbug.com/351114
+  @test.Disabled('android', 'chromeos', 'win')
+  def BenchmarkSmokeTest(self):
+    # Only measure a single page so that this test cycles reasonably quickly.
+    benchmark.options['pageset_repeat'] = 1
+    benchmark.options['page_repeat'] = 1
+
+    class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
+      def CreatePageSet(self, options):
+        # pylint: disable=E1002
+        ps = super(SinglePageBenchmark, self).CreatePageSet(options)
+        ps.pages = ps.pages[:1]
+        return ps
+
+    # Set the benchmark's default arguments.
+    options = options_for_unittests.GetCopy()
+    options.output_format = 'none'
+    parser = options.CreateParser()
+
+    benchmark.AddCommandLineArgs(parser)
+    test.AddCommandLineArgs(parser)
+    benchmark.SetArgumentDefaults(parser)
+    options.MergeDefaultValues(parser.get_default_values())
+
+    benchmark.ProcessCommandLineArgs(None, options)
+    test.ProcessCommandLineArgs(None, options)
+
+    self.assertEqual(0, SinglePageBenchmark().Run(options),
+                     msg='Failed: %s' % benchmark)
+
+  return BenchmarkSmokeTest
+
+
+def load_tests(_, _2, _3):
+  suite = gtest_testrunner.GTestTestSuite()
+
+  benchmarks_dir = os.path.dirname(__file__)
+  top_level_dir = os.path.dirname(benchmarks_dir)
+  measurements_dir = os.path.join(top_level_dir, 'measurements')
+
+  all_measurements = discover.DiscoverClasses(
+      measurements_dir, top_level_dir, page_measurement.PageMeasurement,
+      pattern='*.py').values()
+  all_benchmarks = discover.DiscoverClasses(
+      benchmarks_dir, top_level_dir, test.Test, pattern='*.py').values()
+
+  for benchmark in all_benchmarks:
+    if benchmark.PageTestClass() not in all_measurements:
+      # If the benchmark is not in measurements, then it is not composable.
+      # Ideally we'd like to test these as well, but the non-composable
+      # benchmarks are usually long-running benchmarks.
+      continue
+
+    if hasattr(benchmark, 'generated_profile_archive'):
+      # We'd like to test these, but don't know how yet.
+      continue
+
+    class BenchmarkSmokeTest(unittest.TestCase):
+      pass
+    setattr(BenchmarkSmokeTest, benchmark.Name(), SmokeTestGenerator(benchmark))
+    suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
+
+  return suite
diff --git a/tools/perf/measurements/measurement_unittest.py b/tools/perf/measurements/measurement_unittest.py
deleted file mode 100644
index 7fbcad0..0000000
--- a/tools/perf/measurements/measurement_unittest.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import logging
-import os
-import unittest
-
-from telemetry import test
-from telemetry.core import discover
-from telemetry.page import page_measurement
-from telemetry.unittest import options_for_unittests
-
-
-class MeasurementUnitTest(unittest.TestCase):
-
-  # TODO(achuith): Fix crbug.com/351114.
-  @test.Disabled('android', 'chromeos', 'win')
-  def testMeasurementSmoke(self):
-    # Run all Measurements against the first Page in the PageSet of the first
-    # Benchmark that uses them.
-    #
-    # Ideally this test would be comprehensive, but the above serves as a
-    # kind of smoke test.
-    measurements_dir = os.path.dirname(__file__)
-    top_level_dir = os.path.dirname(measurements_dir)
-    benchmarks_dir = os.path.join(top_level_dir, 'benchmarks')
-
-    all_measurements = discover.DiscoverClasses(
-        measurements_dir, top_level_dir, page_measurement.PageMeasurement,
-        pattern='*.py').values()
-    all_benchmarks = discover.DiscoverClasses(
-        benchmarks_dir, top_level_dir, test.Test, pattern='*.py').values()
-
-    for benchmark in all_benchmarks:
-      if benchmark.test not in all_measurements:
-        # If the benchmark is not in measurements, then it is not composable.
-        # Ideally we'd like to test these as well, but the non-composable
-        # benchmarks are usually long-running benchmarks.
-        continue
-
-      if hasattr(benchmark, 'generated_profile_archive'):
-        # We'd like to test these, but don't know how yet.
-        continue
-
-      # Only measure a single page so that this test cycles reasonably quickly.
-      benchmark.options['pageset_repeat'] = 1
-      benchmark.options['page_repeat'] = 1
-
-      class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
-        def CreatePageSet(self, options):
-          # pylint: disable=E1002
-          ps = super(SinglePageBenchmark, self).CreatePageSet(options)
-          ps.pages = ps.pages[:1]
-          return ps
-
-      logging.info('running: %s', benchmark)
-
-      # Set the benchmark's default arguments.
-      options = options_for_unittests.GetCopy()
-      options.output_format = 'none'
-      parser = options.CreateParser()
-
-      benchmark.AddCommandLineArgs(parser)
-      test.AddCommandLineArgs(parser)
-      benchmark.SetArgumentDefaults(parser)
-      options.MergeDefaultValues(parser.get_default_values())
-
-      benchmark.ProcessCommandLineArgs(None, options)
-      test.ProcessCommandLineArgs(None, options)
-
-      self.assertEqual(0, SinglePageBenchmark().Run(options),
-                       msg='Failed: %s' % benchmark)
diff --git a/tools/telemetry/telemetry/page/gtest_test_results.py b/tools/telemetry/telemetry/page/gtest_test_results.py
index f70866c..ada5474 100644
--- a/tools/telemetry/telemetry/page/gtest_test_results.py
+++ b/tools/telemetry/telemetry/page/gtest_test_results.py
@@ -24,7 +24,7 @@ class GTestTestResults(page_test_results.PageTestResults):
   @staticmethod
   def _formatTestname(test):
     if isinstance(test, unittest.TestCase):
-      chunks = test.id().split('.')[-2:]
+      chunks = test.id().split('.')[2:]
       return '.'.join(chunks)
     else:
       return str(test)