[Telemetry] Teach Telemetry to find and run reference builds.

This allows users to do --browser=reference instead of looking up that long
platform-specific path name. It also causes test_runner to report reference
build commands for platforms that support it.

BUG=416705

Review URL: https://codereview.chromium.org/593983002

Cr-Commit-Position: refs/heads/master@{#296291}

2 files changed, 53 insertions, 22 deletions

diff --git a/tools/telemetry/telemetry/core/backends/chrome/desktop_browser_finder.py b/tools/telemetry/telemetry/core/backends/chrome/desktop_browser_finder.py
index b92fc8f..dcd34cb 100644
--- a/tools/telemetry/telemetry/core/backends/chrome/desktop_browser_finder.py
+++ b/tools/telemetry/telemetry/core/backends/chrome/desktop_browser_finder.py
@@ -90,6 +90,7 @@ def CanFindAvailableBrowsers():
 def FindAllBrowserTypes(_):
   return [
       'exact',
+      'reference',
       'release',
       'release_x64',
       'debug',
@@ -178,12 +179,17 @@ def FindAllAvailableBrowsers(finder_options):
     AddIfFound('content-shell-' + build_type.lower(), build_dir, build_type,
                content_shell_app_name, True)
 
+  reference_build_root = os.path.join(
+     chrome_root, 'chrome', 'tools', 'test', 'reference_build')
+
   # Mac-specific options.
   if sys.platform == 'darwin':
     mac_canary_root = '/Applications/Google Chrome Canary.app/'
     mac_canary = mac_canary_root + 'Contents/MacOS/Google Chrome Canary'
     mac_system_root = '/Applications/Google Chrome.app'
     mac_system = mac_system_root + '/Contents/MacOS/Google Chrome'
+    mac_reference_root = reference_build_root + '/chrome_mac/Google Chrome.app/'
+    mac_reference = mac_reference_root + 'Contents/MacOS/Google Chrome'
     if path.IsExecutable(mac_canary):
       browsers.append(PossibleDesktopBrowser('canary', finder_options,
                                              mac_canary, None, False,
@@ -193,6 +199,10 @@ def FindAllAvailableBrowsers(finder_options):
       browsers.append(PossibleDesktopBrowser('system', finder_options,
                                              mac_system, None, False,
                                              mac_system_root))
+    if path.IsExecutable(mac_reference):
+      browsers.append(PossibleDesktopBrowser('reference', finder_options,
+                                             mac_reference, None, False,
+                                             mac_reference_root))
 
   # Linux specific options.
   if sys.platform.startswith('linux'):
@@ -208,12 +218,19 @@ def FindAllAvailableBrowsers(finder_options):
       browsers.append(PossibleDesktopBrowser('system', finder_options,
                                              'google-chrome', None, False,
                                              '/opt/google/chrome'))
+    linux_reference = os.path.join(reference_build_root, 'chrome_linux',
+                                   'chrome')
+    if path.IsExecutable(linux_reference):
+      browsers.append(PossibleDesktopBrowser('reference', finder_options,
+                                             'chrome', None, False,
+                                             linux_reference))
 
   # Win32-specific options.
   if sys.platform.startswith('win'):
     app_paths = (
         ('system', os.path.join('Google', 'Chrome', 'Application')),
         ('canary', os.path.join('Google', 'Chrome SxS', 'Application')),
+        ('reference', os.path.join(reference_build_root, 'chrome_win')),
     )
 
     for browser_name, app_path in app_paths:
diff --git a/tools/telemetry/telemetry/test_runner.py b/tools/telemetry/telemetry/test_runner.py
index d549a38..4a3f548 100644
--- a/tools/telemetry/telemetry/test_runner.py
+++ b/tools/telemetry/telemetry/test_runner.py
@@ -89,8 +89,10 @@ class List(command_line.OptparseCommand):
   def Run(self, args):
     if args.json_output_file:
       possible_browser = browser_finder.FindBrowser(args)
+      has_ref = 'reference' in browser_finder.GetAllAvailableBrowserTypes(args)
       with open(args.json_output_file, 'w') as f:
-        f.write(_GetJsonTestList(possible_browser, args.tests, args.num_shards))
+        f.write(_GetJsonTestList(possible_browser, has_ref, args.tests,
+                                 args.num_shards))
     else:
       _PrintTestList(args.tests)
     return 0
@@ -253,25 +255,23 @@ def _MatchTestName(input_test_name, exact_matches=True):
           if _Matches(input_test_name, test_class.Name())]
 
 
-def _GetJsonTestList(possible_browser, test_classes, num_shards):
+def _GetJsonTestList(possible_browser, has_reference, test_classes, num_shards):
   """Returns a list of all enabled tests in a JSON format expected by buildbots.
 
   JSON format (see build/android/pylib/perf/test_runner.py):
-  { "version": int,
+  { "version": <int>,
     "steps": {
-      "foo": {
-        "device_affinity": int,
-        "cmd": "script_to_execute foo"
+      <string>: {
+        "device_affinity": <int>,
+        "cmd": <string>,
+        "perf_dashboard_id": <string>,
       },
-      "bar": {
-        "device_affinity": int,
-        "cmd": "script_to_execute bar"
-      }
+      ...
     }
   }
   """
   output = {
-    'version': 1,
+    'version': 2,
     'steps': {
     }
   }
@@ -280,18 +280,32 @@ def _GetJsonTestList(possible_browser, test_classes, num_shards):
       continue
     if not decorators.IsEnabled(test_class, possible_browser):
       continue
-    name = test_class.Name()
-    output['steps'][name] = {
-      'cmd': ' '.join([sys.executable, os.path.realpath(sys.argv[0]),
-                       '--browser=%s' % possible_browser.browser_type,
-                       '-v', '--output-format=buildbot', name]),
-      # TODO(tonyg): Currently we set the device affinity to a stable hash of
-      # the test name. This somewhat evenly distributes benchmarks among the
-      # requested number of shards. However, it is far from optimal in terms of
-      # cycle time. We should add a test size decorator (e.g. small, medium,
-      # large) and let that inform sharding.
-      'device_affinity': int(hashlib.sha1(name).hexdigest(), 16) % num_shards
+
+    base_name = test_class.Name()
+    base_cmd = [sys.executable, os.path.realpath(sys.argv[0]),
+                '-v', '--output-format=buildbot', base_name]
+    perf_dashboard_id = base_name
+    # TODO(tonyg): Currently we set the device affinity to a stable hash of the
+    # test name. This somewhat evenly distributes benchmarks among the requested
+    # number of shards. However, it is far from optimal in terms of cycle time.
+    # We should add a test size decorator (e.g. small, medium, large) and let
+    # that inform sharding.
+    device_affinity = int(hashlib.sha1(base_name).hexdigest(), 16) % num_shards
+
+    output['steps'][base_name] = {
+      'cmd': ' '.join(base_cmd + [
+            '--browser=%s' % possible_browser.browser_type]),
+      'device_affinity': device_affinity,
+      'perf_dashboard_id': perf_dashboard_id,
     }
+    if has_reference:
+      output['steps'][base_name + '.reference'] = {
+        'cmd': ' '.join(base_cmd + [
+              '--browser=reference', '--output-trace-tag=_ref']),
+        'device_affinity': device_affinity,
+        'perf_dashboard_id': perf_dashboard_id,
+      }
+
   return json.dumps(output, indent=2, sort_keys=True)