Add a buildbot script to run Mojo apptests.

Add testing/scripts/mojo_apptest.py to run mojo apptests.
Support Chromium JSON test result output in apptest_runner.
TODO: Use Chromium and TYP testing infrastructure.
TODO: Use GTest Suite.Fixture names, not the apptest names.

Add the mojo_apptest.py script entry to the following bots:
  "Linux GN" (main waterfall release bot)
  "linux_chromium_gn_rel" (CQ bot)

BUG=478244
TEST=Mojo apptests can run on the CQ and the main waterfall.
R=jam@chromium.org,dpranke@chromium.org,phajdan.jr@chromium.org

Review URL: https://codereview.chromium.org/1155683002

Cr-Commit-Position: refs/heads/master@{#331122}

4 files changed, 105 insertions, 0 deletions

diff --git a/mojo/tools/apptest_runner.py b/mojo/tools/apptest_runner.py
index b1c69e4..d4a8068 100755
--- a/mojo/tools/apptest_runner.py
+++ b/mojo/tools/apptest_runner.py
@@ -6,8 +6,10 @@
 """A test runner for gtest application tests."""
 
 import argparse
+import json
 import logging
 import sys
+import time
 
 from mopy import gtest
 from mopy.android import AndroidShell
@@ -21,6 +23,8 @@ def main():
                       help="a file listing apptests to run")
   parser.add_argument("build_dir", type=str, help="the build output directory")
   parser.add_argument("--verbose", default=False, action='store_true')
+  parser.add_argument('--write-full-results-to', metavar='FILENAME',
+                      help='Path to write the JSON list of full results.')
   args = parser.parse_args()
 
   gtest.set_color()
@@ -69,8 +73,55 @@ def main():
   print "[  FAILED  ] %d apptests" % len(failed),
   print ": %s" % ", ".join(failed) if failed else ""
 
+  if args.write_full_results_to:
+    _WriteJSONResults(tests, failed, args.write_full_results_to)
+
   return 1 if failed else 0
 
 
+def _WriteJSONResults(tests, failed, write_full_results_to):
+  """Write the apptest results in the Chromium JSON test results format.
+     See <http://www.chromium.org/developers/the-json-test-results-format>
+     TODO(msw): Use Chromium and TYP testing infrastructure.
+     TODO(msw): Use GTest Suite.Fixture names, not the apptest names.
+     Adapted from chrome/test/mini_installer/test_installer.py
+  """
+  results = {
+    'interrupted': False,
+    'path_delimiter': '.',
+    'version': 3,
+    'seconds_since_epoch': time.time(),
+    'num_failures_by_type': {
+      'FAIL': len(failed),
+      'PASS': len(tests) - len(failed),
+    },
+    'tests': {}
+  }
+
+  for test in tests:
+    value = {
+      'expected': 'PASS',
+      'actual': 'FAIL' if test in failed else 'PASS',
+      'is_unexpected': True if test in failed else False,
+    }
+    _AddPathToTrie(results['tests'], test, value)
+
+  with open(write_full_results_to, 'w') as fp:
+    json.dump(results, fp, indent=2)
+    fp.write('\n')
+
+  return results
+
+
+def _AddPathToTrie(trie, path, value):
+  if '.' not in path:
+    trie[path] = value
+    return
+  directory, rest = path.split('.', 1)
+  if directory not in trie:
+    trie[directory] = {}
+  _AddPathToTrie(trie[directory], rest, value)
+
+
 if __name__ == '__main__':
   sys.exit(main())
diff --git a/testing/buildbot/chromium.linux.json b/testing/buildbot/chromium.linux.json
index cadc084..6a81d7e 100644
--- a/testing/buildbot/chromium.linux.json
+++ b/testing/buildbot/chromium.linux.json
@@ -382,6 +382,10 @@
         ],
         "name": "gyp_flag_compare",
         "script": "gyp_flag_compare.py"
+      },
+      {
+        "name": "mojo_apptest",
+        "script": "mojo_apptest.py"
       }
     ]
   },
diff --git a/testing/buildbot/tryserver.chromium.linux.json b/testing/buildbot/tryserver.chromium.linux.json
index 0a6c686..a63bca9 100644
--- a/testing/buildbot/tryserver.chromium.linux.json
+++ b/testing/buildbot/tryserver.chromium.linux.json
@@ -370,6 +370,10 @@
         ],
         "name": "gyp_flag_compare",
         "script": "gyp_flag_compare.py"
+      },
+      {
+        "name": "mojo_apptest",
+        "script": "mojo_apptest.py"
       }
     ]
   }
diff --git a/testing/scripts/mojo_apptest.py b/testing/scripts/mojo_apptest.py
new file mode 100755
index 0000000..09e3f23
--- /dev/null
+++ b/testing/scripts/mojo_apptest.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import os
+import sys
+
+
+import common
+
+
+def main_run(args):
+  runner = os.path.join(common.SRC_DIR, 'mojo', 'tools', 'apptest_runner.py')
+  tests = os.path.join(common.SRC_DIR, 'mojo', 'tools', 'data', 'apptests')
+  build_dir = os.path.join(common.SRC_DIR, 'out', args.build_config_fs)
+
+  with common.temporary_file() as tempfile_path:
+    rc = common.run_command([runner, tests, build_dir, '--verbose',
+                             '--write-full-results-to', tempfile_path])
+    with open(tempfile_path) as f:
+      results = json.load(f)
+
+  parsed_results = common.parse_common_test_results(results, test_separator='.')
+  failures = parsed_results['unexpected_failures']
+
+  json.dump({
+      'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and
+                   ((rc == 0) or failures)),
+      'failures': failures.keys(),
+  }, args.output)
+
+  return rc
+
+
+def main_compile_targets(args):
+  json.dump(['mandoline:tests'], args.output)
+
+
+if __name__ == '__main__':
+  funcs = {
+    'run': main_run,
+    'compile_targets': main_compile_targets,
+  }
+  sys.exit(common.run_script(sys.argv[1:], funcs))