Add json output to Junit tests.

This change will allow us to get the test results information
to our testrunner without having to parse stdout. This will fix
the issue of junit tests not showing up on the test report step
on our bots.

BUG=465755

Review URL: https://codereview.chromium.org/1003463002

Cr-Commit-Position: refs/heads/master@{#323255}

8 files changed, 271 insertions, 24 deletions

diff --git a/build/android/pylib/junit/test_dispatcher.py b/build/android/pylib/junit/test_dispatcher.py
index b821b75..6e0d865 100644
--- a/build/android/pylib/junit/test_dispatcher.py
+++ b/build/android/pylib/junit/test_dispatcher.py
@@ -2,6 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from pylib import constants
+from pylib.base import base_test_result
+
 def RunTests(tests, runner_factory):
   """Runs a set of java tests on the host.
 
@@ -11,9 +14,15 @@ def RunTests(tests, runner_factory):
   def run(t):
     runner = runner_factory(None, None)
     runner.SetUp()
-    result = runner.RunTest(t)
+    results_list, return_code = runner.RunTest(t)
     runner.TearDown()
-    return result == 0
-
-  return (None, 0 if all(run(t) for t in tests) else 1)
+    return (results_list, return_code == 0)
 
+  test_run_results = base_test_result.TestRunResults()
+  exit_code = 0
+  for t in tests:
+    results_list, passed = run(t)
+    test_run_results.AddResults(results_list)
+    if not passed:
+      exit_code = constants.ERROR_EXIT_CODE
+  return (test_run_results, exit_code)
\ No newline at end of file
diff --git a/build/android/pylib/junit/test_runner.py b/build/android/pylib/junit/test_runner.py
index b85967b..bf97412 100644
--- a/build/android/pylib/junit/test_runner.py
+++ b/build/android/pylib/junit/test_runner.py
@@ -2,10 +2,14 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import json
 import os
+import tempfile
 
 from pylib import cmd_helper
 from pylib import constants
+from pylib.base import base_test_result
+from pylib.results import json_results
 
 class JavaTestRunner(object):
   """Runs java tests on the host."""
@@ -22,22 +26,25 @@ class JavaTestRunner(object):
 
   def RunTest(self, _test):
     """Runs junit tests from |self._test_suite|."""
-
-    command = ['java',
-               '-Drobolectric.dependency.dir=%s' %
-                    os.path.join(constants.GetOutDirectory(), 'lib.java'),
-               '-jar', os.path.join(constants.GetOutDirectory(), 'lib.java',
-                                    '%s.jar' % self._test_suite)]
-
-    if self._test_filter:
-      command.extend(['-gtest-filter', self._test_filter])
-    if self._package_filter:
-      command.extend(['-package-filter', self._package_filter])
-    if self._runner_filter:
-      command.extend(['-runner-filter', self._runner_filter])
-    if self._sdk_version:
-      command.extend(['-sdk-version', self._sdk_version])
-    return cmd_helper.RunCmd(command)
+    with tempfile.NamedTemporaryFile() as json_file:
+      command = ['java',
+                 '-Drobolectric.dependency.dir=%s' %
+                      os.path.join(constants.GetOutDirectory(), 'lib.java'),
+                 '-jar', os.path.join(constants.GetOutDirectory(), 'lib.java',
+                                      '%s.jar' % self._test_suite),
+                 '-json-results-file', json_file.name]
+      if self._test_filter:
+        command.extend(['-gtest-filter', self._test_filter])
+      if self._package_filter:
+        command.extend(['-package-filter', self._package_filter])
+      if self._runner_filter:
+        command.extend(['-runner-filter', self._runner_filter])
+      if self._sdk_version:
+        command.extend(['-sdk-version', self._sdk_version])
+      return_code = cmd_helper.RunCmd(command)
+      results_list = json_results.ParseResultsFromJson(
+          json.loads(json_file.read()))
+      return (results_list, return_code)
 
   def TearDown(self):
     pass
diff --git a/build/android/pylib/results/json_results.py b/build/android/pylib/results/json_results.py
index c34244e..65664e3 100644
--- a/build/android/pylib/results/json_results.py
+++ b/build/android/pylib/results/json_results.py
@@ -16,6 +16,38 @@ def GenerateResultsDict(test_run_result):
     A results dict that mirrors the one generated by
       base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON.
   """
+  # Example json output.
+  # {
+  #   "global_tags": [],
+  #   "all_tests": [
+  #     "test1",
+  #     "test2",
+  #    ],
+  #   "disabled_tests": [],
+  #   "per_iteration_data": [
+  #     {
+  #       "test1": [
+  #         {
+  #           "status": "SUCCESS",
+  #           "elapsed_time_ms": 1,
+  #           "output_snippet": "",
+  #           "output_snippet_base64": "",
+  #           "losless_snippet": "",
+  #         },
+  #       ],
+  #       "test2": [
+  #         {
+  #           "status": "FAILURE",
+  #           "elapsed_time_ms": 12,
+  #           "output_snippet": "",
+  #           "output_snippet_base64": "",
+  #           "losless_snippet": "",
+  #         },
+  #       ],
+  #     },
+  #   ],
+  # }
+
   assert isinstance(test_run_result, base_test_result.TestRunResults)
 
   def status_as_string(s):
@@ -71,3 +103,37 @@ def GenerateJsonResultsFile(test_run_result, file_path):
   with open(file_path, 'w') as json_result_file:
     json_result_file.write(json.dumps(GenerateResultsDict(test_run_result)))
 
+
+def ParseResultsFromJson(json_results):
+  """Creates a list of BaseTestResult objects from JSON.
+
+  Args:
+    json_results: A JSON dict in the format created by
+                  GenerateJsonResultsFile.
+  """
+
+  def string_as_status(s):
+    if s == 'SUCCESS':
+      return base_test_result.ResultType.PASS
+    elif s == 'SKIPPED':
+      return base_test_result.ResultType.SKIP
+    elif s == 'FAILURE':
+      return base_test_result.ResultType.FAIL
+    elif s == 'CRASH':
+      return base_test_result.ResultType.CRASH
+    elif s == 'TIMEOUT':
+      return base_test_result.ResultType.TIMEOUT
+    else:
+      return base_test_result.ResultType.UNKNOWN
+
+  results_list = []
+  testsuite_runs = json_results['per_iteration_data']
+  for testsuite_run in testsuite_runs:
+    for test, test_runs in testsuite_run.iteritems():
+      results_list.extend(
+          [base_test_result.BaseTestResult(test,
+                                           string_as_status(tr['status']),
+                                           duration=tr['elapsed_time_ms'])
+          for tr in test_runs])
+  return results_list
+
diff --git a/build/android/test_runner.py b/build/android/test_runner.py
index 368ac06..87f4cec 100755
--- a/build/android/test_runner.py
+++ b/build/android/test_runner.py
@@ -768,7 +768,13 @@ def _RunUIAutomatorTests(args, devices):
 def _RunJUnitTests(args):
   """Subcommand of RunTestsCommand which runs junit tests."""
   runner_factory, tests = junit_setup.Setup(args)
-  _, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
+  results, exit_code = junit_dispatcher.RunTests(tests, runner_factory)
+
+  report_results.LogFull(
+      results=results,
+      test_type='JUnit',
+      test_package=args.test_suite)
+
   return exit_code
 
 
diff --git a/testing/android/junit/java/src/org/chromium/testing/local/JsonListener.java b/testing/android/junit/java/src/org/chromium/testing/local/JsonListener.java
new file mode 100644
index 0000000..3d501b3
--- /dev/null
+++ b/testing/android/junit/java/src/org/chromium/testing/local/JsonListener.java
@@ -0,0 +1,54 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.testing.local;
+
+import org.junit.runner.Description;
+import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunListener;
+
+/** A json RunListener that creates a Json file with test run information.
+ */
+public class JsonListener extends RunListener {
+
+    private final JsonLogger mJsonLogger;
+    private long mTestStartTimeMillis;
+    private boolean mCurrentTestPassed;
+
+    public JsonListener(JsonLogger jsonLogger) {
+        mJsonLogger = jsonLogger;
+    }
+
+    /** Called after all tests run.
+     */
+    @Override
+    public void testRunFinished(Result r) throws Exception {
+        mJsonLogger.writeJsonToFile();
+    }
+
+    /** Called when a test is about to start.
+     */
+    @Override
+    public void testStarted(Description d) throws Exception {
+        mCurrentTestPassed = true;
+        mTestStartTimeMillis = System.currentTimeMillis();
+    }
+
+    /** Called when a test has just finished.
+     */
+    @Override
+    public void testFinished(Description d) throws Exception {
+        long testElapsedTimeMillis = System.currentTimeMillis() - mTestStartTimeMillis;
+        mJsonLogger.addTestResultInfo(d, mCurrentTestPassed, testElapsedTimeMillis);
+    }
+
+    /** Called when a test fails.
+     */
+    @Override
+    public void testFailure(Failure f) throws Exception {
+        mCurrentTestPassed = false;
+    }
+}
+
diff --git a/testing/android/junit/java/src/org/chromium/testing/local/JsonLogger.java b/testing/android/junit/java/src/org/chromium/testing/local/JsonLogger.java
new file mode 100644
index 0000000..c253ba3
--- /dev/null
+++ b/testing/android/junit/java/src/org/chromium/testing/local/JsonLogger.java
@@ -0,0 +1,89 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.testing.local;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.runner.Description;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+
+/**
+ *  Creates json file with junit test information. Format of the json file mirrors the
+ *  json generated by build/android/pylib/results/json_results.py.
+ */
+public class JsonLogger {
+
+    private final JSONObject mBaseJsonObject;
+    private final JSONObject mBaseTestInfoJsonObject;
+    private final File mOutputFile;
+
+    public JsonLogger(File outputFile) {
+        mBaseJsonObject = new JSONObject();
+        mBaseTestInfoJsonObject = new JSONObject();
+        mOutputFile = outputFile;
+
+        try {
+            mBaseJsonObject.put("global_tags", new JSONArray());
+            mBaseJsonObject.put("all_tests", new JSONArray());
+            mBaseJsonObject.put("disabled_tests", new JSONArray());
+            mBaseJsonObject.put("per_iteration_data",
+                                new JSONArray().put(mBaseTestInfoJsonObject));
+        } catch (JSONException e) {
+            System.err.println("Unable to create json output.");
+        }
+    }
+
+    /**
+     *  Add the results of a test run to the json output.
+     */
+    public void addTestResultInfo(Description test, boolean passed, long elapsedTimeMillis) {
+        JSONObject testInfoJsonObject = new JSONObject();
+
+        try {
+            testInfoJsonObject.put("status", (passed ? "SUCCESS" : "FAILURE"));
+            testInfoJsonObject.put("elapsed_time_ms", elapsedTimeMillis);
+            testInfoJsonObject.put("output_snippet", "");
+            testInfoJsonObject.put("output_snippet_base64", "");
+            testInfoJsonObject.put("losless_snippet", "");
+
+            if (mBaseTestInfoJsonObject.optJSONArray(testName(test)) == null) {
+                mBaseTestInfoJsonObject.put(testName(test), new JSONArray());
+                mBaseJsonObject.getJSONArray("all_tests").put(testName(test));
+            }
+            mBaseTestInfoJsonObject.getJSONArray(testName(test)).put(testInfoJsonObject);
+        } catch (JSONException e) {
+            System.err.println("Unable to log test to json output: " + testName(test));
+        }
+    }
+
+    /**
+     *  Writes the json output to a file.
+     */
+    public void writeJsonToFile() {
+        try {
+            PrintStream stream = new PrintStream(new FileOutputStream(mOutputFile));
+            try {
+                stream.print(mBaseJsonObject);
+            } finally {
+                try {
+                    stream.close();
+                } catch (RuntimeException e) {
+                    System.err.println("Unable to close output file: " + mOutputFile.getPath());
+                }
+            }
+        } catch (FileNotFoundException e) {
+            System.err.println("File not found: " + mOutputFile.getPath());
+        }
+    }
+
+    private String testName(Description test) {
+        return test.getClassName() + "#" + test.getMethodName();
+    }
+}
\ No newline at end of file
diff --git a/testing/android/junit/java/src/org/chromium/testing/local/JunitTestArgParser.java b/testing/android/junit/java/src/org/chromium/testing/local/JunitTestArgParser.java
index 9de7874..2837976 100644
--- a/testing/android/junit/java/src/org/chromium/testing/local/JunitTestArgParser.java
+++ b/testing/android/junit/java/src/org/chromium/testing/local/JunitTestArgParser.java
@@ -4,6 +4,7 @@
 
 package org.chromium.testing.local;
 
+import java.io.File;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -15,6 +16,7 @@ public class JunitTestArgParser {
     private final Set<String> mPackageFilters;
     private final Set<Class<?>> mRunnerFilters;
     private final Set<String> mGtestFilters;
+    private File mJsonOutput;
 
     public static JunitTestArgParser parse(String[] args) {
 
@@ -38,6 +40,9 @@ public class JunitTestArgParser {
                     } else if ("gtest-filter".equals(argName)) {
                         // Read the command line argument after the flag.
                         parsed.addGtestFilter(args[++i]);
+                    } else if ("json-results-file".equals(argName)) {
+                        // Read the command line argument after the flag.
+                        parsed.setJsonOutputFile(args[++i]);
                     } else {
                         System.out.println("Ignoring flag: \"" + argName + "\"");
                     }
@@ -60,6 +65,7 @@ public class JunitTestArgParser {
         mPackageFilters = new HashSet<String>();
         mRunnerFilters = new HashSet<Class<?>>();
         mGtestFilters = new HashSet<String>();
+        mJsonOutput = null;
     }
 
     public Set<String> getPackageFilters() {
@@ -74,6 +80,10 @@ public class JunitTestArgParser {
         return mGtestFilters;
     }
 
+    public File getJsonOutputFile() {
+        return mJsonOutput;
+    }
+
     private void addPackageFilter(String packageFilter) {
         mPackageFilters.add(packageFilter);
     }
@@ -86,5 +96,9 @@ public class JunitTestArgParser {
         mGtestFilters.add(gtestFilter);
     }
 
+    private void setJsonOutputFile(String path) {
+        mJsonOutput = new File(path);
+    }
+
 }
 
diff --git a/testing/android/junit/java/src/org/chromium/testing/local/JunitTestMain.java b/testing/android/junit/java/src/org/chromium/testing/local/JunitTestMain.java
index d9c468d..3a305fa 100644
--- a/testing/android/junit/java/src/org/chromium/testing/local/JunitTestMain.java
+++ b/testing/android/junit/java/src/org/chromium/testing/local/JunitTestMain.java
@@ -76,10 +76,12 @@ public final class JunitTestMain {
         JunitTestArgParser parser = JunitTestArgParser.parse(args);
 
         JUnitCore core = new JUnitCore();
-        GtestLogger logger = new GtestLogger(System.out);
-        core.addListener(new GtestListener(logger));
+        GtestLogger gtestLogger = new GtestLogger(System.out);
+        core.addListener(new GtestListener(gtestLogger));
+        JsonLogger jsonLogger = new JsonLogger(parser.getJsonOutputFile());
+        core.addListener(new JsonListener(jsonLogger));
         Class[] classes = findClassesFromClasspath();
-        Request testRequest = Request.classes(new GtestComputer(logger), classes);
+        Request testRequest = Request.classes(new GtestComputer(gtestLogger), classes);
         for (String packageFilter : parser.getPackageFilters()) {
             testRequest = testRequest.filterWith(new PackageFilter(packageFilter));
         }