diff options
8 files changed, 271 insertions, 24 deletions
diff --git a/build/android/pylib/junit/test_dispatcher.py b/build/android/pylib/junit/test_dispatcher.py index b821b75..6e0d865 100644 --- a/build/android/pylib/junit/test_dispatcher.py +++ b/build/android/pylib/junit/test_dispatcher.py @@ -2,6 +2,9 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from pylib import constants +from pylib.base import base_test_result + def RunTests(tests, runner_factory): """Runs a set of java tests on the host. @@ -11,9 +14,15 @@ def RunTests(tests, runner_factory): def run(t): runner = runner_factory(None, None) runner.SetUp() - result = runner.RunTest(t) + results_list, return_code = runner.RunTest(t) runner.TearDown() - return result == 0 - - return (None, 0 if all(run(t) for t in tests) else 1) + return (results_list, return_code == 0) + test_run_results = base_test_result.TestRunResults() + exit_code = 0 + for t in tests: + results_list, passed = run(t) + test_run_results.AddResults(results_list) + if not passed: + exit_code = constants.ERROR_EXIT_CODE + return (test_run_results, exit_code)
\ No newline at end of file diff --git a/build/android/pylib/junit/test_runner.py b/build/android/pylib/junit/test_runner.py index b85967b..bf97412 100644 --- a/build/android/pylib/junit/test_runner.py +++ b/build/android/pylib/junit/test_runner.py @@ -2,10 +2,14 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import json import os +import tempfile from pylib import cmd_helper from pylib import constants +from pylib.base import base_test_result +from pylib.results import json_results class JavaTestRunner(object): """Runs java tests on the host.""" @@ -22,22 +26,25 @@ class JavaTestRunner(object): def RunTest(self, _test): """Runs junit tests from |self._test_suite|.""" - - command = ['java', - '-Drobolectric.dependency.dir=%s' % - os.path.join(constants.GetOutDirectory(), 'lib.java'), - '-jar', os.path.join(constants.GetOutDirectory(), 'lib.java', - '%s.jar' % self._test_suite)] - - if self._test_filter: - command.extend(['-gtest-filter', self._test_filter]) - if self._package_filter: - command.extend(['-package-filter', self._package_filter]) - if self._runner_filter: - command.extend(['-runner-filter', self._runner_filter]) - if self._sdk_version: - command.extend(['-sdk-version', self._sdk_version]) - return cmd_helper.RunCmd(command) + with tempfile.NamedTemporaryFile() as json_file: + command = ['java', + '-Drobolectric.dependency.dir=%s' % + os.path.join(constants.GetOutDirectory(), 'lib.java'), + '-jar', os.path.join(constants.GetOutDirectory(), 'lib.java', + '%s.jar' % self._test_suite), + '-json-results-file', json_file.name] + if self._test_filter: + command.extend(['-gtest-filter', self._test_filter]) + if self._package_filter: + command.extend(['-package-filter', self._package_filter]) + if self._runner_filter: + command.extend(['-runner-filter', self._runner_filter]) + if self._sdk_version: + command.extend(['-sdk-version', self._sdk_version]) + return_code = cmd_helper.RunCmd(command) + results_list = json_results.ParseResultsFromJson( + json.loads(json_file.read())) + return (results_list, return_code) def TearDown(self): pass diff --git a/build/android/pylib/results/json_results.py b/build/android/pylib/results/json_results.py index c34244e..65664e3 100644 --- a/build/android/pylib/results/json_results.py +++ b/build/android/pylib/results/json_results.py @@ -16,6 +16,38 @@ def GenerateResultsDict(test_run_result): A results dict that mirrors the one generated by base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON. """ + # Example json output. + # { + # "global_tags": [], + # "all_tests": [ + # "test1", + # "test2", + # ], + # "disabled_tests": [], + # "per_iteration_data": [ + # { + # "test1": [ + # { + # "status": "SUCCESS", + # "elapsed_time_ms": 1, + # "output_snippet": "", + # "output_snippet_base64": "", + # "losless_snippet": "", + # }, + # ], + # "test2": [ + # { + # "status": "FAILURE", + # "elapsed_time_ms": 12, + # "output_snippet": "", + # "output_snippet_base64": "", + # "losless_snippet": "", + # }, + # ], + # }, + # ], + # } + assert isinstance(test_run_result, base_test_result.TestRunResults) def status_as_string(s): @@ -71,3 +103,37 @@ def GenerateJsonResultsFile(test_run_result, file_path): with open(file_path, 'w') as json_result_file: json_result_file.write(json.dumps(GenerateResultsDict(test_run_result))) + +def ParseResultsFromJson(json_results): + """Creates a list of BaseTestResult objects from JSON. + + Args: + json_results: A JSON dict in the format created by + GenerateJsonResultsFile. + """ + + def string_as_status(s): + if s == 'SUCCESS': + return base_test_result.ResultType.PASS + elif s == 'SKIPPED': + return base_test_result.ResultType.SKIP + elif s == 'FAILURE': + return base_test_result.ResultType.FAIL + elif s == 'CRASH': + return base_test_result.ResultType.CRASH + elif s == 'TIMEOUT': + return base_test_result.ResultType.TIMEOUT + else: + return base_test_result.ResultType.UNKNOWN + + results_list = [] + testsuite_runs = json_results['per_iteration_data'] + for testsuite_run in testsuite_runs: + for test, test_runs in testsuite_run.iteritems(): + results_list.extend( + [base_test_result.BaseTestResult(test, + string_as_status(tr['status']), + duration=tr['elapsed_time_ms']) + for tr in test_runs]) + return results_list + diff --git a/build/android/test_runner.py b/build/android/test_runner.py index 368ac06..87f4cec 100755 --- a/build/android/test_runner.py +++ b/build/android/test_runner.py @@ -768,7 +768,13 @@ def _RunUIAutomatorTests(args, devices): def _RunJUnitTests(args): """Subcommand of RunTestsCommand which runs junit tests.""" runner_factory, tests = junit_setup.Setup(args) - _, exit_code = junit_dispatcher.RunTests(tests, runner_factory) + results, exit_code = junit_dispatcher.RunTests(tests, runner_factory) + + report_results.LogFull( + results=results, + test_type='JUnit', + test_package=args.test_suite) + return exit_code diff --git a/testing/android/junit/java/src/org/chromium/testing/local/JsonListener.java b/testing/android/junit/java/src/org/chromium/testing/local/JsonListener.java new file mode 100644 index 0000000..3d501b3 --- /dev/null +++ b/testing/android/junit/java/src/org/chromium/testing/local/JsonListener.java @@ -0,0 +1,54 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.testing.local; + +import org.junit.runner.Description; +import org.junit.runner.Result; +import org.junit.runner.notification.Failure; +import org.junit.runner.notification.RunListener; + +/** A json RunListener that creates a Json file with test run information. + */ +public class JsonListener extends RunListener { + + private final JsonLogger mJsonLogger; + private long mTestStartTimeMillis; + private boolean mCurrentTestPassed; + + public JsonListener(JsonLogger jsonLogger) { + mJsonLogger = jsonLogger; + } + + /** Called after all tests run. + */ + @Override + public void testRunFinished(Result r) throws Exception { + mJsonLogger.writeJsonToFile(); + } + + /** Called when a test is about to start. + */ + @Override + public void testStarted(Description d) throws Exception { + mCurrentTestPassed = true; + mTestStartTimeMillis = System.currentTimeMillis(); + } + + /** Called when a test has just finished. + */ + @Override + public void testFinished(Description d) throws Exception { + long testElapsedTimeMillis = System.currentTimeMillis() - mTestStartTimeMillis; + mJsonLogger.addTestResultInfo(d, mCurrentTestPassed, testElapsedTimeMillis); + } + + /** Called when a test fails. + */ + @Override + public void testFailure(Failure f) throws Exception { + mCurrentTestPassed = false; + } +} + diff --git a/testing/android/junit/java/src/org/chromium/testing/local/JsonLogger.java b/testing/android/junit/java/src/org/chromium/testing/local/JsonLogger.java new file mode 100644 index 0000000..c253ba3 --- /dev/null +++ b/testing/android/junit/java/src/org/chromium/testing/local/JsonLogger.java @@ -0,0 +1,89 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package org.chromium.testing.local; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.runner.Description; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PrintStream; + +/** + * Creates json file with junit test information. Format of the json file mirrors the + * json generated by build/android/pylib/results/json_results.py. + */ +public class JsonLogger { + + private final JSONObject mBaseJsonObject; + private final JSONObject mBaseTestInfoJsonObject; + private final File mOutputFile; + + public JsonLogger(File outputFile) { + mBaseJsonObject = new JSONObject(); + mBaseTestInfoJsonObject = new JSONObject(); + mOutputFile = outputFile; + + try { + mBaseJsonObject.put("global_tags", new JSONArray()); + mBaseJsonObject.put("all_tests", new JSONArray()); + mBaseJsonObject.put("disabled_tests", new JSONArray()); + mBaseJsonObject.put("per_iteration_data", + new JSONArray().put(mBaseTestInfoJsonObject)); + } catch (JSONException e) { + System.err.println("Unable to create json output."); + } + } + + /** + * Add the results of a test run to the json output. + */ + public void addTestResultInfo(Description test, boolean passed, long elapsedTimeMillis) { + JSONObject testInfoJsonObject = new JSONObject(); + + try { + testInfoJsonObject.put("status", (passed ? "SUCCESS" : "FAILURE")); + testInfoJsonObject.put("elapsed_time_ms", elapsedTimeMillis); + testInfoJsonObject.put("output_snippet", ""); + testInfoJsonObject.put("output_snippet_base64", ""); + testInfoJsonObject.put("losless_snippet", ""); + + if (mBaseTestInfoJsonObject.optJSONArray(testName(test)) == null) { + mBaseTestInfoJsonObject.put(testName(test), new JSONArray()); + mBaseJsonObject.getJSONArray("all_tests").put(testName(test)); + } + mBaseTestInfoJsonObject.getJSONArray(testName(test)).put(testInfoJsonObject); + } catch (JSONException e) { + System.err.println("Unable to log test to json output: " + testName(test)); + } + } + + /** + * Writes the json output to a file. + */ + public void writeJsonToFile() { + try { + PrintStream stream = new PrintStream(new FileOutputStream(mOutputFile)); + try { + stream.print(mBaseJsonObject); + } finally { + try { + stream.close(); + } catch (RuntimeException e) { + System.err.println("Unable to close output file: " + mOutputFile.getPath()); + } + } + } catch (FileNotFoundException e) { + System.err.println("File not found: " + mOutputFile.getPath()); + } + } + + private String testName(Description test) { + return test.getClassName() + "#" + test.getMethodName(); + } +}
\ No newline at end of file diff --git a/testing/android/junit/java/src/org/chromium/testing/local/JunitTestArgParser.java b/testing/android/junit/java/src/org/chromium/testing/local/JunitTestArgParser.java index 9de7874..2837976 100644 --- a/testing/android/junit/java/src/org/chromium/testing/local/JunitTestArgParser.java +++ b/testing/android/junit/java/src/org/chromium/testing/local/JunitTestArgParser.java @@ -4,6 +4,7 @@ package org.chromium.testing.local; +import java.io.File; import java.util.HashSet; import java.util.Set; @@ -15,6 +16,7 @@ public class JunitTestArgParser { private final Set<String> mPackageFilters; private final Set<Class<?>> mRunnerFilters; private final Set<String> mGtestFilters; + private File mJsonOutput; public static JunitTestArgParser parse(String[] args) { @@ -38,6 +40,9 @@ public class JunitTestArgParser { } else if ("gtest-filter".equals(argName)) { // Read the command line argument after the flag. parsed.addGtestFilter(args[++i]); + } else if ("json-results-file".equals(argName)) { + // Read the command line argument after the flag. + parsed.setJsonOutputFile(args[++i]); } else { System.out.println("Ignoring flag: \"" + argName + "\""); } @@ -60,6 +65,7 @@ public class JunitTestArgParser { mPackageFilters = new HashSet<String>(); mRunnerFilters = new HashSet<Class<?>>(); mGtestFilters = new HashSet<String>(); + mJsonOutput = null; } public Set<String> getPackageFilters() { @@ -74,6 +80,10 @@ public class JunitTestArgParser { return mGtestFilters; } + public File getJsonOutputFile() { + return mJsonOutput; + } + private void addPackageFilter(String packageFilter) { mPackageFilters.add(packageFilter); } @@ -86,5 +96,9 @@ public class JunitTestArgParser { mGtestFilters.add(gtestFilter); } + private void setJsonOutputFile(String path) { + mJsonOutput = new File(path); + } + } diff --git a/testing/android/junit/java/src/org/chromium/testing/local/JunitTestMain.java b/testing/android/junit/java/src/org/chromium/testing/local/JunitTestMain.java index d9c468d..3a305fa 100644 --- a/testing/android/junit/java/src/org/chromium/testing/local/JunitTestMain.java +++ b/testing/android/junit/java/src/org/chromium/testing/local/JunitTestMain.java @@ -76,10 +76,12 @@ public final class JunitTestMain { JunitTestArgParser parser = JunitTestArgParser.parse(args); JUnitCore core = new JUnitCore(); - GtestLogger logger = new GtestLogger(System.out); - core.addListener(new GtestListener(logger)); + GtestLogger gtestLogger = new GtestLogger(System.out); + core.addListener(new GtestListener(gtestLogger)); + JsonLogger jsonLogger = new JsonLogger(parser.getJsonOutputFile()); + core.addListener(new JsonListener(jsonLogger)); Class[] classes = findClassesFromClasspath(); - Request testRequest = Request.classes(new GtestComputer(logger), classes); + Request testRequest = Request.classes(new GtestComputer(gtestLogger), classes); for (String packageFilter : parser.getPackageFilters()) { testRequest = testRequest.filterWith(new PackageFilter(packageFilter)); } |