diff options
3 files changed, 58 insertions, 32 deletions
diff --git a/webkit/tools/layout_tests/layout_package/json_results_generator.py b/webkit/tools/layout_tests/layout_package/json_results_generator.py index 9682473..8a2e973 100644 --- a/webkit/tools/layout_tests/layout_package/json_results_generator.py +++ b/webkit/tools/layout_tests/layout_package/json_results_generator.py @@ -5,14 +5,13 @@ import logging import os import re +import sys from layout_package import path_utils from layout_package import test_failures -class ResultAndTime: - """A holder for a single result and runtime for a test.""" - time = 0 - result = "N" +sys.path.append(path_utils.PathFromBase('third_party')) +import simplejson class JSONResultsGenerator: @@ -23,9 +22,11 @@ class JSONResultsGenerator: JSON_SUFFIX = ");" WEBKIT_PATH = "WebKit" LAYOUT_TESTS_PATH = "layout_tests" + PASS_RESULT = "P" + NO_DATA_RESULT = "N" def __init__(self, failures, individual_test_timings, builder_name, - build_number, results_file_path): + build_number, results_file_path, all_tests): """ failures: Map of test name to list of failures. individual_test_times: Map of test name to a tuple containing the @@ -33,6 +34,7 @@ class JSONResultsGenerator: builder_name: The name of the builder the tests are being run on. build_number: The build number for this run. results_file_path: Absolute path to the results json file. + all_tests: List of all the tests that were run. """ # Make sure all test paths are relative to the layout test root directory. self._failures = {} @@ -40,6 +42,9 @@ class JSONResultsGenerator: test_path = self._GetPathRelativeToLayoutTestRoot(test) self._failures[test_path] = failures[test] + self._all_tests = [self._GetPathRelativeToLayoutTestRoot(test) + for test in all_tests] + self._test_timings = {} for test_tuple in individual_test_timings: test_path = self._GetPathRelativeToLayoutTestRoot(test_tuple.filename) @@ -76,12 +81,12 @@ class JSONResultsGenerator: """Gets the results for the results.json file.""" failures_for_json = {} for test in self._failures: - failures_for_json[test] = ResultAndTime() + failures_for_json[test] = ResultAndTime(test, self._all_tests) failures_for_json[test].result = self._GetResultsCharForFailure(test) for test in self._test_timings: if not test in failures_for_json: - failures_for_json[test] = ResultAndTime() + failures_for_json[test] = ResultAndTime(test, self._all_tests) # Floor for now to get time in seconds. # TODO(ojan): As we make tests faster, reduce to tenth of a second # granularity. @@ -94,7 +99,12 @@ class JSONResultsGenerator: # Strip the prefix and suffix so we can get the actual JSON object. old_results = old_results[ len(self.JSON_PREFIX) : len(old_results) - len(self.JSON_SUFFIX)] - results_json = eval(old_results) + + try: + results_json = simplejson.loads(old_results) + except: + # The JSON file is not valid JSON. Just clobber the results. + results_json = {} if self._builder_name not in results_json: logging.error("Builder name (%s) is not in the results.json file." % @@ -116,7 +126,7 @@ class JSONResultsGenerator: if test in failures_for_json: result_and_time = failures_for_json[test] else: - result_and_time = ResultAndTime() + result_and_time = ResultAndTime(test, self._all_tests) if test not in tests: tests[test] = self._CreateResultsAndTimesJSON() @@ -130,11 +140,9 @@ class JSONResultsGenerator: results_json[self._builder_name]["buildNumbers"].insert(0, self._build_number) - # Generate the JSON and strip whitespace to keep filesize down. - # TODO(ojan): Generate the JSON using a JSON library should someone ever - # add a non-primitive type to results_json. - results_str = self.JSON_PREFIX + repr(results_json) + self.JSON_SUFFIX - return re.sub(r'\s+', '', results_str) + # Specify separators in order to get compact encoding. + results_str = simplejson.dumps(results_json, separators=(',', ':')) + return self.JSON_PREFIX + results_str + self.JSON_SUFFIX def _CreateResultsAndTimesJSON(self): results_and_times = {} @@ -193,7 +201,7 @@ class JSONResultsGenerator: times = times[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG] elif num_results < self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: num_to_pad = self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG - num_results - results = results + num_to_pad * 'N' + results = results + num_to_pad * self.NO_DATA_RESULT times.extend(num_to_pad * [0]) test["results"] = results @@ -203,8 +211,10 @@ class JSONResultsGenerator: # times that take less than a second, remove it from the results to reduce # noise and filesize. if (max(times) >= self.MIN_TIME and num_results and - (results == self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG * 'P' or - results == self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG * 'N')): + (results == self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG * + self.PASS_RESULT or + results == self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG * + self.NO_DATA_RESULT)): del tests[test_path] # Remove tests that don't exist anymore. @@ -212,3 +222,13 @@ class JSONResultsGenerator: full_path = os.path.normpath(full_path) if not os.path.exists(full_path): del tests[test_path] + +class ResultAndTime: + """A holder for a single result and runtime for a test.""" + def __init__(self, test, all_tests): + self.time = 0 + # If the test was run, then we don't want to default the result to nodata. + if test in all_tests: + self.result = JSONResultsGenerator.PASS_RESULT + else: + self.result = JSONResultsGenerator.NO_DATA_RESULT diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py index f6831fb..d3beea8 100644 --- a/webkit/tools/layout_tests/layout_package/test_expectations.py +++ b/webkit/tools/layout_tests/layout_package/test_expectations.py @@ -14,6 +14,8 @@ import time import path_utils import compare_failures +sys.path.append(path_utils.PathFromBase('third_party')) +import simplejson # Test expectation and modifier constants. (PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, REBASELINE, NONE) = \ @@ -36,8 +38,8 @@ class TestExpectations: # TODO(ojan): Replace the Get* calls here with the more sane API exposed # by TestExpectationsFile below. Maybe merge the two classes entirely? - def GetExpectationsForAllPlatforms(self): - return self._expected_failures.GetExpectationsForAllPlatforms() + def GetExpectationsJsonForAllPlatforms(self): + return self._expected_failures.GetExpectationsJsonForAllPlatforms() def GetFixable(self): return self._expected_failures.GetTestSet(NONE) @@ -154,9 +156,14 @@ class ModifiersAndExpectations: self.modifiers = modifiers self.expectations = expectations - def __repr__(self): - return ("{modifiers:'" + self.modifiers + "', expectations:'" + - self.expectations + "'}") +class ExpectationsJsonEncoder(simplejson.JSONEncoder): + """JSON encoder that can handle ModifiersAndExpectations objects. + """ + def default(self, obj): + if isinstance(obj, ModifiersAndExpectations): + return {"modifiers": obj.modifiers, "expectations": obj.expectations} + else: + return JSONEncoder.default(self, obj) class TestExpectationsFile: """Test expectation files consist of lines with specifications of what @@ -280,8 +287,10 @@ class TestExpectationsFile: def GetExpectations(self, test): return self._test_to_expectations[test] - def GetExpectationsForAllPlatforms(self): - return self._all_expectations + def GetExpectationsJsonForAllPlatforms(self): + # Specify separators in order to get compact encoding. + return ExpectationsJsonEncoder(separators=(',', ':')).encode( + self._all_expectations) def Contains(self, test): return test in self._test_to_expectations diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py index c157ac9..449cc68 100755 --- a/webkit/tools/layout_tests/run_webkit_tests.py +++ b/webkit/tools/layout_tests/run_webkit_tests.py @@ -585,11 +585,7 @@ class TestRunner: # dashboard. expectations_file = open(os.path.join(self._options.results_directory, "expectations.json"), "w") - # TODO(ojan): Generate JSON using a JSON library instead of relying on - # GetExpectationsForAllPlatforms returning an object that only uses - # primitive types. - expectations_json = repr( - self._expectations.GetExpectationsForAllPlatforms()) + expectations_json = self._expectations.GetExpectationsJsonForAllPlatforms() expectations_file.write(("ADD_EXPECTATIONS(" + expectations_json + ");")) expectations_file.close() @@ -598,7 +594,8 @@ class TestRunner: builder_name = self._options.builder_name # "WebKitBuilder" build_number = self._options.build_number # "12346" json_generator = json_results_generator.JSONResultsGenerator(failures, - individual_test_timings, builder_name, build_number, results_file_path) + individual_test_timings, builder_name, build_number, + results_file_path, self._test_files_list) results_json = json_generator.GetJSON() results_file = open(results_file_path, "w") @@ -1175,10 +1172,10 @@ if '__main__' == __name__: help=("Run a the tests in batches (n), after every " "n tests, the test shell is relaunched.")) option_parser.add_option("", "--builder-name", - default=None, + default="DUMMY_BUILDER_NAME", help="The name of the builder running this script.") option_parser.add_option("", "--build-number", - default=None, + default="DUMMY_BUILD_NUMBER", help=("The build number of the builder running" "this script.")) option_parser.add_option("", "--find-baselines", action="store_true", |