diff options
Diffstat (limited to 'webkit/tools')
3 files changed, 44 insertions, 20 deletions
diff --git a/webkit/tools/layout_tests/layout_package/compare_failures.py b/webkit/tools/layout_tests/layout_package/compare_failures.py index 28efeb6..80f435a 100644 --- a/webkit/tools/layout_tests/layout_package/compare_failures.py +++ b/webkit/tools/layout_tests/layout_package/compare_failures.py @@ -2,9 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""A helper class for comparing the failures and crashes between layout test -runs. The results from the last test run are stored in expected-failures.txt -and expected-crashes.txt in the layout test results directory.""" +"""A helper class for comparing the failures and crashes from a test run +against what we expected to happen (as specified in test_expectations.txt).""" import errno import os @@ -22,7 +21,7 @@ def PrintFilesFromSet(filenames, header_text, output, opt_expectations=None, filenames: a list of absolute filenames header_text: a string to display before the list of filenames output: file descriptor to write the results to. - opt_expectations: expecations that failed for this test + opt_expectations: expectations that failed for this test """ if not len(filenames): return @@ -44,6 +43,8 @@ class CompareFailures: # A list of which TestFailure classes count as a failure vs a crash. FAILURE_TYPES = (test_failures.FailureTextMismatch, test_failures.FailureImageHashMismatch) + TEXT_FAILURE_TYPES = (test_failures.FailureTextMismatch,) + IMAGE_FAILURE_TYPES = (test_failures.FailureImageHashMismatch,) CRASH_TYPES = (test_failures.FailureCrash,) HANG_TYPES = (test_failures.FailureTimeout,) MISSING_TYPES = (test_failures.FailureMissingResult, @@ -51,7 +52,7 @@ class CompareFailures: def __init__(self, test_files, test_failures, expectations): - """Read the past layout test run's failures from disk. + """Calculate the regressions in this test run. Args: test_files is a set of the filenames of all the test cases we ran @@ -121,17 +122,21 @@ class CompareFailures: missing = set() failures = set() - for test, failure_types in self._test_failures.iteritems(): + for test, failure_type_instances in self._test_failures.iteritems(): # Although each test can have multiple test_failures, we only put them # into one list (either the crash list or the failure list). We give # priority to a crash/timeout over others, and to missing results over # a text mismatch. - is_crash = [True for f in failure_types if type(f) in self.CRASH_TYPES] - is_hang = [True for f in failure_types if type(f) in self.HANG_TYPES] - is_missing = [True for f in failure_types - if type(f) in self.MISSING_TYPES] - is_failure = [True for f in failure_types - if type(f) in self.FAILURE_TYPES] + failure_types = [type(f) for f in failure_type_instances] + is_crash = [True for f in failure_types if f in self.CRASH_TYPES] + is_hang = [True for f in failure_types if f in self.HANG_TYPES] + is_missing = [True for f in failure_types if f in self.MISSING_TYPES] + is_failure = [True for f in failure_types if f in self.FAILURE_TYPES] + is_text_failure = [True for f in failure_types if f in + self.TEXT_FAILURE_TYPES] + is_image_failure = [True for f in failure_types if f in + self.IMAGE_FAILURE_TYPES] + expectations = self._expectations.GetExpectations(test) if is_crash: if not test_expectations.CRASH in expectations: crashes.add(test) @@ -141,8 +146,20 @@ class CompareFailures: # expected files. elif is_missing and not self._expectations.IsRebaselining(test): missing.add(test) + elif is_image_failure and is_text_failure: + if (not test_expectations.FAIL in expectations and + not test_expectations.IMAGE_PLUS_TEXT in expectations): + failures.add(test) + elif is_image_failure: + if (not test_expectations.FAIL in expectations and + not test_expectations.IMAGE in expectations): + failures.add(test) + elif is_text_failure: + if (not test_expectations.FAIL in expectations and + not test_expectations.TEXT in expectations): + failures.add(test) elif is_failure: - if not test_expectations.FAIL in expectations: failures.add(test) + raise ValueError('unexpected failure type:' + f) worklist.remove(test) for test in worklist: @@ -156,7 +173,6 @@ class CompareFailures: self._missing = missing self._regressed_failures = failures - def GetRegressions(self): """Returns a set of regressions from the test expectations. This is used to determine which tests to list in results.html and the diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py index 7e885fd..39a0e79 100644 --- a/webkit/tools/layout_tests/layout_package/test_expectations.py +++ b/webkit/tools/layout_tests/layout_package/test_expectations.py @@ -18,8 +18,8 @@ sys.path.append(path_utils.PathFromBase('third_party')) import simplejson # Test expectation and modifier constants. -(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, REBASELINE, NONE) = \ - range(10) +(PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX, + DEFER, SLOW, REBASELINE, NONE) = range(13) # Test expectation file update action constants (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4) @@ -195,6 +195,10 @@ class TestExpectationsFile: Notes: -A test cannot be both SLOW and TIMEOUT -A test cannot be both DEFER and WONTFIX + -A test should only be one of IMAGE, TEXT, IMAGE+TEXT, or FAIL. FAIL is + a migratory state that currently means either IMAGE, TEXT, or IMAGE+TEXT. + Once we have finished migrating the expectations, we will change FAIL + to have the meaning of IMAGE+TEXT and remove the IMAGE+TEXT identifier. -A test can be included twice, but not via the same path. -If a test is included twice, then the more precise path wins. -CRASH tests cannot be DEFER or WONTFIX @@ -202,6 +206,9 @@ class TestExpectationsFile: EXPECTATIONS = { 'pass': PASS, 'fail': FAIL, + 'text': TEXT, + 'image': IMAGE, + 'image+text': IMAGE_PLUS_TEXT, 'timeout': TIMEOUT, 'crash': CRASH } diff --git a/webkit/tools/layout_tests/test_expectations.txt b/webkit/tools/layout_tests/test_expectations.txt index e11a57d..73c18b0 100755 --- a/webkit/tools/layout_tests/test_expectations.txt +++ b/webkit/tools/layout_tests/test_expectations.txt @@ -543,8 +543,9 @@ BUG8630 WIN LINUX : LayoutTests/fast/canvas/canvas-incremental-repaint-2.html = BUG8630 WIN LINUX : LayoutTests/fast/canvas/canvas-incremental-repaint.html = FAIL BUG8630 WIN LINUX : LayoutTests/fast/backgrounds/solid-color-context-restore.html = FAIL BUG8630 WIN LINUX : LayoutTests/fast/dynamic/containing-block-change.html = FAIL -BUG8630 WIN LINUX : LayoutTests/fast/forms/hidden-listbox.html = FAIL -BUG8630 WIN LINUX : LayoutTests/fast/forms/listbox-clip.html = FAIL +BUG8630 WIN LINUX : LayoutTests/fast/forms/hidden-listbox.html = IMAGE+TEXT +BUG8630 WIN : LayoutTests/fast/forms/listbox-clip.html = IMAGE +BUG8630 LINUX : LayoutTests/fast/forms/listbox-clip.html = IMAGE+TEXT BUG8630 WIN LINUX : LayoutTests/fast/overflow/hidden-scrollbar-resize.html = FAIL BUG8630 WIN LINUX : LayoutTests/fast/reflections/inline-crash.html = FAIL BUG8630 WIN LINUX : LayoutTests/fast/text/font-initial.html = FAIL @@ -680,8 +681,8 @@ BUG19625 SKIP : LayoutTests/fast/eventsource = TIMEOUT BUG19625 SKIP : LayoutTests/http/tests/eventsource = TIMEOUT // Implement HTML5 datalist element. -BUG20226 : LayoutTests/fast/forms/datalist-nonoption-child.html = FAIL -BUG20226 : LayoutTests/fast/forms/datalist.html = FAIL +BUG20226 : LayoutTests/fast/forms/datalist-nonoption-child.html = TEXT +BUG20226 : LayoutTests/fast/forms/datalist.html = TEXT BUG20292 : LayoutTests/fast/forms/input-list.html = FAIL BUG20292 : LayoutTests/fast/forms/input-selectedoption.html = FAIL |