summaryrefslogtreecommitdiffstats
path: root/webkit/tools
diff options
context:
space:
mode:
Diffstat (limited to 'webkit/tools')
-rw-r--r--webkit/tools/layout_tests/layout_package/compare_failures.py44
-rw-r--r--webkit/tools/layout_tests/layout_package/test_expectations.py11
-rwxr-xr-xwebkit/tools/layout_tests/test_expectations.txt9
3 files changed, 44 insertions, 20 deletions
diff --git a/webkit/tools/layout_tests/layout_package/compare_failures.py b/webkit/tools/layout_tests/layout_package/compare_failures.py
index 28efeb6..80f435a 100644
--- a/webkit/tools/layout_tests/layout_package/compare_failures.py
+++ b/webkit/tools/layout_tests/layout_package/compare_failures.py
@@ -2,9 +2,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-"""A helper class for comparing the failures and crashes between layout test
-runs. The results from the last test run are stored in expected-failures.txt
-and expected-crashes.txt in the layout test results directory."""
+"""A helper class for comparing the failures and crashes from a test run
+against what we expected to happen (as specified in test_expectations.txt)."""
import errno
import os
@@ -22,7 +21,7 @@ def PrintFilesFromSet(filenames, header_text, output, opt_expectations=None,
filenames: a list of absolute filenames
header_text: a string to display before the list of filenames
output: file descriptor to write the results to.
- opt_expectations: expecations that failed for this test
+ opt_expectations: expectations that failed for this test
"""
if not len(filenames):
return
@@ -44,6 +43,8 @@ class CompareFailures:
# A list of which TestFailure classes count as a failure vs a crash.
FAILURE_TYPES = (test_failures.FailureTextMismatch,
test_failures.FailureImageHashMismatch)
+ TEXT_FAILURE_TYPES = (test_failures.FailureTextMismatch,)
+ IMAGE_FAILURE_TYPES = (test_failures.FailureImageHashMismatch,)
CRASH_TYPES = (test_failures.FailureCrash,)
HANG_TYPES = (test_failures.FailureTimeout,)
MISSING_TYPES = (test_failures.FailureMissingResult,
@@ -51,7 +52,7 @@ class CompareFailures:
def __init__(self, test_files, test_failures, expectations):
- """Read the past layout test run's failures from disk.
+ """Calculate the regressions in this test run.
Args:
test_files is a set of the filenames of all the test cases we ran
@@ -121,17 +122,21 @@ class CompareFailures:
missing = set()
failures = set()
- for test, failure_types in self._test_failures.iteritems():
+ for test, failure_type_instances in self._test_failures.iteritems():
# Although each test can have multiple test_failures, we only put them
# into one list (either the crash list or the failure list). We give
# priority to a crash/timeout over others, and to missing results over
# a text mismatch.
- is_crash = [True for f in failure_types if type(f) in self.CRASH_TYPES]
- is_hang = [True for f in failure_types if type(f) in self.HANG_TYPES]
- is_missing = [True for f in failure_types
- if type(f) in self.MISSING_TYPES]
- is_failure = [True for f in failure_types
- if type(f) in self.FAILURE_TYPES]
+ failure_types = [type(f) for f in failure_type_instances]
+ is_crash = [True for f in failure_types if f in self.CRASH_TYPES]
+ is_hang = [True for f in failure_types if f in self.HANG_TYPES]
+ is_missing = [True for f in failure_types if f in self.MISSING_TYPES]
+ is_failure = [True for f in failure_types if f in self.FAILURE_TYPES]
+ is_text_failure = [True for f in failure_types if f in
+ self.TEXT_FAILURE_TYPES]
+ is_image_failure = [True for f in failure_types if f in
+ self.IMAGE_FAILURE_TYPES]
+
expectations = self._expectations.GetExpectations(test)
if is_crash:
if not test_expectations.CRASH in expectations: crashes.add(test)
@@ -141,8 +146,20 @@ class CompareFailures:
# expected files.
elif is_missing and not self._expectations.IsRebaselining(test):
missing.add(test)
+ elif is_image_failure and is_text_failure:
+ if (not test_expectations.FAIL in expectations and
+ not test_expectations.IMAGE_PLUS_TEXT in expectations):
+ failures.add(test)
+ elif is_image_failure:
+ if (not test_expectations.FAIL in expectations and
+ not test_expectations.IMAGE in expectations):
+ failures.add(test)
+ elif is_text_failure:
+ if (not test_expectations.FAIL in expectations and
+ not test_expectations.TEXT in expectations):
+ failures.add(test)
elif is_failure:
- if not test_expectations.FAIL in expectations: failures.add(test)
+ raise ValueError('unexpected failure type:' + f)
worklist.remove(test)
for test in worklist:
@@ -156,7 +173,6 @@ class CompareFailures:
self._missing = missing
self._regressed_failures = failures
-
def GetRegressions(self):
"""Returns a set of regressions from the test expectations. This is
used to determine which tests to list in results.html and the
diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py
index 7e885fd..39a0e79 100644
--- a/webkit/tools/layout_tests/layout_package/test_expectations.py
+++ b/webkit/tools/layout_tests/layout_package/test_expectations.py
@@ -18,8 +18,8 @@ sys.path.append(path_utils.PathFromBase('third_party'))
import simplejson
# Test expectation and modifier constants.
-(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, REBASELINE, NONE) = \
- range(10)
+(PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX,
+ DEFER, SLOW, REBASELINE, NONE) = range(13)
# Test expectation file update action constants
(NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4)
@@ -195,6 +195,10 @@ class TestExpectationsFile:
Notes:
-A test cannot be both SLOW and TIMEOUT
-A test cannot be both DEFER and WONTFIX
+ -A test should only be one of IMAGE, TEXT, IMAGE+TEXT, or FAIL. FAIL is
+ a migratory state that currently means either IMAGE, TEXT, or IMAGE+TEXT.
+ Once we have finished migrating the expectations, we will change FAIL
+ to have the meaning of IMAGE+TEXT and remove the IMAGE+TEXT identifier.
-A test can be included twice, but not via the same path.
-If a test is included twice, then the more precise path wins.
-CRASH tests cannot be DEFER or WONTFIX
@@ -202,6 +206,9 @@ class TestExpectationsFile:
EXPECTATIONS = { 'pass': PASS,
'fail': FAIL,
+ 'text': TEXT,
+ 'image': IMAGE,
+ 'image+text': IMAGE_PLUS_TEXT,
'timeout': TIMEOUT,
'crash': CRASH }
diff --git a/webkit/tools/layout_tests/test_expectations.txt b/webkit/tools/layout_tests/test_expectations.txt
index e11a57d..73c18b0 100755
--- a/webkit/tools/layout_tests/test_expectations.txt
+++ b/webkit/tools/layout_tests/test_expectations.txt
@@ -543,8 +543,9 @@ BUG8630 WIN LINUX : LayoutTests/fast/canvas/canvas-incremental-repaint-2.html =
BUG8630 WIN LINUX : LayoutTests/fast/canvas/canvas-incremental-repaint.html = FAIL
BUG8630 WIN LINUX : LayoutTests/fast/backgrounds/solid-color-context-restore.html = FAIL
BUG8630 WIN LINUX : LayoutTests/fast/dynamic/containing-block-change.html = FAIL
-BUG8630 WIN LINUX : LayoutTests/fast/forms/hidden-listbox.html = FAIL
-BUG8630 WIN LINUX : LayoutTests/fast/forms/listbox-clip.html = FAIL
+BUG8630 WIN LINUX : LayoutTests/fast/forms/hidden-listbox.html = IMAGE+TEXT
+BUG8630 WIN : LayoutTests/fast/forms/listbox-clip.html = IMAGE
+BUG8630 LINUX : LayoutTests/fast/forms/listbox-clip.html = IMAGE+TEXT
BUG8630 WIN LINUX : LayoutTests/fast/overflow/hidden-scrollbar-resize.html = FAIL
BUG8630 WIN LINUX : LayoutTests/fast/reflections/inline-crash.html = FAIL
BUG8630 WIN LINUX : LayoutTests/fast/text/font-initial.html = FAIL
@@ -680,8 +681,8 @@ BUG19625 SKIP : LayoutTests/fast/eventsource = TIMEOUT
BUG19625 SKIP : LayoutTests/http/tests/eventsource = TIMEOUT
// Implement HTML5 datalist element.
-BUG20226 : LayoutTests/fast/forms/datalist-nonoption-child.html = FAIL
-BUG20226 : LayoutTests/fast/forms/datalist.html = FAIL
+BUG20226 : LayoutTests/fast/forms/datalist-nonoption-child.html = TEXT
+BUG20226 : LayoutTests/fast/forms/datalist.html = TEXT
BUG20292 : LayoutTests/fast/forms/input-list.html = FAIL
BUG20292 : LayoutTests/fast/forms/input-selectedoption.html = FAIL