From f360e7daa5377b5701d5520427c90bf83b80f82e Mon Sep 17 00:00:00 2001 From: "ojan@google.com" Date: Tue, 6 Jan 2009 21:40:14 +0000 Subject: Readd the concept of deferring tests till a future release and defer all the tests we had deferred for the beta release. I figure we shouldn't block the current release on tests that didn't block the beta. That said, I think we should eventually try to make defer be just for unimplemented features. Review URL: http://codereview.chromium.org/16547 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@7618 0039d316-1c4b-4281-b951-d872f2087c98 --- .../layout_package/compare_failures.py | 8 +- .../layout_package/test_expectations.py | 60 ++++- webkit/tools/layout_tests/run_webkit_tests.py | 42 +++- webkit/tools/layout_tests/test_lists/README | 1 + .../layout_tests/test_lists/tests_fixable.txt | 259 +++++++++++---------- 5 files changed, 221 insertions(+), 149 deletions(-) diff --git a/webkit/tools/layout_tests/layout_package/compare_failures.py b/webkit/tools/layout_tests/layout_package/compare_failures.py index 9b42bb6..5850fda 100644 --- a/webkit/tools/layout_tests/layout_package/compare_failures.py +++ b/webkit/tools/layout_tests/layout_package/compare_failures.py @@ -86,7 +86,13 @@ class CompareFailures: PrintFilesFromSet(passes & self._expectations.GetIgnoredTimeouts(), "Expected to timeout (ignored), but passed", output) - + # Crashes should never be deferred. + PrintFilesFromSet(passes & self._expectations.GetFixableDeferredFailures(), + "Expected to fail (deferred), but passed", + output) + PrintFilesFromSet(passes & self._expectations.GetFixableDeferredTimeouts(), + "Expected to timeout (deferred), but passed", + output) # Print real regressions. PrintFilesFromSet(self._regressed_failures, "Regressions: Unexpected failures", diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py index e24ae30..3a42008 100644 --- a/webkit/tools/layout_tests/layout_package/test_expectations.py +++ b/webkit/tools/layout_tests/layout_package/test_expectations.py @@ -34,23 +34,41 @@ class TestExpectations: self._ValidateLists() def GetFixable(self): - return self._fixable.GetTests() + return (self._fixable.GetTests() - + self._fixable.GetNonSkippedDeferred() - + self._fixable.GetSkippedDeferred()) def GetFixableSkipped(self): return self._fixable.GetSkipped() + def GetFixableSkippedDeferred(self): + return self._fixable.GetSkippedDeferred() + def GetFixableFailures(self): return (self._fixable.GetTestsExpectedTo(FAIL) - self._fixable.GetTestsExpectedTo(TIMEOUT) - - self._fixable.GetTestsExpectedTo(CRASH)) + self._fixable.GetTestsExpectedTo(CRASH) - + self._fixable.GetNonSkippedDeferred()) def GetFixableTimeouts(self): return (self._fixable.GetTestsExpectedTo(TIMEOUT) - - self._fixable.GetTestsExpectedTo(CRASH)) + self._fixable.GetTestsExpectedTo(CRASH) - + self._fixable.GetNonSkippedDeferred()) def GetFixableCrashes(self): return self._fixable.GetTestsExpectedTo(CRASH) + def GetFixableDeferred(self): + return self._fixable.GetNonSkippedDeferred() + + def GetFixableDeferredFailures(self): + return (self._fixable.GetNonSkippedDeferred() & + self._fixable.GetTestsExpectedTo(FAIL)) + + def GetFixableDeferredTimeouts(self): + return (self._fixable.GetNonSkippedDeferred() & + self._fixable.GetTestsExpectedTo(TIMEOUT)) + def GetIgnored(self): return self._ignored.GetTests() @@ -72,10 +90,12 @@ class TestExpectations: return set([PASS]) def IsFixable(self, test): - return self._fixable.Contains(test) + return (self._fixable.Contains(test) and + test not in self._fixable.GetNonSkippedDeferred()) def IsIgnored(self, test): - return self._ignored.Contains(test) + return (self._ignored.Contains(test) and + test not in self._fixable.GetNonSkippedDeferred()) def _ReadFiles(self): self._fixable = self._GetExpectationsFile(self.FIXABLE) @@ -133,9 +153,10 @@ class TestExpectationsFile: DEBUG : LayoutTests/fast/js/no-good.js = TIMEOUT PASS DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS LINUX DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS - LINUX WIN : LayoutTests/fast/js/no-good.js = TIMEOUT PASS + DEFER LINUX WIN : LayoutTests/fast/js/no-good.js = TIMEOUT PASS SKIP: Doesn't run the test. + DEFER: Test does not count in our statistics for the current release. DEBUG: Expectations apply only to the debug build. RELEASE: Expectations apply only to release build. LINUX/WIN/MAC: Expectations apply only to these platforms. @@ -163,6 +184,8 @@ class TestExpectationsFile: self._full_test_list = full_test_list self._skipped = set() + self._skipped_deferred = set() + self._non_skipped_deferred = set() self._expectations = {} self._test_list_paths = {} self._tests = {} @@ -176,6 +199,12 @@ class TestExpectationsFile: def GetSkipped(self): return self._skipped + def GetNonSkippedDeferred(self): + return self._non_skipped_deferred + + def GetSkippedDeferred(self): + return self._skipped_deferred + def GetExpectations(self, test): return self._expectations[test] @@ -225,11 +254,13 @@ class TestExpectationsFile: if line.find(':') is -1: test_and_expectations = line is_skipped = False + is_deferred = False else: parts = line.split(':') test_and_expectations = parts[1] options = self._GetOptionsList(parts[0]) is_skipped = 'skip' in options + is_deferred = 'defer' in options if 'release' in options or 'debug' in options: if self._is_debug_mode and 'debug' not in options: continue @@ -266,9 +297,10 @@ class TestExpectationsFile: tests = self._ExpandTests(test_list_path) if is_skipped: - self._AddSkippedTests(tests) + self._AddSkippedTests(tests, is_deferred) else: - self._AddTests(tests, expectations, test_list_path, lineno) + self._AddTests(tests, expectations, test_list_path, lineno, + is_deferred) if len(self._errors) is not 0: print "\nFAILURES FOR PLATFORM: %s, IS_DEBUG_MODE: %s" \ @@ -302,7 +334,8 @@ class TestExpectationsFile: if test.startswith(path): result.append(test) return result - def _AddTests(self, tests, expectations, test_list_path, lineno): + def _AddTests(self, tests, expectations, test_list_path, lineno, + is_deferred): for test in tests: if test in self._test_list_paths: prev_base_path = self._test_list_paths[test] @@ -323,11 +356,18 @@ class TestExpectationsFile: self._expectations[test] = expectations self._test_list_paths[test] = os.path.normpath(test_list_path) + if is_deferred: + self._non_skipped_deferred.add(test) + for expectation in expectations: + if expectation == CRASH and is_deferred: + self._AddError(lineno, 'Crashes cannot be deferred.', test) self._tests[expectation].add(test) - def _AddSkippedTests(self, tests): + def _AddSkippedTests(self, tests, is_deferred): for test in tests: + if is_deferred: + self._skipped_deferred.add(test) self._skipped.add(test) def _AddError(self, lineno, msg, path): diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py index 66c0da2..e94483c 100755 --- a/webkit/tools/layout_tests/run_webkit_tests.py +++ b/webkit/tools/layout_tests/run_webkit_tests.py @@ -219,16 +219,22 @@ class TestRunner: else: logging.info('Run: %d tests' % len(self._test_files)) + logging.info('Deferred: %d tests' % + len(self._expectations.GetFixableDeferred())) logging.info('Expected passes: %d tests' % len(self._test_files - self._expectations.GetFixable() - self._expectations.GetIgnored())) - logging.info(('Expected failures: %d fixable, %d ignored') % + logging.info(('Expected failures: %d fixable, %d ignored ' + 'and %d deferred tests') % (len(self._expectations.GetFixableFailures()), - len(self._expectations.GetIgnoredFailures()))) - logging.info(('Expected timeouts: %d fixable, %d ignored') % + len(self._expectations.GetIgnoredFailures()), + len(self._expectations.GetFixableDeferredFailures()))) + logging.info(('Expected timeouts: %d fixable, %d ignored ' + 'and %d deferred tests') % (len(self._expectations.GetFixableTimeouts()), - len(self._expectations.GetIgnoredTimeouts()))) + len(self._expectations.GetIgnoredTimeouts()), + len(self._expectations.GetFixableDeferredTimeouts()))) logging.info('Expected crashes: %d fixable tests' % len(self._expectations.GetFixableCrashes())) @@ -395,9 +401,11 @@ class TestRunner: """ failure_counts = {} + deferred_counts = {} fixable_counts = {} non_ignored_counts = {} fixable_failures = set() + deferred_failures = set() non_ignored_failures = set() # Aggregate failures in a dictionary (TestFailure -> frequency), @@ -420,20 +428,31 @@ class TestRunner: # Print breakdown of tests we need to fix and want to pass. # Include skipped fixable tests in the statistics. - skipped = self._expectations.GetFixableSkipped().copy() + skipped = (self._expectations.GetFixableSkipped() - + self._expectations.GetFixableSkippedDeferred()) - self._PrintResultSummary("=> Tests to be fixed", + self._PrintResultSummary("=> Tests to be fixed for the current release", self._expectations.GetFixable(), fixable_failures, fixable_counts, - skipped, output) + skipped, + output) - self._PrintResultSummary("=> Tests we want to pass", + self._PrintResultSummary("=> Tests we want to pass for the current release", (self._test_files - - self._expectations.GetIgnored()), + self._expectations.GetIgnored() - + self._expectations.GetFixableDeferred()), non_ignored_failures, non_ignored_counts, - skipped, output) + skipped, + output) + + self._PrintResultSummary("=> Tests to be fixed for a future release", + self._expectations.GetFixableDeferred(), + deferred_failures, + deferred_counts, + self._expectations.GetFixableSkippedDeferred(), + output) # Print breakdown of all tests including all skipped tests. skipped |= self._expectations.GetIgnoredSkipped() @@ -441,7 +460,8 @@ class TestRunner: self._test_files, test_failures, failure_counts, - skipped, output) + skipped, + output) print def _PrintResultSummary(self, heading, all, failed, failure_counts, skipped, diff --git a/webkit/tools/layout_tests/test_lists/README b/webkit/tools/layout_tests/test_lists/README index 19b9b73..cf035a9 100644 --- a/webkit/tools/layout_tests/test_lists/README +++ b/webkit/tools/layout_tests/test_lists/README @@ -9,6 +9,7 @@ file a bug, then add it to tests_fixable.txt. Tests can have any combination of the following metadata associated with them (optional): SKIP : We don't want to run the test (perhaps because it hangs). + DEFER : Test does not count in our statistics for the current release. DEBUG : Expectations apply only to the Debug build. RELEASE : Expectations apply only to the Release build. LINUX/WIN/MAC: Expectations apply only to the listed platform(s). diff --git a/webkit/tools/layout_tests/test_lists/tests_fixable.txt b/webkit/tools/layout_tests/test_lists/tests_fixable.txt index 9fabeb1..4dc8179 100644 --- a/webkit/tools/layout_tests/test_lists/tests_fixable.txt +++ b/webkit/tools/layout_tests/test_lists/tests_fixable.txt @@ -3,24 +3,24 @@ // Bug 1124548: Copying with no selection is sometimes supposed to work // Also skipped by Apple on Windows (rdar://problem/5015941) -LayoutTests/editing/execCommand/copy-without-selection.html = FAIL +DEFER : LayoutTests/editing/execCommand/copy-without-selection.html = FAIL // onload race condition due to poorly designed test. // Works fine when run stand-alone. Not needed for Beta. // Also skipped by Apple on Windows, due to intermittent failure // (rdar://5313536) -LINUX WIN : LayoutTests/fast/dom/frame-loading-via-document-write.html = FAIL TIMEOUT +DEFER LINUX WIN : LayoutTests/fast/dom/frame-loading-via-document-write.html = FAIL TIMEOUT // Bug 877986. // WebKit's CSS counters are somewhat broken, thus expected results are failures // Our high-precision timers make these tests flakey. // We could fork these tests, but we'll just unfork them as soon as // our high-precision timers are public. -LayoutTests/css2.1/t1204-increment-00-c-o.html = FAIL -LayoutTests/css2.1/t1204-increment-01-c-o.html = FAIL -LayoutTests/css2.1/t1204-increment-02-c-o.html = FAIL -LayoutTests/css2.1/t1204-reset-00-c-o.html = FAIL -WIN MAC : LayoutTests/css2.1/t1204-reset-01-c-o.html = FAIL +DEFER : LayoutTests/css2.1/t1204-increment-00-c-o.html = FAIL +DEFER : LayoutTests/css2.1/t1204-increment-01-c-o.html = FAIL +DEFER : LayoutTests/css2.1/t1204-increment-02-c-o.html = FAIL +DEFER : LayoutTests/css2.1/t1204-reset-00-c-o.html = FAIL +DEFER WIN MAC : LayoutTests/css2.1/t1204-reset-01-c-o.html = FAIL // This are failing for different reasons under our new lighttpd configuration // TODO(deanm): Address all of these via lighttpd if possible, otherwise fork. @@ -49,56 +49,57 @@ LayoutTests/http/tests/xmlhttprequest/upload-progress-events.html = FAIL // Bug 1316221: fail now that we use the same font code path in test_shell // as in Chrome -LINUX WIN : LayoutTests/css2.1/t1202-counter-04-b.html = FAIL -LINUX WIN : LayoutTests/css2.1/t1202-counters-04-b.html = FAIL +DEFER LINUX WIN : LayoutTests/css2.1/t1202-counter-04-b.html = FAIL +DEFER LINUX WIN : LayoutTests/css2.1/t1202-counters-04-b.html = FAIL // Bug 1124513: the max length is being applied correctly, but the over- and // under-lines aren't placed properly over the "x". +// Can defer this test once the crash is fixed. LayoutTests/fast/forms/input-text-maxlength.html = FAIL CRASH -LayoutTests/fast/forms/input-text-paste-maxlength.html = FAIL +DEFER : LayoutTests/fast/forms/input-text-paste-maxlength.html = FAIL // Font differences, requiring overriden metrics, not a real bug, not fixing for Beta -LINUX WIN : LayoutTests/fast/text/international/bidi-AN-after-L.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/international/bidi-AN-after-L.html = FAIL // Bug: 1145880 // Parethesis missing, metrics wrong. -LINUX WIN : LayoutTests/fast/text/international/bidi-neutral-run.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/international/bidi-neutral-run.html = FAIL // Bug: 628529: complex text effects // This is a real bug, but not one we're fixing for Beta. -LINUX WIN : LayoutTests/fast/text/stroking-decorations.html = FAIL -LINUX WIN : LayoutTests/fast/text/stroking.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/stroking-decorations.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/stroking.html = FAIL // Bug: 1124522 // Incrorect results, in incorrect international font metrics. // Fixing these overrides does not help us to Beta, deffering -LINUX WIN : LayoutTests/fast/text/atsui-multiple-renderers.html = FAIL -LINUX WIN : LayoutTests/fast/text/atsui-pointtooffset-calls-cg.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/atsui-multiple-renderers.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/atsui-pointtooffset-calls-cg.html = FAIL // Bug: 1143381 // This test checks that we hack around a bug in helvetica. We fail to. -LINUX WIN : LayoutTests/fast/text/wide-zero-width-space.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/wide-zero-width-space.html = FAIL // Font-size differences in international text cause the wrong character // to be under the (x,y) click location used by the test. See bug 850411 // on faking international font sizes like we do for Latin fonts. -LINUX WIN : LayoutTests/fast/text/atsui-rtl-override-selection.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/atsui-rtl-override-selection.html = FAIL // Bug: 1124542 // More missing international text overides, not needed for Beta. // Capitalization results match Safari, even if "not fully correct" -LINUX WIN : LayoutTests/fast/text/capitalize-boundaries.html = FAIL +DEFER LINUX WIN : LayoutTests/fast/text/capitalize-boundaries.html = FAIL // Bug: 1145887 // Different button line-heights, our behavior looks wrong. -LayoutTests/fast/forms/control-restrict-line-height.html = FAIL -LayoutTests/fast/replaced/table-percent-height.html = FAIL +DEFER : LayoutTests/fast/forms/control-restrict-line-height.html = FAIL +DEFER : LayoutTests/fast/replaced/table-percent-height.html = FAIL // Bug 992930: Unable to load file:/// URLs from data: URLs. -RELEASE : LayoutTests/fast/events/standalone-image-drag-to-editable.html = FAIL +DEFER RELEASE : LayoutTests/fast/events/standalone-image-drag-to-editable.html = FAIL // Bug 1187672. Two font faces should be identical but aren't. Punting SVG. -LINUX WIN : LayoutTests/svg/custom/font-face-simple.svg = FAIL +DEFER LINUX WIN : LayoutTests/svg/custom/font-face-simple.svg = FAIL LINUX WIN : LayoutTests/svg/custom/use-on-non-svg-namespaced-element.svg = FAIL @@ -112,12 +113,12 @@ LINUX WIN : LayoutTests/svg/custom/use-on-non-svg-namespaced-element.svg = FAIL // Implicit expectation in this test is that you can "set query" on a data URL, // and it should replace the first "?" substring. This makes absolutely no sense. -LayoutTests/fast/events/stopPropagation-submit.html = FAIL +DEFER : LayoutTests/fast/events/stopPropagation-submit.html = FAIL // Expected results has a terminal "?", since "set query" on about:blank is allowed. // This is strange since query should have no meaning in non-standard urls -LINUX WIN : LayoutTests/http/tests/navigation/onload-navigation-iframe-timeout.html = FAIL -LINUX WIN : LayoutTests/http/tests/navigation/onload-navigation-iframe.html = FAIL +DEFER LINUX WIN : LayoutTests/http/tests/navigation/onload-navigation-iframe-timeout.html = FAIL +DEFER LINUX WIN : LayoutTests/http/tests/navigation/onload-navigation-iframe.html = FAIL // ----------------------------------------------------------------- // PENDING TESTS (forked to pending/, need to be sent upstream) @@ -153,7 +154,8 @@ LayoutTests/http/tests/navigation/javascriptlink-frames.html = FAIL // Bug: 1143492 // Window status should always return a string object // WebKit does this to match IE, FF also fails this test. -LayoutTests/fast/dom/assign-to-window-status.html = FAIL +// Obscure, not sure we care. DEFER for now. +DEFER : LayoutTests/fast/dom/assign-to-window-status.html = FAIL // Bug 905894 // Getting parseerror (probably wrong svn:eol-style) @@ -165,29 +167,29 @@ LINUX WIN : LayoutTests/fast/xsl/xslt-enc16to16.xml = FAIL // Mac Safari under certain circumstances automatically places // a caret in editable document even when none was requested programatically. // We don't intend to copy this feature (at least not for Beta). -LINUX WIN : LayoutTests/editing/selection/designmode-no-caret.html = FAIL +DEFER LINUX WIN : LayoutTests/editing/selection/designmode-no-caret.html = FAIL // Bug: 742182, 845388, 960092 // Platform-specific: simulates command-{arrow} input to modify selection // Our Event-Sender isn't robust enough to support this. // Not required for Beta. This may also be related to known home/end issues // which are intended to be fixed for Beta. -LINUX WIN : LayoutTests/editing/selection/move-begin-end.html = FAIL +DEFER LINUX WIN : LayoutTests/editing/selection/move-begin-end.html = FAIL // Bug 845400 // The end result looks right, but the event messages differ. -LayoutTests/editing/pasteboard/paste-xml.xhtml = FAIL +DEFER : LayoutTests/editing/pasteboard/paste-xml.xhtml = FAIL // Bug 849441 // Directionality of mixed-direction text in selected choice should // match that in the