Add the ability to distinguish between tests that fail due to a difference

in the text output, tests that fail due to differences in the image output,
and tests that fail both.

This adds the additional failure types 'IMAGE', 'TEXT', and 'BOTH' to 
test_expectations.txt. Tests that are marked 'FAIL' should eventually be
migrated to one of the above three, but for now FAIL means that any of the
three types might happen. 

Note that we do not distinguish between a text diff and a simplified text
diff; 'TEXT' will cover either or both types of failures.

  BUG=none
  TEST=none
  R=ojan@chromium.org

Review URL: http://codereview.chromium.org/235016

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@27305 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 44 insertions, 20 deletions

diff --git a/webkit/tools/layout_tests/layout_package/compare_failures.py b/webkit/tools/layout_tests/layout_package/compare_failures.py
index 28efeb6..80f435a 100644
--- a/webkit/tools/layout_tests/layout_package/compare_failures.py
+++ b/webkit/tools/layout_tests/layout_package/compare_failures.py
@@ -2,9 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""A helper class for comparing the failures and crashes between layout test
-runs.  The results from the last test run are stored in expected-failures.txt
-and expected-crashes.txt in the layout test results directory."""
+"""A helper class for comparing the failures and crashes from a test run
+against what we expected to happen (as specified in test_expectations.txt)."""
 
 import errno
 import os
@@ -22,7 +21,7 @@ def PrintFilesFromSet(filenames, header_text, output, opt_expectations=None,
   filenames: a list of absolute filenames
   header_text: a string to display before the list of filenames
   output: file descriptor to write the results to.
-  opt_expectations: expecations that failed for this test
+  opt_expectations: expectations that failed for this test
   """
   if not len(filenames):
     return
@@ -44,6 +43,8 @@ class CompareFailures:
   # A list of which TestFailure classes count as a failure vs a crash.
   FAILURE_TYPES = (test_failures.FailureTextMismatch,
                    test_failures.FailureImageHashMismatch)
+  TEXT_FAILURE_TYPES = (test_failures.FailureTextMismatch,)
+  IMAGE_FAILURE_TYPES = (test_failures.FailureImageHashMismatch,)
   CRASH_TYPES = (test_failures.FailureCrash,)
   HANG_TYPES = (test_failures.FailureTimeout,)
   MISSING_TYPES = (test_failures.FailureMissingResult,
@@ -51,7 +52,7 @@ class CompareFailures:
 
 
   def __init__(self, test_files, test_failures, expectations):
-    """Read the past layout test run's failures from disk.
+    """Calculate the regressions in this test run.
 
     Args:
       test_files is a set of the filenames of all the test cases we ran
@@ -121,17 +122,21 @@ class CompareFailures:
     missing = set()
     failures = set()
 
-    for test, failure_types in self._test_failures.iteritems():
+    for test, failure_type_instances in self._test_failures.iteritems():
       # Although each test can have multiple test_failures, we only put them
       # into one list (either the crash list or the failure list).  We give
       # priority to a crash/timeout over others, and to missing results over
       # a text mismatch.
-      is_crash = [True for f in failure_types if type(f) in self.CRASH_TYPES]
-      is_hang = [True for f in failure_types if type(f) in self.HANG_TYPES]
-      is_missing = [True for f in failure_types
-                    if type(f) in self.MISSING_TYPES]
-      is_failure = [True for f in failure_types
-                    if type(f) in self.FAILURE_TYPES]
+      failure_types = [type(f) for f in failure_type_instances]
+      is_crash = [True for f in failure_types if f in self.CRASH_TYPES]
+      is_hang = [True for f in failure_types if f in self.HANG_TYPES]
+      is_missing = [True for f in failure_types if f in self.MISSING_TYPES]
+      is_failure = [True for f in failure_types if f in self.FAILURE_TYPES]
+      is_text_failure = [True for f in failure_types if f in
+         self.TEXT_FAILURE_TYPES]
+      is_image_failure = [True for f in failure_types if f in
+         self.IMAGE_FAILURE_TYPES]
+
       expectations = self._expectations.GetExpectations(test)
       if is_crash:
         if not test_expectations.CRASH in expectations: crashes.add(test)
@@ -141,8 +146,20 @@ class CompareFailures:
       # expected files.
       elif is_missing and not self._expectations.IsRebaselining(test):
         missing.add(test)
+      elif is_image_failure and is_text_failure:
+        if (not test_expectations.FAIL in expectations and
+            not test_expectations.IMAGE_PLUS_TEXT in expectations):
+            failures.add(test)
+      elif is_image_failure:
+        if (not test_expectations.FAIL in expectations and
+            not test_expectations.IMAGE in expectations):
+            failures.add(test)
+      elif is_text_failure:
+        if (not test_expectations.FAIL in expectations and
+            not test_expectations.TEXT in expectations):
+            failures.add(test)
       elif is_failure:
-        if not test_expectations.FAIL in expectations: failures.add(test)
+        raise ValueError('unexpected failure type:' + f)
       worklist.remove(test)
 
     for test in worklist:
@@ -156,7 +173,6 @@ class CompareFailures:
     self._missing = missing
     self._regressed_failures = failures
 
-
   def GetRegressions(self):
     """Returns a set of regressions from the test expectations. This is
     used to determine which tests to list in results.html and the
diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py
index 7e885fd..39a0e79 100644
--- a/webkit/tools/layout_tests/layout_package/test_expectations.py
+++ b/webkit/tools/layout_tests/layout_package/test_expectations.py
@@ -18,8 +18,8 @@ sys.path.append(path_utils.PathFromBase('third_party'))
 import simplejson
 
 # Test expectation and modifier constants.
-(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, REBASELINE, NONE) = \
-    range(10)
+(PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX,
+ DEFER, SLOW, REBASELINE, NONE) = range(13)
 
 # Test expectation file update action constants
 (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4)
@@ -195,6 +195,10 @@ class TestExpectationsFile:
   Notes:
     -A test cannot be both SLOW and TIMEOUT
     -A test cannot be both DEFER and WONTFIX
+    -A test should only be one of IMAGE, TEXT, IMAGE+TEXT, or FAIL. FAIL is
+     a migratory state that currently means either IMAGE, TEXT, or IMAGE+TEXT.
+     Once we have finished migrating the expectations, we will change FAIL
+     to have the meaning of IMAGE+TEXT and remove the IMAGE+TEXT identifier.
     -A test can be included twice, but not via the same path.
     -If a test is included twice, then the more precise path wins.
     -CRASH tests cannot be DEFER or WONTFIX
@@ -202,6 +206,9 @@ class TestExpectationsFile:
 
   EXPECTATIONS = { 'pass': PASS,
                    'fail': FAIL,
+                   'text': TEXT,
+                   'image': IMAGE,
+                   'image+text': IMAGE_PLUS_TEXT,
                    'timeout': TIMEOUT,
                    'crash': CRASH }
 
diff --git a/webkit/tools/layout_tests/test_expectations.txt b/webkit/tools/layout_tests/test_expectations.txt
index e11a57d..73c18b0 100755
--- a/webkit/tools/layout_tests/test_expectations.txt
+++ b/webkit/tools/layout_tests/test_expectations.txt
@@ -543,8 +543,9 @@ BUG8630 WIN LINUX : LayoutTests/fast/canvas/canvas-incremental-repaint-2.html =
 BUG8630 WIN LINUX : LayoutTests/fast/canvas/canvas-incremental-repaint.html = FAIL
 BUG8630 WIN LINUX : LayoutTests/fast/backgrounds/solid-color-context-restore.html = FAIL
 BUG8630 WIN LINUX : LayoutTests/fast/dynamic/containing-block-change.html = FAIL
-BUG8630 WIN LINUX : LayoutTests/fast/forms/hidden-listbox.html = FAIL
-BUG8630 WIN LINUX : LayoutTests/fast/forms/listbox-clip.html = FAIL
+BUG8630 WIN LINUX : LayoutTests/fast/forms/hidden-listbox.html = IMAGE+TEXT
+BUG8630 WIN : LayoutTests/fast/forms/listbox-clip.html = IMAGE
+BUG8630 LINUX : LayoutTests/fast/forms/listbox-clip.html = IMAGE+TEXT
 BUG8630 WIN LINUX : LayoutTests/fast/overflow/hidden-scrollbar-resize.html = FAIL
 BUG8630 WIN LINUX : LayoutTests/fast/reflections/inline-crash.html = FAIL
 BUG8630 WIN LINUX : LayoutTests/fast/text/font-initial.html = FAIL
@@ -680,8 +681,8 @@ BUG19625 SKIP : LayoutTests/fast/eventsource = TIMEOUT
 BUG19625 SKIP : LayoutTests/http/tests/eventsource = TIMEOUT
 
 // Implement HTML5 datalist element.
-BUG20226 : LayoutTests/fast/forms/datalist-nonoption-child.html = FAIL
-BUG20226 : LayoutTests/fast/forms/datalist.html = FAIL
+BUG20226 : LayoutTests/fast/forms/datalist-nonoption-child.html = TEXT
+BUG20226 : LayoutTests/fast/forms/datalist.html = TEXT
 BUG20292 : LayoutTests/fast/forms/input-list.html = FAIL
 BUG20292 : LayoutTests/fast/forms/input-selectedoption.html = FAIL