Distinguish between different types of failures in the JSON

output of run_webkit_tests. This is so the dashboards
can match test_expectations.txt semantics. Eventually,
we can use this data to replace all the FAIL expectations
with TEXT/IMAGE/IMAGE+TEXT as appropriate.

Add a TODO to run_webkit_tests to get statistics right
now that we have IMAGE+TEXT expectations.

Treat simplified diff failures as TEXT failures for the JSON
output since noone seems to care about tracking it anymore.
We really should just stop spitting out simplified diffs at all.
Review URL: http://codereview.chromium.org/251096

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@28171 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 15 insertions, 4 deletions

diff --git a/webkit/tools/layout_tests/layout_package/json_results_generator.py b/webkit/tools/layout_tests/layout_package/json_results_generator.py
index e18b7c9..a9508cc 100644
--- a/webkit/tools/layout_tests/layout_package/json_results_generator.py
+++ b/webkit/tools/layout_tests/layout_package/json_results_generator.py
@@ -43,7 +43,15 @@ class JSONResultsGenerator:
   FIXABLE = "fixableCounts"
   ALL_FIXABLE_COUNT = "allFixableCount"
   FIXABLE_COUNT = "fixableCount"
-  FAILURE_CHARS = ("C", "T", "I", "S", "F", "O")
+  """
+    C = CRASH
+    T = TIMEOUT
+    I = IMAGE
+    F = TEXT
+    O = OTHER
+    Z = TEXT+IMAGE
+  """
+  FAILURE_CHARS = ("C", "T", "I", "F", "O", "Z")
   BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
   RESULTS_FILENAME = "results.json"
 
@@ -371,15 +379,18 @@ class JSONResultsGenerator:
     """Returns the worst failure from the list of failures
     since we can only show one failure per run for each test on the dashboard.
     """
+    has_text_failure = (test_failures.FailureTextMismatch in failures or
+        test_failures.FailureSimplifiedTextMismatch in failures)
+
     if test_failures.FailureCrash in failures:
       return "C"
     elif test_failures.FailureTimeout in failures:
       return "T"
     elif test_failures.FailureImageHashMismatch in failures:
+      if has_text_failure:
+        return "Z"
       return "I"
-    elif test_failures.FailureSimplifiedTextMismatch in failures:
-      return "S"
-    elif test_failures.FailureTextMismatch in failures:
+    elif has_text_failure:
       return "F"
     else:
       return "O"