Rollback r33709, r33640 until we can figure out why the output is getting

interleaved in the buildbots on linux ...

  BUG=none
  R=ojan
  TEST=none

Review URL: http://codereview.chromium.org/464022

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@33715 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 1006 insertions, 594 deletions

diff --git a/webkit/tools/layout_tests/layout_package/compare_failures.py b/webkit/tools/layout_tests/layout_package/compare_failures.py
new file mode 100644
index 0000000..010a584
--- /dev/null
+++ b/webkit/tools/layout_tests/layout_package/compare_failures.py
@@ -0,0 +1,216 @@
+# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A helper class for comparing the failures and crashes from a test run
+against what we expected to happen (as specified in test_expectations.txt)."""
+
+import errno
+import os
+
+import path_utils
+import test_failures
+import test_expectations
+
+class CompareFailures:
+  # A list of which TestFailure classes count as a failure vs a crash.
+  FAILURE_TYPES = (test_failures.FailureTextMismatch,
+                   test_failures.FailureImageHashMismatch)
+  TEXT_FAILURE_TYPES = (test_failures.FailureTextMismatch,)
+  IMAGE_FAILURE_TYPES = (test_failures.FailureImageHashMismatch,)
+  CRASH_TYPES = (test_failures.FailureCrash,)
+  HANG_TYPES = (test_failures.FailureTimeout,)
+  MISSING_TYPES = (test_failures.FailureMissingResult,
+                   test_failures.FailureMissingImage,
+                   test_failures.FailureMissingImageHash)
+
+
+  def __init__(self, test_files, test_failures, expectations, is_flaky):
+    """Calculate the regressions in this test run.
+
+    Args:
+      test_files is a set of the filenames of all the test cases we ran
+      test_failures is a dictionary mapping the test filename to a list of
+          TestFailure objects if the test failed
+      expectations is a TestExpectations object representing the
+          current test status
+      is_flaky is whether this set of failures represents tests that failed
+          the first time around, but passed on a subsequent run
+    """
+    self._test_files = test_files
+    self._test_failures = test_failures
+    self._expectations = expectations
+    self._is_flaky = is_flaky
+    self._CalculateRegressions()
+
+
+  def PrintFilesFromSet(self, filenames, header_text, output,
+      opt_expectations=None):
+    """A helper method to print a list of files to output.
+
+    Args:
+    filenames: a list of absolute filenames
+    header_text: a string to display before the list of filenames
+    output: file descriptor to write the results to.
+    opt_expectations: expectations that failed for this test
+    """
+    if not len(filenames):
+      return
+
+    filenames = list(filenames)
+    filenames.sort()
+
+    # Print real regressions.
+    if opt_expectations:
+      if self._is_flaky:
+        header_text = "Unexpected flakiness: " + header_text
+      else:
+        header_text = "Regressions: Unexpected " + header_text
+
+    output.write("\n%s (%d):\n" % (header_text, len(filenames)))
+    for filename in filenames:
+      output_string = "  %s" % path_utils.RelativeTestFilename(filename)
+      if opt_expectations:
+        if self._is_flaky:
+          opt_expectations += (" " +
+              self._expectations.GetExpectationsString(filename))
+        output_string += " = %s" % opt_expectations
+
+      output.write(output_string + "\n")
+
+  def PrintRegressions(self, output):
+    """Write the regressions computed by _CalculateRegressions() to output. """
+
+    # Print unexpected passes by category.
+    passes = self._regressed_passes
+    self.PrintFilesFromSet(passes & self._expectations.GetFixableFailures(),
+                           "Expected to fail, but passed",
+                           output)
+    self.PrintFilesFromSet(passes & self._expectations.GetFixableTimeouts(),
+                           "Expected to timeout, but passed",
+                           output)
+    self.PrintFilesFromSet(passes & self._expectations.GetFixableCrashes(),
+                           "Expected to crash, but passed",
+                           output)
+
+    self.PrintFilesFromSet(passes & self._expectations.GetWontFixFailures(),
+                           "Expected to fail (ignored), but passed",
+                           output)
+    self.PrintFilesFromSet(passes & self._expectations.GetWontFixTimeouts(),
+                           "Expected to timeout (ignored), but passed",
+                           output)
+    # Crashes should never be deferred.
+    self.PrintFilesFromSet(passes & self._expectations.GetDeferredFailures(),
+                           "Expected to fail (deferred), but passed",
+                           output)
+    self.PrintFilesFromSet(passes & self._expectations.GetDeferredTimeouts(),
+                           "Expected to timeout (deferred), but passed",
+                           output)
+
+    self.PrintFilesFromSet(self._regressed_text_failures,
+                           "text failures",
+                           output,
+                           'TEXT')
+    self.PrintFilesFromSet(self._regressed_image_failures,
+                           "image failures",
+                           output,
+                           'IMAGE')
+    self.PrintFilesFromSet(self._regressed_image_plus_text_failures,
+                           "image + text failures",
+                           output,
+                           'IMAGE+TEXT')
+    self.PrintFilesFromSet(self._regressed_hangs,
+                           "timeouts",
+                           output,
+                           'TIMEOUT')
+    self.PrintFilesFromSet(self._regressed_crashes,
+                           "crashes",
+                           output,
+                           'CRASH')
+    self.PrintFilesFromSet(self._missing,
+                           "missing expected results",
+                           output,
+                           'MISSING')
+
+  def _CalculateRegressions(self):
+    """Calculate regressions from this run through the layout tests."""
+    worklist = self._test_files.copy()
+
+    passes = set()
+    crashes = set()
+    hangs = set()
+    missing = set()
+    image_failures = set()
+    text_failures  = set()
+    image_plus_text_failures = set()
+
+    for test, failure_type_instances in self._test_failures.iteritems():
+      # Although each test can have multiple test_failures, we only put them
+      # into one list (either the crash list or the failure list).  We give
+      # priority to a crash/timeout over others, and to missing results over
+      # a text mismatch.
+      failure_types = [type(f) for f in failure_type_instances]
+      is_crash = [True for f in failure_types if f in self.CRASH_TYPES]
+      is_hang = [True for f in failure_types if f in self.HANG_TYPES]
+      is_missing = [True for f in failure_types if f in self.MISSING_TYPES]
+      is_failure = [True for f in failure_types if f in self.FAILURE_TYPES]
+      is_text_failure = [True for f in failure_types if f in
+         self.TEXT_FAILURE_TYPES]
+      is_image_failure = [True for f in failure_types if f in
+         self.IMAGE_FAILURE_TYPES]
+
+      expectations = self._expectations.GetExpectations(test)
+      if is_crash:
+        if not test_expectations.CRASH in expectations: crashes.add(test)
+      elif is_hang:
+        if not test_expectations.TIMEOUT in expectations: hangs.add(test)
+      # Do not add to the missing list if a test is rebaselining and missing
+      # expected files.
+      elif (is_missing and
+            not self._expectations.IsRebaselining(test) and
+            not self._expectations.IsIgnored(test) and
+            (test_expectations.MISSING not in
+             self._expectations.GetExpectations(test))):
+        missing.add(test)
+      elif is_image_failure and is_text_failure:
+        if (not test_expectations.FAIL in expectations and
+            not test_expectations.IMAGE_PLUS_TEXT in expectations):
+            image_plus_text_failures.add(test)
+      elif is_image_failure:
+        if (not test_expectations.FAIL in expectations and
+            not test_expectations.IMAGE in expectations):
+            image_failures.add(test)
+      elif is_text_failure:
+        if (not test_expectations.FAIL in expectations and
+            not test_expectations.TEXT in expectations):
+            text_failures.add(test)
+      elif is_failure:
+        raise ValueError('unexpected failure type:' + f)
+      worklist.remove(test)
+
+    # When processing flaky failures, the list of actual failures is excluded,
+    # so don't look for unexpected passes.
+    if not self._is_flaky:
+      for test in worklist:
+        # Check that all passing tests are expected to pass.
+        expectations = self._expectations.GetExpectations(test)
+        if not test_expectations.PASS in expectations:
+          passes.add(test)
+
+    self._regressed_passes = passes
+    self._regressed_crashes = crashes
+    self._regressed_hangs = hangs
+    self._missing = missing
+    self._regressed_image_failures = image_failures
+    self._regressed_text_failures = text_failures
+    self._regressed_image_plus_text_failures = image_plus_text_failures
+
+  def GetRegressions(self):
+    """Returns a set of regressions from the test expectations. This is
+    used to determine which tests to list in results.html and the
+    right script exit code for the build bots. The list does not
+    include the unexpected passes."""
+    return (self._regressed_text_failures | self._regressed_image_failures |
+            self._regressed_image_plus_text_failures | self._regressed_hangs |
+            self._regressed_crashes | self._missing)
+
diff --git a/webkit/tools/layout_tests/layout_package/compare_failures_unittest.py b/webkit/tools/layout_tests/layout_package/compare_failures_unittest.py
new file mode 100644
index 0000000..202b195f7
--- /dev/null
+++ b/webkit/tools/layout_tests/layout_package/compare_failures_unittest.py
@@ -0,0 +1,281 @@
+#!/usr/bin/env python
+# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unittests to make sure we generate and update the expected-*.txt files
+properly after running layout tests."""
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import compare_failures
+import path_utils
+import test_failures
+
+class CompareFailuresUnittest(unittest.TestCase):
+  def setUp(self):
+    """Makes a temporary results directory and puts expected-failures.txt and
+    expected-crashes.txt into it."""
+    self._tempdir = tempfile.mkdtemp()
+    # copy over expected-*.txt files
+    testdatadir = self.GetTestDataDir()
+    filenames = ("expected-passing.txt", "expected-failures.txt",
+                 "expected-crashes.txt")
+    for filename in filenames:
+      # copyfile doesn't copy file permissions so we can delete the files later
+      shutil.copyfile(os.path.join(testdatadir, filename),
+                      os.path.join(self._tempdir, filename))
+
+  def tearDown(self):
+    """Remove temp directory."""
+    shutil.rmtree(self._tempdir)
+    self._tempdir = None
+
+  ###########################################################################
+  # Tests
+  def testGenerateNewBaseline(self):
+    """Test the generation of new expected-*.txt files when either they don't
+    exist or the user explicitly asks to make new files."""
+    failures = self.GetTestFailures()
+
+    # Test to make sure we generate baseline files if the file doesn't exist.
+    os.remove(os.path.join(self.GetTmpDir(), 'expected-passing.txt'))
+    os.remove(os.path.join(self.GetTmpDir(), 'expected-failures.txt'))
+    os.remove(os.path.join(self.GetTmpDir(), 'expected-crashes.txt'))
+
+    # Test force generation of new baseline files with a new failure and one
+    # less passing.
+    pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html')
+    failures[pass_file] = [test_failures.FailureTextMismatch(None)]
+
+    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
+                                          set(), set(),
+                                          self.GetTmpDir(), False)
+    cf.UpdateFailuresOnDisk()
+    self.CheckOutputWithExpectedFiles('expected-passing-new-baseline.txt',
+                                      'expected-failures-added.txt',
+                                      'expected-crashes.txt')
+
+  def testPassingToFailure(self):
+    """When there's a new failure, we don't add it to the baseline."""
+    failures = self.GetTestFailures()
+
+    # Test case where we don't update new baseline.  We have a new failure,
+    # but it shouldn't be added to the expected-failures.txt file.
+    pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html')
+    failures[pass_file] = [test_failures.FailureTextMismatch(None)]
+    self.CheckNoChanges(failures)
+
+    # Same thing as before: pass -> crash
+    failures[pass_file] = [test_failures.FailureCrash()]
+    self.CheckNoChanges(failures)
+
+  def testFailureToCrash(self):
+    """When there's a new crash, we don't add it to the baseline or remove it
+    from the failure list."""
+    failures = self.GetTestFailures()
+
+    # Test case where we don't update new baseline.  A failure moving to a
+    # crash shouldn't be added to the expected-crashes.txt file.
+    failure_file = os.path.join(path_utils.LayoutTestsDir(),
+                                'fast', 'foo', 'fail1.html')
+    failures[failure_file] = [test_failures.FailureCrash()]
+    self.CheckNoChanges(failures)
+
+  def testFailureToPassing(self):
+    """This is better than before, so we should update the failure list."""
+    failures = self.GetTestFailures()
+
+    # Remove one of the failing test cases from the failures dictionary.  This
+    # makes failure_file considered to be passing.
+    failure_file = os.path.join(path_utils.LayoutTestsDir(),
+                                'fast', 'bar', 'fail2.html')
+    del failures[failure_file]
+
+    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
+                                          set(), set(),
+                                          self.GetTmpDir(), False)
+    cf.UpdateFailuresOnDisk()
+    self.CheckOutputWithExpectedFiles('expected-passing-new-passing2.txt',
+                                      'expected-failures-new-passing.txt',
+                                      'expected-crashes.txt')
+
+  def testCrashToPassing(self):
+    """This is better than before, so we update the crashes file."""
+    failures = self.GetTestFailures()
+
+    crash_file = os.path.join(path_utils.LayoutTestsDir(),
+                              'fast', 'bar', 'betz', 'crash3.html')
+    del failures[crash_file]
+    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
+                                          set(), set(),
+                                          self.GetTmpDir(), False)
+    cf.UpdateFailuresOnDisk()
+    self.CheckOutputWithExpectedFiles('expected-passing-new-passing.txt',
+                                      'expected-failures.txt',
+                                      'expected-crashes-new-passing.txt')
+
+  def testCrashToFailure(self):
+    """This is better than before, so we should update both lists."""
+    failures = self.GetTestFailures()
+
+    crash_file = os.path.join(path_utils.LayoutTestsDir(),
+                              'fast', 'bar', 'betz', 'crash3.html')
+    failures[crash_file] = [test_failures.FailureTextMismatch(None)]
+    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
+                                          set(), set(),
+                                          self.GetTmpDir(), False)
+    cf.UpdateFailuresOnDisk()
+    self.CheckOutputWithExpectedFiles('expected-passing.txt',
+                                      'expected-failures-new-crash.txt',
+                                      'expected-crashes-new-passing.txt')
+
+  def testNewTestPass(self):
+    """After a merge, we need to update new passing tests properly."""
+    files = self.GetTestFiles()
+    new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html")
+    files.add(new_test_file)
+    failures = self.GetTestFailures()
+
+    # New test file passing
+    cf = compare_failures.CompareFailures(files, failures, set(), set(),
+                                          self.GetTmpDir(), False)
+    cf.UpdateFailuresOnDisk()
+    self.CheckOutputWithExpectedFiles('expected-passing-new-test.txt',
+                                      'expected-failures.txt',
+                                      'expected-crashes.txt')
+
+  def testNewTestFail(self):
+    """After a merge, we need to update new failing tests properly."""
+    files = self.GetTestFiles()
+    new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html")
+    files.add(new_test_file)
+    failures = self.GetTestFailures()
+
+    # New test file failing
+    failures[new_test_file] = [test_failures.FailureTextMismatch(None)]
+    cf = compare_failures.CompareFailures(files, failures, set(), set(),
+                                          self.GetTmpDir(), False)
+    cf.UpdateFailuresOnDisk()
+    self.CheckOutputWithExpectedFiles('expected-passing.txt',
+                                      'expected-failures-new-test.txt',
+                                      'expected-crashes.txt')
+
+  def testNewTestCrash(self):
+    """After a merge, we need to update new crashing tests properly."""
+    files = self.GetTestFiles()
+    new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html")
+    files.add(new_test_file)
+    failures = self.GetTestFailures()
+
+    # New test file crashing
+    failures[new_test_file] = [test_failures.FailureCrash()]
+    cf = compare_failures.CompareFailures(files, failures, set(), set(),
+                                          self.GetTmpDir(), False)
+    cf.UpdateFailuresOnDisk()
+    self.CheckOutputWithExpectedFiles('expected-passing.txt',
+                                      'expected-failures.txt',
+                                      'expected-crashes-new-test.txt')
+
+  def testHasNewFailures(self):
+    files = self.GetTestFiles()
+    failures = self.GetTestFailures()
+
+    # no changes, no new failures
+    cf = compare_failures.CompareFailures(files, failures, set(), set(),
+                                          self.GetTmpDir(), False)
+    self.failUnless(not cf.HasNewFailures())
+
+    # test goes from passing to failing
+    pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html')
+    failures[pass_file] = [test_failures.FailureTextMismatch(None)]
+    cf = compare_failures.CompareFailures(files, failures, set(), set(),
+                                          self.GetTmpDir(), False)
+    self.failUnless(cf.HasNewFailures())
+
+    # Failing to passing
+    failures = self.GetTestFailures()
+    failure_file = os.path.join(path_utils.LayoutTestsDir(),
+                                'fast', 'bar', 'fail2.html')
+    del failures[failure_file]
+    cf = compare_failures.CompareFailures(files, failures, set(), set(),
+                                          self.GetTmpDir(), False)
+    self.failUnless(not cf.HasNewFailures())
+
+    # A new test that fails, this doesn't count as a new failure.
+    new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html")
+    files.add(new_test_file)
+    failures = self.GetTestFailures()
+    failures[new_test_file] = [test_failures.FailureCrash()]
+    cf = compare_failures.CompareFailures(files, failures, set(), set(),
+                                          self.GetTmpDir(), False)
+    self.failUnless(not cf.HasNewFailures())
+
+
+  ###########################################################################
+  # Helper methods
+  def CheckOutputEqualsExpectedFile(self, output, expected):
+    """Compares a file in our output dir against a file from the testdata
+    directory."""
+    output = os.path.join(self.GetTmpDir(), output)
+    expected = os.path.join(self.GetTestDataDir(), expected)
+    self.failUnlessEqual(open(output).read(), open(expected).read())
+
+  def CheckOutputWithExpectedFiles(self, passing, failing, crashing):
+    """Compare all three output files against three provided expected
+    files."""
+    self.CheckOutputEqualsExpectedFile('expected-passing.txt', passing)
+    self.CheckOutputEqualsExpectedFile('expected-failures.txt', failing)
+    self.CheckOutputEqualsExpectedFile('expected-crashes.txt', crashing)
+
+  def CheckNoChanges(self, failures):
+    """Verify that none of the expected-*.txt files have changed."""
+    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
+                                          set(), set(),
+                                          self.GetTmpDir(), False)
+    cf.UpdateFailuresOnDisk()
+    self.CheckOutputWithExpectedFiles('expected-passing.txt',
+                                      'expected-failures.txt',
+                                      'expected-crashes.txt')
+
+  def GetTestDataDir(self):
+    return os.path.abspath('testdata')
+
+  def GetTmpDir(self):
+    return self._tempdir
+
+  def GetTestFiles(self):
+    """Get a set of files that includes the expected crashes and failures
+    along with two passing tests."""
+    layout_dir = path_utils.LayoutTestsDir()
+    files = [
+      'fast\\pass1.html',
+      'fast\\foo\\pass2.html',
+      'fast\\foo\\crash1.html',
+      'fast\\bar\\crash2.html',
+      'fast\\bar\\betz\\crash3.html',
+      'fast\\foo\\fail1.html',
+      'fast\\bar\\fail2.html',
+      'fast\\bar\\betz\\fail3.html',
+    ]
+
+    return set([os.path.join(layout_dir, f) for f in files])
+
+  def GetTestFailures(self):
+    """Get a dictionary representing the crashes and failures in the
+    expected-*.txt files."""
+    failures = {}
+    for filename in self.GetTestFiles():
+      if filename.find('crash') != -1:
+        failures[filename] = [test_failures.FailureCrash()]
+      elif filename.find('fail') != -1:
+        failures[filename] = [test_failures.FailureTextMismatch(None)]
+
+    return failures
+
+if '__main__' == __name__:
+  unittest.main()
+
diff --git a/webkit/tools/layout_tests/layout_package/json_results_generator.py b/webkit/tools/layout_tests/layout_package/json_results_generator.py
index b2bf2ae5..e88957f 100644
--- a/webkit/tools/layout_tests/layout_package/json_results_generator.py
+++ b/webkit/tools/layout_tests/layout_package/json_results_generator.py
@@ -12,7 +12,6 @@ import urllib2
 import xml.dom.minidom
 
 from layout_package import path_utils
-from layout_package import test_expectations
 from layout_package import test_failures
 
 sys.path.append(path_utils.PathFromBase('third_party'))
@@ -42,44 +41,41 @@ class JSONResultsGenerator:
   WONTFIX = "wontfixCounts"
   DEFERRED = "deferredCounts"
   FIXABLE = "fixableCounts"
+  ALL_FIXABLE_COUNT = "allFixableCount"
   FIXABLE_COUNT = "fixableCount"
-
-  # Note that we omit test_expectations.FAIL from this list because
-  # it should never show up (it's a legacy input expectation, never
-  # an output expectation).
-  FAILURE_TO_CHAR = {
-    test_expectations.CRASH : "C",
-    test_expectations.TIMEOUT : "T",
-    test_expectations.IMAGE : "I",
-    test_expectations.TEXT : "F",
-    test_expectations.MISSING : "O",
-    test_expectations.IMAGE_PLUS_TEXT : "Z"
-  }
-  FAILURE_CHARS = FAILURE_TO_CHAR.values()
-
+  """
+    C = CRASH
+    T = TIMEOUT
+    I = IMAGE
+    F = TEXT
+    O = OTHER
+    Z = TEXT+IMAGE
+  """
+  FAILURE_CHARS = ("C", "T", "I", "F", "O", "Z")
   BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
   RESULTS_FILENAME = "results.json"
 
-  def __init__(self, options, result_summary, individual_test_timings,
-      results_file_base_path, all_tests):
+  def __init__(self, options, failures, individual_test_timings,
+      results_file_base_path, all_tests, result_summary):
     """Modifies the results.json file. Grabs it off the archive directory if it
     is not found locally.
 
     Args
       options: a dictionary of command line options
-      result_summary: ResultsSummary object containing failure counts for
-          different groups of tests.
+      failures: Map of test name to list of failures.
       individual_test_times: Map of test name to a tuple containing the
           test_run-time.
       results_file_base_path: Absolute path to the directory containing the
           results json file.
       all_tests: List of all the tests that were run.
+      result_summary: ResultsSummary object containing failure counts for
+          different groups of tests.
     """
     # Make sure all test paths are relative to the layout test root directory.
     self._failures = {}
-    for test in result_summary.failures:
+    for test in failures:
       test_path = self._GetPathRelativeToLayoutTestRoot(test)
-      self._failures[test_path] = result_summary.failures[test]
+      self._failures[test_path] = failures[test]
 
     self._all_tests = [self._GetPathRelativeToLayoutTestRoot(test)
         for test in all_tests]
@@ -265,46 +261,43 @@ class JSONResultsGenerator:
           builder.
     """
     self._InsertItemIntoRawList(results_for_builder,
-        len(set(self._result_summary.failures.keys()) &
-            self._result_summary.tests_by_timeline[test_expectations.NOW]),
+        self._result_summary.fixable_count,
         self.FIXABLE_COUNT)
     self._InsertItemIntoRawList(results_for_builder,
-        self._GetFailureSummaryEntry(test_expectations.DEFER),
+        self._result_summary.all_fixable_count,
+        self.ALL_FIXABLE_COUNT)
+
+    self._InsertItemIntoRawList(results_for_builder,
+        self._GetFailureSummaryEntry(self._result_summary.deferred),
         self.DEFERRED)
     self._InsertItemIntoRawList(results_for_builder,
-        self._GetFailureSummaryEntry(test_expectations.WONTFIX),
+        self._GetFailureSummaryEntry(self._result_summary.wontfix),
         self.WONTFIX)
     self._InsertItemIntoRawList(results_for_builder,
-        self._GetFailureSummaryEntry(test_expectations.NOW),
+        self._GetFailureSummaryEntry(self._result_summary.fixable),
         self.FIXABLE)
 
-  def _GetFailureSummaryEntry(self, timeline):
+  def _GetFailureSummaryEntry(self, result_summary_entry):
     """Creates a summary object to insert into the JSON.
 
     Args:
-      summary   ResultSummary object with test results
-      timeline  current test_expectations timeline to build entry for (e.g.,
-                test_expectations.NOW, etc.)
+      result_summary_entry: ResultSummaryEntry for a group of tests
+          (e.g. deferred tests).
     """
     entry = {}
-    summary = self._result_summary
-    timeline_tests = summary.tests_by_timeline[timeline]
-    entry[self.SKIP_RESULT] = len(
-        summary.tests_by_expectation[test_expectations.SKIP] & timeline_tests)
-    entry[self.SKIP_RESULT] = len(
-        summary.tests_by_expectation[test_expectations.PASS] & timeline_tests)
-    for failure_type in summary.tests_by_expectation.keys():
-      if failure_type not in self.FAILURE_TO_CHAR:
-        continue
-      count = len(summary.tests_by_expectation[failure_type] & timeline_tests)
-      entry[self.FAILURE_TO_CHAR[failure_type]] = count
+    entry[self.SKIP_RESULT] = result_summary_entry.skip_count
+    entry[self.PASS_RESULT] = result_summary_entry.pass_count
+    for char in self.FAILURE_CHARS:
+      # There can be multiple failures that map to "O", so keep existing entry
+      # values if they already exist.
+      count = entry.get(char, 0)
+
+      for failure in result_summary_entry.failure_counts:
+        if char == self._GetResultsCharForFailure([failure]):
+          count = result_summary_entry.failure_counts[failure]
+      entry[char] = count
     return entry
 
-  def _GetResultsCharForTest(self, test):
-    """Returns a single character description of the test result."""
-    result = test_failures.DetermineResultType(self._failures[test])
-    return self.FAILURE_TO_CHAR[result]
-
   def _InsertItemIntoRawList(self, results_for_builder, item, key):
     """Inserts the item into the list with the given key in the results for
     this builder. Creates the list if no such list exists.
@@ -362,6 +355,32 @@ class JSONResultsGenerator:
     results_for_builder[self.TESTS] = {}
     return results_for_builder
 
+  def _GetResultsCharForTest(self, test):
+    """Returns the worst failure from the list of failures for this test
+    since we can only show one failure per run for each test on the dashboard.
+    """
+    failures = [failure.__class__ for failure in self._failures[test]]
+    return self._GetResultsCharForFailure(failures)
+
+  def _GetResultsCharForFailure(self, failures):
+    """Returns the worst failure from the list of failures
+    since we can only show one failure per run for each test on the dashboard.
+    """
+    has_text_failure = test_failures.FailureTextMismatch in failures
+
+    if test_failures.FailureCrash in failures:
+      return "C"
+    elif test_failures.FailureTimeout in failures:
+      return "T"
+    elif test_failures.FailureImageHashMismatch in failures:
+      if has_text_failure:
+        return "Z"
+      return "I"
+    elif has_text_failure:
+      return "F"
+    else:
+      return "O"
+
   def _RemoveItemsOverMaxNumberOfBuilds(self, encoded_list):
     """Removes items from the run-length encoded list after the final item that
     exceeds the max number of builds to track.
diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py
index 1578a57..ca78647 100644
--- a/webkit/tools/layout_tests/layout_package/test_expectations.py
+++ b/webkit/tools/layout_tests/layout_package/test_expectations.py
@@ -12,13 +12,14 @@ import re
 import sys
 import time
 import path_utils
+import compare_failures
 
 sys.path.append(path_utils.PathFromBase('third_party'))
 import simplejson
 
 # Test expectation and modifier constants.
 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX,
- DEFER, SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(16)
+ DEFER, SLOW, REBASELINE, MISSING, NONE) = range(14)
 
 # Test expectation file update action constants
 (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4)
@@ -40,6 +41,85 @@ class TestExpectations:
   def GetExpectationsJsonForAllPlatforms(self):
     return self._expected_failures.GetExpectationsJsonForAllPlatforms()
 
+  def GetFixable(self):
+    return self._expected_failures.GetTestSet(NONE)
+
+  def GetFixableFailures(self):
+    # Avoid including TIMEOUT CRASH FAIL tests in the fail numbers since
+    # crashes and timeouts are higher priority.
+    return ((self._expected_failures.GetTestSet(NONE, FAIL) |
+             self._expected_failures.GetTestSet(NONE, IMAGE) |
+             self._expected_failures.GetTestSet(NONE, TEXT) |
+             self._expected_failures.GetTestSet(NONE, IMAGE_PLUS_TEXT)) -
+            self._expected_failures.GetTestSet(NONE, TIMEOUT) -
+            self._expected_failures.GetTestSet(NONE, CRASH))
+
+  def GetFixableTimeouts(self):
+    # Avoid including TIMEOUT CRASH tests in the timeout numbers since crashes
+    # are higher priority.
+    return (self._expected_failures.GetTestSet(NONE, TIMEOUT) -
+            self._expected_failures.GetTestSet(NONE, CRASH))
+
+  def GetFixableCrashes(self):
+    return self._expected_failures.GetTestSet(NONE, CRASH)
+
+  def GetFixableSkipped(self):
+    return (self._expected_failures.GetTestSet(SKIP) -
+            self._expected_failures.GetTestSet(WONTFIX) -
+            self._expected_failures.GetTestSet(DEFER))
+
+  def GetDeferred(self):
+    return self._expected_failures.GetTestSet(DEFER, include_skips=False)
+
+  def GetDeferredSkipped(self):
+    return (self._expected_failures.GetTestSet(SKIP) &
+            self._expected_failures.GetTestSet(DEFER))
+
+  def GetDeferredFailures(self):
+    return (
+        self._expected_failures.GetTestSet(DEFER, FAIL, include_skips=False) |
+        self._expected_failures.GetTestSet(DEFER, IMAGE, include_skips=False) |
+        self._expected_failures.GetTestSet(DEFER, TEXT, include_skips=False) |
+        self._expected_failures.GetTestSet(DEFER, IMAGE_PLUS_TEXT,
+                                           include_skips=False))
+
+
+  def GetDeferredTimeouts(self):
+    return self._expected_failures.GetTestSet(DEFER, TIMEOUT,
+        include_skips=False)
+
+  def GetWontFix(self):
+    return self._expected_failures.GetTestSet(WONTFIX, include_skips=False)
+
+  def GetWontFixSkipped(self):
+    return (self._expected_failures.GetTestSet(WONTFIX) &
+            self._expected_failures.GetTestSet(SKIP))
+
+  def GetWontFixFailures(self):
+    # Avoid including TIMEOUT CRASH FAIL tests in the fail numbers since
+    # crashes and timeouts are higher priority.
+    return (
+        (self._expected_failures.GetTestSet(WONTFIX, FAIL,
+                                            include_skips=False) |
+         self._expected_failures.GetTestSet(WONTFIX, IMAGE,
+                                            include_skips=False) |
+         self._expected_failures.GetTestSet(WONTFIX, TEXT,
+                                            include_skips=False) |
+         self._expected_failures.GetTestSet(WONTFIX, IMAGE_PLUS_TEXT,
+                                            include_skips=False)) -
+        self._expected_failures.GetTestSet(WONTFIX, TIMEOUT,
+                                           include_skips=False) -
+        self._expected_failures.GetTestSet(WONTFIX, CRASH,
+                                           include_skips=False))
+
+  def GetWontFixTimeouts(self):
+    # Avoid including TIMEOUT CRASH tests in the timeout numbers since crashes
+    # are higher priority.
+    return (self._expected_failures.GetTestSet(WONTFIX, TIMEOUT,
+                include_skips=False) -
+            self._expected_failures.GetTestSet(WONTFIX, CRASH,
+                include_skips=False))
+
   def GetRebaseliningFailures(self):
     return (self._expected_failures.GetTestSet(REBASELINE, FAIL) |
             self._expected_failures.GetTestSet(REBASELINE, IMAGE) |
@@ -47,7 +127,12 @@ class TestExpectations:
             self._expected_failures.GetTestSet(REBASELINE, IMAGE_PLUS_TEXT))
 
   def GetExpectations(self, test):
-    return self._expected_failures.GetExpectations(test)
+    if self._expected_failures.Contains(test):
+      return self._expected_failures.GetExpectations(test)
+
+    # If the test file is not listed in any of the expectations lists
+    # we expect it to pass (and nothing else).
+    return set([PASS])
 
   def GetExpectationsString(self, test):
     """Returns the expectatons for the given test as an uppercase string.
@@ -63,22 +148,19 @@ class TestExpectations:
 
     return " ".join(retval).upper()
 
-  def GetTimelineForTest(self, test):
-    return self._expected_failures.GetTimelineForTest(test)
+  def GetModifiers(self, test):
+    if self._expected_failures.Contains(test):
+      return self._expected_failures.GetModifiers(test)
+    return []
 
-  def GetTestsWithResultType(self, result_type):
-    return self._expected_failures.GetTestsWithResultType(result_type)
+  def IsDeferred(self, test):
+    return self._expected_failures.HasModifier(test, DEFER)
 
-  def GetTestsWithTimeline(self, timeline):
-    return self._expected_failures.GetTestsWithTimeline(timeline)
+  def IsFixable(self, test):
+    return self._expected_failures.HasModifier(test, NONE)
 
-  def MatchesAnExpectedResult(self, test, result):
-    """Returns whether we got one of the expected results for this test."""
-    return (result in self._expected_failures.GetExpectations(test) or
-            (result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and
-            FAIL in self._expected_failures.GetExpectations(test)) or
-            result == MISSING and self.IsRebaselining(test) or
-            result == SKIP and self._expected_failures.HasModifier(test, SKIP))
+  def IsIgnored(self, test):
+    return self._expected_failures.HasModifier(test, WONTFIX)
 
   def IsRebaselining(self, test):
     return self._expected_failures.HasModifier(test, REBASELINE)
@@ -169,27 +251,10 @@ class TestExpectationsFile:
                    'crash': CRASH,
                    'missing': MISSING }
 
-  EXPECTATION_DESCRIPTIONS = { SKIP : ('Skipped', 'Skipped'),
-                               PASS : ('passes', 'passes'),
-                               FAIL : ('failure', 'failures'),
-                               TEXT : ('text diff mismatch',
-                                       'text diff mismatch'),
-                               IMAGE : ('image mismatch', 'image mismatch'),
-                               IMAGE_PLUS_TEXT : ('image and text mismatch',
-                                                  'image and text mismatch'),
-                               CRASH : ('test shell crash',
-                                        'test shell crashes'),
-                               TIMEOUT : ('test timed out', 'tests timed out'),
-                               MISSING : ('No expected result found',
-                                          'No expected results found') }
-
-  EXPECTATION_ORDER = ( PASS, CRASH, TIMEOUT, MISSING, IMAGE_PLUS_TEXT,
-     TEXT, IMAGE, FAIL, SKIP )
-
-  BASE_PLATFORMS = ( 'linux', 'mac', 'win' )
-  PLATFORMS = BASE_PLATFORMS + ( 'win-xp', 'win-vista', 'win-7' )
+  BASE_PLATFORMS = [ 'linux', 'mac', 'win' ]
+  PLATFORMS = BASE_PLATFORMS + [ 'win-xp', 'win-vista', 'win-7' ]
 
-  BUILD_TYPES = ( 'debug', 'release' )
+  BUILD_TYPES = [ 'debug', 'release' ]
 
   MODIFIERS = { 'skip': SKIP,
                 'wontfix': WONTFIX,
@@ -198,16 +263,6 @@ class TestExpectationsFile:
                 'rebaseline': REBASELINE,
                 'none': NONE }
 
-  TIMELINES = { 'wontfix': WONTFIX,
-                'now': NOW,
-                'defer': DEFER }
-
-  RESULT_TYPES = { 'skip': SKIP,
-                   'pass': PASS,
-                   'fail': FAIL,
-                   'flaky': FLAKY }
-
-
   def __init__(self, path, full_test_list, platform, is_debug_mode,
       is_lint_mode, expectations_as_str=None, suppress_errors=False):
     """
@@ -243,30 +298,23 @@ class TestExpectationsFile:
     # Maps a test to its list of expectations.
     self._test_to_expectations = {}
 
-    # Maps a test to its list of options (string values)
-    self._test_to_options = {}
-
-    # Maps a test to its list of modifiers: the constants associated with
-    # the options minus any bug or platform strings
+    # Maps a test to its list of modifiers.
     self._test_to_modifiers = {}
 
     # Maps a test to the base path that it was listed with in the test list.
     self._test_list_paths = {}
 
-    self._modifier_to_tests = self._DictOfSets(self.MODIFIERS)
-    self._expectation_to_tests = self._DictOfSets(self.EXPECTATIONS)
-    self._timeline_to_tests = self._DictOfSets(self.TIMELINES)
-    self._result_type_to_tests = self._DictOfSets(self.RESULT_TYPES)
+    # Maps a modifier to a set of tests.
+    self._modifier_to_tests = {}
+    for modifier in self.MODIFIERS.itervalues():
+      self._modifier_to_tests[modifier] = set()
 
-    self._Read(self._GetIterableExpectations())
+    # Maps an expectation to a set of tests.
+    self._expectation_to_tests = {}
+    for expectation in self.EXPECTATIONS.itervalues():
+      self._expectation_to_tests[expectation] = set()
 
-  def _DictOfSets(self, strings_to_constants):
-    """Takes a dict of strings->constants and returns a dict mapping
-    each constant to an empty set."""
-    d = {}
-    for c in strings_to_constants.values():
-      d[c] = set()
-    return d
+    self._Read(self._GetIterableExpectations())
 
   def _GetIterableExpectations(self):
     """Returns an object that can be iterated over. Allows for not caring about
@@ -304,15 +352,12 @@ class TestExpectationsFile:
 
     return tests
 
-  def GetTestsWithResultType(self, result_type):
-    return self._result_type_to_tests[result_type]
-
-  def GetTestsWithTimeline(self, timeline):
-    return self._timeline_to_tests[timeline]
-
   def HasModifier(self, test, modifier):
     return test in self._modifier_to_tests[modifier]
 
+  def GetModifiers(self, test):
+    return self._test_to_modifiers[test]
+
   def GetExpectations(self, test):
     return self._test_to_expectations[test]
 
@@ -405,7 +450,9 @@ class TestExpectationsFile:
 
   def ParseExpectationsLine(self, line):
     """Parses a line from test_expectations.txt and returns a tuple with the
-    test path, options as a list, expectations as a list."""
+    test path, options as a list, expectations as a list, options as a string
+    and expectations as a string.
+    """
     line = StripComments(line)
     if not line:
       return (None, None, None)
@@ -577,7 +624,7 @@ class TestExpectationsFile:
         tests = self._ExpandTests(test_list_path)
 
       self._AddTests(tests, expectations, test_list_path, lineno,
-                     modifiers, options)
+          modifiers, options)
 
     if not self._suppress_errors and (
         len(self._errors) or len(self._non_fatal_errors)):
@@ -593,14 +640,6 @@ class TestExpectationsFile:
       if len(self._errors):
         raise SyntaxError('\n'.join(map(str, self._errors)))
 
-    # Now add in the tests that weren't present in the expectations file
-    expectations = set([PASS])
-    options = []
-    modifiers = []
-    for test in self._full_test_list:
-      if not test in self._test_list_paths:
-        self._AddTest(test, modifiers, expectations, options)
-
   def _GetOptionsList(self, listString):
     return [part.strip().lower() for part in listString.strip().split(' ')]
 
@@ -632,77 +671,42 @@ class TestExpectationsFile:
     return result
 
   def _AddTests(self, tests, expectations, test_list_path, lineno, modifiers,
-                options):
+      options):
     for test in tests:
       if self._AlreadySeenTest(test, test_list_path, lineno):
         continue
 
       self._ClearExpectationsForTest(test, test_list_path)
-      self._AddTest(test, modifiers, expectations, options)
+      self._test_to_expectations[test] = expectations
+      self._test_to_modifiers[test] = options
 
-  def _AddTest(self, test, modifiers, expectations, options):
-    """Sets the expected state for a given test.
-
-    This routine assumes the test has not been added before. If it has,
-    use _ClearExpectationsForTest() to reset the state prior to calling this.
+      if len(modifiers) is 0:
+        self._AddTest(test, NONE, expectations)
+      else:
+        for modifier in modifiers:
+          self._AddTest(test, self.MODIFIERS[modifier], expectations)
 
-    Args:
-      test: test to add
-      modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.)
-      expectations: sequence of expectations (PASS, IMAGE, etc.)
-      options: sequence of keywords and bug identifiers."""
-    self._test_to_expectations[test] = expectations
+  def _AddTest(self, test, modifier, expectations):
+    self._modifier_to_tests[modifier].add(test)
     for expectation in expectations:
       self._expectation_to_tests[expectation].add(test)
 
-    self._test_to_options[test] = options
-    self._test_to_modifiers[test] = set()
-    for modifier in modifiers:
-      mod_value = self.MODIFIERS[modifier]
-      self._modifier_to_tests[mod_value].add(test)
-      self._test_to_modifiers[test].add(mod_value)
-
-    if 'wontfix' in modifiers:
-      self._timeline_to_tests[WONTFIX].add(test)
-    elif 'defer' in modifiers:
-      self._timeline_to_tests[DEFER].add(test)
-    else:
-      self._timeline_to_tests[NOW].add(test)
-
-    if 'skip' in modifiers:
-      self._result_type_to_tests[SKIP].add(test)
-    elif expectations == set([PASS]):
-      self._result_type_to_tests[PASS].add(test)
-    elif len(expectations) > 1:
-      self._result_type_to_tests[FLAKY].add(test)
-    else:
-      self._result_type_to_tests[FAIL].add(test)
-
-
   def _ClearExpectationsForTest(self, test, test_list_path):
-    """Remove prexisting expectations for this test.
+    """Remove prexisiting expectations for this test.
     This happens if we are seeing a more precise path
     than a previous listing.
     """
     if test in self._test_list_paths:
       self._test_to_expectations.pop(test, '')
-      self._RemoveFromSets(test, self._expectation_to_tests)
-      self._RemoveFromSets(test, self._modifier_to_tests)
-      self._RemoveFromSets(test, self._timeline_to_tests)
-      self._RemoveFromSets(test, self._result_type_to_tests)
+      for expectation in self.EXPECTATIONS.itervalues():
+        if test in self._expectation_to_tests[expectation]:
+          self._expectation_to_tests[expectation].remove(test)
+      for modifier in self.MODIFIERS.itervalues():
+        if test in self._modifier_to_tests[modifier]:
+          self._modifier_to_tests[modifier].remove(test)
 
     self._test_list_paths[test] = os.path.normpath(test_list_path)
 
-  def _RemoveFromSets(self, test, dict):
-    """Removes the given test from the sets in the dictionary.
-
-    Args:
-      test: test to look for
-      dict: dict of sets of files"""
-    for set_of_tests in dict.itervalues():
-      if test in set_of_tests:
-        set_of_tests.remove(test)
-
   def _AlreadySeenTest(self, test, test_list_path, lineno):
     """Returns true if we've already seen a more precise path for this test
     than the test_list_path.
diff --git a/webkit/tools/layout_tests/layout_package/test_failures.py b/webkit/tools/layout_tests/layout_package/test_failures.py
index e590492..6f1d807 100644
--- a/webkit/tools/layout_tests/layout_package/test_failures.py
+++ b/webkit/tools/layout_tests/layout_package/test_failures.py
@@ -5,39 +5,29 @@
 """Classes for failures that occur during tests."""
 
 import os
-import test_expectations
-
-def DetermineResultType(failure_list):
-  """Takes a set of test_failures and returns which result type best fits
-  the list of failures. "Best fits" means we use the worst type of failure.
-
-  Returns:
-    one of the test_expectations result types - PASS, TEXT, CRASH, etc."""
-
-  if not failure_list or len(failure_list) == 0:
-    return test_expectations.PASS
-
-  failure_types = [type(f) for f in failure_list]
-  if FailureCrash in failure_types:
-    return test_expectations.CRASH
-  elif FailureTimeout in failure_types:
-    return test_expectations.TIMEOUT
-  elif (FailureMissingResult in failure_types or
-        FailureMissingImage in failure_types or
-        FailureMissingImageHash in failure_types):
-    return test_expectations.MISSING
-  else:
-    is_text_failure = FailureTextMismatch in failure_types
-    is_image_failure = (FailureImageHashIncorrect in failure_types or
-                        FailureImageHashMismatch in failure_types)
-    if is_text_failure and is_image_failure:
-      return test_expectations.IMAGE_PLUS_TEXT
-    elif is_text_failure:
-      return test_expectations.TEXT
-    elif is_image_failure:
-      return test_expectations.IMAGE
-    else:
-      raise ValueError("unclassifiable set of failures: " + str(failure_types))
+
+class FailureSort(object):
+  """A repository for failure sort orders and tool to facilitate sorting."""
+
+  # Each failure class should have an entry in this dictionary. Sort order 1
+  # will be sorted first in the list. Failures with the same numeric sort
+  # order will be sorted alphabetically by Message().
+  SORT_ORDERS = {
+    'FailureTextMismatch': 1,
+    'FailureImageHashMismatch': 2,
+    'FailureTimeout': 3,
+    'FailureCrash': 4,
+    'FailureMissingImageHash': 5,
+    'FailureMissingImage': 6,
+    'FailureMissingResult': 7,
+    'FailureImageHashIncorrect': 8,
+  }
+
+  @staticmethod
+  def SortOrder(failure_type):
+    """Returns a tuple of the class's numeric sort order and its message."""
+    order = FailureSort.SORT_ORDERS.get(failure_type.__name__, -1)
+    return (order, failure_type.Message())
 
 
 class TestFailure(object):
diff --git a/webkit/tools/layout_tests/layout_package/test_shell_thread.py b/webkit/tools/layout_tests/layout_package/test_shell_thread.py
index a8c5fa9..3c7e52f 100644
--- a/webkit/tools/layout_tests/layout_package/test_shell_thread.py
+++ b/webkit/tools/layout_tests/layout_package/test_shell_thread.py
@@ -185,15 +185,13 @@ class SingleTestThread(threading.Thread):
 
 class TestShellThread(threading.Thread):
 
-  def __init__(self, filename_list_queue, result_queue, test_shell_command,
-               test_types, test_args, shell_args, options):
+  def __init__(self, filename_list_queue, test_shell_command, test_types,
+               test_args, shell_args, options):
     """Initialize all the local state for this test shell thread.
 
     Args:
       filename_list_queue: A thread safe Queue class that contains lists of
           tuples of (filename, uri) pairs.
-      result_queue: A thread safe Queue class that will contain tuples of
-        (test, failure lists) for the test results.
       test_shell_command: A list specifying the command+args for test_shell
       test_types: A list of TestType objects to run the test output against.
       test_args: A TestArguments object to pass to each TestType.
@@ -204,7 +202,6 @@ class TestShellThread(threading.Thread):
     """
     threading.Thread.__init__(self)
     self._filename_list_queue = filename_list_queue
-    self._result_queue = result_queue
     self._filename_list = []
     self._test_shell_command = test_shell_command
     self._test_types = test_types
@@ -212,6 +209,7 @@ class TestShellThread(threading.Thread):
     self._test_shell_proc = None
     self._shell_args = shell_args
     self._options = options
+    self._failures = {}
     self._canceled = False
     self._exception_info = None
     self._directory_timing_stats = {}
@@ -227,6 +225,11 @@ class TestShellThread(threading.Thread):
     # Time at which we started running tests from self._current_dir.
     self._current_dir_start_time = None
 
+  def GetFailures(self):
+    """Returns a dictionary mapping test filename to a list of
+    TestFailures."""
+    return self._failures
+
   def GetDirectoryTimingStats(self):
     """Returns a dictionary mapping test directory to a tuple of
     (number of tests in that directory, time to run the tests)"""
@@ -337,9 +340,10 @@ class TestShellThread(threading.Thread):
         error_str = '\n'.join(['  ' + f.Message() for f in failures])
         logging.error("%s failed:\n%s" %
                       (path_utils.RelativeTestFilename(filename), error_str))
+        # Group the errors for reporting.
+        self._failures[filename] = failures
       else:
         logging.debug(path_utils.RelativeTestFilename(filename) + " passed")
-      self._result_queue.put((filename, failures))
 
       if batch_size > 0 and batch_count > batch_size:
         # Bounce the shell and reset count.
diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py
index b9244de..33ec3bd 100755
--- a/webkit/tools/layout_tests/run_webkit_tests.py
+++ b/webkit/tools/layout_tests/run_webkit_tests.py
@@ -36,6 +36,7 @@ import time
 import traceback
 
 from layout_package import apache_http_server
+from layout_package import compare_failures
 from layout_package import test_expectations
 from layout_package import http_server
 from layout_package import json_results_generator
@@ -49,8 +50,6 @@ from test_types import image_diff
 from test_types import test_type_base
 from test_types import text_diff
 
-TestExpectationsFile = test_expectations.TestExpectationsFile
-
 class TestInfo:
   """Groups information about a test for easy passing of data."""
   def __init__(self, filename, timeout):
@@ -71,40 +70,43 @@ class TestInfo:
       self.image_hash = None
 
 
-class ResultSummary(object):
-  """A class for partitioning the test results we get into buckets.
-
-  This class is basically a glorified struct and it's private to this file
-  so we don't bother with any information hiding."""
-  def __init__(self, expectations, test_files):
-    self.total = len(test_files)
-    self.expectations = expectations
-    self.tests_by_expectation = {}
-    self.tests_by_timeline = {}
-    self.results = {}
-    self.unexpected_results = {}
-    self.failures = {}
-    self.tests_by_expectation[test_expectations.SKIP] = set()
-    for expectation in TestExpectationsFile.EXPECTATIONS.values():
-      self.tests_by_expectation[expectation] = set()
-    for timeline in TestExpectationsFile.TIMELINES.values():
-      self.tests_by_timeline[timeline] = expectations.GetTestsWithTimeline(
-          timeline)
-
-  def Add(self, test, failures, result):
-    """Add a result into the appropriate bin.
+class ResultSummaryEntry:
+  def __init__(self, all, failed, failure_counts, skipped):
+    """Resolves result counts.
 
     Args:
-      test: test file name
-      failures: list of failure objects from test execution
-      result: result of test (PASS, IMAGE, etc.)."""
+      all: list of all tests in this category
+      failed: list of failing tests in this category
+      skipped: list of skipped tests
+      failure_counts: dictionary of (TestFailure -> frequency)
+    """
+    self.skip_count = len(skipped)
+    self.total_count = len(all | skipped)
+    self.pass_count = self.total_count - self.skip_count - len(failed)
+    self.failure_counts = failure_counts;
+
 
-    self.tests_by_expectation[result].add(test)
-    self.results[test] = result
-    if len(failures):
-      self.failures[test] = failures
-    if not self.expectations.MatchesAnExpectedResult(test, result):
-      self.unexpected_results[test] = result
+class ResultSummary:
+  """Container for all result summaries for this test run.
+
+  Args:
+    deferred: ResultSummary object for deferred tests.
+    wontfix: ResultSummary object for wontfix tests.
+    fixable: ResultSummary object for tests marked in test_expectations.txt as
+        needing fixing but are not deferred/wontfix (i.e. should be fixed
+        for the next release).
+    fixable_count: Count of all fixable and skipped tests. This is essentially
+        a deduped sum of all the failures that are not deferred/wontfix.
+    all_fixable_count: Count of all the tests that are not deferred/wontfix.
+        This includes passing tests.
+  """
+  def __init__(self, deferred, wontfix, fixable, fixable_count,
+      all_fixable_count):
+    self.deferred = deferred
+    self.wontfix = wontfix
+    self.fixable = fixable
+    self.fixable_count = fixable_count
+    self.all_fixable_count = all_fixable_count
 
 
 class TestRunner:
@@ -157,7 +159,6 @@ class TestRunner:
     self._test_files = set()
     self._test_files_list = None
     self._file_dir = path_utils.GetAbsolutePath(os.path.dirname(sys.argv[0]))
-    self._result_queue = Queue.Queue()
 
   def __del__(self):
     logging.info("flushing stdout")
@@ -174,7 +175,7 @@ class TestRunner:
   def GatherFilePaths(self, paths):
     """Find all the files to test.
 
-    Args:
+    args:
       paths: a list of globs to use instead of the defaults."""
     self._test_files = test_files.GatherTestFiles(paths)
     logging.info('Found: %d tests' % len(self._test_files))
@@ -201,19 +202,21 @@ class TestRunner:
         raise err
 
   def PrepareListsAndPrintOutput(self):
-    """Create appropriate subsets of test lists and returns a ResultSummary
-    object. Also prints expected test counts."""
-
-    result_summary = ResultSummary(self._expectations, self._test_files)
+    """Create appropriate subsets of test lists and print test counts.
 
+    Create appropriate subsets of self._tests_files in
+    self._ignored_failures, self._fixable_failures, and self._fixable_crashes.
+    Also remove skipped files from self._test_files, extract a subset of tests
+    if desired, and create the sorted self._test_files_list.
+    """
     # Remove skipped - both fixable and ignored - files from the
     # top-level list of files to test.
     skipped = set()
+    # If there was only one test file, we'll run it even if it was skipped.
     if len(self._test_files) > 1 and not self._options.force:
-      skipped = self._expectations.GetTestsWithResultType(
-          test_expectations.SKIP)
-      for test in skipped:
-        result_summary.Add(test, [], test_expectations.SKIP)
+      skipped = (self._expectations.GetFixableSkipped() |
+                 self._expectations.GetDeferredSkipped() |
+                 self._expectations.GetWontFixSkipped())
       self._test_files -= skipped
 
     if self._options.force:
@@ -305,15 +308,23 @@ class TestRunner:
     else:
       logging.info('Run: %d tests' % len(self._test_files))
 
-    self._PrintExpectedResultsOfType(result_summary, test_expectations.PASS,
-        "passes")
-    self._PrintExpectedResultsOfType(result_summary, test_expectations.FAIL,
-        "failures")
-    self._PrintExpectedResultsOfType(result_summary, test_expectations.FLAKY,
-        "flaky")
-    self._PrintExpectedResultsOfType(result_summary, test_expectations.SKIP,
-        "skipped")
-    return result_summary
+    logging.info('Deferred: %d tests' % len(self._expectations.GetDeferred()))
+    logging.info('Expected passes: %d tests' %
+                 len(self._test_files -
+                     self._expectations.GetFixable() -
+                     self._expectations.GetWontFix()))
+    logging.info(('Expected failures: %d fixable, %d ignored '
+                  'and %d deferred tests') %
+                 (len(self._expectations.GetFixableFailures()),
+                  len(self._expectations.GetWontFixFailures()),
+                  len(self._expectations.GetDeferredFailures())))
+    logging.info(('Expected timeouts: %d fixable, %d ignored '
+                  'and %d deferred tests') %
+                 (len(self._expectations.GetFixableTimeouts()),
+                  len(self._expectations.GetWontFixTimeouts()),
+                  len(self._expectations.GetDeferredTimeouts())))
+    logging.info('Expected crashes: %d fixable tests' %
+                 len(self._expectations.GetFixableCrashes()))
 
   def AddTestType(self, test_type):
     """Add a TestType to the TestRunner."""
@@ -373,7 +384,6 @@ class TestRunner:
     Return:
       The Queue of lists of TestInfo objects.
     """
-
     if self._options.experimental_fully_parallel:
       filename_queue = Queue.Queue()
       for test_file in test_files:
@@ -474,7 +484,6 @@ class TestRunner:
 
       test_args, shell_args = self._GetTestShellArgs(i)
       thread = test_shell_thread.TestShellThread(filename_queue,
-                                                 self._result_queue,
                                                  test_shell_command,
                                                  test_types,
                                                  test_args,
@@ -493,7 +502,7 @@ class TestRunner:
       proc.stdin.close()
       proc.wait()
 
-  def _RunTests(self, test_shell_binary, file_list, result_summary):
+  def _RunTests(self, test_shell_binary, file_list):
     """Runs the tests in the file_list.
 
     Return: A tuple (failures, thread_timings, test_timings,
@@ -505,7 +514,6 @@ class TestRunner:
           form [time, directory_name, num_tests]
         individual_test_timings is a list of run times for each test in the form
           {filename:filename, test_run_time:test_run_time}
-        result_summary: summary object to populate with the results
     """
     threads = self._InstantiateTestShellThreads(test_shell_binary, file_list)
 
@@ -522,7 +530,7 @@ class TestRunner:
           # suffices to not use an indefinite blocking join for it to
           # be interruptible by KeyboardInterrupt.
           thread.join(1.0)
-          self._UpdateSummary(result_summary)
+        failures.update(thread.GetFailures())
         thread_timings.append({ 'name': thread.getName(),
                                 'num_tests': thread.GetNumTests(),
                                 'total_time': thread.GetTotalTime()});
@@ -542,19 +550,14 @@ class TestRunner:
         # would be assumed to have passed.
         raise exception_info[0], exception_info[1], exception_info[2]
 
-    # Make sure we pick up any remaining tests.
-    self._UpdateSummary(result_summary)
-    return (thread_timings, test_timings, individual_test_timings)
+    return (failures, thread_timings, test_timings, individual_test_timings)
 
-  def Run(self, result_summary):
+  def Run(self):
     """Run all our tests on all our test files.
 
     For each test file, we run each test type.  If there are any failures, we
     collect them for reporting.
 
-    Args:
-      result_summary: a summary object tracking the test results.
-
     Return:
       We return nonzero if there are regressions compared to the last run.
     """
@@ -597,36 +600,25 @@ class TestRunner:
       self._websocket_server.Start()
       # self._websocket_secure_server.Start()
 
-    thread_timings, test_timings, individual_test_timings = (
-        self._RunTests(test_shell_binary, self._test_files_list,
-                       result_summary))
-    original_results_by_test, original_results_by_type = self._CompareResults(
-        result_summary)
-    final_results_by_test = original_results_by_test
-    final_results_by_type = original_results_by_type
-    failed_results_by_test, failed_results_by_type = self._OnlyFailures(
-        original_results_by_test)
-    num_passes = len(original_results_by_test) - len(failed_results_by_test)
+    original_failures, thread_timings, test_timings, individual_test_timings = (
+        self._RunTests(test_shell_binary, self._test_files_list))
 
     retries = 0
-    while (retries < self.NUM_RETRY_ON_UNEXPECTED_FAILURE and
-        len(failed_results_by_test)):
-      logging.info("Retrying %d unexpected failure(s)" %
-                   len(failed_results_by_test))
+    final_failures = original_failures
+    original_regressions = self._CompareFailures(final_failures)
+    regressions = original_regressions
+
+    while retries < self.NUM_RETRY_ON_UNEXPECTED_FAILURE and len(regressions):
+      logging.info("Retrying %d unexpected failure(s)" % len(regressions))
       retries += 1
-      retry_summary = ResultSummary(self._expectations,
-                                    failed_results_by_test.keys())
-      self._RunTests(test_shell_binary, failed_results_by_test.keys(),
-                     retry_summary)
-      final_results_by_test, final_results_by_type = self._CompareResults(
-          retry_summary)
-      failed_results_by_test, failed_results_by_type = self._OnlyFailures(
-          final_results_by_test)
+      final_failures = self._RunTests(test_shell_binary, list(regressions))[0]
+      regressions = self._CompareFailures(final_failures)
 
     self._StopLayoutTestHelper(layout_test_helper_proc)
 
     print
     end_time = time.time()
+
     logging.info("%6.2f total testing time" % (end_time - start_time))
     cuml_time = 0
     logging.debug("Thread timing:")
@@ -641,20 +633,20 @@ class TestRunner:
 
     print
     self._PrintTimingStatistics(test_timings, individual_test_timings,
-                                result_summary)
+        original_failures)
 
-    self._PrintUnexpectedResults(original_results_by_test,
-                                 original_results_by_type,
-                                 final_results_by_test,
-                                 final_results_by_type)
+    self._PrintRegressions(original_failures, original_regressions,
+        final_failures)
 
     # Write summaries to stdout.
-    # The summaries should include flaky tests, so use the original summary,
-    # not the final one.
+    # The summary should include flaky tests, so use original_failures, not
+    # final_failures.
+    result_summary = self._GetResultSummary(original_failures)
     self._PrintResultSummary(result_summary, sys.stdout)
 
     if self._options.verbose:
-      self._WriteJSONFiles(result_summary, individual_test_timings);
+      self._WriteJSONFiles(original_failures, individual_test_timings,
+          result_summary);
 
     # Write the same data to a log file.
     out_filename = os.path.join(self._options.results_directory, "score.txt")
@@ -664,62 +656,40 @@ class TestRunner:
 
     # Write the summary to disk (results.html) and maybe open the test_shell
     # to this file.
-    wrote_results = self._WriteResultsHtmlFile(result_summary)
+    wrote_results = self._WriteResultsHtmlFile(original_failures,
+        original_regressions)
     if not self._options.noshow_results and wrote_results:
       self._ShowResultsHtmlFile()
 
     sys.stdout.flush()
     sys.stderr.flush()
-    # Ignore flaky failures and unexpected passes so we don't turn the
-    # bot red for those.
-    return len(failed_results_by_test)
-
-  def _UpdateSummary(self, result_summary):
-    """Update the summary while running tests."""
-    while True:
-      try:
-        (test, fail_list) = self._result_queue.get_nowait()
-        result = test_failures.DetermineResultType(fail_list)
-        result_summary.Add(test, fail_list, result)
-      except Queue.Empty:
-        return
-
-  def _CompareResults(self, result_summary):
-    """Determine if the results in this test run are unexpected.
+    # Ignore flaky failures so we don't turn the bot red for those.
+    return len(regressions)
 
-    Returns:
-      A dict of files -> results and a dict of results -> sets of files
+  def _PrintRegressions(self, original_failures, original_regressions,
+      final_failures):
+    """Prints the regressions from the test run.
+    Args:
+      original_failures: Failures from the first test run.
+      original_regressions: Regressions from the first test run.
+      final_failures: Failures after retrying the failures from the first run.
     """
-    results_by_test = {}
-    results_by_type = {}
-    for result in TestExpectationsFile.EXPECTATIONS.values():
-      results_by_type[result] = set()
-    for test, result in result_summary.unexpected_results.iteritems():
-      results_by_test[test] = result
-      results_by_type[result].add(test)
-    return results_by_test, results_by_type
-
-  def _OnlyFailures(self, results_by_test):
-    """Filters a dict of results and returns only the failures.
+    print "-" * 78
 
-    Args:
-      results_by_test: a dict of files -> results
+    flaky_failures = {}
+    non_flaky_failures = {}
+    for failure in original_failures:
+      if failure not in original_regressions or failure in final_failures:
+        non_flaky_failures[failure] = original_failures[failure]
+      else:
+        flaky_failures[failure] = original_failures[failure]
 
-    Returns:
-      a dict of files -> results and results -> sets of files.
-    """
-    failed_results_by_test = {}
-    failed_results_by_type = {}
-    for test, result in results_by_test.iteritems():
-      if result != test_expectations.PASS:
-        failed_results_by_test[test] = result
-        if result in failed_results_by_type:
-          failed_results_by_type.add(test)
-        else:
-          failed_results_by_type = set([test])
-    return failed_results_by_test, failed_results_by_type
+    self._CompareFailures(non_flaky_failures, print_regressions=True)
+    self._CompareFailures(flaky_failures, print_regressions=True, is_flaky=True)
 
-  def _WriteJSONFiles(self, result_summary, individual_test_timings):
+    print "-" * 78
+
+  def _WriteJSONFiles(self, failures, individual_test_timings, result_summary):
     logging.debug("Writing JSON files in %s." % self._options.results_directory)
     # Write a json file of the test_expectations.txt file for the layout tests
     # dashboard.
@@ -729,47 +699,16 @@ class TestRunner:
     expectations_file.write(("ADD_EXPECTATIONS(" + expectations_json + ");"))
     expectations_file.close()
 
-    json_results_generator.JSONResultsGenerator(self._options,
-        result_summary, individual_test_timings,
-        self._options.results_directory, self._test_files_list)
+    json_results_generator.JSONResultsGenerator(self._options, failures,
+        individual_test_timings, self._options.results_directory,
+        self._test_files_list, result_summary)
 
     logging.debug("Finished writing JSON files.")
 
-  def _PrintExpectedResultsOfType(self, result_summary, result_type,
-      result_type_str):
-    """Print the number of the tests in a given result class.
-
-    Args:
-      result_summary - the object containg all the results to report on
-      result_type - the particular result type to report in the summary.
-      result_type_str - a string description of the result_type.
-    """
-    tests = self._expectations.GetTestsWithResultType(result_type)
-    now = result_summary.tests_by_timeline[test_expectations.NOW]
-    wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
-    defer = result_summary.tests_by_timeline[test_expectations.DEFER]
-
-    # We use a fancy format string in order to print the data out in a
-    # nicely-aligned table.
-    fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd defer, %%%dd wontfix)" %
-              (self._NumDigits(now), self._NumDigits(defer),
-               self._NumDigits(wontfix)))
-    logging.info(fmtstr %
-        (len(tests), result_type_str, len(tests & now), len(tests & defer),
-         len(tests & wontfix)))
-
-  def _NumDigits(self, num):
-    """Returns the number of digits needed to represent the length of a
-    sequence."""
-    ndigits = 1
-    if len(num):
-      ndigits = int(math.log10(len(num))) + 1
-    return ndigits
-
   def _PrintTimingStatistics(self, directory_test_timings,
-      individual_test_timings, result_summary):
+      individual_test_timings, failures):
     self._PrintAggregateTestStatistics(individual_test_timings)
-    self._PrintIndividualTestTimes(individual_test_timings, result_summary)
+    self._PrintIndividualTestTimes(individual_test_timings, failures)
     self._PrintDirectoryTimings(directory_test_timings)
 
   def _PrintAggregateTestStatistics(self, individual_test_timings):
@@ -803,12 +742,12 @@ class TestRunner:
           "PER TEST TIMES BY TEST TYPE: %s" % test_type,
           times_per_test_type[test_type])
 
-  def _PrintIndividualTestTimes(self, individual_test_timings, result_summary):
+  def _PrintIndividualTestTimes(self, individual_test_timings, failures):
     """Prints the run times for slow, timeout and crash tests.
     Args:
       individual_test_timings: List of test_shell_thread.TestStats for all
           tests.
-      result_summary: Object containing the results of all the tests.
+      failures: Dictionary mapping test filenames to list of test_failures.
     """
     # Reverse-sort by the time spent in test_shell.
     individual_test_timings.sort(lambda a, b:
@@ -825,10 +764,13 @@ class TestRunner:
         is_timeout_crash_or_slow = True
         slow_tests.append(test_tuple)
 
-      if (result_summary.results[filename] == test_expectations.TIMEOUT or
-          result_summary.results[filename] == test_expectations.CRASH):
-        is_timeout_crash_or_slow = True
-        timeout_or_crash_tests.append(test_tuple)
+      if filename in failures:
+        for failure in failures[filename]:
+          if (failure.__class__ == test_failures.FailureTimeout or
+              failure.__class__ == test_failures.FailureCrash):
+            is_timeout_crash_or_slow = True
+            timeout_or_crash_tests.append(test_tuple)
+            break
 
       if (not is_timeout_crash_or_slow and
           num_printed < self._options.num_slow_tests_to_log):
@@ -896,225 +838,181 @@ class TestRunner:
         "99th percentile: %s, Standard deviation: %s\n" % (
         median, mean, percentile90, percentile99, std_deviation)))
 
-  def _PrintResultSummary(self, result_summary, output):
-    """Print a short summary to the output file about how many tests passed.
+  def _GetResultSummary(self, failures):
+    """Returns a ResultSummary object with failure counts.
 
     Args:
-      output: a file or stream to write to
+      failures: dictionary mapping the test filename to a list of
+          TestFailure objects if the test failed
     """
-    failed = len(result_summary.failures)
-    skipped = len(result_summary.tests_by_expectation[test_expectations.SKIP])
-    total = result_summary.total
-    passed = total - failed - skipped
-    pct_passed = 0.0
-    if total > 0:
-      pct_passed = float(passed) * 100 / total
-
-    output.write("\n");
-    output.write("=> Total: %d/%d tests passed (%.1f%%)\n" %
-                 (passed, total, pct_passed))
-
-    output.write("\n");
-    self._PrintResultSummaryEntry(result_summary, test_expectations.NOW,
-       "Tests to be fixed for the current release", output)
-
-    output.write("\n");
-    self._PrintResultSummaryEntry(result_summary, test_expectations.DEFER,
-       "Tests we'll fix in the future if they fail (DEFER)", output)
-
-    output.write("\n");
-    self._PrintResultSummaryEntry(result_summary, test_expectations.WONTFIX,
-       "Tests that won't be fixed if they fail (WONTFIX)", output)
-
-  def _PrintResultSummaryEntry(self, result_summary, timeline, heading, output):
-    """Print a summary block of results for a particular timeline of test.
+    fixable_counts = {}
+    deferred_counts = {}
+    wontfix_counts = {}
+
+    fixable_failures = set()
+    deferred_failures = set()
+    wontfix_failures = set()
+
+    # Aggregate failures in a dictionary (TestFailure -> frequency),
+    # with known (fixable and wontfix) failures separated out.
+    def AddFailure(dictionary, key):
+      if key in dictionary:
+        dictionary[key] += 1
+      else:
+        dictionary[key] = 1
+
+    for test, failure_types in failures.iteritems():
+      for failure_type in failure_types:
+        # TODO(ojan): Now that we have IMAGE+TEXT, IMAGE and TEXT,
+        # we can avoid adding multiple failures per test since there should
+        # be a unique type of failure for each test. This would make the
+        # statistics printed at the end easier to grok.
+        if self._expectations.IsDeferred(test):
+          count_group = deferred_counts
+          failure_group = deferred_failures
+        elif self._expectations.IsIgnored(test):
+          count_group = wontfix_counts
+          failure_group = wontfix_failures
+        else:
+          count_group = fixable_counts
+          failure_group = fixable_failures
+
+        AddFailure(count_group, failure_type.__class__)
+        failure_group.add(test)
+
+    # Here and below, use the prechuncked expectations object for numbers of
+    # skipped tests. Chunking removes the skipped tests before creating the
+    # expectations file.
+    #
+    # This is a bit inaccurate, since the number of skipped tests will be
+    # duplicated across all shard, but it's the best we can reasonably do.
+
+    deduped_fixable_count = len(fixable_failures |
+        self.prechunk_expectations.GetFixableSkipped())
+    all_fixable_count = len(self._test_files -
+        self._expectations.GetWontFix() - self._expectations.GetDeferred())
+
+    # Breakdown of tests we need to fix and want to pass.
+    # Include skipped fixable tests in the statistics.
+    fixable_result_summary = ResultSummaryEntry(
+        self._expectations.GetFixable() | fixable_failures,
+        fixable_failures,
+        fixable_counts,
+        self.prechunk_expectations.GetFixableSkipped())
+
+    deferred_result_summary = ResultSummaryEntry(
+        self._expectations.GetDeferred(),
+        deferred_failures,
+        deferred_counts,
+        self.prechunk_expectations.GetDeferredSkipped())
+
+    wontfix_result_summary = ResultSummaryEntry(
+        self._expectations.GetWontFix(),
+        wontfix_failures,
+        wontfix_counts,
+        self.prechunk_expectations.GetWontFixSkipped())
+
+    return ResultSummary(deferred_result_summary, wontfix_result_summary,
+        fixable_result_summary, deduped_fixable_count, all_fixable_count)
 
-    Args:
-      result_summary: summary to print results for
-      timeline: the timeline to print results for (NOT, WONTFIX, etc.)
-      heading: a textual description of the timeline
-      output: a stream to write to
-    """
-    total = len(result_summary.tests_by_timeline[timeline])
-    not_passing = (total -
-       len(result_summary.tests_by_expectation[test_expectations.PASS] &
-           result_summary.tests_by_timeline[timeline]))
-    output.write("=> %s (%d):\n" % (heading, not_passing))
-    for result in TestExpectationsFile.EXPECTATION_ORDER:
-      if result == test_expectations.PASS:
-        continue
-      results = (result_summary.tests_by_expectation[result] &
-                 result_summary.tests_by_timeline[timeline])
-      desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
-      plural = ["", "s"]
-      if not_passing and len(results):
-        pct = len(results) * 100.0 / not_passing
-        output.write("%d test case%s (%4.1f%%) %s\n" %
-                     (len(results), plural[len(results) != 1], pct,
-                      desc[len(results) != 1]))
-
-  def _PrintUnexpectedResults(self, original_results_by_test,
-                              original_results_by_type,
-                              final_results_by_test,
-                              final_results_by_type):
-    """Print unexpected results (regressions) to stdout and a file.
+  def _PrintResultSummary(self, result_summary, output):
+    """Print a short summary to stdout about how many tests passed.
 
     Args:
-      original_results_by_test: dict mapping tests -> results for the first
-        (original) test run
-      original_results_by_type: dict mapping results -> sets of tests for the
-        first test run
-      final_results_by_test: dict of tests->results after the retries
-        eliminated any flakiness
-      final_results_by_type: dict of results->tests after the retries
-        eliminated any flakiness
+      result_summary: ResultSummary object with failure counts.
+      output: file descriptor to write the results to. For example, sys.stdout.
     """
-    print "-" * 78
+    failed = result_summary.fixable_count
+    total = result_summary.all_fixable_count
+    passed = 0
+    if total > 0:
+      passed = float(total - failed) * 100 / total
+    output.write(
+        "\nTest summary: %.1f%% Passed | %s Failures | %s Tests to pass for "
+        "this release\n" % (
+        passed, failed, total))
+
+    self._PrintResultSummaryEntry(
+        "Tests to be fixed for the current release",
+        result_summary.fixable,
+        output)
+    self._PrintResultSummaryEntry(
+        "Tests to be fixed for a future release (DEFER)",
+        result_summary.deferred,
+        output)
+    self._PrintResultSummaryEntry("Tests never to be fixed (WONTFIX)",
+        result_summary.wontfix,
+        output)
     print
-    flaky_results_by_type = {}
-    non_flaky_results_by_type = {}
-
-    for test, result in original_results_by_test.iteritems():
-      if (result == test_expectations.PASS or
-          (test in final_results_by_test and
-           result == final_results_by_test[test])):
-        if result in non_flaky_results_by_type:
-          non_flaky_results_by_type[result].add(test)
-        else:
-          non_flaky_results_by_type[result] = set([test])
-      else:
-        if result in flaky_results_by_type:
-          flaky_results_by_type[result].add(test)
-        else:
-          flaky_results_by_type[result] = set([test])
 
-    self._PrintUnexpectedResultsByType(non_flaky_results_by_type, False,
-                                       sys.stdout)
-    self._PrintUnexpectedResultsByType(flaky_results_by_type, True, sys.stdout)
-
-    out_filename = os.path.join(self._options.results_directory,
-                                "regressions.txt")
-    output_file = open(out_filename, "w")
-    self._PrintUnexpectedResultsByType(non_flaky_results_by_type, False,
-                                       output_file)
-    output_file.close()
-
-    print "-" * 78
-
-  def _PrintUnexpectedResultsByType(self, results_by_type, is_flaky, output):
-    """Helper method to print a set of unexpected results to an output stream
-    sorted by the result type.
+  def _PrintResultSummaryEntry(self, heading, result_summary, output):
+    """Print a summary block of results for a particular category of test.
 
     Args:
-      results_by_type: dict(result_type -> list of files)
-      is_flaky: where these results flaky or not (changes the output)
-      output: stream to write output to
+      heading: text to print before the block, followed by the total count
+      result_summary: ResultSummaryEntry object with the result counts
+      output: file descriptor to write the results to
     """
-    descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS
-    keywords = {}
-    for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
-      keywords[v] = k.upper()
-
-    for result in TestExpectationsFile.EXPECTATION_ORDER:
-      if result in results_by_type:
-        self._PrintUnexpectedResultSet(output, results_by_type[result],
-                                       is_flaky, descriptions[result][1],
-                                       keywords[result])
-
-  def _PrintUnexpectedResultSet(self, output, filenames, is_flaky,
-                                header_text, keyword):
-    """A helper method to print one set of results (all of the same type)
-    to a stream.
+    output.write("\n=> %s (%d):\n" % (heading, result_summary.total_count))
+    self._PrintResultLine(result_summary.pass_count, result_summary.total_count,
+        "Passed", output)
+    self._PrintResultLine(result_summary.skip_count, result_summary.total_count,
+        "Skipped", output)
+    sorted_keys = sorted(result_summary.failure_counts.keys(),
+                         key=test_failures.FailureSort.SortOrder)
+    for failure in sorted_keys:
+      self._PrintResultLine(result_summary.failure_counts[failure],
+                            result_summary.total_count,
+                            failure.Message(),
+                            output)
+
+  def _PrintResultLine(self, count, total, message, output):
+    if count == 0: return
+    output.write(
+        ("%(count)d test case%(plural)s (%(percent).1f%%) %(message)s\n" %
+            { 'count'   : count,
+              'plural'  : ('s', '')[count == 1],
+              'percent' : float(count) * 100 / total,
+              'message' : message }))
+
+  def _CompareFailures(self, failures, print_regressions=False, is_flaky=False):
+    """Determine if the failures in this test run are unexpected.
 
     Args:
-      output: a stream or file object to write() to
-      filenames: a list of absolute filenames
-      header_text: a string to display before the list of filenames
-      keyword: expectation keyword
-    """
-    filenames = list(filenames)
-    filenames.sort()
-    if not is_flaky and keyword == 'PASS':
-      self._PrintUnexpectedPasses(output, filenames)
-      return
-
-    if is_flaky:
-      prefix = "Unexpected flakiness:"
-    else:
-      prefix = "Regressions: Unexpected"
-    output.write("%s %s (%d):\n" % (prefix, header_text, len(filenames)))
-    for filename in filenames:
-      flaky_text = ""
-      if is_flaky:
-        flaky_text = " " + self._expectations.GetExpectationsString(filename)
-
-      filename = path_utils.RelativeTestFilename(filename)
-      output.write("  %s = %s%s\n" % (filename, keyword, flaky_text))
-    output.write("\n")
-
-  def _PrintUnexpectedPasses(self, output, filenames):
-    """Prints the list of files that passed unexpectedly.
-
-    TODO(dpranke): This routine is a bit of a hack, since it's not clear
-    what the best way to output this is. Each unexpected pass might have
-    multiple expected results, and printing all the combinations would
-    be pretty clunky. For now we sort them into three buckets, crashes,
-    timeouts, and everything else.
+      failures: a dictionary mapping the test filename to a list of
+          TestFailure objects if the test failed
+      print_regressions: whether to print the regressions to stdout
+      is_flaky: whether this set of failures represents tests that failed
+          the first time around, but passed on a subsequent run
 
-    Args:
-      output: a stream to write to
-      filenames: list of files that passed
+    Return:
+      A set of regressions (unexpected failures, hangs, or crashes)
     """
-    crashes = []
-    timeouts = []
-    failures = []
-    for filename in filenames:
-      expectations = self._expectations.GetExpectations(filename)
-      if test_expectations.CRASH in expectations:
-        crashes.append(filename)
-      elif test_expectations.TIMEOUT in expectations:
-        timeouts.append(filename)
-      else:
-        failures.append(filename)
+    cf = compare_failures.CompareFailures(self._test_files,
+                                          failures,
+                                          self._expectations,
+                                          is_flaky)
+    if print_regressions:
+      cf.PrintRegressions(sys.stdout)
 
-    self._PrintPassSet(output, "crash", crashes)
-    self._PrintPassSet(output, "timeout", timeouts)
-    self._PrintPassSet(output, "fail", failures)
+    return cf.GetRegressions()
 
-  def _PrintPassSet(self, output, expected_str, filenames):
-    """Print a set of unexpected passes.
-
-    Args:
-      output: stream to write to
-      expected_str: worst expectation for the given file
-      filenames: list of files in that set
-    """
-    if len(filenames):
-      filenames.sort()
-      output.write("Expected to %s, but passed: (%d)\n" %
-                   (expected_str, len(filenames)))
-      for filename in filenames:
-        filename = path_utils.RelativeTestFilename(filename)
-        output.write("  %s\n" % filename)
-      output.write("\n")
-
-  def _WriteResultsHtmlFile(self, result_summary):
+  def _WriteResultsHtmlFile(self, failures, regressions):
     """Write results.html which is a summary of tests that failed.
 
     Args:
-      result_summary: a summary of the results :)
+      failures: a dictionary mapping the test filename to a list of
+          TestFailure objects if the test failed
+      regressions: a set of test filenames that regressed
 
     Returns:
       True if any results were written (since expected failures may be omitted)
     """
     # test failures
-    failures = result_summary.failures
-    failed_results_by_test, failed_results_by_type = self._OnlyFailures(
-        result_summary.unexpected_results)
     if self._options.full_results_html:
       test_files = failures.keys()
     else:
-      test_files = failed_results_by_test.keys()
+      test_files = list(regressions)
     if not len(test_files):
       return False
 
@@ -1264,15 +1162,15 @@ def main(options, args):
   if options.lint_test_files:
     # Creating the expecations for each platform/target pair does all the
     # test list parsing and ensures it's correct syntax (e.g. no dupes).
-    for platform in TestExpectationsFile.PLATFORMS:
+    for platform in test_expectations.TestExpectationsFile.PLATFORMS:
       test_runner.ParseExpectations(platform, is_debug_mode=True)
       test_runner.ParseExpectations(platform, is_debug_mode=False)
     print ("If there are no fail messages, errors or exceptions, then the "
         "lint succeeded.")
-    sys.exit(0)
+    return
   else:
     test_runner.ParseExpectations(options.platform, options.target == 'Debug')
-    result_summary = test_runner.PrepareListsAndPrintOutput()
+    test_runner.PrepareListsAndPrintOutput()
 
   if options.find_baselines:
     # Record where we found each baseline, then exit.
@@ -1326,7 +1224,7 @@ def main(options, args):
     test_runner.AddTestType(image_diff.ImageDiff)
     if options.fuzzy_pixel_tests:
       test_runner.AddTestType(fuzzy_image_diff.FuzzyImageDiff)
-  has_new_failures = test_runner.Run(result_summary)
+  has_new_failures = test_runner.Run()
   logging.info("Exit status: %d" % has_new_failures)
   sys.exit(has_new_failures)
 
diff --git a/webkit/tools/layout_tests/update_expectations_from_dashboard.py b/webkit/tools/layout_tests/update_expectations_from_dashboard.py
index 5d0b14e..5bfb8cc 100644
--- a/webkit/tools/layout_tests/update_expectations_from_dashboard.py
+++ b/webkit/tools/layout_tests/update_expectations_from_dashboard.py
@@ -368,7 +368,7 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile):
       if not has_non_pass_expectation:
         expectations = ["pass"]
 
-      missing_build_types = list(self.BUILD_TYPES)
+      missing_build_types = self.BUILD_TYPES[:]
       sentinal = None
       for build_type in update.build_info:
         if not sentinal:
@@ -406,7 +406,7 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile):
       platforms: List of lower-case platform names.
     """
     platforms.sort()
-    if platforms == list(self.BASE_PLATFORMS):
+    if platforms == self.BASE_PLATFORMS:
       return ""
     else:
       return " ".join(platforms)
@@ -449,4 +449,4 @@ def main():
   open(path_to_expectations, 'w').write(new_expectations)
 
 if '__main__' == __name__:
-  main()
+  main()
\ No newline at end of file