Clean up the statistics reported by run_webkit_tests to be easier to follow.

The most important changes are that we bin the failures into mutually
exclusive buckets, so that they aren't double-reported like before. As a
result of this, compare_failures became much simpler and I ended up folding
it into run_webkit_tests and test_expectations

In addition, a lot of the data structures for holding different sets of
failures have been revised to be more consistent and extensible. This allows
me to have the output match the LTTF dashboard as well.

  R=ojan@chromium.org,tc@chromium.org,pam@chromium.org
  BUG=none
  TEST=none

Review URL: http://codereview.chromium.org/414066

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@33640 0039d316-1c4b-4281-b951-d872f2087c98

7 files changed, 587 insertions, 991 deletions

diff --git a/webkit/tools/layout_tests/layout_package/compare_failures.py b/webkit/tools/layout_tests/layout_package/compare_failures.py
deleted file mode 100644
index 010a584..0000000
--- a/webkit/tools/layout_tests/layout_package/compare_failures.py
+++ /dev/null
@@ -1,216 +0,0 @@
-# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""A helper class for comparing the failures and crashes from a test run
-against what we expected to happen (as specified in test_expectations.txt)."""
-
-import errno
-import os
-
-import path_utils
-import test_failures
-import test_expectations
-
-class CompareFailures:
-  # A list of which TestFailure classes count as a failure vs a crash.
-  FAILURE_TYPES = (test_failures.FailureTextMismatch,
-                   test_failures.FailureImageHashMismatch)
-  TEXT_FAILURE_TYPES = (test_failures.FailureTextMismatch,)
-  IMAGE_FAILURE_TYPES = (test_failures.FailureImageHashMismatch,)
-  CRASH_TYPES = (test_failures.FailureCrash,)
-  HANG_TYPES = (test_failures.FailureTimeout,)
-  MISSING_TYPES = (test_failures.FailureMissingResult,
-                   test_failures.FailureMissingImage,
-                   test_failures.FailureMissingImageHash)
-
-
-  def __init__(self, test_files, test_failures, expectations, is_flaky):
-    """Calculate the regressions in this test run.
-
-    Args:
-      test_files is a set of the filenames of all the test cases we ran
-      test_failures is a dictionary mapping the test filename to a list of
-          TestFailure objects if the test failed
-      expectations is a TestExpectations object representing the
-          current test status
-      is_flaky is whether this set of failures represents tests that failed
-          the first time around, but passed on a subsequent run
-    """
-    self._test_files = test_files
-    self._test_failures = test_failures
-    self._expectations = expectations
-    self._is_flaky = is_flaky
-    self._CalculateRegressions()
-
-
-  def PrintFilesFromSet(self, filenames, header_text, output,
-      opt_expectations=None):
-    """A helper method to print a list of files to output.
-
-    Args:
-    filenames: a list of absolute filenames
-    header_text: a string to display before the list of filenames
-    output: file descriptor to write the results to.
-    opt_expectations: expectations that failed for this test
-    """
-    if not len(filenames):
-      return
-
-    filenames = list(filenames)
-    filenames.sort()
-
-    # Print real regressions.
-    if opt_expectations:
-      if self._is_flaky:
-        header_text = "Unexpected flakiness: " + header_text
-      else:
-        header_text = "Regressions: Unexpected " + header_text
-
-    output.write("\n%s (%d):\n" % (header_text, len(filenames)))
-    for filename in filenames:
-      output_string = "  %s" % path_utils.RelativeTestFilename(filename)
-      if opt_expectations:
-        if self._is_flaky:
-          opt_expectations += (" " +
-              self._expectations.GetExpectationsString(filename))
-        output_string += " = %s" % opt_expectations
-
-      output.write(output_string + "\n")
-
-  def PrintRegressions(self, output):
-    """Write the regressions computed by _CalculateRegressions() to output. """
-
-    # Print unexpected passes by category.
-    passes = self._regressed_passes
-    self.PrintFilesFromSet(passes & self._expectations.GetFixableFailures(),
-                           "Expected to fail, but passed",
-                           output)
-    self.PrintFilesFromSet(passes & self._expectations.GetFixableTimeouts(),
-                           "Expected to timeout, but passed",
-                           output)
-    self.PrintFilesFromSet(passes & self._expectations.GetFixableCrashes(),
-                           "Expected to crash, but passed",
-                           output)
-
-    self.PrintFilesFromSet(passes & self._expectations.GetWontFixFailures(),
-                           "Expected to fail (ignored), but passed",
-                           output)
-    self.PrintFilesFromSet(passes & self._expectations.GetWontFixTimeouts(),
-                           "Expected to timeout (ignored), but passed",
-                           output)
-    # Crashes should never be deferred.
-    self.PrintFilesFromSet(passes & self._expectations.GetDeferredFailures(),
-                           "Expected to fail (deferred), but passed",
-                           output)
-    self.PrintFilesFromSet(passes & self._expectations.GetDeferredTimeouts(),
-                           "Expected to timeout (deferred), but passed",
-                           output)
-
-    self.PrintFilesFromSet(self._regressed_text_failures,
-                           "text failures",
-                           output,
-                           'TEXT')
-    self.PrintFilesFromSet(self._regressed_image_failures,
-                           "image failures",
-                           output,
-                           'IMAGE')
-    self.PrintFilesFromSet(self._regressed_image_plus_text_failures,
-                           "image + text failures",
-                           output,
-                           'IMAGE+TEXT')
-    self.PrintFilesFromSet(self._regressed_hangs,
-                           "timeouts",
-                           output,
-                           'TIMEOUT')
-    self.PrintFilesFromSet(self._regressed_crashes,
-                           "crashes",
-                           output,
-                           'CRASH')
-    self.PrintFilesFromSet(self._missing,
-                           "missing expected results",
-                           output,
-                           'MISSING')
-
-  def _CalculateRegressions(self):
-    """Calculate regressions from this run through the layout tests."""
-    worklist = self._test_files.copy()
-
-    passes = set()
-    crashes = set()
-    hangs = set()
-    missing = set()
-    image_failures = set()
-    text_failures  = set()
-    image_plus_text_failures = set()
-
-    for test, failure_type_instances in self._test_failures.iteritems():
-      # Although each test can have multiple test_failures, we only put them
-      # into one list (either the crash list or the failure list).  We give
-      # priority to a crash/timeout over others, and to missing results over
-      # a text mismatch.
-      failure_types = [type(f) for f in failure_type_instances]
-      is_crash = [True for f in failure_types if f in self.CRASH_TYPES]
-      is_hang = [True for f in failure_types if f in self.HANG_TYPES]
-      is_missing = [True for f in failure_types if f in self.MISSING_TYPES]
-      is_failure = [True for f in failure_types if f in self.FAILURE_TYPES]
-      is_text_failure = [True for f in failure_types if f in
-         self.TEXT_FAILURE_TYPES]
-      is_image_failure = [True for f in failure_types if f in
-         self.IMAGE_FAILURE_TYPES]
-
-      expectations = self._expectations.GetExpectations(test)
-      if is_crash:
-        if not test_expectations.CRASH in expectations: crashes.add(test)
-      elif is_hang:
-        if not test_expectations.TIMEOUT in expectations: hangs.add(test)
-      # Do not add to the missing list if a test is rebaselining and missing
-      # expected files.
-      elif (is_missing and
-            not self._expectations.IsRebaselining(test) and
-            not self._expectations.IsIgnored(test) and
-            (test_expectations.MISSING not in
-             self._expectations.GetExpectations(test))):
-        missing.add(test)
-      elif is_image_failure and is_text_failure:
-        if (not test_expectations.FAIL in expectations and
-            not test_expectations.IMAGE_PLUS_TEXT in expectations):
-            image_plus_text_failures.add(test)
-      elif is_image_failure:
-        if (not test_expectations.FAIL in expectations and
-            not test_expectations.IMAGE in expectations):
-            image_failures.add(test)
-      elif is_text_failure:
-        if (not test_expectations.FAIL in expectations and
-            not test_expectations.TEXT in expectations):
-            text_failures.add(test)
-      elif is_failure:
-        raise ValueError('unexpected failure type:' + f)
-      worklist.remove(test)
-
-    # When processing flaky failures, the list of actual failures is excluded,
-    # so don't look for unexpected passes.
-    if not self._is_flaky:
-      for test in worklist:
-        # Check that all passing tests are expected to pass.
-        expectations = self._expectations.GetExpectations(test)
-        if not test_expectations.PASS in expectations:
-          passes.add(test)
-
-    self._regressed_passes = passes
-    self._regressed_crashes = crashes
-    self._regressed_hangs = hangs
-    self._missing = missing
-    self._regressed_image_failures = image_failures
-    self._regressed_text_failures = text_failures
-    self._regressed_image_plus_text_failures = image_plus_text_failures
-
-  def GetRegressions(self):
-    """Returns a set of regressions from the test expectations. This is
-    used to determine which tests to list in results.html and the
-    right script exit code for the build bots. The list does not
-    include the unexpected passes."""
-    return (self._regressed_text_failures | self._regressed_image_failures |
-            self._regressed_image_plus_text_failures | self._regressed_hangs |
-            self._regressed_crashes | self._missing)
-
diff --git a/webkit/tools/layout_tests/layout_package/compare_failures_unittest.py b/webkit/tools/layout_tests/layout_package/compare_failures_unittest.py
deleted file mode 100644
index 202b195f7..0000000
--- a/webkit/tools/layout_tests/layout_package/compare_failures_unittest.py
+++ /dev/null
@@ -1,281 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Unittests to make sure we generate and update the expected-*.txt files
-properly after running layout tests."""
-
-import os
-import shutil
-import tempfile
-import unittest
-
-import compare_failures
-import path_utils
-import test_failures
-
-class CompareFailuresUnittest(unittest.TestCase):
-  def setUp(self):
-    """Makes a temporary results directory and puts expected-failures.txt and
-    expected-crashes.txt into it."""
-    self._tempdir = tempfile.mkdtemp()
-    # copy over expected-*.txt files
-    testdatadir = self.GetTestDataDir()
-    filenames = ("expected-passing.txt", "expected-failures.txt",
-                 "expected-crashes.txt")
-    for filename in filenames:
-      # copyfile doesn't copy file permissions so we can delete the files later
-      shutil.copyfile(os.path.join(testdatadir, filename),
-                      os.path.join(self._tempdir, filename))
-
-  def tearDown(self):
-    """Remove temp directory."""
-    shutil.rmtree(self._tempdir)
-    self._tempdir = None
-
-  ###########################################################################
-  # Tests
-  def testGenerateNewBaseline(self):
-    """Test the generation of new expected-*.txt files when either they don't
-    exist or the user explicitly asks to make new files."""
-    failures = self.GetTestFailures()
-
-    # Test to make sure we generate baseline files if the file doesn't exist.
-    os.remove(os.path.join(self.GetTmpDir(), 'expected-passing.txt'))
-    os.remove(os.path.join(self.GetTmpDir(), 'expected-failures.txt'))
-    os.remove(os.path.join(self.GetTmpDir(), 'expected-crashes.txt'))
-
-    # Test force generation of new baseline files with a new failure and one
-    # less passing.
-    pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html')
-    failures[pass_file] = [test_failures.FailureTextMismatch(None)]
-
-    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
-                                          set(), set(),
-                                          self.GetTmpDir(), False)
-    cf.UpdateFailuresOnDisk()
-    self.CheckOutputWithExpectedFiles('expected-passing-new-baseline.txt',
-                                      'expected-failures-added.txt',
-                                      'expected-crashes.txt')
-
-  def testPassingToFailure(self):
-    """When there's a new failure, we don't add it to the baseline."""
-    failures = self.GetTestFailures()
-
-    # Test case where we don't update new baseline.  We have a new failure,
-    # but it shouldn't be added to the expected-failures.txt file.
-    pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html')
-    failures[pass_file] = [test_failures.FailureTextMismatch(None)]
-    self.CheckNoChanges(failures)
-
-    # Same thing as before: pass -> crash
-    failures[pass_file] = [test_failures.FailureCrash()]
-    self.CheckNoChanges(failures)
-
-  def testFailureToCrash(self):
-    """When there's a new crash, we don't add it to the baseline or remove it
-    from the failure list."""
-    failures = self.GetTestFailures()
-
-    # Test case where we don't update new baseline.  A failure moving to a
-    # crash shouldn't be added to the expected-crashes.txt file.
-    failure_file = os.path.join(path_utils.LayoutTestsDir(),
-                                'fast', 'foo', 'fail1.html')
-    failures[failure_file] = [test_failures.FailureCrash()]
-    self.CheckNoChanges(failures)
-
-  def testFailureToPassing(self):
-    """This is better than before, so we should update the failure list."""
-    failures = self.GetTestFailures()
-
-    # Remove one of the failing test cases from the failures dictionary.  This
-    # makes failure_file considered to be passing.
-    failure_file = os.path.join(path_utils.LayoutTestsDir(),
-                                'fast', 'bar', 'fail2.html')
-    del failures[failure_file]
-
-    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
-                                          set(), set(),
-                                          self.GetTmpDir(), False)
-    cf.UpdateFailuresOnDisk()
-    self.CheckOutputWithExpectedFiles('expected-passing-new-passing2.txt',
-                                      'expected-failures-new-passing.txt',
-                                      'expected-crashes.txt')
-
-  def testCrashToPassing(self):
-    """This is better than before, so we update the crashes file."""
-    failures = self.GetTestFailures()
-
-    crash_file = os.path.join(path_utils.LayoutTestsDir(),
-                              'fast', 'bar', 'betz', 'crash3.html')
-    del failures[crash_file]
-    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
-                                          set(), set(),
-                                          self.GetTmpDir(), False)
-    cf.UpdateFailuresOnDisk()
-    self.CheckOutputWithExpectedFiles('expected-passing-new-passing.txt',
-                                      'expected-failures.txt',
-                                      'expected-crashes-new-passing.txt')
-
-  def testCrashToFailure(self):
-    """This is better than before, so we should update both lists."""
-    failures = self.GetTestFailures()
-
-    crash_file = os.path.join(path_utils.LayoutTestsDir(),
-                              'fast', 'bar', 'betz', 'crash3.html')
-    failures[crash_file] = [test_failures.FailureTextMismatch(None)]
-    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
-                                          set(), set(),
-                                          self.GetTmpDir(), False)
-    cf.UpdateFailuresOnDisk()
-    self.CheckOutputWithExpectedFiles('expected-passing.txt',
-                                      'expected-failures-new-crash.txt',
-                                      'expected-crashes-new-passing.txt')
-
-  def testNewTestPass(self):
-    """After a merge, we need to update new passing tests properly."""
-    files = self.GetTestFiles()
-    new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html")
-    files.add(new_test_file)
-    failures = self.GetTestFailures()
-
-    # New test file passing
-    cf = compare_failures.CompareFailures(files, failures, set(), set(),
-                                          self.GetTmpDir(), False)
-    cf.UpdateFailuresOnDisk()
-    self.CheckOutputWithExpectedFiles('expected-passing-new-test.txt',
-                                      'expected-failures.txt',
-                                      'expected-crashes.txt')
-
-  def testNewTestFail(self):
-    """After a merge, we need to update new failing tests properly."""
-    files = self.GetTestFiles()
-    new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html")
-    files.add(new_test_file)
-    failures = self.GetTestFailures()
-
-    # New test file failing
-    failures[new_test_file] = [test_failures.FailureTextMismatch(None)]
-    cf = compare_failures.CompareFailures(files, failures, set(), set(),
-                                          self.GetTmpDir(), False)
-    cf.UpdateFailuresOnDisk()
-    self.CheckOutputWithExpectedFiles('expected-passing.txt',
-                                      'expected-failures-new-test.txt',
-                                      'expected-crashes.txt')
-
-  def testNewTestCrash(self):
-    """After a merge, we need to update new crashing tests properly."""
-    files = self.GetTestFiles()
-    new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html")
-    files.add(new_test_file)
-    failures = self.GetTestFailures()
-
-    # New test file crashing
-    failures[new_test_file] = [test_failures.FailureCrash()]
-    cf = compare_failures.CompareFailures(files, failures, set(), set(),
-                                          self.GetTmpDir(), False)
-    cf.UpdateFailuresOnDisk()
-    self.CheckOutputWithExpectedFiles('expected-passing.txt',
-                                      'expected-failures.txt',
-                                      'expected-crashes-new-test.txt')
-
-  def testHasNewFailures(self):
-    files = self.GetTestFiles()
-    failures = self.GetTestFailures()
-
-    # no changes, no new failures
-    cf = compare_failures.CompareFailures(files, failures, set(), set(),
-                                          self.GetTmpDir(), False)
-    self.failUnless(not cf.HasNewFailures())
-
-    # test goes from passing to failing
-    pass_file = os.path.join(path_utils.LayoutTestsDir(), 'fast', 'pass1.html')
-    failures[pass_file] = [test_failures.FailureTextMismatch(None)]
-    cf = compare_failures.CompareFailures(files, failures, set(), set(),
-                                          self.GetTmpDir(), False)
-    self.failUnless(cf.HasNewFailures())
-
-    # Failing to passing
-    failures = self.GetTestFailures()
-    failure_file = os.path.join(path_utils.LayoutTestsDir(),
-                                'fast', 'bar', 'fail2.html')
-    del failures[failure_file]
-    cf = compare_failures.CompareFailures(files, failures, set(), set(),
-                                          self.GetTmpDir(), False)
-    self.failUnless(not cf.HasNewFailures())
-
-    # A new test that fails, this doesn't count as a new failure.
-    new_test_file = os.path.join(path_utils.LayoutTestsDir(), "new-test.html")
-    files.add(new_test_file)
-    failures = self.GetTestFailures()
-    failures[new_test_file] = [test_failures.FailureCrash()]
-    cf = compare_failures.CompareFailures(files, failures, set(), set(),
-                                          self.GetTmpDir(), False)
-    self.failUnless(not cf.HasNewFailures())
-
-
-  ###########################################################################
-  # Helper methods
-  def CheckOutputEqualsExpectedFile(self, output, expected):
-    """Compares a file in our output dir against a file from the testdata
-    directory."""
-    output = os.path.join(self.GetTmpDir(), output)
-    expected = os.path.join(self.GetTestDataDir(), expected)
-    self.failUnlessEqual(open(output).read(), open(expected).read())
-
-  def CheckOutputWithExpectedFiles(self, passing, failing, crashing):
-    """Compare all three output files against three provided expected
-    files."""
-    self.CheckOutputEqualsExpectedFile('expected-passing.txt', passing)
-    self.CheckOutputEqualsExpectedFile('expected-failures.txt', failing)
-    self.CheckOutputEqualsExpectedFile('expected-crashes.txt', crashing)
-
-  def CheckNoChanges(self, failures):
-    """Verify that none of the expected-*.txt files have changed."""
-    cf = compare_failures.CompareFailures(self.GetTestFiles(), failures,
-                                          set(), set(),
-                                          self.GetTmpDir(), False)
-    cf.UpdateFailuresOnDisk()
-    self.CheckOutputWithExpectedFiles('expected-passing.txt',
-                                      'expected-failures.txt',
-                                      'expected-crashes.txt')
-
-  def GetTestDataDir(self):
-    return os.path.abspath('testdata')
-
-  def GetTmpDir(self):
-    return self._tempdir
-
-  def GetTestFiles(self):
-    """Get a set of files that includes the expected crashes and failures
-    along with two passing tests."""
-    layout_dir = path_utils.LayoutTestsDir()
-    files = [
-      'fast\\pass1.html',
-      'fast\\foo\\pass2.html',
-      'fast\\foo\\crash1.html',
-      'fast\\bar\\crash2.html',
-      'fast\\bar\\betz\\crash3.html',
-      'fast\\foo\\fail1.html',
-      'fast\\bar\\fail2.html',
-      'fast\\bar\\betz\\fail3.html',
-    ]
-
-    return set([os.path.join(layout_dir, f) for f in files])
-
-  def GetTestFailures(self):
-    """Get a dictionary representing the crashes and failures in the
-    expected-*.txt files."""
-    failures = {}
-    for filename in self.GetTestFiles():
-      if filename.find('crash') != -1:
-        failures[filename] = [test_failures.FailureCrash()]
-      elif filename.find('fail') != -1:
-        failures[filename] = [test_failures.FailureTextMismatch(None)]
-
-    return failures
-
-if '__main__' == __name__:
-  unittest.main()
-
diff --git a/webkit/tools/layout_tests/layout_package/json_results_generator.py b/webkit/tools/layout_tests/layout_package/json_results_generator.py
index e88957f..b2bf2ae5 100644
--- a/webkit/tools/layout_tests/layout_package/json_results_generator.py
+++ b/webkit/tools/layout_tests/layout_package/json_results_generator.py
@@ -12,6 +12,7 @@ import urllib2
 import xml.dom.minidom
 
 from layout_package import path_utils
+from layout_package import test_expectations
 from layout_package import test_failures
 
 sys.path.append(path_utils.PathFromBase('third_party'))
@@ -41,41 +42,44 @@ class JSONResultsGenerator:
   WONTFIX = "wontfixCounts"
   DEFERRED = "deferredCounts"
   FIXABLE = "fixableCounts"
-  ALL_FIXABLE_COUNT = "allFixableCount"
   FIXABLE_COUNT = "fixableCount"
-  """
-    C = CRASH
-    T = TIMEOUT
-    I = IMAGE
-    F = TEXT
-    O = OTHER
-    Z = TEXT+IMAGE
-  """
-  FAILURE_CHARS = ("C", "T", "I", "F", "O", "Z")
+
+  # Note that we omit test_expectations.FAIL from this list because
+  # it should never show up (it's a legacy input expectation, never
+  # an output expectation).
+  FAILURE_TO_CHAR = {
+    test_expectations.CRASH : "C",
+    test_expectations.TIMEOUT : "T",
+    test_expectations.IMAGE : "I",
+    test_expectations.TEXT : "F",
+    test_expectations.MISSING : "O",
+    test_expectations.IMAGE_PLUS_TEXT : "Z"
+  }
+  FAILURE_CHARS = FAILURE_TO_CHAR.values()
+
   BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
   RESULTS_FILENAME = "results.json"
 
-  def __init__(self, options, failures, individual_test_timings,
-      results_file_base_path, all_tests, result_summary):
+  def __init__(self, options, result_summary, individual_test_timings,
+      results_file_base_path, all_tests):
     """Modifies the results.json file. Grabs it off the archive directory if it
     is not found locally.
 
     Args
       options: a dictionary of command line options
-      failures: Map of test name to list of failures.
+      result_summary: ResultsSummary object containing failure counts for
+          different groups of tests.
       individual_test_times: Map of test name to a tuple containing the
           test_run-time.
       results_file_base_path: Absolute path to the directory containing the
           results json file.
       all_tests: List of all the tests that were run.
-      result_summary: ResultsSummary object containing failure counts for
-          different groups of tests.
     """
     # Make sure all test paths are relative to the layout test root directory.
     self._failures = {}
-    for test in failures:
+    for test in result_summary.failures:
       test_path = self._GetPathRelativeToLayoutTestRoot(test)
-      self._failures[test_path] = failures[test]
+      self._failures[test_path] = result_summary.failures[test]
 
     self._all_tests = [self._GetPathRelativeToLayoutTestRoot(test)
         for test in all_tests]
@@ -261,43 +265,46 @@ class JSONResultsGenerator:
           builder.
     """
     self._InsertItemIntoRawList(results_for_builder,
-        self._result_summary.fixable_count,
+        len(set(self._result_summary.failures.keys()) &
+            self._result_summary.tests_by_timeline[test_expectations.NOW]),
         self.FIXABLE_COUNT)
     self._InsertItemIntoRawList(results_for_builder,
-        self._result_summary.all_fixable_count,
-        self.ALL_FIXABLE_COUNT)
-
-    self._InsertItemIntoRawList(results_for_builder,
-        self._GetFailureSummaryEntry(self._result_summary.deferred),
+        self._GetFailureSummaryEntry(test_expectations.DEFER),
         self.DEFERRED)
     self._InsertItemIntoRawList(results_for_builder,
-        self._GetFailureSummaryEntry(self._result_summary.wontfix),
+        self._GetFailureSummaryEntry(test_expectations.WONTFIX),
         self.WONTFIX)
     self._InsertItemIntoRawList(results_for_builder,
-        self._GetFailureSummaryEntry(self._result_summary.fixable),
+        self._GetFailureSummaryEntry(test_expectations.NOW),
         self.FIXABLE)
 
-  def _GetFailureSummaryEntry(self, result_summary_entry):
+  def _GetFailureSummaryEntry(self, timeline):
     """Creates a summary object to insert into the JSON.
 
     Args:
-      result_summary_entry: ResultSummaryEntry for a group of tests
-          (e.g. deferred tests).
+      summary   ResultSummary object with test results
+      timeline  current test_expectations timeline to build entry for (e.g.,
+                test_expectations.NOW, etc.)
     """
     entry = {}
-    entry[self.SKIP_RESULT] = result_summary_entry.skip_count
-    entry[self.PASS_RESULT] = result_summary_entry.pass_count
-    for char in self.FAILURE_CHARS:
-      # There can be multiple failures that map to "O", so keep existing entry
-      # values if they already exist.
-      count = entry.get(char, 0)
-
-      for failure in result_summary_entry.failure_counts:
-        if char == self._GetResultsCharForFailure([failure]):
-          count = result_summary_entry.failure_counts[failure]
-      entry[char] = count
+    summary = self._result_summary
+    timeline_tests = summary.tests_by_timeline[timeline]
+    entry[self.SKIP_RESULT] = len(
+        summary.tests_by_expectation[test_expectations.SKIP] & timeline_tests)
+    entry[self.SKIP_RESULT] = len(
+        summary.tests_by_expectation[test_expectations.PASS] & timeline_tests)
+    for failure_type in summary.tests_by_expectation.keys():
+      if failure_type not in self.FAILURE_TO_CHAR:
+        continue
+      count = len(summary.tests_by_expectation[failure_type] & timeline_tests)
+      entry[self.FAILURE_TO_CHAR[failure_type]] = count
     return entry
 
+  def _GetResultsCharForTest(self, test):
+    """Returns a single character description of the test result."""
+    result = test_failures.DetermineResultType(self._failures[test])
+    return self.FAILURE_TO_CHAR[result]
+
   def _InsertItemIntoRawList(self, results_for_builder, item, key):
     """Inserts the item into the list with the given key in the results for
     this builder. Creates the list if no such list exists.
@@ -355,32 +362,6 @@ class JSONResultsGenerator:
     results_for_builder[self.TESTS] = {}
     return results_for_builder
 
-  def _GetResultsCharForTest(self, test):
-    """Returns the worst failure from the list of failures for this test
-    since we can only show one failure per run for each test on the dashboard.
-    """
-    failures = [failure.__class__ for failure in self._failures[test]]
-    return self._GetResultsCharForFailure(failures)
-
-  def _GetResultsCharForFailure(self, failures):
-    """Returns the worst failure from the list of failures
-    since we can only show one failure per run for each test on the dashboard.
-    """
-    has_text_failure = test_failures.FailureTextMismatch in failures
-
-    if test_failures.FailureCrash in failures:
-      return "C"
-    elif test_failures.FailureTimeout in failures:
-      return "T"
-    elif test_failures.FailureImageHashMismatch in failures:
-      if has_text_failure:
-        return "Z"
-      return "I"
-    elif has_text_failure:
-      return "F"
-    else:
-      return "O"
-
   def _RemoveItemsOverMaxNumberOfBuilds(self, encoded_list):
     """Removes items from the run-length encoded list after the final item that
     exceeds the max number of builds to track.
diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py
index ca78647..1578a57 100644
--- a/webkit/tools/layout_tests/layout_package/test_expectations.py
+++ b/webkit/tools/layout_tests/layout_package/test_expectations.py
@@ -12,14 +12,13 @@ import re
 import sys
 import time
 import path_utils
-import compare_failures
 
 sys.path.append(path_utils.PathFromBase('third_party'))
 import simplejson
 
 # Test expectation and modifier constants.
 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX,
- DEFER, SLOW, REBASELINE, MISSING, NONE) = range(14)
+ DEFER, SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(16)
 
 # Test expectation file update action constants
 (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4)
@@ -41,85 +40,6 @@ class TestExpectations:
   def GetExpectationsJsonForAllPlatforms(self):
     return self._expected_failures.GetExpectationsJsonForAllPlatforms()
 
-  def GetFixable(self):
-    return self._expected_failures.GetTestSet(NONE)
-
-  def GetFixableFailures(self):
-    # Avoid including TIMEOUT CRASH FAIL tests in the fail numbers since
-    # crashes and timeouts are higher priority.
-    return ((self._expected_failures.GetTestSet(NONE, FAIL) |
-             self._expected_failures.GetTestSet(NONE, IMAGE) |
-             self._expected_failures.GetTestSet(NONE, TEXT) |
-             self._expected_failures.GetTestSet(NONE, IMAGE_PLUS_TEXT)) -
-            self._expected_failures.GetTestSet(NONE, TIMEOUT) -
-            self._expected_failures.GetTestSet(NONE, CRASH))
-
-  def GetFixableTimeouts(self):
-    # Avoid including TIMEOUT CRASH tests in the timeout numbers since crashes
-    # are higher priority.
-    return (self._expected_failures.GetTestSet(NONE, TIMEOUT) -
-            self._expected_failures.GetTestSet(NONE, CRASH))
-
-  def GetFixableCrashes(self):
-    return self._expected_failures.GetTestSet(NONE, CRASH)
-
-  def GetFixableSkipped(self):
-    return (self._expected_failures.GetTestSet(SKIP) -
-            self._expected_failures.GetTestSet(WONTFIX) -
-            self._expected_failures.GetTestSet(DEFER))
-
-  def GetDeferred(self):
-    return self._expected_failures.GetTestSet(DEFER, include_skips=False)
-
-  def GetDeferredSkipped(self):
-    return (self._expected_failures.GetTestSet(SKIP) &
-            self._expected_failures.GetTestSet(DEFER))
-
-  def GetDeferredFailures(self):
-    return (
-        self._expected_failures.GetTestSet(DEFER, FAIL, include_skips=False) |
-        self._expected_failures.GetTestSet(DEFER, IMAGE, include_skips=False) |
-        self._expected_failures.GetTestSet(DEFER, TEXT, include_skips=False) |
-        self._expected_failures.GetTestSet(DEFER, IMAGE_PLUS_TEXT,
-                                           include_skips=False))
-
-
-  def GetDeferredTimeouts(self):
-    return self._expected_failures.GetTestSet(DEFER, TIMEOUT,
-        include_skips=False)
-
-  def GetWontFix(self):
-    return self._expected_failures.GetTestSet(WONTFIX, include_skips=False)
-
-  def GetWontFixSkipped(self):
-    return (self._expected_failures.GetTestSet(WONTFIX) &
-            self._expected_failures.GetTestSet(SKIP))
-
-  def GetWontFixFailures(self):
-    # Avoid including TIMEOUT CRASH FAIL tests in the fail numbers since
-    # crashes and timeouts are higher priority.
-    return (
-        (self._expected_failures.GetTestSet(WONTFIX, FAIL,
-                                            include_skips=False) |
-         self._expected_failures.GetTestSet(WONTFIX, IMAGE,
-                                            include_skips=False) |
-         self._expected_failures.GetTestSet(WONTFIX, TEXT,
-                                            include_skips=False) |
-         self._expected_failures.GetTestSet(WONTFIX, IMAGE_PLUS_TEXT,
-                                            include_skips=False)) -
-        self._expected_failures.GetTestSet(WONTFIX, TIMEOUT,
-                                           include_skips=False) -
-        self._expected_failures.GetTestSet(WONTFIX, CRASH,
-                                           include_skips=False))
-
-  def GetWontFixTimeouts(self):
-    # Avoid including TIMEOUT CRASH tests in the timeout numbers since crashes
-    # are higher priority.
-    return (self._expected_failures.GetTestSet(WONTFIX, TIMEOUT,
-                include_skips=False) -
-            self._expected_failures.GetTestSet(WONTFIX, CRASH,
-                include_skips=False))
-
   def GetRebaseliningFailures(self):
     return (self._expected_failures.GetTestSet(REBASELINE, FAIL) |
             self._expected_failures.GetTestSet(REBASELINE, IMAGE) |
@@ -127,12 +47,7 @@ class TestExpectations:
             self._expected_failures.GetTestSet(REBASELINE, IMAGE_PLUS_TEXT))
 
   def GetExpectations(self, test):
-    if self._expected_failures.Contains(test):
-      return self._expected_failures.GetExpectations(test)
-
-    # If the test file is not listed in any of the expectations lists
-    # we expect it to pass (and nothing else).
-    return set([PASS])
+    return self._expected_failures.GetExpectations(test)
 
   def GetExpectationsString(self, test):
     """Returns the expectatons for the given test as an uppercase string.
@@ -148,19 +63,22 @@ class TestExpectations:
 
     return " ".join(retval).upper()
 
-  def GetModifiers(self, test):
-    if self._expected_failures.Contains(test):
-      return self._expected_failures.GetModifiers(test)
-    return []
+  def GetTimelineForTest(self, test):
+    return self._expected_failures.GetTimelineForTest(test)
 
-  def IsDeferred(self, test):
-    return self._expected_failures.HasModifier(test, DEFER)
+  def GetTestsWithResultType(self, result_type):
+    return self._expected_failures.GetTestsWithResultType(result_type)
 
-  def IsFixable(self, test):
-    return self._expected_failures.HasModifier(test, NONE)
+  def GetTestsWithTimeline(self, timeline):
+    return self._expected_failures.GetTestsWithTimeline(timeline)
 
-  def IsIgnored(self, test):
-    return self._expected_failures.HasModifier(test, WONTFIX)
+  def MatchesAnExpectedResult(self, test, result):
+    """Returns whether we got one of the expected results for this test."""
+    return (result in self._expected_failures.GetExpectations(test) or
+            (result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and
+            FAIL in self._expected_failures.GetExpectations(test)) or
+            result == MISSING and self.IsRebaselining(test) or
+            result == SKIP and self._expected_failures.HasModifier(test, SKIP))
 
   def IsRebaselining(self, test):
     return self._expected_failures.HasModifier(test, REBASELINE)
@@ -251,10 +169,27 @@ class TestExpectationsFile:
                    'crash': CRASH,
                    'missing': MISSING }
 
-  BASE_PLATFORMS = [ 'linux', 'mac', 'win' ]
-  PLATFORMS = BASE_PLATFORMS + [ 'win-xp', 'win-vista', 'win-7' ]
+  EXPECTATION_DESCRIPTIONS = { SKIP : ('Skipped', 'Skipped'),
+                               PASS : ('passes', 'passes'),
+                               FAIL : ('failure', 'failures'),
+                               TEXT : ('text diff mismatch',
+                                       'text diff mismatch'),
+                               IMAGE : ('image mismatch', 'image mismatch'),
+                               IMAGE_PLUS_TEXT : ('image and text mismatch',
+                                                  'image and text mismatch'),
+                               CRASH : ('test shell crash',
+                                        'test shell crashes'),
+                               TIMEOUT : ('test timed out', 'tests timed out'),
+                               MISSING : ('No expected result found',
+                                          'No expected results found') }
+
+  EXPECTATION_ORDER = ( PASS, CRASH, TIMEOUT, MISSING, IMAGE_PLUS_TEXT,
+     TEXT, IMAGE, FAIL, SKIP )
+
+  BASE_PLATFORMS = ( 'linux', 'mac', 'win' )
+  PLATFORMS = BASE_PLATFORMS + ( 'win-xp', 'win-vista', 'win-7' )
 
-  BUILD_TYPES = [ 'debug', 'release' ]
+  BUILD_TYPES = ( 'debug', 'release' )
 
   MODIFIERS = { 'skip': SKIP,
                 'wontfix': WONTFIX,
@@ -263,6 +198,16 @@ class TestExpectationsFile:
                 'rebaseline': REBASELINE,
                 'none': NONE }
 
+  TIMELINES = { 'wontfix': WONTFIX,
+                'now': NOW,
+                'defer': DEFER }
+
+  RESULT_TYPES = { 'skip': SKIP,
+                   'pass': PASS,
+                   'fail': FAIL,
+                   'flaky': FLAKY }
+
+
   def __init__(self, path, full_test_list, platform, is_debug_mode,
       is_lint_mode, expectations_as_str=None, suppress_errors=False):
     """
@@ -298,24 +243,31 @@ class TestExpectationsFile:
     # Maps a test to its list of expectations.
     self._test_to_expectations = {}
 
-    # Maps a test to its list of modifiers.
+    # Maps a test to its list of options (string values)
+    self._test_to_options = {}
+
+    # Maps a test to its list of modifiers: the constants associated with
+    # the options minus any bug or platform strings
     self._test_to_modifiers = {}
 
     # Maps a test to the base path that it was listed with in the test list.
     self._test_list_paths = {}
 
-    # Maps a modifier to a set of tests.
-    self._modifier_to_tests = {}
-    for modifier in self.MODIFIERS.itervalues():
-      self._modifier_to_tests[modifier] = set()
-
-    # Maps an expectation to a set of tests.
-    self._expectation_to_tests = {}
-    for expectation in self.EXPECTATIONS.itervalues():
-      self._expectation_to_tests[expectation] = set()
+    self._modifier_to_tests = self._DictOfSets(self.MODIFIERS)
+    self._expectation_to_tests = self._DictOfSets(self.EXPECTATIONS)
+    self._timeline_to_tests = self._DictOfSets(self.TIMELINES)
+    self._result_type_to_tests = self._DictOfSets(self.RESULT_TYPES)
 
     self._Read(self._GetIterableExpectations())
 
+  def _DictOfSets(self, strings_to_constants):
+    """Takes a dict of strings->constants and returns a dict mapping
+    each constant to an empty set."""
+    d = {}
+    for c in strings_to_constants.values():
+      d[c] = set()
+    return d
+
   def _GetIterableExpectations(self):
     """Returns an object that can be iterated over. Allows for not caring about
     whether we're iterating over a file or a new-line separated string.
@@ -352,12 +304,15 @@ class TestExpectationsFile:
 
     return tests
 
+  def GetTestsWithResultType(self, result_type):
+    return self._result_type_to_tests[result_type]
+
+  def GetTestsWithTimeline(self, timeline):
+    return self._timeline_to_tests[timeline]
+
   def HasModifier(self, test, modifier):
     return test in self._modifier_to_tests[modifier]
 
-  def GetModifiers(self, test):
-    return self._test_to_modifiers[test]
-
   def GetExpectations(self, test):
     return self._test_to_expectations[test]
 
@@ -450,9 +405,7 @@ class TestExpectationsFile:
 
   def ParseExpectationsLine(self, line):
     """Parses a line from test_expectations.txt and returns a tuple with the
-    test path, options as a list, expectations as a list, options as a string
-    and expectations as a string.
-    """
+    test path, options as a list, expectations as a list."""
     line = StripComments(line)
     if not line:
       return (None, None, None)
@@ -624,7 +577,7 @@ class TestExpectationsFile:
         tests = self._ExpandTests(test_list_path)
 
       self._AddTests(tests, expectations, test_list_path, lineno,
-          modifiers, options)
+                     modifiers, options)
 
     if not self._suppress_errors and (
         len(self._errors) or len(self._non_fatal_errors)):
@@ -640,6 +593,14 @@ class TestExpectationsFile:
       if len(self._errors):
         raise SyntaxError('\n'.join(map(str, self._errors)))
 
+    # Now add in the tests that weren't present in the expectations file
+    expectations = set([PASS])
+    options = []
+    modifiers = []
+    for test in self._full_test_list:
+      if not test in self._test_list_paths:
+        self._AddTest(test, modifiers, expectations, options)
+
   def _GetOptionsList(self, listString):
     return [part.strip().lower() for part in listString.strip().split(' ')]
 
@@ -671,42 +632,77 @@ class TestExpectationsFile:
     return result
 
   def _AddTests(self, tests, expectations, test_list_path, lineno, modifiers,
-      options):
+                options):
     for test in tests:
       if self._AlreadySeenTest(test, test_list_path, lineno):
         continue
 
       self._ClearExpectationsForTest(test, test_list_path)
-      self._test_to_expectations[test] = expectations
-      self._test_to_modifiers[test] = options
+      self._AddTest(test, modifiers, expectations, options)
 
-      if len(modifiers) is 0:
-        self._AddTest(test, NONE, expectations)
-      else:
-        for modifier in modifiers:
-          self._AddTest(test, self.MODIFIERS[modifier], expectations)
+  def _AddTest(self, test, modifiers, expectations, options):
+    """Sets the expected state for a given test.
+
+    This routine assumes the test has not been added before. If it has,
+    use _ClearExpectationsForTest() to reset the state prior to calling this.
 
-  def _AddTest(self, test, modifier, expectations):
-    self._modifier_to_tests[modifier].add(test)
+    Args:
+      test: test to add
+      modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.)
+      expectations: sequence of expectations (PASS, IMAGE, etc.)
+      options: sequence of keywords and bug identifiers."""
+    self._test_to_expectations[test] = expectations
     for expectation in expectations:
       self._expectation_to_tests[expectation].add(test)
 
+    self._test_to_options[test] = options
+    self._test_to_modifiers[test] = set()
+    for modifier in modifiers:
+      mod_value = self.MODIFIERS[modifier]
+      self._modifier_to_tests[mod_value].add(test)
+      self._test_to_modifiers[test].add(mod_value)
+
+    if 'wontfix' in modifiers:
+      self._timeline_to_tests[WONTFIX].add(test)
+    elif 'defer' in modifiers:
+      self._timeline_to_tests[DEFER].add(test)
+    else:
+      self._timeline_to_tests[NOW].add(test)
+
+    if 'skip' in modifiers:
+      self._result_type_to_tests[SKIP].add(test)
+    elif expectations == set([PASS]):
+      self._result_type_to_tests[PASS].add(test)
+    elif len(expectations) > 1:
+      self._result_type_to_tests[FLAKY].add(test)
+    else:
+      self._result_type_to_tests[FAIL].add(test)
+
+
   def _ClearExpectationsForTest(self, test, test_list_path):
-    """Remove prexisiting expectations for this test.
+    """Remove prexisting expectations for this test.
     This happens if we are seeing a more precise path
     than a previous listing.
     """
     if test in self._test_list_paths:
       self._test_to_expectations.pop(test, '')
-      for expectation in self.EXPECTATIONS.itervalues():
-        if test in self._expectation_to_tests[expectation]:
-          self._expectation_to_tests[expectation].remove(test)
-      for modifier in self.MODIFIERS.itervalues():
-        if test in self._modifier_to_tests[modifier]:
-          self._modifier_to_tests[modifier].remove(test)
+      self._RemoveFromSets(test, self._expectation_to_tests)
+      self._RemoveFromSets(test, self._modifier_to_tests)
+      self._RemoveFromSets(test, self._timeline_to_tests)
+      self._RemoveFromSets(test, self._result_type_to_tests)
 
     self._test_list_paths[test] = os.path.normpath(test_list_path)
 
+  def _RemoveFromSets(self, test, dict):
+    """Removes the given test from the sets in the dictionary.
+
+    Args:
+      test: test to look for
+      dict: dict of sets of files"""
+    for set_of_tests in dict.itervalues():
+      if test in set_of_tests:
+        set_of_tests.remove(test)
+
   def _AlreadySeenTest(self, test, test_list_path, lineno):
     """Returns true if we've already seen a more precise path for this test
     than the test_list_path.
diff --git a/webkit/tools/layout_tests/layout_package/test_failures.py b/webkit/tools/layout_tests/layout_package/test_failures.py
index 6f1d807..e590492 100644
--- a/webkit/tools/layout_tests/layout_package/test_failures.py
+++ b/webkit/tools/layout_tests/layout_package/test_failures.py
@@ -5,29 +5,39 @@
 """Classes for failures that occur during tests."""
 
 import os
-
-class FailureSort(object):
-  """A repository for failure sort orders and tool to facilitate sorting."""
-
-  # Each failure class should have an entry in this dictionary. Sort order 1
-  # will be sorted first in the list. Failures with the same numeric sort
-  # order will be sorted alphabetically by Message().
-  SORT_ORDERS = {
-    'FailureTextMismatch': 1,
-    'FailureImageHashMismatch': 2,
-    'FailureTimeout': 3,
-    'FailureCrash': 4,
-    'FailureMissingImageHash': 5,
-    'FailureMissingImage': 6,
-    'FailureMissingResult': 7,
-    'FailureImageHashIncorrect': 8,
-  }
-
-  @staticmethod
-  def SortOrder(failure_type):
-    """Returns a tuple of the class's numeric sort order and its message."""
-    order = FailureSort.SORT_ORDERS.get(failure_type.__name__, -1)
-    return (order, failure_type.Message())
+import test_expectations
+
+def DetermineResultType(failure_list):
+  """Takes a set of test_failures and returns which result type best fits
+  the list of failures. "Best fits" means we use the worst type of failure.
+
+  Returns:
+    one of the test_expectations result types - PASS, TEXT, CRASH, etc."""
+
+  if not failure_list or len(failure_list) == 0:
+    return test_expectations.PASS
+
+  failure_types = [type(f) for f in failure_list]
+  if FailureCrash in failure_types:
+    return test_expectations.CRASH
+  elif FailureTimeout in failure_types:
+    return test_expectations.TIMEOUT
+  elif (FailureMissingResult in failure_types or
+        FailureMissingImage in failure_types or
+        FailureMissingImageHash in failure_types):
+    return test_expectations.MISSING
+  else:
+    is_text_failure = FailureTextMismatch in failure_types
+    is_image_failure = (FailureImageHashIncorrect in failure_types or
+                        FailureImageHashMismatch in failure_types)
+    if is_text_failure and is_image_failure:
+      return test_expectations.IMAGE_PLUS_TEXT
+    elif is_text_failure:
+      return test_expectations.TEXT
+    elif is_image_failure:
+      return test_expectations.IMAGE
+    else:
+      raise ValueError("unclassifiable set of failures: " + str(failure_types))
 
 
 class TestFailure(object):
diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py
index f643688..ddd2822 100755
--- a/webkit/tools/layout_tests/run_webkit_tests.py
+++ b/webkit/tools/layout_tests/run_webkit_tests.py
@@ -36,7 +36,6 @@ import time
 import traceback
 
 from layout_package import apache_http_server
-from layout_package import compare_failures
 from layout_package import test_expectations
 from layout_package import http_server
 from layout_package import json_results_generator
@@ -50,6 +49,8 @@ from test_types import image_diff
 from test_types import test_type_base
 from test_types import text_diff
 
+TestExpectationsFile = test_expectations.TestExpectationsFile
+
 class TestInfo:
   """Groups information about a test for easy passing of data."""
   def __init__(self, filename, timeout):
@@ -70,43 +71,40 @@ class TestInfo:
       self.image_hash = None
 
 
-class ResultSummaryEntry:
-  def __init__(self, all, failed, failure_counts, skipped):
-    """Resolves result counts.
+class ResultSummary(object):
+  """A class for partitioning the test results we get into buckets.
+
+  This class is basically a glorified struct and it's private to this file
+  so we don't bother with any information hiding."""
+  def __init__(self, expectations, test_files):
+    self.total = len(test_files)
+    self.expectations = expectations
+    self.tests_by_expectation = {}
+    self.tests_by_timeline = {}
+    self.results = {}
+    self.unexpected_results = {}
+    self.failures = {}
+    self.tests_by_expectation[test_expectations.SKIP] = set()
+    for expectation in TestExpectationsFile.EXPECTATIONS.values():
+      self.tests_by_expectation[expectation] = set()
+    for timeline in TestExpectationsFile.TIMELINES.values():
+      self.tests_by_timeline[timeline] = expectations.GetTestsWithTimeline(
+          timeline)
+
+  def Add(self, test, failures, result):
+    """Add a result into the appropriate bin.
 
     Args:
-      all: list of all tests in this category
-      failed: list of failing tests in this category
-      skipped: list of skipped tests
-      failure_counts: dictionary of (TestFailure -> frequency)
-    """
-    self.skip_count = len(skipped)
-    self.total_count = len(all | skipped)
-    self.pass_count = self.total_count - self.skip_count - len(failed)
-    self.failure_counts = failure_counts;
-
+      test: test file name
+      failures: list of failure objects from test execution
+      result: result of test (PASS, IMAGE, etc.)."""
 
-class ResultSummary:
-  """Container for all result summaries for this test run.
-
-  Args:
-    deferred: ResultSummary object for deferred tests.
-    wontfix: ResultSummary object for wontfix tests.
-    fixable: ResultSummary object for tests marked in test_expectations.txt as
-        needing fixing but are not deferred/wontfix (i.e. should be fixed
-        for the next release).
-    fixable_count: Count of all fixable and skipped tests. This is essentially
-        a deduped sum of all the failures that are not deferred/wontfix.
-    all_fixable_count: Count of all the tests that are not deferred/wontfix.
-        This includes passing tests.
-  """
-  def __init__(self, deferred, wontfix, fixable, fixable_count,
-      all_fixable_count):
-    self.deferred = deferred
-    self.wontfix = wontfix
-    self.fixable = fixable
-    self.fixable_count = fixable_count
-    self.all_fixable_count = all_fixable_count
+    self.tests_by_expectation[result].add(test)
+    self.results[test] = result
+    if len(failures):
+      self.failures[test] = failures
+    if not self.expectations.MatchesAnExpectedResult(test, result):
+      self.unexpected_results[test] = result
 
 
 class TestRunner:
@@ -166,7 +164,7 @@ class TestRunner:
   def GatherFilePaths(self, paths):
     """Find all the files to test.
 
-    args:
+    Args:
       paths: a list of globs to use instead of the defaults."""
     self._test_files = test_files.GatherTestFiles(paths)
     logging.info('Found: %d tests' % len(self._test_files))
@@ -193,21 +191,19 @@ class TestRunner:
         raise err
 
   def PrepareListsAndPrintOutput(self):
-    """Create appropriate subsets of test lists and print test counts.
+    """Create appropriate subsets of test lists and returns a ResultSummary
+    object. Also prints expected test counts."""
+
+    result_summary = ResultSummary(self._expectations, self._test_files)
 
-    Create appropriate subsets of self._tests_files in
-    self._ignored_failures, self._fixable_failures, and self._fixable_crashes.
-    Also remove skipped files from self._test_files, extract a subset of tests
-    if desired, and create the sorted self._test_files_list.
-    """
     # Remove skipped - both fixable and ignored - files from the
     # top-level list of files to test.
     skipped = set()
-    # If there was only one test file, we'll run it even if it was skipped.
     if len(self._test_files) > 1 and not self._options.force:
-      skipped = (self._expectations.GetFixableSkipped() |
-                 self._expectations.GetDeferredSkipped() |
-                 self._expectations.GetWontFixSkipped())
+      skipped = self._expectations.GetTestsWithResultType(
+          test_expectations.SKIP)
+      for test in skipped:
+        result_summary.Add(test, [], test_expectations.SKIP)
       self._test_files -= skipped
 
     if self._options.force:
@@ -299,23 +295,15 @@ class TestRunner:
     else:
       logging.info('Run: %d tests' % len(self._test_files))
 
-    logging.info('Deferred: %d tests' % len(self._expectations.GetDeferred()))
-    logging.info('Expected passes: %d tests' %
-                 len(self._test_files -
-                     self._expectations.GetFixable() -
-                     self._expectations.GetWontFix()))
-    logging.info(('Expected failures: %d fixable, %d ignored '
-                  'and %d deferred tests') %
-                 (len(self._expectations.GetFixableFailures()),
-                  len(self._expectations.GetWontFixFailures()),
-                  len(self._expectations.GetDeferredFailures())))
-    logging.info(('Expected timeouts: %d fixable, %d ignored '
-                  'and %d deferred tests') %
-                 (len(self._expectations.GetFixableTimeouts()),
-                  len(self._expectations.GetWontFixTimeouts()),
-                  len(self._expectations.GetDeferredTimeouts())))
-    logging.info('Expected crashes: %d fixable tests' %
-                 len(self._expectations.GetFixableCrashes()))
+    self._PrintExpectedResultsOfType(result_summary, test_expectations.PASS,
+        "passes")
+    self._PrintExpectedResultsOfType(result_summary, test_expectations.FAIL,
+        "failures")
+    self._PrintExpectedResultsOfType(result_summary, test_expectations.FLAKY,
+        "flaky")
+    self._PrintExpectedResultsOfType(result_summary, test_expectations.SKIP,
+        "skipped")
+    return result_summary
 
   def AddTestType(self, test_type):
     """Add a TestType to the TestRunner."""
@@ -375,6 +363,7 @@ class TestRunner:
     Return:
       The Queue of lists of TestInfo objects.
     """
+
     if self._options.experimental_fully_parallel:
       filename_queue = Queue.Queue()
       for test_file in test_files:
@@ -493,7 +482,7 @@ class TestRunner:
       proc.stdin.close()
       proc.wait()
 
-  def _RunTests(self, test_shell_binary, file_list):
+  def _RunTests(self, test_shell_binary, file_list, result_summary):
     """Runs the tests in the file_list.
 
     Return: A tuple (failures, thread_timings, test_timings,
@@ -505,6 +494,7 @@ class TestRunner:
           form [time, directory_name, num_tests]
         individual_test_timings is a list of run times for each test in the form
           {filename:filename, test_run_time:test_run_time}
+        result_summary: summary object to populate with the results
     """
     threads = self._InstantiateTestShellThreads(test_shell_binary, file_list)
 
@@ -541,14 +531,18 @@ class TestRunner:
         # would be assumed to have passed.
         raise exception_info[0], exception_info[1], exception_info[2]
 
+    self._PopulateResultSummary(result_summary, failures)
     return (failures, thread_timings, test_timings, individual_test_timings)
 
-  def Run(self):
+  def Run(self, result_summary):
     """Run all our tests on all our test files.
 
     For each test file, we run each test type.  If there are any failures, we
     collect them for reporting.
 
+    Args:
+      result_summary: a summary object tracking the test results.
+
     Return:
       We return nonzero if there are regressions compared to the last run.
     """
@@ -592,24 +586,37 @@ class TestRunner:
       # self._websocket_secure_server.Start()
 
     original_failures, thread_timings, test_timings, individual_test_timings = (
-        self._RunTests(test_shell_binary, self._test_files_list))
+        self._RunTests(test_shell_binary, self._test_files_list,
+                       result_summary))
+    original_results_by_test, original_results_by_type = self._CompareResults(
+        result_summary)
+    final_results_by_test = original_results_by_test
+    final_results_by_type = original_results_by_type
+    failed_results_by_test, failed_results_by_type = self._OnlyFailures(
+        original_results_by_test)
+    num_passes = len(original_results_by_test) - len(failed_results_by_test)
 
     retries = 0
-    final_failures = original_failures
-    original_regressions = self._CompareFailures(final_failures)
-    regressions = original_regressions
-
-    while retries < self.NUM_RETRY_ON_UNEXPECTED_FAILURE and len(regressions):
-      logging.info("Retrying %d unexpected failure(s)" % len(regressions))
+    while (retries < self.NUM_RETRY_ON_UNEXPECTED_FAILURE and
+        len(failed_results_by_test)):
+      logging.info("Retrying %d unexpected failure(s)" %
+                   len(failed_results_by_test))
       retries += 1
-      final_failures = self._RunTests(test_shell_binary, list(regressions))[0]
-      regressions = self._CompareFailures(final_failures)
+      retry_summary = ResultSummary(self._expectations,
+                                    failed_results_by_test.keys())
+      final_failures = self._RunTests(test_shell_binary,
+                                      failed_results_by_test.keys(),
+                                      retry_summary)[0]
+      self._PopulateResultSummary(retry_summary, final_failures)
+      final_results_by_test, final_results_by_type = self._CompareResults(
+          retry_summary)
+      failed_results_by_test, failed_results_by_type = self._OnlyFailures(
+          final_results_by_test)
 
     self._StopLayoutTestHelper(layout_test_helper_proc)
 
     print
     end_time = time.time()
-
     logging.info("%6.2f total testing time" % (end_time - start_time))
     cuml_time = 0
     logging.debug("Thread timing:")
@@ -624,20 +631,20 @@ class TestRunner:
 
     print
     self._PrintTimingStatistics(test_timings, individual_test_timings,
-        original_failures)
+                                result_summary)
 
-    self._PrintRegressions(original_failures, original_regressions,
-        final_failures)
+    self._PrintUnexpectedResults(original_results_by_test,
+                                 original_results_by_type,
+                                 final_results_by_test,
+                                 final_results_by_type)
 
     # Write summaries to stdout.
-    # The summary should include flaky tests, so use original_failures, not
-    # final_failures.
-    result_summary = self._GetResultSummary(original_failures)
+    # The summaries should include flaky tests, so use the original summary,
+    # not the final one.
     self._PrintResultSummary(result_summary, sys.stdout)
 
     if self._options.verbose:
-      self._WriteJSONFiles(original_failures, individual_test_timings,
-          result_summary);
+      self._WriteJSONFiles(result_summary, individual_test_timings);
 
     # Write the same data to a log file.
     out_filename = os.path.join(self._options.results_directory, "score.txt")
@@ -647,40 +654,67 @@ class TestRunner:
 
     # Write the summary to disk (results.html) and maybe open the test_shell
     # to this file.
-    wrote_results = self._WriteResultsHtmlFile(original_failures,
-        original_regressions)
+    wrote_results = self._WriteResultsHtmlFile(result_summary)
     if not self._options.noshow_results and wrote_results:
       self._ShowResultsHtmlFile()
 
     sys.stdout.flush()
     sys.stderr.flush()
-    # Ignore flaky failures so we don't turn the bot red for those.
-    return len(regressions)
+    # Ignore flaky failures and unexpected passes so we don't turn the
+    # bot red for those.
+    return len(failed_results_by_test)
+
+  def _PopulateResultSummary(self, result_summary, failures):
+    """Populate ResultSummary object with test results.
 
-  def _PrintRegressions(self, original_failures, original_regressions,
-      final_failures):
-    """Prints the regressions from the test run.
     Args:
-      original_failures: Failures from the first test run.
-      original_regressions: Regressions from the first test run.
-      final_failures: Failures after retrying the failures from the first run.
+      result_summary: summary object to populate.
+      failures: dictionary mapping the test filename to a list of
+          TestFailure objects if the test failed.
     """
-    print "-" * 78
+    for test, fail_list in failures.iteritems():
+      result = test_failures.DetermineResultType(fail_list)
+      result_summary.Add(test, fail_list, result)
+    for test in self._test_files:
+      if not test in failures:
+        result_summary.Add(test, [], test_expectations.PASS)
 
-    flaky_failures = {}
-    non_flaky_failures = {}
-    for failure in original_failures:
-      if failure not in original_regressions or failure in final_failures:
-        non_flaky_failures[failure] = original_failures[failure]
-      else:
-        flaky_failures[failure] = original_failures[failure]
+  def _CompareResults(self, result_summary):
+    """Determine if the results in this test run are unexpected.
 
-    self._CompareFailures(non_flaky_failures, print_regressions=True)
-    self._CompareFailures(flaky_failures, print_regressions=True, is_flaky=True)
+    Returns:
+      A dict of files -> results and a dict of results -> sets of files
+    """
+    results_by_test = {}
+    results_by_type = {}
+    for result in TestExpectationsFile.EXPECTATIONS.values():
+      results_by_type[result] = set()
+    for test, result in result_summary.unexpected_results.iteritems():
+      results_by_test[test] = result
+      results_by_type[result].add(test)
+    return results_by_test, results_by_type
+
+  def _OnlyFailures(self, results_by_test):
+    """Filters a dict of results and returns only the failures.
 
-    print "-" * 78
+    Args:
+      results_by_test: a dict of files -> results
+
+    Returns:
+      a dict of files -> results and results -> sets of files.
+    """
+    failed_results_by_test = {}
+    failed_results_by_type = {}
+    for test, result in results_by_test.iteritems():
+      if result != test_expectations.PASS:
+        failed_results_by_test[test] = result
+        if result in failed_results_by_type:
+          failed_results_by_type.add(test)
+        else:
+          failed_results_by_type = set([test])
+    return failed_results_by_test, failed_results_by_type
 
-  def _WriteJSONFiles(self, failures, individual_test_timings, result_summary):
+  def _WriteJSONFiles(self, result_summary, individual_test_timings):
     logging.debug("Writing JSON files in %s." % self._options.results_directory)
     # Write a json file of the test_expectations.txt file for the layout tests
     # dashboard.
@@ -690,16 +724,47 @@ class TestRunner:
     expectations_file.write(("ADD_EXPECTATIONS(" + expectations_json + ");"))
     expectations_file.close()
 
-    json_results_generator.JSONResultsGenerator(self._options, failures,
-        individual_test_timings, self._options.results_directory,
-        self._test_files_list, result_summary)
+    json_results_generator.JSONResultsGenerator(self._options,
+        result_summary, individual_test_timings,
+        self._options.results_directory, self._test_files_list)
 
     logging.debug("Finished writing JSON files.")
 
+  def _PrintExpectedResultsOfType(self, result_summary, result_type,
+      result_type_str):
+    """Print the number of the tests in a given result class.
+
+    Args:
+      result_summary - the object containg all the results to report on
+      result_type - the particular result type to report in the summary.
+      result_type_str - a string description of the result_type.
+    """
+    tests = self._expectations.GetTestsWithResultType(result_type)
+    now = result_summary.tests_by_timeline[test_expectations.NOW]
+    wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
+    defer = result_summary.tests_by_timeline[test_expectations.DEFER]
+
+    # We use a fancy format string in order to print the data out in a
+    # nicely-aligned table.
+    fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd defer, %%%dd wontfix)" %
+              (self._NumDigits(now), self._NumDigits(defer),
+               self._NumDigits(wontfix)))
+    logging.info(fmtstr %
+        (len(tests), result_type_str, len(tests & now), len(tests & defer),
+         len(tests & wontfix)))
+
+  def _NumDigits(self, num):
+    """Returns the number of digits needed to represent the length of a
+    sequence."""
+    ndigits = 1
+    if len(num):
+      ndigits = int(math.log10(len(num))) + 1
+    return ndigits
+
   def _PrintTimingStatistics(self, directory_test_timings,
-      individual_test_timings, failures):
+      individual_test_timings, result_summary):
     self._PrintAggregateTestStatistics(individual_test_timings)
-    self._PrintIndividualTestTimes(individual_test_timings, failures)
+    self._PrintIndividualTestTimes(individual_test_timings, result_summary)
     self._PrintDirectoryTimings(directory_test_timings)
 
   def _PrintAggregateTestStatistics(self, individual_test_timings):
@@ -733,12 +798,12 @@ class TestRunner:
           "PER TEST TIMES BY TEST TYPE: %s" % test_type,
           times_per_test_type[test_type])
 
-  def _PrintIndividualTestTimes(self, individual_test_timings, failures):
+  def _PrintIndividualTestTimes(self, individual_test_timings, result_summary):
     """Prints the run times for slow, timeout and crash tests.
     Args:
       individual_test_timings: List of test_shell_thread.TestStats for all
           tests.
-      failures: Dictionary mapping test filenames to list of test_failures.
+      result_summary: Object containing the results of all the tests.
     """
     # Reverse-sort by the time spent in test_shell.
     individual_test_timings.sort(lambda a, b:
@@ -755,13 +820,10 @@ class TestRunner:
         is_timeout_crash_or_slow = True
         slow_tests.append(test_tuple)
 
-      if filename in failures:
-        for failure in failures[filename]:
-          if (failure.__class__ == test_failures.FailureTimeout or
-              failure.__class__ == test_failures.FailureCrash):
-            is_timeout_crash_or_slow = True
-            timeout_or_crash_tests.append(test_tuple)
-            break
+      if (result_summary.results[filename] == test_expectations.TIMEOUT or
+          result_summary.results[filename] == test_expectations.CRASH):
+        is_timeout_crash_or_slow = True
+        timeout_or_crash_tests.append(test_tuple)
 
       if (not is_timeout_crash_or_slow and
           num_printed < self._options.num_slow_tests_to_log):
@@ -829,181 +891,225 @@ class TestRunner:
         "99th percentile: %s, Standard deviation: %s\n" % (
         median, mean, percentile90, percentile99, std_deviation)))
 
-  def _GetResultSummary(self, failures):
-    """Returns a ResultSummary object with failure counts.
+  def _PrintResultSummary(self, result_summary, output):
+    """Print a short summary to the output file about how many tests passed.
 
     Args:
-      failures: dictionary mapping the test filename to a list of
-          TestFailure objects if the test failed
+      output: a file or stream to write to
     """
-    fixable_counts = {}
-    deferred_counts = {}
-    wontfix_counts = {}
-
-    fixable_failures = set()
-    deferred_failures = set()
-    wontfix_failures = set()
-
-    # Aggregate failures in a dictionary (TestFailure -> frequency),
-    # with known (fixable and wontfix) failures separated out.
-    def AddFailure(dictionary, key):
-      if key in dictionary:
-        dictionary[key] += 1
-      else:
-        dictionary[key] = 1
-
-    for test, failure_types in failures.iteritems():
-      for failure_type in failure_types:
-        # TODO(ojan): Now that we have IMAGE+TEXT, IMAGE and TEXT,
-        # we can avoid adding multiple failures per test since there should
-        # be a unique type of failure for each test. This would make the
-        # statistics printed at the end easier to grok.
-        if self._expectations.IsDeferred(test):
-          count_group = deferred_counts
-          failure_group = deferred_failures
-        elif self._expectations.IsIgnored(test):
-          count_group = wontfix_counts
-          failure_group = wontfix_failures
-        else:
-          count_group = fixable_counts
-          failure_group = fixable_failures
-
-        AddFailure(count_group, failure_type.__class__)
-        failure_group.add(test)
-
-    # Here and below, use the prechuncked expectations object for numbers of
-    # skipped tests. Chunking removes the skipped tests before creating the
-    # expectations file.
-    #
-    # This is a bit inaccurate, since the number of skipped tests will be
-    # duplicated across all shard, but it's the best we can reasonably do.
-
-    deduped_fixable_count = len(fixable_failures |
-        self.prechunk_expectations.GetFixableSkipped())
-    all_fixable_count = len(self._test_files -
-        self._expectations.GetWontFix() - self._expectations.GetDeferred())
-
-    # Breakdown of tests we need to fix and want to pass.
-    # Include skipped fixable tests in the statistics.
-    fixable_result_summary = ResultSummaryEntry(
-        self._expectations.GetFixable() | fixable_failures,
-        fixable_failures,
-        fixable_counts,
-        self.prechunk_expectations.GetFixableSkipped())
-
-    deferred_result_summary = ResultSummaryEntry(
-        self._expectations.GetDeferred(),
-        deferred_failures,
-        deferred_counts,
-        self.prechunk_expectations.GetDeferredSkipped())
-
-    wontfix_result_summary = ResultSummaryEntry(
-        self._expectations.GetWontFix(),
-        wontfix_failures,
-        wontfix_counts,
-        self.prechunk_expectations.GetWontFixSkipped())
-
-    return ResultSummary(deferred_result_summary, wontfix_result_summary,
-        fixable_result_summary, deduped_fixable_count, all_fixable_count)
+    failed = len(result_summary.failures)
+    skipped = len(result_summary.tests_by_expectation[test_expectations.SKIP])
+    total = result_summary.total
+    passed = total - failed - skipped
+    pct_passed = 0.0
+    if total > 0:
+      pct_passed = float(passed) * 100 / total
 
-  def _PrintResultSummary(self, result_summary, output):
-    """Print a short summary to stdout about how many tests passed.
+    output.write("\n");
+    output.write("=> Total: %d/%d tests passed (%.1f%%)\n" %
+                 (passed, total, pct_passed))
+
+    output.write("\n");
+    self._PrintResultSummaryEntry(result_summary, test_expectations.NOW,
+       "Tests to be fixed for the current release", output)
+
+    output.write("\n");
+    self._PrintResultSummaryEntry(result_summary, test_expectations.DEFER,
+       "Tests we'll fix in the future if they fail (DEFER)", output)
+
+    output.write("\n");
+    self._PrintResultSummaryEntry(result_summary, test_expectations.WONTFIX,
+       "Tests that won't be fixed if they fail (WONTFIX)", output)
+
+  def _PrintResultSummaryEntry(self, result_summary, timeline, heading, output):
+    """Print a summary block of results for a particular timeline of test.
 
     Args:
-      result_summary: ResultSummary object with failure counts.
-      output: file descriptor to write the results to. For example, sys.stdout.
+      result_summary: summary to print results for
+      timeline: the timeline to print results for (NOT, WONTFIX, etc.)
+      heading: a textual description of the timeline
+      output: a stream to write to
     """
-    failed = result_summary.fixable_count
-    total = result_summary.all_fixable_count
-    passed = 0
-    if total > 0:
-      passed = float(total - failed) * 100 / total
-    output.write(
-        "\nTest summary: %.1f%% Passed | %s Failures | %s Tests to pass for "
-        "this release\n" % (
-        passed, failed, total))
-
-    self._PrintResultSummaryEntry(
-        "Tests to be fixed for the current release",
-        result_summary.fixable,
-        output)
-    self._PrintResultSummaryEntry(
-        "Tests to be fixed for a future release (DEFER)",
-        result_summary.deferred,
-        output)
-    self._PrintResultSummaryEntry("Tests never to be fixed (WONTFIX)",
-        result_summary.wontfix,
-        output)
+    total = len(result_summary.tests_by_timeline[timeline])
+    not_passing = (total -
+       len(result_summary.tests_by_expectation[test_expectations.PASS] &
+           result_summary.tests_by_timeline[timeline]))
+    output.write("=> %s (%d):\n" % (heading, not_passing))
+    for result in TestExpectationsFile.EXPECTATION_ORDER:
+      if result == test_expectations.PASS:
+        continue
+      results = (result_summary.tests_by_expectation[result] &
+                 result_summary.tests_by_timeline[timeline])
+      desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
+      plural = ["", "s"]
+      if not_passing and len(results):
+        pct = len(results) * 100.0 / not_passing
+        output.write("%d test case%s (%4.1f%%) %s\n" %
+                     (len(results), plural[len(results) != 1], pct,
+                      desc[len(results) != 1]))
+
+  def _PrintUnexpectedResults(self, original_results_by_test,
+                              original_results_by_type,
+                              final_results_by_test,
+                              final_results_by_type):
+    """Print unexpected results (regressions) to stdout and a file.
+
+    Args:
+      original_results_by_test: dict mapping tests -> results for the first
+        (original) test run
+      original_results_by_type: dict mapping results -> sets of tests for the
+        first test run
+      final_results_by_test: dict of tests->results after the retries
+        eliminated any flakiness
+      final_results_by_type: dict of results->tests after the retries
+        eliminated any flakiness
+    """
+    print "-" * 78
     print
+    flaky_results_by_type = {}
+    non_flaky_results_by_type = {}
+
+    for test, result in original_results_by_test.iteritems():
+      if (result == test_expectations.PASS or
+          (test in final_results_by_test and
+           result == final_results_by_test[test])):
+        if result in non_flaky_results_by_type:
+          non_flaky_results_by_type[result].add(test)
+        else:
+          non_flaky_results_by_type[result] = set([test])
+      else:
+        if result in flaky_results_by_type:
+          flaky_results_by_type[result].add(test)
+        else:
+          flaky_results_by_type[result] = set([test])
 
-  def _PrintResultSummaryEntry(self, heading, result_summary, output):
-    """Print a summary block of results for a particular category of test.
+    self._PrintUnexpectedResultsByType(non_flaky_results_by_type, False,
+                                       sys.stdout)
+    self._PrintUnexpectedResultsByType(flaky_results_by_type, True, sys.stdout)
+
+    out_filename = os.path.join(self._options.results_directory,
+                                "regressions.txt")
+    output_file = open(out_filename, "w")
+    self._PrintUnexpectedResultsByType(non_flaky_results_by_type, False,
+                                       output_file)
+    output_file.close()
+
+    print "-" * 78
+
+  def _PrintUnexpectedResultsByType(self, results_by_type, is_flaky, output):
+    """Helper method to print a set of unexpected results to an output stream
+    sorted by the result type.
 
     Args:
-      heading: text to print before the block, followed by the total count
-      result_summary: ResultSummaryEntry object with the result counts
-      output: file descriptor to write the results to
+      results_by_type: dict(result_type -> list of files)
+      is_flaky: where these results flaky or not (changes the output)
+      output: stream to write output to
     """
-    output.write("\n=> %s (%d):\n" % (heading, result_summary.total_count))
-    self._PrintResultLine(result_summary.pass_count, result_summary.total_count,
-        "Passed", output)
-    self._PrintResultLine(result_summary.skip_count, result_summary.total_count,
-        "Skipped", output)
-    sorted_keys = sorted(result_summary.failure_counts.keys(),
-                         key=test_failures.FailureSort.SortOrder)
-    for failure in sorted_keys:
-      self._PrintResultLine(result_summary.failure_counts[failure],
-                            result_summary.total_count,
-                            failure.Message(),
-                            output)
-
-  def _PrintResultLine(self, count, total, message, output):
-    if count == 0: return
-    output.write(
-        ("%(count)d test case%(plural)s (%(percent).1f%%) %(message)s\n" %
-            { 'count'   : count,
-              'plural'  : ('s', '')[count == 1],
-              'percent' : float(count) * 100 / total,
-              'message' : message }))
-
-  def _CompareFailures(self, failures, print_regressions=False, is_flaky=False):
-    """Determine if the failures in this test run are unexpected.
+    descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS
+    keywords = {}
+    for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
+      keywords[v] = k.upper()
+
+    for result in TestExpectationsFile.EXPECTATION_ORDER:
+      if result in results_by_type:
+        self._PrintUnexpectedResultSet(output, results_by_type[result],
+                                       is_flaky, descriptions[result][1],
+                                       keywords[result])
+
+  def _PrintUnexpectedResultSet(self, output, filenames, is_flaky,
+                                header_text, keyword):
+    """A helper method to print one set of results (all of the same type)
+    to a stream.
 
     Args:
-      failures: a dictionary mapping the test filename to a list of
-          TestFailure objects if the test failed
-      print_regressions: whether to print the regressions to stdout
-      is_flaky: whether this set of failures represents tests that failed
-          the first time around, but passed on a subsequent run
+      output: a stream or file object to write() to
+      filenames: a list of absolute filenames
+      header_text: a string to display before the list of filenames
+      keyword: expectation keyword
+    """
+    filenames = list(filenames)
+    filenames.sort()
+    if not is_flaky and keyword == 'PASS':
+      self._PrintUnexpectedPasses(output, filenames)
+      return
+
+    if is_flaky:
+      prefix = "Unexpected flakiness:"
+    else:
+      prefix = "Regressions: Unexpected"
+    output.write("%s %s (%d):\n" % (prefix, header_text, len(filenames)))
+    for filename in filenames:
+      flaky_text = ""
+      if is_flaky:
+        flaky_text = " " + self._expectations.GetExpectationsString(filename)
+
+      filename = path_utils.RelativeTestFilename(filename)
+      output.write("  %s = %s%s\n" % (filename, keyword, flaky_text))
+    output.write("\n")
+
+  def _PrintUnexpectedPasses(self, output, filenames):
+    """Prints the list of files that passed unexpectedly.
+
+    TODO(dpranke): This routine is a bit of a hack, since it's not clear
+    what the best way to output this is. Each unexpected pass might have
+    multiple expected results, and printing all the combinations would
+    be pretty clunky. For now we sort them into three buckets, crashes,
+    timeouts, and everything else.
 
-    Return:
-      A set of regressions (unexpected failures, hangs, or crashes)
+    Args:
+      output: a stream to write to
+      filenames: list of files that passed
     """
-    cf = compare_failures.CompareFailures(self._test_files,
-                                          failures,
-                                          self._expectations,
-                                          is_flaky)
-    if print_regressions:
-      cf.PrintRegressions(sys.stdout)
+    crashes = []
+    timeouts = []
+    failures = []
+    for filename in filenames:
+      expectations = self._expectations.GetExpectations(filename)
+      if test_expectations.CRASH in expectations:
+        crashes.append(filename)
+      elif test_expectations.TIMEOUT in expectations:
+        timeouts.append(filename)
+      else:
+        failures.append(filename)
 
-    return cf.GetRegressions()
+    self._PrintPassSet(output, "crash", crashes)
+    self._PrintPassSet(output, "timeout", timeouts)
+    self._PrintPassSet(output, "fail", failures)
 
-  def _WriteResultsHtmlFile(self, failures, regressions):
+  def _PrintPassSet(self, output, expected_str, filenames):
+    """Print a set of unexpected passes.
+
+    Args:
+      output: stream to write to
+      expected_str: worst expectation for the given file
+      filenames: list of files in that set
+    """
+    if len(filenames):
+      filenames.sort()
+      output.write("Expected to %s, but passed: (%d)\n" %
+                   (expected_str, len(filenames)))
+      for filename in filenames:
+        filename = path_utils.RelativeTestFilename(filename)
+        output.write("  %s\n" % filename)
+      output.write("\n")
+
+  def _WriteResultsHtmlFile(self, result_summary):
     """Write results.html which is a summary of tests that failed.
 
     Args:
-      failures: a dictionary mapping the test filename to a list of
-          TestFailure objects if the test failed
-      regressions: a set of test filenames that regressed
+      result_summary: a summary of the results :)
 
     Returns:
       True if any results were written (since expected failures may be omitted)
     """
     # test failures
+    failures = result_summary.failures
+    failed_results_by_test, failed_results_by_type = self._OnlyFailures(
+        result_summary.unexpected_results)
     if self._options.full_results_html:
       test_files = failures.keys()
     else:
-      test_files = list(regressions)
+      test_files = failed_results_by_test.keys()
     if not len(test_files):
       return False
 
@@ -1153,15 +1259,15 @@ def main(options, args):
   if options.lint_test_files:
     # Creating the expecations for each platform/target pair does all the
     # test list parsing and ensures it's correct syntax (e.g. no dupes).
-    for platform in test_expectations.TestExpectationsFile.PLATFORMS:
+    for platform in TestExpectationsFile.PLATFORMS:
       test_runner.ParseExpectations(platform, is_debug_mode=True)
       test_runner.ParseExpectations(platform, is_debug_mode=False)
     print ("If there are no fail messages, errors or exceptions, then the "
         "lint succeeded.")
-    return
+    sys.exit(0)
   else:
     test_runner.ParseExpectations(options.platform, options.target == 'Debug')
-    test_runner.PrepareListsAndPrintOutput()
+    result_summary = test_runner.PrepareListsAndPrintOutput()
 
   if options.find_baselines:
     # Record where we found each baseline, then exit.
@@ -1215,7 +1321,7 @@ def main(options, args):
     test_runner.AddTestType(image_diff.ImageDiff)
     if options.fuzzy_pixel_tests:
       test_runner.AddTestType(fuzzy_image_diff.FuzzyImageDiff)
-  has_new_failures = test_runner.Run()
+  has_new_failures = test_runner.Run(result_summary)
   logging.info("Exit status: %d" % has_new_failures)
   sys.exit(has_new_failures)
 
diff --git a/webkit/tools/layout_tests/update_expectations_from_dashboard.py b/webkit/tools/layout_tests/update_expectations_from_dashboard.py
index 5bfb8cc..5d0b14e 100644
--- a/webkit/tools/layout_tests/update_expectations_from_dashboard.py
+++ b/webkit/tools/layout_tests/update_expectations_from_dashboard.py
@@ -368,7 +368,7 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile):
       if not has_non_pass_expectation:
         expectations = ["pass"]
 
-      missing_build_types = self.BUILD_TYPES[:]
+      missing_build_types = list(self.BUILD_TYPES)
       sentinal = None
       for build_type in update.build_info:
         if not sentinal:
@@ -406,7 +406,7 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile):
       platforms: List of lower-case platform names.
     """
     platforms.sort()
-    if platforms == self.BASE_PLATFORMS:
+    if platforms == list(self.BASE_PLATFORMS):
       return ""
     else:
       return " ".join(platforms)
@@ -449,4 +449,4 @@ def main():
   open(path_to_expectations, 'w').write(new_expectations)
 
 if '__main__' == __name__:
-  main()
\ No newline at end of file
+  main()