Adds a new unit tests for layout tests formatting script.

R=ojan
BUG=17595
TEST=Run test_output_formatter.bat -r, all tests should pass

Review URL: http://codereview.chromium.org/160054

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@21845 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 254 insertions, 17 deletions

diff --git a/webkit/tools/layout_tests/layout_package/failure_finder.py b/webkit/tools/layout_tests/layout_package/failure_finder.py
index 342c7ba..f79612f 100644
--- a/webkit/tools/layout_tests/layout_package/failure_finder.py
+++ b/webkit/tools/layout_tests/layout_package/failure_finder.py
@@ -181,7 +181,7 @@ class FailureFinder(object):
                verbose):
     self.build = build
     # TODO(gwilson): add full url-encoding for the platform.
-    self.platform = builder_name.replace(" ", "%20")
+    self.SetPlatform(builder_name)
     self.exclude_known_failures = exclude_known_failures
     self.test_regex = test_regex
     self.output_dir = output_dir
@@ -190,6 +190,11 @@ class FailureFinder(object):
     self.fyi_builder = False
     self._flaky_test_cache = {}
     self._test_expectations_cache = None
+    # If true, scraping will still happen but no files will be downloaded.
+    self.dont_download = False
+
+  def SetPlatform(self, platform):
+    self.platform = platform.replace(" ", "%20")
 
   # TODO(gwilson): Change this to get the last build that finished
   # successfully.
@@ -214,7 +219,8 @@ class FailureFinder(object):
       print "Using build number %s" % self.build
 
     self.failures = self._GetFailuresFromBuilder()
-    if self.failures and self._DownloadResultResources():
+    if (self.failures and
+        (self._DownloadResultResources() or self.dont_download)):
       return self.failures
     return None
 
@@ -313,11 +319,7 @@ class FailureFinder(object):
     Finds and downloads/extracts all of the test results (pixel/text output)
     for all of the given failures.
     """
-    content = ScrapeURL(GetArchiveURL(self.build,
-                                      self.platform,
-                                      self.fyi_builder))
-    revision = ExtractFirstValue(content, ARCHIVE_URL_REGEX)
-    build_name = ExtractFirstValue(content, BUILD_NAME_REGEX)
+    revision, build_name = self._GetRevisionAndBuildFromArchiveStep()
 
     target_zip = "%s/layout-test-results-%s.zip" % (self.output_dir,
                                                     self.build)
@@ -326,7 +328,8 @@ class FailureFinder(object):
       print "Downloading zip file from %s to %s" % (zip_url, target_zip)
     filename = self._DownloadFile(zip_url, target_zip, "b")
     if not filename:
-      print "Could not download zip file from %s.  Does it exist?" % zip_url
+      if self.verbose:
+        print "Could not download zip file from %s.  Does it exist?" % zip_url
       return False
 
     if zipfile.is_zipfile(filename):
@@ -351,9 +354,18 @@ class FailureFinder(object):
       os.remove(filename)
       return True
     else:
-      print "Downloaded file '%s' doesn't look like a zip file." % filename
+      if self.verbose:
+        print "Downloaded file '%s' doesn't look like a zip file." % filename
       return False
 
+  def _GetRevisionAndBuildFromArchiveStep(self):
+    content = ScrapeURL(GetArchiveURL(self.build,
+                                      self.platform,
+                                      self.fyi_builder))
+    revision = ExtractFirstValue(content, ARCHIVE_URL_REGEX)
+    build_name = ExtractFirstValue(content, BUILD_NAME_REGEX)
+    return (revision, build_name)
+
   def _PopulateTextFailure(self, failure, directory, zip):
     baselines = self._GetBaseline(failure.GetExpectedTextFilename(),
                                   directory)
@@ -363,9 +375,10 @@ class FailureFinder(object):
       self._GetFileAge(failure.GetTextBaselineTracHome()))
     failure.text_actual_local = "%s/%s" % (directory,
                                            failure.GetActualTextFilename())
-    if self._ExtractFileFromZip(zip,
-                                failure.GetTextResultLocationInZipFile(),
-                                failure.text_actual_local):
+    if (not self.dont_download and
+        self._ExtractFileFromZip(zip,
+                                 failure.GetTextResultLocationInZipFile(),
+                                 failure.text_actual_local)):
       GenerateTextDiff(failure.text_baseline_local,
                        failure.text_actual_local,
                        directory + "/" + failure.GetTextDiffFilename())
@@ -383,10 +396,11 @@ class FailureFinder(object):
       self._ExtractFileFromZip(zip,
                                failure.GetImageResultLocationInZipFile(),
                                failure.image_actual_local)
-      if not GeneratePNGDiff("./" + failure.image_baseline_local,
+      if (not GeneratePNGDiff("./" + failure.image_baseline_local,
                              "./" + failure.image_actual_local,
                              "./%s/%s" %
-                             (directory, failure.GetImageDiffFilename())):
+                             (directory, failure.GetImageDiffFilename()))
+          and self.verbose):
         print "Could not generate PNG diff for %s" % failure.test_path
       if failure.IsImageBaselineInChromium():
         upstream_baselines = (
@@ -419,7 +433,8 @@ class FailureFinder(object):
     if local_filename.endswith(".png"):
       download_file_modifiers = "b"  # binary file
 
-    CreateDirectory(local_filename[0:local_filename.rfind("/")])
+    if not self.dont_download:
+      CreateDirectory(local_filename[0:local_filename.rfind("/")])
 
     webkit_mac_location = (
       self._MangleWebkitPixelTestLocation(WEBKIT_IMAGE_BASELINE_BASE_URL_MAC,
@@ -493,7 +508,8 @@ class FailureFinder(object):
         return ExtractSingleRegexAtURL(url + "?view=log",
                                        CHROMIUM_FILE_AGE_REGEX)
     except:
-      print "Could not find age for %s. Does the file exist?" % url
+      if self.verbose:
+        print "Could not find age for %s. Does the file exist?" % url
       return None
 
   # Returns a flakiness on a scale of 1-50.
@@ -573,7 +589,10 @@ class FailureFinder(object):
       if local_filename == None:
         local_filename = url.split('/')[-1]
       if os.path.isfile(local_filename) and not force:
-        print "File at %s already exists." % local_filename
+        if self.verbose:
+          print "File at %s already exists." % local_filename
+        return local_filename
+      if self.dont_download:
         return local_filename
       webFile = urllib2.urlopen(url)
       localFile = open(local_filename, ("w%s" % modifiers))
diff --git a/webkit/tools/layout_tests/layout_package/failure_finder_test.py b/webkit/tools/layout_tests/layout_package/failure_finder_test.py
new file mode 100644
index 0000000..2e761ed
--- /dev/null
+++ b/webkit/tools/layout_tests/layout_package/failure_finder_test.py
@@ -0,0 +1,210 @@
+#!/bin/env/python
+# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+
+from failure_finder import FailureFinder
+
+TEST_BUILDER_OUTPUT = """090723 10:38:22 test_shell_thread.py:289
+            ERROR chrome/fast/forms/textarea-metrics.html failed:
+              Text diff mismatch
+              Simplified text diff mismatch
+            090723 10:38:21 test_shell_thread.py:289
+            ERROR chrome/fast/dom/xss-DENIED-javascript-variations.html failed:
+              Text diff mismatch
+              Simplified text diff mismatch
+            090723 10:37:58 test_shell_thread.py:289
+            ERROR LayoutTests/plugins/bindings-test.html failed:
+              Text diff mismatch
+              Simplified text diff mismatch
+------------------------------------------------------------------------------
+Expected to crash, but passed (1):
+  chrome/fast/forms/textarea-metrics.html
+
+Regressions: Unexpected failures (2):
+  chrome/fast/dom/xss-DENIED-javascript-variations.html = FAIL
+  LayoutTests/plugins/bindings-test.html = FAIL
+------------------------------------------------------------------------------
+"""
+
+WEBKIT_BUILDER_NUMBER = "9800"
+WEBKIT_FAILURES = (
+  ["LayoutTests/fast/backgrounds/animated-svg-as-mask.html",
+   "LayoutTests/fast/backgrounds/background-clip-text.html",
+   "LayoutTests/fast/backgrounds/mask-composite.html",
+   "LayoutTests/fast/backgrounds/repeat/mask-negative-offset-repeat.html",
+   "LayoutTests/fast/backgrounds/svg-as-background-3.html",
+   "LayoutTests/fast/backgrounds/svg-as-background-6.html",
+   "LayoutTests/fast/backgrounds/svg-as-mask.html",
+   "LayoutTests/fast/block/float/013.html",
+   "LayoutTests/fast/block/float/nested-clearance.html",
+   "LayoutTests/fast/block/positioning/047.html"])
+
+CHROMIUM_BASELINE = "chrome/fast/forms/basic-buttons.html"
+EXPECTED_CHROMIUM_LOCAL_BASELINE = "./chrome/fast/forms/basic-buttons.html"
+EXPECTED_CHROMIUM_URL_BASELINE = ("http://src.chromium.org/viewvc/chrome/"
+                                  "trunk/src/webkit/data/layout_tests/chrome/"
+                                  "fast/forms/basic-buttons.html")
+
+WEBKIT_BASELINE = "LayoutTests/fast/forms/11423.html"
+EXPECTED_WEBKIT_LOCAL_BASELINE = "./LayoutTests/fast/forms/11423.html"
+EXPECTED_WEBKIT_URL_BASELINE = ("http://svn.webkit.org/repository/webkit/trunk/"
+                                "LayoutTests/fast/forms/11423.html")
+
+TEST_ZIP_FILE = ("http://build.chromium.org/buildbot/layout_test_results/"
+                 "webkit-rel/21432/layout-test-results.zip")
+
+EXPECTED_REVISION = "20861"
+EXPECTED_BUILD_NAME = "webkit-rel"
+
+class FailureFinderTest(object):
+
+  def runTests(self):
+    all_tests_passed = True
+
+    tests = ["testWhitespaceInBuilderName",
+             "testGetLastBuild",
+             "testFindMatchesInBuilderOutput",
+             "testScrapeBuilderOutput",
+             "testGetChromiumBaseline",
+             "testGetWebkitBaseline",
+             "testZipDownload",
+             "testTranslateBuildToZip",
+             "testFull"]
+
+    for test in tests:
+      print test
+      result = eval(test + "()")
+      if result:
+        print "PASS"
+      else:
+        all_tests_passed = False
+        print "FAIL"
+    return all_tests_passed
+
+def _getBasicFailureFinder():
+  return FailureFinder(None, "Webkit", False, "", ".", 10, False)
+
+def _testLastBuild(failure_finder):
+  try:
+    last_build = failure_finder.GetLastBuild()
+    # Verify that last_build is not empty and is a number.
+    build = int(last_build)
+    return (build > 0)
+  except:
+    return False
+
+def testGetLastBuild():
+  test = _getBasicFailureFinder()
+  return _testLastBuild(test)
+
+def testWhitespaceInBuilderName():
+  test = _getBasicFailureFinder()
+  test.SetPlatform("Webkit (webkit.org)")
+  return _testLastBuild(test)
+
+def testScrapeBuilderOutput():
+
+  # Try on the default builder.
+  test = _getBasicFailureFinder()
+  test.build = "9800"
+  output = test._ScrapeBuilderOutput()
+  if not output:
+    return False
+
+  # Try on a crazy builder on the FYI waterfall.
+  test = _getBasicFailureFinder()
+  test.build = "1766"
+  test.SetPlatform("Webkit Linux (webkit.org)")
+  output = test._ScrapeBuilderOutput()
+  if not output:
+    return False
+
+  return True
+
+def testFindMatchesInBuilderOutput():
+  test =  _getBasicFailureFinder()
+  test.exclude_known_failures = True
+  matches = test._FindMatchesInBuilderOutput(TEST_BUILDER_OUTPUT)
+  # Verify that we found x matches.
+  if len(matches) != 2:
+    print "Did not find all unexpected failures."
+    return False
+
+  test.exclude_known_failures = False
+  matches = test._FindMatchesInBuilderOutput(TEST_BUILDER_OUTPUT)
+  if len(matches) != 3:
+    print "Did not find all failures."
+    return False
+  return True
+
+def _testBaseline(test_name, expected_local, expected_url):
+  test = _getBasicFailureFinder()
+  # Test baseline that is obviously in Chromium's tree.
+  local, url = test._GetBaseline(test_name, ".", False)
+  try:
+    os.remove(local)
+    if (local != expected_local or url != expected_url):
+      return False
+  except:
+    return False
+  return True
+
+def testGetChromiumBaseline():
+  return _testBaseline(CHROMIUM_BASELINE, EXPECTED_CHROMIUM_LOCAL_BASELINE,
+                       EXPECTED_CHROMIUM_URL_BASELINE)
+
+def testGetWebkitBaseline():
+  return _testBaseline(WEBKIT_BASELINE, EXPECTED_WEBKIT_LOCAL_BASELINE,
+                       EXPECTED_WEBKIT_URL_BASELINE)
+
+# TODO(gwilson): implement support for using local log files instead of
+# scraping the buildbots.
+def testUseLocalOutput():
+  return True
+
+def testZipDownload():
+  test = _getBasicFailureFinder()
+  try:
+    test._DownloadFile(TEST_ZIP_FILE, "test.zip", "b") # "b" -> binary
+    os.remove("test.zip")
+    return True
+  except:
+    return False
+
+def testTranslateBuildToZip():
+  test = _getBasicFailureFinder()
+  test.build = WEBKIT_BUILDER_NUMBER
+  revision, build_name = test._GetRevisionAndBuildFromArchiveStep()
+  if revision != EXPECTED_REVISION or build_name != EXPECTED_BUILD_NAME:
+    return False
+  return True
+
+def testFull():
+  """ Verifies that the entire system works end-to-end. """
+  test = _getBasicFailureFinder()
+  test.build = WEBKIT_BUILDER_NUMBER
+  test.dont_download = True  # Dry run only, no downloading needed.
+  failures = test.GetFailures()
+  # Verify that the max failures parameter works.
+  if not failures or len(failures) > 10:
+    "Got no failures or too many failures."
+    return False
+
+  # Verify the failures match the list of expected failures.
+  for failure in failures:
+    if not (failure.test_path in WEBKIT_FAILURES):
+      print "Found a failure I did not expect to see."
+      return False
+
+  return True
+
+if __name__ == "__main__":
+  fft = FailureFinderTest()
+  result = fft.runTests()
+  if result:
+    print "All tests passed."
+  else:
+    print "Not all tests passed."
diff --git a/webkit/tools/layout_tests/test_output_formatter.py b/webkit/tools/layout_tests/test_output_formatter.py
index 217708f..89cdb15 100644
--- a/webkit/tools/layout_tests/test_output_formatter.py
+++ b/webkit/tools/layout_tests/test_output_formatter.py
@@ -12,12 +12,17 @@ import optparse
 from layout_package import test_expectations
 from layout_package import failure
 from layout_package import failure_finder
+from layout_package import failure_finder_test
 from layout_package import html_generator
 
 DEFAULT_BUILDER = "Webkit"
 
 def main(options, args):
 
+  if options.run_tests:
+    fft = failure_finder_test.FailureFinderTest()
+    return fft.runTests()
+
   # TODO(gwilson): Add a check that verifies the given platform exists.
 
   finder = failure_finder.FailureFinder(options.build_number,
@@ -66,6 +71,9 @@ if __name__ == "__main__":
   option_parser.add_option("-m", "--max-failures",
                            default = 100,
                            help = "Limit the maximum number of failures")
+  option_parser.add_option("-r", "--run-tests", action = "store_true",
+                           default = False,
+                           help = "Runs unit tests")
   options, args = option_parser.parse_args()
   main(options, args)