Add an image diff link to failed layout tests.

Review URL: http://codereview.chromium.org/18023

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@8119 0039d316-1c4b-4281-b951-d872f2087c98

10 files changed, 76 insertions, 12 deletions

diff --git a/webkit/tools/layout_tests/layout_package/platform_utils_linux.py b/webkit/tools/layout_tests/layout_package/platform_utils_linux.py
index f8cb79d..7965fb3 100644
--- a/webkit/tools/layout_tests/layout_package/platform_utils_linux.py
+++ b/webkit/tools/layout_tests/layout_package/platform_utils_linux.py
@@ -164,6 +164,11 @@ class PlatformUtility(object):
     """
     return 'wdiff'
 
+  def ImageCompareExecutablePath(self, target):
+    """Path to the image_diff binary."""
+    # See mmoss TODO below.
+    return PathFromBase('chrome', 'Hammer', 'image_diff')
+
   def TestShellBinary(self):
     """The name of the binary for TestShell."""
     return 'test_shell'
diff --git a/webkit/tools/layout_tests/layout_package/platform_utils_mac.py b/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
index 0a5e090..dbc6963 100644
--- a/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
+++ b/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
@@ -133,6 +133,9 @@ class PlatformUtility(object):
     """
     return 'wdiff'
 
+  def ImageCompareExecutablePath(self, target):
+    return PathFromBase('xcodebuild', target, 'image_diff')
+
   def TestShellBinary(self):
     """The name of the binary for TestShell."""
     return 'TestShell'
diff --git a/webkit/tools/layout_tests/layout_package/platform_utils_win.py b/webkit/tools/layout_tests/layout_package/platform_utils_win.py
index 06bc445..c6bd525 100644
--- a/webkit/tools/layout_tests/layout_package/platform_utils_win.py
+++ b/webkit/tools/layout_tests/layout_package/platform_utils_win.py
@@ -185,6 +185,9 @@ class PlatformUtility(google.platform_utils_win.PlatformUtility):
     """Path to the WDiff executable, whose binary is checked in on Win"""
     return PathFromBase('third_party', 'cygwin', 'bin', 'wdiff.exe')
 
+  def ImageCompareExecutablePath(self, target):
+    return PathFromBase('chrome', target, 'image_diff.exe')
+
   def TestShellBinary(self):
     """The name of the binary for TestShell."""
     return 'test_shell.exe'
diff --git a/webkit/tools/layout_tests/layout_package/test_failures.py b/webkit/tools/layout_tests/layout_package/test_failures.py
index 8779a68..134f2e3 100644
--- a/webkit/tools/layout_tests/layout_package/test_failures.py
+++ b/webkit/tools/layout_tests/layout_package/test_failures.py
@@ -194,7 +194,7 @@ class FailureMissingImage(FailureWithType):
 
 class FailureImageHashMismatch(FailureWithType):
   """Image hashes didn't match."""
-  OUT_FILENAMES = ["-actual-win.png", "-expected.png"]
+  OUT_FILENAMES = ["-actual-win.png", "-expected.png", "-diff-win.png"]
 
   @staticmethod
   def Message():
diff --git a/webkit/tools/layout_tests/layout_package/test_shell_thread.py b/webkit/tools/layout_tests/layout_package/test_shell_thread.py
index fde21de..eb2faf7 100644
--- a/webkit/tools/layout_tests/layout_package/test_shell_thread.py
+++ b/webkit/tools/layout_tests/layout_package/test_shell_thread.py
@@ -27,7 +27,7 @@ import test_failures
 # test_shell.exe.
 DEFAULT_TEST_TIMEOUT_MS = 10 * 1000
 
-def ProcessOutput(proc, filename, test_uri, test_types, test_args):
+def ProcessOutput(proc, filename, test_uri, test_types, test_args, target):
   """Receives the output from a test_shell process, subjects it to a number
   of tests, and returns a list of failure types the test produced.
 
@@ -36,6 +36,7 @@ def ProcessOutput(proc, filename, test_uri, test_types, test_args):
     filename: path of the test file being run
     test_types: list of test types to subject the output to
     test_args: arguments to be passed to each test
+    target: Debug or Release
 
   Returns: a list of failure objects for the test being processed
   """
@@ -82,7 +83,8 @@ def ProcessOutput(proc, filename, test_uri, test_types, test_args):
   for test_type in test_types:
     new_failures = test_type.CompareOutput(filename, proc,
                                            ''.join(outlines),
-                                           local_test_args)
+                                           local_test_args,
+                                           target)
     # Don't add any more failures if we already have a crash or timeout, so
     # we don't double-report those tests.
     if not crash_or_timeout:
@@ -254,7 +256,8 @@ class TestShellThread(threading.Thread):
 
     # ...and read the response
     return ProcessOutput(self._test_shell_proc, filename, test_uri,
-                         self._test_types, self._test_args)
+                         self._test_types, self._test_args,
+                         self._options.target)
 
 
   def _EnsureTestShellIsRunning(self):
diff --git a/webkit/tools/layout_tests/test_types/fuzzy_image_diff.py b/webkit/tools/layout_tests/test_types/fuzzy_image_diff.py
index 579a30a..6a87e22 100644
--- a/webkit/tools/layout_tests/test_types/fuzzy_image_diff.py
+++ b/webkit/tools/layout_tests/test_types/fuzzy_image_diff.py
@@ -17,7 +17,7 @@ from layout_package import test_failures
 from test_types import test_type_base
 
 class FuzzyImageDiff(test_type_base.TestTypeBase):
-  def CompareOutput(self, filename, proc, output, test_args):
+  def CompareOutput(self, filename, proc, output, test_args, target):
     """Implementation of CompareOutput that checks the output image and
     checksum against the expected files from the LayoutTest directory.
     """
diff --git a/webkit/tools/layout_tests/test_types/image_diff.py b/webkit/tools/layout_tests/test_types/image_diff.py
index 07c7561..077e0c4 100644
--- a/webkit/tools/layout_tests/test_types/image_diff.py
+++ b/webkit/tools/layout_tests/test_types/image_diff.py
@@ -13,11 +13,17 @@ import errno
 import logging
 import os
 import shutil
+import subprocess
 
 from layout_package import path_utils
+from layout_package import platform_utils
 from layout_package import test_failures
 from test_types import test_type_base
 
+# Cache whether we have the image_diff executable available.
+_compare_available = True
+_compare_msg_printed = False
+
 class ImageDiff(test_type_base.TestTypeBase):
   def _CopyOutputPNGs(self, filename, actual_png, expected_png):
     """Copies result files into the output directory with appropriate names.
@@ -53,7 +59,48 @@ class ImageDiff(test_type_base.TestTypeBase):
     self._SaveBaselineData(filename, png_data, ".png")
     self._SaveBaselineData(filename, checksum, ".checksum")
 
-  def CompareOutput(self, filename, proc, output, test_args):
+  def _CreateImageDiff(self, filename, target):
+    """Creates the visual diff of the expected/actual PNGs.
+
+    Args:
+      filename: the name of the test
+      target: Debug or Release
+    """
+    diff_filename = self.OutputFilename(filename,
+      self.FILENAME_SUFFIX_COMPARE)
+    actual_filename = self.OutputFilename(filename,
+      self.FILENAME_SUFFIX_ACTUAL + '.png')
+    expected_win_filename = self.OutputFilename(filename,
+      self.FILENAME_SUFFIX_EXPECTED + '.png')
+    platform_util = platform_utils.PlatformUtility('')
+
+    global _compare_available
+    cmd = ''
+
+    try:
+      executable = platform_util.ImageCompareExecutablePath(target)
+      cmd = [executable, '--diff', actual_filename, expected_win_filename,
+             diff_filename]
+    except Exception, e:
+      _compare_available = False
+
+    if _compare_available:
+      try:
+        subprocess.call(cmd)
+      except OSError, e:
+        if e.errno == errno.ENOENT or e.errno == errno.EACCES:
+          _compare_available = False
+        else:
+          raise e
+
+    global _compare_msg_printed
+
+    if not _compare_available and not _compare_msg_printed:
+      _compare_msg_printed = True
+      print('image_diff not found. Make sure you have a ' + target +
+            ' build of the image_diff executable.')
+
+  def CompareOutput(self, filename, proc, output, test_args, target):
     """Implementation of CompareOutput that checks the output image and
     checksum against the expected files from the LayoutTest directory.
     """
@@ -102,10 +149,11 @@ class ImageDiff(test_type_base.TestTypeBase):
 
     # If anything was wrong, write the output files.
     if len(failures):
-      self.WriteOutputFiles(filename, '', '.checksum', test_args.hash,
-                            expected_hash, diff=False)
       self._CopyOutputPNGs(filename, test_args.png_path,
-                          expected_png_file)
+                           expected_png_file)
+      self._CreateImageDiff(filename, target)
+      self.WriteOutputFiles(filename, '', '.checksum', test_args.hash,
+                            expected_hash, diff=False, wdiff=False)
 
     return failures
 
diff --git a/webkit/tools/layout_tests/test_types/simplified_text_diff.py b/webkit/tools/layout_tests/test_types/simplified_text_diff.py
index bd1de3e..00af6c1 100644
--- a/webkit/tools/layout_tests/test_types/simplified_text_diff.py
+++ b/webkit/tools/layout_tests/test_types/simplified_text_diff.py
@@ -83,7 +83,7 @@ class SimplifiedTextDiff(text_diff.TestTextDiff):
 
     return text
 
-  def CompareOutput(self, filename, proc, output, test_args):
+  def CompareOutput(self, filename, proc, output, test_args, target):
     """Implementation of CompareOutput that removes most numbers before
     computing the diff.
 
diff --git a/webkit/tools/layout_tests/test_types/test_type_base.py b/webkit/tools/layout_tests/test_types/test_type_base.py
index fd7558f..7750b6e 100644
--- a/webkit/tools/layout_tests/test_types/test_type_base.py
+++ b/webkit/tools/layout_tests/test_types/test_type_base.py
@@ -46,6 +46,7 @@ class TestTypeBase(object):
   FILENAME_SUFFIX_EXPECTED = "-expected"
   FILENAME_SUFFIX_DIFF = "-diff-win"
   FILENAME_SUFFIX_WDIFF = "-wdiff-win.html"
+  FILENAME_SUFFIX_COMPARE = "-diff-win.png"
 
   def __init__(self, platform, root_output_dir):
     """Initialize a TestTypeBase object.
@@ -119,7 +120,7 @@ class TestTypeBase(object):
     """
     return os.path.splitext(filename)[0] + modifier
 
-  def CompareOutput(self, filename, proc, output, test_args):
+  def CompareOutput(self, filename, proc, output, test_args, target):
     """Method that compares the output from the test with the expected value.
     
     This is an abstract method to be implemented by all sub classes.
@@ -129,6 +130,7 @@ class TestTypeBase(object):
       proc: a reference to the test_shell process
       output: a string containing the output of the test
       test_args: a TestArguments object holding optional additional arguments
+      target: Debug or Release
     
     Return:
       a list of TestFailure objects, empty if the test passes
diff --git a/webkit/tools/layout_tests/test_types/text_diff.py b/webkit/tools/layout_tests/test_types/text_diff.py
index 7cf6627..d23a912 100644
--- a/webkit/tools/layout_tests/test_types/text_diff.py
+++ b/webkit/tools/layout_tests/test_types/text_diff.py
@@ -45,7 +45,7 @@ class TestTextDiff(test_type_base.TestTypeBase):
     # Normalize line endings
     return expected.strip("\r\n").replace("\r\n", "\n") + "\n"
 
-  def CompareOutput(self, filename, proc, output, test_args):
+  def CompareOutput(self, filename, proc, output, test_args, target):
     """Implementation of CompareOutput that checks the output text against the
     expected text from the LayoutTest directory."""
     failures = []