Rebaseline tool update

- Do not add unnecessary baselines
  Check whether a baseline is duplicate and can fallback to
  same baseline for another platform. For example, if a test
  has same baseline on linux and windows, then we only store
  windows baseline and linux baseline will fallback to the
  windows version.
- Set new baseline svn property.

TEST=rebaseline tool
BUG=18906

Review URL: http://codereview.chromium.org/174567

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@25355 0039d316-1c4b-4281-b951-d872f2087c98

6 files changed, 245 insertions, 30 deletions

diff --git a/webkit/tools/layout_tests/layout_package/path_utils.py b/webkit/tools/layout_tests/layout_package/path_utils.py
index f2c0090..756ff0a 100644
--- a/webkit/tools/layout_tests/layout_package/path_utils.py
+++ b/webkit/tools/layout_tests/layout_package/path_utils.py
@@ -13,6 +13,9 @@ us including a few things that don't really have anything to do
 import errno
 import os
 import platform_utils
+import platform_utils_win
+import platform_utils_mac
+import platform_utils_linux
 
 # Cache some values so we don't have to recalculate them. _basedir is
 # used by PathFromBase() and caches the full (native) path to the top
@@ -49,8 +52,23 @@ def WebKitBaselinePath(platform):
   return PathFromBase('third_party', 'WebKit', 'LayoutTests',
                       'platform', platform)
 
+def BaselineSearchPath(platform=None):
+  """Returns the list of directories to search for baselines/results for a
+  given platform, in order of preference. Paths are relative to the top of the
+  source tree. If parameter platform is None, returns the list for the current
+  platform that the script is running on."""
+  if platform is None:
+    return platform_utils.BaselineSearchPath()
+  elif platform.startswith('mac'):
+    return platform_utils_mac.BaselineSearchPath()
+  elif platform.startswith('win'):
+    return platform_utils_win.BaselineSearchPath()
+  elif platform.startswith('linux'):
+    return platform_utils_linux.BaselineSearchPath()
+  else:
+    return platform_utils.BaselineSearchPath()
 
-def ExpectedBaseline(filename, suffix):
+def ExpectedBaseline(filename, suffix, platform=None, all_baselines=False):
   """Given a test name, finds where the baseline result is located. The
   result is returned as a pair of values, the absolute path to top of the test
   results directory, and the relative path from there to the results file.
@@ -65,13 +83,20 @@ def ExpectedBaseline(filename, suffix):
      filename: absolute filename to test file
      suffix: file suffix of the expected results, including dot; e.g. '.txt'
          or '.png'.  This should not be None, but may be an empty string.
+     platform: layout test platform: 'win', 'linux' or 'mac'. Defaults to the
+               current platform.
+     all_baselines: If True, return an ordered list of all baseline paths
+                    for the given platform. If False, return only the first
+                    one.
   Returns
-     platform_dir - abs path to the top of the results tree (or test tree)
-     results_filename - relative path from top of tree to the results file
+     a list of ( platform_dir, results_filename ), where
+       platform_dir - abs path to the top of the results tree (or test tree)
+       results_filename - relative path from top of tree to the results file
          (os.path.join of the two gives you the full path to the file, unless
           None was returned.)
   """
   global _baseline_search_path
+  global _search_path_platform
   testname = os.path.splitext(RelativeTestFilename(filename))[0]
 
   # While we still have tests in LayoutTests/, chrome/, and pending/, we need
@@ -80,24 +105,42 @@ def ExpectedBaseline(filename, suffix):
   platform_filename = testname + '-expected' + suffix
   testdir, base_filename = platform_filename.split('/', 1)
 
-  if _baseline_search_path is None:
-    _baseline_search_path = platform_utils.BaselineSearchPath()
+  if (_baseline_search_path is None) or (_search_path_platform != platform):
+    _baseline_search_path = BaselineSearchPath(platform)
+    _search_path_platform = platform
+
+  current_platform_dir = ChromiumBaselinePath(PlatformName(platform))
+
+  baselines = []
+  foundCurrentPlatform = False
   for platform_dir in _baseline_search_path:
-    # TODO(pamg): Clean this up once we upstream everything in chrome/ and
-    # pending/.
-    if os.path.basename(platform_dir).startswith('chromium'):
-      if os.path.exists(os.path.join(platform_dir, platform_filename)):
-        return platform_dir, platform_filename
-    else:
-      if os.path.exists(os.path.join(platform_dir, base_filename)):
-        return platform_dir, base_filename
+    # Find current platform from baseline search paths and start from there.
+    if platform_dir == current_platform_dir:
+      foundCurrentPlatform = True
+
+    if foundCurrentPlatform:
+      # TODO(pamg): Clean this up once we upstream everything in chrome/ and
+      # pending/.
+      if os.path.basename(platform_dir).startswith('chromium'):
+        if os.path.exists(os.path.join(platform_dir, platform_filename)):
+          baselines.append((platform_dir, platform_filename))
+      else:
+        if os.path.exists(os.path.join(platform_dir, base_filename)):
+          baselines.append((platform_dir, base_filename))
+
+      if not all_baselines and baselines:
+        return baselines
 
   # If it wasn't found in a platform directory, return the expected result
   # in the test directory, even if no such file actually exists.
   platform_dir = LayoutTestsDir(filename)
   if os.path.exists(os.path.join(platform_dir, platform_filename)):
-    return platform_dir, platform_filename
-  return None, platform_filename
+    baselines.append((platform_dir, platform_filename))
+
+  if baselines:
+    return baselines
+
+  return [(None, platform_filename)]
 
 def ExpectedFilename(filename, suffix):
   """Given a test name, returns an absolute path to its expected results.
@@ -115,7 +158,7 @@ def ExpectedFilename(filename, suffix):
          search list of directories, e.g., 'chromium-win', or
          'chromium-mac-leopard' (we follow the WebKit format)
   """
-  platform_dir, platform_filename = ExpectedBaseline(filename, suffix)
+  platform_dir, platform_filename = ExpectedBaseline(filename, suffix)[0]
   if platform_dir:
     return os.path.join(platform_dir, platform_filename)
   return os.path.join(LayoutTestsDir(filename), platform_filename)
diff --git a/webkit/tools/layout_tests/layout_package/platform_utils_mac.py b/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
index 5c3eb3d..51c8506 100644
--- a/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
+++ b/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
@@ -23,6 +23,9 @@ def PlatformVersion():
   '-snowleopard'. If the platform does not distinguish between
   minor versions, it returns ''."""
   os_version_string = platform.mac_ver()[0]  # e.g. "10.5.6"
+  if not os_version_string:
+    return '-leopard'
+
   release_version = int(os_version_string.split('.')[1])
 
   # we don't support 'tiger' or earlier releases
@@ -30,8 +33,11 @@ def PlatformVersion():
     return '-leopard'
   elif release_version == 6:
     return '-snowleopard'
+
   return ''
 
+# TODO: We should add leopard and snowleopard to the list of paths to check
+# once we start running the tests from snowleopard.
 def BaselineSearchPath():
   """Returns the list of directories to search for baselines/results, in
   order of preference. Paths are relative to the top of the source tree."""
diff --git a/webkit/tools/layout_tests/rebaseline.py b/webkit/tools/layout_tests/rebaseline.py
index 2c78828..8ea4edb 100644
--- a/webkit/tools/layout_tests/rebaseline.py
+++ b/webkit/tools/layout_tests/rebaseline.py
@@ -32,9 +32,11 @@ import zipfile
 
 from layout_package import path_utils
 from layout_package import test_expectations
+from test_types import image_diff
+from test_types import text_diff
 
 BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
-
+REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'linux']
 
 def RunShell(command, print_output=False):
   """Executes a command and returns the output.
@@ -354,6 +356,15 @@ class Rebaseliner(object):
         f.write(data)
         f.close()
 
+        # TODO(victorw): for now, the rebaselining tool checks whether
+        # or not THIS baseline is duplicate and should be skipped.
+        # We could improve the tool to check all baselines in upper and lower
+        # levels and remove all duplicated baselines.
+        if self._IsDupBaseline(expected_fullpath, test, suffix, self._platform):
+          # Clean up the duplicate baseline.
+          self._DeleteBaseline(expected_fullpath)
+          continue
+
         if not self._SvnAdd(expected_fullpath):
           svn_error = True
         elif suffix != '.checksum':
@@ -375,6 +386,82 @@ class Rebaseliner(object):
 
     return self._rebaselined_tests
 
+  def _IsDupBaseline(self, baseline_path, test, suffix, platform):
+    """Check whether a baseline is duplicate and can fallback to same
+       baseline for another platform. For example, if a test has same baseline
+       on linux and windows, then we only store windows baseline and linux
+       baseline will fallback to the windows version.
+
+    Args:
+      expected_filename: baseline expectation file name.
+      test: test name.
+      suffix: file suffix of the expected results, including dot; e.g. '.txt'
+              or '.png'.
+      platform: baseline platform 'mac', 'win' or 'linux'.
+
+    Returns:
+      True if the baseline is unnecessary.
+      False otherwise.
+    """
+    test_filepath = os.path.join(path_utils.LayoutTestsDir(), test)
+    all_baselines = path_utils.ExpectedBaseline(test_filepath,
+                                                suffix,
+                                                platform,
+                                                True)
+    for (fallback_dir, fallback_file) in all_baselines:
+      if fallback_dir and fallback_file:
+        fallback_fullpath = os.path.normpath(
+            os.path.join(fallback_dir, fallback_file))
+        if fallback_fullpath.lower() != baseline_path.lower():
+          if not self._DiffBaselines(baseline_path, fallback_fullpath):
+            logging.info('  Found same baseline at %s', fallback_fullpath)
+            return True
+          else:
+            return False
+
+    return False
+
+  def _DiffBaselines(self, file1, file2):
+    """Check whether two baselines are different.
+
+    Args:
+      file1, file2: full paths of the baselines to compare.
+
+    Returns:
+      True if two files are different or have different extensions.
+      False otherwise.
+    """
+
+    ext1 = os.path.splitext(file1)[1].upper()
+    ext2 = os.path.splitext(file2)[1].upper()
+    if ext1 != ext2:
+      logging.warn('Files to compare have different ext. File1: %s; File2: %s',
+                   file1, file2)
+      return True
+
+    if ext1 == '.PNG':
+      return image_diff.ImageDiff(self._platform, '').DiffFiles(file1,
+                                                                file2)
+    else:
+      return text_diff.TestTextDiff(self._platform, '').DiffFiles(file1,
+                                                                  file2)
+
+  def _DeleteBaseline(self, filename):
+    """Remove the file from SVN repository and delete it from disk.
+
+    Args:
+      filename: full path of the file to delete.
+    """
+
+    if not filename:
+      return
+
+    parent_dir, basename = os.path.split(filename)
+    original_dir = os.getcwd()
+    os.chdir(parent_dir)
+    status_output = RunShell(['svn', 'delete', '--force', basename], False)
+    os.chdir(original_dir)
+
   def _UpdateRebaselinedTestsInFile(self, backup):
     """Update the rebaselined tests in test expectations file.
 
@@ -432,6 +519,7 @@ class Rebaseliner(object):
     output = add_output.upper().rstrip()
     if output.startswith('A') and output.find(basename.upper()) >= 0:
       logging.info('  Added new file: "%s"', filename)
+      self._SvnPropSet(filename)
       return True
 
     if (not status_output) and (add_output.upper().find(
@@ -444,6 +532,32 @@ class Rebaseliner(object):
     logging.warn('  Svn add output: "%s"', add_output)
     return False
 
+  def _SvnPropSet(self, filename):
+    """Set the baseline property
+
+    Args:
+      filename: full path of the file to add.
+
+    Returns:
+      True if the file already exists in SVN or is sucessfully added to SVN.
+      False otherwise.
+    """
+    ext = os.path.splitext(filename)[1].upper()
+    if ext != '.TXT' and ext != '.PNG' and ext != '.CHECKSUM':
+      return
+
+    parent_dir, basename = os.path.split(filename)
+    original_dir = os.getcwd()
+    os.chdir(parent_dir)
+    if ext == '.PNG':
+      cmd = [ 'svn', 'pset', 'svn:mime-type', 'image/png', basename ]
+    else:
+      cmd = [ 'svn', 'pset', 'svn:eol-style', 'LF', basename ]
+
+    logging.debug('  Set svn prop: %s', ' '.join(cmd))
+    RunShell(cmd, False)
+    os.chdir(original_dir)
+
   def _CreateHtmlBaselineFiles(self, baseline_fullpath):
     """Create baseline files (old, new and diff) in html directory.
 
@@ -731,7 +845,7 @@ def main():
                            help='include debug-level logging.')
 
   option_parser.add_option('-p', '--platforms',
-                           default='win,mac,linux',
+                           default='mac,win,linux',
                            help=('Comma delimited list of platforms that need '
                                  'rebaselining.'))
 
@@ -803,17 +917,26 @@ def main():
     sys.exit(1)
   platforms = [p.strip().lower() for p in options.platforms.split(',')]
   for platform in platforms:
-    if not platform in test_expectations.TestExpectationsFile.PLATFORMS:
-      logging.error('Invalid platform platform: "%s"' % (platform))
+    if not platform in REBASELINE_PLATFORM_ORDER:
+      logging.error('Invalid platform: "%s"' % (platform))
       sys.exit(1)
 
+  # Adjust the platform order so rebaseline tool is running at the order of
+  # 'mac', 'win' and 'linux'. This is in same order with layout test baseline
+  # search paths. It simplifies how the rebaseline tool detects duplicate
+  # baselines. Check _IsDupBaseline method for details.
+  rebaseline_platforms = []
+  for platform in REBASELINE_PLATFORM_ORDER:
+    if platform in platforms:
+      rebaseline_platforms.append(platform)
+
   if not options.no_html_results:
     options.html_directory = SetupHtmlDirectory(options.html_directory,
                                                 options.clean_html_directory)
 
   rebaselining_tests = set()
   backup = options.backup
-  for platform in platforms:
+  for platform in rebaseline_platforms:
     rebaseliner = Rebaseliner(platform, options)
 
     logging.info('')
@@ -830,7 +953,9 @@ def main():
   if not options.no_html_results:
     logging.info('')
     LogDashedString('Rebaselining result comparison started', None)
-    html_generator = HtmlGenerator(options, platforms, rebaselining_tests)
+    html_generator = HtmlGenerator(options,
+                                   rebaseline_platforms,
+                                   rebaselining_tests)
     html_generator.GenerateHtml()
     html_generator.ShowHtml()
     LogDashedString('Rebaselining result comparison done', None)
diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py
index 97792aa..00191fa 100755
--- a/webkit/tools/layout_tests/run_webkit_tests.py
+++ b/webkit/tools/layout_tests/run_webkit_tests.py
@@ -1029,12 +1029,12 @@ def main(options, args):
     print("html,txt,checksum,png");
     for t in test_runner._test_files:
       (expected_txt_dir, expected_txt_file) = path_utils.ExpectedBaseline(
-          t, '.txt')
+          t, '.txt')[0]
       (expected_png_dir, expected_png_file) = path_utils.ExpectedBaseline(
-          t, '.png')
+          t, '.png')[0]
       (expected_checksum_dir,
           expected_checksum_file) = path_utils.ExpectedBaseline(
-          t, '.checksum')
+          t, '.checksum')[0]
       print("%s,%s,%s,%s" % (path_utils.RelativeTestFilename(t),
             expected_txt_dir, expected_checksum_dir, expected_png_dir))
     return
diff --git a/webkit/tools/layout_tests/test_types/image_diff.py b/webkit/tools/layout_tests/test_types/image_diff.py
index d8f0a16..a14cb4c 100644
--- a/webkit/tools/layout_tests/test_types/image_diff.py
+++ b/webkit/tools/layout_tests/test_types/image_diff.py
@@ -161,3 +161,30 @@ class ImageDiff(test_type_base.TestTypeBase):
                           expected_hash, diff=False, wdiff=False)
 
     return failures
+
+  def DiffFiles(self, file1, file2):
+    """Diff two image files.
+
+    Args:
+      file1, file2: full paths of the files to compare.
+
+    Returns:
+      True if two files are different.
+      False otherwise.
+    """
+
+    try:
+      executable = path_utils.ImageDiffPath('Debug')
+    except Exception, e:
+      logging.warn('Failed to find image diff executable.')
+      return True
+
+    cmd = [ executable, file1, file2 ]
+    result = 1
+    try:
+      result = subprocess.call(cmd);
+    except OSError, e:
+      logging.warn('Failed to compare image diff: %s', e)
+      return True
+
+    return result == 1
diff --git a/webkit/tools/layout_tests/test_types/text_diff.py b/webkit/tools/layout_tests/test_types/text_diff.py
index 576d03d..0ae3c94 100644
--- a/webkit/tools/layout_tests/test_types/text_diff.py
+++ b/webkit/tools/layout_tests/test_types/text_diff.py
@@ -34,19 +34,21 @@ class TestTextDiff(test_type_base.TestTypeBase):
     if the expected output file was not found."""
     # Read the platform-specific expected text.
     expected_filename = path_utils.ExpectedFilename(filename, '.txt')
-
     if show_sources:
       logging.debug('Using %s' % expected_filename)
+
+    return self.GetNormalizedText(expected_filename)
+
+  def GetNormalizedText(self, filename):
     try:
-      expected = open(expected_filename).read()
+      text = open(filename).read()
     except IOError, e:
       if errno.ENOENT != e.errno:
         raise
-      expected = ''
-      return expected
+      return ''
 
     # Normalize line endings
-    return expected.strip("\r\n").replace("\r\n", "\n") + "\n"
+    return text.strip("\r\n").replace("\r\n", "\n") + "\n"
 
   def _SaveBaselineData(self, filename, data, modifier):
     """Saves a new baseline file into the platform directory.
@@ -105,3 +107,15 @@ class TestTextDiff(test_type_base.TestTypeBase):
 
     return failures
 
+  def DiffFiles(self, file1, file2):
+    """Diff two text files.
+
+    Args:
+      file1, file2: full paths of the files to compare.
+
+    Returns:
+      True if two files are different.
+      False otherwise.
+    """
+
+    return self.GetNormalizedText(file1) != self.GetNormalizedText(file2)