Rebaselining tool that automatically produces baselines for all platforms.

The script does the following for each platform specified:
  1. Compile a list of tests that need rebaseline.
  2. Download test result archive from buildbot for the platform.
  3. Extract baselines from the archive file for all identified files.
  4. Add new baselines to SVN repository.
  5. For each test that has been rebaselined, remove this platform option from
     the test in test_expectation.txt. If no other platforms remaining after
     removal, delete the rebaselined test from the file.

BUG=11744

Review URL: http://codereview.chromium.org/115666

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@17317 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 603 insertions, 7 deletions

diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py
index 7ba9ace..3f1feb6 100644
--- a/webkit/tools/layout_tests/layout_package/test_expectations.py
+++ b/webkit/tools/layout_tests/layout_package/test_expectations.py
@@ -10,12 +10,18 @@ import logging
 import os
 import re
 import sys
+import time
 import path_utils
 import compare_failures
 
 
 # Test expectation and modifier constants.
-(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, NONE) = range(9)
+(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, REBASELINE, NONE) = \
+    range(10)
+
+# Test expectation file update action constants
+(NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4)
+
 
 class TestExpectations:
   TEST_LIST = "test_expectations.txt"
@@ -95,6 +101,9 @@ class TestExpectations:
             self._expected_failures.GetTestSet(WONTFIX, CRASH,
                 include_skips=False))
 
+  def GetRebaseliningFailures(self):
+    return self._expected_failures.GetTestSet(REBASELINE, FAIL)
+
   def GetExpectations(self, test):
     if self._expected_failures.Contains(test):
       return self._expected_failures.GetExpectations(test)
@@ -115,6 +124,11 @@ class TestExpectations:
   def HasModifier(self, test, modifier):
     return self._expected_failures.HasModifier(test, modifier)
 
+  def RemovePlatformFromFile(self, tests, platform, backup=False):
+    return self._expected_failures.RemovePlatformFromFile(tests,
+                                                          platform,
+                                                          backup)
+
 def StripComments(line):
   """Strips comments from a line and return None if the line is empty
   or else the contents of line with leading and trailing spaces removed
@@ -169,13 +183,14 @@ class TestExpectationsFile:
                    'crash': CRASH }
 
   PLATFORMS = [ 'mac', 'linux', 'win' ]
-  
+
   BUILD_TYPES = [ 'debug', 'release' ]
 
   MODIFIERS = { 'skip': SKIP,
                 'wontfix': WONTFIX,
                 'defer': DEFER,
                 'slow': SLOW,
+                'rebaseline': REBASELINE,
                 'none': NONE }
 
   def __init__(self, path, full_test_list, platform, is_debug_mode):
@@ -188,12 +203,13 @@ class TestExpectationsFile:
     is_debug_mode: Whether we testing a test_shell built debug mode.
     """
 
+    self._path = path
     self._full_test_list = full_test_list
     self._errors = []
     self._non_fatal_errors = []
     self._platform = platform
     self._is_debug_mode = is_debug_mode
-    
+
     # Maps a test to its list of expectations.
     self._test_to_expectations = {}
 
@@ -218,10 +234,10 @@ class TestExpectationsFile:
     else:
       tests = (self._expectation_to_tests[expectation] &
           self._modifier_to_tests[modifier])
-    
+
     if not include_skips:
       tests = tests - self.GetTestSet(SKIP, expectation)
-    
+
     return tests
 
   def HasModifier(self, test, modifier):
@@ -233,9 +249,133 @@ class TestExpectationsFile:
   def Contains(self, test):
     return test in self._test_to_expectations
 
+  def RemovePlatformFromFile(self, tests, platform, backup=False):
+    """Remove the platform option from test expectations file.
+
+    If a test is in the test list and has an option that matches the given
+    platform, remove the matching platform and save the updated test back
+    to the file. If no other platforms remaining after removal, delete the
+    test from the file.
+
+    Args:
+      tests: list of tests that need to update..
+      platform: which platform option to remove.
+      backup: if true, the original test expectations file is saved as
+              [self.TEST_LIST].orig.YYYYMMDDHHMMSS
+
+    Returns:
+      no
+    """
+
+    new_file = self._path + '.new'
+    logging.debug('Original file: "%s"', self._path)
+    logging.debug('New file: "%s"', new_file)
+    f_orig = open(self._path)
+    f_new = open(new_file, 'w')
+
+    tests_removed = 0
+    tests_updated = 0
+    for line in f_orig:
+      action = self._GetPlatformUpdateAction(line, tests, platform)
+      if action == NO_CHANGE:
+        # Save the original line back to the file
+        logging.debug('No change to test: %s', line)
+        f_new.write(line)
+      elif action == REMOVE_TEST:
+        tests_removed += 1
+        logging.info('Test removed: %s', line)
+      elif action == REMOVE_PLATFORM:
+        parts = line.split(':')
+        new_options = parts[0].replace(platform.upper() + ' ', '', 1)
+        new_line = ('%s:%s' % (new_options, parts[1]))
+        f_new.write(new_line)
+        tests_updated += 1
+        logging.info('Test updated: ')
+        logging.info('  old: %s', line)
+        logging.info('  new: %s', new_line)
+      elif action == ADD_PLATFORMS_EXCEPT_THIS:
+        parts = line.split(':')
+        new_options = parts[0]
+        for p in self.PLATFORMS:
+          if not p == platform:
+            new_options += p.upper() + ' '
+        new_line = ('%s:%s' % (new_options, parts[1]))
+        f_new.write(new_line)
+        tests_updated += 1
+        logging.info('Test updated: ')
+        logging.info('  old: %s', line)
+        logging.info('  new: %s', new_line)
+      else:
+        logging.error('Unknown update action: %d; line: %s', action, line)
+
+    logging.info('Total tests removed: %d', tests_removed)
+    logging.info('Total tests updated: %d', tests_updated)
+
+    f_orig.close()
+    f_new.close()
+
+    if backup:
+      date_suffix = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
+      backup_file = ('%s.orig.%s' % (self._path, date_suffix))
+      if os.path.exists(backup_file):
+        os.remove(backup_file)
+      logging.info('Saving original file to "%s"', backup_file)
+      os.rename(self._path, backup_file)
+    else:
+      os.remove(self._path)
+
+    logging.debug('Saving new file to "%s"', self._path)
+    os.rename(new_file, self._path)
+    return True
+
+  def _GetPlatformUpdateAction(self, line, tests, platform):
+    """Check the platform option and return the action needs to be taken.
+
+    Args:
+      line: current line in test expectations file.
+      tests: list of tests that need to update..
+      platform: which platform option to remove.
+
+    Returns:
+      NO_CHANGE: no change to the line (comments, test not in the list etc)
+      REMOVE_TEST: remove the test from file.
+      REMOVE_PLATFORM: remove this platform option from the test.
+      ADD_PLATFORMS_EXCEPT_THIS: add all the platforms except this one.
+    """
+
+    line = StripComments(line)
+    if not line:
+      return NO_CHANGE
+
+    options = []
+    if line.find(':') is -1:
+      test_and_expecation = line.split('=')
+    else:
+      parts = line.split(':')
+      options = self._GetOptionsList(parts[0])
+      test_and_expecation = parts[1].split('=')
+
+    test = test_and_expecation[0].strip()
+    if not test in tests:
+      return NO_CHANGE
+
+    has_any_platform = False
+    for option in options:
+      if option in self.PLATFORMS:
+        has_any_platform = True
+        if not option == platform:
+          return REMOVE_PLATFORM
+
+    # If there is no platform specified, then it means apply to all platforms.
+    # Return the action to add all the platforms except this one.
+    if not has_any_platform:
+      return ADD_PLATFORMS_EXCEPT_THIS
+
+    return REMOVE_TEST
+
   def _HasValidModifiersForCurrentPlatform(self, options, lineno,
       test_and_expectations, modifiers):
-    """ Returns true if the current platform is in the options list or if no
+    """Returns true if the current platform is in the options list or if no
     platforms are listed and if there are no fatal errors in the options list.
 
     Args:
@@ -372,7 +512,7 @@ class TestExpectationsFile:
     for test in self._full_test_list:
       if test.startswith(path): result.append(test)
     return result
-    
+
   def _AddTests(self, tests, expectations, test_list_path, lineno, modifiers):
     for test in tests:
       if self._AlreadySeenTest(test, test_list_path, lineno):
diff --git a/webkit/tools/layout_tests/rebaseline.bat b/webkit/tools/layout_tests/rebaseline.bat
new file mode 100644
index 0000000..341b66d
--- /dev/null
+++ b/webkit/tools/layout_tests/rebaseline.bat
@@ -0,0 +1 @@
+%~dp0..\..\..\third_party\python_24\python.exe %~dp0rebaseline.py %*
diff --git a/webkit/tools/layout_tests/rebaseline.py b/webkit/tools/layout_tests/rebaseline.py
new file mode 100644
index 0000000..5e6e48d
--- /dev/null
+++ b/webkit/tools/layout_tests/rebaseline.py
@@ -0,0 +1,436 @@
+#!/bin/env python

+# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.

+# Use of this source code is governed by a BSD-style license that can be

+# found in the LICENSE file.

+

+"""Rebaselining tool that automatically produces baselines for all platforms.

+

+The script does the following for each platform specified:

+  1. Compile a list of tests that need rebaselining.

+  2. Download test result archive from buildbot for the platform.

+  3. Extract baselines from the archive file for all identified files.

+  4. Add new baselines to SVN repository.

+  5. For each test that has been rebaselined, remove this platform option from

+     the test in test_expectation.txt. If no other platforms remain after

+     removal, delete the rebaselined test from the file.

+"""

+

+import logging

+import optparse

+import os

+import re

+import subprocess

+import sys

+import urllib

+import zipfile

+

+from layout_package import path_utils

+from layout_package import platform_utils_linux

+from layout_package import platform_utils_mac

+from layout_package import platform_utils_win

+from layout_package import test_expectations

+

+

+def RunShell(command, print_output=False):

+  """Executes a command and returns the output.

+

+  Args:

+    command: program and arguments.

+    print_output: if true, print the command results to standard output.

+

+  Returns:

+    command output

+  """

+

+  # Use a shell for subcommands on Windows to get a PATH search.

+  use_shell = sys.platform.startswith('win')

+  p = subprocess.Popen(command, stdout=subprocess.PIPE,

+                       stderr=subprocess.STDOUT, shell=use_shell,

+                       universal_newlines=True)

+  if print_output:

+    output_array = []

+    while True:

+      line = p.stdout.readline()

+      if not line:

+        break

+      if print_output:

+        print line.strip('\n')

+      output_array.append(line)

+    output = ''.join(output_array)

+  else:

+    output = p.stdout.read()

+  p.wait()

+  p.stdout.close()

+  return output

+

+

+def LogDashedString(text, platform, logging_level=logging.INFO):

+  """Log text message with dashes on both sides."""

+

+  msg = '%s: %s' % (text, platform)

+  if len(msg) < 78:

+    dashes = '-' * ((78 - len(msg)) / 2)

+    msg = '%s %s %s' % (dashes, msg, dashes)

+

+  if logging_level == logging.ERROR:

+    logging.error(msg)

+  elif logging_level == logging.WARNING:

+    logging.warn(msg)

+  else:

+    logging.info(msg)

+

+

+class Rebaseliner(object):

+  """Class to produce new baselines for a given platform."""

+

+  BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']

+  REVISION_REGEX = r'<a href=\"(\d+)/\">'

+

+  def __init__(self, platform, options):

+    self._file_dir = path_utils.GetAbsolutePath(os.path.dirname(sys.argv[0]))

+    self._platform = platform

+    self._options = options

+

+    # Create tests and expectations helper which is used to:

+    #   -. compile list of tests that need rebaselining.

+    #   -. update the tests in test_expectations file after rebaseline is done.

+    self._test_expectations = test_expectations.TestExpectations(None,

+                                                                 self._file_dir,

+                                                                 platform,

+                                                                 False)

+

+  def Run(self, backup):

+    """Run rebaseline process."""

+

+    LogDashedString('Compiling rebaselining tests', self._platform)

+    rebaselining_tests = self._CompileRebaseliningTests()

+    logging.info('')

+    if rebaselining_tests is None:

+      return True

+

+    LogDashedString('Downloading archive', self._platform)

+    archive_file = self._DownloadBuildBotArchive()

+    logging.info('')

+    if not archive_file:

+      logging.error('No archive found.')

+      return False

+

+    rebaselined_tests = []

+    LogDashedString('Extracting and adding new baselines', self._platform)

+    rebaselined_tests = self._ExtractAndAddNewBaselines(archive_file,

+                                                        rebaselining_tests)

+    logging.info('')

+

+    LogDashedString('Updating rebaselined tests in file', self._platform)

+    self._UpdateRebaselinedTestsInFile(rebaselined_tests, backup)

+    logging.info('')

+

+    if len(rebaselining_tests) != len(rebaselined_tests):

+      logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '

+                      'REBASELINED.')

+      logging.warning('  Total tests needing rebaselining: %d',

+                      len(rebaselining_tests))

+      logging.warning('  Total tests rebaselined: %d',

+                      len(rebaselined_tests))

+      return False

+

+    logging.warning('All tests needing rebaselining were successfully '

+                    'rebaselined.')

+

+    return True

+

+  def _CompileRebaseliningTests(self):

+    """Compile list of tests that need rebaselining for the platform.

+

+    Returns:

+      List of tests that need rebaselining or

+      None if there is no such test.

+    """

+

+    rebaselining_tests = self._test_expectations.GetRebaseliningFailures()

+    if not rebaselining_tests:

+      logging.warn('No tests found that need rebaselining.')

+      return None

+

+    logging.info('Total number of tests needing rebaselining for "%s": "%d"',

+                 self._platform, len(rebaselining_tests))

+

+    test_no = 1

+    for test in rebaselining_tests:

+      logging.info('  %d: %s', test_no, test)

+      test_no += 1

+

+    return rebaselining_tests

+

+  def _GetLatestRevision(self, url):

+    """Get the latest layout test revision number from buildbot.

+

+    Args:

+      url: Url to retrieve layout test revision numbers.

+

+    Returns:

+      latest revision or

+      None on failure.

+    """

+

+    logging.debug('Url to retrieve revision: "%s"', url)

+

+    f = urllib.urlopen(url)

+    content = f.read()

+    f.close()

+

+    revisions = re.findall(self.REVISION_REGEX, content)

+    if not revisions:

+      logging.error('Failed to find revision, content: "%s"', content)

+      return None

+

+    revisions.sort(key=int)

+    logging.info('Latest revision: "%s"', revisions[len(revisions) - 1])

+    return revisions[len(revisions) - 1]

+

+  def _GetArchiveUrl(self):

+    """Generate the url to download latest layout test archive.

+

+    Returns:

+      Url to download archive or

+      None on failure

+    """

+

+    platform_name = self._options.buildbot_platform_dir_basename

+    if not self._platform == 'win':

+      platform_name += '-' + self._platform

+    logging.debug('Buildbot platform dir name: "%s"', platform_name)

+

+    url_base = '%s/%s/' % (self._options.archive_url, platform_name)

+    latest_revision = self._GetLatestRevision(url_base)

+    if latest_revision is None or latest_revision <= 0:

+      return None

+

+    archive_url = ('%s%s/%s.zip' % (url_base,

+                                    latest_revision,

+                                    self._options.archive_name))

+    logging.info('Archive url: "%s"', archive_url)

+    return archive_url

+

+  def _DownloadBuildBotArchive(self):

+    """Download layout test archive file from buildbot.

+

+    Returns:

+      True if download succeeded or

+      False otherwise.

+    """

+

+    url = self._GetArchiveUrl()

+    if url is None:

+      return None

+

+    fn = urllib.urlretrieve(url)[0]

+    logging.info('Archive downloaded and saved to file: "%s"', fn)

+    return fn

+

+  def _GetPlatformNewResultsDir(self):

+    """Get the dir name to extract new baselines for the given platform."""

+

+    if self._platform == 'win':

+      return platform_utils_win.PlatformUtility(None).PlatformNewResultsDir()

+    elif self._platform == 'mac':

+      return platform_utils_mac.PlatformUtility(None).PlatformNewResultsDir()

+    elif self._platform == 'linux':

+      return platform_utils_linux.PlatformUtility(None).PlatformNewResultsDir()

+

+    return None

+

+  def _ExtractAndAddNewBaselines(self, archive_file, rebaselining_tests):

+    """Extract new baselines from archive and add them to SVN repository.

+

+    Args:

+      archive_file: full path to the archive file.

+      rebaselining_tests: list of tests that need rebaselining.

+

+    Returns:

+      List of tests that have been rebaselined or

+      None on failure.

+    """

+

+    zip_file = zipfile.ZipFile(archive_file, 'r')

+    zip_namelist = zip_file.namelist()

+

+    logging.debug('zip file namelist:')

+    for name in zip_namelist:

+      logging.debug('  ' + name)

+

+    platform_dir = self._GetPlatformNewResultsDir()

+    if not platform_dir:

+      logging.error('Invalid platform new results dir, platform: "%s"',

+                    self._platform)

+      return None

+

+    logging.debug('Platform new results dir: "%s"', platform_dir)

+

+    test_no = 1

+    rebaselined_tests = []

+    for test in rebaselining_tests:

+      logging.info('Test %d: %s', test_no, test)

+

+      found = False

+      svn_error = False

+      test_basename = os.path.splitext(test)[0]

+      for suffix in self.BASELINE_SUFFIXES:

+        archive_test_name = '%s/%s-actual%s' % (self._options.archive_name,

+                                                test_basename,

+                                                suffix)

+        logging.debug('  Archive test file name: "%s"', archive_test_name)

+        if not archive_test_name in zip_namelist:

+          logging.info('  %s file not in archive.', suffix)

+          continue

+

+        found = True

+        logging.info('  %s file found in archive.', suffix)

+

+        expected_filename = '%s-expected%s' % (test_basename, suffix)

+        expected_fullpath = os.path.join(

+            path_utils.ChromiumPlatformResultsEnclosingDir(),

+            platform_dir,

+            expected_filename)

+        expected_fullpath = os.path.normcase(expected_fullpath)

+        logging.debug('  Expected file full path: "%s"', expected_fullpath)

+

+        data = zip_file.read(archive_test_name)

+        f = open(expected_fullpath, 'wb')

+        f.write(data)

+        f.close()

+

+        if not self._SvnAdd(expected_fullpath):

+          svn_error = True

+

+      if not found:

+        logging.warn('  No new baselines found in archive.')

+      else:

+        if svn_error:

+          logging.warn('  Failed to add baselines to SVN.')

+        else:

+          logging.info('  Rebaseline succeeded.')

+          rebaselined_tests.append(test)

+

+      test_no += 1

+

+    zip_file.close()

+    os.remove(archive_file)

+

+    return rebaselined_tests

+

+  def _UpdateRebaselinedTestsInFile(self, rebaselined_tests, backup):

+    """Update the rebaselined tests in test expectations file.

+

+    Args:

+      rebaselined_tests: list of tests that have been rebaselined.

+      backup: if True, backup the original test expectations file.

+

+    Returns:

+      no

+    """

+

+    if rebaselined_tests:

+      self._test_expectations.RemovePlatformFromFile(rebaselined_tests,

+                                                     self._platform,

+                                                     backup)

+    else:

+      logging.info('No test was rebaselined so nothing to remove.')

+

+  def _SvnAdd(self, filename):

+    """Add the file to SVN repository.

+

+    Args:

+      filename: full path of the file to add.

+

+    Returns:

+      True if the file already exists in SVN or is sucessfully added to SVN.

+      False otherwise.

+    """

+

+    output = RunShell(['svn', 'status', filename], False)

+    logging.debug('  Svn status output: "%s"', output)

+    if output.startswith('A') or output.startswith('M'):

+      logging.info('  File already added to SVN: "%s"', filename)

+      return True

+

+    output = RunShell(['svn', 'add', filename], True)

+    logging.debug('  Svn add output: "%s"', output)

+    if output.startswith('A') and output.endswith(filename):

+      logging.info('  Added new file: "%s"', filename)

+      return True

+

+    logging.warn('  Failed to add file to SVN: "%s"', filename)

+    return False

+

+

+def main():

+  """Main function to produce new baselines."""

+

+  option_parser = optparse.OptionParser()

+  option_parser.add_option('-v', '--verbose',

+                           action='store_true',

+                           default=False,

+                           help='include debug-level logging.')

+

+  option_parser.add_option('-p', '--platforms',

+                           default='win,mac,linux',

+                           help=('Comma delimited list of platforms that need '

+                                 'rebaselining.'))

+

+  option_parser.add_option('-u', '--archive_url',

+                           default=('http://build.chromium.org/buildbot/'

+                                    'layout_test_results'),

+                           help=('Url to find the layout test result archive '

+                                 'file.'))

+

+  option_parser.add_option('-t', '--archive_name',

+                           default='layout-test-results',

+                           help='Layout test result archive name.')

+

+  option_parser.add_option('-n', '--buildbot_platform_dir_basename',

+                           default='webkit-rel',

+                           help=('Base name of buildbot platform directory '

+                                 'that stores the layout test results.'))

+

+  option_parser.add_option('-b', '--backup',

+                           action='store_true',

+                           default=False,

+                           help=('Whether or not to backup the original test '

+                                 'expectations file after rebaseline.'))

+

+  options = option_parser.parse_args()[0]

+

+  # Set up our logging format.

+  log_level = logging.INFO

+  if options.verbose:

+    log_level = logging.DEBUG

+  logging.basicConfig(level=log_level,

+                      format=('%(asctime)s %(filename)s:%(lineno)-3d '

+                              '%(levelname)s %(message)s'),

+                      datefmt='%y%m%d %H:%M:%S')

+

+  # Verify 'platforms' option is valid

+  if not options.platforms:

+    logging.error('Invalid "platforms" option. --platforms must be specified '

+                  'in order to rebaseline.')

+    sys.exit(1)

+

+  backup = options.backup

+  platforms = [p.strip().lower() for p in options.platforms.split(',')]

+  for platform in platforms:

+    rebaseliner = Rebaseliner(platform, options)

+

+    logging.info('')

+    LogDashedString('Rebaseline started', platform)

+    if rebaseliner.Run(backup):

+      # Only need to backup one original copy of test expectation file.

+      backup = False

+      LogDashedString('Rebaseline done', platform)

+    else:

+      LogDashedString('Rebaseline failed', platform, logging.ERROR)

+

+  sys.exit(0)

+

+if '__main__' == __name__:

+  main()

diff --git a/webkit/tools/layout_tests/rebaseline.sh b/webkit/tools/layout_tests/rebaseline.sh
new file mode 100644
index 0000000..1499bc5
--- /dev/null
+++ b/webkit/tools/layout_tests/rebaseline.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+exec_dir=$(dirname $0)
+
+if [ "$OSTYPE" = "cygwin" ]; then
+  system_root=`cygpath "$SYSTEMROOT"`
+  PATH="/usr/bin:$system_root/system32:$system_root:$system_root/system32/WBEM"
+  export PATH
+  PYTHON_PROG="$exec_dir/../../../third_party/python_24/python.exe"
+else
+  PYTHON_PROG=python
+  # When not using the included python, we don't get automatic site.py paths.
+  # Specifically, run_webkit_tests needs the paths in:
+  # third_party/python_24/Lib/site-packages/google.pth
+  PYTHONPATH="${exec_dir}/../../../tools/python:$PYTHONPATH"
+  export PYTHONPATH
+fi
+
+"$PYTHON_PROG" "$exec_dir/rebaseline.py" "$@"