diff options
| -rw-r--r-- | webkit/tools/layout_tests/layout_package/test_expectations.py | 154 | ||||
| -rw-r--r-- | webkit/tools/layout_tests/rebaseline.bat | 1 | ||||
| -rw-r--r-- | webkit/tools/layout_tests/rebaseline.py | 436 | ||||
| -rw-r--r-- | webkit/tools/layout_tests/rebaseline.sh | 19 |
4 files changed, 603 insertions, 7 deletions
diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py index 7ba9ace..3f1feb6 100644 --- a/webkit/tools/layout_tests/layout_package/test_expectations.py +++ b/webkit/tools/layout_tests/layout_package/test_expectations.py @@ -10,12 +10,18 @@ import logging import os import re import sys +import time import path_utils import compare_failures # Test expectation and modifier constants. -(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, NONE) = range(9) +(PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, SLOW, REBASELINE, NONE) = \ + range(10) + +# Test expectation file update action constants +(NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4) + class TestExpectations: TEST_LIST = "test_expectations.txt" @@ -95,6 +101,9 @@ class TestExpectations: self._expected_failures.GetTestSet(WONTFIX, CRASH, include_skips=False)) + def GetRebaseliningFailures(self): + return self._expected_failures.GetTestSet(REBASELINE, FAIL) + def GetExpectations(self, test): if self._expected_failures.Contains(test): return self._expected_failures.GetExpectations(test) @@ -115,6 +124,11 @@ class TestExpectations: def HasModifier(self, test, modifier): return self._expected_failures.HasModifier(test, modifier) + def RemovePlatformFromFile(self, tests, platform, backup=False): + return self._expected_failures.RemovePlatformFromFile(tests, + platform, + backup) + def StripComments(line): """Strips comments from a line and return None if the line is empty or else the contents of line with leading and trailing spaces removed @@ -169,13 +183,14 @@ class TestExpectationsFile: 'crash': CRASH } PLATFORMS = [ 'mac', 'linux', 'win' ] - + BUILD_TYPES = [ 'debug', 'release' ] MODIFIERS = { 'skip': SKIP, 'wontfix': WONTFIX, 'defer': DEFER, 'slow': SLOW, + 'rebaseline': REBASELINE, 'none': NONE } def __init__(self, path, full_test_list, platform, is_debug_mode): @@ -188,12 +203,13 @@ class TestExpectationsFile: is_debug_mode: Whether we testing a test_shell built debug mode. """ + self._path = path self._full_test_list = full_test_list self._errors = [] self._non_fatal_errors = [] self._platform = platform self._is_debug_mode = is_debug_mode - + # Maps a test to its list of expectations. self._test_to_expectations = {} @@ -218,10 +234,10 @@ class TestExpectationsFile: else: tests = (self._expectation_to_tests[expectation] & self._modifier_to_tests[modifier]) - + if not include_skips: tests = tests - self.GetTestSet(SKIP, expectation) - + return tests def HasModifier(self, test, modifier): @@ -233,9 +249,133 @@ class TestExpectationsFile: def Contains(self, test): return test in self._test_to_expectations + def RemovePlatformFromFile(self, tests, platform, backup=False): + """Remove the platform option from test expectations file. + + If a test is in the test list and has an option that matches the given + platform, remove the matching platform and save the updated test back + to the file. If no other platforms remaining after removal, delete the + test from the file. + + Args: + tests: list of tests that need to update.. + platform: which platform option to remove. + backup: if true, the original test expectations file is saved as + [self.TEST_LIST].orig.YYYYMMDDHHMMSS + + Returns: + no + """ + + new_file = self._path + '.new' + logging.debug('Original file: "%s"', self._path) + logging.debug('New file: "%s"', new_file) + f_orig = open(self._path) + f_new = open(new_file, 'w') + + tests_removed = 0 + tests_updated = 0 + for line in f_orig: + action = self._GetPlatformUpdateAction(line, tests, platform) + if action == NO_CHANGE: + # Save the original line back to the file + logging.debug('No change to test: %s', line) + f_new.write(line) + elif action == REMOVE_TEST: + tests_removed += 1 + logging.info('Test removed: %s', line) + elif action == REMOVE_PLATFORM: + parts = line.split(':') + new_options = parts[0].replace(platform.upper() + ' ', '', 1) + new_line = ('%s:%s' % (new_options, parts[1])) + f_new.write(new_line) + tests_updated += 1 + logging.info('Test updated: ') + logging.info(' old: %s', line) + logging.info(' new: %s', new_line) + elif action == ADD_PLATFORMS_EXCEPT_THIS: + parts = line.split(':') + new_options = parts[0] + for p in self.PLATFORMS: + if not p == platform: + new_options += p.upper() + ' ' + new_line = ('%s:%s' % (new_options, parts[1])) + f_new.write(new_line) + tests_updated += 1 + logging.info('Test updated: ') + logging.info(' old: %s', line) + logging.info(' new: %s', new_line) + else: + logging.error('Unknown update action: %d; line: %s', action, line) + + logging.info('Total tests removed: %d', tests_removed) + logging.info('Total tests updated: %d', tests_updated) + + f_orig.close() + f_new.close() + + if backup: + date_suffix = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) + backup_file = ('%s.orig.%s' % (self._path, date_suffix)) + if os.path.exists(backup_file): + os.remove(backup_file) + logging.info('Saving original file to "%s"', backup_file) + os.rename(self._path, backup_file) + else: + os.remove(self._path) + + logging.debug('Saving new file to "%s"', self._path) + os.rename(new_file, self._path) + return True + + def _GetPlatformUpdateAction(self, line, tests, platform): + """Check the platform option and return the action needs to be taken. + + Args: + line: current line in test expectations file. + tests: list of tests that need to update.. + platform: which platform option to remove. + + Returns: + NO_CHANGE: no change to the line (comments, test not in the list etc) + REMOVE_TEST: remove the test from file. + REMOVE_PLATFORM: remove this platform option from the test. + ADD_PLATFORMS_EXCEPT_THIS: add all the platforms except this one. + """ + + line = StripComments(line) + if not line: + return NO_CHANGE + + options = [] + if line.find(':') is -1: + test_and_expecation = line.split('=') + else: + parts = line.split(':') + options = self._GetOptionsList(parts[0]) + test_and_expecation = parts[1].split('=') + + test = test_and_expecation[0].strip() + if not test in tests: + return NO_CHANGE + + has_any_platform = False + for option in options: + if option in self.PLATFORMS: + has_any_platform = True + if not option == platform: + return REMOVE_PLATFORM + + # If there is no platform specified, then it means apply to all platforms. + # Return the action to add all the platforms except this one. + if not has_any_platform: + return ADD_PLATFORMS_EXCEPT_THIS + + return REMOVE_TEST + def _HasValidModifiersForCurrentPlatform(self, options, lineno, test_and_expectations, modifiers): - """ Returns true if the current platform is in the options list or if no + """Returns true if the current platform is in the options list or if no platforms are listed and if there are no fatal errors in the options list. Args: @@ -372,7 +512,7 @@ class TestExpectationsFile: for test in self._full_test_list: if test.startswith(path): result.append(test) return result - + def _AddTests(self, tests, expectations, test_list_path, lineno, modifiers): for test in tests: if self._AlreadySeenTest(test, test_list_path, lineno): diff --git a/webkit/tools/layout_tests/rebaseline.bat b/webkit/tools/layout_tests/rebaseline.bat new file mode 100644 index 0000000..341b66d --- /dev/null +++ b/webkit/tools/layout_tests/rebaseline.bat @@ -0,0 +1 @@ +%~dp0..\..\..\third_party\python_24\python.exe %~dp0rebaseline.py %* diff --git a/webkit/tools/layout_tests/rebaseline.py b/webkit/tools/layout_tests/rebaseline.py new file mode 100644 index 0000000..5e6e48d --- /dev/null +++ b/webkit/tools/layout_tests/rebaseline.py @@ -0,0 +1,436 @@ +#!/bin/env python
+# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Rebaselining tool that automatically produces baselines for all platforms.
+
+The script does the following for each platform specified:
+ 1. Compile a list of tests that need rebaselining.
+ 2. Download test result archive from buildbot for the platform.
+ 3. Extract baselines from the archive file for all identified files.
+ 4. Add new baselines to SVN repository.
+ 5. For each test that has been rebaselined, remove this platform option from
+ the test in test_expectation.txt. If no other platforms remain after
+ removal, delete the rebaselined test from the file.
+"""
+
+import logging
+import optparse
+import os
+import re
+import subprocess
+import sys
+import urllib
+import zipfile
+
+from layout_package import path_utils
+from layout_package import platform_utils_linux
+from layout_package import platform_utils_mac
+from layout_package import platform_utils_win
+from layout_package import test_expectations
+
+
+def RunShell(command, print_output=False):
+ """Executes a command and returns the output.
+
+ Args:
+ command: program and arguments.
+ print_output: if true, print the command results to standard output.
+
+ Returns:
+ command output
+ """
+
+ # Use a shell for subcommands on Windows to get a PATH search.
+ use_shell = sys.platform.startswith('win')
+ p = subprocess.Popen(command, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, shell=use_shell,
+ universal_newlines=True)
+ if print_output:
+ output_array = []
+ while True:
+ line = p.stdout.readline()
+ if not line:
+ break
+ if print_output:
+ print line.strip('\n')
+ output_array.append(line)
+ output = ''.join(output_array)
+ else:
+ output = p.stdout.read()
+ p.wait()
+ p.stdout.close()
+ return output
+
+
+def LogDashedString(text, platform, logging_level=logging.INFO):
+ """Log text message with dashes on both sides."""
+
+ msg = '%s: %s' % (text, platform)
+ if len(msg) < 78:
+ dashes = '-' * ((78 - len(msg)) / 2)
+ msg = '%s %s %s' % (dashes, msg, dashes)
+
+ if logging_level == logging.ERROR:
+ logging.error(msg)
+ elif logging_level == logging.WARNING:
+ logging.warn(msg)
+ else:
+ logging.info(msg)
+
+
+class Rebaseliner(object):
+ """Class to produce new baselines for a given platform."""
+
+ BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
+ REVISION_REGEX = r'<a href=\"(\d+)/\">'
+
+ def __init__(self, platform, options):
+ self._file_dir = path_utils.GetAbsolutePath(os.path.dirname(sys.argv[0]))
+ self._platform = platform
+ self._options = options
+
+ # Create tests and expectations helper which is used to:
+ # -. compile list of tests that need rebaselining.
+ # -. update the tests in test_expectations file after rebaseline is done.
+ self._test_expectations = test_expectations.TestExpectations(None,
+ self._file_dir,
+ platform,
+ False)
+
+ def Run(self, backup):
+ """Run rebaseline process."""
+
+ LogDashedString('Compiling rebaselining tests', self._platform)
+ rebaselining_tests = self._CompileRebaseliningTests()
+ logging.info('')
+ if rebaselining_tests is None:
+ return True
+
+ LogDashedString('Downloading archive', self._platform)
+ archive_file = self._DownloadBuildBotArchive()
+ logging.info('')
+ if not archive_file:
+ logging.error('No archive found.')
+ return False
+
+ rebaselined_tests = []
+ LogDashedString('Extracting and adding new baselines', self._platform)
+ rebaselined_tests = self._ExtractAndAddNewBaselines(archive_file,
+ rebaselining_tests)
+ logging.info('')
+
+ LogDashedString('Updating rebaselined tests in file', self._platform)
+ self._UpdateRebaselinedTestsInFile(rebaselined_tests, backup)
+ logging.info('')
+
+ if len(rebaselining_tests) != len(rebaselined_tests):
+ logging.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
+ 'REBASELINED.')
+ logging.warning(' Total tests needing rebaselining: %d',
+ len(rebaselining_tests))
+ logging.warning(' Total tests rebaselined: %d',
+ len(rebaselined_tests))
+ return False
+
+ logging.warning('All tests needing rebaselining were successfully '
+ 'rebaselined.')
+
+ return True
+
+ def _CompileRebaseliningTests(self):
+ """Compile list of tests that need rebaselining for the platform.
+
+ Returns:
+ List of tests that need rebaselining or
+ None if there is no such test.
+ """
+
+ rebaselining_tests = self._test_expectations.GetRebaseliningFailures()
+ if not rebaselining_tests:
+ logging.warn('No tests found that need rebaselining.')
+ return None
+
+ logging.info('Total number of tests needing rebaselining for "%s": "%d"',
+ self._platform, len(rebaselining_tests))
+
+ test_no = 1
+ for test in rebaselining_tests:
+ logging.info(' %d: %s', test_no, test)
+ test_no += 1
+
+ return rebaselining_tests
+
+ def _GetLatestRevision(self, url):
+ """Get the latest layout test revision number from buildbot.
+
+ Args:
+ url: Url to retrieve layout test revision numbers.
+
+ Returns:
+ latest revision or
+ None on failure.
+ """
+
+ logging.debug('Url to retrieve revision: "%s"', url)
+
+ f = urllib.urlopen(url)
+ content = f.read()
+ f.close()
+
+ revisions = re.findall(self.REVISION_REGEX, content)
+ if not revisions:
+ logging.error('Failed to find revision, content: "%s"', content)
+ return None
+
+ revisions.sort(key=int)
+ logging.info('Latest revision: "%s"', revisions[len(revisions) - 1])
+ return revisions[len(revisions) - 1]
+
+ def _GetArchiveUrl(self):
+ """Generate the url to download latest layout test archive.
+
+ Returns:
+ Url to download archive or
+ None on failure
+ """
+
+ platform_name = self._options.buildbot_platform_dir_basename
+ if not self._platform == 'win':
+ platform_name += '-' + self._platform
+ logging.debug('Buildbot platform dir name: "%s"', platform_name)
+
+ url_base = '%s/%s/' % (self._options.archive_url, platform_name)
+ latest_revision = self._GetLatestRevision(url_base)
+ if latest_revision is None or latest_revision <= 0:
+ return None
+
+ archive_url = ('%s%s/%s.zip' % (url_base,
+ latest_revision,
+ self._options.archive_name))
+ logging.info('Archive url: "%s"', archive_url)
+ return archive_url
+
+ def _DownloadBuildBotArchive(self):
+ """Download layout test archive file from buildbot.
+
+ Returns:
+ True if download succeeded or
+ False otherwise.
+ """
+
+ url = self._GetArchiveUrl()
+ if url is None:
+ return None
+
+ fn = urllib.urlretrieve(url)[0]
+ logging.info('Archive downloaded and saved to file: "%s"', fn)
+ return fn
+
+ def _GetPlatformNewResultsDir(self):
+ """Get the dir name to extract new baselines for the given platform."""
+
+ if self._platform == 'win':
+ return platform_utils_win.PlatformUtility(None).PlatformNewResultsDir()
+ elif self._platform == 'mac':
+ return platform_utils_mac.PlatformUtility(None).PlatformNewResultsDir()
+ elif self._platform == 'linux':
+ return platform_utils_linux.PlatformUtility(None).PlatformNewResultsDir()
+
+ return None
+
+ def _ExtractAndAddNewBaselines(self, archive_file, rebaselining_tests):
+ """Extract new baselines from archive and add them to SVN repository.
+
+ Args:
+ archive_file: full path to the archive file.
+ rebaselining_tests: list of tests that need rebaselining.
+
+ Returns:
+ List of tests that have been rebaselined or
+ None on failure.
+ """
+
+ zip_file = zipfile.ZipFile(archive_file, 'r')
+ zip_namelist = zip_file.namelist()
+
+ logging.debug('zip file namelist:')
+ for name in zip_namelist:
+ logging.debug(' ' + name)
+
+ platform_dir = self._GetPlatformNewResultsDir()
+ if not platform_dir:
+ logging.error('Invalid platform new results dir, platform: "%s"',
+ self._platform)
+ return None
+
+ logging.debug('Platform new results dir: "%s"', platform_dir)
+
+ test_no = 1
+ rebaselined_tests = []
+ for test in rebaselining_tests:
+ logging.info('Test %d: %s', test_no, test)
+
+ found = False
+ svn_error = False
+ test_basename = os.path.splitext(test)[0]
+ for suffix in self.BASELINE_SUFFIXES:
+ archive_test_name = '%s/%s-actual%s' % (self._options.archive_name,
+ test_basename,
+ suffix)
+ logging.debug(' Archive test file name: "%s"', archive_test_name)
+ if not archive_test_name in zip_namelist:
+ logging.info(' %s file not in archive.', suffix)
+ continue
+
+ found = True
+ logging.info(' %s file found in archive.', suffix)
+
+ expected_filename = '%s-expected%s' % (test_basename, suffix)
+ expected_fullpath = os.path.join(
+ path_utils.ChromiumPlatformResultsEnclosingDir(),
+ platform_dir,
+ expected_filename)
+ expected_fullpath = os.path.normcase(expected_fullpath)
+ logging.debug(' Expected file full path: "%s"', expected_fullpath)
+
+ data = zip_file.read(archive_test_name)
+ f = open(expected_fullpath, 'wb')
+ f.write(data)
+ f.close()
+
+ if not self._SvnAdd(expected_fullpath):
+ svn_error = True
+
+ if not found:
+ logging.warn(' No new baselines found in archive.')
+ else:
+ if svn_error:
+ logging.warn(' Failed to add baselines to SVN.')
+ else:
+ logging.info(' Rebaseline succeeded.')
+ rebaselined_tests.append(test)
+
+ test_no += 1
+
+ zip_file.close()
+ os.remove(archive_file)
+
+ return rebaselined_tests
+
+ def _UpdateRebaselinedTestsInFile(self, rebaselined_tests, backup):
+ """Update the rebaselined tests in test expectations file.
+
+ Args:
+ rebaselined_tests: list of tests that have been rebaselined.
+ backup: if True, backup the original test expectations file.
+
+ Returns:
+ no
+ """
+
+ if rebaselined_tests:
+ self._test_expectations.RemovePlatformFromFile(rebaselined_tests,
+ self._platform,
+ backup)
+ else:
+ logging.info('No test was rebaselined so nothing to remove.')
+
+ def _SvnAdd(self, filename):
+ """Add the file to SVN repository.
+
+ Args:
+ filename: full path of the file to add.
+
+ Returns:
+ True if the file already exists in SVN or is sucessfully added to SVN.
+ False otherwise.
+ """
+
+ output = RunShell(['svn', 'status', filename], False)
+ logging.debug(' Svn status output: "%s"', output)
+ if output.startswith('A') or output.startswith('M'):
+ logging.info(' File already added to SVN: "%s"', filename)
+ return True
+
+ output = RunShell(['svn', 'add', filename], True)
+ logging.debug(' Svn add output: "%s"', output)
+ if output.startswith('A') and output.endswith(filename):
+ logging.info(' Added new file: "%s"', filename)
+ return True
+
+ logging.warn(' Failed to add file to SVN: "%s"', filename)
+ return False
+
+
+def main():
+ """Main function to produce new baselines."""
+
+ option_parser = optparse.OptionParser()
+ option_parser.add_option('-v', '--verbose',
+ action='store_true',
+ default=False,
+ help='include debug-level logging.')
+
+ option_parser.add_option('-p', '--platforms',
+ default='win,mac,linux',
+ help=('Comma delimited list of platforms that need '
+ 'rebaselining.'))
+
+ option_parser.add_option('-u', '--archive_url',
+ default=('http://build.chromium.org/buildbot/'
+ 'layout_test_results'),
+ help=('Url to find the layout test result archive '
+ 'file.'))
+
+ option_parser.add_option('-t', '--archive_name',
+ default='layout-test-results',
+ help='Layout test result archive name.')
+
+ option_parser.add_option('-n', '--buildbot_platform_dir_basename',
+ default='webkit-rel',
+ help=('Base name of buildbot platform directory '
+ 'that stores the layout test results.'))
+
+ option_parser.add_option('-b', '--backup',
+ action='store_true',
+ default=False,
+ help=('Whether or not to backup the original test '
+ 'expectations file after rebaseline.'))
+
+ options = option_parser.parse_args()[0]
+
+ # Set up our logging format.
+ log_level = logging.INFO
+ if options.verbose:
+ log_level = logging.DEBUG
+ logging.basicConfig(level=log_level,
+ format=('%(asctime)s %(filename)s:%(lineno)-3d '
+ '%(levelname)s %(message)s'),
+ datefmt='%y%m%d %H:%M:%S')
+
+ # Verify 'platforms' option is valid
+ if not options.platforms:
+ logging.error('Invalid "platforms" option. --platforms must be specified '
+ 'in order to rebaseline.')
+ sys.exit(1)
+
+ backup = options.backup
+ platforms = [p.strip().lower() for p in options.platforms.split(',')]
+ for platform in platforms:
+ rebaseliner = Rebaseliner(platform, options)
+
+ logging.info('')
+ LogDashedString('Rebaseline started', platform)
+ if rebaseliner.Run(backup):
+ # Only need to backup one original copy of test expectation file.
+ backup = False
+ LogDashedString('Rebaseline done', platform)
+ else:
+ LogDashedString('Rebaseline failed', platform, logging.ERROR)
+
+ sys.exit(0)
+
+if '__main__' == __name__:
+ main()
diff --git a/webkit/tools/layout_tests/rebaseline.sh b/webkit/tools/layout_tests/rebaseline.sh new file mode 100644 index 0000000..1499bc5 --- /dev/null +++ b/webkit/tools/layout_tests/rebaseline.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +exec_dir=$(dirname $0) + +if [ "$OSTYPE" = "cygwin" ]; then + system_root=`cygpath "$SYSTEMROOT"` + PATH="/usr/bin:$system_root/system32:$system_root:$system_root/system32/WBEM" + export PATH + PYTHON_PROG="$exec_dir/../../../third_party/python_24/python.exe" +else + PYTHON_PROG=python + # When not using the included python, we don't get automatic site.py paths. + # Specifically, run_webkit_tests needs the paths in: + # third_party/python_24/Lib/site-packages/google.pth + PYTHONPATH="${exec_dir}/../../../tools/python:$PYTHONPATH" + export PYTHONPATH +fi + +"$PYTHON_PROG" "$exec_dir/rebaseline.py" "$@" |