Modify tools to look for results in LayoutTests/platform/ directory.

(Until we can upload our tests, the chromium-win/ and any other chromium-*/
platform directories are located in webkit/data/layout_tests/platform/
rather than in webkit/data/layout_tests/LayoutTests/platform/.)

Add --sources option to run_webkit_tests to show which expected results file
is being cmopared to the actual results for each test.

Clean up the use of PlatformUtility a bit in path_utils.py.

Start removing build_type now that we don't support 'kjs' anymore, only 'v8'.
Only enough of this to allow it to be replaced by 'platform' was done.

BUG=4207
TEST=covered by layout tests once we copy the results over
Review URL: http://codereview.chromium.org/10702

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@5408 0039d316-1c4b-4281-b951-d872f2087c98

9 files changed, 182 insertions, 119 deletions

diff --git a/webkit/tools/layout_tests/layout_package/path_utils.py b/webkit/tools/layout_tests/layout_package/path_utils.py
index 742b40c..f7207b6 100644
--- a/webkit/tools/layout_tests/layout_package/path_utils.py
+++ b/webkit/tools/layout_tests/layout_package/path_utils.py
@@ -19,9 +19,12 @@ class PathNotFound(Exception): pass
 # Save some paths here so we don't keep re-evaling.
 _webkit_root = None
 _layout_data_dir = None
-_expected_results_dir = None
+# A map from platform description to directory list.
 _platform_results_dirs = {}
 
+# An instance of the PlatformUtility for use by methods mapped from there.
+_platform_util = None
+
 # TODO this should probably be moved into path_utils as ToUnixPath().
 def WinPathToUnix(path):
   """Convert a windows path to use unix-style path separators (a/b/c)."""
@@ -48,86 +51,111 @@ def LayoutDataDir():
                                                   'data', 'layout_tests')
   return _layout_data_dir
 
-def ExpectedResultsDir():
-  """Gets the full path to the custom_results directory.  Raises
-  PathNotFound if we're unable to find it."""
-  global _expected_results_dir
-  if _expected_results_dir:
-    return _expected_results_dir
-  _expected_results_dir = google.path_utils.FindUpward(WebKitRoot(), 'webkit',
-                                                       'data',
-                                                       'layout_test_results')
-  return _expected_results_dir
-
-def CustomExpectedResultsDir(custom_id):
-  """Gets the full path to the directory in which custom expected results for
-  this app and build type are located.
-
-  Args:
-    custom_id: a string specifying the particular set of results to use (e.g.,
-      'v8' or 'kjs')
+def ChromiumPlatformResultsDir():
+  """Returns the full path to the directory containing Chromium platform
+  result directories. Raises PathNotFound if we're unable to find it.
   """
-  return os.path.join(ExpectedResultsDir(), custom_id)
+  # TODO(pamg): Once we move platform/chromium-* into LayoutTests/platform/,
+  # remove this and use PlatformResultsDir() for everything.
+  return os.path.join(LayoutDataDir(), 'platform')
 
-def PlatformResultsDir(name):
-  """Gets the full path to a platform-specific results directory.  Raises
+def WebKitPlatformResultsDir():
+  """Gets the full path to the platform results directory.  Raises
   PathNotFound if we're unable to find it."""
-  global _platform_results_dirs
-  if _platform_results_dirs.get(name):
-    return _platform_results_dirs[name]
-  _platform_results_dirs[name] = google.path_utils.FindUpward(WebKitRoot(),
-      'webkit', 'data', 'layout_tests', 'LayoutTests', 'platform', name)
-  return _platform_results_dirs[name]
-
-def ExpectedFilename(filename, suffix, custom_result_id):
-  """Given a test name, returns an absolute filename to the most specific
-  applicable file of expected results.
+  return os.path.join(LayoutDataDir(), 'LayoutTests', 'platform')
+
+def PlatformResultsDir(platform):
+  """Gets the full path to the results directory for this platform."""
+  if platform.startswith('chromium'):
+    return ChromiumPlatformResultsDir()
+  return WebKitPlatformResultsDir()
+
+def ExpectedFilename(filename, suffix, platform):
+  """Given a test name, returns an absolute path to its expected results.
+
+  The result will be sought in the hierarchical platform directories, in the
+  corresponding WebKit platform directories, in the WebKit platform/mac/
+  directory, and finally next to the test file.
+
+  Suppose that the |platform| is 'chromium-win-xp'.  In that case, the
+  following directories are searched in order, if they exist, and the first
+  match is returned:
+    LayoutTests/platform/chromium-win-xp/
+    LayoutTests/platform/chromium-win/
+    LayoutTests/platform/chromium/
+    LayoutTests/platform/win-xp/
+    LayoutTests/platform/win/
+    LayoutTests/platform/mac/
+    the directory in which the test itself is located
+
+  If the |platform| is 'chromium-mac-leopard', the sequence will be as follows:
+    LayoutTests/platform/chromium-mac-leopard/
+    LayoutTests/platform/chromium-mac/
+    LayoutTests/platform/chromium/
+    LayoutTests/platform/mac-leopard/
+    LayoutTests/platform/mac/
+    the directory in which the test itself is located
+
+  If no expected results are found in any of the searched directories, the
+  directory in which the test itself is located will be returned.
 
   Args:
     filename: absolute filename to test file
     suffix: file suffix of the expected results, including dot; e.g. '.txt'
         or '.png'.  This should not be None, but may be an empty string.
-    custom_result_id: Tells us where to look for custom results.  Currently
-        this is either kjs or v8.
-
-  Return:
-    If a file named <testname>-expected<suffix> exists in the subdirectory
-    of the ExpectedResultsDir() specified by this platform's identifier,
-    return its absolute path.  Otherwise, return a path to a
-    <testname>-expected<suffix> file under the MacExpectedResultsDir() or
-    <testname>-expected<suffix> file under the MacLeopardExpectedResultsDir()
-    or (if not found there) in the same directory as the test file (even if
-    that default file does not exist).
+    platform: a hyphen-separated list of platform descriptors from least to
+        most specific, matching the WebKit format, that will be used to find
+        the platform/ directories to look in. For example, 'chromium-win' or
+        'chromium-mac-leopard'.
+
+  Returns:
+    An absolute path to the most specific matching result file for the given
+    test, following the search rules described above.
   """
   testname = os.path.splitext(RelativeTestFilename(filename))[0]
+  # While we still have tests in LayoutTests/, chrome/, and pending/, we need
+  # to strip that outer directory.
+  # TODO(pamg): Once we upstream all of chrome/ and pending/, clean this up.
+  testdir, testname = testname.split('/', 1)
   results_filename = testname + '-expected' + suffix
-  results_dirs = [
-    CustomExpectedResultsDir(custom_result_id),
-    CustomExpectedResultsDir('common'),
-    LayoutDataDir()
-  ]
-
-  for results_dir in results_dirs:
-    platform_file = os.path.join(results_dir, results_filename)
+
+  # Use the cached directory list if we have one.
+  global _platform_results_dirs
+  platform_dirs = _platform_results_dirs.get(platform, [])
+  if len(platform_dirs) == 0:
+    # Build the list of platform directories: chromium-foo-bar, chromium-foo,
+    # chromium.
+    segments = platform.split('-')
+    for length in range(len(segments), 0, -1):
+      platform_dirs.append('-'.join(segments[:length]))
+
+    # Append corresponding WebKit platform directories too.
+    if platform.startswith('chromium-'):
+      for length in range(len(segments), 1, -1):
+        platform_dirs.append('-'.join(segments[1:length]))
+
+    # Finally, append platform/mac/ to all searches.
+    if 'mac' not in platform_dirs:
+      platform_dirs.append('mac')
+
+    platform_dirs = [os.path.join(PlatformResultsDir(x), x)
+                     for x in platform_dirs]
+    _platform_results_dirs[platform] = platform_dirs
+
+  for platform_dir in platform_dirs:
+    # TODO(pamg): Clean this up once we upstream everything in chrome/ and
+    # pending/.
+    if os.path.basename(platform_dir).startswith('chromium'):
+      platform_file = os.path.join(platform_dir, testdir, results_filename)
+    else:
+      platform_file = os.path.join(platform_dir, results_filename)
     if os.path.exists(platform_file):
       return platform_file
 
-  # for 'base' tests, we need to look for mac-specific results
-  if testname.startswith('LayoutTests'):
-    layout_test_results_dirs = [
-      PlatformResultsDir('mac'),
-      PlatformResultsDir('mac-leopard'),
-      PlatformResultsDir('mac-tiger')
-    ]
-    rel_testname = testname[len('LayoutTests') + 1:]
-    rel_filename = rel_testname + '-expected' + suffix
-    for results_dir in layout_test_results_dirs:
-      platform_file = os.path.join(results_dir, rel_filename)
-      if os.path.exists(platform_file):
-        return platform_file
-
-  # Failed to find the results anywhere, return default path anyway
-  return os.path.join(results_dirs[0], results_filename)
+  # If it wasn't found in a platform directory, return the expected result
+  # in the test's directory, even if no such file actually exists.
+  return os.path.join(os.path.dirname(filename),
+                      os.path.basename(results_filename))
 
 def TestShellBinaryPath(target):
   """Gets the full path to the test_shell binary for the target build
@@ -147,23 +175,27 @@ def RelativeTestFilename(filename):
   directory as a unix style path (a/b/c)."""
   return WinPathToUnix(filename[len(LayoutDataDir()) + 1:])
 
+def GetPlatformUtil():
+  """Returns a singleton instance of the PlatformUtility."""
+  global _platform_util
+  if not _platform_util:
+    # Avoid circular import by delaying it.
+    import layout_package.platform_utils
+    _platform_util = (
+        layout_package.platform_utils.PlatformUtility(WebKitRoot()))
+  return _platform_util
+
 # Map platform specific path utility functions.  We do this as a convenience
 # so importing path_utils will get all path related functions even if they are
 # platform specific.
 def GetAbsolutePath(path):
-  # Avoid circular import by delaying it.
-  import layout_package.platform_utils
-  platform_util = layout_package.platform_utils.PlatformUtility(WebKitRoot())
-  return platform_util.GetAbsolutePath(path)
+  return GetPlatformUtil().GetAbsolutePath(path)
 
 def FilenameToUri(path):
-  # Avoid circular import by delaying it.
-  import layout_package.platform_utils
-  platform_util = layout_package.platform_utils.PlatformUtility(WebKitRoot())
-  return platform_util.FilenameToUri(path)
+  return GetPlatformUtil().FilenameToUri(path)
 
 def TestListPlatformDir():
-  # Avoid circular import by delaying it.
-  import layout_package.platform_utils
-  platform_util = layout_package.platform_utils.PlatformUtility(WebKitRoot())
-  return platform_util.TestListPlatformDir()
+  return GetPlatformUtil().TestListPlatformDir()
+
+def PlatformDir():
+  return GetPlatformUtil().PlatformDir()
diff --git a/webkit/tools/layout_tests/layout_package/platform_utils_mac.py b/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
index 2215503..2674ffe 100644
--- a/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
+++ b/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
@@ -137,5 +137,11 @@ class PlatformUtility(object):
                         'Contents', 'MacOS', self.TestShellBinary())
 
   def TestListPlatformDir(self):
-    """Return the platform-specific directory for where the test lists live"""
+    """Return the platform-specific directory for where the test lists live."""
     return 'mac'
+
+  def PlatformDir(self):
+    """Returns the most specific directory name where platform-specific
+    results live.
+    """
+    return 'chromium-mac'
diff --git a/webkit/tools/layout_tests/layout_package/platform_utils_win.py b/webkit/tools/layout_tests/layout_package/platform_utils_win.py
index 8f8c23d..fe75706 100644
--- a/webkit/tools/layout_tests/layout_package/platform_utils_win.py
+++ b/webkit/tools/layout_tests/layout_package/platform_utils_win.py
@@ -188,5 +188,11 @@ class PlatformUtility(google.platform_utils_win.PlatformUtility):
     return PathFromBase('chrome', target, self.TestShellBinary())
 
   def TestListPlatformDir(self):
-    """Return the platform-specific directory for where the test lists live"""
+    """Return the platform-specific directory for where the test lists live."""
     return 'win'
+
+  def PlatformDir(self):
+    """Returns the most specific directory name where platform-specific
+    results live.
+    """
+    return 'chromium-win'
diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py
index 5d7bbd5..1ab1f16 100644
--- a/webkit/tools/layout_tests/layout_package/test_expectations.py
+++ b/webkit/tools/layout_tests/layout_package/test_expectations.py
@@ -3,7 +3,11 @@
 # found in the LICENSE file.
 
 """A helper class for reading in and dealing with tests expectations
-for layout tests. """
+for layout tests.
+"""
+
+# TODO(pamg): Excise build_type (v8 and kjs) from this file and the test lists
+# now that we only support v8.
 
 import os
 import re
@@ -22,7 +26,7 @@ class TestExpectations:
   FIXABLE = "tests_fixable.txt"
   IGNORED = "tests_ignored.txt"
 
-  def __init__(self, tests, directory, build_type):
+  def __init__(self, tests, directory, build_type='v8'):
     """Reads the test expectations files from the given directory."""
     self._tests = tests
     self._directory = directory
diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py
index 9a1d7f2..dfe1a88 100644
--- a/webkit/tools/layout_tests/run_webkit_tests.py
+++ b/webkit/tools/layout_tests/run_webkit_tests.py
@@ -132,16 +132,17 @@ class TestRunner:
     file_dir = os.path.join(os.path.dirname(sys.argv[0]), TEST_FILE_DIR)
     file_dir = os.path.join(file_dir, path_utils.TestListPlatformDir())
     file_dir = path_utils.GetAbsolutePath(file_dir)
-    
-    expectations = test_expectations.TestExpectations(self._test_files, 
-                                                      file_dir, 
-                                                      self._options.build_type)
+
+    expectations = test_expectations.TestExpectations(self._test_files,
+                                                      file_dir)
 
     # Remove skipped - both fixable and ignored - files from the
     # top-level list of files to test.
-    skipped = expectations.GetFixableSkipped() | expectations.GetIgnoredSkipped()
+    skipped = (expectations.GetFixableSkipped() |
+               expectations.GetIgnoredSkipped())
+
     self._test_files -= skipped
-      
+
     # If there was only one test file, run the test even if it was skipped.
     if len(saved_test_files):
       self._test_files = saved_test_files
@@ -228,18 +229,19 @@ class TestRunner:
         shell_args.append("--pixel-tests=" + png_path)
         test_args.png_path = png_path
 
-      if self._options.wdiff:
-        test_args.wdiff = True
+      test_args.wdiff = self._options.wdiff
 
       if self._options.new_baseline:
         test_args.new_baseline = self._options.new_baseline
         if not self._options.pixel_tests:
           test_args.text_baseline = True
 
+      test_args.show_sources = self._options.sources
+
       # Create separate TestTypes instances for each thread.
       test_types = []
       for t in self._test_types:
-        test_types.append(t(self._options.build_type,
+        test_types.append(t(self._options.platform,
                             self._options.results_directory))
 
       if self._options.startup_dialog:
@@ -467,6 +469,9 @@ def main(options, args):
     options: a dictionary of command line options
     args: a list of sub directories or files to test
   """
+  if options.sources:
+    options.verbose = True
+
   # Set up our logging format.
   log_level = logging.INFO
   if options.verbose:
@@ -482,12 +487,8 @@ def main(options, args):
     else:
       options.target = "Release"
 
-  if (options.pixel_tests and
-      options.build_type != 'v8' and
-      not options.new_baseline):
-    logging.warn('Pixel tests disabled: no expected results for %s builds' %
-                 options.build_type)
-    options.pixel_tests = False
+  if options.platform is None:
+    options.platform = path_utils.PlatformDir()
 
   if options.results_directory.startswith("/"):
     # Assume it's an absolute path and normalize
@@ -505,13 +506,12 @@ def main(options, args):
   try:
     test_shell_binary_path = path_utils.TestShellBinaryPath(options.target)
   except:
-    print "\nERROR: test_shell is not found. Be sure that you have built it and"
-    print "that you are using the correct build. This script will run the"
-    print "release one by default. Use --debug to use the debug build.\n"
+    print "\nERROR: test_shell is not found. Be sure that you have built it"
+    print "and that you are using the correct build. This script will run the"
+    print "Release one by default. Use --debug to use the Debug build.\n"
     sys.exit(1)
 
-  logging.info("Using expected results from %s" %
-                path_utils.CustomExpectedResultsDir(options.build_type))
+  logging.info("Using platform '%s'" % options.platform)
   logging.info("Placing test results in %s" % options.results_directory)
   logging.info("Using %s build at %s" %
                (options.target, test_shell_binary_path))
@@ -592,14 +592,17 @@ if '__main__' == __name__:
   option_parser.add_option("", "--debug", action="store_true", default=False,
                            help="use the debug binary instead of the release "
                                 "binary")
-  option_parser.add_option("", "--build-type", default="v8",
-                           help="use these test lists and expected results "
-                                "('kjs' or 'v8')")
+  option_parser.add_option("", "--platform",
+                           help="Override the platform for expected results")
   option_parser.add_option("", "--target", default="",
                            help="Set the build target configuration (overrides"
                                  "--debug)")
+  # TODO(pamg): Support multiple levels of verbosity, and remove --sources.
   option_parser.add_option("-v", "--verbose", action="store_true",
-                           default=False, help="include debug level logging")
+                           default=False, help="include debug-level logging")
+  option_parser.add_option("", "--sources", action="store_true",
+                           help="show expected result file path for each test "
+                                "(implies --verbose)")
   option_parser.add_option("", "--startup-dialog", action="store_true",
                            default=False, 
                            help="create a dialog on test_shell.exe startup")
diff --git a/webkit/tools/layout_tests/test_types/image_diff.py b/webkit/tools/layout_tests/test_types/image_diff.py
index 4f951cf..0964266 100644
--- a/webkit/tools/layout_tests/test_types/image_diff.py
+++ b/webkit/tools/layout_tests/test_types/image_diff.py
@@ -10,6 +10,7 @@ test results directory.
 """
 
 import errno
+import logging
 import os
 import shutil
 
@@ -72,11 +73,14 @@ class ImageDiff(test_type_base.TestTypeBase):
     # Compare hashes.
     expected_hash_file = path_utils.ExpectedFilename(filename,
                                                      '.checksum',
-                                                     self._custom_result_id)
+                                                     self._platform)
 
     expected_png_file = path_utils.ExpectedFilename(filename,
                                                     '.png',
-                                                    self._custom_result_id)
+                                                    self._platform)
+
+    if test_args.show_sources:
+      logging.debug('Using %s' % expected_png_file)
 
     try:
       expected_hash = open(expected_hash_file, "r").read()
diff --git a/webkit/tools/layout_tests/test_types/simplified_text_diff.py b/webkit/tools/layout_tests/test_types/simplified_text_diff.py
index 3b0731b..46e7e8a 100644
--- a/webkit/tools/layout_tests/test_types/simplified_text_diff.py
+++ b/webkit/tools/layout_tests/test_types/simplified_text_diff.py
@@ -94,7 +94,9 @@ class SimplifiedTextDiff(text_diff.TestTextDiff):
 
     # Normalize text to diff
     output = self.GetNormalizedOutputText(output)
-    expected = self.GetNormalizedExpectedText(filename)
+    # The full text diff already gave the result file if requested, so we'll
+    # ignore the |show_sources| option here.
+    expected = self.GetNormalizedExpectedText(filename, show_sources=False)
 
     # Don't bother with the simplified text diff if we match before simplifying
     # the text.
diff --git a/webkit/tools/layout_tests/test_types/test_type_base.py b/webkit/tools/layout_tests/test_types/test_type_base.py
index d748818..9e1a163 100644
--- a/webkit/tools/layout_tests/test_types/test_type_base.py
+++ b/webkit/tools/layout_tests/test_types/test_type_base.py
@@ -36,6 +36,9 @@ class TestArguments(object):
   # Whether to use wdiff to generate by-word diffs.
   wdiff = False
 
+  # Whether to report the locations of the expected result files used.
+  show_sources = False
+
 class TestTypeBase(object):
   # Filename pieces when writing failures to the test results directory.
   FILENAME_SUFFIX_ACTUAL = "-actual-win"
@@ -43,16 +46,16 @@ class TestTypeBase(object):
   FILENAME_SUFFIX_DIFF = "-diff-win"
   FILENAME_SUFFIX_WDIFF = "-wdiff-win.html"
 
-  def __init__(self, custom_result_id, root_output_dir):
+  def __init__(self, platform, root_output_dir):
     """Initialize a TestTypeBase object.
 
     Args:
-      custom_result_id: the string (generally 'kjs' or 'v8') identifying the
-        custom expected results to be used
+      platform: the platform (e.g., 'chromium-mac-leopard') identifying the
+        platform-specific results to be used
       root_output_dir: The unix style path to the output dir.
     """
     self._root_output_dir = root_output_dir
-    self._custom_result_id = custom_result_id
+    self._platform = platform
   
   def _MakeOutputDirectory(self, filename):
     """Creates the output directory (if needed) for a given test filename."""
diff --git a/webkit/tools/layout_tests/test_types/text_diff.py b/webkit/tools/layout_tests/test_types/text_diff.py
index 0b0086a..8b188cd 100644
--- a/webkit/tools/layout_tests/test_types/text_diff.py
+++ b/webkit/tools/layout_tests/test_types/text_diff.py
@@ -9,6 +9,7 @@ files into the layout test results directory.
 """
 
 import errno
+import logging
 import os.path
 
 from layout_package import path_utils
@@ -23,14 +24,16 @@ class TestTextDiff(test_type_base.TestTypeBase):
     norm = output.replace("\r\r\n", "\r\n").strip("\r\n").replace("\r\n", "\n")
     return norm + "\n"
 
-  def GetNormalizedExpectedText(self, filename):
+  def GetNormalizedExpectedText(self, filename, show_sources):
     """Given the filename of the test, read the expected output from a file
     and normalize the text.  Returns a string with the expected text, or ''
     if the expected output file was not found."""
-    # Read the platform-specific expected text, or the Mac default if no
-    # platform result exists.
+    # Read the platform-specific expected text.
     expected_filename = path_utils.ExpectedFilename(filename, '.txt',
-                                                    self._custom_result_id)
+                                                    self._platform)
+
+    if show_sources:
+      logging.debug('Using %s' % expected_filename)
     try:
       expected = open(expected_filename).read()
     except IOError, e:
@@ -55,7 +58,7 @@ class TestTextDiff(test_type_base.TestTypeBase):
 
     # Normalize text to diff
     output = self.GetNormalizedOutputText(output)
-    expected = self.GetNormalizedExpectedText(filename)
+    expected = self.GetNormalizedExpectedText(filename, test_args.show_sources)
 
     # Write output files for new tests, too.
     if output != expected: