By popular demand, mv test_lists/tests_fixable.txt to test_expectations.txt.

This will require updating a couple scripts. I've updated run_webkit_tests.
I think jonc and sandholm's scripts will need updating once this is committed.

Also, while I'm modifying run_webkit_tests.py, re-adds --num-test-shells argument as I'm trying to make it work.
Review URL: http://codereview.chromium.org/49030

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@12496 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 19 insertions, 48 deletions

diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py
index 3dbb0eb..8fb283c 100644
--- a/webkit/tools/layout_tests/layout_package/test_expectations.py
+++ b/webkit/tools/layout_tests/layout_package/test_expectations.py
@@ -17,7 +17,7 @@ import compare_failures
 (PASS, FAIL, TIMEOUT, CRASH, SKIP, WONTFIX, DEFER, NONE) = range(8)
 
 class TestExpectations:
-  TEST_LIST = "tests_fixable.txt"
+  TEST_LIST = "test_expectations.txt"
 
   def __init__(self, tests, directory, platform, is_debug_mode):
     """Reads the test expectations files from the given directory."""
diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py
index 64a3fa3..c4dd11e 100755
--- a/webkit/tools/layout_tests/run_webkit_tests.py
+++ b/webkit/tools/layout_tests/run_webkit_tests.py
@@ -48,11 +48,6 @@ from test_types import text_diff
 from test_types import simplified_text_diff
 
 
-# The test list files are found in this subdirectory, which must be a sibling
-# to this script itself.
-TEST_FILE_DIR = 'test_lists'
-
-
 class TestRunner:
   """A class for managing running a series of tests on a series of test
   files."""
@@ -81,8 +76,7 @@ class TestRunner:
     # a set of test files, and the same tests as a list
     self._test_files = set()
     self._test_files_list = None
-    self._file_dir = os.path.join(os.path.dirname(sys.argv[0]), TEST_FILE_DIR)
-    self._file_dir = path_utils.GetAbsolutePath(self._file_dir)
+    self._file_dir = path_utils.GetAbsolutePath(os.path.dirname(sys.argv[0]))
 
     if options.lint_test_files:
       # Creating the expecations for each platform/target pair does all the
@@ -740,11 +734,6 @@ def main(options, args):
   if os.path.exists(cachedir):
     shutil.rmtree(cachedir)
 
-  # This was an experimental feature where we would run more than one
-  # test_shell in parallel.  For some reason, this would result in different
-  # layout test results, so just use 1 test_shell for now.
-  options.num_test_shells = 1
-
   test_runner.AddTestType(text_diff.TestTextDiff)
   test_runner.AddTestType(simplified_text_diff.SimplifiedTextDiff)
   if not options.no_pixel_tests:
@@ -788,6 +777,10 @@ if '__main__' == __name__:
                            help="Disable comparison to the last test run. "
                                 "When enabled, show stats on how many tests "
                                 "newly pass or fail.")
+  option_parser.add_option("", "--num-test-shells",
+                           default=1,
+                           help="Experimental. Number of testshells to run in "
+                                "parallel.")
   option_parser.add_option("", "--time-out-ms",
                            default=None,
                            help="Set the timeout for each test")
diff --git a/webkit/tools/layout_tests/test_lists/tests_fixable.txt b/webkit/tools/layout_tests/test_expectations.txt
index b7fafa9..aae1101 100644
--- a/webkit/tools/layout_tests/test_lists/tests_fixable.txt
+++ b/webkit/tools/layout_tests/test_expectations.txt
@@ -22,8 +22,19 @@
 // RELEASE: Expectations apply only to release build.
 // LINUX/WIN/MAC: Expectations apply only to these platforms.
 //
-// A test can be included twice, but not via the same path. If a test is
-// included twice, then the more precise path wins.
+// The format of a test line is as follows:
+// METADATA : relative/path/to/test.html = EXPECTATIONS
+//
+// For example,
+// LINUX : LayoutTests/media = PASS FAIL
+//
+// The above means that all the media tests are flaky, but only on Linux.
+//
+// Notes:
+// -A test cannot be both DEFER and WONTFIX
+// -A test can be included twice, but not via the same path.
+// -If a test is included twice, then the more precise path wins.
+// -CRASH tests cannot be DEFER or WONTFIX
 
 // -----------------------------------------------------------------
 // SKIPPED TESTS
diff --git a/webkit/tools/layout_tests/test_lists/README b/webkit/tools/layout_tests/test_lists/README
deleted file mode 100644
index cf035a9..0000000
--- a/webkit/tools/layout_tests/test_lists/README
+++ /dev/null
@@ -1,33 +0,0 @@
-The files in this directory are read by the run_webkit_tests.py script
-and used to determine which tests to run and which failures to
-consider regressions.
-
-If we *never* expect to pass a test (e.g. a test for Apple's Objective-C API),
-then add it to tests_ignore.txt. If we might ever want to pass the test, first
-file a bug, then add it to tests_fixable.txt.
-
-Tests can have any combination of the following metadata associated with
-them (optional):
-  SKIP : We don't want to run the test (perhaps because it hangs).
-  DEFER : Test does not count in our statistics for the current release.
-  DEBUG  : Expectations apply only to the Debug build.
-  RELEASE : Expectations apply only to the Release build.
-  LINUX/WIN/MAC: Expectations apply only to the listed platform(s).
-
-Tests can also have one or more expectations (required):
-  PASS
-  FAIL
-  TIMEOUT
-  CRASH
-
-The format of a test line is as follows:
-METADATA : relative/path/to/test.html = EXPECTATIONS
-
-For example,
-LINUX : LayoutTests/media = PASS FAIL
-
-The above means that all the media tests are flaky, but only on Linux.
-
-Note that tests_fixable.txt and tests_ignored.txt aren't allowed to share any
-lines. Except for skipped tests, all tests are run. In calculating success
-metrics, we ignore tests in tests_ignored.txt and deferred tests.