Add an option to the test_shell to enable it to load each test several times

We have sees bugs which could have been caught by reloading a page one or more times and finding that the results for one of the subsequent loads is not the same as the expected output. The new option--multiple-loads=X will load each test X time. The output dumped by test_shell is the result of the last load. The default when the option is specified without a value is 2 times in debug mode and 5 times in release mode.

To be able to have more fine-grained control of how the JavaScript engine behaves for each load the existing flag --js-flags can now specify a list of flag-sets like this

  --js-flags="--xxx,--noxxx --yyy,--noyyy"

First load will run with --xxx, the second with --yyy and the third without any (the 'no' prefix is handled by V8 to turn off the flag).

The changes to the Python test runner will be in a separate change.

BUG=none
TEST=none
Review URL: http://codereview.chromium.org/4016002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@63863 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 126 insertions, 19 deletions

diff --git a/webkit/tools/test_shell/test_shell.cc b/webkit/tools/test_shell/test_shell.cc
index 64af309..ee1ad6b 100644
--- a/webkit/tools/test_shell/test_shell.cc
+++ b/webkit/tools/test_shell/test_shell.cc
@@ -115,6 +115,9 @@ bool TestShell::allow_external_pages_ = false;
 int TestShell::file_test_timeout_ms_ = kDefaultFileTestTimeoutMillisecs;
 bool TestShell::test_is_preparing_ = false;
 bool TestShell::test_is_pending_ = false;
+int TestShell::load_count_ = 1;
+std::vector<std::string> TestShell::js_flags_;
+bool TestShell::dump_when_finished_ = true;
 bool TestShell::accelerated_2d_canvas_enabled_ = false;
 bool TestShell::accelerated_compositing_enabled_ = false;
 
@@ -657,7 +660,7 @@ void TestShell::LoadFile(const FilePath& file) {
 void TestShell::LoadURL(const GURL& url) {
   // Used as a sentinal for run_webkit_tests.py to know when to start reading
   // test output for this test and so we know we're not getting out of sync.
-  if (layout_test_mode_ && test_params())
+  if (layout_test_mode_ && dump_when_finished_ && test_params())
     printf("#URL:%s\n", test_params()->test_url.c_str());
 
   LoadURLForFrame(url, std::wstring());
diff --git a/webkit/tools/test_shell/test_shell.h b/webkit/tools/test_shell/test_shell.h
index bf139d4..7c320a2 100644
--- a/webkit/tools/test_shell/test_shell.h
+++ b/webkit/tools/test_shell/test_shell.h
@@ -30,6 +30,7 @@
 
 #include <string>
 #include <list>
+#include <vector>
 
 #include "base/basictypes.h"
 #include "base/file_path.h"
@@ -301,7 +302,34 @@ public:
     // Make it a bit longer than the regular timeout to avoid killing the
     // TestShell process unless we really need to.
     static int GetLayoutTestTimeoutForWatchDog() {
-      return file_test_timeout_ms_ + 1000;
+      return (load_count_ * file_test_timeout_ms_) + 1000;
+    }
+
+    // Set the JavaScript flags to use. This is a vector as when multiple loads
+    // are specified each load can have different flags passed.
+    static void SetJavaScriptFlags(std::vector<std::string> js_flags) {
+      js_flags_ = js_flags;
+    }
+
+    // Set the number of times to load each URL.
+    static void SetMultipleLoad(int load_count) {
+      load_count_ = load_count;
+    }
+
+    // Get the number of times to load each URL.
+    static int GetLoadCount() { return load_count_; }
+
+    // Get the JavaScript flags for a specific load
+    static std::string GetJSFlagsForLoad(size_t load) {
+      if (load >= js_flags_.size()) return "";
+      return js_flags_[load];
+    }
+
+    // Set whether to dump when the loaded page has finished processing. This is
+    // used with multiple load testing where normally we only want to have the
+    // output from the last load.
+    static void SetDumpWhenFinished(bool dump_when_finished) {
+      dump_when_finished_ = dump_when_finished;
     }
 
 #if defined(OS_WIN)
@@ -443,6 +471,17 @@ private:
     // True while a test is running
     static bool test_is_pending_;
 
+    // Number of times to load each URL.
+    static int load_count_;
+
+    // JavaScript flags. Each element in the vector contains a set of flags as
+    // a string (e.g. "--xxx --yyy"). Each flag set is used for separate loads
+    // of each URL.
+    static std::vector<std::string> js_flags_;
+
+    // True if a test shell dump should be made when the test is finished.
+    static bool dump_when_finished_;
+
     // True if we're testing the accelerated canvas 2d path.
     static bool accelerated_2d_canvas_enabled_;
 
diff --git a/webkit/tools/test_shell/test_shell_gtk.cc b/webkit/tools/test_shell/test_shell_gtk.cc
index 3d08253..3aafb3d 100644
--- a/webkit/tools/test_shell/test_shell_gtk.cc
+++ b/webkit/tools/test_shell/test_shell_gtk.cc
@@ -376,10 +376,12 @@ void TestShell::TestFinished() {
     return;
 
   test_is_pending_ = false;
-  GtkWindow* window = *(TestShell::windowList()->begin());
-  TestShell* shell = static_cast<TestShell*>(g_object_get_data(G_OBJECT(window),
-                                             "test-shell"));
-  TestShell::Dump(shell);
+  if (dump_when_finished_) {
+    GtkWindow* window = *(TestShell::windowList()->begin());
+    TestShell* shell = static_cast<TestShell*>(
+        g_object_get_data(G_OBJECT(window), "test-shell"));
+    TestShell::Dump(shell);
+  }
   MessageLoop::current()->Quit();
 }
 
diff --git a/webkit/tools/test_shell/test_shell_mac.mm b/webkit/tools/test_shell/test_shell_mac.mm
index d2b90a7..30eab0f 100644
--- a/webkit/tools/test_shell/test_shell_mac.mm
+++ b/webkit/tools/test_shell/test_shell_mac.mm
@@ -370,10 +370,12 @@ void TestShell::TestFinished() {
     return;  // reached when running under test_shell_tests
 
   test_is_pending_ = false;
-  NSWindow* window = *(TestShell::windowList()->begin());
-  WindowMap::iterator it = window_map_.Get().find(window);
-  if (it != window_map_.Get().end())
-    TestShell::Dump(it->second);
+  if (dump_when_finished_) {
+    NSWindow* window = *(TestShell::windowList()->begin());
+    WindowMap::iterator it = window_map_.Get().find(window);
+    if (it != window_map_.Get().end())
+      TestShell::Dump(it->second);
+  }
   MessageLoop::current()->Quit();
 }
 
diff --git a/webkit/tools/test_shell/test_shell_main.cc b/webkit/tools/test_shell/test_shell_main.cc
index 8d7d3b9..5d43e0b 100644
--- a/webkit/tools/test_shell/test_shell_main.cc
+++ b/webkit/tools/test_shell/test_shell_main.cc
@@ -26,6 +26,7 @@
 #include "net/base/net_module.h"
 #include "net/base/net_util.h"
 #include "net/http/http_cache.h"
+#include "net/http/http_util.h"
 #include "net/test/test_server.h"
 #include "net/url_request/url_request_context.h"
 #include "third_party/WebKit/WebKit/chromium/public/WebKit.h"
@@ -131,6 +132,21 @@ int main(int argc, char* argv[]) {
   if (parsed_command_line.HasSwitch(test_shell::kEnableAccelCompositing))
     TestShell::SetAcceleratedCompositingEnabled(true);
 
+  if (parsed_command_line.HasSwitch(test_shell::kMultipleLoads)) {
+    const std::string multiple_loads_str =
+        parsed_command_line.GetSwitchValueASCII(test_shell::kMultipleLoads);
+    int load_count;
+    base::StringToInt(multiple_loads_str, &load_count);
+    if (load_count <= 0) {
+  #ifndef NDEBUG
+      load_count = 2;
+  #else
+      load_count = 5;
+  #endif
+    }
+    TestShell::SetMultipleLoad(load_count);
+  }
+
   TestShell::InitLogging(suppress_error_dialogs,
                          layout_test_mode,
                          enable_gp_fault_error_box);
@@ -247,11 +263,31 @@ int main(int argc, char* argv[]) {
     }
   }
 
+  // Get the JavaScript flags. The test runner might send a quoted string which
+  // needs to be unquoted before further processing.
   std::string js_flags =
       parsed_command_line.GetSwitchValueASCII(test_shell::kJavaScriptFlags);
-  // Test shell always exposes the GC.
-  js_flags += " --expose-gc";
-  webkit_glue::SetJavaScriptFlags(js_flags);
+  js_flags = net::HttpUtil::Unquote(js_flags);
+  // Split the JavaScript flags into a list.
+  std::vector<std::string> js_flags_list;
+  size_t start = 0;
+  while (true) {
+    size_t comma_pos = js_flags.find_first_of(',', start);
+    std::string flags;
+    if (comma_pos == std::string::npos) {
+      flags = js_flags.substr(start, js_flags.length() - start);
+    } else {
+      flags = js_flags.substr(start, comma_pos - start);
+      start = comma_pos + 1;
+    }
+    // Test shell always exposes the GC.
+    flags += " --expose-gc";
+    js_flags_list.push_back(flags);
+    if (comma_pos == std::string::npos)
+      break;
+  }
+  TestShell::SetJavaScriptFlags(js_flags_list);
+
   // Expose GCController to JavaScript.
   WebScriptController::registerExtension(extensions_v8::GCExtension::Get());
 
@@ -361,7 +397,23 @@ int main(int argc, char* argv[]) {
               params.pixel_hash = pixel_hash;
           }
 
-          if (!TestShell::RunFileTest(params))
+          // Load the page the number of times specified.
+          bool fatal_error = false;
+          for (int i = 0; i < TestShell::GetLoadCount(); i++) {
+            // Set the JavaScript flags specified for this load.
+            webkit_glue::SetJavaScriptFlags(TestShell::GetJSFlagsForLoad(i));
+
+            // Only dump for the last load.
+            bool is_last_load = (i == (TestShell::GetLoadCount() - 1));
+            TestShell::SetDumpWhenFinished(is_last_load);
+
+            if (!TestShell::RunFileTest(params)) {
+              fatal_error = true;
+              break;
+            }
+          }
+
+          if (fatal_error)
             break;
 
           TestShell::SetFileTestTimeout(old_timeout_ms);
diff --git a/webkit/tools/test_shell/test_shell_switches.cc b/webkit/tools/test_shell/test_shell_switches.cc
index b3ed0bc..29d8328 100644
--- a/webkit/tools/test_shell/test_shell_switches.cc
+++ b/webkit/tools/test_shell/test_shell_switches.cc
@@ -38,7 +38,13 @@ const char kStartupDialog[] = "testshell-startup-dialog";
 // it possible to attach a crashed test shell to a debugger.
 const char kGPFaultErrorBox[] = "gp-fault-error-box";
 
-// JavaScript flags passed to engine.
+// Make the test shell load the test URL multiple times. The output dump will
+// only be made from the default last run.
+const char kMultipleLoads[] = "multiple-loads";
+
+// JavaScript flags passed to engine. If multiple loads has been specified this
+// can be a list separated by commas. Each set of flags are passed to the engine
+// in the corresponding load.
 const char kJavaScriptFlags[] = "js-flags";
 
 // Run the http cache in record mode.
diff --git a/webkit/tools/test_shell/test_shell_switches.h b/webkit/tools/test_shell/test_shell_switches.h
index 6f2afc0f..cb5d722 100644
--- a/webkit/tools/test_shell/test_shell_switches.h
+++ b/webkit/tools/test_shell/test_shell_switches.h
@@ -21,6 +21,7 @@ extern const char kTestShellTimeOut[];
 extern const char kStartupDialog[];
 extern const char kGPFaultErrorBox[];
 extern const char kJavaScriptFlags[];
+extern const char kMultipleLoads[];
 extern const char kRecordMode[];
 extern const char kPlaybackMode[];
 extern const char kNoEvents[];
diff --git a/webkit/tools/test_shell/test_shell_win.cc b/webkit/tools/test_shell/test_shell_win.cc
index 3853755..bb333c1 100644
--- a/webkit/tools/test_shell/test_shell_win.cc
+++ b/webkit/tools/test_shell/test_shell_win.cc
@@ -426,10 +426,12 @@ void TestShell::TestFinished() {
     return;  // reached when running under test_shell_tests
 
   test_is_pending_ = false;
-  HWND hwnd = *(TestShell::windowList()->begin());
-  TestShell* shell =
-      static_cast<TestShell*>(win_util::GetWindowUserData(hwnd));
-  TestShell::Dump(shell);
+  if (dump_when_finished_) {
+    HWND hwnd = *(TestShell::windowList()->begin());
+    TestShell* shell =
+        static_cast<TestShell*>(win_util::GetWindowUserData(hwnd));
+    TestShell::Dump(shell);
+  }
 
   UINT_PTR timer_id = reinterpret_cast<UINT_PTR>(this);
   KillTimer(mainWnd(), timer_id);