- Add UI test for the V8 Benchmark Suite.

- Modify the V8 Benchmark Suite to work with the UI test framework.
- Add utils for JavaScript UI tests.
- Move automation.js files for SunSpider and V8 Benchmark to the
  src/chrome/test/ui directory.
- Overload UITest::PrintResult for convenience.

BUG=9255
Review URL: http://codereview.chromium.org/42628

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@12823 0039d316-1c4b-4281-b951-d872f2087c98

10 files changed, 335 insertions, 37 deletions

diff --git a/chrome/test/ui/javascript_test_util.cc b/chrome/test/ui/javascript_test_util.cc
new file mode 100644
index 0000000..17928a2
--- /dev/null
+++ b/chrome/test/ui/javascript_test_util.cc
@@ -0,0 +1,55 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/test/ui/javascript_test_util.h"
+
+#include "base/logging.h"
+#include "base/scoped_ptr.h"
+#include "base/string_util.h"
+#include "base/values.h"
+#include "chrome/common/json_value_serializer.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+bool JsonDictionaryToMap(const std::string& json,
+                         std::map<std::string, std::string>* results) {
+  DCHECK(results != NULL);
+  JSONStringValueSerializer deserializer(json);
+  scoped_ptr<Value> root(deserializer.Deserialize(NULL));
+
+  // Note that we don't use ASSERT_TRUE here (and in some other places) as it
+  // doesn't work inside a function with a return type other than void.
+  EXPECT_TRUE(root.get());
+  if (!root.get())
+    return false;
+
+  EXPECT_TRUE(root->IsType(Value::TYPE_DICTIONARY));
+  if (!root->IsType(Value::TYPE_DICTIONARY))
+    return false;
+
+  DictionaryValue* dict = static_cast<DictionaryValue*>(root.get());
+
+  DictionaryValue::key_iterator it = dict->begin_keys();
+  for (; it != dict->end_keys(); ++it) {
+    Value* value = NULL;
+    bool succeeded = dict->Get(*it, &value);
+
+    EXPECT_TRUE(succeeded);
+    if (!succeeded)
+      continue;
+
+    EXPECT_TRUE(value->IsType(Value::TYPE_STRING));
+    if (value->IsType(Value::TYPE_STRING)) {
+      std::string key = WideToUTF8(*it);
+
+      std::string result;
+      succeeded = value->GetAsString(&result);
+      EXPECT_TRUE(succeeded);
+
+      if (succeeded)
+        results->insert(std::make_pair(key, result));
+    }
+  }
+
+  return true;
+}
diff --git a/chrome/test/ui/javascript_test_util.h b/chrome/test/ui/javascript_test_util.h
new file mode 100644
index 0000000..1619fc0
--- /dev/null
+++ b/chrome/test/ui/javascript_test_util.h
@@ -0,0 +1,20 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_TEST_UI_JAVASCRIPT_TEST_UTIL_H_
+#define CHROME_TEST_UI_JAVASCRIPT_TEST_UTIL_H_
+
+#include <string>
+#include <map>
+
+// This file provides a common set of utilities that are useful to UI tests
+// that interact with JavaScript.
+
+// Given a JSON encoded representation of a dictionary, parses the string and
+// fills in a map with the results. No attempt is made to clear the map.
+// Returns a bool indicating success or failure of the operation.
+bool JsonDictionaryToMap(const std::string& json,
+                         std::map<std::string, std::string>* results);
+
+#endif  // CHROME_TEST_UI_JAVASCRIPT_TEST_UTIL_H_
diff --git a/chrome/test/ui/sunspider_uitest.cc b/chrome/test/ui/sunspider_uitest.cc
index 0fe82be..31bb621 100644
--- a/chrome/test/ui/sunspider_uitest.cc
+++ b/chrome/test/ui/sunspider_uitest.cc
@@ -12,6 +12,7 @@
 #include "chrome/common/chrome_switches.h"
 #include "chrome/common/json_value_serializer.h"
 #include "chrome/test/automation/tab_proxy.h"
+#include "chrome/test/ui/javascript_test_util.h"
 #include "chrome/test/ui/ui_test.h"
 #include "googleurl/src/gurl.h"
 #include "net/base/net_util.h"
@@ -91,42 +92,7 @@ class SunSpiderTest : public UITest {
       return false;
 
     std::string json = WideToUTF8(json_wide);
-    JSONStringValueSerializer deserializer(json);
-    scoped_ptr<Value> root(deserializer.Deserialize(NULL));
-
-    EXPECT_TRUE(root.get());
-    if (!root.get())
-      return false;
-
-    EXPECT_TRUE(root->IsType(Value::TYPE_DICTIONARY));
-    if (!root->IsType(Value::TYPE_DICTIONARY))
-      return false;
-
-    DictionaryValue* dict = static_cast<DictionaryValue*>(root.get());
-
-    DictionaryValue::key_iterator it = dict->begin_keys();
-    for (; it != dict->end_keys(); ++it) {
-      Value* value = NULL;
-      succeeded = dict->Get(*it, &value);
-
-      EXPECT_TRUE(succeeded);
-      if (!succeeded)
-        continue;
-
-      EXPECT_TRUE(value->IsType(Value::TYPE_STRING));
-      if (value->IsType(Value::TYPE_STRING)) {
-        std::string key = WideToUTF8(*it);
-
-        std::string result;
-        succeeded = value->GetAsString(&result);
-        EXPECT_TRUE(succeeded);
-
-        if (succeeded)
-          results->insert(std::make_pair(key, result));
-      }
-    }
-
-    return true;
+    return JsonDictionaryToMap(json, results);
   }
 
   void PrintResults(TabProxy* tab) {
diff --git a/chrome/test/ui/sunspider_uitest.js b/chrome/test/ui/sunspider_uitest.js
new file mode 100644
index 0000000..8e488d2
--- /dev/null
+++ b/chrome/test/ui/sunspider_uitest.js
@@ -0,0 +1,31 @@
+/*
+  Copyright (c) 2009 The Chromium Authors. All rights reserved.
+  Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file.
+*/
+
+// Automation utilities for running SunSpider as a UI test.
+
+function Automation() {
+}
+
+Automation.prototype.SetDone = function() {
+  document.cookie = '__done=1; path=/';
+}
+
+Automation.prototype.GetTotal = function() {
+  // Calculated in sunspider-analyze-results.js.
+  return mean + ',' + stdDev;
+}
+
+Automation.prototype.GetResults = function() {
+  var results = {};
+
+  var input = eval("(" + decodeURI(location.search.substring(1)) + ")");
+  for (test in input)
+    results[test] = input[test].toString();
+
+  return results;
+}
+
+automation = new Automation();
diff --git a/chrome/test/ui/ui_test.cc b/chrome/test/ui/ui_test.cc
index c4d4cbc..2ae2955 100644
--- a/chrome/test/ui/ui_test.cc
+++ b/chrome/test/ui/ui_test.cc
@@ -363,7 +363,8 @@ void UITest::LaunchBrowser(const CommandLine& arguments, bool clear_profile) {
     CommandLine wrapped_command(ASCIIToWide(browser_wrapper));
     wrapped_command.AppendArguments(command_line, true);
     command_line = wrapped_command;
-    LOG(INFO) << "BROWSER_WRAPPER was set, prefixing command_line with " << browser_wrapper;
+    LOG(INFO) << "BROWSER_WRAPPER was set, prefixing command_line with "
+              << browser_wrapper;
   }
 
   bool started = base::LaunchApp(command_line.argv(),
@@ -812,6 +813,16 @@ void UITest::PrintResult(const std::string& measurement,
                    "", "", units, important);
 }
 
+void UITest::PrintResult(const std::string& measurement,
+                         const std::string& modifier,
+                         const std::string& trace,
+                         const std::string& value,
+                         const std::string& units,
+                         bool important) {
+  PrintResultsImpl(measurement, modifier, trace, value, "", "", units,
+                   important);
+}
+
 void UITest::PrintResultMeanAndError(const std::string& measurement,
                                      const std::string& modifier,
                                      const std::string& trace,
diff --git a/chrome/test/ui/ui_test.h b/chrome/test/ui/ui_test.h
index 6f35cdc..c059ded 100644
--- a/chrome/test/ui/ui_test.h
+++ b/chrome/test/ui/ui_test.h
@@ -201,6 +201,15 @@ class UITest : public testing::Test {
                    const std::string& units,
                    bool important);
 
+  // Like the above version of PrintResult(), but takes a std::string value
+  // instead of a size_t.
+  void PrintResult(const std::string& measurement,
+                   const std::string& modifier,
+                   const std::string& trace,
+                   const std::string& value,
+                   const std::string& units,
+                   bool important);
+
   // Like PrintResult(), but prints a (mean, standard deviation) result pair.
   // The |<values>| should be two comma-seaprated numbers, the mean and
   // standard deviation (or other error metric) of the measurement.
diff --git a/chrome/test/ui/ui_tests.scons b/chrome/test/ui/ui_tests.scons
index 85f7cf3..a3f7d51 100644
--- a/chrome/test/ui/ui_tests.scons
+++ b/chrome/test/ui/ui_tests.scons
@@ -98,6 +98,8 @@ input_files = ChromeFileList([
     '$CHROME_DIR/tools/build/win/precompiled_wtl$OBJSUFFIX',
     '$CHROME_DIR/tools/build/win/precompiled_wtl.h',
     'run_all_unittests.cc',
+    'javascript_test_util.cc',
+    'javascript_test_util.h',
     '$CHROME_DIR/test/testing_browser_process.h',
     'ui_test.cc',
     'ui_test.h',
@@ -232,6 +234,9 @@ input_files = ChromeFileList([
   MSVSFilter('TestSunSpider', [
     'sunspider_uitest.cc',
   ]),
+  MSVSFilter('TestV8Benchmark', [
+    'v8_benchmark_uitest.cc',
+  ]),
 ])
 
 if not env.Bit('windows'):
@@ -243,6 +248,7 @@ if not env.Bit('windows'):
       'omnibox_uitest.cc',
       'dom_checker_uitest.cc',
       'sunspider_uitest.cc',
+      'v8_benchmark_uitest.cc',
 
       'sandbox_uitests.cc',
 
diff --git a/chrome/test/ui/ui_tests.vcproj b/chrome/test/ui/ui_tests.vcproj
index 29e906e..5a3ed55 100644
--- a/chrome/test/ui/ui_tests.vcproj
+++ b/chrome/test/ui/ui_tests.vcproj
@@ -178,6 +178,14 @@
 				>
 			</File>
 			<File
+				RelativePath=".\javascript_test_util.cc"
+				>
+			</File>
+			<File
+				RelativePath=".\javascript_test_util.h"
+				>
+			</File>
+			<File
 				RelativePath=".\ui_test.cc"
 				>
 			</File>
@@ -558,6 +566,14 @@
 				>
 			</File>
 		</Filter>
+		<Filter
+			Name="TestV8Benchmark"
+			>
+			<File
+				RelativePath=".\v8_benchmark_uitest.cc"
+				>
+			</File>
+		</Filter>
 	</Files>
 	<Globals>
 	</Globals>
diff --git a/chrome/test/ui/v8_benchmark_uitest.cc b/chrome/test/ui/v8_benchmark_uitest.cc
new file mode 100644
index 0000000..19656de
--- /dev/null
+++ b/chrome/test/ui/v8_benchmark_uitest.cc
@@ -0,0 +1,150 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/command_line.h"
+#include "base/file_path.h"
+#include "base/file_util.h"
+#include "base/path_service.h"
+#include "base/string_util.h"
+#include "base/values.h"
+#include "chrome/common/chrome_paths.h"
+#include "chrome/common/chrome_switches.h"
+#include "chrome/common/json_value_serializer.h"
+#include "chrome/test/automation/tab_proxy.h"
+#include "chrome/test/ui/javascript_test_util.h"
+#include "chrome/test/ui/ui_test.h"
+#include "googleurl/src/gurl.h"
+#include "net/base/net_util.h"
+
+namespace {
+
+static const FilePath::CharType kStartFile[] =
+    FILE_PATH_LITERAL("run.html");
+
+const wchar_t kRunV8Benchmark[] = L"run-v8-benchmark";
+
+class V8BenchmarkTest : public UITest {
+ public:
+  typedef std::map<std::string, std::string> ResultsMap;
+
+  V8BenchmarkTest() : reference_(false) {
+    dom_automation_enabled_ = true;
+    show_window_ = true;
+  }
+
+  void RunTest() {
+    FilePath::StringType start_file(kStartFile);
+    FilePath test_path = GetV8BenchmarkDir();
+    test_path = test_path.Append(start_file);
+    GURL test_url(net::FilePathToFileURL(test_path));
+
+    scoped_ptr<TabProxy> tab(GetActiveTab());
+    tab->NavigateToURL(test_url);
+
+    // Wait for the test to finish.
+    ASSERT_TRUE(WaitUntilTestCompletes(tab.get(), test_url));
+
+    PrintResults(tab.get());
+  }
+
+ protected:
+  bool reference_;  // True if this is a reference build.
+
+ private:
+  // Return the path to the V8 benchmark directory on the local filesystem.
+  FilePath GetV8BenchmarkDir() {
+    FilePath test_dir;
+    PathService::Get(chrome::DIR_TEST_DATA, &test_dir);
+    return test_dir.AppendASCII("v8_benchmark");
+  }
+
+  bool WaitUntilTestCompletes(TabProxy* tab, const GURL& test_url) {
+    return WaitUntilCookieValue(tab, test_url, "__done", 1000,
+                                UITest::test_timeout_ms(), "1");
+  }
+
+  bool GetScore(TabProxy* tab, std::string* score) {
+    std::wstring score_wide;
+    bool succeeded = tab->ExecuteAndExtractString(L"",
+        L"window.domAutomationController.send(automation.GetScore());",
+        &score_wide);
+
+    // Note that we don't use ASSERT_TRUE here (and in some other places) as it
+    // doesn't work inside a function with a return type other than void.
+    EXPECT_TRUE(succeeded);
+    if (!succeeded)
+      return false;
+
+    score->assign(WideToUTF8(score_wide));
+    return true;
+  }
+
+  bool GetResults(TabProxy* tab, ResultsMap* results) {
+    std::wstring json_wide;
+    bool succeeded = tab->ExecuteAndExtractString(L"",
+        L"window.domAutomationController.send("
+        L"    JSON.stringify(automation.GetResults()));",
+        &json_wide);
+
+    EXPECT_TRUE(succeeded);
+    if (!succeeded)
+      return false;
+
+    std::string json = WideToUTF8(json_wide);
+    return JsonDictionaryToMap(json, results);
+  }
+
+  void PrintResults(TabProxy* tab) {
+    std::string score;
+    ASSERT_TRUE(GetScore(tab, &score));
+
+    ResultsMap results;
+    ASSERT_TRUE(GetResults(tab, &results));
+
+    std::string trace_name = reference_ ? "score_ref" : "score";
+    std::string unit_name = "score (bigger is better)";
+
+    PrintResult("score", "", trace_name, score, unit_name, true);
+
+    ResultsMap::const_iterator it = results.begin();
+    for (; it != results.end(); ++it)
+      PrintResult(it->first, "", trace_name, it->second, unit_name, false);
+  }
+
+  DISALLOW_COPY_AND_ASSIGN(V8BenchmarkTest);
+};
+
+class V8BenchmarkReferenceTest : public V8BenchmarkTest {
+ public:
+  V8BenchmarkReferenceTest() : V8BenchmarkTest() {
+    reference_ = true;
+  }
+
+  // Override the browser directory that is used by UITest::SetUp to cause it
+  // to use the reference build instead.
+  void SetUp() {
+    FilePath dir;
+    PathService::Get(chrome::DIR_TEST_TOOLS, &dir);
+    dir = dir.AppendASCII("reference_build");
+    dir = dir.AppendASCII("chrome");
+    browser_directory_ = dir.ToWStringHack();
+    UITest::SetUp();
+  }
+};
+
+}  // namespace
+
+TEST_F(V8BenchmarkTest, Perf) {
+  if (!CommandLine::ForCurrentProcess()->HasSwitch(kRunV8Benchmark))
+    return;
+
+  RunTest();
+}
+
+TEST_F(V8BenchmarkReferenceTest, Perf) {
+  if (!CommandLine::ForCurrentProcess()->HasSwitch(kRunV8Benchmark))
+    return;
+
+  RunTest();
+}
diff --git a/chrome/test/ui/v8_benchmark_uitest.js b/chrome/test/ui/v8_benchmark_uitest.js
new file mode 100644
index 0000000..9950c5a
--- /dev/null
+++ b/chrome/test/ui/v8_benchmark_uitest.js
@@ -0,0 +1,34 @@
+/*
+  Copyright (c) 2009 The Chromium Authors. All rights reserved.
+  Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file.
+*/
+
+// Automation utilities for running the V8 Benchmark as a UI test.
+
+function Automation() {
+  this.score = '';
+  this.results = {};
+}
+
+Automation.prototype.SetDone = function() {
+  document.cookie = '__done=1; path=/';
+}
+
+Automation.prototype.SetScore = function (score) {
+  this.score = score;
+}
+
+Automation.prototype.GetScore = function() {
+  return this.score;
+}
+
+Automation.prototype.AddResult = function(name, result) {
+  this.results[name] = result;
+}
+
+Automation.prototype.GetResults = function() {
+  return this.results;
+}
+
+automation = new Automation();