Add some shutdown perf tests. We measure shutdown for the combination of

shutdown method: {Close all windows, clicking exit, terminating process}
and number of tabs open: {just about:blank, twenty tabs}

BUG=23118
TEST=This CL adds a test.
Review URL: http://codereview.chromium.org/1594008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@43758 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 233 insertions, 70 deletions

diff --git a/chrome/browser/automation/automation_provider.cc b/chrome/browser/automation/automation_provider.cc
index addc2fd..7e96223 100644
--- a/chrome/browser/automation/automation_provider.cc
+++ b/chrome/browser/automation/automation_provider.cc
@@ -917,9 +917,9 @@ void AutomationProvider::ExecuteBrowserCommand(
     IDC_HOME,
     IDC_SELECT_NEXT_TAB,
     IDC_SELECT_PREVIOUS_TAB,
+    IDC_SHOW_BOOKMARK_MANAGER,
     IDC_SHOW_DOWNLOADS,
     IDC_SHOW_HISTORY,
-    IDC_SHOW_BOOKMARK_MANAGER,
   };
   if (browser_tracker_->ContainsHandle(handle)) {
     Browser* browser = browser_tracker_->GetResource(handle);
@@ -1707,7 +1707,8 @@ class SetProxyConfigTask : public Task {
  public:
   SetProxyConfigTask(URLRequestContextGetter* request_context_getter,
                      const std::string& new_proxy_config)
-      : request_context_getter_(request_context_getter), proxy_config_(new_proxy_config) {}
+      : request_context_getter_(request_context_getter),
+        proxy_config_(new_proxy_config) {}
   virtual void Run() {
     // First, deserialize the JSON string. If this fails, log and bail.
     JSONStringValueSerializer deserializer(proxy_config_);
diff --git a/chrome/chrome_tests.gypi b/chrome/chrome_tests.gypi
index cc21528..3cd313c 100644
--- a/chrome/chrome_tests.gypi
+++ b/chrome/chrome_tests.gypi
@@ -1410,6 +1410,7 @@
       ],
       'sources': [
         'test/startup/feature_startup_test.cc',
+        'test/startup/shutdown_test.cc',
         'test/startup/startup_test.cc',
       ],
       'conditions': [
diff --git a/chrome/common/env_vars.cc b/chrome/common/env_vars.cc
index 9a0457f..dc2648c 100644
--- a/chrome/common/env_vars.cc
+++ b/chrome/common/env_vars.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -34,4 +34,7 @@ const char kLtrLocale[] = "LEFT_TO_RIGHT";
 // according to the process.
 const char kNoOOBreakpad[] = "NO_OO_BREAKPAD";
 
+// Number of times to run a given startup_tests unit test.
+const char kStartupTestsNumCycles[] = "STARTUP_TESTS_NUMCYCLES";
+
 }  // namespace env_vars
diff --git a/chrome/common/env_vars.h b/chrome/common/env_vars.h
index ed83d1e..1c5bc5d 100644
--- a/chrome/common/env_vars.h
+++ b/chrome/common/env_vars.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -17,6 +17,7 @@ extern const char kRestartInfo[];
 extern const char kRtlLocale[];
 extern const char kLtrLocale[];
 extern const char kNoOOBreakpad[];
+extern const char kStartupTestsNumCycles[];
 
 }  // namespace env_vars
 
diff --git a/chrome/test/startup/shutdown_test.cc b/chrome/test/startup/shutdown_test.cc
new file mode 100644
index 0000000..fcb620a
--- /dev/null
+++ b/chrome/test/startup/shutdown_test.cc
@@ -0,0 +1,122 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/env_var.h"
+#include "base/file_util.h"
+#include "base/path_service.h"
+#include "base/platform_thread.h"
+#include "base/string_util.h"
+#include "base/sys_info.h"
+#include "base/time.h"
+#include "chrome/common/chrome_constants.h"
+#include "chrome/common/chrome_paths.h"
+#include "chrome/common/chrome_switches.h"
+#include "chrome/common/env_vars.h"
+#include "chrome/test/ui/ui_test.h"
+#include "chrome/test/ui_test_utils.h"
+
+using base::TimeDelta;
+
+namespace {
+
+class ShutdownTest : public UITest {
+ public:
+  ShutdownTest() {
+    show_window_ = true;
+  }
+  void SetUp() {}
+  void TearDown() {}
+
+  void SetUpTwentyTabs() {
+    const FilePath kFastShutdownDir(FILE_PATH_LITERAL("fast_shutdown"));
+    const FilePath kCurrentDir(FilePath::kCurrentDirectory);
+    const FilePath test_cases[] = {
+      ui_test_utils::GetTestFilePath(kFastShutdownDir,
+          FilePath(FILE_PATH_LITERAL("on_before_unloader.html"))),
+      ui_test_utils::GetTestFilePath(kCurrentDir,
+          FilePath(FILE_PATH_LITERAL("animated-gifs.html"))),
+      ui_test_utils::GetTestFilePath(kCurrentDir,
+          FilePath(FILE_PATH_LITERAL("french_page.html"))),
+      ui_test_utils::GetTestFilePath(kCurrentDir,
+          FilePath(FILE_PATH_LITERAL("onunload_cookie.html"))),
+    };
+
+    for (size_t i = 0; i < arraysize(test_cases); i++) {
+      ASSERT_TRUE(file_util::PathExists(test_cases[i]));
+      for (size_t j = 0; j < 5; j++) {
+        launch_arguments_.AppendLooseValue(test_cases[i].ToWStringHack());
+      }
+    }
+  }
+
+  void RunShutdownTest(const char* graph, const char* trace, bool important,
+                       UITest::ShutdownType shutdown_type) {
+    const int kNumCyclesMax = 20;
+    int numCycles = kNumCyclesMax;
+    scoped_ptr<base::EnvVarGetter> env(base::EnvVarGetter::Create());
+    std::string numCyclesEnv;
+    if (env->GetEnv(env_vars::kStartupTestsNumCycles, &numCyclesEnv) &&
+        StringToInt(numCyclesEnv, &numCycles)) {
+      LOG(INFO) << env_vars::kStartupTestsNumCycles << " set in environment, "
+                << "so setting numCycles to " << numCycles;
+    }
+
+    TimeDelta timings[kNumCyclesMax];
+    for (int i = 0; i < numCycles; ++i) {
+      UITest::SetUp();
+      set_shutdown_type(shutdown_type);
+      UITest::TearDown();
+      timings[i] = browser_quit_time_;
+
+      if (i == 0) {
+        // Re-use the profile data after first run so that the noise from
+        // creating databases doesn't impact all the runs.
+        clear_profile_ = false;
+        // Clear template_user_data_ so we don't try to copy it over each time
+        // through.
+        set_template_user_data(FilePath());
+      }
+    }
+
+    std::string times;
+    for (int i = 0; i < numCycles; ++i)
+      StringAppendF(&times, "%.2f,", timings[i].InMillisecondsF());
+    PrintResultList(graph, "", trace, times, "ms", important);
+  }
+};
+
+TEST_F(ShutdownTest, SimpleWindowClose) {
+  RunShutdownTest("shutdown", "simple-window-close",
+                  true, /* important */ UITest::WINDOW_CLOSE);
+}
+
+TEST_F(ShutdownTest, SimpleUserQuit) {
+  RunShutdownTest("shutdown", "simple-user-quit",
+                  true, /* important */ UITest::USER_QUIT);
+}
+
+TEST_F(ShutdownTest, SimpleSessionEnding) {
+  RunShutdownTest("shutdown", "simple-session-ending",
+                  true, /* important */ UITest::SESSION_ENDING);
+}
+
+TEST_F(ShutdownTest, TwentyTabsWindowClose) {
+  SetUpTwentyTabs();
+  RunShutdownTest("shutdown", "twentytabs-window-close",
+                  true, /* important */ UITest::WINDOW_CLOSE);
+}
+
+TEST_F(ShutdownTest, TwentyTabsUserQuit) {
+  SetUpTwentyTabs();
+  RunShutdownTest("shutdown", "twentytabs-user-quit",
+                  true, /* important */ UITest::USER_QUIT);
+}
+
+TEST_F(ShutdownTest, TwentyTabsSessionEnding) {
+  SetUpTwentyTabs();
+  RunShutdownTest("shutdown", "twentytabs-session-ending",
+                  true, /* important */ UITest::SESSION_ENDING);
+}
+
+}  // namespace
diff --git a/chrome/test/startup/startup_test.cc b/chrome/test/startup/startup_test.cc
index 9611bc1..ff9ec7e 100644
--- a/chrome/test/startup/startup_test.cc
+++ b/chrome/test/startup/startup_test.cc
@@ -1,18 +1,21 @@
-// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include "base/env_var.h"
 #include "base/file_util.h"
 #include "base/path_service.h"
 #include "base/platform_thread.h"
 #include "base/string_util.h"
+#include "base/sys_info.h"
 #include "base/test/test_file_util.h"
 #include "base/time.h"
 #include "chrome/common/chrome_constants.h"
 #include "chrome/common/chrome_paths.h"
 #include "chrome/common/chrome_switches.h"
+#include "chrome/common/env_vars.h"
 #include "chrome/test/ui/ui_test.h"
-#include "net/base/net_util.h"
+#include "chrome/test/ui_test_utils.h"
 
 using base::TimeDelta;
 using base::TimeTicks;
@@ -23,7 +26,6 @@ class StartupTest : public UITest {
  public:
   StartupTest() {
     show_window_ = true;
-    pages_ = "about:blank";
   }
   void SetUp() {}
   void TearDown() {}
@@ -31,13 +33,11 @@ class StartupTest : public UITest {
   // Load a file on startup rather than about:blank.  This tests a longer
   // startup path, including resource loading and the loading of gears.dll.
   void SetUpWithFileURL() {
-    FilePath file_url;
-    ASSERT_TRUE(PathService::Get(chrome::DIR_TEST_DATA, &file_url));
-    file_url = file_url.AppendASCII("simple.html");
+    const FilePath file_url = ui_test_utils::GetTestFilePath(
+        FilePath(FilePath::kCurrentDirectory),
+        FilePath(FILE_PATH_LITERAL("simple.html")));
     ASSERT_TRUE(file_util::PathExists(file_url));
     launch_arguments_.AppendLooseValue(file_url.ToWStringHack());
-
-    pages_ = WideToUTF8(file_url.ToWStringHack());
   }
 
   // Use the given profile in the test data extensions/profiles dir.  This tests
@@ -66,14 +66,13 @@ class StartupTest : public UITest {
 
     const int kNumCyclesMax = 20;
     int numCycles = kNumCyclesMax;
-// It's ok for unit test code to use getenv(), isn't it?
-#if defined(OS_WIN)
-#pragma warning( disable : 4996 )
-#endif
-    const char* numCyclesEnv = getenv("STARTUP_TESTS_NUMCYCLES");
-    if (numCyclesEnv && StringToInt(numCyclesEnv, &numCycles))
-      LOG(INFO) << "STARTUP_TESTS_NUMCYCLES set in environment, "
+    scoped_ptr<base::EnvVarGetter> env(base::EnvVarGetter::Create());
+    std::string numCyclesEnv;
+    if (env->GetEnv(env_vars::kStartupTestsNumCycles, &numCyclesEnv) &&
+        StringToInt(numCyclesEnv, &numCycles)) {
+      LOG(INFO) << env_vars::kStartupTestsNumCycles << " set in environment, "
                 << "so setting numCycles to " << numCycles;
+    }
 
     TimeDelta timings[kNumCyclesMax];
     for (int i = 0; i < numCycles; ++i) {
@@ -102,9 +101,6 @@ class StartupTest : public UITest {
       UITest::SetUp();
       TimeTicks end_time = TimeTicks::Now();
       timings[i] = end_time - browser_launch_time_;
-      // TODO(beng): Can't shut down so quickly. Figure out why, and fix. If we
-      // do, we crash.
-      PlatformThread::Sleep(50);
       UITest::TearDown();
 
       if (i == 0) {
@@ -122,9 +118,6 @@ class StartupTest : public UITest {
       StringAppendF(&times, "%.2f,", timings[i].InMillisecondsF());
     PrintResultList(graph, "", trace, times, "ms", important);
   }
-
- protected:
-  std::string pages_;
 };
 
 TEST_F(StartupTest, PerfWarm) {
diff --git a/chrome/test/ui/ui_test.cc b/chrome/test/ui/ui_test.cc
index 9ad2aa1..9c3c778 100644
--- a/chrome/test/ui/ui_test.cc
+++ b/chrome/test/ui/ui_test.cc
@@ -1,9 +1,14 @@
-// Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/test/ui/ui_test.h"
 
+#if defined(OS_POSIX)
+#include <signal.h>
+#include <sys/types.h>
+#endif
+
 #include <set>
 #include <vector>
 
@@ -40,6 +45,8 @@
 #endif
 
 
+using base::Time;
+using base::TimeDelta;
 using base::TimeTicks;
 
 // Delay to let browser complete a requested action.
@@ -112,7 +119,8 @@ UITestBase::UITestBase()
       use_existing_browser_(default_use_existing_browser_),
       enable_file_cookies_(true),
       profile_type_(UITestBase::DEFAULT_THEME),
-      test_start_time_(base::Time::NowFromSystemTime()),
+      shutdown_type_(UITestBase::WINDOW_CLOSE),
+      test_start_time_(Time::NowFromSystemTime()),
       command_execution_timeout_ms_(kCommandExecutionTimeout),
       action_timeout_ms_(kWaitForActionMsec),
       action_max_timeout_ms_(kWaitForActionMaxMsec),
@@ -137,7 +145,8 @@ UITestBase::UITestBase(MessageLoop::Type msg_loop_type)
       use_existing_browser_(default_use_existing_browser_),
       enable_file_cookies_(true),
       profile_type_(UITestBase::DEFAULT_THEME),
-      test_start_time_(base::Time::NowFromSystemTime()),
+      shutdown_type_(UITestBase::WINDOW_CLOSE),
+      test_start_time_(Time::NowFromSystemTime()),
       command_execution_timeout_ms_(kCommandExecutionTimeout),
       action_timeout_ms_(kWaitForActionMsec),
       action_max_timeout_ms_(kWaitForActionMaxMsec),
@@ -412,41 +421,58 @@ bool UITestBase::LaunchAnotherBrowserBlockUntilClosed(
 }
 
 void UITestBase::QuitBrowser() {
+  if (SESSION_ENDING == shutdown_type_) {
+    TerminateBrowser();
+    return;
+  }
+
   // There's nothing to do here if the browser is not running.
   if (IsBrowserRunning()) {
+    TimeTicks quit_start = TimeTicks::Now();
     EXPECT_TRUE(automation()->SetFilteredInet(false));
 
-    int window_count = 0;
-    EXPECT_TRUE(automation()->GetBrowserWindowCount(&window_count));
+    if (WINDOW_CLOSE == shutdown_type_) {
+      int window_count = 0;
+      EXPECT_TRUE(automation()->GetBrowserWindowCount(&window_count));
+
+      // Synchronously close all but the last browser window. Closing them
+      // one-by-one may help with stability.
+      while (window_count > 1) {
+        scoped_refptr<BrowserProxy> browser_proxy =
+            automation()->GetBrowserWindow(0);
+        EXPECT_TRUE(browser_proxy.get());
+        if (browser_proxy.get()) {
+          EXPECT_TRUE(browser_proxy->RunCommand(IDC_CLOSE_WINDOW));
+          EXPECT_TRUE(automation()->GetBrowserWindowCount(&window_count));
+        } else {
+          break;
+        }
+      }
 
-    // Synchronously close all but the last browser window. Closing them
-    // one-by-one may help with stability.
-    while (window_count > 1) {
+      // Close the last window asynchronously, because the browser may
+      // shutdown faster than it will be able to send a synchronous response
+      // to our message.
       scoped_refptr<BrowserProxy> browser_proxy =
           automation()->GetBrowserWindow(0);
       EXPECT_TRUE(browser_proxy.get());
       if (browser_proxy.get()) {
-        EXPECT_TRUE(browser_proxy->RunCommand(IDC_CLOSE_WINDOW));
-        EXPECT_TRUE(automation()->GetBrowserWindowCount(&window_count));
-      } else {
-        break;
+        EXPECT_TRUE(browser_proxy->ApplyAccelerator(IDC_CLOSE_WINDOW));
+        browser_proxy = NULL;
       }
-    }
-
-    // Close the last window asynchronously, because the browser may shutdown
-    // faster than it will be able to send a synchronous response to our
-    // message.
-    scoped_refptr<BrowserProxy> browser_proxy =
-        automation()->GetBrowserWindow(0);
-    EXPECT_TRUE(browser_proxy.get());
-    if (browser_proxy.get()) {
-      EXPECT_TRUE(browser_proxy->ApplyAccelerator(IDC_CLOSE_WINDOW));
-      browser_proxy = NULL;
+    } else if (USER_QUIT == shutdown_type_) {
+      scoped_refptr<BrowserProxy> browser_proxy =
+          automation()->GetBrowserWindow(0);
+      EXPECT_TRUE(browser_proxy.get());
+      if (browser_proxy.get()) {
+        EXPECT_TRUE(browser_proxy->RunCommandAsync(IDC_EXIT));
+      }
+    } else {
+      NOTREACHED() << "Invalid shutdown type " << shutdown_type_;
     }
 
     // Now, drop the automation IPC channel so that the automation provider in
     // the browser notices and drops its reference to the browser process.
-    server_->Disconnect();
+    automation()->Disconnect();
 
     // Wait for the browser process to quit. It should quit once all tabs have
     // been closed.
@@ -459,6 +485,7 @@ void UITestBase::QuitBrowser() {
       // enough. Take no chance and kill every chrome processes.
       CleanupAppProcesses();
     }
+    browser_quit_time_ = TimeTicks::Now() - quit_start;
   }
 
   // Don't forget to close the handle
@@ -468,34 +495,34 @@ void UITestBase::QuitBrowser() {
 }
 
 void UITestBase::TerminateBrowser() {
-  // There's nothing to do here if the browser is not running.
-  if (!IsBrowserRunning())
-    return;
-
-  EXPECT_TRUE(automation()->SetFilteredInet(false));
+  if (IsBrowserRunning()) {
+    TimeTicks quit_start = TimeTicks::Now();
+    EXPECT_TRUE(automation()->SetFilteredInet(false));
 #if defined(OS_WIN)
-  scoped_refptr<BrowserProxy> browser(automation()->GetBrowserWindow(0));
-  ASSERT_TRUE(browser.get());
-  ASSERT_TRUE(browser->TerminateSession());
+    scoped_refptr<BrowserProxy> browser(automation()->GetBrowserWindow(0));
+    ASSERT_TRUE(browser.get());
+    ASSERT_TRUE(browser->TerminateSession());
 #endif  // defined(OS_WIN)
 
-  // Now, drop the automation IPC channel so that the automation provider in
-  // the browser notices and drops its reference to the browser process.
-  automation()->Disconnect();
+    // Now, drop the automation IPC channel so that the automation provider in
+    // the browser notices and drops its reference to the browser process.
+    automation()->Disconnect();
 
 #if defined(OS_POSIX)
-  EXPECT_EQ(kill(process_, SIGTERM), 0);
+    EXPECT_EQ(kill(process_, SIGTERM), 0);
 #endif  // OS_POSIX
 
-  // Wait for the browser process to quit.
-  int timeout = terminate_timeout_ms_;
+    // Wait for the browser process to quit.
+    int timeout = terminate_timeout_ms_;
 #ifdef WAIT_FOR_DEBUGGER_ON_OPEN
-  timeout = 500000;
+    timeout = 500000;
 #endif
-  if (!base::WaitForSingleProcess(process_, timeout)) {
-    // We need to force the browser to quit because it didn't quit fast
-    // enough. Take no chance and kill every chrome processes.
-    CleanupAppProcesses();
+    if (!base::WaitForSingleProcess(process_, timeout)) {
+      // We need to force the browser to quit because it didn't quit fast
+      // enough. Take no chance and kill every chrome processes.
+      CleanupAppProcesses();
+    }
+    browser_quit_time_ = TimeTicks::Now() - quit_start;
   }
 
   // Don't forget to close the handle
@@ -1224,7 +1251,7 @@ void UITestBase::UpdateHistoryDates() {
     return;
 
   ASSERT_TRUE(db.Open(history));
-  base::Time yesterday = base::Time::Now() - base::TimeDelta::FromDays(1);
+  Time yesterday = Time::Now() - TimeDelta::FromDays(1);
   std::string yesterday_str = Int64ToString(yesterday.ToInternalValue());
   std::string query = StringPrintf(
       "UPDATE segment_usage "
diff --git a/chrome/test/ui/ui_test.h b/chrome/test/ui/ui_test.h
index 50ab471..3d939f6 100644
--- a/chrome/test/ui/ui_test.h
+++ b/chrome/test/ui/ui_test.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -441,6 +441,18 @@ class UITestBase {
     homepage_ = homepage;
   }
 
+  // Different ways to quit the browser.
+  typedef enum {
+    WINDOW_CLOSE,
+    USER_QUIT,
+    SESSION_ENDING,
+  } ShutdownType;
+
+  // Sets the shutdown type, which defaults to WINDOW_CLOSE.
+  void set_shutdown_type(ShutdownType value) {
+    shutdown_type_ = value;
+  }
+
   // Count the number of active browser processes launched by this test.
   // The count includes browser sub-processes.
   int GetBrowserProcessCount();
@@ -547,6 +559,7 @@ class UITestBase {
   bool wait_for_initial_loads_;         // Wait for initial loads to complete
                                         // in SetUp() before running test body.
   base::TimeTicks browser_launch_time_; // Time when the browser was run.
+  base::TimeDelta browser_quit_time_;   // How long the shutdown took.
   bool dom_automation_enabled_;         // This can be set to true to have the
                                         // test run the dom automation case.
   FilePath template_user_data_;         // See set_template_user_data().
@@ -567,6 +580,8 @@ class UITestBase {
   ProfileType profile_type_;            // Are we using a profile with a
                                         // complex theme?
   FilePath websocket_pid_file_;         // PID file for websocket server.
+  ShutdownType shutdown_type_;          // The method for shutting down
+                                        // the browser. Used in ShutdownTest.
 
  private:
   bool LaunchBrowserHelper(const CommandLine& arguments,