Startup hang - Dump the call stacks of all threads if

ThreadWatcherdoesn't start watching all threads in 5 mins.

In Debug mode, it doesa DCHECK and in release mode it dumps
the callstack on windows, butdoesn't crash the browser.

TODO: need to implement code to get stack traces on other
platforms. This is a partial fix to chromium-os:20487.

BUG=97997, chromium-os:20487
Test=ThreadWatcher and startup tests
R=jar

Review URL: http://codereview.chromium.org/8041003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@104644 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 104 insertions, 6 deletions

diff --git a/chrome/browser/chrome_browser_main.cc b/chrome/browser/chrome_browser_main.cc
index 98d594b..4ca0e85 100644
--- a/chrome/browser/chrome_browser_main.cc
+++ b/chrome/browser/chrome_browser_main.cc
@@ -1674,6 +1674,10 @@ int ChromeBrowserMainParts::PreMainMessageLoopRunInternal() {
   // testing against a bunch of special cases that are taken care early on.
   PrepareRestartOnCrashEnviroment(parsed_command_line());
 
+  // Start watching for hangs during startup. We disarm this hang detector when
+  // ThreadWatcher takes over or when browser is shutdown.
+  StartupTimeBomb::Arm(base::TimeDelta::FromSeconds(300));
+
 #if defined(OS_WIN)
   // Registers Chrome with the Windows Restart Manager, which will restore the
   // Chrome session when the computer is restarted after a system update.
@@ -1955,6 +1959,9 @@ void ChromeBrowserMainParts::PostMainMessageLoopRun() {
   // |shutdown_watcher_| object is destructed.
   shutdown_watcher_->Arm(base::TimeDelta::FromSeconds(90));
 
+  // Disarm the startup hang detector time bomb if it is still Arm'ed.
+  StartupTimeBomb::Disarm();
+
 #if defined(OS_WIN)
   // If it's the first run, log the search engine chosen.  We wait until
   // shutdown because otherwise we can't be sure the user has finished
diff --git a/chrome/browser/metrics/thread_watcher.cc b/chrome/browser/metrics/thread_watcher.cc
index fb4f4bc..272bbc9 100644
--- a/chrome/browser/metrics/thread_watcher.cc
+++ b/chrome/browser/metrics/thread_watcher.cc
@@ -14,6 +14,7 @@
 #include "chrome/browser/metrics/metrics_service.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/common/chrome_version_info.h"
+#include "chrome/common/logging_chrome.h"
 #include "content/common/notification_service.h"
 
 #if defined(OS_WIN)
@@ -492,6 +493,11 @@ void ThreadWatcherList::InitializeAndStartWatching(
   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
                 unresponsive_threshold, crash_on_hang_thread_names,
                 live_threads_threshold);
+
+  BrowserThread::PostTask(
+      BrowserThread::UI,
+      FROM_HERE,
+      NewRunnableFunction(StartupTimeBomb::Disarm));
 }
 
 // static
@@ -677,9 +683,34 @@ void WatchDogThread::CleanUp() {
 
 namespace {
 
+// StartupWatchDogThread methods and members.
+//
+// Class for detecting hangs during startup.
+class StartupWatchDogThread : public base::Watchdog {
+ public:
+  // Constructor specifies how long the StartupWatchDogThread will wait before
+  // alarming.
+  explicit StartupWatchDogThread(const base::TimeDelta& duration)
+      : base::Watchdog(duration, "Startup watchdog thread", true) {
+  }
+
+  // Alarm is called if the time expires after an Arm() without someone calling
+  // Disarm(). When Alarm goes off, in release mode we get the crash dump
+  // without crashing and in debug mode we break into the debugger.
+  virtual void Alarm() {
+#ifndef NDEBUG
+    DCHECK(false);
+#else
+    logging::DumpWithoutCrashing();
+#endif
+  }
+
+  DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
+};
+
 // ShutdownWatchDogThread methods and members.
 //
-// Class for watching the jank during shutdown.
+// Class for detecting hangs during shutdown.
 class ShutdownWatchDogThread : public base::Watchdog {
  public:
   // Constructor specifies how long the ShutdownWatchDogThread will wait before
@@ -698,9 +729,35 @@ class ShutdownWatchDogThread : public base::Watchdog {
 };
 }  // namespace
 
+// StartupTimeBomb methods and members.
+//
+// static
+base::Watchdog* StartupTimeBomb::startup_watchdog_ = NULL;
+
+// static
+void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  DCHECK(!startup_watchdog_);
+  startup_watchdog_ = new StartupWatchDogThread(duration);
+  startup_watchdog_->Arm();
+}
+
+// static
+void StartupTimeBomb::Disarm() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  if (startup_watchdog_) {
+    startup_watchdog_->Disarm();
+    // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
+    // very fast.
+    base::ThreadRestrictions::SetIOAllowed(true);
+    delete startup_watchdog_;
+    startup_watchdog_ = NULL;
+  }
+}
+
 // ShutdownWatcherHelper methods and members.
 //
-// ShutdownWatcherHelper is a wrapper class for watching the jank during
+// ShutdownWatcherHelper is a wrapper class for detecting hangs during
 // shutdown.
 ShutdownWatcherHelper::ShutdownWatcherHelper() : shutdown_watchdog_(NULL) {
 }
diff --git a/chrome/browser/metrics/thread_watcher.h b/chrome/browser/metrics/thread_watcher.h
index 56e87ad..703713a 100644
--- a/chrome/browser/metrics/thread_watcher.h
+++ b/chrome/browser/metrics/thread_watcher.h
@@ -64,6 +64,7 @@
 #include "content/common/notification_registrar.h"
 
 class CustomThreadWatcher;
+class StartupTimeBomb;
 class ThreadWatcherList;
 class ThreadWatcherObserver;
 
@@ -358,7 +359,7 @@ class ThreadWatcherList {
 
   // This constructs the |ThreadWatcherList| singleton and starts watching
   // browser threads by calling StartWatching() on each browser thread that is
-  // watched.
+  // watched. It disarms StartupTimeBomb.
   static void InitializeAndStartWatching(
       uint32 unresponsive_threshold,
       const std::set<std::string>& crash_on_hang_thread_names,
@@ -493,7 +494,27 @@ class WatchDogThread : public base::Thread {
   DISALLOW_COPY_AND_ASSIGN(WatchDogThread);
 };
 
-// This is a wrapper class for watching the jank during shutdown.
+// This is a wrapper class for getting the crash dumps of the hangs during
+// startup.
+class StartupTimeBomb {
+ public:
+  // Constructs |startup_watchdog_| which spawns a thread and starts timer.
+  // |duration| specifies how long |startup_watchdog_| will wait before it
+  // calls alarm.
+  static void Arm(const base::TimeDelta& duration);
+
+  // Disarms |startup_watchdog_| thread and then deletes it which stops the
+  // Watchdog thread.
+  static void Disarm();
+
+ private:
+  // Watches for hangs during startup until it is disarm'ed.
+  static base::Watchdog* startup_watchdog_;
+
+  DISALLOW_COPY_AND_ASSIGN(StartupTimeBomb);
+};
+
+// This is a wrapper class for detecting hangs during shutdown.
 class ShutdownWatcherHelper {
  public:
   // Create an empty holder for |shutdown_watchdog_|.
@@ -507,7 +528,7 @@ class ShutdownWatcherHelper {
   void Arm(const base::TimeDelta& duration);
 
  private:
-  // shutdown_watchdog_ watches the jank during shutdown.
+  // shutdown_watchdog_ watches for hangs during shutdown.
   base::Watchdog* shutdown_watchdog_;
 
   DISALLOW_COPY_AND_ASSIGN(ShutdownWatcherHelper);
diff --git a/chrome/common/logging_chrome.cc b/chrome/common/logging_chrome.cc
index 8c21213..0b7f726 100644
--- a/chrome/common/logging_chrome.cc
+++ b/chrome/common/logging_chrome.cc
@@ -449,4 +449,14 @@ size_t GetFatalAssertions(AssertionList* assertions) {
   return assertion_count;
 }
 
+
+void DumpWithoutCrashing() {
+#if defined(OS_WIN)
+  std::string str;
+  DumpProcessAssertHandler(str);
+#else
+  NOTIMPLEMENTED();
+#endif  // OS_WIN
+}
+
 }  // namespace logging
diff --git a/chrome/common/logging_chrome.h b/chrome/common/logging_chrome.h
index 6fc1389..74f5835 100644
--- a/chrome/common/logging_chrome.h
+++ b/chrome/common/logging_chrome.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -66,6 +66,9 @@ typedef std::vector<std::wstring> AssertionList;
 // the program writing the log has terminated.
 size_t GetFatalAssertions(AssertionList* assertions);
 
+// Handler to silently dump the current process without crashing.
+void DumpWithoutCrashing();
+
 } // namespace logging
 
 #endif  // CHROME_COMMON_LOGGING_CHROME_H_