GPU process terminates on hang.

I added a watchdog thread that intermitently checks the main thread can respond to tasks posted on its message queue.

I fixed some bugs that preventede GGL from failing when the GPU channel was lost.

Added a command line swith to disable the watchdog thread for debugging purposes.

TEST=try, check WebGL works, check about:gpuhang terminates process.
BUG=38739,53871

Review URL: http://codereview.chromium.org/3528012

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@61718 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 154 insertions, 0 deletions

diff --git a/chrome/gpu/gpu_main.cc b/chrome/gpu/gpu_main.cc
index 5ffd39f..e1dd911 100644
--- a/chrome/gpu/gpu_main.cc
+++ b/chrome/gpu/gpu_main.cc
@@ -11,6 +11,7 @@
 #include "chrome/gpu/gpu_config.h"
 #include "chrome/gpu/gpu_process.h"
 #include "chrome/gpu/gpu_thread.h"
+#include "chrome/gpu/gpu_watchdog_thread.h"
 
 #if defined(USE_LINUX_BREAKPAD)
 #include "chrome/app/breakpad_linux.h"
@@ -65,12 +66,21 @@ int GpuMain(const MainFunctionParams& parameters) {
   GpuProcess gpu_process;
   gpu_process.set_main_thread(new GpuThread());
 
+  scoped_refptr<GpuWatchdogThread> watchdog_thread(
+      new GpuWatchdogThread(MessageLoop::current()));
+
+  if (!command_line.HasSwitch(switches::kDisableGpuWatchdog))
+    watchdog_thread->Start();
+
 #if defined(USE_X11)
   SetGpuX11ErrorHandlers();
 #endif
 
   main_message_loop.Run();
 
+  if (!command_line.HasSwitch(switches::kDisableGpuWatchdog))
+    watchdog_thread->Stop();
+
   return 0;
 }
 
diff --git a/chrome/gpu/gpu_watchdog_thread.cc b/chrome/gpu/gpu_watchdog_thread.cc
new file mode 100644
index 0000000..ccd9ff3
--- /dev/null
+++ b/chrome/gpu/gpu_watchdog_thread.cc
@@ -0,0 +1,104 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#if defined(OS_WIN)
+#include <windows.h>
+#endif
+
+#include "chrome/gpu/gpu_watchdog_thread.h"
+
+#include "base/compiler_specific.h"
+#include "build/build_config.h"
+
+namespace {
+const int64 kCheckPeriod = 2000;
+const int64 kTimeoutPeriod = 2000;
+}
+
+GpuWatchdogThread::GpuWatchdogThread(MessageLoop* watched_message_loop)
+    : base::Thread("Watchdog"),
+      watched_message_loop_(watched_message_loop) {
+  DCHECK(watched_message_loop);
+}
+
+GpuWatchdogThread::~GpuWatchdogThread() {
+  // Verify that the thread was explicitly stopped. If the thread is stopped
+  // implicitly by the destructor, CleanUp() will not be called.
+  DCHECK(!method_factory_.get());
+}
+
+void GpuWatchdogThread::Init() {
+  // The method factory must be created on the watchdog thread.
+  method_factory_.reset(new MethodFactory(this));
+
+  // Schedule the first check.
+  OnCheck();
+}
+
+void GpuWatchdogThread::CleanUp() {
+  // The method factory must be destroyed on the watchdog thread.
+  method_factory_->RevokeAll();
+  method_factory_.reset();
+
+  // Prevent any more delayed tasks from being posted.
+  watched_message_loop_ = NULL;
+}
+
+void GpuWatchdogThread::OnAcknowledge() {
+  // Revoke any pending OnExit.
+  method_factory_->RevokeAll();
+
+  // The monitored thread has responded. Post a task to check it again.
+  if (watched_message_loop_) {
+    message_loop()->PostDelayedTask(
+        FROM_HERE,
+        method_factory_->NewRunnableMethod(&GpuWatchdogThread::OnCheck),
+        kCheckPeriod);
+  }
+}
+
+void GpuWatchdogThread::OnCheck() {
+  if (watched_message_loop_) {
+    // Post a task to the monitored thread that simply responds with a task that
+    // calls OnAcknowldge.
+    watched_message_loop_->PostTask(
+        FROM_HERE,
+        NewRunnableMethod(this, &GpuWatchdogThread::PostAcknowledge));
+
+    // Post a task to the watchdog thread to exit if the nmonitored thread does
+    // not respond in time.
+    message_loop()->PostDelayedTask(
+        FROM_HERE,
+        method_factory_->NewRunnableMethod(&GpuWatchdogThread::OnExit),
+        kTimeoutPeriod);
+  }
+}
+
+void GpuWatchdogThread::PostAcknowledge() {
+  // Called on the monitored thread. Responds with OnAcknowledge. Cannot use
+  // the method factory. Rely on reference counting instead.
+  message_loop()->PostTask(
+      FROM_HERE,
+      NewRunnableMethod(this, &GpuWatchdogThread::OnAcknowledge));
+}
+
+// Use the --disable-gpu-watchdog command line switch to disable this.
+void GpuWatchdogThread::OnExit() {
+  // For minimal developer annoyance, don't keep crashing.
+  static bool crashed = false;
+  if (crashed)
+    return;
+
+#if defined(OS_WIN)
+  if (IsDebuggerPresent())
+    return;
+#endif
+
+  LOG(ERROR) << "The GPU process hung. Restarting.";
+
+  volatile int* null_pointer = NULL;
+  *null_pointer = 0xb0a710ad;  // Boatload (of cycles)
+
+  crashed = true;
+}
diff --git a/chrome/gpu/gpu_watchdog_thread.h b/chrome/gpu/gpu_watchdog_thread.h
new file mode 100644
index 0000000..6dbbc32
--- /dev/null
+++ b/chrome/gpu/gpu_watchdog_thread.h
@@ -0,0 +1,40 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_GPU_GPU_WATCHDOG_THREAD_H_
+#define CHROME_GPU_GPU_WATCHDOG_THREAD_H_
+
+#include "base/ref_counted.h"
+#include "base/scoped_ptr.h"
+#include "base/task.h"
+#include "base/thread.h"
+
+// A thread that intermitently sends tasks to a group of watched message loops
+// and deliberately crashes if one of them does not respond after a timeout.
+class GpuWatchdogThread : public base::Thread,
+                          public base::RefCountedThreadSafe<GpuWatchdogThread> {
+ public:
+  explicit GpuWatchdogThread(MessageLoop* watched_message_loop);
+  virtual ~GpuWatchdogThread();
+
+ protected:
+  virtual void Init();
+  virtual void CleanUp();
+
+ private:
+  void OnAcknowledge();
+  void OnCheck();
+  void PostAcknowledge();
+  void OnExit();
+  void Disable();
+
+  MessageLoop* watched_message_loop_;
+
+  typedef ScopedRunnableMethodFactory<GpuWatchdogThread> MethodFactory;
+  scoped_ptr<MethodFactory> method_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(GpuWatchdogThread);
+};
+
+#endif  // CHROME_GPU_GPU_WATCHDOG_THREAD_H_