GPU thread acknowledges an armed watchdog every time it returns to the message loop.

It used to only acknowledge upon receiving a particular kind of Task from the watchdog, which could potentially be delayed by other Tasks.

TEST=WebGL locally, check recovery from about:gpuhang locally, try
BUG=none

Review URL: http://codereview.chromium.org/5278006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@67184 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 89 insertions, 14 deletions

diff --git a/chrome/gpu/gpu_watchdog_thread.cc b/chrome/gpu/gpu_watchdog_thread.cc
index e262c79..36f7e6b 100644
--- a/chrome/gpu/gpu_watchdog_thread.cc
+++ b/chrome/gpu/gpu_watchdog_thread.cc
@@ -13,21 +13,38 @@
 
 namespace {
 const int64 kCheckPeriod = 2000;
+
+void DoNothing() {
+}
 }
 
 GpuWatchdogThread::GpuWatchdogThread(MessageLoop* watched_message_loop,
                                      int timeout)
     : base::Thread("Watchdog"),
       watched_message_loop_(watched_message_loop),
-      timeout_(timeout) {
+      timeout_(timeout),
+      armed_(false),
+      ALLOW_THIS_IN_INITIALIZER_LIST(task_observer_(this)) {
   DCHECK(watched_message_loop);
   DCHECK(timeout >= 0);
+
+  watched_message_loop_->AddTaskObserver(&task_observer_);
 }
 
 GpuWatchdogThread::~GpuWatchdogThread() {
   // Verify that the thread was explicitly stopped. If the thread is stopped
   // implicitly by the destructor, CleanUp() will not be called.
   DCHECK(!method_factory_.get());
+
+  watched_message_loop_->RemoveTaskObserver(&task_observer_);
+}
+
+void GpuWatchdogThread::PostAcknowledge() {
+  // Called on the monitored thread. Responds with OnAcknowledge. Cannot use
+  // the method factory. Rely on reference counting instead.
+  message_loop()->PostTask(
+      FROM_HERE,
+      NewRunnableMethod(this, &GpuWatchdogThread::OnAcknowledge));
 }
 
 void GpuWatchdogThread::Init() {
@@ -47,9 +64,46 @@ void GpuWatchdogThread::CleanUp() {
   watched_message_loop_ = NULL;
 }
 
+GpuWatchdogThread::GpuWatchdogTaskObserver::GpuWatchdogTaskObserver(
+    GpuWatchdogThread* watchdog)
+  : watchdog_(watchdog) {
+}
+
+GpuWatchdogThread::GpuWatchdogTaskObserver::~GpuWatchdogTaskObserver() {
+}
+
+void GpuWatchdogThread::GpuWatchdogTaskObserver::WillProcessTask(
+    const Task* task)
+{
+  CheckArmed();
+}
+
+void GpuWatchdogThread::GpuWatchdogTaskObserver::DidProcessTask(
+    const Task* task)
+{
+  CheckArmed();
+}
+
+void GpuWatchdogThread::GpuWatchdogTaskObserver::CheckArmed()
+{
+  // Acknowledge the watchdog if it has armed itself. The watchdog will not
+  // change its armed state until it is acknowledged.
+  if (watchdog_->armed()) {
+    watchdog_->PostAcknowledge();
+  }
+}
+
 void GpuWatchdogThread::OnAcknowledge() {
+  // The check has already been acknowledged and another has already been
+  // scheduled by a previous call to OnAcknowledge. It is normal for a
+  // watched thread to see armed_ being true multiple times before
+  // the OnAcknowledge task is run on the watchdog thread.
+  if (!armed_)
+    return;
+
   // Revoke any pending OnExit.
   method_factory_->RevokeAll();
+  armed_ = false;
 
   // The monitored thread has responded. Post a task to check it again.
   if (watched_message_loop_) {
@@ -62,13 +116,19 @@ void GpuWatchdogThread::OnAcknowledge() {
 
 void GpuWatchdogThread::OnCheck() {
   if (watched_message_loop_) {
-    // Post a task to the monitored thread that simply responds with a task that
-    // calls OnAcknowldge.
+    // Must set armed before posting the task. This task might be the only task
+    // that will activate the TaskObserver on the watched thread and it must not
+    // miss the false -> true transition.
+    armed_ = true;
+
+    // Post a task to the monitored thread that does nothing but wake up the
+    // TaskObserver. Any other tasks that are pending on the watched thread will
+    // also wake up the observer. This simply ensures there is at least one.
     watched_message_loop_->PostTask(
         FROM_HERE,
-        NewRunnableMethod(this, &GpuWatchdogThread::PostAcknowledge));
+        NewRunnableFunction(DoNothing));
 
-    // Post a task to the watchdog thread to exit if the nmonitored thread does
+    // Post a task to the watchdog thread to exit if the monitored thread does
     // not respond in time.
     message_loop()->PostDelayedTask(
         FROM_HERE,
@@ -77,14 +137,6 @@ void GpuWatchdogThread::OnCheck() {
   }
 }
 
-void GpuWatchdogThread::PostAcknowledge() {
-  // Called on the monitored thread. Responds with OnAcknowledge. Cannot use
-  // the method factory. Rely on reference counting instead.
-  message_loop()->PostTask(
-      FROM_HERE,
-      NewRunnableMethod(this, &GpuWatchdogThread::OnAcknowledge));
-}
-
 // Use the --disable-gpu-watchdog command line switch to disable this.
 void GpuWatchdogThread::OnExit() {
   // Make sure the timeout period is on the stack before crashing.
diff --git a/chrome/gpu/gpu_watchdog_thread.h b/chrome/gpu/gpu_watchdog_thread.h
index d6e1117..19cc41f 100644
--- a/chrome/gpu/gpu_watchdog_thread.h
+++ b/chrome/gpu/gpu_watchdog_thread.h
@@ -5,6 +5,7 @@
 #ifndef CHROME_GPU_GPU_WATCHDOG_THREAD_H_
 #define CHROME_GPU_GPU_WATCHDOG_THREAD_H_
 
+#include "base/message_loop.h"
 #include "base/ref_counted.h"
 #include "base/scoped_ptr.h"
 #include "base/task.h"
@@ -18,19 +19,41 @@ class GpuWatchdogThread : public base::Thread,
   GpuWatchdogThread(MessageLoop* watched_message_loop, int timeout);
   virtual ~GpuWatchdogThread();
 
+  // Accessible on watched thread but only modified by watchdog thread.
+  bool armed() const { return armed_; }
+  void PostAcknowledge();
+
  protected:
   virtual void Init();
   virtual void CleanUp();
 
  private:
+
+  // An object of this type intercepts the reception and completion of all tasks
+  // on the watched thread and checks whether the watchdog is armed.
+  class GpuWatchdogTaskObserver : public MessageLoop::TaskObserver {
+   public:
+    explicit GpuWatchdogTaskObserver(GpuWatchdogThread* watchdog);
+    virtual ~GpuWatchdogTaskObserver();
+
+    // Implements MessageLoop::TaskObserver.
+    virtual void WillProcessTask(const Task* task);
+    virtual void DidProcessTask(const Task* task);
+
+   private:
+    void CheckArmed();
+    GpuWatchdogThread* watchdog_;
+  };
+
   void OnAcknowledge();
   void OnCheck();
-  void PostAcknowledge();
   void OnExit();
   void Disable();
 
   MessageLoop* watched_message_loop_;
   int timeout_;
+  volatile bool armed_;
+  GpuWatchdogTaskObserver task_observer_;
 
   typedef ScopedRunnableMethodFactory<GpuWatchdogThread> MethodFactory;
   scoped_ptr<MethodFactory> method_factory_;