Windows: fix failure to recover from device lost in AcceleratedSurface.

When device lost is detected, null out or recreate all the resources and shaders used by the old device. They will not work on the new device. The old shaders created by the AcceleratedSurfaceTransformer need to be recreated. It should also recreate the shared texture and swap chain because they won't otherwise get recreated without window resizing and / or tab switching.

I moved the lock from AcceleratedPresenter to PresentThread so that one AcceratedPresenter can destroy resources of others that use the same D3D device without risking deadlock.

Check for device lost while polling the event query. Sometimes GetData does not seem to report the loss of the device.

Also, treat S_PRESENT_MODE_CHANGED as a lost device because sometimes CheckDeviceState returns that after a GPU hang.

I backed out the change to terminate the GPU process on any D3D errors in DoPresentAndAcknowledge. I don't think that was the real issue.

BUG=170767,170875

Review URL: https://chromiumcodereview.appspot.com/12252053

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@182984 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 112 insertions, 53 deletions

diff --git a/ui/surface/accelerated_surface_transformer_win.cc b/ui/surface/accelerated_surface_transformer_win.cc
index 30fd1af..834c5b8 100644
--- a/ui/surface/accelerated_surface_transformer_win.cc
+++ b/ui/surface/accelerated_surface_transformer_win.cc
@@ -181,35 +181,35 @@ bool AcceleratedSurfaceTransformer::DoInit(IDirect3DDevice9* device) {
 
 bool AcceleratedSurfaceTransformer::CompileShaderCombo(
     ShaderCombo shader) {
-  if (!vertex_shaders_[shader]) {
-    HRESULT hr = device_->CreateVertexShader(
-        reinterpret_cast<const DWORD*>(vertex_shader_sources_[shader]),
-        vertex_shaders_[shader].Receive());
+  vertex_shaders_[shader] = NULL;
 
-    if (FAILED(hr))
-      return false;
+  HRESULT hr = device_->CreateVertexShader(
+      reinterpret_cast<const DWORD*>(vertex_shader_sources_[shader]),
+      vertex_shaders_[shader].Receive());
+
+  if (FAILED(hr))
+    return false;
 
-    for (int i = 0; i < NUM_SHADERS; ++i) {
-      if (vertex_shader_sources_[i] == vertex_shader_sources_[shader] &&
-          i != shader) {
-        vertex_shaders_[i] = vertex_shaders_[shader];
-      }
+  for (int i = 0; i < NUM_SHADERS; ++i) {
+    if (vertex_shader_sources_[i] == vertex_shader_sources_[shader] &&
+        i != shader) {
+      vertex_shaders_[i] = vertex_shaders_[shader];
     }
   }
 
-  if (!pixel_shaders_[shader]) {
-    HRESULT hr = device_->CreatePixelShader(
-        reinterpret_cast<const DWORD*>(pixel_shader_sources_[shader]),
-        pixel_shaders_[shader].Receive());
+  pixel_shaders_[shader] = NULL;
 
-    if (FAILED(hr))
-      return false;
+  hr = device_->CreatePixelShader(
+      reinterpret_cast<const DWORD*>(pixel_shader_sources_[shader]),
+      pixel_shaders_[shader].Receive());
+
+  if (FAILED(hr))
+    return false;
 
-    for (int i = 0; i < NUM_SHADERS; ++i) {
-      if (pixel_shader_sources_[i] == pixel_shader_sources_[shader] &&
-          i != shader) {
-        pixel_shaders_[i] = pixel_shaders_[shader];
-      }
+  for (int i = 0; i < NUM_SHADERS; ++i) {
+    if (pixel_shader_sources_[i] == pixel_shader_sources_[shader] &&
+        i != shader) {
+      pixel_shaders_[i] = pixel_shaders_[shader];
     }
   }
 
diff --git a/ui/surface/accelerated_surface_win.cc b/ui/surface/accelerated_surface_win.cc
index a3ecb03..2b6adc8 100644
--- a/ui/surface/accelerated_surface_win.cc
+++ b/ui/surface/accelerated_surface_win.cc
@@ -115,6 +115,11 @@ class PresentThread : public base::Thread,
 
   void InitDevice();
   void ResetDevice();
+  bool IsDeviceLost();
+
+  base::Lock* lock() {
+    return &lock_;
+  }
 
  protected:
   virtual void Init();
@@ -125,6 +130,10 @@ class PresentThread : public base::Thread,
 
   ~PresentThread();
 
+  // The lock is taken while any thread is calling an AcceleratedPresenter
+  // associated with this thread.
+  base::Lock lock_;
+
   base::ScopedNativeLibrary d3d_module_;
   base::win::ScopedComPtr<IDirect3DDevice9Ex> device_;
   // This query is used to wait until a certain amount of progress has been
@@ -163,6 +172,11 @@ class AcceleratedPresenterMap {
   scoped_refptr<AcceleratedPresenter> GetPresenter(
       gfx::PluginWindowHandle window);
 
+
+  // Destroy any D3D resources owned by the given present thread. Called on
+  // the given present thread.
+  void ResetPresentThread(PresentThread* present_thread);
+
  private:
   base::Lock lock_;
   typedef std::map<gfx::PluginWindowHandle, AcceleratedPresenter*> PresenterMap;
@@ -191,11 +205,18 @@ void PresentThread::InitDevice() {
 void PresentThread::ResetDevice() {
   TRACE_EVENT0("gpu", "PresentThread::ResetDevice");
 
+  LOG(ERROR) << "Reseting D3D device";
+
+  // The D3D device must be created on the present thread.
+  CHECK(message_loop() == MessageLoop::current());
+
   // This will crash some Intel drivers but we can't render anything without
   // reseting the device, which would be disappointing.
   query_ = NULL;
   device_ = NULL;
 
+  g_accelerated_presenter_map.Pointer()->ResetPresentThread(this);
+
   if (!d3d_utils::CreateDevice(d3d_module_,
                                D3DDEVTYPE_HAL,
                                GetPresentationInterval(),
@@ -218,6 +239,11 @@ void PresentThread::ResetDevice() {
   }
 }
 
+bool PresentThread::IsDeviceLost() {
+  HRESULT hr = device_->CheckDeviceState(NULL);
+  return FAILED(hr) || hr == S_PRESENT_MODE_CHANGED;
+}
+
 void PresentThread::Init() {
   TRACE_EVENT0("gpu", "Initialize thread");
 }
@@ -296,6 +322,17 @@ scoped_refptr<AcceleratedPresenter> AcceleratedPresenterMap::GetPresenter(
   return it->second;
 }
 
+void AcceleratedPresenterMap::ResetPresentThread(
+    PresentThread* present_thread) {
+  base::AutoLock locked(lock_);
+
+  for (PresenterMap::iterator it = presenters_.begin();
+      it != presenters_.end();
+      ++it) {
+    it->second->ResetPresentThread(present_thread);
+  }
+}
+
 AcceleratedPresenter::AcceleratedPresenter(gfx::PluginWindowHandle window)
     : present_thread_(g_present_thread_pool.Pointer()->NextThread()),
       window_(window),
@@ -335,7 +372,7 @@ void AcceleratedPresenter::AsyncPresentAndAcknowledge(
 void AcceleratedPresenter::Present(HDC dc) {
   TRACE_EVENT0("gpu", "Present");
 
-  base::AutoLock locked(lock_);
+  base::AutoLock locked(*present_thread_->lock());
 
   // If invalidated, do nothing. The window is gone.
   if (!window_)
@@ -406,7 +443,7 @@ bool AcceleratedPresenter::DoCopyToARGB(const gfx::Rect& requested_src_subrect,
       "width", dst_size.width(),
       "height", dst_size.height());
 
-  base::AutoLock locked(lock_);
+  base::AutoLock locked(*present_thread_->lock());
 
   if (!swap_chain_)
     return false;
@@ -503,7 +540,7 @@ bool AcceleratedPresenter::DoCopyToYUV(
       "width", dst_size.width(),
       "height", dst_size.height());
 
-  base::AutoLock locked(lock_);
+  base::AutoLock locked(*present_thread_->lock());
 
   if (!swap_chain_)
     return false;
@@ -589,7 +626,7 @@ void AcceleratedPresenter::Suspend() {
 }
 
 void AcceleratedPresenter::WasHidden() {
-  base::AutoLock locked(lock_);
+  base::AutoLock locked(*present_thread_->lock());
   hidden_ = true;
 }
 
@@ -609,10 +646,25 @@ void AcceleratedPresenter::Invalidate() {
   // last pending task has been ignored, the reference count on the presenter
   // will go to zero and the presenter, and potentially also the present thread
   // it has a reference count on, will be destroyed.
-  base::AutoLock locked(lock_);
+  base::AutoLock locked(*present_thread_->lock());
   window_ = NULL;
 }
 
+void AcceleratedPresenter::ResetPresentThread(
+    PresentThread* present_thread) {
+  TRACE_EVENT0("gpu", "ResetPresentThread");
+
+  // present_thread_ can be accessed without the lock because it is immutable.
+  if (present_thread_ != present_thread)
+    return;
+
+  present_thread_->lock()->AssertAcquired();
+
+  source_texture_ = NULL;
+  swap_chain_ = NULL;
+  quantized_size_ = gfx::Size();
+}
+
 #if defined(USE_AURA)
 void AcceleratedPresenter::SetNewTargetWindow(gfx::PluginWindowHandle window) {
   window_ = window;
@@ -633,7 +685,7 @@ void AcceleratedPresenter::DoPresentAndAcknowledge(
 
   HRESULT hr;
 
-  base::AutoLock locked(lock_);
+  base::AutoLock locked(*present_thread_->lock());
 
   // Initialize the device lazily since calling Direct3D can crash bots.
   present_thread_->InitDevice();
@@ -644,10 +696,13 @@ void AcceleratedPresenter::DoPresentAndAcknowledge(
     return;
   }
 
+  // Ensure the task is acknowledged on early out after this point.
+  base::ScopedClosureRunner scoped_completion_runner(
+      base::Bind(completion_task, true, base::TimeTicks(), base::TimeDelta()));
+
   // If invalidated, do nothing, the window is gone.
   if (!window_) {
     TRACE_EVENT0("gpu", "EarlyOut_NoWindow");
-    completion_task.Run(true, base::TimeTicks(), base::TimeDelta());
     return;
   }
 
@@ -673,15 +728,10 @@ void AcceleratedPresenter::DoPresentAndAcknowledge(
     TRACE_EVENT2("gpu", "EarlyOut_WrongWindowSize2",
                  "windowwidth", window_size.width(),
                  "windowheight", window_size.height());
-    completion_task.Run(true, base::TimeTicks(), base::TimeDelta());
     return;
   }
 #endif
 
-  // Ensure the task is notified of failure on early out after this point.
-  base::ScopedClosureRunner scoped_completion_runner(
-      base::Bind(completion_task, false, base::TimeTicks(), base::TimeDelta()));
-
   // Round up size so the swap chain is not continuously resized with the
   // surface, which could lead to memory fragmentation.
   const int kRound = 64;
@@ -781,10 +831,10 @@ void AcceleratedPresenter::DoPresentAndAcknowledge(
     // For latency_tests.cc:
     UNSHIPPED_TRACE_EVENT_INSTANT0("test_gpu", "CompositorSwapBuffersComplete");
 
-    if (FAILED(hr) &&
-        FAILED(present_thread_->device()->CheckDeviceState(window_))) {
-      LOG(ERROR) << "Reseting D3D device";
-      present_thread_->ResetDevice();
+    if (FAILED(hr)) {
+      if (present_thread_->IsDeviceLost())
+        present_thread_->ResetDevice();
+      return;
     }
   } else {
     HDC dc = GetDC(window_);
@@ -792,12 +842,6 @@ void AcceleratedPresenter::DoPresentAndAcknowledge(
     ReleaseDC(window_, dc);
   }
 
-  // Early out if failed to reset device.
-  if (!present_thread_->device()) {
-    LOG(ERROR) << "Failed to reset device";
-    return;
-  }
-
   hidden_ = false;
 
   D3DDISPLAYMODE display_mode;
@@ -841,11 +885,17 @@ void AcceleratedPresenter::DoPresentAndAcknowledge(
   // that it is safe to write to its backing store again.
   {
     TRACE_EVENT0("gpu", "spin");
+
     do {
       hr = present_thread_->query()->GetData(NULL, 0, D3DGETDATA_FLUSH);
-
-      if (hr == S_FALSE)
+      if (hr == S_FALSE) {
         Sleep(1);
+
+        if (present_thread_->IsDeviceLost()) {
+          present_thread_->ResetDevice();
+          return;
+        }
+      }
     } while (hr == S_FALSE);
   }
 
@@ -854,12 +904,12 @@ void AcceleratedPresenter::DoPresentAndAcknowledge(
 }
 
 void AcceleratedPresenter::DoSuspend() {
-  base::AutoLock locked(lock_);
+  base::AutoLock locked(*present_thread_->lock());
   swap_chain_ = NULL;
 }
 
 void AcceleratedPresenter::DoReleaseSurface() {
-  base::AutoLock locked(lock_);
+  base::AutoLock locked(*present_thread_->lock());
   present_thread_->InitDevice();
   source_texture_.Release();
 }
@@ -909,6 +959,16 @@ void AcceleratedPresenter::PresentWithGDI(HDC dc) {
     TRACE_EVENT0("gpu", "GetRenderTargetData");
     hr = present_thread_->device()->GetRenderTargetData(back_buffer,
                                                         system_surface);
+
+    if (FAILED(hr)) {
+      if (present_thread_->IsDeviceLost()) {
+        present_thread_->message_loop()->PostTask(
+            FROM_HERE,
+            base::Bind(&PresentThread::ResetDevice, present_thread_));
+      }
+      return;
+    }
+
     DCHECK(SUCCEEDED(hr));
   }
 
diff --git a/ui/surface/accelerated_surface_win.h b/ui/surface/accelerated_surface_win.h
index eebaf1a..454f9f5d 100644
--- a/ui/surface/accelerated_surface_win.h
+++ b/ui/surface/accelerated_surface_win.h
@@ -76,6 +76,10 @@ class SURFACE_EXPORT AcceleratedPresenter
       const base::Callback<void(bool)>& callback);
   void Invalidate();
 
+  // Destroy any D3D resources owned by the given present thread. Called on
+  // the given present thread.
+  void ResetPresentThread(PresentThread* present_thread);
+
 #if defined(USE_AURA)
   // TODO(scottmg): This is a temporary hack until we have a two-worlds ash/aura
   // separation.
@@ -127,11 +131,6 @@ class SURFACE_EXPORT AcceleratedPresenter
   // The window that is presented to.
   gfx::PluginWindowHandle window_;
 
-  // The lock is taken while any thread is calling the object, except those that
-  // simply post from the main thread to the present thread via the immutable
-  // present_thread_ member.
-  base::Lock lock_;
-
   // UI thread can wait on this event to ensure a present is finished.
   base::WaitableEvent event_;