Re-land of http://codereview.chromium.org/6094009 with Mac build fix.

Perform GPU-related initialization in GPU process in response to an
IPC from the browser. Because Chromium's child process host detects
that the child has crashed by watching for IPC channel errors, it is
imperative that the GPU process's IPC channel be set up before it does
any work that might cause it to crash. If initialization fails, the
GPU process quits its message loop and cooperatively exits.

Fixed a bug in the GpuProcessHost where it would not unblock renderers
waiting for GPU process initialization if the GPU process exited.

BUG=65369
TEST=ran test case from bug on machine with no GPU hardware and
verified that it no longer hangs the renderer, and that the GPU
process exits

Review URL: http://codereview.chromium.org/6124002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@70747 0039d316-1c4b-4281-b951-d872f2087c98

6 files changed, 164 insertions, 104 deletions

diff --git a/chrome/browser/gpu_process_host.cc b/chrome/browser/gpu_process_host.cc
index b0d7931..8e7ac68 100644
--- a/chrome/browser/gpu_process_host.cc
+++ b/chrome/browser/gpu_process_host.cc
@@ -97,6 +97,9 @@ bool GpuProcessHost::EnsureInitialized() {
   if (!initialized_) {
     initialized_ = true;
     initialized_successfully_ = Init();
+    if (initialized_successfully_) {
+      Send(new GpuMsg_Initialize());
+    }
   }
   return initialized_successfully_;
 }
@@ -228,10 +231,13 @@ void GpuProcessHost::OnChannelEstablished(
 }
 
 void GpuProcessHost::OnSynchronizeReply() {
-  const SynchronizationRequest& request =
-      queued_synchronization_replies_.front();
-  SendSynchronizationReply(request.reply, request.filter);
-  queued_synchronization_replies_.pop();
+  // Guard against race conditions in abrupt GPU process termination.
+  if (queued_synchronization_replies_.size() > 0) {
+    const SynchronizationRequest& request =
+        queued_synchronization_replies_.front();
+    SendSynchronizationReply(request.reply, request.filter);
+    queued_synchronization_replies_.pop();
+  }
 }
 
 #if defined(OS_LINUX)
@@ -480,11 +486,26 @@ void GpuProcessHost::SendSynchronizationReply(
   filter->Send(reply);
 }
 
+void GpuProcessHost::SendOutstandingReplies() {
+  // First send empty channel handles for all EstablishChannel requests.
+  while (!sent_requests_.empty()) {
+    const ChannelRequest& request = sent_requests_.front();
+    SendEstablishChannelReply(IPC::ChannelHandle(), GPUInfo(), request.filter);
+    sent_requests_.pop();
+  }
+
+  // Now unblock all renderers waiting for synchronization replies.
+  while (!queued_synchronization_replies_.empty()) {
+    OnSynchronizeReply();
+  }
+}
+
 bool GpuProcessHost::CanShutdown() {
   return true;
 }
 
 void GpuProcessHost::OnChildDied() {
+  SendOutstandingReplies();
   // Located in OnChildDied because OnProcessCrashed suffers from a race
   // condition on Linux. The GPU process will only die if it crashes.
   UMA_HISTOGRAM_ENUMERATION("GPU.GPUProcessLifetimeEvents",
@@ -493,6 +514,7 @@ void GpuProcessHost::OnChildDied() {
 }
 
 void GpuProcessHost::OnProcessCrashed(int exit_code) {
+  SendOutstandingReplies();
   if (++g_gpu_crash_count >= kGpuMaxCrashCount) {
     // The gpu process is too unstable to use. Disable it for current session.
     RenderViewHostDelegateHelper::set_gpu_enabled(false);
diff --git a/chrome/browser/gpu_process_host.h b/chrome/browser/gpu_process_host.h
index e3bc66a..9a7a90d 100644
--- a/chrome/browser/gpu_process_host.h
+++ b/chrome/browser/gpu_process_host.h
@@ -107,6 +107,11 @@ class GpuProcessHost : public BrowserChildProcessHost,
   void SendSynchronizationReply(IPC::Message* reply,
                                 RenderMessageFilter* filter);
 
+  // Sends outstanding replies to renderer processes. This is only called
+  // in error situations like the GPU process crashing -- but is necessary
+  // to prevent the renderer process from hanging.
+  void SendOutstandingReplies();
+
   virtual bool CanShutdown();
   virtual void OnChildDied();
   virtual void OnProcessCrashed(int exit_code);
diff --git a/chrome/common/gpu_messages_internal.h b/chrome/common/gpu_messages_internal.h
index 22f2f9d..22da39a 100644
--- a/chrome/common/gpu_messages_internal.h
+++ b/chrome/common/gpu_messages_internal.h
@@ -25,6 +25,14 @@ class GPUInfo;
 //------------------------------------------------------------------------------
 // GPU Messages
 // These are messages from the browser to the GPU process.
+
+// Tells the GPU process to initialize itself. The browser explicitly
+// requests this be done so that we are guaranteed that the channel is set
+// up between the browser and GPU process before doing any work that might
+// potentially crash the GPU process. Detection of the child process
+// exiting abruptly is predicated on having the IPC channel set up.
+IPC_MESSAGE_CONTROL0(GpuMsg_Initialize)
+
 // Tells the GPU process to create a new channel for communication with a
 // given renderer.  The channel name is returned in a
 // GpuHostMsg_ChannelEstablished message.  The renderer ID is passed so that
diff --git a/chrome/gpu/gpu_main.cc b/chrome/gpu/gpu_main.cc
index 5d0e090..9bd6568 100644
--- a/chrome/gpu/gpu_main.cc
+++ b/chrome/gpu/gpu_main.cc
@@ -4,9 +4,6 @@
 
 #include <stdlib.h>
 
-#include "app/app_switches.h"
-#include "app/gfx/gl/gl_context.h"
-#include "app/gfx/gl/gl_implementation.h"
 #include "app/win/scoped_com_initializer.h"
 #include "base/environment.h"
 #include "base/message_loop.h"
@@ -20,7 +17,6 @@
 #include "chrome/gpu/gpu_config.h"
 #include "chrome/gpu/gpu_process.h"
 #include "chrome/gpu/gpu_thread.h"
-#include "chrome/gpu/gpu_watchdog_thread.h"
 
 #if defined(USE_LINUX_BREAKPAD)
 #include "chrome/app/breakpad_linux.h"
@@ -28,31 +24,12 @@
 
 #if defined(OS_MACOSX)
 #include "chrome/common/chrome_application_mac.h"
-#include "chrome/common/sandbox_mac.h"
 #endif
 
 #if defined(USE_X11)
 #include "gfx/gtk_util.h"
 #endif
 
-const int kGpuTimeout = 10000;
-
-namespace {
-
-bool InitializeGpuSandbox() {
-#if defined(OS_MACOSX)
-  CommandLine* parsed_command_line = CommandLine::ForCurrentProcess();
-  SandboxInitWrapper sandbox_wrapper;
-  return sandbox_wrapper.InitializeSandbox(*parsed_command_line,
-                                           switches::kGpuProcess);
-#else
-  // TODO(port): Create GPU sandbox for linux and windows.
-  return true;
-#endif
-}
-
-}  // namespace
-
 // Main function for starting the Gpu process.
 int GpuMain(const MainFunctionParams& parameters) {
   base::Time start_time = base::Time::Now();
@@ -86,69 +63,21 @@ int GpuMain(const MainFunctionParams& parameters) {
   gfx::GtkInitFromCommandLine(command_line);
 #endif
 
-  // Note that kNoSandbox will also disable the GPU sandbox.
-  bool no_gpu_sandbox = command_line.HasSwitch(switches::kNoGpuSandbox);
-  if (!no_gpu_sandbox) {
-    if (!InitializeGpuSandbox()) {
-      LOG(ERROR) << "Failed to initialize the GPU sandbox";
-      return EXIT_FAILURE;
-    }
-  } else {
-    LOG(ERROR) << "Running without GPU sandbox";
-  }
-
-  // Load the GL implementation and locate the bindings before starting the GPU
-  // watchdog because this can take a lot of time and the GPU watchdog might
-  // terminate the GPU process.
-  if (!gfx::GLContext::InitializeOneOff())
-    return EXIT_FAILURE;
-
-  // Do this soon before running the message loop so accurate
-  // initialization time is recorded in the GPU info. Don't do it before
-  // starting the watchdog thread since it can take a significant amount of
-  // time to collect GPU information in GpuThread::Init.
+  // We can not tolerate early returns from this code, because the
+  // detection of early return of a child process is implemented using
+  // an IPC channel error. If the IPC channel is not fully set up
+  // between the browser and GPU process, and the GPU process crashes
+  // or exits early, the browser process will never detect it.  For
+  // this reason we defer all work related to the GPU until receiving
+  // the GpuMsg_Initialize message from the browser.
   GpuProcess gpu_process;
-  GpuThread* gpu_thread = new GpuThread;
+  GpuThread* gpu_thread = new GpuThread(command_line);
   gpu_thread->Init(start_time);
   gpu_process.set_main_thread(gpu_thread);
 
-
-  // In addition to disabling the watchdog if the command line switch is
-  // present, disable it in two other cases. OSMesa is expected to run very
-  // slowly.  Also disable the watchdog on valgrind because the code is expected
-  // to run slowly in that case.
-  bool enable_watchdog =
-      !command_line.HasSwitch(switches::kDisableGpuWatchdog) &&
-      gfx::GetGLImplementation() != gfx::kGLImplementationOSMesaGL &&
-      !RunningOnValgrind();
-
-  // Disable the watchdog in debug builds because they tend to only be run by
-  // developers who will not appreciate the watchdog killing the GPU process.
-#ifndef NDEBUG
-  enable_watchdog = false;
-#endif
-
-  // Disable the watchdog for Windows. It tends to abort when the GPU process
-  // is not hung but still taking a long time to do something. Instead, the
-  // browser process displays a dialog when it notices that the child window
-  // is hung giving the user an opportunity to terminate it. This is the
-  // same mechanism used to abort hung plugins.
-#if defined(OS_WIN)
-  enable_watchdog = false;
-#endif
-
-  // Start the GPU watchdog only after anything that is expected to be time
-  // consuming has completed, otherwise the process is liable to be aborted.
-  scoped_refptr<GpuWatchdogThread> watchdog_thread;
-  if (enable_watchdog) {
-    watchdog_thread = new GpuWatchdogThread(kGpuTimeout);
-    watchdog_thread->Start();
-  }
-
   main_message_loop.Run();
 
-  if (enable_watchdog)
-    watchdog_thread->Stop();
+  gpu_thread->StopWatchdog();
 
   return 0;
 }
diff --git a/chrome/gpu/gpu_thread.cc b/chrome/gpu/gpu_thread.cc
index c72de9e..58dd088 100644
--- a/chrome/gpu/gpu_thread.cc
+++ b/chrome/gpu/gpu_thread.cc
@@ -8,41 +8,50 @@
 #include <vector>
 
 #include "app/gfx/gl/gl_context.h"
+#include "app/gfx/gl/gl_implementation.h"
 #include "app/win/scoped_com_initializer.h"
 #include "base/command_line.h"
 #include "base/threading/worker_pool.h"
 #include "build/build_config.h"
 #include "chrome/common/child_process.h"
 #include "chrome/common/child_process_logging.h"
+#include "chrome/common/chrome_switches.h"
 #include "chrome/common/gpu_messages.h"
 #include "chrome/gpu/gpu_info_collector.h"
+#include "chrome/gpu/gpu_watchdog_thread.h"
 #include "ipc/ipc_channel_handle.h"
 
-GpuThread::GpuThread() {
+#if defined(OS_MACOSX)
+#include "chrome/common/sandbox_init_wrapper.h"
+#include "chrome/common/sandbox_mac.h"
+#endif
+
+const int kGpuTimeout = 10000;
+
+namespace {
+
+bool InitializeGpuSandbox() {
+#if defined(OS_MACOSX)
+  CommandLine* parsed_command_line = CommandLine::ForCurrentProcess();
+  SandboxInitWrapper sandbox_wrapper;
+  return sandbox_wrapper.InitializeSandbox(*parsed_command_line,
+                                           switches::kGpuProcess);
+#else
+  // TODO(port): Create GPU sandbox for linux and windows.
+  return true;
+#endif
 }
 
+}  // namespace
+
+GpuThread::GpuThread(const CommandLine& command_line)
+    : command_line_(command_line) {}
+
 GpuThread::~GpuThread() {
 }
 
 void GpuThread::Init(const base::Time& process_start_time) {
-  gpu_info_collector::CollectGraphicsInfo(&gpu_info_);
-  child_process_logging::SetGpuInfo(gpu_info_);
-
-#if defined(OS_WIN)
-  // Asynchronously collect the DirectX diagnostics because this can take a
-  // couple of seconds.
-  if (!base::WorkerPool::PostTask(
-      FROM_HERE,
-      NewRunnableFunction(&GpuThread::CollectDxDiagnostics, this),
-      true)) {
-    // Flag GPU info as complete if the DirectX diagnostics cannot be collected.
-    gpu_info_.SetProgress(GPUInfo::kComplete);
-  }
-#endif
-
-  // Record initialization only after collecting the GPU info because that can
-  // take a significant amount of time.
-  gpu_info_.SetInitializationTime(base::Time::Now() - process_start_time);
+  process_start_time_ = process_start_time;
 }
 
 void GpuThread::RemoveChannel(int renderer_id) {
@@ -53,6 +62,7 @@ bool GpuThread::OnControlMessageReceived(const IPC::Message& msg) {
   bool msg_is_ok = true;
   bool handled = true;
   IPC_BEGIN_MESSAGE_MAP_EX(GpuThread, msg, msg_is_ok)
+    IPC_MESSAGE_HANDLER(GpuMsg_Initialize, OnInitialize)
     IPC_MESSAGE_HANDLER(GpuMsg_EstablishChannel, OnEstablishChannel)
     IPC_MESSAGE_HANDLER(GpuMsg_CloseChannel, OnCloseChannel)
     IPC_MESSAGE_HANDLER(GpuMsg_Synchronize, OnSynchronize)
@@ -70,6 +80,83 @@ bool GpuThread::OnControlMessageReceived(const IPC::Message& msg) {
   return handled;
 }
 
+void GpuThread::OnInitialize() {
+  // Load the GL implementation and locate the bindings before starting the GPU
+  // watchdog because this can take a lot of time and the GPU watchdog might
+  // terminate the GPU process.
+  if (!gfx::GLContext::InitializeOneOff()) {
+    MessageLoop::current()->Quit();
+    return;
+  }
+  gpu_info_collector::CollectGraphicsInfo(&gpu_info_);
+  child_process_logging::SetGpuInfo(gpu_info_);
+
+#if defined(OS_WIN)
+  // Asynchronously collect the DirectX diagnostics because this can take a
+  // couple of seconds.
+  if (!base::WorkerPool::PostTask(
+      FROM_HERE,
+      NewRunnableFunction(&GpuThread::CollectDxDiagnostics, this),
+      true)) {
+    // Flag GPU info as complete if the DirectX diagnostics cannot be collected.
+    gpu_info_.SetProgress(GPUInfo::kComplete);
+  }
+#endif
+
+  // Record initialization only after collecting the GPU info because that can
+  // take a significant amount of time.
+  gpu_info_.SetInitializationTime(base::Time::Now() - process_start_time_);
+
+  // Note that kNoSandbox will also disable the GPU sandbox.
+  bool no_gpu_sandbox = command_line_.HasSwitch(switches::kNoGpuSandbox);
+  if (!no_gpu_sandbox) {
+    if (!InitializeGpuSandbox()) {
+      LOG(ERROR) << "Failed to initialize the GPU sandbox";
+      MessageLoop::current()->Quit();
+      return;
+    }
+  } else {
+    LOG(ERROR) << "Running without GPU sandbox";
+  }
+
+  // In addition to disabling the watchdog if the command line switch is
+  // present, disable it in two other cases. OSMesa is expected to run very
+  // slowly.  Also disable the watchdog on valgrind because the code is expected
+  // to run slowly in that case.
+  bool enable_watchdog =
+      !command_line_.HasSwitch(switches::kDisableGpuWatchdog) &&
+      gfx::GetGLImplementation() != gfx::kGLImplementationOSMesaGL &&
+      !RunningOnValgrind();
+
+  // Disable the watchdog in debug builds because they tend to only be run by
+  // developers who will not appreciate the watchdog killing the GPU process.
+#ifndef NDEBUG
+  enable_watchdog = false;
+#endif
+
+  // Disable the watchdog for Windows. It tends to abort when the GPU process
+  // is not hung but still taking a long time to do something. Instead, the
+  // browser process displays a dialog when it notices that the child window
+  // is hung giving the user an opportunity to terminate it. This is the
+  // same mechanism used to abort hung plugins.
+#if defined(OS_WIN)
+  enable_watchdog = false;
+#endif
+
+  // Start the GPU watchdog only after anything that is expected to be time
+  // consuming has completed, otherwise the process is liable to be aborted.
+  if (enable_watchdog) {
+    watchdog_thread_ = new GpuWatchdogThread(kGpuTimeout);
+    watchdog_thread_->Start();
+  }
+}
+
+void GpuThread::StopWatchdog() {
+  if (watchdog_thread_.get()) {
+    watchdog_thread_->Stop();
+  }
+}
+
 void GpuThread::OnEstablishChannel(int renderer_id) {
   scoped_refptr<GpuChannel> channel;
   IPC::ChannelHandle channel_handle;
diff --git a/chrome/gpu/gpu_thread.h b/chrome/gpu/gpu_thread.h
index 07302d3..1d0f434 100644
--- a/chrome/gpu/gpu_thread.h
+++ b/chrome/gpu/gpu_thread.h
@@ -7,6 +7,7 @@
 #pragma once
 
 #include "base/basictypes.h"
+#include "base/command_line.h"
 #include "base/scoped_ptr.h"
 #include "base/time.h"
 #include "build/build_config.h"
@@ -21,12 +22,15 @@ namespace IPC {
 struct ChannelHandle;
 }
 
+class GpuWatchdogThread;
+
 class GpuThread : public ChildThread {
  public:
-  GpuThread();
+  explicit GpuThread(const CommandLine& command_line);
   ~GpuThread();
 
   void Init(const base::Time& process_start_time);
+  void StopWatchdog();
 
   // Remove the channel for a particular renderer.
   void RemoveChannel(int renderer_id);
@@ -36,6 +40,7 @@ class GpuThread : public ChildThread {
   virtual bool OnControlMessageReceived(const IPC::Message& msg);
 
   // Message handlers.
+  void OnInitialize();
   void OnEstablishChannel(int renderer_id);
   void OnCloseChannel(const IPC::ChannelHandle& channel_handle);
   void OnSynchronize();
@@ -53,6 +58,10 @@ class GpuThread : public ChildThread {
   static void SetDxDiagnostics(GpuThread* thread, const DxDiagNode& node);
 #endif
 
+  CommandLine command_line_;
+  base::Time process_start_time_;
+  scoped_refptr<GpuWatchdogThread> watchdog_thread_;
+
   typedef base::hash_map<int, scoped_refptr<GpuChannel> > GpuChannelMap;
   GpuChannelMap gpu_channels_;