Fix possible deadlock in PluginChannel.

This occurs when the renderer sends an async message with the unblock flag, and then a sync message right after.  If the plugin process just made a sync (with no unblock) call to the renderer, it'll dispatch the first message, and if that leads to a sync call to the renderer, then the unblock flag won't get sent and a deadlock occurs.

BUG=43617
Review URL: http://codereview.chromium.org/2045012

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@47063 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 26 insertions, 20 deletions

diff --git a/chrome/plugin/plugin_channel.cc b/chrome/plugin/plugin_channel.cc
index 3e5b3af..33faff9a 100644
--- a/chrome/plugin/plugin_channel.cc
+++ b/chrome/plugin/plugin_channel.cc
@@ -167,7 +167,7 @@ PluginChannel::PluginChannel()
       in_send_(0),
       off_the_record_(false),
       filter_(new MessageFilter()) {
-  SendUnblockingOnlyDuringSyncDispatch();
+  set_send_unblocking_only_during_unblock_dispatch();
   ChildProcess::current()->AddRefProcess();
   const CommandLine* command_line = CommandLine::ForCurrentProcess();
   log_messages_ = command_line->HasSwitch(switches::kLogPluginMessages);
diff --git a/chrome/plugin/plugin_channel_base.cc b/chrome/plugin/plugin_channel_base.cc
index f104fea..712c697 100644
--- a/chrome/plugin/plugin_channel_base.cc
+++ b/chrome/plugin/plugin_channel_base.cc
@@ -66,8 +66,8 @@ PluginChannelBase::PluginChannelBase()
       peer_pid_(0),
       in_remove_route_(false),
       channel_valid_(false),
-      in_sync_dispatch_(0),
-      send_unblocking_only_during_sync_dispatch_(false) {
+      in_unblock_dispatch_(0),
+      send_unblocking_only_during_unblock_dispatch_(false) {
 }
 
 PluginChannelBase::~PluginChannelBase() {
@@ -119,7 +119,8 @@ bool PluginChannelBase::Send(IPC::Message* message) {
     return false;
   }
 
-  if (send_unblocking_only_during_sync_dispatch_ && in_sync_dispatch_ == 0 &&
+  if (send_unblocking_only_during_unblock_dispatch_ &&
+      in_unblock_dispatch_ == 0 &&
       message->is_sync()) {
     message->set_unblock(false);
   }
@@ -137,8 +138,8 @@ void PluginChannelBase::OnMessageReceived(const IPC::Message& message) {
   lazy_plugin_channel_stack_.Pointer()->push(
       scoped_refptr<PluginChannelBase>(this));
 
-  if (message.is_sync())
-    in_sync_dispatch_++;
+  if (message.should_unblock())
+    in_unblock_dispatch_++;
   if (message.routing_id() == MSG_ROUTING_CONTROL) {
     OnControlMessageReceived(message);
   } else {
@@ -151,8 +152,8 @@ void PluginChannelBase::OnMessageReceived(const IPC::Message& message) {
       Send(reply);
     }
   }
-  if (message.is_sync())
-    in_sync_dispatch_--;
+  if (message.should_unblock())
+    in_unblock_dispatch_--;
 
   lazy_plugin_channel_stack_.Pointer()->pop();
 }
@@ -237,7 +238,3 @@ void PluginChannelBase::OnChannelError() {
 #endif
   channel_valid_ = false;
 }
-
-void PluginChannelBase::SendUnblockingOnlyDuringSyncDispatch() {
-  send_unblocking_only_during_sync_dispatch_ = true;
-}
diff --git a/chrome/plugin/plugin_channel_base.h b/chrome/plugin/plugin_channel_base.h
index 079dcc5..b1f5e3d 100644
--- a/chrome/plugin/plugin_channel_base.h
+++ b/chrome/plugin/plugin_channel_base.h
@@ -94,9 +94,9 @@ class PluginChannelBase : public IPC::Channel::Listener,
   virtual void OnChannelConnected(int32 peer_pid);
   virtual void OnChannelError();
 
-  // If this is set, sync messages that are sent will only unblock the receiver
-  // if this channel is in the middle of a sync dispatch.
-  void SendUnblockingOnlyDuringSyncDispatch();
+  void set_send_unblocking_only_during_unblock_dispatch() {
+      send_unblocking_only_during_unblock_dispatch_ = true;
+  }
 
   virtual bool Init(MessageLoop* ipc_message_loop, bool create_pipe_now);
 
@@ -124,13 +124,22 @@ class PluginChannelBase : public IPC::Channel::Listener,
   // error. This flag is used to indicate the same.
   bool channel_valid_;
 
-  // Track whether we're within a synchronous dispatch; works like a refcount,
-  // 0 when we're not.
-  int in_sync_dispatch_;
+  // Track whether we're dispatching a message with the unblock flag; works like
+  // a refcount, 0 when we're not.
+  int in_unblock_dispatch_;
 
   // If true, sync messages will only be marked as unblocking if the channel is
-  // in the middle of dispatching a synchronous message.
-  bool send_unblocking_only_during_sync_dispatch_;
+  // in the middle of dispatching an unblocking message.
+  // The plugin process wants to avoid setting the unblock flag on its sync
+  // messages unless necessary, since it can potentially introduce reentrancy
+  // into WebKit in ways that it doesn't expect (i.e. causing layoutout during
+  // paint).  However to avoid deadlock, we must ensure that any message that's
+  // sent as a result of a sync call from the renderer must unblock the
+  // renderer.  We additionally have to do this for async messages from the
+  // renderer that have the unblock flag set, since they could be followed by a
+  // sync message that won't get dispatched until the call to the renderer is
+  // complete.
+  bool send_unblocking_only_during_unblock_dispatch_;
 
   DISALLOW_COPY_AND_ASSIGN(PluginChannelBase);
 };