Fix Pepper2D on the Mac so that it runs in the sandbox. Note that trusted

plugins still have to run outside of the sandbox (this is not a regression).
This CL allows untrusted Pepper 2D plugins to run in the sandbox on the Mac.

BUG=40701
TEST=pepper_test_plugin (has to run w/ --no-sandbox on Mac), run any untrusted
    .nexe that uses Pepper 2D or 3D (examples are inthe NaCl SDK).

Review URL: http://codereview.chromium.org/1558032

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@44016 0039d316-1c4b-4281-b951-d872f2087c98

10 files changed, 48 insertions, 27 deletions

diff --git a/chrome/browser/renderer_host/render_widget_helper.cc b/chrome/browser/renderer_host/render_widget_helper.cc
index a6fbd5c..5c379e7 100644
--- a/chrome/browser/renderer_host/render_widget_helper.cc
+++ b/chrome/browser/renderer_host/render_widget_helper.cc
@@ -263,7 +263,7 @@ TransportDIB* RenderWidgetHelper::MapTransportDIB(TransportDIB::Id dib_id) {
 }
 
 void RenderWidgetHelper::AllocTransportDIB(
-    size_t size, TransportDIB::Handle* result) {
+    size_t size, bool cache_in_browser, TransportDIB::Handle* result) {
   scoped_ptr<base::SharedMemory> shared_memory(new base::SharedMemory());
   if (!shared_memory->Create(L"", false /* read write */,
                              false /* do not open existing */, size)) {
@@ -274,9 +274,11 @@ void RenderWidgetHelper::AllocTransportDIB(
 
   shared_memory->GiveToProcess(0 /* pid, not needed */, result);
 
-  // Keep a copy of the file descriptor around
-  AutoLock locked(allocated_dibs_lock_);
-  allocated_dibs_[shared_memory->id()] = dup(result->fd);
+  if (cache_in_browser) {
+    // Keep a copy of the file descriptor around
+    AutoLock locked(allocated_dibs_lock_);
+    allocated_dibs_[shared_memory->id()] = dup(result->fd);
+  }
 }
 
 void RenderWidgetHelper::FreeTransportDIB(TransportDIB::Id dib_id) {
diff --git a/chrome/browser/renderer_host/render_widget_helper.h b/chrome/browser/renderer_host/render_widget_helper.h
index 728bb7c..53b336c 100644
--- a/chrome/browser/renderer_host/render_widget_helper.h
+++ b/chrome/browser/renderer_host/render_widget_helper.h
@@ -128,8 +128,14 @@ class RenderWidgetHelper
                        int* route_id);
 
 #if defined(OS_MACOSX)
-  // Called on the IO thread to handle the allocation of a transport DIB
-  void AllocTransportDIB(size_t size, TransportDIB::Handle* result);
+  // Called on the IO thread to handle the allocation of a TransportDIB.  If
+  // |cache_in_browser| is |true|, then a copy of the shmem is kept by the
+  // browser, and it is the caller's repsonsibility to call
+  // FreeTransportDIB().  In all cases, the caller is responsible for deleting
+  // the resulting TransportDIB.
+  void AllocTransportDIB(size_t size,
+                         bool cache_in_browser,
+                         TransportDIB::Handle* result);
 
   // Called on the IO thread to handle the freeing of a transport DIB
   void FreeTransportDIB(TransportDIB::Id dib_id);
diff --git a/chrome/browser/renderer_host/resource_message_filter.cc b/chrome/browser/renderer_host/resource_message_filter.cc
index 951ed86..39706a9 100644
--- a/chrome/browser/renderer_host/resource_message_filter.cc
+++ b/chrome/browser/renderer_host/resource_message_filter.cc
@@ -1256,8 +1256,8 @@ void ResourceMessageFilter::OnRendererHistograms(
 
 #if defined(OS_MACOSX)
 void ResourceMessageFilter::OnAllocTransportDIB(
-    size_t size, TransportDIB::Handle* handle) {
-  render_widget_helper_->AllocTransportDIB(size, handle);
+    size_t size, bool cache_in_browser, TransportDIB::Handle* handle) {
+  render_widget_helper_->AllocTransportDIB(size, cache_in_browser, handle);
 }
 
 void ResourceMessageFilter::OnFreeTransportDIB(
diff --git a/chrome/browser/renderer_host/resource_message_filter.h b/chrome/browser/renderer_host/resource_message_filter.h
index c673c7c..77b8eb7 100644
--- a/chrome/browser/renderer_host/resource_message_filter.h
+++ b/chrome/browser/renderer_host/resource_message_filter.h
@@ -298,6 +298,7 @@ class ResourceMessageFilter : public IPC::ChannelProxy::MessageFilter,
 #endif
   // Browser side transport DIB allocation
   void OnAllocTransportDIB(size_t size,
+                           bool cache_in_browser,
                            TransportDIB::Handle* result);
   void OnFreeTransportDIB(TransportDIB::Id dib_id);
 
diff --git a/chrome/common/render_messages_internal.h b/chrome/common/render_messages_internal.h
index 1654850..17e5689 100644
--- a/chrome/common/render_messages_internal.h
+++ b/chrome/common/render_messages_internal.h
@@ -1869,8 +1869,14 @@ IPC_BEGIN_MESSAGES(ViewHost)
   // On OSX, we cannot allocated shared memory from within the sandbox, so
   // this call exists for the renderer to ask the browser to allocate memory
   // on its behalf. We return a file descriptor to the POSIX shared memory.
-  IPC_SYNC_MESSAGE_CONTROL1_1(ViewHostMsg_AllocTransportDIB,
+  // If the |cache_in_browser| flag is |true|, then a copy of the shmem is kept
+  // by the browser, and it is the caller's repsonsibility to send a
+  // ViewHostMsg_FreeTransportDIB message in order to release the cached shmem.
+  // In all cases, the caller is responsible for deleting the resulting
+  // TransportDIB.
+  IPC_SYNC_MESSAGE_CONTROL2_1(ViewHostMsg_AllocTransportDIB,
                               size_t, /* bytes requested */
+                              bool, /* cache in the browser */
                               TransportDIB::Handle /* DIB */)
 
   // Since the browser keeps handles to the allocated transport DIBs, this
diff --git a/chrome/renderer/pepper_devices.cc b/chrome/renderer/pepper_devices.cc
index 72027eb..12e34d1 100644
--- a/chrome/renderer/pepper_devices.cc
+++ b/chrome/renderer/pepper_devices.cc
@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 #include "chrome/renderer/pepper_devices.h"
+#include "chrome/renderer/render_thread.h"
 #include "chrome/renderer/webplugin_delegate_pepper.h"
 #include "skia/ext/platform_canvas.h"
 #include "webkit/glue/plugins/plugin_instance.h"
@@ -30,19 +31,24 @@ NPError Graphics2DDeviceContext::Initialize(
 
   // Allocate the transport DIB and the PlatformCanvas pointing to it.
 #if defined(OS_MACOSX)
-  // On the Mac, there is no clean way to create a TransportDIB::Handle and
-  // then Map() it.  Using TransportDIB::Create() followed by
-  // TransportDIB::Map() will leak a TransportDIB object (you can't Create()
-  // then Map(), then delete because the file descriptor used by the underlying
-  // shared memory object gets closed.)  Work around this issue by creating
-  // a SharedMemory object then pass that into TransportDIB::Map().
-  scoped_ptr<base::SharedMemory> shared_memory(new base::SharedMemory());
-  if (!shared_memory->Create(L"", false /* read write */,
-                             false /* do not open existing */, buffer_size)) {
-    return NPERR_OUT_OF_MEMORY_ERROR;
-  }
+  // On the Mac, shared memory has to be created in the browser in order to
+  // work in the sandbox.  Do this by sending a message to the browser
+  // requesting a TransportDIB (see also
+  // chrome/renderer/webplugin_delegate_proxy.cc, method
+  // WebPluginDelegateProxy::CreateBitmap() for similar code).  Note that the
+  // TransportDIB is _not_ cached in the browser; this is because this memory
+  // gets flushed by the renderer into another TransportDIB that represents the
+  // page, which is then in turn flushed to the screen by the browser process.
+  // When |transport_dib_| goes out of scope in the dtor, all of its shared
+  // memory gets reclaimed.
   TransportDIB::Handle dib_handle;
-  shared_memory->GiveToProcess(0 /* pid, not needed */, &dib_handle);
+  IPC::Message* msg = new ViewHostMsg_AllocTransportDIB(buffer_size,
+                                                        false,
+                                                        &dib_handle);
+  if (!RenderThread::current()->Send(msg))
+    return NPERR_GENERIC_ERROR;
+  if (!TransportDIB::is_valid(dib_handle))
+    return NPERR_OUT_OF_MEMORY_ERROR;
   transport_dib_.reset(TransportDIB::Map(dib_handle));
 #else
   transport_dib_.reset(TransportDIB::Create(buffer_size, ++next_buffer_id_));
diff --git a/chrome/renderer/render_process.cc b/chrome/renderer/render_process.cc
index 07e642e..85c374e 100644
--- a/chrome/renderer/render_process.cc
+++ b/chrome/renderer/render_process.cc
@@ -171,9 +171,9 @@ TransportDIB* RenderProcess::CreateTransportDIB(size_t size) {
   return TransportDIB::Create(size, sequence_number_++);
 #elif defined(OS_MACOSX)  // defined(OS_WIN) || defined(OS_LINUX)
   // Mac creates transport DIBs in the browser, so we need to do a sync IPC to
-  // get one.
+  // get one.  The TransportDIB is cached in the browser.
   TransportDIB::Handle handle;
-  IPC::Message* msg = new ViewHostMsg_AllocTransportDIB(size, &handle);
+  IPC::Message* msg = new ViewHostMsg_AllocTransportDIB(size, true, &handle);
   if (!main_thread()->Send(msg))
     return NULL;
   if (handle.fd < 0)
diff --git a/chrome/renderer/render_process_impl.cc b/chrome/renderer/render_process_impl.cc
index 5b4c383..e82fed9c 100644
--- a/chrome/renderer/render_process_impl.cc
+++ b/chrome/renderer/render_process_impl.cc
@@ -192,9 +192,9 @@ TransportDIB* RenderProcessImpl::CreateTransportDIB(size_t size) {
   return TransportDIB::Create(size, transport_dib_next_sequence_number_++);
 #elif defined(OS_MACOSX)  // defined(OS_WIN) || defined(OS_LINUX)
   // Mac creates transport DIBs in the browser, so we need to do a sync IPC to
-  // get one.
+  // get one.  The TransportDIB is cached in the browser.
   TransportDIB::Handle handle;
-  IPC::Message* msg = new ViewHostMsg_AllocTransportDIB(size, &handle);
+  IPC::Message* msg = new ViewHostMsg_AllocTransportDIB(size, true, &handle);
   if (!main_thread()->Send(msg))
     return NULL;
   if (handle.fd < 0)
diff --git a/chrome/renderer/render_view.cc b/chrome/renderer/render_view.cc
index ba8c80f..c1025ab37 100644
--- a/chrome/renderer/render_view.cc
+++ b/chrome/renderer/render_view.cc
@@ -4874,7 +4874,7 @@ TransportDIB::Handle RenderView::AcceleratedSurfaceAllocTransportDIB(
     size_t size) {
   TransportDIB::Handle dib_handle;
   // Assume this is a synchronous RPC.
-  if (Send(new ViewHostMsg_AllocTransportDIB(size, &dib_handle)))
+  if (Send(new ViewHostMsg_AllocTransportDIB(size, true, &dib_handle)))
     return dib_handle;
   // Return an invalid handle if Send() fails.
   return TransportDIB::DefaultHandleValue();
diff --git a/chrome/renderer/webplugin_delegate_proxy.cc b/chrome/renderer/webplugin_delegate_proxy.cc
index fbadb86..2d27522 100644
--- a/chrome/renderer/webplugin_delegate_proxy.cc
+++ b/chrome/renderer/webplugin_delegate_proxy.cc
@@ -617,7 +617,7 @@ bool WebPluginDelegateProxy::CreateBitmap(
 #endif
 #if defined(OS_MACOSX)
   TransportDIB::Handle handle;
-  IPC::Message* msg = new ViewHostMsg_AllocTransportDIB(size, &handle);
+  IPC::Message* msg = new ViewHostMsg_AllocTransportDIB(size, true, &handle);
   if (!RenderThread::current()->Send(msg))
     return false;
   if (handle.fd < 0)