Reland "Transfer media::VideoFrame using mojo shared memory"

Now that MojoSharedBufferVideoFrame exists and uses mojo shared memory,
pass the shared memory handle when passing using mojo. TypeConverter
for this interface updated to handle the shared memory.

BUG=510088
TEST=update media_mojo_unittests pass

Review URL: https://codereview.chromium.org/1783663003

Cr-Commit-Position: refs/heads/master@{#380536}

6 files changed, 161 insertions, 106 deletions

diff --git a/media/mojo/common/BUILD.gn b/media/mojo/common/BUILD.gn
index 1d0fa81..2e3e230 100644
--- a/media/mojo/common/BUILD.gn
+++ b/media/mojo/common/BUILD.gn
@@ -44,6 +44,7 @@ test("media_mojo_unittests") {
     "//mojo/edk/test:run_all_unittests",
     "//testing/gmock",
     "//testing/gtest",
+    "//ui/gfx:test_support",
   ]
 }
 
diff --git a/media/mojo/common/media_type_converters.cc b/media/mojo/common/media_type_converters.cc
index 5735cb0..f1bf033 100644
--- a/media/mojo/common/media_type_converters.cc
+++ b/media/mojo/common/media_type_converters.cc
@@ -20,8 +20,10 @@
 #include "media/base/media_keys.h"
 #include "media/base/video_decoder_config.h"
 #include "media/base/video_frame.h"
+#include "media/mojo/common/mojo_shared_buffer_video_frame.h"
 #include "media/mojo/interfaces/demuxer_stream.mojom.h"
 #include "mojo/converters/geometry/geometry_type_converters.h"
+#include "mojo/public/cpp/system/buffer.h"
 
 namespace mojo {
 
@@ -293,54 +295,6 @@ ASSERT_CDM_MESSAGE_TYPE(LICENSE_REQUEST);
 ASSERT_CDM_MESSAGE_TYPE(LICENSE_RENEWAL);
 ASSERT_CDM_MESSAGE_TYPE(LICENSE_RELEASE);
 
-namespace {
-
-// Copy the data for plane |plane| from |input| into the vector |dest|. This
-// function only copies the actual frame data. Any padding data is skipped.
-void CopyPlaneDataToVector(const scoped_refptr<media::VideoFrame>& input,
-                           size_t plane,
-                           std::vector<uint8_t>* dest) {
-  DCHECK(dest->empty());
-  uint8_t* source = input->data(plane);
-  size_t num_rows = input->rows(plane);
-  size_t stride = input->stride(plane);
-  size_t row_bytes = input->row_bytes(plane);
-  DCHECK_GE(stride, row_bytes);
-
-  // Copy |row_bytes| for each row, but increment by |stride| to point at the
-  // subsequent row.
-  dest->reserve(num_rows * row_bytes);
-  for (size_t i = 0; i < num_rows; ++i) {
-    dest->insert(dest->end(), source, source + row_bytes);
-    source += stride;
-  }
-}
-
-// Copy the data from |input| into the plane |plane| of |frame|. If there is
-// padding in |frame|, it is unchanged.
-void CopyPlaneData(const std::vector<uint8_t>& input,
-                   size_t plane,
-                   const scoped_refptr<media::VideoFrame>& frame) {
-  const uint8_t* source = input.data();
-  uint8_t* dest = frame->data(plane);
-  size_t num_rows = frame->rows(plane);
-  size_t stride = frame->stride(plane);
-  size_t row_bytes = frame->row_bytes(plane);
-  DCHECK_GE(stride, row_bytes);
-  DCHECK_EQ(input.size(), num_rows * row_bytes);
-
-  // Copy |row_bytes| for each row. |input| contains only the data bytes, so
-  // |source| is only incremented by |row_bytes|. |dest| may contain padding,
-  // so increment by |stride| to point at the subsequent row.
-  for (size_t i = 0; i < num_rows; ++i) {
-    memcpy(dest, source, row_bytes);
-    source += row_bytes;
-    dest += stride;
-  }
-}
-
-}  // namespace
-
 // static
 media::interfaces::SubsampleEntryPtr TypeConverter<
     media::interfaces::SubsampleEntryPtr,
@@ -643,36 +597,38 @@ media::interfaces::VideoFramePtr
 TypeConverter<media::interfaces::VideoFramePtr,
               scoped_refptr<media::VideoFrame>>::
     Convert(const scoped_refptr<media::VideoFrame>& input) {
-  media::interfaces::VideoFramePtr buffer(media::interfaces::VideoFrame::New());
-  buffer->end_of_stream =
+  media::interfaces::VideoFramePtr frame(media::interfaces::VideoFrame::New());
+  frame->end_of_stream =
       input->metadata()->IsTrue(media::VideoFrameMetadata::END_OF_STREAM);
-  if (buffer->end_of_stream)
-    return buffer;
-
-  // handle non EOS buffer.
-  buffer->format = static_cast<media::interfaces::VideoFormat>(input->format());
-  buffer->coded_size = Size::From(input->coded_size());
-  buffer->visible_rect = Rect::From(input->visible_rect());
-  buffer->natural_size = Size::From(input->natural_size());
-  buffer->timestamp_usec = input->timestamp().InMicroseconds();
-
-  if (!input->coded_size().IsEmpty()) {
-    // TODO(jrummell): Use a shared buffer rather than copying the data for
-    // each plane.
-    std::vector<uint8_t> y_data;
-    CopyPlaneDataToVector(input, media::VideoFrame::kYPlane, &y_data);
-    buffer->y_data.Swap(&y_data);
-
-    std::vector<uint8_t> u_data;
-    CopyPlaneDataToVector(input, media::VideoFrame::kUPlane, &u_data);
-    buffer->u_data.Swap(&u_data);
-
-    std::vector<uint8_t> v_data;
-    CopyPlaneDataToVector(input, media::VideoFrame::kVPlane, &v_data);
-    buffer->v_data.Swap(&v_data);
-  }
-
-  return buffer;
+  if (frame->end_of_stream)
+    return frame;
+
+  // Handle non EOS frame. It must be a MojoSharedBufferVideoFrame.
+  // TODO(jrummell): Support other types of VideoFrame.
+  CHECK_EQ(media::VideoFrame::STORAGE_MOJO_SHARED_BUFFER,
+           input->storage_type());
+  media::MojoSharedBufferVideoFrame* input_frame =
+      static_cast<media::MojoSharedBufferVideoFrame*>(input.get());
+  mojo::ScopedSharedBufferHandle duplicated_handle;
+  const MojoResult result =
+      DuplicateBuffer(input_frame->Handle(), nullptr, &duplicated_handle);
+  CHECK_EQ(MOJO_RESULT_OK, result);
+  CHECK(duplicated_handle.is_valid());
+
+  frame->format = static_cast<media::interfaces::VideoFormat>(input->format());
+  frame->coded_size = Size::From(input->coded_size());
+  frame->visible_rect = Rect::From(input->visible_rect());
+  frame->natural_size = Size::From(input->natural_size());
+  frame->timestamp_usec = input->timestamp().InMicroseconds();
+  frame->frame_data = std::move(duplicated_handle);
+  frame->frame_data_size = input_frame->MappedSize();
+  frame->y_stride = input_frame->stride(media::VideoFrame::kYPlane);
+  frame->u_stride = input_frame->stride(media::VideoFrame::kUPlane);
+  frame->v_stride = input_frame->stride(media::VideoFrame::kVPlane);
+  frame->y_offset = input_frame->PlaneOffset(media::VideoFrame::kYPlane);
+  frame->u_offset = input_frame->PlaneOffset(media::VideoFrame::kUPlane);
+  frame->v_offset = input_frame->PlaneOffset(media::VideoFrame::kVPlane);
+  return frame;
 }
 
 // static
@@ -683,16 +639,16 @@ TypeConverter<scoped_refptr<media::VideoFrame>,
   if (input->end_of_stream)
     return media::VideoFrame::CreateEOSFrame();
 
-  scoped_refptr<media::VideoFrame> frame = media::VideoFrame::CreateFrame(
+  return media::MojoSharedBufferVideoFrame::Create(
       static_cast<media::VideoPixelFormat>(input->format),
       input->coded_size.To<gfx::Size>(), input->visible_rect.To<gfx::Rect>(),
-      input->natural_size.To<gfx::Size>(),
+      input->natural_size.To<gfx::Size>(), std::move(input->frame_data),
+      base::saturated_cast<size_t>(input->frame_data_size),
+      base::saturated_cast<size_t>(input->y_offset),
+      base::saturated_cast<size_t>(input->u_offset),
+      base::saturated_cast<size_t>(input->v_offset), input->y_stride,
+      input->u_stride, input->v_stride,
       base::TimeDelta::FromMicroseconds(input->timestamp_usec));
-  CopyPlaneData(input->y_data.storage(), media::VideoFrame::kYPlane, frame);
-  CopyPlaneData(input->u_data.storage(), media::VideoFrame::kUPlane, frame);
-  CopyPlaneData(input->v_data.storage(), media::VideoFrame::kVPlane, frame);
-
-  return frame;
 }
 
 }  // namespace mojo
diff --git a/media/mojo/common/media_type_converters_unittest.cc b/media/mojo/common/media_type_converters_unittest.cc
index f9de05d..9858cd0 100644
--- a/media/mojo/common/media_type_converters_unittest.cc
+++ b/media/mojo/common/media_type_converters_unittest.cc
@@ -17,6 +17,7 @@
 #include "media/base/sample_format.h"
 #include "media/base/test_helpers.h"
 #include "media/base/video_frame.h"
+#include "media/mojo/common/mojo_shared_buffer_video_frame.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace media {
@@ -90,9 +91,8 @@ void CompareVideoPlane(size_t plane,
 
 void CompareVideoFrames(const scoped_refptr<VideoFrame>& original,
                         const scoped_refptr<VideoFrame>& result) {
-  if (original->metadata()->IsTrue(media::VideoFrameMetadata::END_OF_STREAM)) {
-    EXPECT_TRUE(
-        result->metadata()->IsTrue(media::VideoFrameMetadata::END_OF_STREAM));
+  if (original->metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM)) {
+    EXPECT_TRUE(result->metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM));
     return;
   }
 
@@ -104,9 +104,69 @@ void CompareVideoFrames(const scoped_refptr<VideoFrame>& original,
   EXPECT_EQ(original->natural_size().height(), result->natural_size().height());
   EXPECT_EQ(original->natural_size().width(), result->natural_size().width());
 
-  CompareVideoPlane(media::VideoFrame::kYPlane, original, result);
-  CompareVideoPlane(media::VideoFrame::kUPlane, original, result);
-  CompareVideoPlane(media::VideoFrame::kVPlane, original, result);
+  CompareVideoPlane(VideoFrame::kYPlane, original, result);
+  CompareVideoPlane(VideoFrame::kUPlane, original, result);
+  CompareVideoPlane(VideoFrame::kVPlane, original, result);
+}
+
+// Returns a color VideoFrame that stores the data in a
+// mojo::SharedBufferHandle.
+scoped_refptr<VideoFrame> CreateMojoSharedBufferColorFrame() {
+  // Create a color VideoFrame to use as reference (data will need to be copied
+  // to a mojo::SharedBufferHandle).
+  const int kWidth = 16;
+  const int kHeight = 9;
+  const base::TimeDelta kTimestamp = base::TimeDelta::FromSeconds(26);
+  scoped_refptr<VideoFrame> color_frame(VideoFrame::CreateColorFrame(
+      gfx::Size(kWidth, kHeight), 255, 128, 24, kTimestamp));
+
+  // Allocate a mojo::SharedBufferHandle big enough to contain
+  // |color_frame|'s data.
+  const size_t allocation_size = VideoFrame::AllocationSize(
+      color_frame->format(), color_frame->coded_size());
+  mojo::ScopedSharedBufferHandle handle;
+  const MojoResult mojo_result =
+      mojo::CreateSharedBuffer(nullptr, allocation_size, &handle);
+  EXPECT_EQ(mojo_result, MOJO_RESULT_OK);
+
+  // Create a MojoSharedBufferVideoFrame whose dimensions match |color_frame|.
+  const size_t y_plane_size = color_frame->rows(VideoFrame::kYPlane) *
+                              color_frame->stride(VideoFrame::kYPlane);
+  const size_t u_plane_size = color_frame->rows(VideoFrame::kUPlane) *
+                              color_frame->stride(VideoFrame::kUPlane);
+  const size_t v_plane_size = color_frame->rows(VideoFrame::kVPlane) *
+                              color_frame->stride(VideoFrame::kVPlane);
+  scoped_refptr<VideoFrame> frame(MojoSharedBufferVideoFrame::Create(
+      color_frame->format(), color_frame->coded_size(),
+      color_frame->visible_rect(), color_frame->natural_size(),
+      std::move(handle), allocation_size, 0, y_plane_size,
+      y_plane_size + u_plane_size, color_frame->stride(VideoFrame::kYPlane),
+      color_frame->stride(VideoFrame::kUPlane),
+      color_frame->stride(VideoFrame::kVPlane), color_frame->timestamp()));
+  EXPECT_EQ(color_frame->coded_size(), frame->coded_size());
+  EXPECT_EQ(color_frame->visible_rect(), frame->visible_rect());
+  EXPECT_EQ(color_frame->natural_size(), frame->natural_size());
+  EXPECT_EQ(color_frame->rows(VideoFrame::kYPlane),
+            frame->rows(VideoFrame::kYPlane));
+  EXPECT_EQ(color_frame->rows(VideoFrame::kUPlane),
+            frame->rows(VideoFrame::kUPlane));
+  EXPECT_EQ(color_frame->rows(VideoFrame::kVPlane),
+            frame->rows(VideoFrame::kVPlane));
+  EXPECT_EQ(color_frame->stride(VideoFrame::kYPlane),
+            frame->stride(VideoFrame::kYPlane));
+  EXPECT_EQ(color_frame->stride(VideoFrame::kUPlane),
+            frame->stride(VideoFrame::kUPlane));
+  EXPECT_EQ(color_frame->stride(VideoFrame::kVPlane),
+            frame->stride(VideoFrame::kVPlane));
+
+  // Copy all the data from |color_frame| into |frame|.
+  memcpy(frame->data(VideoFrame::kYPlane),
+         color_frame->data(VideoFrame::kYPlane), y_plane_size);
+  memcpy(frame->data(VideoFrame::kUPlane),
+         color_frame->data(VideoFrame::kUPlane), u_plane_size);
+  memcpy(frame->data(VideoFrame::kVPlane),
+         color_frame->data(VideoFrame::kVPlane), v_plane_size);
+  return frame;
 }
 
 }  // namespace
@@ -372,30 +432,30 @@ TEST(MediaTypeConvertersTest, ConvertVideoFrame_EOS) {
   CompareVideoFrames(buffer, result);
 }
 
-TEST(MediaTypeConvertersTest, ConvertVideoFrame_BlackFrame) {
+TEST(MediaTypeConvertersTest, ConvertVideoFrame_EmptyFrame) {
   // Original.
-  scoped_refptr<VideoFrame> buffer(
-      VideoFrame::CreateBlackFrame(gfx::Size(100, 100)));
+  scoped_refptr<VideoFrame> frame(MojoSharedBufferVideoFrame::CreateDefaultI420(
+      gfx::Size(100, 100), base::TimeDelta::FromSeconds(100)));
 
   // Convert to and back.
-  interfaces::VideoFramePtr ptr(interfaces::VideoFrame::From(buffer));
+  interfaces::VideoFramePtr ptr(interfaces::VideoFrame::From(frame));
   scoped_refptr<VideoFrame> result(ptr.To<scoped_refptr<VideoFrame>>());
+  EXPECT_NE(result.get(), nullptr);
 
   // Compare.
-  CompareVideoFrames(buffer, result);
+  CompareVideoFrames(frame, result);
 }
 
 TEST(MediaTypeConvertersTest, ConvertVideoFrame_ColorFrame) {
-  // Original.
-  scoped_refptr<VideoFrame> buffer(VideoFrame::CreateColorFrame(
-      gfx::Size(50, 100), 255, 128, 128, base::TimeDelta::FromSeconds(26)));
+  scoped_refptr<VideoFrame> frame(CreateMojoSharedBufferColorFrame());
 
   // Convert to and back.
-  interfaces::VideoFramePtr ptr(interfaces::VideoFrame::From(buffer));
+  interfaces::VideoFramePtr ptr(interfaces::VideoFrame::From(frame));
   scoped_refptr<VideoFrame> result(ptr.To<scoped_refptr<VideoFrame>>());
+  EXPECT_NE(result.get(), nullptr);
 
   // Compare.
-  CompareVideoFrames(buffer, result);
+  CompareVideoFrames(frame, result);
 }
 
 }  // namespace media
diff --git a/media/mojo/common/mojo_shared_buffer_video_frame.cc b/media/mojo/common/mojo_shared_buffer_video_frame.cc
index fa226c2..3fca408 100644
--- a/media/mojo/common/mojo_shared_buffer_video_frame.cc
+++ b/media/mojo/common/mojo_shared_buffer_video_frame.cc
@@ -144,4 +144,12 @@ size_t MojoSharedBufferVideoFrame::PlaneOffset(size_t plane) const {
   return offsets_[plane];
 }
 
+const mojo::SharedBufferHandle& MojoSharedBufferVideoFrame::Handle() const {
+  return shared_buffer_handle_.get();
+}
+
+size_t MojoSharedBufferVideoFrame::MappedSize() const {
+  return shared_buffer_size_;
+}
+
 }  // namespace media
diff --git a/media/mojo/common/mojo_shared_buffer_video_frame.h b/media/mojo/common/mojo_shared_buffer_video_frame.h
index d27a85f..bc49c3f 100644
--- a/media/mojo/common/mojo_shared_buffer_video_frame.h
+++ b/media/mojo/common/mojo_shared_buffer_video_frame.h
@@ -15,8 +15,19 @@
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/size.h"
 
+namespace mojo {
+template <typename T, typename U>
+struct TypeConverter;
+template <typename T>
+class StructPtr;
+};
+
 namespace media {
 
+namespace interfaces {
+class VideoFrame;
+}
+
 // A derived class of media::VideoFrame holding a mojo::SharedBufferHandle
 // which is mapped on constructor and remains so for the lifetime of the
 // object. These frames are ref-counted.
@@ -53,6 +64,11 @@ class MojoSharedBufferVideoFrame : public VideoFrame {
   size_t PlaneOffset(size_t plane) const;
 
  private:
+  // mojo::TypeConverter added as a friend so that MojoSharedBufferVideoFrame
+  // can be transferred across a mojo connection.
+  friend struct mojo::TypeConverter<mojo::StructPtr<interfaces::VideoFrame>,
+                                    scoped_refptr<VideoFrame>>;
+
   MojoSharedBufferVideoFrame(VideoPixelFormat format,
                              const gfx::Size& coded_size,
                              const gfx::Rect& visible_rect,
@@ -71,6 +87,14 @@ class MojoSharedBufferVideoFrame : public VideoFrame {
             size_t u_offset,
             size_t v_offset);
 
+  // Returns the mojo shared memory handle. This object continues to own the
+  // handle. Caller should call duplicate the handle if they want to keep a
+  // copy of the shared memory.
+  const mojo::SharedBufferHandle& Handle() const;
+
+  // Returns the size of the shared memory.
+  size_t MappedSize() const;
+
   mojo::ScopedSharedBufferHandle shared_buffer_handle_;
   size_t shared_buffer_size_;
   uint8_t* shared_buffer_data_;
diff --git a/media/mojo/interfaces/media_types.mojom b/media/mojo/interfaces/media_types.mojom
index 085256e..44e6cdf 100644
--- a/media/mojo/interfaces/media_types.mojom
+++ b/media/mojo/interfaces/media_types.mojom
@@ -266,8 +266,6 @@ struct AudioBuffer {
 };
 
 // This defines a mojo transport format for media::VideoFrame.
-// TODO(jrummell): Support shared memory based VideoFrame to avoid copying
-// the data multiple times.
 struct VideoFrame {
   // Format of the frame.
   VideoFormat format;
@@ -287,8 +285,16 @@ struct VideoFrame {
   // Timestamp in microseconds of the associated frame.
   int64 timestamp_usec;
 
-  // Frame data for each plane. Will be null for EOS buffers.
-  array<uint8>? y_data;
-  array<uint8>? u_data;
-  array<uint8>? v_data;
+  // Reference to the shared memory containing the frame's data.
+  handle<shared_buffer> frame_data;
+  uint64 frame_data_size;
+
+  // Stride and offsets for each plane. Offsets are relative to the start
+  // of |frame_data|.
+  int32 y_stride;
+  int32 u_stride;
+  int32 v_stride;
+  uint64 y_offset;
+  uint64 u_offset;
+  uint64 v_offset;
 };