Add support for converting I420 software frames into NV12 hardware frames

Enhance MaybeCreateHardwareFrame() to be able to create a NV12 hardware
frame backed by a YUV_420_BIPLANAR GpuMemoryBuffer.

This code path is not enabled yet.

BUG=524582
CQ_INCLUDE_TRYBOTS=tryserver.blink:linux_blink_rel

Review URL: https://codereview.chromium.org/1307853003

Cr-Commit-Position: refs/heads/master@{#347864}

5 files changed, 165 insertions, 52 deletions

diff --git a/media/base/video_frame.cc b/media/base/video_frame.cc
index 996a261..b69f4f5 100644
--- a/media/base/video_frame.cc
+++ b/media/base/video_frame.cc
@@ -220,7 +220,9 @@ scoped_refptr<VideoFrame> VideoFrame::WrapNativeTexture(
     const gfx::Rect& visible_rect,
     const gfx::Size& natural_size,
     base::TimeDelta timestamp) {
-  if (format != PIXEL_FORMAT_ARGB && format != PIXEL_FORMAT_UYVY) {
+  if (format != PIXEL_FORMAT_ARGB &&
+      format != PIXEL_FORMAT_UYVY &&
+      format != PIXEL_FORMAT_NV12) {
     DLOG(ERROR) << "Unsupported pixel format supported, got "
                 << VideoPixelFormatToString(format);
     return nullptr;
diff --git a/media/blink/skcanvas_video_renderer.cc b/media/blink/skcanvas_video_renderer.cc
index 2559b825..7385e5c 100644
--- a/media/blink/skcanvas_video_renderer.cc
+++ b/media/blink/skcanvas_video_renderer.cc
@@ -137,6 +137,7 @@ skia::RefPtr<SkImage> NewSkImageFromVideoFrameNative(
     VideoFrame* video_frame,
     const Context3D& context_3d) {
   DCHECK(PIXEL_FORMAT_ARGB == video_frame->format() ||
+         PIXEL_FORMAT_NV12 == video_frame->format() ||
          PIXEL_FORMAT_UYVY == video_frame->format());
 
   const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0);
diff --git a/media/renderers/mock_gpu_video_accelerator_factories.cc b/media/renderers/mock_gpu_video_accelerator_factories.cc
index 5230cdd..5f2d64d 100644
--- a/media/renderers/mock_gpu_video_accelerator_factories.cc
+++ b/media/renderers/mock_gpu_video_accelerator_factories.cc
@@ -4,6 +4,7 @@
 
 #include "media/renderers/mock_gpu_video_accelerator_factories.h"
 
+#include "ui/gfx/buffer_format_util.h"
 #include "ui/gfx/gpu_memory_buffer.h"
 
 namespace media {
@@ -13,16 +14,23 @@ namespace {
 class GpuMemoryBufferImpl : public gfx::GpuMemoryBuffer {
  public:
   GpuMemoryBufferImpl(const gfx::Size& size, gfx::BufferFormat format)
-      : format_(format), size_(size) {
+      : format_(format), size_(size),
+        num_planes_(gfx::NumberOfPlanesForBufferFormat(format)) {
     DCHECK(gfx::BufferFormat::R_8 == format_ ||
+           gfx::BufferFormat::YUV_420_BIPLANAR == format_ ||
            gfx::BufferFormat::UYVY_422 == format_);
-    bytes_.resize(size_.GetArea() *
-                  (format_ == gfx::BufferFormat::UYVY_422 ? 2 : 1));
+    DCHECK(num_planes_ <= kMaxPlanes);
+    for (int i = 0; i < static_cast<int>(num_planes_); ++i) {
+      bytes_[i].resize(
+          gfx::RowSizeForBufferFormat(size_.width(), format_, i) *
+          size_.height() / gfx::SubsamplingFactorForBufferFormat(format_, i));
+    }
   }
 
   // Overridden from gfx::GpuMemoryBuffer:
   bool Map(void** data) override {
-    data[0] = &bytes_[0];
+    for (size_t plane = 0; plane < num_planes_; ++plane)
+      data[plane] = &bytes_[plane][0];
     return true;
   }
   void Unmap() override{};
@@ -31,12 +39,13 @@ class GpuMemoryBufferImpl : public gfx::GpuMemoryBuffer {
     return false;
   }
   gfx::BufferFormat GetFormat() const override {
-    NOTREACHED();
-    return gfx::BufferFormat::R_8;
+    return format_;
   }
-  void GetStride(int* stride) const override {
-    stride[0] =
-        size_.width() * (format_ == gfx::BufferFormat::UYVY_422 ? 2 : 1);
+  void GetStride(int* strides) const override {
+    for (int plane = 0; plane < static_cast<int>(num_planes_); ++plane) {
+      strides[plane] = static_cast<int>(
+          gfx::RowSizeForBufferFormat(size_.width(), format_, plane));
+    }
   }
   gfx::GpuMemoryBufferId GetId() const override {
     NOTREACHED();
@@ -51,9 +60,12 @@ class GpuMemoryBufferImpl : public gfx::GpuMemoryBuffer {
   }
 
  private:
+  static const size_t kMaxPlanes = 3;
+
   gfx::BufferFormat format_;
-  std::vector<unsigned char> bytes_;
   const gfx::Size size_;
+  size_t num_planes_;
+  std::vector<uint8> bytes_[kMaxPlanes];
 };
 
 }  // unnamed namespace
diff --git a/media/video/gpu_memory_buffer_video_frame_pool.cc b/media/video/gpu_memory_buffer_video_frame_pool.cc
index 739bcc7..ec93342 100644
--- a/media/video/gpu_memory_buffer_video_frame_pool.cc
+++ b/media/video/gpu_memory_buffer_video_frame_pool.cc
@@ -161,6 +161,9 @@ gfx::BufferFormat GpuMemoryBufferFormat(VideoPixelFormat format, size_t plane) {
     case PIXEL_FORMAT_I420:
       DCHECK_LE(plane, 2u);
       return gfx::BufferFormat::R_8;
+    case PIXEL_FORMAT_NV12:
+      DCHECK_LE(plane, 1u);
+      return gfx::BufferFormat::YUV_420_BIPLANAR;
     case PIXEL_FORMAT_UYVY:
       DCHECK_EQ(0u, plane);
       return gfx::BufferFormat::UYVY_422;
@@ -175,6 +178,10 @@ unsigned ImageInternalFormat(VideoPixelFormat format, size_t plane) {
     case PIXEL_FORMAT_I420:
       DCHECK_LE(plane, 2u);
       return GL_R8_EXT;
+    case PIXEL_FORMAT_NV12:
+      DCHECK_LE(plane, 1u);
+      DLOG(WARNING) << "NV12 format not supported yet";
+      return 0;  // TODO(andresantoso): Implement extension for NV12.
     case PIXEL_FORMAT_UYVY:
       DCHECK_EQ(0u, plane);
       return GL_RGB_YCBCR_422_CHROMIUM;
@@ -184,6 +191,31 @@ unsigned ImageInternalFormat(VideoPixelFormat format, size_t plane) {
   }
 }
 
+// The number of output planes to be copied in each iteration.
+size_t PlanesPerCopy(VideoPixelFormat format) {
+  switch (format) {
+    case PIXEL_FORMAT_I420:
+    case PIXEL_FORMAT_UYVY:
+      return 1;
+    case PIXEL_FORMAT_NV12:
+      return 2;
+    default:
+      NOTREACHED();
+      return 0;
+  }
+}
+
+// The number of output rows to be copied in each iteration.
+int RowsPerCopy(size_t plane, VideoPixelFormat format, int width) {
+  int bytes_per_row = VideoFrame::RowBytes(plane, format, width);
+  if (format == PIXEL_FORMAT_NV12) {
+    DCHECK_EQ(0u, plane);
+    bytes_per_row += VideoFrame::RowBytes(1, format, width);
+  }
+  // Copy an even number of lines, and at least one.
+  return std::max<size_t>((kBytesPerCopyTarget / bytes_per_row) & ~1, 1);
+}
+
 void CopyRowsToI420Buffer(int first_row,
                           int rows,
                           int bytes_per_row,
@@ -204,6 +236,38 @@ void CopyRowsToI420Buffer(int first_row,
   done.Run();
 }
 
+void CopyRowsToNV12Buffer(int first_row,
+                          int rows,
+                          int bytes_per_row,
+                          const scoped_refptr<VideoFrame>& source_frame,
+                          uint8* dest_y,
+                          int dest_stride_y,
+                          uint8* dest_uv,
+                          int dest_stride_uv,
+                          const base::Closure& done) {
+  TRACE_EVENT2("media", "CopyRowsToNV12Buffer", "bytes_per_row", bytes_per_row,
+               "rows", rows);
+  DCHECK_NE(dest_stride_y, 0);
+  DCHECK_NE(dest_stride_uv, 0);
+  DCHECK_LE(bytes_per_row, std::abs(dest_stride_y));
+  DCHECK_LE(bytes_per_row, std::abs(dest_stride_uv));
+  DCHECK_EQ(0, first_row % 2);
+  libyuv::I420ToNV12(
+      source_frame->data(VideoFrame::kYPlane) +
+          first_row * source_frame->stride(VideoFrame::kYPlane),
+      source_frame->stride(VideoFrame::kYPlane),
+      source_frame->data(VideoFrame::kUPlane) +
+          first_row / 2 * source_frame->stride(VideoFrame::kUPlane),
+      source_frame->stride(VideoFrame::kUPlane),
+      source_frame->data(VideoFrame::kVPlane) +
+          first_row / 2 * source_frame->stride(VideoFrame::kVPlane),
+      source_frame->stride(VideoFrame::kVPlane),
+      dest_y + first_row * dest_stride_y, dest_stride_y,
+      dest_uv + first_row / 2 * dest_stride_uv, dest_stride_uv,
+      bytes_per_row, rows);
+  done.Run();
+}
+
 void CopyRowsToUYVYBuffer(int first_row,
                           int rows,
                           int width,
@@ -298,9 +362,9 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::OnCopiesDone(
     const scoped_refptr<VideoFrame>& video_frame,
     FrameResources* frame_resources,
     const FrameReadyCB& frame_ready_cb) {
-  const size_t planes = VideoFrame::NumPlanes(output_format_);
-  for (size_t i = 0; i < planes; ++i) {
-    frame_resources->plane_resources[i].gpu_memory_buffer->Unmap();
+  for (const auto& plane_resource : frame_resources->plane_resources) {
+    if (plane_resource.gpu_memory_buffer)
+      plane_resource.gpu_memory_buffer->Unmap();
   }
 
   media_task_runner_->PostTask(
@@ -317,15 +381,13 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
     FrameResources* frame_resources,
     const FrameReadyCB& frame_ready_cb) {
   // Compute the number of tasks to post and create the barrier.
-  const size_t dest_planes = VideoFrame::NumPlanes(output_format_);
+  const size_t num_planes = VideoFrame::NumPlanes(output_format_);
+  const size_t planes_per_copy = PlanesPerCopy(output_format_);
   gfx::Size size = video_frame->visible_rect().size();
   size_t copies = 0;
-  for (size_t i = 0; i < dest_planes; ++i) {
-    int rows = VideoFrame::Rows(i, output_format_, size.height());
-    int bytes_per_row = VideoFrame::RowBytes(i, output_format_, size.width());
-    // Copy a even number of lines, and at least one.
-    int rows_per_copy =
-        std::max<size_t>((kBytesPerCopyTarget / bytes_per_row) & ~1, 1);
+  for (size_t i = 0; i < num_planes; i += planes_per_copy) {
+    const int rows = VideoFrame::Rows(i, output_format_, size.height());
+    const int rows_per_copy = RowsPerCopy(i, output_format_, size.width());
     copies += rows / rows_per_copy;
     if (rows % rows_per_copy)
       ++copies;
@@ -334,40 +396,51 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
       base::Bind(&PoolImpl::OnCopiesDone, this, video_frame, frame_resources,
                  frame_ready_cb);
   base::Closure barrier = base::BarrierClosure(copies, copies_done);
+
   // Post all the async tasks.
-  for (size_t i = 0; i < dest_planes; ++i) {
-    int rows = VideoFrame::Rows(i, output_format_, size.height());
-    int bytes_per_row = VideoFrame::RowBytes(i, output_format_, size.width());
-    int rows_per_copy =
-        std::max<size_t>((kBytesPerCopyTarget / bytes_per_row) & ~1, 1);
-
-    void* data = nullptr;
-    DCHECK_EQ(1u, gfx::NumberOfPlanesForBufferFormat(
-                      GpuMemoryBufferFormat(output_format_, i)));
-    bool rv = frame_resources->plane_resources[i].gpu_memory_buffer->Map(&data);
+  for (size_t i = 0; i < num_planes; i += planes_per_copy) {
+    gfx::GpuMemoryBuffer* buffer =
+        frame_resources->plane_resources[i].gpu_memory_buffer.get();
+    DCHECK(buffer);
+    DCHECK_EQ(planes_per_copy,
+              gfx::NumberOfPlanesForBufferFormat(buffer->GetFormat()));
+    uint8* dest_buffers[VideoFrame::kMaxPlanes];
+    int dest_strides[VideoFrame::kMaxPlanes];
+    bool rv = buffer->Map(reinterpret_cast<void**>(dest_buffers));
     DCHECK(rv);
-    uint8* mapped_buffer = static_cast<uint8*>(data);
+    buffer->GetStride(dest_strides);
 
-    int dest_stride = 0;
-    frame_resources->plane_resources[i].gpu_memory_buffer->GetStride(
-        &dest_stride);
+    const int rows = VideoFrame::Rows(i, output_format_, size.height());
+    const int rows_per_copy = RowsPerCopy(i, output_format_, size.width());
 
     for (int row = 0; row < rows; row += rows_per_copy) {
+      const int rows_to_copy = std::min(rows_per_copy, rows - row);
       switch (output_format_) {
-        case PIXEL_FORMAT_I420:
+        case PIXEL_FORMAT_I420: {
+          const int bytes_per_row =
+              VideoFrame::RowBytes(i, output_format_, size.width());
           worker_task_runner_->PostTask(
               FROM_HERE,
-              base::Bind(&CopyRowsToI420Buffer, row,
-                         std::min(rows_per_copy, rows - row), bytes_per_row,
-                         video_frame->data(i), video_frame->stride(i),
-                         mapped_buffer, dest_stride, barrier));
+              base::Bind(&CopyRowsToI420Buffer, row, rows_to_copy,
+                         bytes_per_row, video_frame->data(i),
+                         video_frame->stride(i), dest_buffers[0],
+                         dest_strides[0], barrier));
+          break;
+        }
+        case PIXEL_FORMAT_NV12:
+          worker_task_runner_->PostTask(
+              FROM_HERE,
+              base::Bind(&CopyRowsToNV12Buffer, row, rows_to_copy,
+                         size.width(), video_frame, dest_buffers[0],
+                         dest_strides[0], dest_buffers[1], dest_strides[1],
+                         barrier));
           break;
         case PIXEL_FORMAT_UYVY:
           worker_task_runner_->PostTask(
               FROM_HERE,
-              base::Bind(&CopyRowsToUYVYBuffer, row,
-                         std::min(rows_per_copy, rows - row), size.width(),
-                         video_frame, mapped_buffer, dest_stride, barrier));
+              base::Bind(&CopyRowsToUYVYBuffer, row, rows_to_copy, size.width(),
+                         video_frame, dest_buffers[0], dest_strides[0],
+                         barrier));
           break;
         default:
           NOTREACHED();
@@ -387,16 +460,18 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::
     return;
   }
 
-  const size_t planes = VideoFrame::NumPlanes(output_format_);
+  const size_t num_planes = VideoFrame::NumPlanes(output_format_);
+  const size_t planes_per_copy = PlanesPerCopy(output_format_);
   const gfx::Size size = video_frame->visible_rect().size();
   gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes];
   // Set up the planes creating the mailboxes needed to refer to the textures.
-  for (size_t i = 0; i < planes; ++i) {
+  for (size_t i = 0; i < num_planes; i += planes_per_copy) {
     PlaneResource& plane_resource = frame_resources->plane_resources[i];
+    DCHECK(plane_resource.gpu_memory_buffer);
     // Bind the texture and create or rebind the image.
     gles2->BindTexture(texture_target_, plane_resource.texture_id);
 
-    if (plane_resource.gpu_memory_buffer && !plane_resource.image_id) {
+    if (!plane_resource.image_id) {
       const size_t width = VideoFrame::Columns(i, output_format_, size.width());
       const size_t height = VideoFrame::Rows(i, output_format_, size.height());
       plane_resource.image_id = gles2->CreateImageCHROMIUM(
@@ -415,9 +490,8 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::
   // mailboxes refer to will be used only after all the previous commands posted
   // in the command buffer have been processed.
   unsigned sync_point = gles2->InsertSyncPointCHROMIUM();
-  for (size_t i = 0; i < planes; ++i) {
+  for (size_t i = 0; i < num_planes; i += planes_per_copy)
     mailbox_holders[i].sync_point = sync_point;
-  }
 
   scoped_refptr<VideoFrame> frame;
   // Create the VideoFrame backed by native textures.
@@ -433,9 +507,10 @@ void GpuMemoryBufferVideoFramePool::PoolImpl::
       if (video_frame->metadata()->IsTrue(VideoFrameMetadata::ALLOW_OVERLAY))
         frame->metadata()->SetBoolean(VideoFrameMetadata::ALLOW_OVERLAY, true);
       break;
+    case PIXEL_FORMAT_NV12:
     case PIXEL_FORMAT_UYVY:
       frame = VideoFrame::WrapNativeTexture(
-          PIXEL_FORMAT_UYVY, mailbox_holders[VideoFrame::kYPlane],
+          output_format_, mailbox_holders[VideoFrame::kYPlane],
           base::Bind(&PoolImpl::MailboxHoldersReleased, this, frame_resources),
           size, video_frame->visible_rect(), video_frame->natural_size(),
           video_frame->timestamp());
@@ -488,17 +563,18 @@ GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(
   if (!gles2)
     return nullptr;
   gles2->ActiveTexture(GL_TEXTURE0);
-  size_t planes = VideoFrame::NumPlanes(format);
+  size_t num_planes = VideoFrame::NumPlanes(format);
   FrameResources* frame_resources = new FrameResources(size);
   resources_pool_.push_back(frame_resources);
-  for (size_t i = 0; i < planes; ++i) {
+  for (size_t i = 0; i < num_planes; i += PlanesPerCopy(format)) {
     PlaneResource& plane_resource = frame_resources->plane_resources[i];
     const size_t width = VideoFrame::Columns(i, format, size.width());
     const size_t height = VideoFrame::Rows(i, format, size.height());
     const gfx::Size plane_size(width, height);
 
+    const gfx::BufferFormat buffer_format = GpuMemoryBufferFormat(format, i);
     plane_resource.gpu_memory_buffer = gpu_factories_->AllocateGpuMemoryBuffer(
-        plane_size, GpuMemoryBufferFormat(format, i), gfx::BufferUsage::MAP);
+        plane_size, buffer_format, gfx::BufferUsage::MAP);
 
     gles2->GenTextures(1, &plane_resource.texture_id);
     gles2->BindTexture(texture_target_, plane_resource.texture_id);
diff --git a/media/video/gpu_memory_buffer_video_frame_pool_unittest.cc b/media/video/gpu_memory_buffer_video_frame_pool_unittest.cc
index acf8c99..6d891b2 100644
--- a/media/video/gpu_memory_buffer_video_frame_pool_unittest.cc
+++ b/media/video/gpu_memory_buffer_video_frame_pool_unittest.cc
@@ -231,4 +231,26 @@ TEST_F(GpuMemoryBufferVideoFramePoolTest, CreateOneHardwareUYUVFrame) {
   EXPECT_EQ(1u, gles2_->gen_textures);
 }
 
+TEST_F(GpuMemoryBufferVideoFramePoolTest, CreateOneHardwareNV12Frame) {
+  scoped_refptr<VideoFrame> software_frame = CreateTestYUVVideoFrame(10);
+  scoped_refptr<MockGpuVideoAcceleratorFactories> mock_gpu_factories(
+      new MockGpuVideoAcceleratorFactories);
+  mock_gpu_factories->SetVideoFrameOutputFormat(PIXEL_FORMAT_NV12);
+  scoped_ptr<GpuMemoryBufferVideoFramePool> gpu_memory_buffer_pool_ =
+      make_scoped_ptr(new GpuMemoryBufferVideoFramePool(
+          media_task_runner_, copy_task_runner_.get(), mock_gpu_factories));
+
+  EXPECT_CALL(*mock_gpu_factories.get(), GetGLES2Interface())
+      .WillRepeatedly(testing::Return(gles2_.get()));
+
+  scoped_refptr<VideoFrame> frame;
+  gpu_memory_buffer_pool_->MaybeCreateHardwareFrame(
+      software_frame, base::Bind(MaybeCreateHardwareFrameCallback, &frame));
+
+  RunUntilIdle();
+
+  EXPECT_NE(software_frame.get(), frame.get());
+  EXPECT_EQ(1u, gles2_->gen_textures);
+}
+
 }  // namespace media