Allow shaped-desktop hosts to send shape only when it changes.

Previously hosts supplying a shaped desktop needed to attach the desktop
shape to every frame, wasting bandwidth since the shape changes
relatively infrequently.

This CL updates the VideoRenderer implementations to preserve the shape
(or lack of one) from the preceding frame if the VideoPacket does not
include the use_desktop_shape field.

Also simplifies FrameConsumerProxy to remove the need for ref-counting,
updates NULL->nullptr throughout remoting/codec/, and removes
unnecessary transparency logic from VideoDecoderVpx.

BUG=446288

Review URL: https://codereview.chromium.org/1236663002

Cr-Commit-Position: refs/heads/master@{#339212}

4 files changed, 78 insertions, 201 deletions

diff --git a/remoting/codec/audio_decoder_opus.cc b/remoting/codec/audio_decoder_opus.cc
index 656d4ac..e7bd5ac 100644
--- a/remoting/codec/audio_decoder_opus.cc
+++ b/remoting/codec/audio_decoder_opus.cc
@@ -26,10 +26,7 @@ const AudioPacket::SamplingRate kSamplingRate =
 }  // namespace
 
 AudioDecoderOpus::AudioDecoderOpus()
-    : sampling_rate_(0),
-      channels_(0),
-      decoder_(nullptr) {
-}
+    : sampling_rate_(0), channels_(0), decoder_(nullptr) {}
 
 AudioDecoderOpus::~AudioDecoderOpus() {
   DestroyDecoder();
diff --git a/remoting/codec/codec_test.cc b/remoting/codec/codec_test.cc
index 7c3dbbc..a97c51f 100644
--- a/remoting/codec/codec_test.cc
+++ b/remoting/codec/codec_test.cc
@@ -205,10 +205,7 @@ class VideoDecoderTester {
 // the message to other subprograms for validaton.
 class VideoEncoderTester {
  public:
-  VideoEncoderTester()
-      : decoder_tester_(nullptr),
-        data_available_(0) {
-  }
+  VideoEncoderTester() : decoder_tester_(nullptr), data_available_(0) {}
 
   ~VideoEncoderTester() {
     EXPECT_GT(data_available_, 0);
diff --git a/remoting/codec/video_decoder_vpx.cc b/remoting/codec/video_decoder_vpx.cc
index 7a8020f..b14d888 100644
--- a/remoting/codec/video_decoder_vpx.cc
+++ b/remoting/codec/video_decoder_vpx.cc
@@ -22,80 +22,25 @@ extern "C" {
 
 namespace remoting {
 
-namespace {
-
-const uint32 kTransparentColor = 0;
-
-// Fills the rectangle |rect| with the given ARGB color |color| in |buffer|.
-void FillRect(uint8* buffer,
-              int stride,
-              const webrtc::DesktopRect& rect,
-              uint32 color) {
-  uint32* ptr = reinterpret_cast<uint32*>(buffer + (rect.top() * stride) +
-      (rect.left() * VideoDecoder::kBytesPerPixel));
-  int width = rect.width();
-  for (int height = rect.height(); height > 0; --height) {
-    std::fill(ptr, ptr + width, color);
-    ptr += stride / VideoDecoder::kBytesPerPixel;
-  }
-}
-
-} // namespace
-
 // static
 scoped_ptr<VideoDecoderVpx> VideoDecoderVpx::CreateForVP8() {
-  ScopedVpxCodec codec(new vpx_codec_ctx_t);
-
-  // TODO(hclam): Scale the number of threads with number of cores of the
-  // machine.
-  vpx_codec_dec_cfg config;
-  config.w = 0;
-  config.h = 0;
-  config.threads = 2;
-  vpx_codec_err_t ret =
-      vpx_codec_dec_init(codec.get(), vpx_codec_vp8_dx(), &config, 0);
-  if (ret != VPX_CODEC_OK) {
-    LOG(ERROR) << "Cannot initialize codec.";
-    return nullptr;
-  }
-
-  return make_scoped_ptr(new VideoDecoderVpx(codec.Pass()));
+  return make_scoped_ptr(new VideoDecoderVpx(vpx_codec_vp8_dx()));
 }
 
 // static
 scoped_ptr<VideoDecoderVpx> VideoDecoderVpx::CreateForVP9() {
-  ScopedVpxCodec codec(new vpx_codec_ctx_t);
-
-  // TODO(hclam): Scale the number of threads with number of cores of the
-  // machine.
-  vpx_codec_dec_cfg config;
-  config.w = 0;
-  config.h = 0;
-  config.threads = 2;
-  vpx_codec_err_t ret =
-      vpx_codec_dec_init(codec.get(), vpx_codec_vp9_dx(), &config, 0);
-  if (ret != VPX_CODEC_OK) {
-    LOG(ERROR) << "Cannot initialize codec.";
-    return nullptr;
-  }
-
-  return make_scoped_ptr(new VideoDecoderVpx(codec.Pass()));
+  return make_scoped_ptr(new VideoDecoderVpx(vpx_codec_vp9_dx()));
 }
 
 VideoDecoderVpx::~VideoDecoderVpx() {}
 
-void VideoDecoderVpx::Initialize(const webrtc::DesktopSize& screen_size) {
-  DCHECK(!screen_size.is_empty());
-
-  screen_size_ = screen_size;
-
-  transparent_region_.SetRect(webrtc::DesktopRect::MakeSize(screen_size_));
+void VideoDecoderVpx::Initialize(const webrtc::DesktopSize& source_size) {
+  // Nothing to do here; the codec handles resizing internally, and returns
+  // the source dimensions as part of the vpx_image_t.
 }
 
 bool VideoDecoderVpx::DecodePacket(const VideoPacket& packet) {
-  DCHECK(!screen_size_.is_empty());
-
-  // Do the actual decoding.
+  // Pass the packet to the codec to process.
   vpx_codec_err_t ret = vpx_codec_decode(
       codec_.get(), reinterpret_cast<const uint8*>(packet.data().data()),
       packet.data().size(), nullptr, 0);
@@ -107,15 +52,16 @@ bool VideoDecoderVpx::DecodePacket(const VideoPacket& packet) {
     return false;
   }
 
-  // Gets the decoded data.
+  // Fetch the decoded video frame.
   vpx_codec_iter_t iter = nullptr;
-  vpx_image_t* image = vpx_codec_get_frame(codec_.get(), &iter);
-  if (!image) {
+  image_ = vpx_codec_get_frame(codec_.get(), &iter);
+  if (!image_) {
     LOG(ERROR) << "No video frame decoded";
     return false;
   }
-  last_image_ = image;
+  DCHECK(!image_size().is_empty());
 
+  // Determine which areas have been updated.
   webrtc::DesktopRegion region;
   for (int i = 0; i < packet.dirty_rects_size(); ++i) {
     Rect remoting_rect = packet.dirty_rects(i);
@@ -123,27 +69,25 @@ bool VideoDecoderVpx::DecodePacket(const VideoPacket& packet) {
         remoting_rect.x(), remoting_rect.y(),
         remoting_rect.width(), remoting_rect.height()));
   }
-
   updated_region_.AddRegion(region);
 
-  // Update the desktop shape region.
-  webrtc::DesktopRegion desktop_shape_region;
+  // Process the frame shape, if supplied.
   if (packet.has_use_desktop_shape()) {
-    for (int i = 0; i < packet.desktop_shape_rects_size(); ++i) {
-      Rect remoting_rect = packet.desktop_shape_rects(i);
-      desktop_shape_region.AddRect(webrtc::DesktopRect::MakeXYWH(
-          remoting_rect.x(), remoting_rect.y(),
-          remoting_rect.width(), remoting_rect.height()));
+    if (packet.use_desktop_shape()) {
+      if (!desktop_shape_)
+        desktop_shape_ = make_scoped_ptr(new webrtc::DesktopRegion);
+      desktop_shape_->Clear();
+      for (int i = 0; i < packet.desktop_shape_rects_size(); ++i) {
+        Rect remoting_rect = packet.desktop_shape_rects(i);
+        desktop_shape_->AddRect(webrtc::DesktopRect::MakeXYWH(
+            remoting_rect.x(), remoting_rect.y(), remoting_rect.width(),
+            remoting_rect.height()));
+      }
+    } else {
+      desktop_shape_.reset();
     }
-  } else {
-    // Fallback for the case when the host didn't include the desktop shape
-    // region.
-    desktop_shape_region =
-        webrtc::DesktopRegion(webrtc::DesktopRect::MakeSize(screen_size_));
   }
 
-  UpdateImageShapeRegion(&desktop_shape_region);
-
   return true;
 }
 
@@ -152,15 +96,8 @@ void VideoDecoderVpx::Invalidate(const webrtc::DesktopSize& view_size,
   DCHECK(!view_size.is_empty());
 
   for (webrtc::DesktopRegion::Iterator i(region); !i.IsAtEnd(); i.Advance()) {
-    updated_region_.AddRect(ScaleRect(i.rect(), view_size, screen_size_));
+    updated_region_.AddRect(ScaleRect(i.rect(), view_size, image_size()));
   }
-
-  // Updated areas outside of the new desktop shape region should be made
-  // transparent, not repainted.
-  webrtc::DesktopRegion difference = updated_region_;
-  difference.Subtract(desktop_shape_);
-  updated_region_.Subtract(difference);
-  transparent_region_.AddRegion(difference);
 }
 
 void VideoDecoderVpx::RenderFrame(const webrtc::DesktopSize& view_size,
@@ -168,21 +105,20 @@ void VideoDecoderVpx::RenderFrame(const webrtc::DesktopSize& view_size,
                                   uint8* image_buffer,
                                   int image_stride,
                                   webrtc::DesktopRegion* output_region) {
-  DCHECK(!screen_size_.is_empty());
+  DCHECK(!image_size().is_empty());
   DCHECK(!view_size.is_empty());
 
   // Early-return and do nothing if we haven't yet decoded any frames.
-  if (!last_image_)
+  if (!image_)
     return;
 
-  webrtc::DesktopRect source_clip =
-      webrtc::DesktopRect::MakeWH(last_image_->d_w, last_image_->d_h);
+  webrtc::DesktopRect source_clip = webrtc::DesktopRect::MakeSize(image_size());
 
   // VP8 only outputs I420 frames, but VP9 can also produce I444.
-  switch (last_image_->fmt) {
+  switch (image_->fmt) {
     case VPX_IMG_FMT_I444: {
       // TODO(wez): Add scaling support to the I444 conversion path.
-      if (view_size.equals(screen_size_)) {
+      if (view_size.equals(image_size())) {
         for (webrtc::DesktopRegion::Iterator i(updated_region_);
              !i.IsAtEnd(); i.Advance()) {
           // Determine the scaled area affected by this rectangle changing.
@@ -194,15 +130,12 @@ void VideoDecoderVpx::RenderFrame(const webrtc::DesktopSize& view_size,
 
           int image_offset = image_stride * rect.top() +
                              rect.left() * VideoDecoder::kBytesPerPixel;
-          int y_offset = last_image_->stride[0] * rect.top() + rect.left();
-          int u_offset = last_image_->stride[1] * rect.top() + rect.left();
-          int v_offset = last_image_->stride[2] * rect.top() + rect.left();
-          libyuv::I444ToARGB(last_image_->planes[0] + y_offset,
-                             last_image_->stride[0],
-                             last_image_->planes[1] + u_offset,
-                             last_image_->stride[1],
-                             last_image_->planes[2] + v_offset,
-                             last_image_->stride[2],
+          int y_offset = image_->stride[0] * rect.top() + rect.left();
+          int u_offset = image_->stride[1] * rect.top() + rect.left();
+          int v_offset = image_->stride[2] * rect.top() + rect.left();
+          libyuv::I444ToARGB(image_->planes[0] + y_offset, image_->stride[0],
+                             image_->planes[1] + u_offset, image_->stride[1],
+                             image_->planes[2] + v_offset, image_->stride[2],
                              image_buffer + image_offset, image_stride,
                              rect.width(), rect.height());
 
@@ -224,7 +157,7 @@ void VideoDecoderVpx::RenderFrame(const webrtc::DesktopSize& view_size,
         // We're scaling only |clip_area| into the |image_buffer|, so we need to
         // work out which source rectangle that corresponds to.
         webrtc::DesktopRect source_rect =
-            ScaleRect(clip_area, view_size, screen_size_);
+            ScaleRect(clip_area, view_size, image_size());
         source_rect = webrtc::DesktopRect::MakeLTRB(
             RoundToTwosMultiple(source_rect.left()),
             RoundToTwosMultiple(source_rect.top()),
@@ -240,23 +173,15 @@ void VideoDecoderVpx::RenderFrame(const webrtc::DesktopSize& view_size,
 
         // Scale & convert the entire clip area.
         int y_offset = CalculateYOffset(source_rect.left(), source_rect.top(),
-                                        last_image_->stride[0]);
+                                        image_->stride[0]);
         int uv_offset = CalculateUVOffset(source_rect.left(), source_rect.top(),
-                                          last_image_->stride[1]);
-        ScaleYUVToRGB32(last_image_->planes[0] + y_offset,
-                        last_image_->planes[1] + uv_offset,
-                        last_image_->planes[2] + uv_offset,
-                        image_buffer,
-                        source_rect.width(),
-                        source_rect.height(),
-                        clip_area.width(),
-                        clip_area.height(),
-                        last_image_->stride[0],
-                        last_image_->stride[1],
-                        image_stride,
-                        media::YV12,
-                        media::ROTATE_0,
-                        media::FILTER_BILINEAR);
+                                          image_->stride[1]);
+        ScaleYUVToRGB32(
+            image_->planes[0] + y_offset, image_->planes[1] + uv_offset,
+            image_->planes[2] + uv_offset, image_buffer, source_rect.width(),
+            source_rect.height(), clip_area.width(), clip_area.height(),
+            image_->stride[0], image_->stride[1], image_stride, media::YV12,
+            media::ROTATE_0, media::FILTER_BILINEAR);
 
         output_region->AddRect(clip_area);
         updated_region_.Subtract(source_rect);
@@ -270,86 +195,51 @@ void VideoDecoderVpx::RenderFrame(const webrtc::DesktopSize& view_size,
         rect.IntersectWith(source_clip);
         if (rect.is_empty())
           continue;
-        rect = ScaleRect(rect, screen_size_, view_size);
+        rect = ScaleRect(rect, image_size(), view_size);
         rect.IntersectWith(clip_area);
         if (rect.is_empty())
           continue;
 
-        ConvertAndScaleYUVToRGB32Rect(last_image_->planes[0],
-                                      last_image_->planes[1],
-                                      last_image_->planes[2],
-                                      last_image_->stride[0],
-                                      last_image_->stride[1],
-                                      screen_size_,
-                                      source_clip,
-                                      image_buffer,
-                                      image_stride,
-                                      view_size,
-                                      clip_area,
-                                      rect);
+        ConvertAndScaleYUVToRGB32Rect(
+            image_->planes[0], image_->planes[1], image_->planes[2],
+            image_->stride[0], image_->stride[1], image_size(), source_clip,
+            image_buffer, image_stride, view_size, clip_area, rect);
 
         output_region->AddRect(rect);
       }
 
-      updated_region_.Subtract(ScaleRect(clip_area, view_size, screen_size_));
+      updated_region_.Subtract(ScaleRect(clip_area, view_size, image_size()));
       break;
     }
     default: {
-      LOG(ERROR) << "Unsupported image format:" << last_image_->fmt;
+      LOG(ERROR) << "Unsupported image format:" << image_->fmt;
       return;
     }
   }
 
-  for (webrtc::DesktopRegion::Iterator i(transparent_region_);
-       !i.IsAtEnd(); i.Advance()) {
-    // Determine the scaled area affected by this rectangle changing.
-    webrtc::DesktopRect rect = i.rect();
-    rect.IntersectWith(source_clip);
-    if (rect.is_empty())
-      continue;
-    rect = ScaleRect(rect, screen_size_, view_size);
-    rect.IntersectWith(clip_area);
-    if (rect.is_empty())
-      continue;
-
-    // Fill the rectange with transparent pixels.
-    FillRect(image_buffer, image_stride, rect, kTransparentColor);
-    output_region->AddRect(rect);
-  }
-
   webrtc::DesktopRect scaled_clip_area =
-      ScaleRect(clip_area, view_size, screen_size_);
+      ScaleRect(clip_area, view_size, image_size());
   updated_region_.Subtract(scaled_clip_area);
-  transparent_region_.Subtract(scaled_clip_area);
 }
 
 const webrtc::DesktopRegion* VideoDecoderVpx::GetImageShape() {
-  return &desktop_shape_;
+  return desktop_shape_.get();
 }
 
-VideoDecoderVpx::VideoDecoderVpx(ScopedVpxCodec codec)
-    : codec_(codec.Pass()),
-      last_image_(nullptr) {
-  DCHECK(codec_);
+VideoDecoderVpx::VideoDecoderVpx(vpx_codec_iface_t* codec) : image_(nullptr) {
+  codec_.reset(new vpx_codec_ctx_t);
+
+  vpx_codec_dec_cfg config;
+  config.w = 0;
+  config.h = 0;
+  config.threads = 2;
+  vpx_codec_err_t ret = vpx_codec_dec_init(codec_.get(), codec, &config, 0);
+  CHECK_EQ(VPX_CODEC_OK, ret);
 }
 
-void VideoDecoderVpx::UpdateImageShapeRegion(
-    webrtc::DesktopRegion* new_desktop_shape) {
-  // Add all areas that have been updated or become transparent to the
-  // transparent region. Exclude anything within the new desktop shape.
-  transparent_region_.AddRegion(desktop_shape_);
-  transparent_region_.AddRegion(updated_region_);
-  transparent_region_.Subtract(*new_desktop_shape);
-
-  // Add newly exposed areas to the update region and limit updates to the new
-  // desktop shape.
-  webrtc::DesktopRegion difference = *new_desktop_shape;
-  difference.Subtract(desktop_shape_);
-  updated_region_.AddRegion(difference);
-  updated_region_.IntersectWith(*new_desktop_shape);
-
-  // Set the new desktop shape region.
-  desktop_shape_.Swap(new_desktop_shape);
+webrtc::DesktopSize VideoDecoderVpx::image_size() const {
+  return image_ ? webrtc::DesktopSize(image_->d_w, image_->d_h)
+                : webrtc::DesktopSize();
 }
 
 }  // namespace remoting
diff --git a/remoting/codec/video_decoder_vpx.h b/remoting/codec/video_decoder_vpx.h
index 9c96998..21a576a 100644
--- a/remoting/codec/video_decoder_vpx.h
+++ b/remoting/codec/video_decoder_vpx.h
@@ -12,6 +12,7 @@
 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
 
+typedef const struct vpx_codec_iface vpx_codec_iface_t;
 typedef struct vpx_image vpx_image_t;
 
 namespace remoting {
@@ -25,7 +26,7 @@ class VideoDecoderVpx : public VideoDecoder {
   ~VideoDecoderVpx() override;
 
   // VideoDecoder interface.
-  void Initialize(const webrtc::DesktopSize& screen_size) override;
+  void Initialize(const webrtc::DesktopSize& source_size) override;
   bool DecodePacket(const VideoPacket& packet) override;
   void Invalidate(const webrtc::DesktopSize& view_size,
                   const webrtc::DesktopRegion& region) override;
@@ -37,29 +38,21 @@ class VideoDecoderVpx : public VideoDecoder {
   const webrtc::DesktopRegion* GetImageShape() override;
 
  private:
-  explicit VideoDecoderVpx(ScopedVpxCodec codec);
+  explicit VideoDecoderVpx(vpx_codec_iface_t* codec);
 
-  // Calculates the difference between the desktop shape regions in two
-  // consecutive frames and updates |updated_region_| and |transparent_region_|
-  // accordingly.
-  void UpdateImageShapeRegion(webrtc::DesktopRegion* new_desktop_shape);
+  // Returns the dimensions of the most recent frame as a DesktopSize.
+  webrtc::DesktopSize image_size() const;
 
   ScopedVpxCodec codec_;
 
-  // Pointer to the last decoded image.
-  vpx_image_t* last_image_;
+  // Pointer to the most recently decoded image.
+  vpx_image_t* image_;
 
-  // The region updated that hasn't been copied to the screen yet.
+  // Area of the source that has changed since the last RenderFrame call.
   webrtc::DesktopRegion updated_region_;
 
-  // Output dimensions.
-  webrtc::DesktopSize screen_size_;
-
-  // The region occupied by the top level windows.
-  webrtc::DesktopRegion desktop_shape_;
-
-  // The region that should be make transparent.
-  webrtc::DesktopRegion transparent_region_;
+  // The shape of the most-recent frame, if any.
+  scoped_ptr<webrtc::DesktopRegion> desktop_shape_;
 
   DISALLOW_COPY_AND_ASSIGN(VideoDecoderVpx);
 };