diff options
author | alexeypa@chromium.org <alexeypa@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-02-24 23:05:56 +0000 |
---|---|---|
committer | alexeypa@chromium.org <alexeypa@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-02-24 23:05:56 +0000 |
commit | 55d3688e99e891b269cc93c1f695c65a00ea3b48 (patch) | |
tree | 31c34f0901866f698d241b7b45a71a2ba6d3cce7 /remoting | |
parent | 1fd0ff3f0814835c474e527b08c6a2f560246e94 (diff) | |
download | chromium_src-55d3688e99e891b269cc93c1f695c65a00ea3b48.zip chromium_src-55d3688e99e891b269cc93c1f695c65a00ea3b48.tar.gz chromium_src-55d3688e99e891b269cc93c1f695c65a00ea3b48.tar.bz2 |
This CL makes several the following improvements to the Chromoting decoder pipeline:
1. Only the clipping area, not the full frame, is drawn. This reduces the risk of out of memory situation on high page zoom levels. Screen updates are also incremental including scrolling scenarios.
2. Decoders now write pixels directly to the Pepper API backing store making it one memcpy less.
3. Scaling and panning is fully controlled by the plugin. The decoder only supplies the pixels it was asked for by the plugin.
BUG=109938
Review URL: http://codereview.chromium.org/9331003
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@123573 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'remoting')
-rw-r--r-- | remoting/base/codec_test.cc | 23 | ||||
-rw-r--r-- | remoting/base/compressor_verbatim.cc | 4 | ||||
-rw-r--r-- | remoting/base/decoder.h | 56 | ||||
-rw-r--r-- | remoting/base/decoder_row_based.cc | 87 | ||||
-rw-r--r-- | remoting/base/decoder_row_based.h | 21 | ||||
-rw-r--r-- | remoting/base/decoder_vp8.cc | 107 | ||||
-rw-r--r-- | remoting/base/decoder_vp8.h | 24 | ||||
-rw-r--r-- | remoting/base/util.cc | 4 | ||||
-rw-r--r-- | remoting/client/chromoting_client.cc | 5 | ||||
-rw-r--r-- | remoting/client/chromoting_client.h | 5 | ||||
-rw-r--r-- | remoting/client/chromoting_view.h | 17 | ||||
-rw-r--r-- | remoting/client/frame_consumer.h | 60 | ||||
-rw-r--r-- | remoting/client/frame_consumer_proxy.cc | 43 | ||||
-rw-r--r-- | remoting/client/frame_consumer_proxy.h | 21 | ||||
-rw-r--r-- | remoting/client/frame_producer.h | 51 | ||||
-rw-r--r-- | remoting/client/plugin/chromoting_instance.cc | 25 | ||||
-rw-r--r-- | remoting/client/plugin/pepper_view.cc | 453 | ||||
-rw-r--r-- | remoting/client/plugin/pepper_view.h | 90 | ||||
-rw-r--r-- | remoting/client/rectangle_update_decoder.cc | 206 | ||||
-rw-r--r-- | remoting/client/rectangle_update_decoder.h | 65 | ||||
-rw-r--r-- | remoting/remoting.gyp | 1 |
21 files changed, 689 insertions, 679 deletions
diff --git a/remoting/base/codec_test.cc b/remoting/base/codec_test.cc index 81f0db8..aa761d8 100644 --- a/remoting/base/codec_test.cc +++ b/remoting/base/codec_test.cc @@ -123,12 +123,9 @@ class DecoderTester { DecoderTester(Decoder* decoder) : strict_(false), decoder_(decoder) { - frame_ = media::VideoFrame::CreateFrame(media::VideoFrame::RGB32, - kWidth, kHeight, - base::TimeDelta(), - base::TimeDelta()); - EXPECT_TRUE(frame_.get()); - decoder_->Initialize(frame_); + image_data_.reset(new uint8[kWidth * kHeight * kBytesPerPixel]); + EXPECT_TRUE(image_data_.get()); + decoder_->Initialize(SkISize::Make(kWidth, kHeight)); } void Reset() { @@ -142,7 +139,11 @@ class DecoderTester { ASSERT_NE(Decoder::DECODE_ERROR, result); if (result == Decoder::DECODE_DONE) { - decoder_->GetUpdatedRegion(&update_region_); + decoder_->RenderFrame(SkISize::Make(kWidth, kHeight), + SkIRect::MakeXYWH(0, 0, kWidth, kHeight), + image_data_.get(), + kWidth * kBytesPerPixel, + &update_region_); } } @@ -169,12 +170,12 @@ class DecoderTester { // Test the content of the update region. EXPECT_EQ(expected_region_, update_region_); for (SkRegion::Iterator i(update_region_); !i.done(); i.next()) { - EXPECT_EQ(frame_->stride(0), capture_data_->data_planes().strides[0]); - const int stride = frame_->stride(0); + const int stride = kWidth * kBytesPerPixel; + EXPECT_EQ(stride, capture_data_->data_planes().strides[0]); const int offset = stride * i.rect().top() + kBytesPerPixel * i.rect().left(); const uint8* original = capture_data_->data_planes().data[0] + offset; - const uint8* decoded = frame_->data(0) + offset; + const uint8* decoded = image_data_.get() + offset; const int row_size = kBytesPerPixel * i.rect().width(); for (int y = 0; y < i.rect().height(); ++y) { EXPECT_EQ(0, memcmp(original, decoded, row_size)) @@ -190,7 +191,7 @@ class DecoderTester { SkRegion expected_region_; SkRegion update_region_; Decoder* decoder_; - scoped_refptr<media::VideoFrame> frame_; + scoped_array<uint8> image_data_; scoped_refptr<CaptureData> capture_data_; DISALLOW_COPY_AND_ASSIGN(DecoderTester); diff --git a/remoting/base/compressor_verbatim.cc b/remoting/base/compressor_verbatim.cc index 7058b52..97b71e1 100644 --- a/remoting/base/compressor_verbatim.cc +++ b/remoting/base/compressor_verbatim.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2010 The Chromium Authors. All rights reserved. +// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -26,7 +26,7 @@ bool CompressorVerbatim::Process(const uint8* input_data, int input_size, // Since we're just a memcpy, consumed and written are the same. *consumed = *written = bytes_to_copy; - return true; + return (flush != CompressorFinish) || (output_size < bytes_to_copy); } } // namespace remoting diff --git a/remoting/base/decoder.h b/remoting/base/decoder.h index 9efc17e..dd3d43e 100644 --- a/remoting/base/decoder.h +++ b/remoting/base/decoder.h @@ -5,10 +5,12 @@ #ifndef REMOTING_BASE_DECODER_H_ #define REMOTING_BASE_DECODER_H_ +#include "base/basictypes.h" #include "base/memory/scoped_ptr.h" -#include "media/base/video_frame.h" #include "remoting/proto/video.pb.h" +#include "third_party/skia/include/core/SkRect.h" #include "third_party/skia/include/core/SkRegion.h" +#include "third_party/skia/include/core/SkSize.h" namespace remoting { @@ -32,41 +34,41 @@ class Decoder { Decoder() {} virtual ~Decoder() {} - // Initializes the decoder to draw into the given |frame|. - virtual void Initialize(scoped_refptr<media::VideoFrame> frame) = 0; + // Initializes the decoder and sets the output dimensions. + virtual void Initialize(const SkISize& screen_size) = 0; // Feeds more data into the decoder. virtual DecodeResult DecodePacket(const VideoPacket* packet) = 0; - // Returns the region affected by the most recent frame. Can be called only - // after DecodePacket returned DECODE_DONE. Caller keeps ownership of - // |region|. - virtual void GetUpdatedRegion(SkRegion* region) = 0; - - // Reset the decoder to an uninitialized state. Release all references to - // the initialized |frame|. Initialize() must be called before the decoder - // is used again. - virtual void Reset() = 0; - // Returns true if decoder is ready to accept data via DecodePacket. virtual bool IsReadyForData() = 0; virtual VideoPacketFormat::Encoding Encoding() = 0; - // Set the output dimensions for the decoder. If the dimensions are empty - // then the source is rendered without scaling. - // Output dimensions are ignored if the decoder doesn't support scaling. - virtual void SetOutputSize(const SkISize& size) {} - - // Set the clipping rectangle to the decoder. Decoder should respect this and - // only output changes in this rectangle. The new clipping rectangle will be - // effective on the next decoded video frame. - virtual void SetClipRect(const SkIRect& clip_rect) {} - - // Force decoder to output a frame based on the specified |region| of the - // most recently decoded video frame. |region| is expressed in video frame - // rather than output coordinates. - virtual void RefreshRegion(const SkRegion& region) {} + // Forces the decoder to include the specified |region| the next time + // RenderFrame() is called. |region| is expressed in output coordinates. + virtual void Invalidate(const SkISize& view_size, + const SkRegion& region) = 0; + + // Copies invalidated pixels of the video frame to |image_buffer|. Both + // decoding a packet or Invalidate() call can result in parts of the frame + // to be invalidated. Only the pixels within |clip_area| are copied. + // Invalidated pixels outside of |clip_area| remain invalidated. + // + // The routine sets |output_region| to indicate the updated areas of + // |image_buffer|. |output_region| is in output buffer coordinates. + // + // |image_buffer| is assumed to be large enough to hold entire |clip_area| + // (RGBA32). The top left corner of the buffer corresponds to the top left + // corner of |clip_area|. |image_stride| specifies the size of a single row + // of the buffer in bytes. + // + // Both |clip_area| and |output_region| are expressed in output coordinates. + virtual void RenderFrame(const SkISize& view_size, + const SkIRect& clip_area, + uint8* image_buffer, + int image_stride, + SkRegion* output_region) = 0; }; } // namespace remoting diff --git a/remoting/base/decoder_row_based.cc b/remoting/base/decoder_row_based.cc index 5c95991..d98a5ca 100644 --- a/remoting/base/decoder_row_based.cc +++ b/remoting/base/decoder_row_based.cc @@ -34,19 +34,13 @@ DecoderRowBased::DecoderRowBased(Decompressor* decompressor, decompressor_(decompressor), encoding_(encoding), row_pos_(0), - row_y_(0) { + row_y_(0), + screen_size_(SkISize::Make(0, 0)) { } DecoderRowBased::~DecoderRowBased() { } -void DecoderRowBased::Reset() { - frame_ = NULL; - decompressor_->Reset(); - state_ = kUninitialized; - updated_region_.setEmpty(); -} - bool DecoderRowBased::IsReadyForData() { switch (state_) { case kUninitialized: @@ -62,17 +56,18 @@ bool DecoderRowBased::IsReadyForData() { return false; } -void DecoderRowBased::Initialize(scoped_refptr<media::VideoFrame> frame) { - // Make sure we are not currently initialized. - CHECK_EQ(kUninitialized, state_); +void DecoderRowBased::Initialize(const SkISize& screen_size) { + decompressor_->Reset(); + updated_region_.setEmpty(); + screen_buffer_.reset(NULL); - if (frame->format() != media::VideoFrame::RGB32) { - LOG(WARNING) << "DecoderRowBased only supports RGB32."; - state_ = kError; - return; + screen_size_ = screen_size; + // Allocate the screen buffer, if necessary. + if (!screen_size_.isEmpty()) { + screen_buffer_.reset(new uint8[ + screen_size_.width() * screen_size_.height() * kBytesPerPixel]); } - frame_ = frame; state_ = kReady; } @@ -85,13 +80,11 @@ Decoder::DecodeResult DecoderRowBased::DecodePacket(const VideoPacket* packet) { const uint8* in = reinterpret_cast<const uint8*>(packet->data().data()); const int in_size = packet->data().size(); - const int row_size = clip_.width() * kBytesPerPixel; - int stride = frame_->stride(media::VideoFrame::kRGBPlane); - uint8* rect_begin = frame_->data(media::VideoFrame::kRGBPlane); - uint8* out = rect_begin + stride * (clip_.fTop + row_y_) + - kBytesPerPixel * clip_.fLeft; + int out_stride = screen_size_.width() * kBytesPerPixel; + uint8* out = screen_buffer_.get() + out_stride * (clip_.top() + row_y_) + + kBytesPerPixel * clip_.left(); // Consume all the data in the message. bool decompress_again = true; @@ -105,8 +98,6 @@ Decoder::DecodeResult DecoderRowBased::DecodePacket(const VideoPacket* packet) { int written = 0; int consumed = 0; - // TODO(ajwong): This assume source and dest stride are the same, which is - // incorrect. decompress_again = decompressor_->Process( in + used, in_size - used, out + row_pos_, row_size - row_pos_, &consumed, &written); @@ -117,7 +108,7 @@ Decoder::DecodeResult DecoderRowBased::DecodePacket(const VideoPacket* packet) { if (row_pos_ == row_size) { ++row_y_; row_pos_ = 0; - out += stride; + out += out_stride; } } @@ -150,11 +141,17 @@ void DecoderRowBased::UpdateStateForPacket(const VideoPacket* packet) { LOG(WARNING) << "Received unexpected FIRST_PACKET."; return; } - state_ = kProcessing; // Reset the buffer location status variables on the first packet. clip_.setXYWH(packet->format().x(), packet->format().y(), packet->format().width(), packet->format().height()); + if (!SkIRect::MakeSize(screen_size_).contains(clip_)) { + state_ = kError; + LOG(WARNING) << "Invalid clipping area received."; + return; + } + + state_ = kProcessing; row_pos_ = 0; row_y_ = 0; } @@ -186,13 +183,43 @@ void DecoderRowBased::UpdateStateForPacket(const VideoPacket* packet) { return; } -void DecoderRowBased::GetUpdatedRegion(SkRegion* region) { - region->swap(updated_region_); - updated_region_.setEmpty(); -} - VideoPacketFormat::Encoding DecoderRowBased::Encoding() { return encoding_; } +void DecoderRowBased::Invalidate(const SkISize& view_size, + const SkRegion& region) { + updated_region_.op(region, SkRegion::kUnion_Op); +} + +void DecoderRowBased::RenderFrame(const SkISize& view_size, + const SkIRect& clip_area, + uint8* image_buffer, + int image_stride, + SkRegion* output_region) { + output_region->setEmpty(); + + // TODO(alexeypa): scaling is not implemented. + SkIRect clip_rect = SkIRect::MakeSize(screen_size_); + if (!clip_rect.intersect(clip_area)) + return; + + int screen_stride = screen_size_.width() * kBytesPerPixel; + + for (SkRegion::Iterator i(updated_region_); !i.done(); i.next()) { + SkIRect rect(i.rect()); + if (!rect.intersect(clip_rect)) + continue; + + CopyRGB32Rect(screen_buffer_.get(), screen_stride, + clip_rect, + image_buffer, image_stride, + clip_area, + rect); + output_region->op(rect, SkRegion::kUnion_Op); + } + + updated_region_.setEmpty(); +} + } // namespace remoting diff --git a/remoting/base/decoder_row_based.h b/remoting/base/decoder_row_based.h index 183086c..a88daa0 100644 --- a/remoting/base/decoder_row_based.h +++ b/remoting/base/decoder_row_based.h @@ -20,11 +20,16 @@ class DecoderRowBased : public Decoder { // Decoder implementation. virtual bool IsReadyForData() OVERRIDE; - virtual void Initialize(scoped_refptr<media::VideoFrame> frame) OVERRIDE; + virtual void Initialize(const SkISize& screen_size) OVERRIDE; virtual DecodeResult DecodePacket(const VideoPacket* packet) OVERRIDE; - virtual void GetUpdatedRegion(SkRegion* region) OVERRIDE; - virtual void Reset() OVERRIDE; virtual VideoPacketFormat::Encoding Encoding() OVERRIDE; + virtual void Invalidate(const SkISize& view_size, + const SkRegion& region) OVERRIDE; + virtual void RenderFrame(const SkISize& view_size, + const SkIRect& clip_area, + uint8* image_buffer, + int image_stride, + SkRegion* output_region) OVERRIDE; private: enum State { @@ -48,9 +53,6 @@ class DecoderRowBased : public Decoder { // Keeps track of the updating rect. SkIRect clip_; - // The video frame to write to. - scoped_refptr<media::VideoFrame> frame_; - // The compression for the input byte stream. scoped_ptr<Decompressor> decompressor_; @@ -63,8 +65,15 @@ class DecoderRowBased : public Decoder { // The current row in the rect that we are updaing. int row_y_; + // The region updated that hasn't been copied to the screen yet. SkRegion updated_region_; + // Size of the remote screen. + SkISize screen_size_; + + // The bitmap holding the remote screen bits. + scoped_array<uint8> screen_buffer_; + DISALLOW_COPY_AND_ASSIGN(DecoderRowBased); }; diff --git a/remoting/base/decoder_vp8.cc b/remoting/base/decoder_vp8.cc index fc6670f..88ea3b6 100644 --- a/remoting/base/decoder_vp8.cc +++ b/remoting/base/decoder_vp8.cc @@ -22,8 +22,7 @@ DecoderVp8::DecoderVp8() : state_(kUninitialized), codec_(NULL), last_image_(NULL), - clip_rect_(SkIRect::MakeEmpty()), - output_size_(SkISize::Make(0, 0)) { + screen_size_(SkISize::Make(0, 0)) { } DecoderVp8::~DecoderVp8() { @@ -34,16 +33,8 @@ DecoderVp8::~DecoderVp8() { delete codec_; } -void DecoderVp8::Initialize(scoped_refptr<media::VideoFrame> frame) { - DCHECK_EQ(kUninitialized, state_); - - if (frame->format() != media::VideoFrame::RGB32) { - LOG(INFO) << "DecoderVp8 only supports RGB32 as output"; - state_ = kError; - return; - } - frame_ = frame; - +void DecoderVp8::Initialize(const SkISize& screen_size) { + screen_size_ = screen_size; state_ = kReady; } @@ -102,19 +93,10 @@ Decoder::DecodeResult DecoderVp8::DecodePacket(const VideoPacket* packet) { region.op(rect, SkRegion::kUnion_Op); } - RefreshRegion(region); + updated_region_.op(region, SkRegion::kUnion_Op); return DECODE_DONE; } -void DecoderVp8::GetUpdatedRegion(SkRegion* region) { - region->swap(updated_region_); -} - -void DecoderVp8::Reset() { - frame_ = NULL; - state_ = kUninitialized; -} - bool DecoderVp8::IsReadyForData() { return state_ == kReady; } @@ -123,68 +105,49 @@ VideoPacketFormat::Encoding DecoderVp8::Encoding() { return VideoPacketFormat::ENCODING_VP8; } -void DecoderVp8::SetOutputSize(const SkISize& size) { - output_size_ = size; -} - -void DecoderVp8::SetClipRect(const SkIRect& clip_rect) { - clip_rect_ = clip_rect; +void DecoderVp8::Invalidate(const SkISize& view_size, + const SkRegion& region) { + for (SkRegion::Iterator i(region); !i.done(); i.next()) { + SkIRect rect = i.rect(); + rect = ScaleRect(rect, view_size, screen_size_); + updated_region_.op(rect, SkRegion::kUnion_Op); + } } -void DecoderVp8::RefreshRegion(const SkRegion& region) { - // TODO(wez): Fix the rest of the decode pipeline not to assume the frame - // size is the host dimensions, since it's not when scaling. If the host - // gets smaller, then the output size will be too big and we'll overrun the - // frame, so currently we render 1:1 in that case; the app will see the - // host size change and resize us if need be. - if (output_size_.width() > static_cast<int>(frame_->width())) - output_size_.set(frame_->width(), output_size_.height()); - if (output_size_.height() > static_cast<int>(frame_->height())) - output_size_.set(output_size_.width(), frame_->height()); - - if (!last_image_) - return; - - updated_region_.setEmpty(); - - // Clip based on both the output dimensions and Pepper clip rect. - // ConvertAndScaleYUVToRGB32Rect() requires even X and Y coordinates, so we - // align |clip_rect| to prevent clipping from breaking alignment. We then - // clamp it to the image dimensions, which may lead to odd width & height, - // which we can cope with. - SkIRect clip_rect = AlignRect(clip_rect_); - if (!clip_rect.intersect(SkIRect::MakeSize(output_size_))) - return; - - SkISize image_size = SkISize::Make(last_image_->d_w, last_image_->d_h); - uint8* output_rgb_buf = frame_->data(media::VideoFrame::kRGBPlane); - const int output_stride = frame_->stride(media::VideoFrame::kRGBPlane); +void DecoderVp8::RenderFrame(const SkISize& view_size, + const SkIRect& clip_area, + uint8* image_buffer, + int image_stride, + SkRegion* output_region) { + SkIRect source_clip = SkIRect::MakeWH(last_image_->d_w, last_image_->d_h); - for (SkRegion::Iterator i(region); !i.done(); i.next()) { + for (SkRegion::Iterator i(updated_region_); !i.done(); i.next()) { // Determine the scaled area affected by this rectangle changing. - // Align the rectangle so the top-left coordinates are even, for - // ConvertAndScaleYUVToRGB32Rect(). - SkIRect output_rect = ScaleRect(AlignRect(i.rect()), - image_size, output_size_); - if (!output_rect.intersect(clip_rect)) + SkIRect rect = i.rect(); + if (!rect.intersect(source_clip)) + continue; + rect = ScaleRect(rect, screen_size_, view_size); + if (!rect.intersect(clip_area)) continue; - // The scaler will not to read outside the input dimensions. ConvertAndScaleYUVToRGB32Rect(last_image_->planes[0], last_image_->planes[1], last_image_->planes[2], last_image_->stride[0], last_image_->stride[1], - image_size, - SkIRect::MakeSize(image_size), - output_rgb_buf, - output_stride, - output_size_, - SkIRect::MakeSize(output_size_), - output_rect); - - updated_region_.op(output_rect, SkRegion::kUnion_Op); + screen_size_, + source_clip, + image_buffer, + image_stride, + view_size, + clip_area, + rect); + + output_region->op(rect, SkRegion::kUnion_Op); } + + updated_region_.op(ScaleRect(clip_area, view_size, screen_size_), + SkRegion::kDifference_Op); } } // namespace remoting diff --git a/remoting/base/decoder_vp8.h b/remoting/base/decoder_vp8.h index 0549f03..c316ba8 100644 --- a/remoting/base/decoder_vp8.h +++ b/remoting/base/decoder_vp8.h @@ -18,15 +18,17 @@ class DecoderVp8 : public Decoder { virtual ~DecoderVp8(); // Decoder implementations. - virtual void Initialize(scoped_refptr<media::VideoFrame> frame) OVERRIDE; + virtual void Initialize(const SkISize& screen_size) OVERRIDE; virtual DecodeResult DecodePacket(const VideoPacket* packet) OVERRIDE; - virtual void GetUpdatedRegion(SkRegion* region) OVERRIDE; virtual bool IsReadyForData() OVERRIDE; - virtual void Reset() OVERRIDE; virtual VideoPacketFormat::Encoding Encoding() OVERRIDE; - virtual void SetOutputSize(const SkISize& size) OVERRIDE; - virtual void SetClipRect(const SkIRect& clip_rect) OVERRIDE; - virtual void RefreshRegion(const SkRegion& region) OVERRIDE; + virtual void Invalidate(const SkISize& view_size, + const SkRegion& region) OVERRIDE; + virtual void RenderFrame(const SkISize& view_size, + const SkIRect& clip_area, + uint8* image_buffer, + int image_stride, + SkRegion* output_region) OVERRIDE; private: enum State { @@ -38,22 +40,16 @@ class DecoderVp8 : public Decoder { // The internal state of the decoder. State state_; - // The video frame to write to. - scoped_refptr<media::VideoFrame> frame_; - vpx_codec_ctx_t* codec_; // Pointer to the last decoded image. vpx_image_t* last_image_; - // The region updated by the most recent decode. + // The region updated that hasn't been copied to the screen yet. SkRegion updated_region_; - // Clipping rect for the output of the decoder. - SkIRect clip_rect_; - // Output dimensions. - SkISize output_size_; + SkISize screen_size_; DISALLOW_COPY_AND_ASSIGN(DecoderVp8); }; diff --git a/remoting/base/util.cc b/remoting/base/util.cc index 30691ea..26886ca 100644 --- a/remoting/base/util.cc +++ b/remoting/base/util.cc @@ -221,9 +221,9 @@ SkIRect ScaleRect(const SkIRect& rect, const SkISize& out_size) { int left = (rect.left() * out_size.width()) / in_size.width(); int top = (rect.top() * out_size.height()) / in_size.height(); - int right = (rect.right() * out_size.width() + out_size.width() - 1) / + int right = (rect.right() * out_size.width() + in_size.width() - 1) / in_size.width(); - int bottom = (rect.bottom() * out_size.height() + out_size.height() - 1) / + int bottom = (rect.bottom() * out_size.height() + in_size.height() - 1) / in_size.height(); return SkIRect::MakeLTRB(left, top, right, bottom); } diff --git a/remoting/client/chromoting_client.cc b/remoting/client/chromoting_client.cc index a5f71ae..77be87e 100644 --- a/remoting/client/chromoting_client.cc +++ b/remoting/client/chromoting_client.cc @@ -103,11 +103,6 @@ ChromotingStats* ChromotingClient::GetStats() { return &stats_; } -void ChromotingClient::Repaint() { - DCHECK(message_loop()->BelongsToCurrentThread()); - view_->Paint(); -} - void ChromotingClient::ProcessVideoPacket(const VideoPacket* packet, const base::Closure& done) { DCHECK(message_loop()->BelongsToCurrentThread()); diff --git a/remoting/client/chromoting_client.h b/remoting/client/chromoting_client.h index 756c388..50a432d 100644 --- a/remoting/client/chromoting_client.h +++ b/remoting/client/chromoting_client.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -49,9 +49,6 @@ class ChromotingClient : public protocol::ConnectionToHost::HostEventCallback, // Return the stats recorded by this client. ChromotingStats* GetStats(); - // Signals that the associated view may need updating. - virtual void Repaint(); - // ConnectionToHost::HostEventCallback implementation. virtual void OnConnectionState( protocol::ConnectionToHost::State state, diff --git a/remoting/client/chromoting_view.h b/remoting/client/chromoting_view.h index e93ab0d..f9f0411 100644 --- a/remoting/client/chromoting_view.h +++ b/remoting/client/chromoting_view.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -10,10 +10,6 @@ namespace remoting { -static const uint32 kCreatedColor = 0xffccccff; -static const uint32 kDisconnectedColor = 0xff00ccff; -static const uint32 kFailedColor = 0xffcc00ff; - // ChromotingView defines the behavior of an object that draws a view of the // remote desktop. Its main function is to render the update stream onto the // screen. @@ -27,17 +23,6 @@ class ChromotingView { // Free up resources allocated by this view. virtual void TearDown() = 0; - // Tells the ChromotingView to paint the current image on the screen. - virtual void Paint() = 0; - - // Fill the screen with one single static color, and ignore updates. - // Useful for debugging. - virtual void SetSolidFill(uint32 color) = 0; - - // Removes a previously set solid fill. If no fill was previous set, this - // does nothing. - virtual void UnsetSolidFill() = 0; - // Record the update the state of the connection, updating the UI as needed. virtual void SetConnectionState(protocol::ConnectionToHost::State state, protocol::ConnectionToHost::Error error) = 0; diff --git a/remoting/client/frame_consumer.h b/remoting/client/frame_consumer.h index de56d81..ce852b4 100644 --- a/remoting/client/frame_consumer.h +++ b/remoting/client/frame_consumer.h @@ -5,7 +5,13 @@ #ifndef REMOTING_CLIENT_FRAME_CONSUMER_H_ #define REMOTING_CLIENT_FRAME_CONSUMER_H_ -#include "remoting/base/decoder.h" // For UpdatedRects +#include "third_party/skia/include/core/SkRect.h" +#include "third_party/skia/include/core/SkRegion.h" +#include "third_party/skia/include/core/SkSize.h" + +namespace pp { +class ImageData; +} // namespace pp namespace remoting { @@ -14,40 +20,26 @@ class FrameConsumer { FrameConsumer() {} virtual ~FrameConsumer() {} - // Request a frame be allocated from the FrameConsumer. - // - // If a frame cannot be allocated to fit the format, and |size| - // requirements, |frame_out| will be set to NULL. - // - // An allocated frame will have at least the |size| requested, but - // may be bigger. Query the retrun frame for the actual frame size, - // stride, etc. - // - // The AllocateFrame call is asynchronous. From invocation, until when the - // |done| callback is invoked, |frame_out| should be considered to be locked - // by the FrameConsumer, must remain a valid pointer, and should not be - // examined or modified. After |done| is called, the |frame_out| will - // contain a result of the allocation. If a frame could not be allocated, - // |frame_out| will be NULL. - // - // All frames retrieved via the AllocateFrame call must be released by a - // corresponding call ReleaseFrame(scoped_refptr<VideoFrame>* frame_out. - virtual void AllocateFrame(media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) = 0; - - virtual void ReleaseFrame(media::VideoFrame* frame) = 0; - - // OnPartialFrameOutput() is called every time at least one rectangle of - // output is produced. The |frame| is guaranteed to have valid data for all - // of |region|. + // Accepts a buffer to be painted to the screen. The buffer's dimensions and + // relative position within the frame are specified by |clip_area|. Only + // pixels falling within |region| and the current clipping area are painted. + // The function assumes that the passed buffer was scaled to fit a window + // having |view_size| dimensions. // - // Both |frame| and |region| are guaranteed to be valid until the |done| - // callback is invoked. - virtual void OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) = 0; + // N.B. Both |clip_area| and |region| are in output coordinates relative to + // the frame. + virtual void ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) = 0; + + // Accepts a buffer that couldn't be used for drawing for any reason (shutdown + // is in progress, the view area has changed, etc.). The accepted buffer can + // be freed or reused for another drawing operation. + virtual void ReturnBuffer(pp::ImageData* buffer) = 0; + + // Set the dimension of the entire host screen. + virtual void SetSourceSize(const SkISize& source_size) = 0; private: DISALLOW_COPY_AND_ASSIGN(FrameConsumer); diff --git a/remoting/client/frame_consumer_proxy.cc b/remoting/client/frame_consumer_proxy.cc index 65b84ed..47bc9cd 100644 --- a/remoting/client/frame_consumer_proxy.cc +++ b/remoting/client/frame_consumer_proxy.cc @@ -6,59 +6,60 @@ #include "base/bind.h" #include "base/message_loop.h" +#include "ppapi/cpp/image_data.h" namespace remoting { FrameConsumerProxy::FrameConsumerProxy( - FrameConsumer* frame_consumer, base::MessageLoopProxy* frame_consumer_message_loop) - : frame_consumer_(frame_consumer), + : frame_consumer_(NULL), frame_consumer_message_loop_(frame_consumer_message_loop) { } FrameConsumerProxy::~FrameConsumerProxy() { } -void FrameConsumerProxy::AllocateFrame( - media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) { +void FrameConsumerProxy::ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) { if (!frame_consumer_message_loop_->BelongsToCurrentThread()) { frame_consumer_message_loop_->PostTask(FROM_HERE, base::Bind( - &FrameConsumerProxy::AllocateFrame, this, format, size, frame_out, - done)); + &FrameConsumerProxy::ApplyBuffer, this, + view_size, clip_area, buffer, region)); return; } - if (frame_consumer_) { - frame_consumer_->AllocateFrame(format, size, frame_out, done); - } + if (frame_consumer_) + frame_consumer_->ApplyBuffer(view_size, clip_area, buffer, region); } -void FrameConsumerProxy::ReleaseFrame(media::VideoFrame* frame) { +void FrameConsumerProxy::ReturnBuffer(pp::ImageData* buffer) { if (!frame_consumer_message_loop_->BelongsToCurrentThread()) { frame_consumer_message_loop_->PostTask(FROM_HERE, base::Bind( - &FrameConsumerProxy::ReleaseFrame, this, make_scoped_refptr(frame))); + &FrameConsumerProxy::ReturnBuffer, this, buffer)); return; } if (frame_consumer_) - frame_consumer_->ReleaseFrame(frame); + frame_consumer_->ReturnBuffer(buffer); } -void FrameConsumerProxy::OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) { +void FrameConsumerProxy::SetSourceSize(const SkISize& source_size) { if (!frame_consumer_message_loop_->BelongsToCurrentThread()) { frame_consumer_message_loop_->PostTask(FROM_HERE, base::Bind( - &FrameConsumerProxy::OnPartialFrameOutput, this, - make_scoped_refptr(frame), region, done)); + &FrameConsumerProxy::SetSourceSize, this, source_size)); return; } if (frame_consumer_) - frame_consumer_->OnPartialFrameOutput(frame, region, done); + frame_consumer_->SetSourceSize(source_size); +} + +void FrameConsumerProxy::Attach(FrameConsumer* frame_consumer) { + DCHECK(frame_consumer_message_loop_->BelongsToCurrentThread()); + DCHECK(frame_consumer_ == NULL); + frame_consumer_ = frame_consumer; } void FrameConsumerProxy::Detach() { diff --git a/remoting/client/frame_consumer_proxy.h b/remoting/client/frame_consumer_proxy.h index d7dd4b2..039f042 100644 --- a/remoting/client/frame_consumer_proxy.h +++ b/remoting/client/frame_consumer_proxy.h @@ -25,19 +25,20 @@ class FrameConsumerProxy public: // Constructs a proxy for |frame_consumer| which will trampoline invocations // to |frame_consumer_message_loop|. - FrameConsumerProxy(FrameConsumer* frame_consumer, - base::MessageLoopProxy* frame_consumer_message_loop); + FrameConsumerProxy(base::MessageLoopProxy* frame_consumer_message_loop); virtual ~FrameConsumerProxy(); // FrameConsumer implementation. - virtual void AllocateFrame(media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) OVERRIDE; - virtual void ReleaseFrame(media::VideoFrame* frame) OVERRIDE; - virtual void OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) OVERRIDE; + virtual void ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) OVERRIDE; + virtual void ReturnBuffer(pp::ImageData* buffer) OVERRIDE; + virtual void SetSourceSize(const SkISize& source_size) OVERRIDE; + + // Attaches to |frame_consumer_|. + // This must only be called from |frame_consumer_message_loop_|. + void Attach(FrameConsumer* frame_consumer); // Detaches from |frame_consumer_|, ensuring no further calls reach it. // This must only be called from |frame_consumer_message_loop_|. diff --git a/remoting/client/frame_producer.h b/remoting/client/frame_producer.h new file mode 100644 index 0000000..644c270 --- /dev/null +++ b/remoting/client/frame_producer.h @@ -0,0 +1,51 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef REMOTING_CLIENT_FRAME_PRODUCER_H_ +#define REMOTING_CLIENT_FRAME_PRODUCER_H_ + +#include "base/callback_forward.h" +#include "third_party/skia/include/core/SkRect.h" +#include "third_party/skia/include/core/SkRegion.h" +#include "third_party/skia/include/core/SkSize.h" + +namespace pp { +class ImageData; +} // namespace pp + +namespace remoting { + +class FrameProducer { + public: + FrameProducer() {} + virtual ~FrameProducer() {} + + // Adds an image buffer to the pool of pending buffers for subsequent drawing. + // Once drawing is completed the buffer will be returned to the consumer via + // the FrameConsumer::ApplyBuffer() call. Alternatively an empty buffer could + // be returned via the FrameConsumer::ReturnBuffer() call. + // + // The passed buffer must be large enough to hold the whole clipping area. + virtual void DrawBuffer(pp::ImageData* buffer) = 0; + + // Requests repainting of the specified |region| of the frame as soon as + // possible. |region| is specified in output coordinates relative to + // the beginning of the frame. + virtual void InvalidateRegion(const SkRegion& region) = 0; + + // Requests returing of all pending buffers to the consumer via + // FrameConsumer::ReturnBuffer() calls. + virtual void RequestReturnBuffers(const base::Closure& done) = 0; + + // Notifies the producer of changes to the output view size or clipping area. + virtual void SetOutputSizeAndClip(const SkISize& view_size, + const SkIRect& clip_area) = 0; + + private: + DISALLOW_COPY_AND_ASSIGN(FrameProducer); +}; + +} // namespace remoting + +#endif // REMOTING_CLIENT_FRAME_PRODUCER_H_ diff --git a/remoting/client/plugin/chromoting_instance.cc b/remoting/client/plugin/chromoting_instance.cc index 1b0b52b..44796db 100644 --- a/remoting/client/plugin/chromoting_instance.cc +++ b/remoting/client/plugin/chromoting_instance.cc @@ -197,13 +197,11 @@ bool ChromotingInstance::Init(uint32_t argc, // Create the chromoting objects that don't depend on the network connection. // Because we decode on a separate thread we need a FrameConsumerProxy to // bounce calls from the RectangleUpdateDecoder back to the plugin thread. - view_.reset(new PepperView(this, &context_)); - consumer_proxy_ = new FrameConsumerProxy(view_.get(), plugin_message_loop_); + consumer_proxy_ = new FrameConsumerProxy(plugin_message_loop_); rectangle_decoder_ = new RectangleUpdateDecoder( context_.decode_message_loop(), consumer_proxy_.get()); - - // Default to a medium grey. - view_->SetSolidFill(0xFFCDCDCD); + view_.reset(new PepperView(this, &context_, rectangle_decoder_.get())); + consumer_proxy_->Attach(view_.get()); return true; } @@ -261,15 +259,14 @@ void ChromotingInstance::DidChangeView(const pp::Rect& position, DCHECK(plugin_message_loop_->BelongsToCurrentThread()); SkISize new_size = SkISize::Make(position.width(), position.height()); - if (view_->SetViewSize(new_size)) { - if (mouse_input_filter_.get()) { - mouse_input_filter_->set_input_size(new_size); - } - rectangle_decoder_->SetOutputSize(new_size); - } + SkIRect new_clip = + SkIRect::MakeXYWH(clip.x(), clip.y(), clip.width(), clip.height()); - rectangle_decoder_->UpdateClipRect( - SkIRect::MakeXYWH(clip.x(), clip.y(), clip.width(), clip.height())); + view_->SetView(new_size, new_clip); + + if (mouse_input_filter_.get()) { + mouse_input_filter_->set_input_size(view_->get_view_size()); + } } bool ChromotingInstance::HandleInputEvent(const pp::InputEvent& event) { @@ -282,7 +279,7 @@ bool ChromotingInstance::HandleInputEvent(const pp::InputEvent& event) { // this there. // If |input_handler_| is valid, then |mouse_input_filter_| must also be // since they are constructed together as part of the input pipeline - mouse_input_filter_->set_output_size(view_->get_host_size()); + mouse_input_filter_->set_output_size(view_->get_screen_size()); return input_handler_->HandleInputEvent(event); } diff --git a/remoting/client/plugin/pepper_view.cc b/remoting/client/plugin/pepper_view.cc index b466fa6..27ce1bc 100644 --- a/remoting/client/plugin/pepper_view.cc +++ b/remoting/client/plugin/pepper_view.cc @@ -4,8 +4,11 @@ #include "remoting/client/plugin/pepper_view.h" +#include <functional> + #include "base/message_loop.h" #include "base/string_util.h" +#include "base/synchronization/waitable_event.h" #include "ppapi/cpp/completion_callback.h" #include "ppapi/cpp/graphics_2d.h" #include "ppapi/cpp/image_data.h" @@ -15,13 +18,19 @@ #include "remoting/base/util.h" #include "remoting/client/chromoting_stats.h" #include "remoting/client/client_context.h" +#include "remoting/client/frame_producer.h" #include "remoting/client/plugin/chromoting_instance.h" #include "remoting/client/plugin/pepper_util.h" +using base::Passed; + namespace remoting { namespace { +// The maximum number of image buffers to be allocated at any point of time. +const size_t kMaxPendingBuffersCount = 2; + ChromotingInstance::ConnectionError ConvertConnectionError( protocol::ConnectionToHost::Error error) { switch (error) { @@ -42,16 +51,25 @@ ChromotingInstance::ConnectionError ConvertConnectionError( } // namespace -PepperView::PepperView(ChromotingInstance* instance, ClientContext* context) +PepperView::PepperView(ChromotingInstance* instance, + ClientContext* context, + FrameProducer* producer) : instance_(instance), context_(context), - flush_blocked_(false), - is_static_fill_(false), - static_fill_color_(0), + producer_(producer), + merge_buffer_(NULL), + merge_clip_area_(SkIRect::MakeEmpty()), + view_size_(SkISize::Make(0, 0)), + clip_area_(SkIRect::MakeEmpty()), + source_size_(SkISize::Make(0, 0)), + flush_pending_(false), + in_teardown_(false), ALLOW_THIS_IN_INITIALIZER_LIST(weak_factory_(this)) { } PepperView::~PepperView() { + DCHECK(merge_buffer_ == NULL); + DCHECK(buffers_.empty()); } bool PepperView::Initialize() { @@ -61,275 +79,286 @@ bool PepperView::Initialize() { void PepperView::TearDown() { DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + // The producer should now return any pending buffers. At this point, however, + // ReturnBuffer() tasks scheduled by the producer will not be delivered, + // so we free all the buffers once the producer's queue is empty. + in_teardown_ = true; + base::WaitableEvent done_event(true, false); + producer_->RequestReturnBuffers( + base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event))); + done_event.Wait(); + + merge_buffer_ = NULL; + while (!buffers_.empty()) { + FreeBuffer(buffers_.front()); + } + weak_factory_.InvalidateWeakPtrs(); } -void PepperView::Paint() { +void PepperView::SetConnectionState(protocol::ConnectionToHost::State state, + protocol::ConnectionToHost::Error error) { DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - if (is_static_fill_) { - VLOG(1) << "Static filling " << static_fill_color_; - pp::ImageData image(instance_, pp::ImageData::GetNativeImageDataFormat(), - pp::Size(graphics2d_.size().width(), - graphics2d_.size().height()), - false); - if (image.is_null()) { - LOG(ERROR) << "Unable to allocate image of size: " - << graphics2d_.size().width() << " x " - << graphics2d_.size().height(); - return; - } + switch (state) { + case protocol::ConnectionToHost::CONNECTING: + instance_->SetConnectionState( + ChromotingInstance::STATE_CONNECTING, + ConvertConnectionError(error)); + break; - for (int y = 0; y < image.size().height(); y++) { - for (int x = 0; x < image.size().width(); x++) { - *image.GetAddr32(pp::Point(x, y)) = static_fill_color_; - } - } + case protocol::ConnectionToHost::CONNECTED: + instance_->SetConnectionState( + ChromotingInstance::STATE_CONNECTED, + ConvertConnectionError(error)); + break; - // For ReplaceContents, make sure the image size matches the device context - // size! Otherwise, this will just silently do nothing. - graphics2d_.ReplaceContents(&image); - FlushGraphics(base::Time::Now()); - } else { - // TODO(ajwong): We need to keep a backing store image of the host screen - // that has the data here which can be redrawn. - return; + case protocol::ConnectionToHost::CLOSED: + instance_->SetConnectionState( + ChromotingInstance::STATE_CLOSED, + ConvertConnectionError(error)); + break; + + case protocol::ConnectionToHost::FAILED: + instance_->SetConnectionState( + ChromotingInstance::STATE_FAILED, + ConvertConnectionError(error)); + break; } } -void PepperView::SetHostSize(const SkISize& host_size) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); +void PepperView::SetView(const SkISize& view_size, + const SkIRect& clip_area) { + bool view_changed = false; - if (host_size_ == host_size) - return; + // TODO(alexeypa): Prevent upscaling because the YUV-to-RGB conversion code + // currently does not support upscaling. Once it does, this code be removed. + SkISize size = SkISize::Make( + std::min(view_size.width(), source_size_.width()), + std::min(view_size.height(), source_size_.height())); - host_size_ = host_size; + if (view_size_ != size) { + view_changed = true; + view_size_ = size; - // Submit an update of desktop size to Javascript. - instance_->SetDesktopSize(host_size.width(), host_size.height()); -} - -void PepperView::PaintFrame(media::VideoFrame* frame, const SkRegion& region) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + pp::Size pp_size = pp::Size(view_size_.width(), view_size_.height()); + graphics2d_ = pp::Graphics2D(instance_, pp_size, true); + bool result = instance_->BindGraphics(graphics2d_); - SetHostSize(SkISize::Make(frame->width(), frame->height())); - - if (!backing_store_.get() || backing_store_->is_null()) { - LOG(ERROR) << "Backing store is not available."; - return; + // There is no good way to handle this error currently. + DCHECK(result) << "Couldn't bind the device context."; } - base::Time start_time = base::Time::Now(); + if (clip_area_ != clip_area) { + view_changed = true; - // Copy updated regions to the backing store and then paint the regions. - bool changes_made = false; - for (SkRegion::Iterator i(region); !i.done(); i.next()) - changes_made |= PaintRect(frame, i.rect()); + // YUV to RGB conversion may require even X and Y coordinates for + // the top left corner of the clipping area. + clip_area_ = AlignRect(clip_area); + clip_area_.intersect(SkIRect::MakeSize(view_size_)); + } - if (changes_made) - FlushGraphics(start_time); + if (view_changed) { + producer_->SetOutputSizeAndClip(view_size_, clip_area_); + InitiateDrawing(); + } } -bool PepperView::PaintRect(media::VideoFrame* frame, const SkIRect& r) { - const uint8* frame_data = frame->data(media::VideoFrame::kRGBPlane); - const int kFrameStride = frame->stride(media::VideoFrame::kRGBPlane); - const int kBytesPerPixel = GetBytesPerPixel(media::VideoFrame::RGB32); +void PepperView::ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) { + DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - pp::Size backing_store_size = backing_store_->size(); - SkIRect rect(r); - if (!rect.intersect(SkIRect::MakeWH(backing_store_size.width(), - backing_store_size.height()))) { - return false; + // Currently we cannot use the data in the buffer is scale factor has changed + // already. + // TODO(alexeypa): We could rescale and draw it (or even draw it without + // rescaling) to reduce the perceived lag while we are waiting for + // the properly scaled data. + if (view_size_ != view_size) { + FreeBuffer(buffer); + InitiateDrawing(); + } else { + FlushBuffer(clip_area, buffer, region); } +} - const uint8* in = - frame_data + - kFrameStride * rect.fTop + // Y offset. - kBytesPerPixel * rect.fLeft; // X offset. - uint8* out = - reinterpret_cast<uint8*>(backing_store_->data()) + - backing_store_->stride() * rect.fTop + // Y offset. - kBytesPerPixel * rect.fLeft; // X offset. - - // TODO(hclam): We really should eliminate this memory copy. - for (int j = 0; j < rect.height(); ++j) { - memcpy(out, in, rect.width() * kBytesPerPixel); - in += kFrameStride; - out += backing_store_->stride(); - } +void PepperView::ReturnBuffer(pp::ImageData* buffer) { + DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - // Pepper Graphics 2D has a strange and badly documented API that the - // point here is the offset from the source rect. Why? - graphics2d_.PaintImageData( - *backing_store_.get(), - pp::Point(0, 0), - pp::Rect(rect.fLeft, rect.fTop, rect.width(), rect.height())); - return true; -} + // Free the buffer if there is nothing to paint. + if (in_teardown_) { + FreeBuffer(buffer); + return; + } -void PepperView::BlankRect(pp::ImageData& image_data, const pp::Rect& rect) { - const int kBytesPerPixel = GetBytesPerPixel(media::VideoFrame::RGB32); - for (int y = rect.y(); y < rect.bottom(); y++) { - uint8* to = reinterpret_cast<uint8*>(image_data.data()) + - (y * image_data.stride()) + (rect.x() * kBytesPerPixel); - memset(to, 0xff, rect.width() * kBytesPerPixel); + // Reuse the buffer if it is large enough, otherwise drop it on the floor + // and allocate a new one. + if (buffer->size().width() >= clip_area_.width() && + buffer->size().height() >= clip_area_.height()) { + producer_->DrawBuffer(buffer); + } else { + FreeBuffer(buffer); + InitiateDrawing(); } } -void PepperView::FlushGraphics(base::Time paint_start) { - scoped_ptr<base::Closure> task( - new base::Closure( - base::Bind(&PepperView::OnPaintDone, weak_factory_.GetWeakPtr(), - paint_start))); +void PepperView::SetSourceSize(const SkISize& source_size) { + DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - // Flag needs to be set here in order to get a proper error code for Flush(). - // Otherwise Flush() will always return PP_OK_COMPLETIONPENDING and the error - // would be hidden. - // - // Note that we can also handle this by providing an actual callback which - // takes the result code. Right now everything goes to the task that doesn't - // result value. - pp::CompletionCallback pp_callback(&CompletionCallbackClosureAdapter, - task.get(), - PP_COMPLETIONCALLBACK_FLAG_OPTIONAL); - int error = graphics2d_.Flush(pp_callback); + if (source_size_ == source_size) + return; - // There is already a flush in progress so set this flag to true so that we - // can flush again later. - // |paint_start| is then discarded but this is fine because we're not aiming - // for precise measurement of timing, otherwise we need to keep a list of - // queued start time(s). - if (error == PP_ERROR_INPROGRESS) - flush_blocked_ = true; - else - flush_blocked_ = false; + source_size_ = source_size; - // If Flush() returns asynchronously then release the task. - if (error == PP_OK_COMPLETIONPENDING) - ignore_result(task.release()); + // Notify JavaScript of the change in source size. + instance_->SetDesktopSize(source_size.width(), source_size.height()); } -void PepperView::SetSolidFill(uint32 color) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - - is_static_fill_ = true; - static_fill_color_ = color; +pp::ImageData* PepperView::AllocateBuffer() { + pp::ImageData* buffer = NULL; + if (buffers_.size() < kMaxPendingBuffersCount) { + pp::Size pp_size = pp::Size(clip_area_.width(), clip_area_.height()); + buffer = new pp::ImageData(instance_, + PP_IMAGEDATAFORMAT_BGRA_PREMUL, + pp_size, + false); + if (buffer->is_null()) { + LOG(WARNING) << "Not enough memory for frame buffers."; + delete buffer; + buffer = NULL; + } else { + buffers_.push_back(buffer); + } + } - Paint(); + return buffer; } -void PepperView::UnsetSolidFill() { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); +void PepperView::FreeBuffer(pp::ImageData* buffer) { + DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end()); - is_static_fill_ = false; + buffers_.remove(buffer); + delete buffer; } -void PepperView::SetConnectionState(protocol::ConnectionToHost::State state, - protocol::ConnectionToHost::Error error) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); +void PepperView::InitiateDrawing() { + // Do not schedule drawing if there is nothing to paint. + if (in_teardown_) + return; - switch (state) { - case protocol::ConnectionToHost::CONNECTING: - SetSolidFill(kCreatedColor); - instance_->SetConnectionState( - ChromotingInstance::STATE_CONNECTING, - ConvertConnectionError(error)); - break; + pp::ImageData* buffer = AllocateBuffer(); + while (buffer) { + producer_->DrawBuffer(buffer); + buffer = AllocateBuffer(); + } +} - case protocol::ConnectionToHost::CONNECTED: - UnsetSolidFill(); - instance_->SetConnectionState( - ChromotingInstance::STATE_CONNECTED, - ConvertConnectionError(error)); - break; +void PepperView::FlushBuffer(const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) { - case protocol::ConnectionToHost::CLOSED: - SetSolidFill(kDisconnectedColor); - instance_->SetConnectionState( - ChromotingInstance::STATE_CLOSED, - ConvertConnectionError(error)); - break; + // Defer drawing if the flush is already in progress. + if (flush_pending_) { + // |merge_buffer_| is guaranteed to be free here because we allocate only + // two buffers simultaneously. If more buffers are allowed this code should + // apply all pending changes to the screen. + DCHECK(merge_buffer_ == NULL); - case protocol::ConnectionToHost::FAILED: - SetSolidFill(kFailedColor); - instance_->SetConnectionState( - ChromotingInstance::STATE_FAILED, - ConvertConnectionError(error)); - break; + merge_clip_area_ = clip_area; + merge_buffer_ = buffer; + merge_region_ = region; + return; } -} -bool PepperView::SetViewSize(const SkISize& view_size) { - if (view_size_ == view_size) - return false; - view_size_ = view_size; + // Notify Pepper API about the updated areas and flush pixels to the screen. + base::Time start_time = base::Time::Now(); - pp::Size pp_size = pp::Size(view_size.width(), view_size.height()); + for (SkRegion::Iterator i(region); !i.done(); i.next()) { + SkIRect rect = i.rect(); - graphics2d_ = pp::Graphics2D(instance_, pp_size, true); - if (!instance_->BindGraphics(graphics2d_)) { - LOG(ERROR) << "Couldn't bind the device context."; - return false; - } + // Re-clip |region| with the current clipping area |clip_area_| because + // the latter could change from the time the buffer was drawn. + if (!rect.intersect(clip_area_)) + continue; - if (view_size.isEmpty()) - return false; - - // Allocate the backing store to save the desktop image. - if ((backing_store_.get() == NULL) || - (backing_store_->size() != pp_size)) { - VLOG(1) << "Allocate backing store: " - << view_size.width() << " x " << view_size.height(); - backing_store_.reset( - new pp::ImageData(instance_, pp::ImageData::GetNativeImageDataFormat(), - pp_size, false)); - DCHECK(backing_store_.get() && !backing_store_->is_null()) - << "Not enough memory for backing store."; + // Specify the rectangle coordinates relative to the clipping area. + rect.offset(-clip_area.left(), -clip_area.top()); + + // Pepper Graphics 2D has a strange and badly documented API that the + // point here is the offset from the source rect. Why? + graphics2d_.PaintImageData( + *buffer, + pp::Point(clip_area.left(), clip_area.top()), + pp::Rect(rect.left(), rect.top(), rect.width(), rect.height())); } - return true; -} -void PepperView::AllocateFrame(media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + // Notify the producer that some parts of the region weren't painted because + // the clipping area has changed already. + if (clip_area != clip_area_) { + SkRegion not_painted = region; + not_painted.op(clip_area_, SkRegion::kDifference_Op); + if (!not_painted.isEmpty()) { + producer_->InvalidateRegion(not_painted); + } + } - *frame_out = media::VideoFrame::CreateFrame( - media::VideoFrame::RGB32, size.width(), size.height(), - base::TimeDelta(), base::TimeDelta()); - (*frame_out)->AddRef(); - done.Run(); -} + // Flush the updated areas to the screen. + scoped_ptr<base::Closure> task( + new base::Closure( + base::Bind(&PepperView::OnFlushDone, weak_factory_.GetWeakPtr(), + start_time, buffer))); -void PepperView::ReleaseFrame(media::VideoFrame* frame) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + // Flag needs to be set here in order to get a proper error code for Flush(). + // Otherwise Flush() will always return PP_OK_COMPLETIONPENDING and the error + // would be hidden. + // + // Note that we can also handle this by providing an actual callback which + // takes the result code. Right now everything goes to the task that doesn't + // result value. + pp::CompletionCallback pp_callback(&CompletionCallbackClosureAdapter, + task.get(), + PP_COMPLETIONCALLBACK_FLAG_OPTIONAL); + int error = graphics2d_.Flush(pp_callback); - if (frame) - frame->Release(); -} + // If Flush() returns asynchronously then release the task. + flush_pending_ = (error == PP_OK_COMPLETIONPENDING); + if (flush_pending_) { + ignore_result(task.release()); + } else { + instance_->GetStats()->video_paint_ms()->Record( + (base::Time::Now() - start_time).InMilliseconds()); -void PepperView::OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + ReturnBuffer(buffer); - // TODO(ajwong): Clean up this API to be async so we don't need to use a - // member variable as a hack. - PaintFrame(frame, *region); - done.Run(); + // Resume painting for the buffer that was previoulsy postponed because of + // pending flush. + if (merge_buffer_ != NULL) { + buffer = merge_buffer_; + merge_buffer_ = NULL; + FlushBuffer(merge_clip_area_, buffer, merge_region_); + } + } } -void PepperView::OnPaintDone(base::Time paint_start) { +void PepperView::OnFlushDone(base::Time paint_start, + pp::ImageData* buffer) { DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + DCHECK(flush_pending_); + instance_->GetStats()->video_paint_ms()->Record( (base::Time::Now() - paint_start).InMilliseconds()); - // If the last flush failed because there was already another one in progress - // then we perform the flush now. - if (flush_blocked_) - FlushGraphics(base::Time::Now()); - return; + flush_pending_ = false; + ReturnBuffer(buffer); + + // Resume painting for the buffer that was previoulsy postponed because of + // pending flush. + if (merge_buffer_ != NULL) { + buffer = merge_buffer_; + merge_buffer_ = NULL; + FlushBuffer(merge_clip_area_, buffer, merge_region_); + } } } // namespace remoting diff --git a/remoting/client/plugin/pepper_view.h b/remoting/client/plugin/pepper_view.h index 426ced0..5bcef54 100644 --- a/remoting/client/plugin/pepper_view.h +++ b/remoting/client/plugin/pepper_view.h @@ -8,11 +8,10 @@ #ifndef REMOTING_CLIENT_PLUGIN_PEPPER_VIEW_H_ #define REMOTING_CLIENT_PLUGIN_PEPPER_VIEW_H_ -#include <vector> +#include <list> #include "base/memory/scoped_ptr.h" #include "base/memory/weak_ptr.h" -#include "media/base/video_frame.h" #include "ppapi/cpp/graphics_2d.h" #include "ppapi/cpp/point.h" #include "remoting/client/chromoting_view.h" @@ -22,64 +21,64 @@ namespace remoting { class ChromotingInstance; class ClientContext; +class FrameProducer; class PepperView : public ChromotingView, public FrameConsumer { public: - // Constructs a PepperView for the |instance|. The |instance| and - // |context| must outlive this class. - PepperView(ChromotingInstance* instance, ClientContext* context); + // Constructs a PepperView for the |instance|. The |instance|, |context| + // and |producer| must outlive this class. + PepperView(ChromotingInstance* instance, + ClientContext* context, + FrameProducer* producer); virtual ~PepperView(); // ChromotingView implementation. virtual bool Initialize() OVERRIDE; virtual void TearDown() OVERRIDE; - virtual void Paint() OVERRIDE; - virtual void SetSolidFill(uint32 color) OVERRIDE; - virtual void UnsetSolidFill() OVERRIDE; virtual void SetConnectionState( protocol::ConnectionToHost::State state, protocol::ConnectionToHost::Error error) OVERRIDE; // FrameConsumer implementation. - virtual void AllocateFrame(media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) OVERRIDE; - virtual void ReleaseFrame(media::VideoFrame* frame) OVERRIDE; - virtual void OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) OVERRIDE; - - // Sets the display size of this view. Returns true if plugin size has - // changed, false otherwise. - bool SetViewSize(const SkISize& plugin_size); + virtual void ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) OVERRIDE; + virtual void ReturnBuffer(pp::ImageData* buffer) OVERRIDE; + virtual void SetSourceSize(const SkISize& source_size) OVERRIDE; + + // Sets the display size and clipping area of this view. + void SetView(const SkISize& view_size, const SkIRect& clip_area); // Return the client view and original host dimensions. const SkISize& get_view_size() const { return view_size_; } - const SkISize& get_host_size() const { - return host_size_; + const SkISize& get_screen_size() const { + return source_size_; } private: - void OnPaintDone(base::Time paint_start); - - // Set the dimension of the entire host screen. - void SetHostSize(const SkISize& host_size); + // This routine allocates an image buffer. + pp::ImageData* AllocateBuffer(); - void PaintFrame(media::VideoFrame* frame, const SkRegion& region); + // This routine frees an image buffer allocated by AllocateBuffer(). + void FreeBuffer(pp::ImageData* buffer); - // Render the rectangle of |frame| to the backing store. - // Returns true if this rectangle is not clipped. - bool PaintRect(media::VideoFrame* frame, const SkIRect& rect); + // This routine makes sure that enough image buffers are in flight to keep + // the decoding pipeline busy. + void InitiateDrawing(); - // Blanks out a rectangle in an image. - void BlankRect(pp::ImageData& image_data, const pp::Rect& rect); + // This routine applies the given image buffer to the screen taking into + // account |clip_area| of the buffer and |region| describing valid parts + // of the buffer. + void FlushBuffer(const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region); - // Perform a flush on the graphics context. - void FlushGraphics(base::Time paint_start); + // This is a completion callback for FlushGraphics(). + void OnFlushDone(base::Time paint_start, pp::ImageData* buffer); // Reference to the creating plugin instance. Needed for interacting with // pepper. Marking explicitly as const since it must be initialized at @@ -91,21 +90,28 @@ class PepperView : public ChromotingView, pp::Graphics2D graphics2d_; - // A backing store that saves the current desktop image. - scoped_ptr<pp::ImageData> backing_store_; + FrameProducer* producer_; + + // List of allocated image buffers. + std::list<pp::ImageData*> buffers_; - // True if there is pending paint commands in Pepper's queue. This is set to - // true if the last flush returns a PP_ERROR_INPROGRESS error. - bool flush_blocked_; + pp::ImageData* merge_buffer_; + SkIRect merge_clip_area_; + SkRegion merge_region_; // The size of the plugin element. SkISize view_size_; + // The current clip area rectangle. + SkIRect clip_area_; + // The size of the host screen. - SkISize host_size_; + SkISize source_size_; + + // True if there is already a Flush() pending on the Graphics2D context. + bool flush_pending_; - bool is_static_fill_; - uint32 static_fill_color_; + bool in_teardown_; base::WeakPtrFactory<PepperView> weak_factory_; diff --git a/remoting/client/rectangle_update_decoder.cc b/remoting/client/rectangle_update_decoder.cc index 673c2a0..6f58564 100644 --- a/remoting/client/rectangle_update_decoder.cc +++ b/remoting/client/rectangle_update_decoder.cc @@ -10,6 +10,7 @@ #include "base/location.h" #include "base/logging.h" #include "base/message_loop_proxy.h" +#include "ppapi/cpp/image_data.h" #include "remoting/base/decoder.h" #include "remoting/base/decoder_row_based.h" #include "remoting/base/decoder_vp8.h" @@ -17,6 +18,7 @@ #include "remoting/client/frame_consumer.h" #include "remoting/protocol/session_config.h" +using base::Passed; using remoting::protocol::ChannelConfig; using remoting::protocol::SessionConfig; @@ -26,9 +28,9 @@ RectangleUpdateDecoder::RectangleUpdateDecoder( base::MessageLoopProxy* message_loop, FrameConsumer* consumer) : message_loop_(message_loop), consumer_(consumer), - screen_size_(SkISize::Make(0, 0)), - clip_rect_(SkIRect::MakeEmpty()), - decoder_needs_reset_(false) { + source_size_(SkISize::Make(0, 0)), + view_size_(SkISize::Make(0, 0)), + clip_area_(SkIRect::MakeEmpty()) { } RectangleUpdateDecoder::~RectangleUpdateDecoder() { @@ -56,67 +58,29 @@ void RectangleUpdateDecoder::DecodePacket(const VideoPacket* packet, this, packet, done)); return; } - AllocateFrame(packet, done); -} -void RectangleUpdateDecoder::AllocateFrame(const VideoPacket* packet, - const base::Closure& done) { - if (!message_loop_->BelongsToCurrentThread()) { - message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::AllocateFrame, - this, packet, done)); - return; - } base::ScopedClosureRunner done_runner(done); + bool decoder_needs_reset = false; // If the packet includes a screen size, store it. if (packet->format().has_screen_width() && packet->format().has_screen_height()) { - screen_size_.set(packet->format().screen_width(), - packet->format().screen_height()); - } - - // If we've never seen a screen size, ignore the packet. - if (screen_size_.isZero()) { - return; - } - - // Ensure the output frame is the right size. - SkISize frame_size = SkISize::Make(0, 0); - if (frame_) - frame_size.set(frame_->width(), frame_->height()); - - // Allocate a new frame, if necessary. - if ((!frame_) || (screen_size_ != frame_size)) { - if (frame_) { - consumer_->ReleaseFrame(frame_); - frame_ = NULL; + SkISize source_size = SkISize::Make(packet->format().screen_width(), + packet->format().screen_height()); + if (source_size_ != source_size) { + source_size_ = source_size; + decoder_needs_reset = true; } - - consumer_->AllocateFrame( - media::VideoFrame::RGB32, screen_size_, &frame_, - base::Bind(&RectangleUpdateDecoder::ProcessPacketData, - this, packet, done_runner.Release())); - decoder_needs_reset_ = true; - return; } - ProcessPacketData(packet, done_runner.Release()); -} -void RectangleUpdateDecoder::ProcessPacketData( - const VideoPacket* packet, const base::Closure& done) { - if (!message_loop_->BelongsToCurrentThread()) { - message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::ProcessPacketData, - this, packet, done)); + // If we've never seen a screen size, ignore the packet. + if (source_size_.isZero()) { return; } - base::ScopedClosureRunner done_runner(done); - if (decoder_needs_reset_) { - decoder_->Reset(); - decoder_->Initialize(frame_); - decoder_needs_reset_ = false; + if (decoder_needs_reset) { + decoder_->Initialize(source_size_); + consumer_->SetSourceSize(source_size_); } if (!decoder_->IsReadyForData()) { @@ -126,109 +90,109 @@ void RectangleUpdateDecoder::ProcessPacketData( } if (decoder_->DecodePacket(packet) == Decoder::DECODE_DONE) - SubmitToConsumer(); + DoPaint(); } -void RectangleUpdateDecoder::SetOutputSize(const SkISize& size) { - if (!message_loop_->BelongsToCurrentThread()) { - message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::SetOutputSize, - this, size)); +void RectangleUpdateDecoder::DoPaint() { + if (buffers_.empty()) return; - } - // TODO(wez): Refresh the frame only if the ratio has changed. - if (frame_) { - SkIRect frame_rect = SkIRect::MakeWH(frame_->width(), frame_->height()); - refresh_region_.op(frame_rect, SkRegion::kUnion_Op); - } + // Draw the invalidated region to the buffer. + pp::ImageData* buffer = buffers_.front(); + SkRegion output_region; + decoder_->RenderFrame(view_size_, clip_area_, + reinterpret_cast<uint8*>(buffer->data()), + buffer->stride(), + &output_region); - // TODO(hclam): If the scale ratio has changed we should reallocate a - // VideoFrame of different size. However if the scale ratio is always - // smaller than 1.0 we can use the same video frame. - if (decoder_.get()) { - decoder_->SetOutputSize(size); - RefreshFullFrame(); + // Notify the consumer that painting is done. + if (!output_region.isEmpty()) { + buffers_.pop_front(); + consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region); } } -void RectangleUpdateDecoder::UpdateClipRect(const SkIRect& new_clip_rect) { +void RectangleUpdateDecoder::RequestReturnBuffers(const base::Closure& done) { if (!message_loop_->BelongsToCurrentThread()) { message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::UpdateClipRect, - this, new_clip_rect)); + FROM_HERE, base::Bind(&RectangleUpdateDecoder::RequestReturnBuffers, + this, done)); return; } - if (new_clip_rect == clip_rect_ || !decoder_.get()) - return; - - // TODO(wez): Only refresh newly-exposed portions of the frame. - if (frame_) { - SkIRect frame_rect = SkIRect::MakeWH(frame_->width(), frame_->height()); - refresh_region_.op(frame_rect, SkRegion::kUnion_Op); + while (!buffers_.empty()) { + consumer_->ReturnBuffer(buffers_.front()); + buffers_.pop_front(); } - clip_rect_ = new_clip_rect; - decoder_->SetClipRect(new_clip_rect); - - // TODO(wez): Defer refresh so that multiple events can be batched. - DoRefresh(); + if (!done.is_null()) + done.Run(); } -void RectangleUpdateDecoder::RefreshFullFrame() { +void RectangleUpdateDecoder::DrawBuffer(pp::ImageData* buffer) { if (!message_loop_->BelongsToCurrentThread()) { message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::RefreshFullFrame, this)); + FROM_HERE, base::Bind(&RectangleUpdateDecoder::DrawBuffer, + this, buffer)); return; } - // If a video frame or the decoder is not allocated yet then don't - // save the refresh rectangle to avoid wasted computation. - if (!frame_ || !decoder_.get()) - return; - - SkIRect frame_rect = SkIRect::MakeWH(frame_->width(), frame_->height()); - refresh_region_.op(frame_rect, SkRegion::kUnion_Op); - - DoRefresh(); -} - -void RectangleUpdateDecoder::SubmitToConsumer() { - // A frame is not allocated yet, we can reach here because of a refresh - // request. - if (!frame_) - return; - - SkRegion* dirty_region = new SkRegion; - decoder_->GetUpdatedRegion(dirty_region); + DCHECK(clip_area_.width() <= buffer->size().width() && + clip_area_.height() <= buffer->size().height()); - consumer_->OnPartialFrameOutput(frame_, dirty_region, base::Bind( - &RectangleUpdateDecoder::OnFrameConsumed, this, dirty_region)); + buffers_.push_back(buffer); + DoPaint(); } -void RectangleUpdateDecoder::DoRefresh() { - DCHECK(message_loop_->BelongsToCurrentThread()); - - if (refresh_region_.isEmpty()) +void RectangleUpdateDecoder::InvalidateRegion(const SkRegion& region) { + if (!message_loop_->BelongsToCurrentThread()) { + message_loop_->PostTask( + FROM_HERE, base::Bind(&RectangleUpdateDecoder::InvalidateRegion, + this, region)); return; + } - decoder_->RefreshRegion(refresh_region_); - refresh_region_.setEmpty(); - SubmitToConsumer(); + if (decoder_.get()) { + decoder_->Invalidate(view_size_, region); + DoPaint(); + } } -void RectangleUpdateDecoder::OnFrameConsumed(SkRegion* region) { +void RectangleUpdateDecoder::SetOutputSizeAndClip(const SkISize& view_size, + const SkIRect& clip_area) { if (!message_loop_->BelongsToCurrentThread()) { message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::OnFrameConsumed, - this, region)); + FROM_HERE, base::Bind(&RectangleUpdateDecoder::SetOutputSizeAndClip, + this, view_size, clip_area)); return; } - delete region; - - DoRefresh(); + // The whole frame needs to be repainted if the scaling factor has changed. + if (view_size_ != view_size && decoder_.get()) { + SkRegion region; + region.op(SkIRect::MakeSize(view_size), SkRegion::kUnion_Op); + decoder_->Invalidate(view_size, region); + } + + if (view_size_ != view_size || + clip_area_ != clip_area) { + view_size_ = view_size; + clip_area_ = clip_area; + + // Return buffers that are smaller than needed to the consumer for + // reuse/reallocation. + std::list<pp::ImageData*>::iterator i = buffers_.begin(); + while (i != buffers_.end()) { + pp::Size buffer_size = (*i)->size(); + if (buffer_size.width() < clip_area_.width() || + buffer_size.height() < clip_area_.height()) { + consumer_->ReturnBuffer(*i); + i = buffers_.erase(i); + } else { + ++i; + } + } + } } } // namespace remoting diff --git a/remoting/client/rectangle_update_decoder.h b/remoting/client/rectangle_update_decoder.h index e2833c2..0ca7046 100644 --- a/remoting/client/rectangle_update_decoder.h +++ b/remoting/client/rectangle_update_decoder.h @@ -5,16 +5,22 @@ #ifndef REMOTING_CLIENT_RECTANGLE_UPDATE_DECODER_H_ #define REMOTING_CLIENT_RECTANGLE_UPDATE_DECODER_H_ +#include <list> + #include "base/callback_forward.h" #include "base/memory/ref_counted.h" #include "base/memory/scoped_ptr.h" -#include "media/base/video_frame.h" #include "remoting/base/decoder.h" +#include "remoting/client/frame_producer.h" namespace base { class MessageLoopProxy; } // namespace base +namespace pp { +class ImageData; +}; + namespace remoting { class FrameConsumer; @@ -28,7 +34,8 @@ class SessionConfig; // conditions on each step are reported. Should they be CHECKs? Logs? Other? // TODO(sergeyu): Rename this class. class RectangleUpdateDecoder : - public base::RefCountedThreadSafe<RectangleUpdateDecoder> { + public base::RefCountedThreadSafe<RectangleUpdateDecoder>, + public FrameProducer { public: RectangleUpdateDecoder(base::MessageLoopProxy* message_loop, FrameConsumer* consumer); @@ -36,55 +43,41 @@ class RectangleUpdateDecoder : // Initializes decoder with the infromation from the protocol config. void Initialize(const protocol::SessionConfig& config); - // Decodes the contents of |packet| calling OnPartialFrameOutput() in the - // regsitered as data is avaialable. DecodePacket may keep a reference to + // Decodes the contents of |packet|. DecodePacket may keep a reference to // |packet| so the |packet| must remain alive and valid until |done| is // executed. void DecodePacket(const VideoPacket* packet, const base::Closure& done); - // Set the output dimensions to scale video output to. - void SetOutputSize(const SkISize& size); - - // Set a new clipping rectangle for the decoder. Decoder should respect - // this clipping rectangle and only decode content in this rectangle and - // report dirty rectangles accordingly to enhance performance. - void UpdateClipRect(const SkIRect& clip_rect); - - // Force the decoder to output the last decoded video frame without any - // clipping. - void RefreshFullFrame(); + // FrameProducer implementation. + virtual void DrawBuffer(pp::ImageData* buffer) OVERRIDE; + virtual void InvalidateRegion(const SkRegion& region) OVERRIDE; + virtual void RequestReturnBuffers(const base::Closure& done) OVERRIDE; + virtual void SetOutputSizeAndClip(const SkISize& view_size, + const SkIRect& clip_area) OVERRIDE; private: friend class base::RefCountedThreadSafe<RectangleUpdateDecoder>; - friend class PartialFrameCleanup; - - ~RectangleUpdateDecoder(); - void AllocateFrame(const VideoPacket* packet, const base::Closure& done); - void ProcessPacketData(const VideoPacket* packet, const base::Closure& done); + virtual ~RectangleUpdateDecoder(); - // Obtain updated rectangles from decoder and submit it to the consumer. - void SubmitToConsumer(); - - // Use |refresh_rects_| to do a refresh to the backing video frame. - // When done the affected rectangles are submitted to the consumer. - void DoRefresh(); - - // Callback for FrameConsumer::OnPartialFrameOutput() - void OnFrameConsumed(SkRegion* region); + // Paint the invalidated region to the next available buffer and return it + // to the consumer. + void DoPaint(); scoped_refptr<base::MessageLoopProxy> message_loop_; FrameConsumer* consumer_; - SkISize screen_size_; - SkIRect clip_rect_; - SkRegion refresh_region_; - scoped_ptr<Decoder> decoder_; - bool decoder_needs_reset_; - // The video frame that the decoder writes to. - scoped_refptr<media::VideoFrame> frame_; + // Remote screen size in pixels. + SkISize source_size_; + + // The current dimentions of the frame consumer view. + SkISize view_size_; + SkIRect clip_area_; + + // The drawing buffers supplied by the frame consumer. + std::list<pp::ImageData*> buffers_; }; } // namespace remoting diff --git a/remoting/remoting.gyp b/remoting/remoting.gyp index 74c4c3f..2c5e546 100644 --- a/remoting/remoting.gyp +++ b/remoting/remoting.gyp @@ -664,6 +664,7 @@ 'client/frame_consumer.h', 'client/frame_consumer_proxy.cc', 'client/frame_consumer_proxy.h', + 'client/frame_producer.h', 'client/mouse_input_filter.cc', 'client/mouse_input_filter.h', 'client/rectangle_update_decoder.cc', |