diff options
author | alexeypa@chromium.org <alexeypa@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-02-24 23:05:56 +0000 |
---|---|---|
committer | alexeypa@chromium.org <alexeypa@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-02-24 23:05:56 +0000 |
commit | 55d3688e99e891b269cc93c1f695c65a00ea3b48 (patch) | |
tree | 31c34f0901866f698d241b7b45a71a2ba6d3cce7 /remoting/client | |
parent | 1fd0ff3f0814835c474e527b08c6a2f560246e94 (diff) | |
download | chromium_src-55d3688e99e891b269cc93c1f695c65a00ea3b48.zip chromium_src-55d3688e99e891b269cc93c1f695c65a00ea3b48.tar.gz chromium_src-55d3688e99e891b269cc93c1f695c65a00ea3b48.tar.bz2 |
This CL makes several the following improvements to the Chromoting decoder pipeline:
1. Only the clipping area, not the full frame, is drawn. This reduces the risk of out of memory situation on high page zoom levels. Screen updates are also incremental including scrolling scenarios.
2. Decoders now write pixels directly to the Pepper API backing store making it one memcpy less.
3. Scaling and panning is fully controlled by the plugin. The decoder only supplies the pixels it was asked for by the plugin.
BUG=109938
Review URL: http://codereview.chromium.org/9331003
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@123573 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'remoting/client')
-rw-r--r-- | remoting/client/chromoting_client.cc | 5 | ||||
-rw-r--r-- | remoting/client/chromoting_client.h | 5 | ||||
-rw-r--r-- | remoting/client/chromoting_view.h | 17 | ||||
-rw-r--r-- | remoting/client/frame_consumer.h | 60 | ||||
-rw-r--r-- | remoting/client/frame_consumer_proxy.cc | 43 | ||||
-rw-r--r-- | remoting/client/frame_consumer_proxy.h | 21 | ||||
-rw-r--r-- | remoting/client/frame_producer.h | 51 | ||||
-rw-r--r-- | remoting/client/plugin/chromoting_instance.cc | 25 | ||||
-rw-r--r-- | remoting/client/plugin/pepper_view.cc | 453 | ||||
-rw-r--r-- | remoting/client/plugin/pepper_view.h | 90 | ||||
-rw-r--r-- | remoting/client/rectangle_update_decoder.cc | 206 | ||||
-rw-r--r-- | remoting/client/rectangle_update_decoder.h | 65 |
12 files changed, 526 insertions, 515 deletions
diff --git a/remoting/client/chromoting_client.cc b/remoting/client/chromoting_client.cc index a5f71ae..77be87e 100644 --- a/remoting/client/chromoting_client.cc +++ b/remoting/client/chromoting_client.cc @@ -103,11 +103,6 @@ ChromotingStats* ChromotingClient::GetStats() { return &stats_; } -void ChromotingClient::Repaint() { - DCHECK(message_loop()->BelongsToCurrentThread()); - view_->Paint(); -} - void ChromotingClient::ProcessVideoPacket(const VideoPacket* packet, const base::Closure& done) { DCHECK(message_loop()->BelongsToCurrentThread()); diff --git a/remoting/client/chromoting_client.h b/remoting/client/chromoting_client.h index 756c388..50a432d 100644 --- a/remoting/client/chromoting_client.h +++ b/remoting/client/chromoting_client.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -49,9 +49,6 @@ class ChromotingClient : public protocol::ConnectionToHost::HostEventCallback, // Return the stats recorded by this client. ChromotingStats* GetStats(); - // Signals that the associated view may need updating. - virtual void Repaint(); - // ConnectionToHost::HostEventCallback implementation. virtual void OnConnectionState( protocol::ConnectionToHost::State state, diff --git a/remoting/client/chromoting_view.h b/remoting/client/chromoting_view.h index e93ab0d..f9f0411 100644 --- a/remoting/client/chromoting_view.h +++ b/remoting/client/chromoting_view.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -10,10 +10,6 @@ namespace remoting { -static const uint32 kCreatedColor = 0xffccccff; -static const uint32 kDisconnectedColor = 0xff00ccff; -static const uint32 kFailedColor = 0xffcc00ff; - // ChromotingView defines the behavior of an object that draws a view of the // remote desktop. Its main function is to render the update stream onto the // screen. @@ -27,17 +23,6 @@ class ChromotingView { // Free up resources allocated by this view. virtual void TearDown() = 0; - // Tells the ChromotingView to paint the current image on the screen. - virtual void Paint() = 0; - - // Fill the screen with one single static color, and ignore updates. - // Useful for debugging. - virtual void SetSolidFill(uint32 color) = 0; - - // Removes a previously set solid fill. If no fill was previous set, this - // does nothing. - virtual void UnsetSolidFill() = 0; - // Record the update the state of the connection, updating the UI as needed. virtual void SetConnectionState(protocol::ConnectionToHost::State state, protocol::ConnectionToHost::Error error) = 0; diff --git a/remoting/client/frame_consumer.h b/remoting/client/frame_consumer.h index de56d81..ce852b4 100644 --- a/remoting/client/frame_consumer.h +++ b/remoting/client/frame_consumer.h @@ -5,7 +5,13 @@ #ifndef REMOTING_CLIENT_FRAME_CONSUMER_H_ #define REMOTING_CLIENT_FRAME_CONSUMER_H_ -#include "remoting/base/decoder.h" // For UpdatedRects +#include "third_party/skia/include/core/SkRect.h" +#include "third_party/skia/include/core/SkRegion.h" +#include "third_party/skia/include/core/SkSize.h" + +namespace pp { +class ImageData; +} // namespace pp namespace remoting { @@ -14,40 +20,26 @@ class FrameConsumer { FrameConsumer() {} virtual ~FrameConsumer() {} - // Request a frame be allocated from the FrameConsumer. - // - // If a frame cannot be allocated to fit the format, and |size| - // requirements, |frame_out| will be set to NULL. - // - // An allocated frame will have at least the |size| requested, but - // may be bigger. Query the retrun frame for the actual frame size, - // stride, etc. - // - // The AllocateFrame call is asynchronous. From invocation, until when the - // |done| callback is invoked, |frame_out| should be considered to be locked - // by the FrameConsumer, must remain a valid pointer, and should not be - // examined or modified. After |done| is called, the |frame_out| will - // contain a result of the allocation. If a frame could not be allocated, - // |frame_out| will be NULL. - // - // All frames retrieved via the AllocateFrame call must be released by a - // corresponding call ReleaseFrame(scoped_refptr<VideoFrame>* frame_out. - virtual void AllocateFrame(media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) = 0; - - virtual void ReleaseFrame(media::VideoFrame* frame) = 0; - - // OnPartialFrameOutput() is called every time at least one rectangle of - // output is produced. The |frame| is guaranteed to have valid data for all - // of |region|. + // Accepts a buffer to be painted to the screen. The buffer's dimensions and + // relative position within the frame are specified by |clip_area|. Only + // pixels falling within |region| and the current clipping area are painted. + // The function assumes that the passed buffer was scaled to fit a window + // having |view_size| dimensions. // - // Both |frame| and |region| are guaranteed to be valid until the |done| - // callback is invoked. - virtual void OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) = 0; + // N.B. Both |clip_area| and |region| are in output coordinates relative to + // the frame. + virtual void ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) = 0; + + // Accepts a buffer that couldn't be used for drawing for any reason (shutdown + // is in progress, the view area has changed, etc.). The accepted buffer can + // be freed or reused for another drawing operation. + virtual void ReturnBuffer(pp::ImageData* buffer) = 0; + + // Set the dimension of the entire host screen. + virtual void SetSourceSize(const SkISize& source_size) = 0; private: DISALLOW_COPY_AND_ASSIGN(FrameConsumer); diff --git a/remoting/client/frame_consumer_proxy.cc b/remoting/client/frame_consumer_proxy.cc index 65b84ed..47bc9cd 100644 --- a/remoting/client/frame_consumer_proxy.cc +++ b/remoting/client/frame_consumer_proxy.cc @@ -6,59 +6,60 @@ #include "base/bind.h" #include "base/message_loop.h" +#include "ppapi/cpp/image_data.h" namespace remoting { FrameConsumerProxy::FrameConsumerProxy( - FrameConsumer* frame_consumer, base::MessageLoopProxy* frame_consumer_message_loop) - : frame_consumer_(frame_consumer), + : frame_consumer_(NULL), frame_consumer_message_loop_(frame_consumer_message_loop) { } FrameConsumerProxy::~FrameConsumerProxy() { } -void FrameConsumerProxy::AllocateFrame( - media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) { +void FrameConsumerProxy::ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) { if (!frame_consumer_message_loop_->BelongsToCurrentThread()) { frame_consumer_message_loop_->PostTask(FROM_HERE, base::Bind( - &FrameConsumerProxy::AllocateFrame, this, format, size, frame_out, - done)); + &FrameConsumerProxy::ApplyBuffer, this, + view_size, clip_area, buffer, region)); return; } - if (frame_consumer_) { - frame_consumer_->AllocateFrame(format, size, frame_out, done); - } + if (frame_consumer_) + frame_consumer_->ApplyBuffer(view_size, clip_area, buffer, region); } -void FrameConsumerProxy::ReleaseFrame(media::VideoFrame* frame) { +void FrameConsumerProxy::ReturnBuffer(pp::ImageData* buffer) { if (!frame_consumer_message_loop_->BelongsToCurrentThread()) { frame_consumer_message_loop_->PostTask(FROM_HERE, base::Bind( - &FrameConsumerProxy::ReleaseFrame, this, make_scoped_refptr(frame))); + &FrameConsumerProxy::ReturnBuffer, this, buffer)); return; } if (frame_consumer_) - frame_consumer_->ReleaseFrame(frame); + frame_consumer_->ReturnBuffer(buffer); } -void FrameConsumerProxy::OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) { +void FrameConsumerProxy::SetSourceSize(const SkISize& source_size) { if (!frame_consumer_message_loop_->BelongsToCurrentThread()) { frame_consumer_message_loop_->PostTask(FROM_HERE, base::Bind( - &FrameConsumerProxy::OnPartialFrameOutput, this, - make_scoped_refptr(frame), region, done)); + &FrameConsumerProxy::SetSourceSize, this, source_size)); return; } if (frame_consumer_) - frame_consumer_->OnPartialFrameOutput(frame, region, done); + frame_consumer_->SetSourceSize(source_size); +} + +void FrameConsumerProxy::Attach(FrameConsumer* frame_consumer) { + DCHECK(frame_consumer_message_loop_->BelongsToCurrentThread()); + DCHECK(frame_consumer_ == NULL); + frame_consumer_ = frame_consumer; } void FrameConsumerProxy::Detach() { diff --git a/remoting/client/frame_consumer_proxy.h b/remoting/client/frame_consumer_proxy.h index d7dd4b2..039f042 100644 --- a/remoting/client/frame_consumer_proxy.h +++ b/remoting/client/frame_consumer_proxy.h @@ -25,19 +25,20 @@ class FrameConsumerProxy public: // Constructs a proxy for |frame_consumer| which will trampoline invocations // to |frame_consumer_message_loop|. - FrameConsumerProxy(FrameConsumer* frame_consumer, - base::MessageLoopProxy* frame_consumer_message_loop); + FrameConsumerProxy(base::MessageLoopProxy* frame_consumer_message_loop); virtual ~FrameConsumerProxy(); // FrameConsumer implementation. - virtual void AllocateFrame(media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) OVERRIDE; - virtual void ReleaseFrame(media::VideoFrame* frame) OVERRIDE; - virtual void OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) OVERRIDE; + virtual void ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) OVERRIDE; + virtual void ReturnBuffer(pp::ImageData* buffer) OVERRIDE; + virtual void SetSourceSize(const SkISize& source_size) OVERRIDE; + + // Attaches to |frame_consumer_|. + // This must only be called from |frame_consumer_message_loop_|. + void Attach(FrameConsumer* frame_consumer); // Detaches from |frame_consumer_|, ensuring no further calls reach it. // This must only be called from |frame_consumer_message_loop_|. diff --git a/remoting/client/frame_producer.h b/remoting/client/frame_producer.h new file mode 100644 index 0000000..644c270 --- /dev/null +++ b/remoting/client/frame_producer.h @@ -0,0 +1,51 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef REMOTING_CLIENT_FRAME_PRODUCER_H_ +#define REMOTING_CLIENT_FRAME_PRODUCER_H_ + +#include "base/callback_forward.h" +#include "third_party/skia/include/core/SkRect.h" +#include "third_party/skia/include/core/SkRegion.h" +#include "third_party/skia/include/core/SkSize.h" + +namespace pp { +class ImageData; +} // namespace pp + +namespace remoting { + +class FrameProducer { + public: + FrameProducer() {} + virtual ~FrameProducer() {} + + // Adds an image buffer to the pool of pending buffers for subsequent drawing. + // Once drawing is completed the buffer will be returned to the consumer via + // the FrameConsumer::ApplyBuffer() call. Alternatively an empty buffer could + // be returned via the FrameConsumer::ReturnBuffer() call. + // + // The passed buffer must be large enough to hold the whole clipping area. + virtual void DrawBuffer(pp::ImageData* buffer) = 0; + + // Requests repainting of the specified |region| of the frame as soon as + // possible. |region| is specified in output coordinates relative to + // the beginning of the frame. + virtual void InvalidateRegion(const SkRegion& region) = 0; + + // Requests returing of all pending buffers to the consumer via + // FrameConsumer::ReturnBuffer() calls. + virtual void RequestReturnBuffers(const base::Closure& done) = 0; + + // Notifies the producer of changes to the output view size or clipping area. + virtual void SetOutputSizeAndClip(const SkISize& view_size, + const SkIRect& clip_area) = 0; + + private: + DISALLOW_COPY_AND_ASSIGN(FrameProducer); +}; + +} // namespace remoting + +#endif // REMOTING_CLIENT_FRAME_PRODUCER_H_ diff --git a/remoting/client/plugin/chromoting_instance.cc b/remoting/client/plugin/chromoting_instance.cc index 1b0b52b..44796db 100644 --- a/remoting/client/plugin/chromoting_instance.cc +++ b/remoting/client/plugin/chromoting_instance.cc @@ -197,13 +197,11 @@ bool ChromotingInstance::Init(uint32_t argc, // Create the chromoting objects that don't depend on the network connection. // Because we decode on a separate thread we need a FrameConsumerProxy to // bounce calls from the RectangleUpdateDecoder back to the plugin thread. - view_.reset(new PepperView(this, &context_)); - consumer_proxy_ = new FrameConsumerProxy(view_.get(), plugin_message_loop_); + consumer_proxy_ = new FrameConsumerProxy(plugin_message_loop_); rectangle_decoder_ = new RectangleUpdateDecoder( context_.decode_message_loop(), consumer_proxy_.get()); - - // Default to a medium grey. - view_->SetSolidFill(0xFFCDCDCD); + view_.reset(new PepperView(this, &context_, rectangle_decoder_.get())); + consumer_proxy_->Attach(view_.get()); return true; } @@ -261,15 +259,14 @@ void ChromotingInstance::DidChangeView(const pp::Rect& position, DCHECK(plugin_message_loop_->BelongsToCurrentThread()); SkISize new_size = SkISize::Make(position.width(), position.height()); - if (view_->SetViewSize(new_size)) { - if (mouse_input_filter_.get()) { - mouse_input_filter_->set_input_size(new_size); - } - rectangle_decoder_->SetOutputSize(new_size); - } + SkIRect new_clip = + SkIRect::MakeXYWH(clip.x(), clip.y(), clip.width(), clip.height()); - rectangle_decoder_->UpdateClipRect( - SkIRect::MakeXYWH(clip.x(), clip.y(), clip.width(), clip.height())); + view_->SetView(new_size, new_clip); + + if (mouse_input_filter_.get()) { + mouse_input_filter_->set_input_size(view_->get_view_size()); + } } bool ChromotingInstance::HandleInputEvent(const pp::InputEvent& event) { @@ -282,7 +279,7 @@ bool ChromotingInstance::HandleInputEvent(const pp::InputEvent& event) { // this there. // If |input_handler_| is valid, then |mouse_input_filter_| must also be // since they are constructed together as part of the input pipeline - mouse_input_filter_->set_output_size(view_->get_host_size()); + mouse_input_filter_->set_output_size(view_->get_screen_size()); return input_handler_->HandleInputEvent(event); } diff --git a/remoting/client/plugin/pepper_view.cc b/remoting/client/plugin/pepper_view.cc index b466fa6..27ce1bc 100644 --- a/remoting/client/plugin/pepper_view.cc +++ b/remoting/client/plugin/pepper_view.cc @@ -4,8 +4,11 @@ #include "remoting/client/plugin/pepper_view.h" +#include <functional> + #include "base/message_loop.h" #include "base/string_util.h" +#include "base/synchronization/waitable_event.h" #include "ppapi/cpp/completion_callback.h" #include "ppapi/cpp/graphics_2d.h" #include "ppapi/cpp/image_data.h" @@ -15,13 +18,19 @@ #include "remoting/base/util.h" #include "remoting/client/chromoting_stats.h" #include "remoting/client/client_context.h" +#include "remoting/client/frame_producer.h" #include "remoting/client/plugin/chromoting_instance.h" #include "remoting/client/plugin/pepper_util.h" +using base::Passed; + namespace remoting { namespace { +// The maximum number of image buffers to be allocated at any point of time. +const size_t kMaxPendingBuffersCount = 2; + ChromotingInstance::ConnectionError ConvertConnectionError( protocol::ConnectionToHost::Error error) { switch (error) { @@ -42,16 +51,25 @@ ChromotingInstance::ConnectionError ConvertConnectionError( } // namespace -PepperView::PepperView(ChromotingInstance* instance, ClientContext* context) +PepperView::PepperView(ChromotingInstance* instance, + ClientContext* context, + FrameProducer* producer) : instance_(instance), context_(context), - flush_blocked_(false), - is_static_fill_(false), - static_fill_color_(0), + producer_(producer), + merge_buffer_(NULL), + merge_clip_area_(SkIRect::MakeEmpty()), + view_size_(SkISize::Make(0, 0)), + clip_area_(SkIRect::MakeEmpty()), + source_size_(SkISize::Make(0, 0)), + flush_pending_(false), + in_teardown_(false), ALLOW_THIS_IN_INITIALIZER_LIST(weak_factory_(this)) { } PepperView::~PepperView() { + DCHECK(merge_buffer_ == NULL); + DCHECK(buffers_.empty()); } bool PepperView::Initialize() { @@ -61,275 +79,286 @@ bool PepperView::Initialize() { void PepperView::TearDown() { DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + // The producer should now return any pending buffers. At this point, however, + // ReturnBuffer() tasks scheduled by the producer will not be delivered, + // so we free all the buffers once the producer's queue is empty. + in_teardown_ = true; + base::WaitableEvent done_event(true, false); + producer_->RequestReturnBuffers( + base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event))); + done_event.Wait(); + + merge_buffer_ = NULL; + while (!buffers_.empty()) { + FreeBuffer(buffers_.front()); + } + weak_factory_.InvalidateWeakPtrs(); } -void PepperView::Paint() { +void PepperView::SetConnectionState(protocol::ConnectionToHost::State state, + protocol::ConnectionToHost::Error error) { DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - if (is_static_fill_) { - VLOG(1) << "Static filling " << static_fill_color_; - pp::ImageData image(instance_, pp::ImageData::GetNativeImageDataFormat(), - pp::Size(graphics2d_.size().width(), - graphics2d_.size().height()), - false); - if (image.is_null()) { - LOG(ERROR) << "Unable to allocate image of size: " - << graphics2d_.size().width() << " x " - << graphics2d_.size().height(); - return; - } + switch (state) { + case protocol::ConnectionToHost::CONNECTING: + instance_->SetConnectionState( + ChromotingInstance::STATE_CONNECTING, + ConvertConnectionError(error)); + break; - for (int y = 0; y < image.size().height(); y++) { - for (int x = 0; x < image.size().width(); x++) { - *image.GetAddr32(pp::Point(x, y)) = static_fill_color_; - } - } + case protocol::ConnectionToHost::CONNECTED: + instance_->SetConnectionState( + ChromotingInstance::STATE_CONNECTED, + ConvertConnectionError(error)); + break; - // For ReplaceContents, make sure the image size matches the device context - // size! Otherwise, this will just silently do nothing. - graphics2d_.ReplaceContents(&image); - FlushGraphics(base::Time::Now()); - } else { - // TODO(ajwong): We need to keep a backing store image of the host screen - // that has the data here which can be redrawn. - return; + case protocol::ConnectionToHost::CLOSED: + instance_->SetConnectionState( + ChromotingInstance::STATE_CLOSED, + ConvertConnectionError(error)); + break; + + case protocol::ConnectionToHost::FAILED: + instance_->SetConnectionState( + ChromotingInstance::STATE_FAILED, + ConvertConnectionError(error)); + break; } } -void PepperView::SetHostSize(const SkISize& host_size) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); +void PepperView::SetView(const SkISize& view_size, + const SkIRect& clip_area) { + bool view_changed = false; - if (host_size_ == host_size) - return; + // TODO(alexeypa): Prevent upscaling because the YUV-to-RGB conversion code + // currently does not support upscaling. Once it does, this code be removed. + SkISize size = SkISize::Make( + std::min(view_size.width(), source_size_.width()), + std::min(view_size.height(), source_size_.height())); - host_size_ = host_size; + if (view_size_ != size) { + view_changed = true; + view_size_ = size; - // Submit an update of desktop size to Javascript. - instance_->SetDesktopSize(host_size.width(), host_size.height()); -} - -void PepperView::PaintFrame(media::VideoFrame* frame, const SkRegion& region) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + pp::Size pp_size = pp::Size(view_size_.width(), view_size_.height()); + graphics2d_ = pp::Graphics2D(instance_, pp_size, true); + bool result = instance_->BindGraphics(graphics2d_); - SetHostSize(SkISize::Make(frame->width(), frame->height())); - - if (!backing_store_.get() || backing_store_->is_null()) { - LOG(ERROR) << "Backing store is not available."; - return; + // There is no good way to handle this error currently. + DCHECK(result) << "Couldn't bind the device context."; } - base::Time start_time = base::Time::Now(); + if (clip_area_ != clip_area) { + view_changed = true; - // Copy updated regions to the backing store and then paint the regions. - bool changes_made = false; - for (SkRegion::Iterator i(region); !i.done(); i.next()) - changes_made |= PaintRect(frame, i.rect()); + // YUV to RGB conversion may require even X and Y coordinates for + // the top left corner of the clipping area. + clip_area_ = AlignRect(clip_area); + clip_area_.intersect(SkIRect::MakeSize(view_size_)); + } - if (changes_made) - FlushGraphics(start_time); + if (view_changed) { + producer_->SetOutputSizeAndClip(view_size_, clip_area_); + InitiateDrawing(); + } } -bool PepperView::PaintRect(media::VideoFrame* frame, const SkIRect& r) { - const uint8* frame_data = frame->data(media::VideoFrame::kRGBPlane); - const int kFrameStride = frame->stride(media::VideoFrame::kRGBPlane); - const int kBytesPerPixel = GetBytesPerPixel(media::VideoFrame::RGB32); +void PepperView::ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) { + DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - pp::Size backing_store_size = backing_store_->size(); - SkIRect rect(r); - if (!rect.intersect(SkIRect::MakeWH(backing_store_size.width(), - backing_store_size.height()))) { - return false; + // Currently we cannot use the data in the buffer is scale factor has changed + // already. + // TODO(alexeypa): We could rescale and draw it (or even draw it without + // rescaling) to reduce the perceived lag while we are waiting for + // the properly scaled data. + if (view_size_ != view_size) { + FreeBuffer(buffer); + InitiateDrawing(); + } else { + FlushBuffer(clip_area, buffer, region); } +} - const uint8* in = - frame_data + - kFrameStride * rect.fTop + // Y offset. - kBytesPerPixel * rect.fLeft; // X offset. - uint8* out = - reinterpret_cast<uint8*>(backing_store_->data()) + - backing_store_->stride() * rect.fTop + // Y offset. - kBytesPerPixel * rect.fLeft; // X offset. - - // TODO(hclam): We really should eliminate this memory copy. - for (int j = 0; j < rect.height(); ++j) { - memcpy(out, in, rect.width() * kBytesPerPixel); - in += kFrameStride; - out += backing_store_->stride(); - } +void PepperView::ReturnBuffer(pp::ImageData* buffer) { + DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - // Pepper Graphics 2D has a strange and badly documented API that the - // point here is the offset from the source rect. Why? - graphics2d_.PaintImageData( - *backing_store_.get(), - pp::Point(0, 0), - pp::Rect(rect.fLeft, rect.fTop, rect.width(), rect.height())); - return true; -} + // Free the buffer if there is nothing to paint. + if (in_teardown_) { + FreeBuffer(buffer); + return; + } -void PepperView::BlankRect(pp::ImageData& image_data, const pp::Rect& rect) { - const int kBytesPerPixel = GetBytesPerPixel(media::VideoFrame::RGB32); - for (int y = rect.y(); y < rect.bottom(); y++) { - uint8* to = reinterpret_cast<uint8*>(image_data.data()) + - (y * image_data.stride()) + (rect.x() * kBytesPerPixel); - memset(to, 0xff, rect.width() * kBytesPerPixel); + // Reuse the buffer if it is large enough, otherwise drop it on the floor + // and allocate a new one. + if (buffer->size().width() >= clip_area_.width() && + buffer->size().height() >= clip_area_.height()) { + producer_->DrawBuffer(buffer); + } else { + FreeBuffer(buffer); + InitiateDrawing(); } } -void PepperView::FlushGraphics(base::Time paint_start) { - scoped_ptr<base::Closure> task( - new base::Closure( - base::Bind(&PepperView::OnPaintDone, weak_factory_.GetWeakPtr(), - paint_start))); +void PepperView::SetSourceSize(const SkISize& source_size) { + DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - // Flag needs to be set here in order to get a proper error code for Flush(). - // Otherwise Flush() will always return PP_OK_COMPLETIONPENDING and the error - // would be hidden. - // - // Note that we can also handle this by providing an actual callback which - // takes the result code. Right now everything goes to the task that doesn't - // result value. - pp::CompletionCallback pp_callback(&CompletionCallbackClosureAdapter, - task.get(), - PP_COMPLETIONCALLBACK_FLAG_OPTIONAL); - int error = graphics2d_.Flush(pp_callback); + if (source_size_ == source_size) + return; - // There is already a flush in progress so set this flag to true so that we - // can flush again later. - // |paint_start| is then discarded but this is fine because we're not aiming - // for precise measurement of timing, otherwise we need to keep a list of - // queued start time(s). - if (error == PP_ERROR_INPROGRESS) - flush_blocked_ = true; - else - flush_blocked_ = false; + source_size_ = source_size; - // If Flush() returns asynchronously then release the task. - if (error == PP_OK_COMPLETIONPENDING) - ignore_result(task.release()); + // Notify JavaScript of the change in source size. + instance_->SetDesktopSize(source_size.width(), source_size.height()); } -void PepperView::SetSolidFill(uint32 color) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); - - is_static_fill_ = true; - static_fill_color_ = color; +pp::ImageData* PepperView::AllocateBuffer() { + pp::ImageData* buffer = NULL; + if (buffers_.size() < kMaxPendingBuffersCount) { + pp::Size pp_size = pp::Size(clip_area_.width(), clip_area_.height()); + buffer = new pp::ImageData(instance_, + PP_IMAGEDATAFORMAT_BGRA_PREMUL, + pp_size, + false); + if (buffer->is_null()) { + LOG(WARNING) << "Not enough memory for frame buffers."; + delete buffer; + buffer = NULL; + } else { + buffers_.push_back(buffer); + } + } - Paint(); + return buffer; } -void PepperView::UnsetSolidFill() { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); +void PepperView::FreeBuffer(pp::ImageData* buffer) { + DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end()); - is_static_fill_ = false; + buffers_.remove(buffer); + delete buffer; } -void PepperView::SetConnectionState(protocol::ConnectionToHost::State state, - protocol::ConnectionToHost::Error error) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); +void PepperView::InitiateDrawing() { + // Do not schedule drawing if there is nothing to paint. + if (in_teardown_) + return; - switch (state) { - case protocol::ConnectionToHost::CONNECTING: - SetSolidFill(kCreatedColor); - instance_->SetConnectionState( - ChromotingInstance::STATE_CONNECTING, - ConvertConnectionError(error)); - break; + pp::ImageData* buffer = AllocateBuffer(); + while (buffer) { + producer_->DrawBuffer(buffer); + buffer = AllocateBuffer(); + } +} - case protocol::ConnectionToHost::CONNECTED: - UnsetSolidFill(); - instance_->SetConnectionState( - ChromotingInstance::STATE_CONNECTED, - ConvertConnectionError(error)); - break; +void PepperView::FlushBuffer(const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) { - case protocol::ConnectionToHost::CLOSED: - SetSolidFill(kDisconnectedColor); - instance_->SetConnectionState( - ChromotingInstance::STATE_CLOSED, - ConvertConnectionError(error)); - break; + // Defer drawing if the flush is already in progress. + if (flush_pending_) { + // |merge_buffer_| is guaranteed to be free here because we allocate only + // two buffers simultaneously. If more buffers are allowed this code should + // apply all pending changes to the screen. + DCHECK(merge_buffer_ == NULL); - case protocol::ConnectionToHost::FAILED: - SetSolidFill(kFailedColor); - instance_->SetConnectionState( - ChromotingInstance::STATE_FAILED, - ConvertConnectionError(error)); - break; + merge_clip_area_ = clip_area; + merge_buffer_ = buffer; + merge_region_ = region; + return; } -} -bool PepperView::SetViewSize(const SkISize& view_size) { - if (view_size_ == view_size) - return false; - view_size_ = view_size; + // Notify Pepper API about the updated areas and flush pixels to the screen. + base::Time start_time = base::Time::Now(); - pp::Size pp_size = pp::Size(view_size.width(), view_size.height()); + for (SkRegion::Iterator i(region); !i.done(); i.next()) { + SkIRect rect = i.rect(); - graphics2d_ = pp::Graphics2D(instance_, pp_size, true); - if (!instance_->BindGraphics(graphics2d_)) { - LOG(ERROR) << "Couldn't bind the device context."; - return false; - } + // Re-clip |region| with the current clipping area |clip_area_| because + // the latter could change from the time the buffer was drawn. + if (!rect.intersect(clip_area_)) + continue; - if (view_size.isEmpty()) - return false; - - // Allocate the backing store to save the desktop image. - if ((backing_store_.get() == NULL) || - (backing_store_->size() != pp_size)) { - VLOG(1) << "Allocate backing store: " - << view_size.width() << " x " << view_size.height(); - backing_store_.reset( - new pp::ImageData(instance_, pp::ImageData::GetNativeImageDataFormat(), - pp_size, false)); - DCHECK(backing_store_.get() && !backing_store_->is_null()) - << "Not enough memory for backing store."; + // Specify the rectangle coordinates relative to the clipping area. + rect.offset(-clip_area.left(), -clip_area.top()); + + // Pepper Graphics 2D has a strange and badly documented API that the + // point here is the offset from the source rect. Why? + graphics2d_.PaintImageData( + *buffer, + pp::Point(clip_area.left(), clip_area.top()), + pp::Rect(rect.left(), rect.top(), rect.width(), rect.height())); } - return true; -} -void PepperView::AllocateFrame(media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + // Notify the producer that some parts of the region weren't painted because + // the clipping area has changed already. + if (clip_area != clip_area_) { + SkRegion not_painted = region; + not_painted.op(clip_area_, SkRegion::kDifference_Op); + if (!not_painted.isEmpty()) { + producer_->InvalidateRegion(not_painted); + } + } - *frame_out = media::VideoFrame::CreateFrame( - media::VideoFrame::RGB32, size.width(), size.height(), - base::TimeDelta(), base::TimeDelta()); - (*frame_out)->AddRef(); - done.Run(); -} + // Flush the updated areas to the screen. + scoped_ptr<base::Closure> task( + new base::Closure( + base::Bind(&PepperView::OnFlushDone, weak_factory_.GetWeakPtr(), + start_time, buffer))); -void PepperView::ReleaseFrame(media::VideoFrame* frame) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + // Flag needs to be set here in order to get a proper error code for Flush(). + // Otherwise Flush() will always return PP_OK_COMPLETIONPENDING and the error + // would be hidden. + // + // Note that we can also handle this by providing an actual callback which + // takes the result code. Right now everything goes to the task that doesn't + // result value. + pp::CompletionCallback pp_callback(&CompletionCallbackClosureAdapter, + task.get(), + PP_COMPLETIONCALLBACK_FLAG_OPTIONAL); + int error = graphics2d_.Flush(pp_callback); - if (frame) - frame->Release(); -} + // If Flush() returns asynchronously then release the task. + flush_pending_ = (error == PP_OK_COMPLETIONPENDING); + if (flush_pending_) { + ignore_result(task.release()); + } else { + instance_->GetStats()->video_paint_ms()->Record( + (base::Time::Now() - start_time).InMilliseconds()); -void PepperView::OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) { - DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + ReturnBuffer(buffer); - // TODO(ajwong): Clean up this API to be async so we don't need to use a - // member variable as a hack. - PaintFrame(frame, *region); - done.Run(); + // Resume painting for the buffer that was previoulsy postponed because of + // pending flush. + if (merge_buffer_ != NULL) { + buffer = merge_buffer_; + merge_buffer_ = NULL; + FlushBuffer(merge_clip_area_, buffer, merge_region_); + } + } } -void PepperView::OnPaintDone(base::Time paint_start) { +void PepperView::OnFlushDone(base::Time paint_start, + pp::ImageData* buffer) { DCHECK(context_->main_message_loop()->BelongsToCurrentThread()); + DCHECK(flush_pending_); + instance_->GetStats()->video_paint_ms()->Record( (base::Time::Now() - paint_start).InMilliseconds()); - // If the last flush failed because there was already another one in progress - // then we perform the flush now. - if (flush_blocked_) - FlushGraphics(base::Time::Now()); - return; + flush_pending_ = false; + ReturnBuffer(buffer); + + // Resume painting for the buffer that was previoulsy postponed because of + // pending flush. + if (merge_buffer_ != NULL) { + buffer = merge_buffer_; + merge_buffer_ = NULL; + FlushBuffer(merge_clip_area_, buffer, merge_region_); + } } } // namespace remoting diff --git a/remoting/client/plugin/pepper_view.h b/remoting/client/plugin/pepper_view.h index 426ced0..5bcef54 100644 --- a/remoting/client/plugin/pepper_view.h +++ b/remoting/client/plugin/pepper_view.h @@ -8,11 +8,10 @@ #ifndef REMOTING_CLIENT_PLUGIN_PEPPER_VIEW_H_ #define REMOTING_CLIENT_PLUGIN_PEPPER_VIEW_H_ -#include <vector> +#include <list> #include "base/memory/scoped_ptr.h" #include "base/memory/weak_ptr.h" -#include "media/base/video_frame.h" #include "ppapi/cpp/graphics_2d.h" #include "ppapi/cpp/point.h" #include "remoting/client/chromoting_view.h" @@ -22,64 +21,64 @@ namespace remoting { class ChromotingInstance; class ClientContext; +class FrameProducer; class PepperView : public ChromotingView, public FrameConsumer { public: - // Constructs a PepperView for the |instance|. The |instance| and - // |context| must outlive this class. - PepperView(ChromotingInstance* instance, ClientContext* context); + // Constructs a PepperView for the |instance|. The |instance|, |context| + // and |producer| must outlive this class. + PepperView(ChromotingInstance* instance, + ClientContext* context, + FrameProducer* producer); virtual ~PepperView(); // ChromotingView implementation. virtual bool Initialize() OVERRIDE; virtual void TearDown() OVERRIDE; - virtual void Paint() OVERRIDE; - virtual void SetSolidFill(uint32 color) OVERRIDE; - virtual void UnsetSolidFill() OVERRIDE; virtual void SetConnectionState( protocol::ConnectionToHost::State state, protocol::ConnectionToHost::Error error) OVERRIDE; // FrameConsumer implementation. - virtual void AllocateFrame(media::VideoFrame::Format format, - const SkISize& size, - scoped_refptr<media::VideoFrame>* frame_out, - const base::Closure& done) OVERRIDE; - virtual void ReleaseFrame(media::VideoFrame* frame) OVERRIDE; - virtual void OnPartialFrameOutput(media::VideoFrame* frame, - SkRegion* region, - const base::Closure& done) OVERRIDE; - - // Sets the display size of this view. Returns true if plugin size has - // changed, false otherwise. - bool SetViewSize(const SkISize& plugin_size); + virtual void ApplyBuffer(const SkISize& view_size, + const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region) OVERRIDE; + virtual void ReturnBuffer(pp::ImageData* buffer) OVERRIDE; + virtual void SetSourceSize(const SkISize& source_size) OVERRIDE; + + // Sets the display size and clipping area of this view. + void SetView(const SkISize& view_size, const SkIRect& clip_area); // Return the client view and original host dimensions. const SkISize& get_view_size() const { return view_size_; } - const SkISize& get_host_size() const { - return host_size_; + const SkISize& get_screen_size() const { + return source_size_; } private: - void OnPaintDone(base::Time paint_start); - - // Set the dimension of the entire host screen. - void SetHostSize(const SkISize& host_size); + // This routine allocates an image buffer. + pp::ImageData* AllocateBuffer(); - void PaintFrame(media::VideoFrame* frame, const SkRegion& region); + // This routine frees an image buffer allocated by AllocateBuffer(). + void FreeBuffer(pp::ImageData* buffer); - // Render the rectangle of |frame| to the backing store. - // Returns true if this rectangle is not clipped. - bool PaintRect(media::VideoFrame* frame, const SkIRect& rect); + // This routine makes sure that enough image buffers are in flight to keep + // the decoding pipeline busy. + void InitiateDrawing(); - // Blanks out a rectangle in an image. - void BlankRect(pp::ImageData& image_data, const pp::Rect& rect); + // This routine applies the given image buffer to the screen taking into + // account |clip_area| of the buffer and |region| describing valid parts + // of the buffer. + void FlushBuffer(const SkIRect& clip_area, + pp::ImageData* buffer, + const SkRegion& region); - // Perform a flush on the graphics context. - void FlushGraphics(base::Time paint_start); + // This is a completion callback for FlushGraphics(). + void OnFlushDone(base::Time paint_start, pp::ImageData* buffer); // Reference to the creating plugin instance. Needed for interacting with // pepper. Marking explicitly as const since it must be initialized at @@ -91,21 +90,28 @@ class PepperView : public ChromotingView, pp::Graphics2D graphics2d_; - // A backing store that saves the current desktop image. - scoped_ptr<pp::ImageData> backing_store_; + FrameProducer* producer_; + + // List of allocated image buffers. + std::list<pp::ImageData*> buffers_; - // True if there is pending paint commands in Pepper's queue. This is set to - // true if the last flush returns a PP_ERROR_INPROGRESS error. - bool flush_blocked_; + pp::ImageData* merge_buffer_; + SkIRect merge_clip_area_; + SkRegion merge_region_; // The size of the plugin element. SkISize view_size_; + // The current clip area rectangle. + SkIRect clip_area_; + // The size of the host screen. - SkISize host_size_; + SkISize source_size_; + + // True if there is already a Flush() pending on the Graphics2D context. + bool flush_pending_; - bool is_static_fill_; - uint32 static_fill_color_; + bool in_teardown_; base::WeakPtrFactory<PepperView> weak_factory_; diff --git a/remoting/client/rectangle_update_decoder.cc b/remoting/client/rectangle_update_decoder.cc index 673c2a0..6f58564 100644 --- a/remoting/client/rectangle_update_decoder.cc +++ b/remoting/client/rectangle_update_decoder.cc @@ -10,6 +10,7 @@ #include "base/location.h" #include "base/logging.h" #include "base/message_loop_proxy.h" +#include "ppapi/cpp/image_data.h" #include "remoting/base/decoder.h" #include "remoting/base/decoder_row_based.h" #include "remoting/base/decoder_vp8.h" @@ -17,6 +18,7 @@ #include "remoting/client/frame_consumer.h" #include "remoting/protocol/session_config.h" +using base::Passed; using remoting::protocol::ChannelConfig; using remoting::protocol::SessionConfig; @@ -26,9 +28,9 @@ RectangleUpdateDecoder::RectangleUpdateDecoder( base::MessageLoopProxy* message_loop, FrameConsumer* consumer) : message_loop_(message_loop), consumer_(consumer), - screen_size_(SkISize::Make(0, 0)), - clip_rect_(SkIRect::MakeEmpty()), - decoder_needs_reset_(false) { + source_size_(SkISize::Make(0, 0)), + view_size_(SkISize::Make(0, 0)), + clip_area_(SkIRect::MakeEmpty()) { } RectangleUpdateDecoder::~RectangleUpdateDecoder() { @@ -56,67 +58,29 @@ void RectangleUpdateDecoder::DecodePacket(const VideoPacket* packet, this, packet, done)); return; } - AllocateFrame(packet, done); -} -void RectangleUpdateDecoder::AllocateFrame(const VideoPacket* packet, - const base::Closure& done) { - if (!message_loop_->BelongsToCurrentThread()) { - message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::AllocateFrame, - this, packet, done)); - return; - } base::ScopedClosureRunner done_runner(done); + bool decoder_needs_reset = false; // If the packet includes a screen size, store it. if (packet->format().has_screen_width() && packet->format().has_screen_height()) { - screen_size_.set(packet->format().screen_width(), - packet->format().screen_height()); - } - - // If we've never seen a screen size, ignore the packet. - if (screen_size_.isZero()) { - return; - } - - // Ensure the output frame is the right size. - SkISize frame_size = SkISize::Make(0, 0); - if (frame_) - frame_size.set(frame_->width(), frame_->height()); - - // Allocate a new frame, if necessary. - if ((!frame_) || (screen_size_ != frame_size)) { - if (frame_) { - consumer_->ReleaseFrame(frame_); - frame_ = NULL; + SkISize source_size = SkISize::Make(packet->format().screen_width(), + packet->format().screen_height()); + if (source_size_ != source_size) { + source_size_ = source_size; + decoder_needs_reset = true; } - - consumer_->AllocateFrame( - media::VideoFrame::RGB32, screen_size_, &frame_, - base::Bind(&RectangleUpdateDecoder::ProcessPacketData, - this, packet, done_runner.Release())); - decoder_needs_reset_ = true; - return; } - ProcessPacketData(packet, done_runner.Release()); -} -void RectangleUpdateDecoder::ProcessPacketData( - const VideoPacket* packet, const base::Closure& done) { - if (!message_loop_->BelongsToCurrentThread()) { - message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::ProcessPacketData, - this, packet, done)); + // If we've never seen a screen size, ignore the packet. + if (source_size_.isZero()) { return; } - base::ScopedClosureRunner done_runner(done); - if (decoder_needs_reset_) { - decoder_->Reset(); - decoder_->Initialize(frame_); - decoder_needs_reset_ = false; + if (decoder_needs_reset) { + decoder_->Initialize(source_size_); + consumer_->SetSourceSize(source_size_); } if (!decoder_->IsReadyForData()) { @@ -126,109 +90,109 @@ void RectangleUpdateDecoder::ProcessPacketData( } if (decoder_->DecodePacket(packet) == Decoder::DECODE_DONE) - SubmitToConsumer(); + DoPaint(); } -void RectangleUpdateDecoder::SetOutputSize(const SkISize& size) { - if (!message_loop_->BelongsToCurrentThread()) { - message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::SetOutputSize, - this, size)); +void RectangleUpdateDecoder::DoPaint() { + if (buffers_.empty()) return; - } - // TODO(wez): Refresh the frame only if the ratio has changed. - if (frame_) { - SkIRect frame_rect = SkIRect::MakeWH(frame_->width(), frame_->height()); - refresh_region_.op(frame_rect, SkRegion::kUnion_Op); - } + // Draw the invalidated region to the buffer. + pp::ImageData* buffer = buffers_.front(); + SkRegion output_region; + decoder_->RenderFrame(view_size_, clip_area_, + reinterpret_cast<uint8*>(buffer->data()), + buffer->stride(), + &output_region); - // TODO(hclam): If the scale ratio has changed we should reallocate a - // VideoFrame of different size. However if the scale ratio is always - // smaller than 1.0 we can use the same video frame. - if (decoder_.get()) { - decoder_->SetOutputSize(size); - RefreshFullFrame(); + // Notify the consumer that painting is done. + if (!output_region.isEmpty()) { + buffers_.pop_front(); + consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region); } } -void RectangleUpdateDecoder::UpdateClipRect(const SkIRect& new_clip_rect) { +void RectangleUpdateDecoder::RequestReturnBuffers(const base::Closure& done) { if (!message_loop_->BelongsToCurrentThread()) { message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::UpdateClipRect, - this, new_clip_rect)); + FROM_HERE, base::Bind(&RectangleUpdateDecoder::RequestReturnBuffers, + this, done)); return; } - if (new_clip_rect == clip_rect_ || !decoder_.get()) - return; - - // TODO(wez): Only refresh newly-exposed portions of the frame. - if (frame_) { - SkIRect frame_rect = SkIRect::MakeWH(frame_->width(), frame_->height()); - refresh_region_.op(frame_rect, SkRegion::kUnion_Op); + while (!buffers_.empty()) { + consumer_->ReturnBuffer(buffers_.front()); + buffers_.pop_front(); } - clip_rect_ = new_clip_rect; - decoder_->SetClipRect(new_clip_rect); - - // TODO(wez): Defer refresh so that multiple events can be batched. - DoRefresh(); + if (!done.is_null()) + done.Run(); } -void RectangleUpdateDecoder::RefreshFullFrame() { +void RectangleUpdateDecoder::DrawBuffer(pp::ImageData* buffer) { if (!message_loop_->BelongsToCurrentThread()) { message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::RefreshFullFrame, this)); + FROM_HERE, base::Bind(&RectangleUpdateDecoder::DrawBuffer, + this, buffer)); return; } - // If a video frame or the decoder is not allocated yet then don't - // save the refresh rectangle to avoid wasted computation. - if (!frame_ || !decoder_.get()) - return; - - SkIRect frame_rect = SkIRect::MakeWH(frame_->width(), frame_->height()); - refresh_region_.op(frame_rect, SkRegion::kUnion_Op); - - DoRefresh(); -} - -void RectangleUpdateDecoder::SubmitToConsumer() { - // A frame is not allocated yet, we can reach here because of a refresh - // request. - if (!frame_) - return; - - SkRegion* dirty_region = new SkRegion; - decoder_->GetUpdatedRegion(dirty_region); + DCHECK(clip_area_.width() <= buffer->size().width() && + clip_area_.height() <= buffer->size().height()); - consumer_->OnPartialFrameOutput(frame_, dirty_region, base::Bind( - &RectangleUpdateDecoder::OnFrameConsumed, this, dirty_region)); + buffers_.push_back(buffer); + DoPaint(); } -void RectangleUpdateDecoder::DoRefresh() { - DCHECK(message_loop_->BelongsToCurrentThread()); - - if (refresh_region_.isEmpty()) +void RectangleUpdateDecoder::InvalidateRegion(const SkRegion& region) { + if (!message_loop_->BelongsToCurrentThread()) { + message_loop_->PostTask( + FROM_HERE, base::Bind(&RectangleUpdateDecoder::InvalidateRegion, + this, region)); return; + } - decoder_->RefreshRegion(refresh_region_); - refresh_region_.setEmpty(); - SubmitToConsumer(); + if (decoder_.get()) { + decoder_->Invalidate(view_size_, region); + DoPaint(); + } } -void RectangleUpdateDecoder::OnFrameConsumed(SkRegion* region) { +void RectangleUpdateDecoder::SetOutputSizeAndClip(const SkISize& view_size, + const SkIRect& clip_area) { if (!message_loop_->BelongsToCurrentThread()) { message_loop_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::OnFrameConsumed, - this, region)); + FROM_HERE, base::Bind(&RectangleUpdateDecoder::SetOutputSizeAndClip, + this, view_size, clip_area)); return; } - delete region; - - DoRefresh(); + // The whole frame needs to be repainted if the scaling factor has changed. + if (view_size_ != view_size && decoder_.get()) { + SkRegion region; + region.op(SkIRect::MakeSize(view_size), SkRegion::kUnion_Op); + decoder_->Invalidate(view_size, region); + } + + if (view_size_ != view_size || + clip_area_ != clip_area) { + view_size_ = view_size; + clip_area_ = clip_area; + + // Return buffers that are smaller than needed to the consumer for + // reuse/reallocation. + std::list<pp::ImageData*>::iterator i = buffers_.begin(); + while (i != buffers_.end()) { + pp::Size buffer_size = (*i)->size(); + if (buffer_size.width() < clip_area_.width() || + buffer_size.height() < clip_area_.height()) { + consumer_->ReturnBuffer(*i); + i = buffers_.erase(i); + } else { + ++i; + } + } + } } } // namespace remoting diff --git a/remoting/client/rectangle_update_decoder.h b/remoting/client/rectangle_update_decoder.h index e2833c2..0ca7046 100644 --- a/remoting/client/rectangle_update_decoder.h +++ b/remoting/client/rectangle_update_decoder.h @@ -5,16 +5,22 @@ #ifndef REMOTING_CLIENT_RECTANGLE_UPDATE_DECODER_H_ #define REMOTING_CLIENT_RECTANGLE_UPDATE_DECODER_H_ +#include <list> + #include "base/callback_forward.h" #include "base/memory/ref_counted.h" #include "base/memory/scoped_ptr.h" -#include "media/base/video_frame.h" #include "remoting/base/decoder.h" +#include "remoting/client/frame_producer.h" namespace base { class MessageLoopProxy; } // namespace base +namespace pp { +class ImageData; +}; + namespace remoting { class FrameConsumer; @@ -28,7 +34,8 @@ class SessionConfig; // conditions on each step are reported. Should they be CHECKs? Logs? Other? // TODO(sergeyu): Rename this class. class RectangleUpdateDecoder : - public base::RefCountedThreadSafe<RectangleUpdateDecoder> { + public base::RefCountedThreadSafe<RectangleUpdateDecoder>, + public FrameProducer { public: RectangleUpdateDecoder(base::MessageLoopProxy* message_loop, FrameConsumer* consumer); @@ -36,55 +43,41 @@ class RectangleUpdateDecoder : // Initializes decoder with the infromation from the protocol config. void Initialize(const protocol::SessionConfig& config); - // Decodes the contents of |packet| calling OnPartialFrameOutput() in the - // regsitered as data is avaialable. DecodePacket may keep a reference to + // Decodes the contents of |packet|. DecodePacket may keep a reference to // |packet| so the |packet| must remain alive and valid until |done| is // executed. void DecodePacket(const VideoPacket* packet, const base::Closure& done); - // Set the output dimensions to scale video output to. - void SetOutputSize(const SkISize& size); - - // Set a new clipping rectangle for the decoder. Decoder should respect - // this clipping rectangle and only decode content in this rectangle and - // report dirty rectangles accordingly to enhance performance. - void UpdateClipRect(const SkIRect& clip_rect); - - // Force the decoder to output the last decoded video frame without any - // clipping. - void RefreshFullFrame(); + // FrameProducer implementation. + virtual void DrawBuffer(pp::ImageData* buffer) OVERRIDE; + virtual void InvalidateRegion(const SkRegion& region) OVERRIDE; + virtual void RequestReturnBuffers(const base::Closure& done) OVERRIDE; + virtual void SetOutputSizeAndClip(const SkISize& view_size, + const SkIRect& clip_area) OVERRIDE; private: friend class base::RefCountedThreadSafe<RectangleUpdateDecoder>; - friend class PartialFrameCleanup; - - ~RectangleUpdateDecoder(); - void AllocateFrame(const VideoPacket* packet, const base::Closure& done); - void ProcessPacketData(const VideoPacket* packet, const base::Closure& done); + virtual ~RectangleUpdateDecoder(); - // Obtain updated rectangles from decoder and submit it to the consumer. - void SubmitToConsumer(); - - // Use |refresh_rects_| to do a refresh to the backing video frame. - // When done the affected rectangles are submitted to the consumer. - void DoRefresh(); - - // Callback for FrameConsumer::OnPartialFrameOutput() - void OnFrameConsumed(SkRegion* region); + // Paint the invalidated region to the next available buffer and return it + // to the consumer. + void DoPaint(); scoped_refptr<base::MessageLoopProxy> message_loop_; FrameConsumer* consumer_; - SkISize screen_size_; - SkIRect clip_rect_; - SkRegion refresh_region_; - scoped_ptr<Decoder> decoder_; - bool decoder_needs_reset_; - // The video frame that the decoder writes to. - scoped_refptr<media::VideoFrame> frame_; + // Remote screen size in pixels. + SkISize source_size_; + + // The current dimentions of the frame consumer view. + SkISize view_size_; + SkIRect clip_area_; + + // The drawing buffers supplied by the frame consumer. + std::list<pp::ImageData*> buffers_; }; } // namespace remoting |