diff options
author | sergeyu@chromium.org <sergeyu@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-01-23 09:32:45 +0000 |
---|---|---|
committer | sergeyu@chromium.org <sergeyu@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-01-23 09:32:45 +0000 |
commit | 46860da6ddd4e6855923a93990317964dba0ecbf (patch) | |
tree | 39e5156f3b0dec10828bcd08ff64c6fb0bb7cf8d | |
parent | 97a5da4f358fed573663fe9b7e19039ef662eeec (diff) | |
download | chromium_src-46860da6ddd4e6855923a93990317964dba0ecbf.zip chromium_src-46860da6ddd4e6855923a93990317964dba0ecbf.tar.gz chromium_src-46860da6ddd4e6855923a93990317964dba0ecbf.tar.bz2 |
Add VideoProcessor interface.
The new VideoProcessor interface abstracts video layer in chromoting client.
RectangleUpdateDecoder is the default implementation. Another implementation
will be added to suppord decoding using <video> and MediaSource API (or
Pepper video decode API if we decide to use it instead in the future).
BUG=321825
Review URL: https://codereview.chromium.org/136763009
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@246547 0039d316-1c4b-4281-b951-d872f2087c98
19 files changed, 327 insertions, 252 deletions
diff --git a/remoting/client/chromoting_client.cc b/remoting/client/chromoting_client.cc index e1c5b14..40eddcd 100644 --- a/remoting/client/chromoting_client.cc +++ b/remoting/client/chromoting_client.cc @@ -10,7 +10,7 @@ #include "remoting/client/audio_player.h" #include "remoting/client/client_context.h" #include "remoting/client/client_user_interface.h" -#include "remoting/client/rectangle_update_decoder.h" +#include "remoting/client/video_renderer.h" #include "remoting/proto/audio.pb.h" #include "remoting/proto/video.pb.h" #include "remoting/protocol/authentication_method.h" @@ -29,18 +29,15 @@ ChromotingClient::ChromotingClient( ClientContext* client_context, protocol::ConnectionToHost* connection, ClientUserInterface* user_interface, - scoped_refptr<FrameConsumerProxy> frame_consumer, + VideoRenderer* video_renderer, scoped_ptr<AudioPlayer> audio_player) : config_(config), task_runner_(client_context->main_task_runner()), connection_(connection), user_interface_(user_interface), + video_renderer_(video_renderer), host_capabilities_received_(false), weak_factory_(this) { - rectangle_decoder_ = - new RectangleUpdateDecoder(client_context->main_task_runner(), - client_context->decode_task_runner(), - frame_consumer); if (audio_player) { audio_decode_scheduler_.reset(new AudioDecodeScheduler( client_context->main_task_runner(), @@ -77,19 +74,10 @@ void ChromotingClient::Start( this, this, this, - rectangle_decoder_.get(), + video_renderer_, audio_decode_scheduler_.get()); } -FrameProducer* ChromotingClient::GetFrameProducer() { - return rectangle_decoder_.get(); -} - -ChromotingStats* ChromotingClient::GetStats() { - DCHECK(task_runner_->BelongsToCurrentThread()); - return rectangle_decoder_->GetStats(); -} - void ChromotingClient::SetCapabilities( const protocol::Capabilities& capabilities) { DCHECK(task_runner_->BelongsToCurrentThread()); @@ -170,7 +158,7 @@ void ChromotingClient::OnAuthenticated() { DCHECK(task_runner_->BelongsToCurrentThread()); // Initialize the decoder. - rectangle_decoder_->Initialize(connection_->config()); + video_renderer_->Initialize(connection_->config()); if (connection_->config().is_audio_enabled()) audio_decode_scheduler_->Initialize(connection_->config()); diff --git a/remoting/client/chromoting_client.h b/remoting/client/chromoting_client.h index dc41499..65f9910 100644 --- a/remoting/client/chromoting_client.h +++ b/remoting/client/chromoting_client.h @@ -36,7 +36,7 @@ class ClientContext; class ClientUserInterface; class FrameConsumerProxy; class FrameProducer; -class RectangleUpdateDecoder; +class VideoRenderer; class SignalStrategy; class ChromotingClient : public protocol::ConnectionToHost::HostEventCallback, @@ -47,7 +47,7 @@ class ChromotingClient : public protocol::ConnectionToHost::HostEventCallback, ClientContext* client_context, protocol::ConnectionToHost* connection, ClientUserInterface* user_interface, - scoped_refptr<FrameConsumerProxy> frame_consumer, + VideoRenderer* video_renderer, scoped_ptr<AudioPlayer> audio_player); virtual ~ChromotingClient(); @@ -57,11 +57,6 @@ class ChromotingClient : public protocol::ConnectionToHost::HostEventCallback, void Start(SignalStrategy* signal_strategy, scoped_ptr<protocol::TransportFactory> transport_factory); - FrameProducer* GetFrameProducer(); - - // Return the stats recorded by this client. - ChromotingStats* GetStats(); - // ClientStub implementation. virtual void SetCapabilities( const protocol::Capabilities& capabilities) OVERRIDE; @@ -98,7 +93,7 @@ class ChromotingClient : public protocol::ConnectionToHost::HostEventCallback, scoped_refptr<base::SingleThreadTaskRunner> task_runner_; protocol::ConnectionToHost* connection_; ClientUserInterface* user_interface_; - scoped_refptr<RectangleUpdateDecoder> rectangle_decoder_; + VideoRenderer* video_renderer_; scoped_ptr<AudioDecodeScheduler> audio_decode_scheduler_; diff --git a/remoting/client/frame_consumer.h b/remoting/client/frame_consumer.h index 165cedd..0996332 100644 --- a/remoting/client/frame_consumer.h +++ b/remoting/client/frame_consumer.h @@ -37,7 +37,8 @@ class FrameConsumer { virtual void ApplyBuffer(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area, webrtc::DesktopFrame* buffer, - const webrtc::DesktopRegion& region) = 0; + const webrtc::DesktopRegion& region, + const webrtc::DesktopRegion& shape) = 0; // Accepts a buffer that couldn't be used for drawing for any reason (shutdown // is in progress, the view area has changed, etc.). The accepted buffer can diff --git a/remoting/client/frame_consumer_proxy.cc b/remoting/client/frame_consumer_proxy.cc index a1b3895..bf70da5 100644 --- a/remoting/client/frame_consumer_proxy.cc +++ b/remoting/client/frame_consumer_proxy.cc @@ -24,16 +24,17 @@ FrameConsumerProxy::FrameConsumerProxy( void FrameConsumerProxy::ApplyBuffer(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area, webrtc::DesktopFrame* buffer, - const webrtc::DesktopRegion& region) { + const webrtc::DesktopRegion& region, + const webrtc::DesktopRegion& shape) { if (!task_runner_->BelongsToCurrentThread()) { task_runner_->PostTask(FROM_HERE, base::Bind( &FrameConsumerProxy::ApplyBuffer, this, - view_size, clip_area, buffer, region)); + view_size, clip_area, buffer, region, shape)); return; } if (frame_consumer_.get()) - frame_consumer_->ApplyBuffer(view_size, clip_area, buffer, region); + frame_consumer_->ApplyBuffer(view_size, clip_area, buffer, region, shape); } void FrameConsumerProxy::ReturnBuffer(webrtc::DesktopFrame* buffer) { diff --git a/remoting/client/frame_consumer_proxy.h b/remoting/client/frame_consumer_proxy.h index f32a313..b42bab5 100644 --- a/remoting/client/frame_consumer_proxy.h +++ b/remoting/client/frame_consumer_proxy.h @@ -33,7 +33,8 @@ class FrameConsumerProxy virtual void ApplyBuffer(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area, webrtc::DesktopFrame* buffer, - const webrtc::DesktopRegion& region) OVERRIDE; + const webrtc::DesktopRegion& region, + const webrtc::DesktopRegion& shape) OVERRIDE; virtual void ReturnBuffer(webrtc::DesktopFrame* buffer) OVERRIDE; virtual void SetSourceSize(const webrtc::DesktopSize& source_size, const webrtc::DesktopVector& dpi) OVERRIDE; diff --git a/remoting/client/frame_producer.h b/remoting/client/frame_producer.h index aa7e63d..2776eb7 100644 --- a/remoting/client/frame_producer.h +++ b/remoting/client/frame_producer.h @@ -42,9 +42,6 @@ class FrameProducer { virtual void SetOutputSizeAndClip(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area) = 0; - // Returns a reference to the shape of the most recently drawn buffer. - virtual const webrtc::DesktopRegion* GetBufferShape() = 0; - protected: virtual ~FrameProducer() {} diff --git a/remoting/client/jni/chromoting_jni_instance.cc b/remoting/client/jni/chromoting_jni_instance.cc index 3c7866b..96f0d35 100644 --- a/remoting/client/jni/chromoting_jni_instance.cc +++ b/remoting/client/jni/chromoting_jni_instance.cc @@ -12,6 +12,7 @@ #include "remoting/client/audio_player.h" #include "remoting/client/jni/android_keymap.h" #include "remoting/client/jni/chromoting_jni_runtime.h" +#include "remoting/client/software_video_renderer.h" #include "remoting/jingle_glue/chromium_port_allocator.h" #include "remoting/jingle_glue/chromium_socket_factory.h" #include "remoting/jingle_glue/network_settings.h" @@ -188,7 +189,7 @@ void ChromotingJniInstance::RecordPaintTime(int64 paint_time_ms) { } if (stats_logging_enabled_) - client_->GetStats()->video_paint_ms()->Record(paint_time_ms); + video_renderer_->GetStats()->video_paint_ms()->Record(paint_time_ms); } void ChromotingJniInstance::OnConnectionState( @@ -298,11 +299,17 @@ void ChromotingJniInstance::ConnectToHostOnNetworkThread() { connection_.reset(new protocol::ConnectionToHost(true)); + SoftwareVideoRenderer* renderer = + new SoftwareVideoRenderer(client_context_->main_task_runner(), + client_context_->decode_task_runner(), + frame_consumer_); + view_->set_frame_producer(renderer); + video_renderer_.reset(renderer); + client_.reset(new ChromotingClient( client_config_, client_context_.get(), connection_.get(), - this, frame_consumer_, scoped_ptr<AudioPlayer>())); + this, video_renderer_.get(), scoped_ptr<AudioPlayer>())); - view_->set_frame_producer(client_->GetFrameProducer()); signaling_.reset(new XmppSignalStrategy( net::ClientSocketFactory::GetDefaultFactory(), @@ -373,7 +380,7 @@ void ChromotingJniInstance::LogPerfStats() { if (!stats_logging_enabled_) return; - ChromotingStats* stats = client_->GetStats(); + ChromotingStats* stats = video_renderer_->GetStats(); __android_log_print(ANDROID_LOG_INFO, "stats", "Bandwidth:%.0f FrameRate:%.1f Capture:%.1f Encode:%.1f " "Decode:%.1f Render:%.1f Latency:%.0f", diff --git a/remoting/client/jni/chromoting_jni_instance.h b/remoting/client/jni/chromoting_jni_instance.h index cfaaed4f..fdb5a4f 100644 --- a/remoting/client/jni/chromoting_jni_instance.h +++ b/remoting/client/jni/chromoting_jni_instance.h @@ -29,6 +29,8 @@ class ClipboardEvent; class CursorShapeInfo; } // namespace protocol +class VideoRenderer; + // ClientUserInterface that indirectly makes and receives JNI calls. class ChromotingJniInstance : public ClientUserInterface, @@ -136,6 +138,7 @@ class ChromotingJniInstance // This group of variables is to be used on the network thread. ClientConfig client_config_; scoped_ptr<ClientContext> client_context_; + scoped_ptr<VideoRenderer> video_renderer_; scoped_ptr<protocol::ConnectionToHost> connection_; scoped_ptr<ChromotingClient> client_; XmppSignalStrategy::XmppServerConfig xmpp_config_; diff --git a/remoting/client/jni/jni_frame_consumer.cc b/remoting/client/jni/jni_frame_consumer.cc index 3362daf..aeb70d0 100644 --- a/remoting/client/jni/jni_frame_consumer.cc +++ b/remoting/client/jni/jni_frame_consumer.cc @@ -45,7 +45,8 @@ void JniFrameConsumer::set_frame_producer(FrameProducer* producer) { void JniFrameConsumer::ApplyBuffer(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area, webrtc::DesktopFrame* buffer, - const webrtc::DesktopRegion& region) { + const webrtc::DesktopRegion& region, + const webrtc::DesktopRegion& shape) { DCHECK(jni_runtime_->display_task_runner()->BelongsToCurrentThread()); if (bitmap_->size().width() != buffer->size().width() || diff --git a/remoting/client/jni/jni_frame_consumer.h b/remoting/client/jni/jni_frame_consumer.h index ceeac09..92e9089 100644 --- a/remoting/client/jni/jni_frame_consumer.h +++ b/remoting/client/jni/jni_frame_consumer.h @@ -43,7 +43,8 @@ class JniFrameConsumer : public FrameConsumer { virtual void ApplyBuffer(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area, webrtc::DesktopFrame* buffer, - const webrtc::DesktopRegion& region) OVERRIDE; + const webrtc::DesktopRegion& region, + const webrtc::DesktopRegion& shape) OVERRIDE; virtual void ReturnBuffer(webrtc::DesktopFrame* buffer) OVERRIDE; virtual void SetSourceSize(const webrtc::DesktopSize& source_size, const webrtc::DesktopVector& dpi) OVERRIDE; diff --git a/remoting/client/plugin/chromoting_instance.cc b/remoting/client/plugin/chromoting_instance.cc index 49c5210..b94c207 100644 --- a/remoting/client/plugin/chromoting_instance.cc +++ b/remoting/client/plugin/chromoting_instance.cc @@ -39,7 +39,7 @@ #include "remoting/client/plugin/pepper_port_allocator.h" #include "remoting/client/plugin/pepper_token_fetcher.h" #include "remoting/client/plugin/pepper_view.h" -#include "remoting/client/rectangle_update_decoder.h" +#include "remoting/client/software_video_renderer.h" #include "remoting/protocol/connection_to_host.h" #include "remoting/protocol/host_stub.h" #include "remoting/protocol/libjingle_transport_factory.h" @@ -634,23 +634,27 @@ void ChromotingInstance::ConnectWithConfig(const ClientConfig& config, view_weak_factory_.reset( new base::WeakPtrFactory<FrameConsumer>(view_.get())); - // RectangleUpdateDecoder runs on a separate thread so for now we wrap + // SoftwareVideoRenderer runs on a separate thread so for now we wrap // PepperView with a ref-counted proxy object. scoped_refptr<FrameConsumerProxy> consumer_proxy = new FrameConsumerProxy(plugin_task_runner_, view_weak_factory_->GetWeakPtr()); + SoftwareVideoRenderer* decoder = + new SoftwareVideoRenderer(context_.main_task_runner(), + context_.decode_task_runner(), + consumer_proxy); + view_->Initialize(decoder); + video_renderer_.reset(decoder); + host_connection_.reset(new protocol::ConnectionToHost(true)); scoped_ptr<AudioPlayer> audio_player(new PepperAudioPlayer(this)); - client_.reset(new ChromotingClient(config, &context_, - host_connection_.get(), this, - consumer_proxy, audio_player.Pass())); - - view_->Initialize(client_->GetFrameProducer()); + client_.reset(new ChromotingClient(config, &context_, host_connection_.get(), + this, video_renderer_.get(), + audio_player.Pass())); - if (!plugin_view_.is_null()) { + if (!plugin_view_.is_null()) view_->SetView(plugin_view_); - } // Connect the input pipeline to the protocol stub & initialize components. mouse_input_filter_.set_input_stub(host_connection_->input_stub()); @@ -906,9 +910,9 @@ void ChromotingInstance::HandleAllowMouseLockMessage() { } ChromotingStats* ChromotingInstance::GetStats() { - if (!client_.get()) + if (!video_renderer_.get()) return NULL; - return client_->GetStats(); + return video_renderer_->GetStats(); } void ChromotingInstance::PostChromotingMessage( @@ -937,7 +941,7 @@ void ChromotingInstance::SendOutgoingIq(const std::string& iq) { } void ChromotingInstance::SendPerfStats() { - if (!client_.get()) { + if (!video_renderer_.get()) { return; } @@ -947,7 +951,7 @@ void ChromotingInstance::SendPerfStats() { base::TimeDelta::FromMilliseconds(kPerfStatsIntervalMs)); scoped_ptr<base::DictionaryValue> data(new base::DictionaryValue()); - ChromotingStats* stats = client_->GetStats(); + ChromotingStats* stats = video_renderer_->GetStats(); data->SetDouble("videoBandwidth", stats->video_bandwidth()->Rate()); data->SetDouble("videoFrameRate", stats->video_frame_rate()->Rate()); data->SetDouble("captureLatency", stats->video_capture_ms()->Average()); diff --git a/remoting/client/plugin/chromoting_instance.h b/remoting/client/plugin/chromoting_instance.h index 0b4a1dd2..3b0ad76 100644 --- a/remoting/client/plugin/chromoting_instance.h +++ b/remoting/client/plugin/chromoting_instance.h @@ -62,6 +62,7 @@ class PepperTokenFetcher; class PepperView; class RectangleUpdateDecoder; class SignalStrategy; +class VideoRenderer; struct ClientConfig; @@ -239,7 +240,7 @@ class ChromotingInstance : PepperPluginThreadDelegate plugin_thread_delegate_; scoped_refptr<PluginThreadTaskRunner> plugin_task_runner_; ClientContext context_; - scoped_refptr<RectangleUpdateDecoder> rectangle_decoder_; + scoped_ptr<VideoRenderer> video_renderer_; scoped_ptr<PepperView> view_; scoped_ptr<base::WeakPtrFactory<FrameConsumer> > view_weak_factory_; pp::View plugin_view_; diff --git a/remoting/client/plugin/pepper_view.cc b/remoting/client/plugin/pepper_view.cc index 16a66a0..d561bfb 100644 --- a/remoting/client/plugin/pepper_view.cc +++ b/remoting/client/plugin/pepper_view.cc @@ -164,7 +164,8 @@ void PepperView::SetView(const pp::View& view) { void PepperView::ApplyBuffer(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area, webrtc::DesktopFrame* buffer, - const webrtc::DesktopRegion& region) { + const webrtc::DesktopRegion& region, + const webrtc::DesktopRegion& shape) { DCHECK(context_->main_task_runner()->BelongsToCurrentThread()); if (!frame_received_) { @@ -181,6 +182,7 @@ void PepperView::ApplyBuffer(const webrtc::DesktopSize& view_size, Initialize(producer_); } else { FlushBuffer(clip_area, buffer, region); + instance_->SetDesktopShape(shape); } } @@ -303,11 +305,6 @@ void PepperView::FlushBuffer(const webrtc::DesktopRect& clip_area, int error = graphics2d_.Flush(callback); CHECK(error == PP_OK_COMPLETIONPENDING); flush_pending_ = true; - - // If the buffer we just rendered has a shape then pass that to JavaScript. - const webrtc::DesktopRegion* buffer_shape = producer_->GetBufferShape(); - if (buffer_shape) - instance_->SetDesktopShape(*buffer_shape); } void PepperView::OnFlushDone(int result, diff --git a/remoting/client/plugin/pepper_view.h b/remoting/client/plugin/pepper_view.h index 2b4e69b..4f35277 100644 --- a/remoting/client/plugin/pepper_view.h +++ b/remoting/client/plugin/pepper_view.h @@ -48,7 +48,8 @@ class PepperView : public FrameConsumer { virtual void ApplyBuffer(const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area, webrtc::DesktopFrame* buffer, - const webrtc::DesktopRegion& region) OVERRIDE; + const webrtc::DesktopRegion& region, + const webrtc::DesktopRegion& shape) OVERRIDE; virtual void ReturnBuffer(webrtc::DesktopFrame* buffer) OVERRIDE; virtual void SetSourceSize(const webrtc::DesktopSize& source_size, const webrtc::DesktopVector& dpi) OVERRIDE; diff --git a/remoting/client/rectangle_update_decoder.h b/remoting/client/rectangle_update_decoder.h deleted file mode 100644 index 3c7468a..0000000 --- a/remoting/client/rectangle_update_decoder.h +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef REMOTING_CLIENT_RECTANGLE_UPDATE_DECODER_H_ -#define REMOTING_CLIENT_RECTANGLE_UPDATE_DECODER_H_ - -#include <list> - -#include "base/callback_forward.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "remoting/codec/video_decoder.h" -#include "remoting/client/chromoting_stats.h" -#include "remoting/client/frame_consumer_proxy.h" -#include "remoting/client/frame_producer.h" -#include "remoting/protocol/video_stub.h" -#include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h" - -namespace base { -class SingleThreadTaskRunner; -} // namespace base - -namespace remoting { - -class ChromotingStats; - -namespace protocol { -class SessionConfig; -} // namespace protocol - -// TODO(ajwong): Re-examine this API, especially with regards to how error -// conditions on each step are reported. Should they be CHECKs? Logs? Other? -// TODO(sergeyu): Rename this class. -class RectangleUpdateDecoder - : public base::RefCountedThreadSafe<RectangleUpdateDecoder>, - public FrameProducer, - public protocol::VideoStub { - public: - // Creates an update decoder on |main_task_runner_| and |decode_task_runner_|, - // outputting to |consumer|. The |main_task_runner_| is responsible for - // receiving and queueing packets. The |decode_task_runner_| is responsible - // for decoding the video packets. - // TODO(wez): Replace the ref-counted proxy with an owned FrameConsumer. - RectangleUpdateDecoder( - scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, - scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, - scoped_refptr<FrameConsumerProxy> consumer); - - // Initializes decoder with the information from the protocol config. - void Initialize(const protocol::SessionConfig& config); - - // FrameProducer implementation. These methods may be called before we are - // Initialize()d, or we know the source screen size. - virtual void DrawBuffer(webrtc::DesktopFrame* buffer) OVERRIDE; - virtual void InvalidateRegion(const webrtc::DesktopRegion& region) OVERRIDE; - virtual void RequestReturnBuffers(const base::Closure& done) OVERRIDE; - virtual void SetOutputSizeAndClip( - const webrtc::DesktopSize& view_size, - const webrtc::DesktopRect& clip_area) OVERRIDE; - virtual const webrtc::DesktopRegion* GetBufferShape() OVERRIDE; - - // VideoStub implementation. - virtual void ProcessVideoPacket(scoped_ptr<VideoPacket> packet, - const base::Closure& done) OVERRIDE; - - // Return the stats recorded by this client. - ChromotingStats* GetStats(); - - private: - friend class base::RefCountedThreadSafe<RectangleUpdateDecoder>; - virtual ~RectangleUpdateDecoder(); - - // Paints the invalidated region to the next available buffer and returns it - // to the consumer. - void SchedulePaint(); - void DoPaint(); - - // Decodes the contents of |packet|. DecodePacket may keep a reference to - // |packet| so the |packet| must remain alive and valid until |done| is - // executed. - void DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done); - - // Callback method when a VideoPacket is processed. |decode_start| contains - // the timestamp when the packet will start to be processed. - void OnPacketDone(base::Time decode_start, const base::Closure& done); - - scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; - scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_; - scoped_refptr<FrameConsumerProxy> consumer_; - scoped_ptr<VideoDecoder> decoder_; - - // Remote screen size in pixels. - webrtc::DesktopSize source_size_; - - // Vertical and horizontal DPI of the remote screen. - webrtc::DesktopVector source_dpi_; - - // The current dimensions of the frame consumer view. - webrtc::DesktopSize view_size_; - webrtc::DesktopRect clip_area_; - - // The drawing buffers supplied by the frame consumer. - std::list<webrtc::DesktopFrame*> buffers_; - - // Flag used to coalesce runs of SchedulePaint()s into a single DoPaint(). - bool paint_scheduled_; - - ChromotingStats stats_; - - // Keep track of the most recent sequence number bounced back from the host. - int64 latest_sequence_number_; -}; - -} // namespace remoting - -#endif // REMOTING_CLIENT_RECTANGLE_UPDATE_DECODER_H_ diff --git a/remoting/client/rectangle_update_decoder.cc b/remoting/client/software_video_renderer.cc index 3482527..4d95e13 100644 --- a/remoting/client/rectangle_update_decoder.cc +++ b/remoting/client/software_video_renderer.cc @@ -1,8 +1,10 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "remoting/client/rectangle_update_decoder.h" +#include "remoting/client/software_video_renderer.h" + +#include <list> #include "base/bind.h" #include "base/callback.h" @@ -74,7 +76,57 @@ class RgbToBgrVideoDecoderFilter : public VideoDecoder { scoped_ptr<VideoDecoder> parent_; }; -RectangleUpdateDecoder::RectangleUpdateDecoder( +class SoftwareVideoRenderer::Core { + public: + Core(scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, + scoped_refptr<FrameConsumerProxy> consumer); + ~Core(); + + void Initialize(const protocol::SessionConfig& config); + void DrawBuffer(webrtc::DesktopFrame* buffer); + void InvalidateRegion(const webrtc::DesktopRegion& region); + void RequestReturnBuffers(const base::Closure& done); + void SetOutputSizeAndClip( + const webrtc::DesktopSize& view_size, + const webrtc::DesktopRect& clip_area); + + // Decodes the contents of |packet|. DecodePacket may keep a reference to + // |packet| so the |packet| must remain alive and valid until |done| is + // executed. + void DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done); + + private: + // Paints the invalidated region to the next available buffer and returns it + // to the consumer. + void SchedulePaint(); + void DoPaint(); + + scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_; + scoped_refptr<FrameConsumerProxy> consumer_; + scoped_ptr<VideoDecoder> decoder_; + + // Remote screen size in pixels. + webrtc::DesktopSize source_size_; + + // Vertical and horizontal DPI of the remote screen. + webrtc::DesktopVector source_dpi_; + + // The current dimensions of the frame consumer view. + webrtc::DesktopSize view_size_; + webrtc::DesktopRect clip_area_; + + // The drawing buffers supplied by the frame consumer. + std::list<webrtc::DesktopFrame*> buffers_; + + // Flag used to coalesce runs of SchedulePaint()s into a single DoPaint(). + bool paint_scheduled_; + + base::WeakPtrFactory<Core> weak_factory_; +}; + +SoftwareVideoRenderer::Core::Core( scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, scoped_refptr<FrameConsumerProxy> consumer) @@ -82,19 +134,14 @@ RectangleUpdateDecoder::RectangleUpdateDecoder( decode_task_runner_(decode_task_runner), consumer_(consumer), paint_scheduled_(false), - latest_sequence_number_(0) { + weak_factory_(this) { } -RectangleUpdateDecoder::~RectangleUpdateDecoder() { +SoftwareVideoRenderer::Core::~Core() { } -void RectangleUpdateDecoder::Initialize(const SessionConfig& config) { - if (!decode_task_runner_->BelongsToCurrentThread()) { - decode_task_runner_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::Initialize, this, - config)); - return; - } +void SoftwareVideoRenderer::Core::Initialize(const SessionConfig& config) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); // Initialize decoder based on the selected codec. ChannelConfig::Codec codec = config.video_config().codec; @@ -115,12 +162,10 @@ void RectangleUpdateDecoder::Initialize(const SessionConfig& config) { } } -void RectangleUpdateDecoder::DecodePacket(scoped_ptr<VideoPacket> packet, - const base::Closure& done) { +void SoftwareVideoRenderer::Core::DecodePacket(scoped_ptr<VideoPacket> packet, + const base::Closure& done) { DCHECK(decode_task_runner_->BelongsToCurrentThread()); - base::ScopedClosureRunner done_runner(done); - bool decoder_needs_reset = false; bool notify_size_or_dpi_change = false; @@ -145,8 +190,10 @@ void RectangleUpdateDecoder::DecodePacket(scoped_ptr<VideoPacket> packet, } // If we've never seen a screen size, ignore the packet. - if (source_size_.is_empty()) + if (source_size_.is_empty()) { + main_task_runner_->PostTask(FROM_HERE, base::Bind(done)); return; + } if (decoder_needs_reset) decoder_->Initialize(source_size_); @@ -158,17 +205,22 @@ void RectangleUpdateDecoder::DecodePacket(scoped_ptr<VideoPacket> packet, } else { LOG(ERROR) << "DecodePacket() failed."; } + + main_task_runner_->PostTask(FROM_HERE, base::Bind(done)); } -void RectangleUpdateDecoder::SchedulePaint() { +void SoftwareVideoRenderer::Core::SchedulePaint() { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); if (paint_scheduled_) return; paint_scheduled_ = true; decode_task_runner_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::DoPaint, this)); + FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DoPaint, + weak_factory_.GetWeakPtr())); } -void RectangleUpdateDecoder::DoPaint() { +void SoftwareVideoRenderer::Core::DoPaint() { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); DCHECK(paint_scheduled_); paint_scheduled_ = false; @@ -184,24 +236,19 @@ void RectangleUpdateDecoder::DoPaint() { webrtc::DesktopFrame* buffer = buffers_.front(); webrtc::DesktopRegion output_region; decoder_->RenderFrame(view_size_, clip_area_, - buffer->data(), - buffer->stride(), - &output_region); + buffer->data(), buffer->stride(), &output_region); // Notify the consumer that painting is done. if (!output_region.is_empty()) { buffers_.pop_front(); - consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region); + consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region, + *decoder_->GetImageShape()); } } -void RectangleUpdateDecoder::RequestReturnBuffers(const base::Closure& done) { - if (!decode_task_runner_->BelongsToCurrentThread()) { - decode_task_runner_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::RequestReturnBuffers, - this, done)); - return; - } +void SoftwareVideoRenderer::Core::RequestReturnBuffers( + const base::Closure& done) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); while (!buffers_.empty()) { consumer_->ReturnBuffer(buffers_.front()); @@ -212,14 +259,8 @@ void RectangleUpdateDecoder::RequestReturnBuffers(const base::Closure& done) { done.Run(); } -void RectangleUpdateDecoder::DrawBuffer(webrtc::DesktopFrame* buffer) { - if (!decode_task_runner_->BelongsToCurrentThread()) { - decode_task_runner_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::DrawBuffer, - this, buffer)); - return; - } - +void SoftwareVideoRenderer::Core::DrawBuffer(webrtc::DesktopFrame* buffer) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); DCHECK(clip_area_.width() <= buffer->size().width() && clip_area_.height() <= buffer->size().height()); @@ -227,14 +268,9 @@ void RectangleUpdateDecoder::DrawBuffer(webrtc::DesktopFrame* buffer) { SchedulePaint(); } -void RectangleUpdateDecoder::InvalidateRegion( +void SoftwareVideoRenderer::Core::InvalidateRegion( const webrtc::DesktopRegion& region) { - if (!decode_task_runner_->BelongsToCurrentThread()) { - decode_task_runner_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::InvalidateRegion, - this, region)); - return; - } + DCHECK(decode_task_runner_->BelongsToCurrentThread()); if (decoder_.get()) { decoder_->Invalidate(view_size_, region); @@ -242,15 +278,10 @@ void RectangleUpdateDecoder::InvalidateRegion( } } -void RectangleUpdateDecoder::SetOutputSizeAndClip( +void SoftwareVideoRenderer::Core::SetOutputSizeAndClip( const webrtc::DesktopSize& view_size, const webrtc::DesktopRect& clip_area) { - if (!decode_task_runner_->BelongsToCurrentThread()) { - decode_task_runner_->PostTask( - FROM_HERE, base::Bind(&RectangleUpdateDecoder::SetOutputSizeAndClip, - this, view_size, clip_area)); - return; - } + DCHECK(decode_task_runner_->BelongsToCurrentThread()); // The whole frame needs to be repainted if the scaling factor has changed. if (!view_size_.equals(view_size) && decoder_.get()) { @@ -281,13 +312,38 @@ void RectangleUpdateDecoder::SetOutputSizeAndClip( } } -const webrtc::DesktopRegion* RectangleUpdateDecoder::GetBufferShape() { - return decoder_->GetImageShape(); +SoftwareVideoRenderer::SoftwareVideoRenderer( + scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, + scoped_refptr<FrameConsumerProxy> consumer) + : decode_task_runner_(decode_task_runner), + core_(new Core(main_task_runner, decode_task_runner, consumer)), + latest_sequence_number_(0), + weak_factory_(this) { + DCHECK(CalledOnValidThread()); } -void RectangleUpdateDecoder::ProcessVideoPacket(scoped_ptr<VideoPacket> packet, +SoftwareVideoRenderer::~SoftwareVideoRenderer() { + DCHECK(CalledOnValidThread()); + decode_task_runner_->DeleteSoon(FROM_HERE, core_.release()); +} + +void SoftwareVideoRenderer::Initialize( + const protocol::SessionConfig& config) { + DCHECK(CalledOnValidThread()); + decode_task_runner_->PostTask( + FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::Initialize, + base::Unretained(core_.get()), config)); +} + +ChromotingStats* SoftwareVideoRenderer::GetStats() { + DCHECK(CalledOnValidThread()); + return &stats_; +} + +void SoftwareVideoRenderer::ProcessVideoPacket(scoped_ptr<VideoPacket> packet, const base::Closure& done) { - DCHECK(main_task_runner_->BelongsToCurrentThread()); + DCHECK(CalledOnValidThread()); // If the video packet is empty then drop it. Empty packets are used to // maintain activity on the network. @@ -317,22 +373,47 @@ void RectangleUpdateDecoder::ProcessVideoPacket(scoped_ptr<VideoPacket> packet, // Measure the latency between the last packet being received and presented. base::Time decode_start = base::Time::Now(); - base::Closure decode_done = base::Bind( - &RectangleUpdateDecoder::OnPacketDone, this, decode_start, done); + base::Closure decode_done = base::Bind(&SoftwareVideoRenderer::OnPacketDone, + weak_factory_.GetWeakPtr(), + decode_start, done); decode_task_runner_->PostTask(FROM_HERE, base::Bind( - &RectangleUpdateDecoder::DecodePacket, this, - base::Passed(&packet), decode_done)); + &SoftwareVideoRenderer::Core::DecodePacket, + base::Unretained(core_.get()), base::Passed(&packet), decode_done)); } -void RectangleUpdateDecoder::OnPacketDone(base::Time decode_start, +void SoftwareVideoRenderer::DrawBuffer(webrtc::DesktopFrame* buffer) { + decode_task_runner_->PostTask( + FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DrawBuffer, + base::Unretained(core_.get()), buffer)); +} + +void SoftwareVideoRenderer::InvalidateRegion( + const webrtc::DesktopRegion& region) { + decode_task_runner_->PostTask( + FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::InvalidateRegion, + base::Unretained(core_.get()), region)); +} + +void SoftwareVideoRenderer::RequestReturnBuffers(const base::Closure& done) { + decode_task_runner_->PostTask( + FROM_HERE, + base::Bind(&SoftwareVideoRenderer::Core::RequestReturnBuffers, + base::Unretained(core_.get()), done)); +} + +void SoftwareVideoRenderer::SetOutputSizeAndClip( + const webrtc::DesktopSize& view_size, + const webrtc::DesktopRect& clip_area) { + decode_task_runner_->PostTask( + FROM_HERE, + base::Bind(&SoftwareVideoRenderer::Core::SetOutputSizeAndClip, + base::Unretained(core_.get()), view_size, clip_area)); +} + +void SoftwareVideoRenderer::OnPacketDone(base::Time decode_start, const base::Closure& done) { - if (!main_task_runner_->BelongsToCurrentThread()) { - main_task_runner_->PostTask(FROM_HERE, base::Bind( - &RectangleUpdateDecoder::OnPacketDone, this, - decode_start, done)); - return; - } + DCHECK(CalledOnValidThread()); // Record the latency between the packet being received and presented. stats_.video_decode_ms()->Record( @@ -341,9 +422,4 @@ void RectangleUpdateDecoder::OnPacketDone(base::Time decode_start, done.Run(); } -ChromotingStats* RectangleUpdateDecoder::GetStats() { - DCHECK(main_task_runner_->BelongsToCurrentThread()); - return &stats_; -} - } // namespace remoting diff --git a/remoting/client/software_video_renderer.h b/remoting/client/software_video_renderer.h new file mode 100644 index 0000000..f03f3e6 --- /dev/null +++ b/remoting/client/software_video_renderer.h @@ -0,0 +1,86 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef REMOTING_CLIENT_SOFTWARE_VIDEO_RENDERER_H_ +#define REMOTING_CLIENT_SOFTWARE_VIDEO_RENDERER_H_ + +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "remoting/client/chromoting_stats.h" +#include "remoting/client/frame_consumer_proxy.h" +#include "remoting/client/frame_producer.h" +#include "remoting/client/video_renderer.h" +#include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h" + +namespace base { +class SingleThreadTaskRunner; +} // namespace base + +namespace remoting { + +class ChromotingStats; + +// Implementation of VideoRenderer interface that decodes frame on CPU (on a +// decode thread) and then passes decoded frames to a FrameConsumer. +// FrameProducer methods can be called on any thread. All other methods must be +// called on the main thread. Owned must ensure that this class outlives +// FrameConsumer (which calls FrameProducer interface). +class SoftwareVideoRenderer : public VideoRenderer, + public FrameProducer, + public base::NonThreadSafe { + public: + // Creates an update decoder on |main_task_runner_| and |decode_task_runner_|, + // outputting to |consumer|. The |main_task_runner_| is responsible for + // receiving and queueing packets. The |decode_task_runner_| is responsible + // for decoding the video packets. + // TODO(wez): Replace the ref-counted proxy with an owned FrameConsumer. + SoftwareVideoRenderer( + scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, + scoped_refptr<FrameConsumerProxy> consumer); + virtual ~SoftwareVideoRenderer(); + + // VideoRenderer implementation. + virtual void Initialize(const protocol::SessionConfig& config) OVERRIDE; + virtual ChromotingStats* GetStats() OVERRIDE; + virtual void ProcessVideoPacket(scoped_ptr<VideoPacket> packet, + const base::Closure& done) OVERRIDE; + + // FrameProducer implementation. These methods may be called before we are + // Initialize()d, or we know the source screen size. These methods may be + // called on any thread. + // + // TODO(sergeyu): On Android a separate display thread is used for drawing. + // FrameConsumer calls FrameProducer on that thread. Can we avoid having a + // separate display thread? E.g. can we do everything on the decode thread? + virtual void DrawBuffer(webrtc::DesktopFrame* buffer) OVERRIDE; + virtual void InvalidateRegion(const webrtc::DesktopRegion& region) OVERRIDE; + virtual void RequestReturnBuffers(const base::Closure& done) OVERRIDE; + virtual void SetOutputSizeAndClip( + const webrtc::DesktopSize& view_size, + const webrtc::DesktopRect& clip_area) OVERRIDE; + + private: + class Core; + + // Callback method when a VideoPacket is processed. |decode_start| contains + // the timestamp when the packet will start to be processed. + void OnPacketDone(base::Time decode_start, const base::Closure& done); + + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_; + scoped_ptr<Core> core_; + + ChromotingStats stats_; + + // Keep track of the most recent sequence number bounced back from the host. + int64 latest_sequence_number_; + + base::WeakPtrFactory<SoftwareVideoRenderer> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(SoftwareVideoRenderer); +}; + +} // namespace remoting + +#endif // REMOTING_CLIENT_SOFTWARE_VIDEO_RENDERER_H_ diff --git a/remoting/client/video_renderer.h b/remoting/client/video_renderer.h new file mode 100644 index 0000000..08ccc40 --- /dev/null +++ b/remoting/client/video_renderer.h @@ -0,0 +1,31 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef REMOTING_CLIENT_VIDEO_RENDERER_H_ +#define REMOTING_CLIENT_VIDEO_RENDERER_H_ + +#include "remoting/protocol/video_stub.h" + +namespace remoting { + +class ChromotingStats; + +namespace protocol { +class SessionConfig; +} // namespace protocol; + +// VideoRenderer is responsible for decoding and displaying incoming video +// stream. +class VideoRenderer : public protocol::VideoStub { + public: + // Initializes the processor with the information from the protocol config. + virtual void Initialize(const protocol::SessionConfig& config) = 0; + + // Return the statistics recorded by this client. + virtual ChromotingStats* GetStats() = 0; +}; + +} // namespace remoting + +#endif // REMOTING_CLIENT_VIDEO_RENDERER_H_ diff --git a/remoting/remoting.gyp b/remoting/remoting.gyp index 51a2213..220c79f 100644 --- a/remoting/remoting.gyp +++ b/remoting/remoting.gyp @@ -2476,8 +2476,9 @@ 'client/frame_producer.h', 'client/key_event_mapper.cc', 'client/key_event_mapper.h', - 'client/rectangle_update_decoder.cc', - 'client/rectangle_update_decoder.h', + 'client/software_video_renderer.cc', + 'client/software_video_renderer.h', + 'client/video_renderer.h', ], }, # end of target 'remoting_client' |