diff options
Diffstat (limited to 'remoting/client/software_video_renderer.cc')
| -rw-r--r-- | remoting/client/software_video_renderer.cc | 425 |
1 files changed, 425 insertions, 0 deletions
diff --git a/remoting/client/software_video_renderer.cc b/remoting/client/software_video_renderer.cc new file mode 100644 index 0000000..4d95e13 --- /dev/null +++ b/remoting/client/software_video_renderer.cc @@ -0,0 +1,425 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "remoting/client/software_video_renderer.h" + +#include <list> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/callback_helpers.h" +#include "base/location.h" +#include "base/logging.h" +#include "base/single_thread_task_runner.h" +#include "remoting/base/util.h" +#include "remoting/client/frame_consumer.h" +#include "remoting/codec/video_decoder.h" +#include "remoting/codec/video_decoder_verbatim.h" +#include "remoting/codec/video_decoder_vpx.h" +#include "remoting/protocol/session_config.h" +#include "third_party/libyuv/include/libyuv/convert_argb.h" +#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h" + +using base::Passed; +using remoting::protocol::ChannelConfig; +using remoting::protocol::SessionConfig; + +namespace remoting { + +// This class wraps a VideoDecoder and byte-swaps the pixels for compatibility +// with the android.graphics.Bitmap class. +// TODO(lambroslambrou): Refactor so that the VideoDecoder produces data +// in the right byte-order, instead of swapping it here. +class RgbToBgrVideoDecoderFilter : public VideoDecoder { + public: + RgbToBgrVideoDecoderFilter(scoped_ptr<VideoDecoder> parent) + : parent_(parent.Pass()) { + } + + virtual void Initialize(const webrtc::DesktopSize& screen_size) OVERRIDE { + parent_->Initialize(screen_size); + } + + virtual bool DecodePacket(const VideoPacket& packet) OVERRIDE { + return parent_->DecodePacket(packet); + } + + virtual void Invalidate(const webrtc::DesktopSize& view_size, + const webrtc::DesktopRegion& region) OVERRIDE { + return parent_->Invalidate(view_size, region); + } + + virtual void RenderFrame(const webrtc::DesktopSize& view_size, + const webrtc::DesktopRect& clip_area, + uint8* image_buffer, + int image_stride, + webrtc::DesktopRegion* output_region) OVERRIDE { + parent_->RenderFrame(view_size, clip_area, image_buffer, image_stride, + output_region); + + for (webrtc::DesktopRegion::Iterator i(*output_region); !i.IsAtEnd(); + i.Advance()) { + webrtc::DesktopRect rect = i.rect(); + uint8* pixels = image_buffer + (rect.top() * image_stride) + + (rect.left() * kBytesPerPixel); + libyuv::ABGRToARGB(pixels, image_stride, pixels, image_stride, + rect.width(), rect.height()); + } + } + + virtual const webrtc::DesktopRegion* GetImageShape() OVERRIDE { + return parent_->GetImageShape(); + } + + private: + scoped_ptr<VideoDecoder> parent_; +}; + +class SoftwareVideoRenderer::Core { + public: + Core(scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, + scoped_refptr<FrameConsumerProxy> consumer); + ~Core(); + + void Initialize(const protocol::SessionConfig& config); + void DrawBuffer(webrtc::DesktopFrame* buffer); + void InvalidateRegion(const webrtc::DesktopRegion& region); + void RequestReturnBuffers(const base::Closure& done); + void SetOutputSizeAndClip( + const webrtc::DesktopSize& view_size, + const webrtc::DesktopRect& clip_area); + + // Decodes the contents of |packet|. DecodePacket may keep a reference to + // |packet| so the |packet| must remain alive and valid until |done| is + // executed. + void DecodePacket(scoped_ptr<VideoPacket> packet, const base::Closure& done); + + private: + // Paints the invalidated region to the next available buffer and returns it + // to the consumer. + void SchedulePaint(); + void DoPaint(); + + scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner_; + scoped_refptr<FrameConsumerProxy> consumer_; + scoped_ptr<VideoDecoder> decoder_; + + // Remote screen size in pixels. + webrtc::DesktopSize source_size_; + + // Vertical and horizontal DPI of the remote screen. + webrtc::DesktopVector source_dpi_; + + // The current dimensions of the frame consumer view. + webrtc::DesktopSize view_size_; + webrtc::DesktopRect clip_area_; + + // The drawing buffers supplied by the frame consumer. + std::list<webrtc::DesktopFrame*> buffers_; + + // Flag used to coalesce runs of SchedulePaint()s into a single DoPaint(). + bool paint_scheduled_; + + base::WeakPtrFactory<Core> weak_factory_; +}; + +SoftwareVideoRenderer::Core::Core( + scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, + scoped_refptr<FrameConsumerProxy> consumer) + : main_task_runner_(main_task_runner), + decode_task_runner_(decode_task_runner), + consumer_(consumer), + paint_scheduled_(false), + weak_factory_(this) { +} + +SoftwareVideoRenderer::Core::~Core() { +} + +void SoftwareVideoRenderer::Core::Initialize(const SessionConfig& config) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); + + // Initialize decoder based on the selected codec. + ChannelConfig::Codec codec = config.video_config().codec; + if (codec == ChannelConfig::CODEC_VERBATIM) { + decoder_.reset(new VideoDecoderVerbatim()); + } else if (codec == ChannelConfig::CODEC_VP8) { + decoder_ = VideoDecoderVpx::CreateForVP8(); + } else if (codec == ChannelConfig::CODEC_VP9) { + decoder_ = VideoDecoderVpx::CreateForVP9(); + } else { + NOTREACHED() << "Invalid Encoding found: " << codec; + } + + if (consumer_->GetPixelFormat() == FrameConsumer::FORMAT_RGBA) { + scoped_ptr<VideoDecoder> wrapper( + new RgbToBgrVideoDecoderFilter(decoder_.Pass())); + decoder_ = wrapper.Pass(); + } +} + +void SoftwareVideoRenderer::Core::DecodePacket(scoped_ptr<VideoPacket> packet, + const base::Closure& done) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); + + bool decoder_needs_reset = false; + bool notify_size_or_dpi_change = false; + + // If the packet includes screen size or DPI information, store them. + if (packet->format().has_screen_width() && + packet->format().has_screen_height()) { + webrtc::DesktopSize source_size(packet->format().screen_width(), + packet->format().screen_height()); + if (!source_size_.equals(source_size)) { + source_size_ = source_size; + decoder_needs_reset = true; + notify_size_or_dpi_change = true; + } + } + if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) { + webrtc::DesktopVector source_dpi(packet->format().x_dpi(), + packet->format().y_dpi()); + if (!source_dpi.equals(source_dpi_)) { + source_dpi_ = source_dpi; + notify_size_or_dpi_change = true; + } + } + + // If we've never seen a screen size, ignore the packet. + if (source_size_.is_empty()) { + main_task_runner_->PostTask(FROM_HERE, base::Bind(done)); + return; + } + + if (decoder_needs_reset) + decoder_->Initialize(source_size_); + if (notify_size_or_dpi_change) + consumer_->SetSourceSize(source_size_, source_dpi_); + + if (decoder_->DecodePacket(*packet.get())) { + SchedulePaint(); + } else { + LOG(ERROR) << "DecodePacket() failed."; + } + + main_task_runner_->PostTask(FROM_HERE, base::Bind(done)); +} + +void SoftwareVideoRenderer::Core::SchedulePaint() { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); + if (paint_scheduled_) + return; + paint_scheduled_ = true; + decode_task_runner_->PostTask( + FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DoPaint, + weak_factory_.GetWeakPtr())); +} + +void SoftwareVideoRenderer::Core::DoPaint() { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); + DCHECK(paint_scheduled_); + paint_scheduled_ = false; + + // If the view size is empty or we have no output buffers ready, return. + if (buffers_.empty() || view_size_.is_empty()) + return; + + // If no Decoder is initialized, or the host dimensions are empty, return. + if (!decoder_.get() || source_size_.is_empty()) + return; + + // Draw the invalidated region to the buffer. + webrtc::DesktopFrame* buffer = buffers_.front(); + webrtc::DesktopRegion output_region; + decoder_->RenderFrame(view_size_, clip_area_, + buffer->data(), buffer->stride(), &output_region); + + // Notify the consumer that painting is done. + if (!output_region.is_empty()) { + buffers_.pop_front(); + consumer_->ApplyBuffer(view_size_, clip_area_, buffer, output_region, + *decoder_->GetImageShape()); + } +} + +void SoftwareVideoRenderer::Core::RequestReturnBuffers( + const base::Closure& done) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); + + while (!buffers_.empty()) { + consumer_->ReturnBuffer(buffers_.front()); + buffers_.pop_front(); + } + + if (!done.is_null()) + done.Run(); +} + +void SoftwareVideoRenderer::Core::DrawBuffer(webrtc::DesktopFrame* buffer) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); + DCHECK(clip_area_.width() <= buffer->size().width() && + clip_area_.height() <= buffer->size().height()); + + buffers_.push_back(buffer); + SchedulePaint(); +} + +void SoftwareVideoRenderer::Core::InvalidateRegion( + const webrtc::DesktopRegion& region) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); + + if (decoder_.get()) { + decoder_->Invalidate(view_size_, region); + SchedulePaint(); + } +} + +void SoftwareVideoRenderer::Core::SetOutputSizeAndClip( + const webrtc::DesktopSize& view_size, + const webrtc::DesktopRect& clip_area) { + DCHECK(decode_task_runner_->BelongsToCurrentThread()); + + // The whole frame needs to be repainted if the scaling factor has changed. + if (!view_size_.equals(view_size) && decoder_.get()) { + webrtc::DesktopRegion region; + region.AddRect(webrtc::DesktopRect::MakeSize(view_size)); + decoder_->Invalidate(view_size, region); + } + + if (!view_size_.equals(view_size) || + !clip_area_.equals(clip_area)) { + view_size_ = view_size; + clip_area_ = clip_area; + + // Return buffers that are smaller than needed to the consumer for + // reuse/reallocation. + std::list<webrtc::DesktopFrame*>::iterator i = buffers_.begin(); + while (i != buffers_.end()) { + if ((*i)->size().width() < clip_area_.width() || + (*i)->size().height() < clip_area_.height()) { + consumer_->ReturnBuffer(*i); + i = buffers_.erase(i); + } else { + ++i; + } + } + + SchedulePaint(); + } +} + +SoftwareVideoRenderer::SoftwareVideoRenderer( + scoped_refptr<base::SingleThreadTaskRunner> main_task_runner, + scoped_refptr<base::SingleThreadTaskRunner> decode_task_runner, + scoped_refptr<FrameConsumerProxy> consumer) + : decode_task_runner_(decode_task_runner), + core_(new Core(main_task_runner, decode_task_runner, consumer)), + latest_sequence_number_(0), + weak_factory_(this) { + DCHECK(CalledOnValidThread()); +} + +SoftwareVideoRenderer::~SoftwareVideoRenderer() { + DCHECK(CalledOnValidThread()); + decode_task_runner_->DeleteSoon(FROM_HERE, core_.release()); +} + +void SoftwareVideoRenderer::Initialize( + const protocol::SessionConfig& config) { + DCHECK(CalledOnValidThread()); + decode_task_runner_->PostTask( + FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::Initialize, + base::Unretained(core_.get()), config)); +} + +ChromotingStats* SoftwareVideoRenderer::GetStats() { + DCHECK(CalledOnValidThread()); + return &stats_; +} + +void SoftwareVideoRenderer::ProcessVideoPacket(scoped_ptr<VideoPacket> packet, + const base::Closure& done) { + DCHECK(CalledOnValidThread()); + + // If the video packet is empty then drop it. Empty packets are used to + // maintain activity on the network. + if (!packet->has_data() || packet->data().size() == 0) { + done.Run(); + return; + } + + // Add one frame to the counter. + stats_.video_frame_rate()->Record(1); + + // Record other statistics received from host. + stats_.video_bandwidth()->Record(packet->data().size()); + if (packet->has_capture_time_ms()) + stats_.video_capture_ms()->Record(packet->capture_time_ms()); + if (packet->has_encode_time_ms()) + stats_.video_encode_ms()->Record(packet->encode_time_ms()); + if (packet->has_client_sequence_number() && + packet->client_sequence_number() > latest_sequence_number_) { + latest_sequence_number_ = packet->client_sequence_number(); + base::TimeDelta round_trip_latency = + base::Time::Now() - + base::Time::FromInternalValue(packet->client_sequence_number()); + stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds()); + } + + // Measure the latency between the last packet being received and presented. + base::Time decode_start = base::Time::Now(); + + base::Closure decode_done = base::Bind(&SoftwareVideoRenderer::OnPacketDone, + weak_factory_.GetWeakPtr(), + decode_start, done); + + decode_task_runner_->PostTask(FROM_HERE, base::Bind( + &SoftwareVideoRenderer::Core::DecodePacket, + base::Unretained(core_.get()), base::Passed(&packet), decode_done)); +} + +void SoftwareVideoRenderer::DrawBuffer(webrtc::DesktopFrame* buffer) { + decode_task_runner_->PostTask( + FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::DrawBuffer, + base::Unretained(core_.get()), buffer)); +} + +void SoftwareVideoRenderer::InvalidateRegion( + const webrtc::DesktopRegion& region) { + decode_task_runner_->PostTask( + FROM_HERE, base::Bind(&SoftwareVideoRenderer::Core::InvalidateRegion, + base::Unretained(core_.get()), region)); +} + +void SoftwareVideoRenderer::RequestReturnBuffers(const base::Closure& done) { + decode_task_runner_->PostTask( + FROM_HERE, + base::Bind(&SoftwareVideoRenderer::Core::RequestReturnBuffers, + base::Unretained(core_.get()), done)); +} + +void SoftwareVideoRenderer::SetOutputSizeAndClip( + const webrtc::DesktopSize& view_size, + const webrtc::DesktopRect& clip_area) { + decode_task_runner_->PostTask( + FROM_HERE, + base::Bind(&SoftwareVideoRenderer::Core::SetOutputSizeAndClip, + base::Unretained(core_.get()), view_size, clip_area)); +} + +void SoftwareVideoRenderer::OnPacketDone(base::Time decode_start, + const base::Closure& done) { + DCHECK(CalledOnValidThread()); + + // Record the latency between the packet being received and presented. + stats_.video_decode_ms()->Record( + (base::Time::Now() - decode_start).InMilliseconds()); + + done.Run(); +} + +} // namespace remoting |