diff options
author | miu <miu@chromium.org> | 2015-02-10 19:28:44 -0800 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-02-11 03:29:24 +0000 |
commit | 9005303cba2d9ab58dd4439295695a93c8801e96 (patch) | |
tree | b628b54da9f7f4c820b3543140c4ab3a5e0262f1 | |
parent | e1ee70db0efad38817a6aa7027b2d1cfafe3af82 (diff) | |
download | chromium_src-9005303cba2d9ab58dd4439295695a93c8801e96.zip chromium_src-9005303cba2d9ab58dd4439295695a93c8801e96.tar.gz chromium_src-9005303cba2d9ab58dd4439295695a93c8801e96.tar.bz2 |
[Cast] Size-Adaptable platform video encoders.
Provides a proxy VideoEncoder implementation that wraps
ExternalVideoEncoder or H264VTEncoder to handle frame size changes.
When the frame size changes, the proxy 1) waits for current encoder
instance to finish encoding any frames in-progress, then 2) destroys the
encoder, and 3) finally re-creates/initializes a new encoder instance.
During this process, frames provided via InsertRawVideoFrame() are
simply dropped.
Merged external_encoder_unittest.cc and video_encoder_impl_unittest.cc
into a single video_encoder_unittest.cc that tests *all* cast
VideoEncoder implementations. Made a number of fixes to allow testing
of the H264VTEncoder on desktop Macs and the trybots.
BUG=451277
Review URL: https://codereview.chromium.org/906403006
Cr-Commit-Position: refs/heads/master@{#315723}
31 files changed, 1346 insertions, 858 deletions
diff --git a/chrome/renderer/media/cast_rtp_stream.cc b/chrome/renderer/media/cast_rtp_stream.cc index 4d51889..a3f970d 100644 --- a/chrome/renderer/media/cast_rtp_stream.cc +++ b/chrome/renderer/media/cast_rtp_stream.cc @@ -214,10 +214,6 @@ bool ToVideoSenderConfig(const CastRtpParams& params, if (config->min_playout_delay > config->max_playout_delay) return false; config->rtp_payload_type = params.payload.payload_type; - config->width = params.payload.width; - config->height = params.payload.height; - if (config->width < 2 || config->height < 2) - return false; config->min_bitrate = config->start_bitrate = params.payload.min_bitrate * kBitrateMultiplier; config->max_bitrate = params.payload.max_bitrate * kBitrateMultiplier; @@ -257,14 +253,11 @@ class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>, public content::MediaStreamVideoSink { public: // |track| provides data for this sink. - // |expected_natural_size| is the expected dimension of the video frame. // |error_callback| is called if video formats don't match. CastVideoSink(const blink::WebMediaStreamTrack& track, - const gfx::Size& expected_natural_size, const CastRtpStream::ErrorCallback& error_callback) : track_(track), sink_added_(false), - expected_natural_size_(expected_natural_size), error_callback_(error_callback) {} ~CastVideoSink() override { @@ -275,24 +268,12 @@ class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>, // This static method is used to forward video frames to |frame_input|. static void OnVideoFrame( // These parameters are already bound when callback is created. - const gfx::Size& expected_natural_size, const CastRtpStream::ErrorCallback& error_callback, const scoped_refptr<media::cast::VideoFrameInput> frame_input, // These parameters are passed for each frame. const scoped_refptr<media::VideoFrame>& frame, const media::VideoCaptureFormat& format, const base::TimeTicks& estimated_capture_time) { - if (frame->natural_size() != expected_natural_size) { - error_callback.Run( - base::StringPrintf("Video frame resolution does not match config." - " Expected %dx%d. Got %dx%d.", - expected_natural_size.width(), - expected_natural_size.height(), - frame->natural_size().width(), - frame->natural_size().height())); - return; - } - base::TimeTicks timestamp; if (estimated_capture_time.is_null()) timestamp = base::TimeTicks::Now(); @@ -318,7 +299,6 @@ class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>, this, base::Bind( &CastVideoSink::OnVideoFrame, - expected_natural_size_, error_callback_, frame_input), track_); @@ -327,7 +307,6 @@ class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>, private: blink::WebMediaStreamTrack track_; bool sink_added_; - gfx::Size expected_natural_size_; CastRtpStream::ErrorCallback error_callback_; DISALLOW_COPY_AND_ASSIGN(CastVideoSink); @@ -543,7 +522,6 @@ void CastRtpStream::Start(const CastRtpParams& params, // See the code for audio above for explanation of callbacks. video_sink_.reset(new CastVideoSink( track_, - gfx::Size(config.width, config.height), media::BindToCurrentLoop(base::Bind(&CastRtpStream::DidEncounterError, weak_factory_.GetWeakPtr())))); cast_session_->StartVideo( diff --git a/chrome/renderer/media/cast_rtp_stream.h b/chrome/renderer/media/cast_rtp_stream.h index 24bf885..61a6bf4 100644 --- a/chrome/renderer/media/cast_rtp_stream.h +++ b/chrome/renderer/media/cast_rtp_stream.h @@ -68,6 +68,7 @@ struct CastRtpPayloadParams { double max_frame_rate; // Width and height of the video content. + // TODO(miu): DEPRECATED. Remove these, as they are ignored. int width; int height; diff --git a/chrome/test/data/extensions/api_test/cast_streaming/rtp_stream_error.js b/chrome/test/data/extensions/api_test/cast_streaming/rtp_stream_error.js index 5cdba8a..9db1949 100644 --- a/chrome/test/data/extensions/api_test/cast_streaming/rtp_stream_error.js +++ b/chrome/test/data/extensions/api_test/cast_streaming/rtp_stream_error.js @@ -39,9 +39,8 @@ chrome.test.runTests([ udpTransport.destroy(udpId); console.log(msg); }.bind(null, audioId, videoId))); - // Invalid width and height. - videoParams.payload.width = 100; - videoParams.payload.height = 100; + // Specify invalid value to trigger error. + videoParams.payload.codecName = "Animated WebP"; udpTransport.setDestination(udpId, {address: "127.0.0.1", port: 2344}); rtpStream.start(videoId, videoParams); diff --git a/media/cast/BUILD.gn b/media/cast/BUILD.gn index 6a250d0..99c3332 100644 --- a/media/cast/BUILD.gn +++ b/media/cast/BUILD.gn @@ -121,6 +121,8 @@ source_set("sender") { "sender/fake_software_video_encoder.cc", "sender/frame_sender.cc", "sender/frame_sender.h", + "sender/size_adaptable_video_encoder_base.cc", + "sender/size_adaptable_video_encoder_base.h", "sender/software_video_encoder.h", "sender/video_encoder.h", "sender/video_encoder.cc", @@ -296,10 +298,9 @@ test("cast_unittests") { "sender/audio_encoder_unittest.cc", "sender/audio_sender_unittest.cc", "sender/congestion_control_unittest.cc", - "sender/external_video_encoder_unittest.cc", "sender/fake_video_encode_accelerator_factory.cc", "sender/fake_video_encode_accelerator_factory.h", - "sender/video_encoder_impl_unittest.cc", + "sender/video_encoder_unittest.cc", "sender/video_sender_unittest.cc", "test/end2end_unittest.cc", "test/fake_receiver_time_offset_estimator.cc", diff --git a/media/cast/cast.gyp b/media/cast/cast.gyp index 0a7c619..82ba1f4 100644 --- a/media/cast/cast.gyp +++ b/media/cast/cast.gyp @@ -162,6 +162,8 @@ 'sender/fake_software_video_encoder.cc', 'sender/frame_sender.cc', 'sender/frame_sender.h', + 'sender/size_adaptable_video_encoder_base.cc', + 'sender/size_adaptable_video_encoder_base.h', 'sender/software_video_encoder.h', 'sender/video_encoder.h', 'sender/video_encoder.cc', diff --git a/media/cast/cast_config.cc b/media/cast/cast_config.cc index 9dda608..7c8a195 100644 --- a/media/cast/cast_config.cc +++ b/media/cast/cast_config.cc @@ -29,8 +29,6 @@ VideoSenderConfig::VideoSenderConfig() base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs)), rtp_payload_type(0), use_external_encoder(false), - width(0), - height(0), congestion_control_back_off(kDefaultCongestionControlBackOff), max_bitrate(5000000), min_bitrate(1000000), diff --git a/media/cast/cast_config.h b/media/cast/cast_config.h index 89af430..9deb678 100644 --- a/media/cast/cast_config.h +++ b/media/cast/cast_config.h @@ -87,8 +87,6 @@ struct VideoSenderConfig { int rtp_payload_type; bool use_external_encoder; - int width; // Incoming frames will be scaled to this size. - int height; float congestion_control_back_off; int max_bitrate; diff --git a/media/cast/cast_sender.h b/media/cast/cast_sender.h index 1ccebd1..7c68913 100644 --- a/media/cast/cast_sender.h +++ b/media/cast/cast_sender.h @@ -21,6 +21,10 @@ #include "media/cast/cast_environment.h" #include "media/cast/net/cast_transport_sender.h" +namespace gfx { +class Size; +} + namespace media { class VideoFrame; @@ -40,14 +44,18 @@ class VideoFrameInput : public base::RefCountedThreadSafe<VideoFrameInput> { // frames offer performance benefits, such as memory copy elimination. The // format is guaranteed to be I420 or NV12. // - // Not every encoder supports this method. Use |ShouldCreateOptimizedFrame| - // to determine if you can and should use this method. Calling - // this method when |ShouldCreateOptimizedFrame| is false will CHECK. - virtual scoped_refptr<VideoFrame> CreateOptimizedFrame( - base::TimeDelta timestamp) = 0; + // Not every encoder supports this method. Use |CanCreateOptimizedFrames| to + // determine if you can and should use this method. + // + // Even if |CanCreateOptimizedFrames| indicates support, there are transient + // conditions during a session where optimized frames cannot be provided. In + // this case, the caller must be able to account for a nullptr return value + // and instantiate its own media::VideoFrames. + virtual scoped_refptr<VideoFrame> MaybeCreateOptimizedFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) = 0; // Returns true if the encoder supports creating optimized frames. - virtual bool SupportsCreateOptimizedFrame() const = 0; + virtual bool CanCreateOptimizedFrames() const = 0; protected: virtual ~VideoFrameInput() {} diff --git a/media/cast/cast_sender_impl.cc b/media/cast/cast_sender_impl.cc index b47ae9c..23e8a23 100644 --- a/media/cast/cast_sender_impl.cc +++ b/media/cast/cast_sender_impl.cc @@ -19,11 +19,12 @@ namespace cast { class LocalVideoFrameInput : public VideoFrameInput { public: LocalVideoFrameInput(scoped_refptr<CastEnvironment> cast_environment, - base::WeakPtr<VideoSender> video_sender, - scoped_ptr<VideoFrameFactory> video_frame_factory) + base::WeakPtr<VideoSender> video_sender) : cast_environment_(cast_environment), video_sender_(video_sender), - video_frame_factory_(video_frame_factory.Pass()) {} + video_frame_factory_( + video_sender.get() ? + video_sender->CreateVideoFrameFactory().release() : nullptr) {} void InsertRawVideoFrame(const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& capture_time) override { @@ -35,13 +36,14 @@ class LocalVideoFrameInput : public VideoFrameInput { capture_time)); } - scoped_refptr<VideoFrame> CreateOptimizedFrame( + scoped_refptr<VideoFrame> MaybeCreateOptimizedFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) override { - DCHECK(video_frame_factory_.get()); - return video_frame_factory_->CreateFrame(timestamp); + return video_frame_factory_ ? + video_frame_factory_->MaybeCreateFrame(frame_size, timestamp) : nullptr; } - bool SupportsCreateOptimizedFrame() const override { + bool CanCreateOptimizedFrames() const override { return video_frame_factory_.get() != nullptr; } @@ -51,9 +53,9 @@ class LocalVideoFrameInput : public VideoFrameInput { private: friend class base::RefCountedThreadSafe<LocalVideoFrameInput>; - scoped_refptr<CastEnvironment> cast_environment_; - base::WeakPtr<VideoSender> video_sender_; - scoped_ptr<VideoFrameFactory> video_frame_factory_; + const scoped_refptr<CastEnvironment> cast_environment_; + const base::WeakPtr<VideoSender> video_sender_; + const scoped_ptr<VideoFrameFactory> video_frame_factory_; DISALLOW_COPY_AND_ASSIGN(LocalVideoFrameInput); }; @@ -194,8 +196,7 @@ void CastSenderImpl::OnVideoStatusChange( DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); if (status == STATUS_INITIALIZED && !video_frame_input_) { video_frame_input_ = - new LocalVideoFrameInput(cast_environment_, video_sender_->AsWeakPtr(), - video_sender_->CreateVideoFrameFactory()); + new LocalVideoFrameInput(cast_environment_, video_sender_->AsWeakPtr()); } status_change_cb.Run(status); } diff --git a/media/cast/cast_testing.gypi b/media/cast/cast_testing.gypi index 67a0f7d..22a2182 100644 --- a/media/cast/cast_testing.gypi +++ b/media/cast/cast_testing.gypi @@ -115,10 +115,9 @@ 'sender/audio_encoder_unittest.cc', 'sender/audio_sender_unittest.cc', 'sender/congestion_control_unittest.cc', - 'sender/external_video_encoder_unittest.cc', 'sender/fake_video_encode_accelerator_factory.cc', 'sender/fake_video_encode_accelerator_factory.h', - 'sender/video_encoder_impl_unittest.cc', + 'sender/video_encoder_unittest.cc', 'sender/video_sender_unittest.cc', 'test/end2end_unittest.cc', 'test/fake_receiver_time_offset_estimator.cc', diff --git a/media/cast/sender/external_video_encoder.cc b/media/cast/sender/external_video_encoder.cc index c5685e5..fcabe5c 100644 --- a/media/cast/sender/external_video_encoder.cc +++ b/media/cast/sender/external_video_encoder.cc @@ -72,7 +72,7 @@ class ExternalVideoEncoder::VEAClientImpl create_video_encode_memory_cb_(create_video_encode_memory_cb), video_encode_accelerator_(vea.Pass()), encoder_active_(false), - last_encoded_frame_id_(kStartFrameId), + next_frame_id_(0u), key_frame_encountered_(false) { } @@ -82,9 +82,9 @@ class ExternalVideoEncoder::VEAClientImpl void Initialize(const gfx::Size& frame_size, VideoCodecProfile codec_profile, - int start_bit_rate) { + int start_bit_rate, + uint32 first_frame_id) { DCHECK(task_runner_->RunsTasksOnCurrentThread()); - DCHECK(!frame_size.IsEmpty()); encoder_active_ = video_encode_accelerator_->Initialize( media::VideoFrame::I420, @@ -92,6 +92,7 @@ class ExternalVideoEncoder::VEAClientImpl codec_profile, start_bit_rate, this); + next_frame_id_ = first_frame_id; UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess", encoder_active_); @@ -203,7 +204,7 @@ class ExternalVideoEncoder::VEAClientImpl scoped_ptr<EncodedFrame> encoded_frame(new EncodedFrame()); encoded_frame->dependency = key_frame ? EncodedFrame::KEY : EncodedFrame::DEPENDENT; - encoded_frame->frame_id = ++last_encoded_frame_id_; + encoded_frame->frame_id = next_frame_id_++; if (key_frame) encoded_frame->referenced_frame_id = encoded_frame->frame_id; else @@ -290,7 +291,7 @@ class ExternalVideoEncoder::VEAClientImpl const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_; scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_; bool encoder_active_; - uint32 last_encoded_frame_id_; + uint32 next_frame_id_; bool key_frame_encountered_; std::string stream_header_; @@ -303,71 +304,61 @@ class ExternalVideoEncoder::VEAClientImpl DISALLOW_COPY_AND_ASSIGN(VEAClientImpl); }; +// static +bool ExternalVideoEncoder::IsSupported(const VideoSenderConfig& video_config) { + if (video_config.codec != CODEC_VIDEO_VP8 && + video_config.codec != CODEC_VIDEO_H264) + return false; + + // TODO(miu): "Layering hooks" are needed to be able to query outside of + // libmedia, to determine whether the system provides a hardware encoder. For + // now, assume that this was already checked by this point. + // http://crbug.com/454029 + return video_config.use_external_encoder; +} + ExternalVideoEncoder::ExternalVideoEncoder( const scoped_refptr<CastEnvironment>& cast_environment, const VideoSenderConfig& video_config, const gfx::Size& frame_size, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb, const CreateVideoEncodeAcceleratorCallback& create_vea_cb, const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb) : cast_environment_(cast_environment), create_video_encode_memory_cb_(create_video_encode_memory_cb), + frame_size_(frame_size), bit_rate_(video_config.start_bitrate), key_frame_requested_(false), weak_factory_(this) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); DCHECK_GT(video_config.max_frame_rate, 0); - DCHECK(!frame_size.IsEmpty()); + DCHECK(!frame_size_.IsEmpty()); DCHECK(!status_change_cb.is_null()); DCHECK(!create_vea_cb.is_null()); DCHECK(!create_video_encode_memory_cb_.is_null()); DCHECK_GT(bit_rate_, 0); - VideoCodecProfile codec_profile; - switch (video_config.codec) { - case CODEC_VIDEO_VP8: - codec_profile = media::VP8PROFILE_ANY; - break; - case CODEC_VIDEO_H264: - codec_profile = media::H264PROFILE_MAIN; - break; - case CODEC_VIDEO_FAKE: - NOTREACHED() << "Fake software video encoder cannot be external"; - // ...flow through to next case... - default: - cast_environment_->PostTask( - CastEnvironment::MAIN, - FROM_HERE, - base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC)); - return; - } - create_vea_cb.Run( base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator, weak_factory_.GetWeakPtr(), - frame_size, - codec_profile, - video_config.max_frame_rate, + video_config, + first_frame_id, status_change_cb)); } ExternalVideoEncoder::~ExternalVideoEncoder() { } -bool ExternalVideoEncoder::CanEncodeVariedFrameSizes() const { - return false; -} - bool ExternalVideoEncoder::EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, const FrameEncodedCallback& frame_encoded_callback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - DCHECK(!video_frame->visible_rect().IsEmpty()); DCHECK(!frame_encoded_callback.is_null()); - if (!client_) - return false; // Not ready. + if (!client_ || video_frame->visible_rect().size() != frame_size_) + return false; client_->task_runner()->PostTask(FROM_HERE, base::Bind(&VEAClientImpl::EncodeVideoFrame, @@ -401,9 +392,8 @@ void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) { } void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator( - const gfx::Size& frame_size, - VideoCodecProfile codec_profile, - int max_frame_rate, + const VideoSenderConfig& video_config, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb, scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner, scoped_ptr<media::VideoEncodeAccelerator> vea) { @@ -420,19 +410,64 @@ void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator( return; } + VideoCodecProfile codec_profile; + switch (video_config.codec) { + case CODEC_VIDEO_VP8: + codec_profile = media::VP8PROFILE_ANY; + break; + case CODEC_VIDEO_H264: + codec_profile = media::H264PROFILE_MAIN; + break; + case CODEC_VIDEO_FAKE: + NOTREACHED() << "Fake software video encoder cannot be external"; + // ...flow through to next case... + default: + cast_environment_->PostTask( + CastEnvironment::MAIN, + FROM_HERE, + base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC)); + return; + } + DCHECK(!client_); client_ = new VEAClientImpl(cast_environment_, encoder_task_runner, vea.Pass(), - max_frame_rate, + video_config.max_frame_rate, status_change_cb, create_video_encode_memory_cb_); client_->task_runner()->PostTask(FROM_HERE, base::Bind(&VEAClientImpl::Initialize, client_, - frame_size, + frame_size_, codec_profile, - bit_rate_)); + bit_rate_, + first_frame_id)); +} + +SizeAdaptableExternalVideoEncoder::SizeAdaptableExternalVideoEncoder( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb, + const CreateVideoEncodeAcceleratorCallback& create_vea_cb, + const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb) + : SizeAdaptableVideoEncoderBase(cast_environment, + video_config, + status_change_cb), + create_vea_cb_(create_vea_cb), + create_video_encode_memory_cb_(create_video_encode_memory_cb) {} + +SizeAdaptableExternalVideoEncoder::~SizeAdaptableExternalVideoEncoder() {} + +scoped_ptr<VideoEncoder> SizeAdaptableExternalVideoEncoder::CreateEncoder() { + return scoped_ptr<VideoEncoder>(new ExternalVideoEncoder( + cast_environment(), + video_config(), + frame_size(), + last_frame_id() + 1, + CreateEncoderStatusChangeCallback(), + create_vea_cb_, + create_video_encode_memory_cb_)); } } // namespace cast diff --git a/media/cast/sender/external_video_encoder.h b/media/cast/sender/external_video_encoder.h index e560c57..ee8b7f4 100644 --- a/media/cast/sender/external_video_encoder.h +++ b/media/cast/sender/external_video_encoder.h @@ -9,6 +9,7 @@ #include "base/memory/weak_ptr.h" #include "media/cast/cast_config.h" #include "media/cast/cast_environment.h" +#include "media/cast/sender/size_adaptable_video_encoder_base.h" #include "media/cast/sender/video_encoder.h" #include "media/video/video_encode_accelerator.h" #include "ui/gfx/geometry/size.h" @@ -21,10 +22,15 @@ namespace cast { // emits media::cast::EncodedFrames. class ExternalVideoEncoder : public VideoEncoder { public: + // Returns true if the current platform and system configuration supports + // using ExternalVideoEncoder with the given |video_config|. + static bool IsSupported(const VideoSenderConfig& video_config); + ExternalVideoEncoder( const scoped_refptr<CastEnvironment>& cast_environment, const VideoSenderConfig& video_config, const gfx::Size& frame_size, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb, const CreateVideoEncodeAcceleratorCallback& create_vea_cb, const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb); @@ -32,7 +38,6 @@ class ExternalVideoEncoder : public VideoEncoder { ~ExternalVideoEncoder() override; // VideoEncoder implementation. - bool CanEncodeVariedFrameSizes() const override; bool EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, @@ -48,15 +53,18 @@ class ExternalVideoEncoder : public VideoEncoder { // VEAClientImpl to own and interface with a new |vea|. Upon return, // |client_| holds a reference to the new VEAClientImpl. void OnCreateVideoEncodeAccelerator( - const gfx::Size& frame_size, - VideoCodecProfile codec_profile, - int max_frame_rate, + const VideoSenderConfig& video_config, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb, scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner, scoped_ptr<media::VideoEncodeAccelerator> vea); const scoped_refptr<CastEnvironment> cast_environment_; const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_; + + // The size of the visible region of the video frames to be encoded. + const gfx::Size frame_size_; + int bit_rate_; bool key_frame_requested_; @@ -69,6 +77,31 @@ class ExternalVideoEncoder : public VideoEncoder { DISALLOW_COPY_AND_ASSIGN(ExternalVideoEncoder); }; +// An implementation of SizeAdaptableVideoEncoderBase to proxy for +// ExternalVideoEncoder instances. +class SizeAdaptableExternalVideoEncoder : public SizeAdaptableVideoEncoderBase { + public: + SizeAdaptableExternalVideoEncoder( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb, + const CreateVideoEncodeAcceleratorCallback& create_vea_cb, + const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb); + + ~SizeAdaptableExternalVideoEncoder() override; + + protected: + scoped_ptr<VideoEncoder> CreateEncoder() override; + + private: + // Special callbacks needed by media::cast::ExternalVideoEncoder. + // TODO(miu): Remove these. http://crbug.com/454029 + const CreateVideoEncodeAcceleratorCallback create_vea_cb_; + const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_; + + DISALLOW_COPY_AND_ASSIGN(SizeAdaptableExternalVideoEncoder); +}; + } // namespace cast } // namespace media diff --git a/media/cast/sender/external_video_encoder_unittest.cc b/media/cast/sender/external_video_encoder_unittest.cc deleted file mode 100644 index 0ade1ed..0000000 --- a/media/cast/sender/external_video_encoder_unittest.cc +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <vector> - -#include "base/bind.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "media/base/video_frame.h" -#include "media/cast/cast_defines.h" -#include "media/cast/cast_environment.h" -#include "media/cast/sender/external_video_encoder.h" -#include "media/cast/sender/fake_video_encode_accelerator_factory.h" -#include "media/cast/test/fake_single_thread_task_runner.h" -#include "media/cast/test/utility/video_utility.h" -#include "media/video/fake_video_encode_accelerator.h" -#include "testing/gmock/include/gmock/gmock.h" - -namespace media { -namespace cast { - -using testing::_; - -namespace { - -class TestVideoEncoderCallback - : public base::RefCountedThreadSafe<TestVideoEncoderCallback> { - public: - TestVideoEncoderCallback() {} - - void SetExpectedResult(uint32 expected_frame_id, - uint32 expected_last_referenced_frame_id, - uint32 expected_rtp_timestamp, - const base::TimeTicks& expected_reference_time) { - expected_frame_id_ = expected_frame_id; - expected_last_referenced_frame_id_ = expected_last_referenced_frame_id; - expected_rtp_timestamp_ = expected_rtp_timestamp; - expected_reference_time_ = expected_reference_time; - } - - void DeliverEncodedVideoFrame( - scoped_ptr<EncodedFrame> encoded_frame) { - if (expected_frame_id_ == expected_last_referenced_frame_id_) { - EXPECT_EQ(EncodedFrame::KEY, encoded_frame->dependency); - } else { - EXPECT_EQ(EncodedFrame::DEPENDENT, - encoded_frame->dependency); - } - EXPECT_EQ(expected_frame_id_, encoded_frame->frame_id); - EXPECT_EQ(expected_last_referenced_frame_id_, - encoded_frame->referenced_frame_id); - EXPECT_EQ(expected_rtp_timestamp_, encoded_frame->rtp_timestamp); - EXPECT_EQ(expected_reference_time_, encoded_frame->reference_time); - } - - protected: - virtual ~TestVideoEncoderCallback() {} - - private: - friend class base::RefCountedThreadSafe<TestVideoEncoderCallback>; - - uint32 expected_frame_id_; - uint32 expected_last_referenced_frame_id_; - uint32 expected_rtp_timestamp_; - base::TimeTicks expected_reference_time_; - - DISALLOW_COPY_AND_ASSIGN(TestVideoEncoderCallback); -}; -} // namespace - -class ExternalVideoEncoderTest : public ::testing::Test { - protected: - ExternalVideoEncoderTest() - : testing_clock_(new base::SimpleTestTickClock()), - task_runner_(new test::FakeSingleThreadTaskRunner(testing_clock_)), - cast_environment_(new CastEnvironment( - scoped_ptr<base::TickClock>(testing_clock_).Pass(), - task_runner_, - task_runner_, - task_runner_)), - vea_factory_(task_runner_), - operational_status_(STATUS_UNINITIALIZED), - test_video_encoder_callback_(new TestVideoEncoderCallback()) { - testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); - - video_config_.ssrc = 1; - video_config_.receiver_ssrc = 2; - video_config_.rtp_payload_type = 127; - video_config_.use_external_encoder = true; - video_config_.max_bitrate = 5000000; - video_config_.min_bitrate = 1000000; - video_config_.start_bitrate = 2000000; - video_config_.max_qp = 56; - video_config_.min_qp = 0; - video_config_.max_frame_rate = 30; - video_config_.max_number_of_video_buffers_used = 3; - video_config_.codec = CODEC_VIDEO_VP8; - const gfx::Size size(320, 240); - video_frame_ = media::VideoFrame::CreateFrame( - VideoFrame::I420, size, gfx::Rect(size), size, base::TimeDelta()); - PopulateVideoFrame(video_frame_.get(), 123); - - video_encoder_.reset(new ExternalVideoEncoder( - cast_environment_, - video_config_, - size, - base::Bind(&ExternalVideoEncoderTest::SaveOperationalStatus, - base::Unretained(this)), - base::Bind( - &FakeVideoEncodeAcceleratorFactory::CreateVideoEncodeAccelerator, - base::Unretained(&vea_factory_)), - base::Bind(&FakeVideoEncodeAcceleratorFactory::CreateSharedMemory, - base::Unretained(&vea_factory_)))); - } - - ~ExternalVideoEncoderTest() override {} - - void AdvanceClockAndVideoFrameTimestamp() { - testing_clock_->Advance(base::TimeDelta::FromMilliseconds(33)); - video_frame_->set_timestamp( - video_frame_->timestamp() + base::TimeDelta::FromMilliseconds(33)); - } - - void SaveOperationalStatus(OperationalStatus result) { - EXPECT_EQ(STATUS_UNINITIALIZED, operational_status_); - operational_status_ = result; - } - - base::SimpleTestTickClock* const testing_clock_; // Owned by CastEnvironment. - const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; - const scoped_refptr<CastEnvironment> cast_environment_; - FakeVideoEncodeAcceleratorFactory vea_factory_; - OperationalStatus operational_status_; - scoped_refptr<TestVideoEncoderCallback> test_video_encoder_callback_; - VideoSenderConfig video_config_; - scoped_ptr<VideoEncoder> video_encoder_; - scoped_refptr<media::VideoFrame> video_frame_; - - DISALLOW_COPY_AND_ASSIGN(ExternalVideoEncoderTest); -}; - -TEST_F(ExternalVideoEncoderTest, EncodePattern30fpsRunningOutOfAck) { - vea_factory_.SetAutoRespond(true); - task_runner_->RunTasks(); // Run the initializer on the correct thread. - EXPECT_EQ(STATUS_INITIALIZED, operational_status_); - - VideoEncoder::FrameEncodedCallback frame_encoded_callback = - base::Bind(&TestVideoEncoderCallback::DeliverEncodedVideoFrame, - test_video_encoder_callback_.get()); - - test_video_encoder_callback_->SetExpectedResult( - 0, 0, TimeDeltaToRtpDelta(video_frame_->timestamp(), kVideoFrequency), - testing_clock_->NowTicks()); - video_encoder_->SetBitRate(2000); - EXPECT_TRUE(video_encoder_->EncodeVideoFrame( - video_frame_, testing_clock_->NowTicks(), frame_encoded_callback)); - task_runner_->RunTasks(); - - for (int i = 0; i < 6; ++i) { - AdvanceClockAndVideoFrameTimestamp(); - test_video_encoder_callback_->SetExpectedResult( - i + 1, - i, - TimeDeltaToRtpDelta(video_frame_->timestamp(), kVideoFrequency), - testing_clock_->NowTicks()); - EXPECT_TRUE(video_encoder_->EncodeVideoFrame( - video_frame_, testing_clock_->NowTicks(), frame_encoded_callback)); - task_runner_->RunTasks(); - } - - ASSERT_EQ(1u, vea_factory_.last_response_vea()->stored_bitrates().size()); - EXPECT_EQ(2000u, vea_factory_.last_response_vea()->stored_bitrates()[0]); - - // We need to run the task to cleanup the GPU instance. - video_encoder_.reset(NULL); - task_runner_->RunTasks(); - - EXPECT_EQ(1, vea_factory_.vea_response_count()); - EXPECT_EQ(3, vea_factory_.shm_response_count()); -} - -TEST_F(ExternalVideoEncoderTest, StreamHeader) { - vea_factory_.SetAutoRespond(true); - task_runner_->RunTasks(); // Run the initializer on the correct thread. - EXPECT_EQ(STATUS_INITIALIZED, operational_status_); - - VideoEncoder::FrameEncodedCallback frame_encoded_callback = - base::Bind(&TestVideoEncoderCallback::DeliverEncodedVideoFrame, - test_video_encoder_callback_.get()); - - // Force the FakeVideoEncodeAccelerator to return a dummy non-key frame first. - vea_factory_.last_response_vea()->SendDummyFrameForTesting(false); - - // Verify the first returned bitstream buffer is still a key frame. - test_video_encoder_callback_->SetExpectedResult( - 0, 0, TimeDeltaToRtpDelta(video_frame_->timestamp(), kVideoFrequency), - testing_clock_->NowTicks()); - EXPECT_TRUE(video_encoder_->EncodeVideoFrame( - video_frame_, testing_clock_->NowTicks(), frame_encoded_callback)); - task_runner_->RunTasks(); - - // We need to run the task to cleanup the GPU instance. - video_encoder_.reset(NULL); - task_runner_->RunTasks(); - - EXPECT_EQ(1, vea_factory_.vea_response_count()); - EXPECT_EQ(3, vea_factory_.shm_response_count()); -} - -// Verify that everything goes well even if ExternalVideoEncoder is destroyed -// before it has a chance to receive the VEA creation callback. -TEST_F(ExternalVideoEncoderTest, DestroyBeforeVEACreatedCallback) { - video_encoder_.reset(); - EXPECT_EQ(0, vea_factory_.vea_response_count()); - vea_factory_.SetAutoRespond(true); - task_runner_->RunTasks(); - EXPECT_EQ(1, vea_factory_.vea_response_count()); - EXPECT_EQ(STATUS_UNINITIALIZED, operational_status_); -} - -} // namespace cast -} // namespace media diff --git a/media/cast/sender/h264_vt_encoder.cc b/media/cast/sender/h264_vt_encoder.cc index 364a416..c8ae82b 100644 --- a/media/cast/sender/h264_vt_encoder.cc +++ b/media/cast/sender/h264_vt_encoder.cc @@ -13,6 +13,7 @@ #include "base/location.h" #include "base/logging.h" #include "base/macros.h" +#include "base/synchronization/lock.h" #include "media/base/mac/corevideo_glue.h" #include "media/base/mac/video_frame_mac.h" #include "media/cast/sender/video_frame_factory.h" @@ -36,25 +37,35 @@ struct InProgressFrameEncode { frame_encoded_callback(callback) {} }; +base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeysAndValues( + CFTypeRef* keys, + CFTypeRef* values, + size_t size) { + return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate( + kCFAllocatorDefault, + keys, + values, + size, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks)); +} + base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key, CFTypeRef value) { CFTypeRef keys[1] = {key}; CFTypeRef values[1] = {value}; - return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate( - kCFAllocatorDefault, keys, values, 1, &kCFTypeDictionaryKeyCallBacks, - &kCFTypeDictionaryValueCallBacks)); + return DictionaryWithKeysAndValues(keys, values, 1); } -base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const std::vector<int>& v) { +base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, size_t size) { std::vector<CFNumberRef> numbers; - numbers.reserve(v.size()); - for (const int i : v) { - numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, &i)); - } + numbers.reserve(size); + for (const int* end = v + size; v < end; ++v) + numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v)); base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate( kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]), numbers.size(), &kCFTypeArrayCallBacks)); - for (CFNumberRef number : numbers) { + for (auto& number : numbers) { CFRelease(number); } return array; @@ -202,42 +213,61 @@ void CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf, class VideoFrameFactoryCVPixelBufferPoolImpl : public VideoFrameFactory { public: VideoFrameFactoryCVPixelBufferPoolImpl( - const base::ScopedCFTypeRef<CVPixelBufferPoolRef>& pool) - : pool_(pool) {} + const base::ScopedCFTypeRef<CVPixelBufferPoolRef>& pool, + const gfx::Size& frame_size) + : pool_(pool), + frame_size_(frame_size) {} ~VideoFrameFactoryCVPixelBufferPoolImpl() override {} - scoped_refptr<VideoFrame> CreateFrame(base::TimeDelta timestamp) override { + scoped_refptr<VideoFrame> MaybeCreateFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) override { + if (frame_size != frame_size_) + return nullptr; // Buffer pool is not a match for requested frame size. + base::ScopedCFTypeRef<CVPixelBufferRef> buffer; - CHECK_EQ(kCVReturnSuccess, - CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool_, - buffer.InitializeInto())); + if (CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool_, + buffer.InitializeInto()) != + kCVReturnSuccess) + return nullptr; // Buffer pool has run out of pixel buffers. + DCHECK(buffer); + return VideoFrame::WrapCVPixelBuffer(buffer, timestamp); } private: - base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool_; + const base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool_; + const gfx::Size frame_size_; DISALLOW_COPY_AND_ASSIGN(VideoFrameFactoryCVPixelBufferPoolImpl); }; } // namespace +// static +bool H264VideoToolboxEncoder::IsSupported( + const VideoSenderConfig& video_config) { + return video_config.codec == CODEC_VIDEO_H264 && VideoToolboxGlue::Get(); +} + H264VideoToolboxEncoder::H264VideoToolboxEncoder( const scoped_refptr<CastEnvironment>& cast_environment, const VideoSenderConfig& video_config, const gfx::Size& frame_size, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb) : cast_environment_(cast_environment), videotoolbox_glue_(VideoToolboxGlue::Get()), - frame_id_(kStartFrameId), + frame_size_(frame_size), + status_change_cb_(status_change_cb), + next_frame_id_(first_frame_id), encode_next_frame_as_keyframe_(false) { - DCHECK(!frame_size.IsEmpty()); - DCHECK(!status_change_cb.is_null()); + DCHECK(!frame_size_.IsEmpty()); + DCHECK(!status_change_cb_.is_null()); OperationalStatus operational_status; if (video_config.codec == CODEC_VIDEO_H264 && videotoolbox_glue_) { - operational_status = Initialize(video_config, frame_size) ? + operational_status = Initialize(video_config) ? STATUS_INITIALIZED : STATUS_INVALID_CONFIGURATION; } else { operational_status = STATUS_UNSUPPORTED_CODEC; @@ -245,7 +275,7 @@ H264VideoToolboxEncoder::H264VideoToolboxEncoder( cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, - base::Bind(status_change_cb, operational_status)); + base::Bind(status_change_cb_, operational_status)); } H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { @@ -253,8 +283,7 @@ H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { } bool H264VideoToolboxEncoder::Initialize( - const VideoSenderConfig& video_config, - const gfx::Size& frame_size) { + const VideoSenderConfig& video_config) { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(!compression_session_); @@ -280,16 +309,40 @@ bool H264VideoToolboxEncoder::Initialize( // Certain encoders prefer kCVPixelFormatType_422YpCbCr8, which is not // supported through VideoFrame. We can force 420 formats to be used instead. const int formats[] = { - kCVPixelFormatType_420YpCbCr8Planar, - CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange}; - base::ScopedCFTypeRef<CFArrayRef> formats_array = ArrayWithIntegers( - std::vector<int>(formats, formats + arraysize(formats))); - base::ScopedCFTypeRef<CFDictionaryRef> buffer_attributes = - DictionaryWithKeyValue(kCVPixelBufferPixelFormatTypeKey, formats_array); + kCVPixelFormatType_420YpCbCr8Planar, + CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + }; + // Keep these attachment settings in-sync with those in ConfigureSession(). + CFTypeRef attachments_keys[] = { + kCVImageBufferColorPrimariesKey, + kCVImageBufferTransferFunctionKey, + kCVImageBufferYCbCrMatrixKey + }; + CFTypeRef attachments_values[] = { + kCVImageBufferColorPrimaries_ITU_R_709_2, + kCVImageBufferTransferFunction_ITU_R_709_2, + kCVImageBufferYCbCrMatrix_ITU_R_709_2 + }; + CFTypeRef buffer_attributes_keys[] = { + kCVPixelBufferPixelFormatTypeKey, + kCVBufferPropagatedAttachmentsKey + }; + CFTypeRef buffer_attributes_values[] = { + ArrayWithIntegers(formats, arraysize(formats)).release(), + DictionaryWithKeysAndValues(attachments_keys, + attachments_values, + arraysize(attachments_keys)).release() + }; + const base::ScopedCFTypeRef<CFDictionaryRef> buffer_attributes = + DictionaryWithKeysAndValues(buffer_attributes_keys, + buffer_attributes_values, + arraysize(buffer_attributes_keys)); + for (auto& v : buffer_attributes_values) + CFRelease(v); VTCompressionSessionRef session; OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate( - kCFAllocatorDefault, frame_size.width(), frame_size.height(), + kCFAllocatorDefault, frame_size_.width(), frame_size_.height(), CoreMediaGlue::kCMVideoCodecType_H264, encoder_spec, buffer_attributes, nullptr /* compressedDataAllocator */, &H264VideoToolboxEncoder::CompressionCallback, @@ -329,6 +382,7 @@ void H264VideoToolboxEncoder::ConfigureSession( SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(), video_config.max_frame_rate); + // Keep these attachment settings in-sync with those in Initialize(). SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(), kCVImageBufferColorPrimaries_ITU_R_709_2); @@ -357,16 +411,11 @@ void H264VideoToolboxEncoder::Teardown() { } } -bool H264VideoToolboxEncoder::CanEncodeVariedFrameSizes() const { - return false; -} - bool H264VideoToolboxEncoder::EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, const FrameEncodedCallback& frame_encoded_callback) { DCHECK(thread_checker_.CalledOnValidThread()); - DCHECK(!video_frame->visible_rect().IsEmpty()); DCHECK(!frame_encoded_callback.is_null()); if (!compression_session_) { @@ -374,6 +423,9 @@ bool H264VideoToolboxEncoder::EncodeVideoFrame( return false; } + if (video_frame->visible_rect().size() != frame_size_) + return false; + // Wrap the VideoFrame in a CVPixelBuffer. In all cases, no data will be // copied. If the VideoFrame was created by this encoder's video frame // factory, then the returned CVPixelBuffer will have been obtained from the @@ -435,12 +487,14 @@ void H264VideoToolboxEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) { scoped_ptr<VideoFrameFactory> H264VideoToolboxEncoder::CreateVideoFrameFactory() { + if (!videotoolbox_glue_ || !compression_session_) + return nullptr; base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool( videotoolbox_glue_->VTCompressionSessionGetPixelBufferPool( compression_session_), base::scoped_policy::RETAIN); return scoped_ptr<VideoFrameFactory>( - new VideoFrameFactoryCVPixelBufferPoolImpl(pool)); + new VideoFrameFactoryCVPixelBufferPoolImpl(pool, frame_size_)); } void H264VideoToolboxEncoder::EmitFrames() { @@ -483,31 +537,38 @@ void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, OSStatus status, VTEncodeInfoFlags info, CMSampleBufferRef sbuf) { - if (status != noErr) { - DLOG(ERROR) << " encoding failed: " << status; - return; - } - if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) { - DVLOG(2) << " frame dropped"; - return; - } - auto encoder = reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque); const scoped_ptr<InProgressFrameEncode> request( reinterpret_cast<InProgressFrameEncode*>(request_opaque)); - auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex( - CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0)); + bool keyframe = false; + bool has_frame_data = false; - // If the NotSync key is not present, it implies Sync, which indicates a - // keyframe (at least I think, VT documentation is, erm, sparse). Could - // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false. - bool keyframe = - !CFDictionaryContainsKey(sample_attachments, - CoreMediaGlue::kCMSampleAttachmentKey_NotSync()); + if (status != noErr) { + DLOG(ERROR) << " encoding failed: " << status; + encoder->cast_environment_->PostTask( + CastEnvironment::MAIN, + FROM_HERE, + base::Bind(encoder->status_change_cb_, STATUS_CODEC_RUNTIME_ERROR)); + } else if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) { + DVLOG(2) << " frame dropped"; + } else { + auto sample_attachments = static_cast<CFDictionaryRef>( + CFArrayGetValueAtIndex( + CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), + 0)); + + // If the NotSync key is not present, it implies Sync, which indicates a + // keyframe (at least I think, VT documentation is, erm, sparse). Could + // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false. + keyframe = !CFDictionaryContainsKey( + sample_attachments, + CoreMediaGlue::kCMSampleAttachmentKey_NotSync()); + has_frame_data = true; + } // Increment the encoder-scoped frame id and assign the new value to this // frame. VideoToolbox calls the output callback serially, so this is safe. - uint32 frame_id = ++encoder->frame_id_; + const uint32 frame_id = encoder->next_frame_id_++; scoped_ptr<EncodedFrame> encoded_frame(new EncodedFrame()); encoded_frame->frame_id = frame_id; @@ -530,7 +591,8 @@ void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, encoded_frame->referenced_frame_id = frame_id - 1; } - CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe); + if (has_frame_data) + CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe); encoder->cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, @@ -538,5 +600,89 @@ void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, base::Passed(&encoded_frame))); } +// A ref-counted structure that is shared to provide concurrent access to the +// VideoFrameFactory instance for the current encoder. OnEncoderReplaced() can +// change |factory| whenever an encoder instance has been replaced, while users +// of CreateVideoFrameFactory() may attempt to read/use |factory| by any thread +// at any time. +struct SizeAdaptableH264VideoToolboxVideoEncoder::FactoryHolder + : public base::RefCountedThreadSafe<FactoryHolder> { + base::Lock lock; + scoped_ptr<VideoFrameFactory> factory; + + private: + friend class base::RefCountedThreadSafe<FactoryHolder>; + ~FactoryHolder() {} +}; + +SizeAdaptableH264VideoToolboxVideoEncoder:: + SizeAdaptableH264VideoToolboxVideoEncoder( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb) + : SizeAdaptableVideoEncoderBase(cast_environment, + video_config, + status_change_cb), + holder_(new FactoryHolder()) {} + +SizeAdaptableH264VideoToolboxVideoEncoder:: + ~SizeAdaptableH264VideoToolboxVideoEncoder() {} + +// A proxy allowing SizeAdaptableH264VideoToolboxVideoEncoder to swap out the +// VideoFrameFactory instance to match one appropriate for the current encoder +// instance. +class SizeAdaptableH264VideoToolboxVideoEncoder::VideoFrameFactoryProxy + : public VideoFrameFactory { + public: + explicit VideoFrameFactoryProxy(const scoped_refptr<FactoryHolder>& holder) + : holder_(holder) {} + + ~VideoFrameFactoryProxy() override {} + + scoped_refptr<VideoFrame> MaybeCreateFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) override { + base::AutoLock auto_lock(holder_->lock); + return holder_->factory ? + holder_->factory->MaybeCreateFrame(frame_size, timestamp) : nullptr; + } + + private: + const scoped_refptr<FactoryHolder> holder_; + + DISALLOW_COPY_AND_ASSIGN(VideoFrameFactoryProxy); +}; + +scoped_ptr<VideoFrameFactory> + SizeAdaptableH264VideoToolboxVideoEncoder::CreateVideoFrameFactory() { + return scoped_ptr<VideoFrameFactory>(new VideoFrameFactoryProxy(holder_)); +} + +scoped_ptr<VideoEncoder> + SizeAdaptableH264VideoToolboxVideoEncoder::CreateEncoder() { + return scoped_ptr<VideoEncoder>(new H264VideoToolboxEncoder( + cast_environment(), + video_config(), + frame_size(), + last_frame_id() + 1, + CreateEncoderStatusChangeCallback())); +} + +void SizeAdaptableH264VideoToolboxVideoEncoder::OnEncoderReplaced( + VideoEncoder* replacement_encoder) { + scoped_ptr<VideoFrameFactory> current_factory( + replacement_encoder->CreateVideoFrameFactory()); + base::AutoLock auto_lock(holder_->lock); + holder_->factory = current_factory.Pass(); + SizeAdaptableVideoEncoderBase::OnEncoderReplaced(replacement_encoder); +} + +void SizeAdaptableH264VideoToolboxVideoEncoder::DestroyEncoder() { + { + base::AutoLock auto_lock(holder_->lock); + holder_->factory.reset(); + } + SizeAdaptableVideoEncoderBase::DestroyEncoder(); +} + } // namespace cast } // namespace media diff --git a/media/cast/sender/h264_vt_encoder.h b/media/cast/sender/h264_vt_encoder.h index 349075a..ece6ab3 100644 --- a/media/cast/sender/h264_vt_encoder.h +++ b/media/cast/sender/h264_vt_encoder.h @@ -8,6 +8,7 @@ #include "base/mac/scoped_cftyperef.h" #include "base/threading/thread_checker.h" #include "media/base/mac/videotoolbox_glue.h" +#include "media/cast/sender/size_adaptable_video_encoder_base.h" #include "media/cast/sender/video_encoder.h" namespace media { @@ -22,15 +23,19 @@ class H264VideoToolboxEncoder : public VideoEncoder { typedef VideoToolboxGlue::VTEncodeInfoFlags VTEncodeInfoFlags; public: + // Returns true if the current platform and system configuration supports + // using H264VideoToolboxEncoder with the given |video_config|. + static bool IsSupported(const VideoSenderConfig& video_config); + H264VideoToolboxEncoder( const scoped_refptr<CastEnvironment>& cast_environment, const VideoSenderConfig& video_config, const gfx::Size& frame_size, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb); ~H264VideoToolboxEncoder() override; // media::cast::VideoEncoder implementation - bool CanEncodeVariedFrameSizes() const override; bool EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, @@ -43,8 +48,7 @@ class H264VideoToolboxEncoder : public VideoEncoder { private: // Initialize the compression session. - bool Initialize(const VideoSenderConfig& video_config, - const gfx::Size& frame_size); + bool Initialize(const VideoSenderConfig& video_config); // Configure the compression session. void ConfigureSession(const VideoSenderConfig& video_config); @@ -68,7 +72,13 @@ class H264VideoToolboxEncoder : public VideoEncoder { const scoped_refptr<CastEnvironment> cast_environment_; // VideoToolboxGlue provides access to VideoToolbox at runtime. - const VideoToolboxGlue* videotoolbox_glue_; + const VideoToolboxGlue* const videotoolbox_glue_; + + // The size of the visible region of the video frames to be encoded. + const gfx::Size frame_size_; + + // Callback used to report initialization status and runtime errors. + const StatusChangeCallback status_change_cb_; // Thread checker to enforce that this object is used on a specific thread. base::ThreadChecker thread_checker_; @@ -77,7 +87,7 @@ class H264VideoToolboxEncoder : public VideoEncoder { base::ScopedCFTypeRef<VTCompressionSessionRef> compression_session_; // Frame identifier counter. - uint32 frame_id_; + uint32 next_frame_id_; // Force next frame to be a keyframe. bool encode_next_frame_as_keyframe_; @@ -85,6 +95,34 @@ class H264VideoToolboxEncoder : public VideoEncoder { DISALLOW_COPY_AND_ASSIGN(H264VideoToolboxEncoder); }; +// An implementation of SizeAdaptableVideoEncoderBase to proxy for +// H264VideoToolboxEncoder instances. +class SizeAdaptableH264VideoToolboxVideoEncoder + : public SizeAdaptableVideoEncoderBase { + public: + SizeAdaptableH264VideoToolboxVideoEncoder( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb); + + ~SizeAdaptableH264VideoToolboxVideoEncoder() override; + + scoped_ptr<VideoFrameFactory> CreateVideoFrameFactory() override; + + protected: + scoped_ptr<VideoEncoder> CreateEncoder() override; + void OnEncoderReplaced(VideoEncoder* replacement_encoder) override; + void DestroyEncoder() override; + + private: + struct FactoryHolder; + class VideoFrameFactoryProxy; + + const scoped_refptr<FactoryHolder> holder_; + + DISALLOW_COPY_AND_ASSIGN(SizeAdaptableH264VideoToolboxVideoEncoder); +}; + } // namespace cast } // namespace media diff --git a/media/cast/sender/h264_vt_encoder_unittest.cc b/media/cast/sender/h264_vt_encoder_unittest.cc index 78012eb..04f3d629 100644 --- a/media/cast/sender/h264_vt_encoder_unittest.cc +++ b/media/cast/sender/h264_vt_encoder_unittest.cc @@ -176,7 +176,9 @@ class EndToEndFrameChecker }; void CreateFrameAndMemsetPlane(VideoFrameFactory* const video_frame_factory) { - auto video_frame = video_frame_factory->CreateFrame(base::TimeDelta()); + const scoped_refptr<media::VideoFrame> video_frame = + video_frame_factory->MaybeCreateFrame( + gfx::Size(kVideoWidth, kVideoHeight), base::TimeDelta()); ASSERT_TRUE(video_frame.get()); auto cv_pixel_buffer = video_frame->cv_pixel_buffer(); ASSERT_TRUE(cv_pixel_buffer); @@ -207,6 +209,7 @@ class H264VideoToolboxEncoderTest : public ::testing::Test { cast_environment_, video_sender_config_, gfx::Size(kVideoWidth, kVideoHeight), + 0u, base::Bind(&SaveOperationalStatus, &operational_status_))); message_loop_.RunUntilIdle(); EXPECT_EQ(STATUS_INITIALIZED, operational_status_); @@ -304,6 +307,8 @@ TEST_F(H264VideoToolboxEncoderTest, CheckVideoFrameFactory) { auto video_frame_factory = encoder_->CreateVideoFrameFactory(); ASSERT_TRUE(video_frame_factory.get()); CreateFrameAndMemsetPlane(video_frame_factory.get()); + // TODO(jfroy): Need to test that the encoder can encode VideoFrames provided + // by the VideoFrameFactory. encoder_.reset(); message_loop_.RunUntilIdle(); CreateFrameAndMemsetPlane(video_frame_factory.get()); diff --git a/media/cast/sender/size_adaptable_video_encoder_base.cc b/media/cast/sender/size_adaptable_video_encoder_base.cc new file mode 100644 index 0000000..96c65a0 --- /dev/null +++ b/media/cast/sender/size_adaptable_video_encoder_base.cc @@ -0,0 +1,168 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/cast/sender/size_adaptable_video_encoder_base.h" + +#include "base/bind.h" +#include "base/location.h" +#include "media/base/video_frame.h" + +namespace media { +namespace cast { + +SizeAdaptableVideoEncoderBase::SizeAdaptableVideoEncoderBase( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb) + : cast_environment_(cast_environment), + video_config_(video_config), + status_change_cb_(status_change_cb), + frames_in_encoder_(0), + last_frame_id_(kStartFrameId), + weak_factory_(this) { + cast_environment_->PostTask( + CastEnvironment::MAIN, + FROM_HERE, + base::Bind(status_change_cb_, STATUS_INITIALIZED)); +} + +SizeAdaptableVideoEncoderBase::~SizeAdaptableVideoEncoderBase() { + DestroyEncoder(); +} + +bool SizeAdaptableVideoEncoderBase::EncodeVideoFrame( + const scoped_refptr<media::VideoFrame>& video_frame, + const base::TimeTicks& reference_time, + const FrameEncodedCallback& frame_encoded_callback) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + + const gfx::Size frame_size = video_frame->visible_rect().size(); + if (frame_size.IsEmpty()) { + DVLOG(1) << "Rejecting empty video frame."; + return false; + } + if (frames_in_encoder_ == kEncoderIsInitializing) { + VLOG(1) << "Dropping frame since encoder initialization is in-progress."; + return false; + } + if (frame_size != frame_size_ || !encoder_) { + VLOG(1) << "Dropping this frame, and future frames until a replacement " + "encoder is spun-up to handle size " << frame_size.ToString(); + TrySpawningReplacementEncoder(frame_size); + return false; + } + + const bool is_frame_accepted = encoder_->EncodeVideoFrame( + video_frame, + reference_time, + base::Bind(&SizeAdaptableVideoEncoderBase::OnEncodedVideoFrame, + weak_factory_.GetWeakPtr(), + frame_encoded_callback)); + if (is_frame_accepted) + ++frames_in_encoder_; + return is_frame_accepted; +} + +void SizeAdaptableVideoEncoderBase::SetBitRate(int new_bit_rate) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + video_config_.start_bitrate = new_bit_rate; + if (encoder_) + encoder_->SetBitRate(new_bit_rate); +} + +void SizeAdaptableVideoEncoderBase::GenerateKeyFrame() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (encoder_) + encoder_->GenerateKeyFrame(); +} + +void SizeAdaptableVideoEncoderBase::LatestFrameIdToReference(uint32 frame_id) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (encoder_) + encoder_->LatestFrameIdToReference(frame_id); +} + +scoped_ptr<VideoFrameFactory> + SizeAdaptableVideoEncoderBase::CreateVideoFrameFactory() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + return nullptr; +} + +void SizeAdaptableVideoEncoderBase::EmitFrames() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (encoder_) + encoder_->EmitFrames(); +} + +StatusChangeCallback + SizeAdaptableVideoEncoderBase::CreateEncoderStatusChangeCallback() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + return base::Bind(&SizeAdaptableVideoEncoderBase::OnEncoderStatusChange, + weak_factory_.GetWeakPtr()); +} + +void SizeAdaptableVideoEncoderBase::OnEncoderReplaced( + VideoEncoder* replacement_encoder) {} + +void SizeAdaptableVideoEncoderBase::DestroyEncoder() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + // The weak pointers are invalidated to prevent future calls back to |this|. + // This effectively cancels any of |encoder_|'s posted tasks that have not yet + // run. + weak_factory_.InvalidateWeakPtrs(); + encoder_.reset(); +} + +void SizeAdaptableVideoEncoderBase::TrySpawningReplacementEncoder( + const gfx::Size& size_needed) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + + // If prior frames are still encoding in the current encoder, let them finish + // first. + if (frames_in_encoder_ > 0) { + encoder_->EmitFrames(); + // Check again, since EmitFrames() is a synchronous operation for some + // encoders. + if (frames_in_encoder_ > 0) + return; + } + + if (frames_in_encoder_ == kEncoderIsInitializing) + return; // Already spawned. + + DestroyEncoder(); + frames_in_encoder_ = kEncoderIsInitializing; + OnEncoderStatusChange(STATUS_CODEC_REINIT_PENDING); + VLOG(1) << "Creating replacement video encoder (for frame size change from " + << frame_size_.ToString() << " to " + << size_needed.ToString() << ")."; + frame_size_ = size_needed; + encoder_ = CreateEncoder().Pass(); + DCHECK(encoder_); +} + +void SizeAdaptableVideoEncoderBase::OnEncoderStatusChange( + OperationalStatus status) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (frames_in_encoder_ == kEncoderIsInitializing && + status == STATUS_INITIALIZED) { + // Begin using the replacement encoder. + frames_in_encoder_ = 0; + OnEncoderReplaced(encoder_.get()); + } + status_change_cb_.Run(status); +} + +void SizeAdaptableVideoEncoderBase::OnEncodedVideoFrame( + const FrameEncodedCallback& frame_encoded_callback, + scoped_ptr<EncodedFrame> encoded_frame) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + --frames_in_encoder_; + DCHECK_GE(frames_in_encoder_, 0); + last_frame_id_ = encoded_frame->frame_id; + frame_encoded_callback.Run(encoded_frame.Pass()); +} + +} // namespace cast +} // namespace media diff --git a/media/cast/sender/size_adaptable_video_encoder_base.h b/media/cast/sender/size_adaptable_video_encoder_base.h new file mode 100644 index 0000000..293a684 --- /dev/null +++ b/media/cast/sender/size_adaptable_video_encoder_base.h @@ -0,0 +1,120 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_CAST_SENDER_SIZE_ADAPTABLE_VIDEO_ENCODER_BASE_H_ +#define MEDIA_CAST_SENDER_SIZE_ADAPTABLE_VIDEO_ENCODER_BASE_H_ + +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/weak_ptr.h" +#include "media/cast/cast_config.h" +#include "media/cast/cast_environment.h" +#include "media/cast/sender/video_encoder.h" +#include "ui/gfx/geometry/size.h" + +namespace media { +namespace cast { + +// Creates and owns a VideoEncoder instance. The owned instance is an +// implementation that does not support changing frame sizes, and so +// SizeAdaptableVideoEncoderBase acts as a proxy to automatically detect when +// the owned instance should be replaced with one that can handle the new frame +// size. +class SizeAdaptableVideoEncoderBase : public VideoEncoder { + public: + SizeAdaptableVideoEncoderBase( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb); + + ~SizeAdaptableVideoEncoderBase() override; + + // VideoEncoder implementation. + bool EncodeVideoFrame( + const scoped_refptr<media::VideoFrame>& video_frame, + const base::TimeTicks& reference_time, + const FrameEncodedCallback& frame_encoded_callback) override; + void SetBitRate(int new_bit_rate) override; + void GenerateKeyFrame() override; + void LatestFrameIdToReference(uint32 frame_id) override; + scoped_ptr<VideoFrameFactory> CreateVideoFrameFactory() override; + void EmitFrames() override; + + protected: + // Accessors for subclasses. + CastEnvironment* cast_environment() const { + return cast_environment_.get(); + } + const VideoSenderConfig& video_config() const { + return video_config_; + } + const gfx::Size& frame_size() const { + return frame_size_; + } + uint32 last_frame_id() const { + return last_frame_id_; + } + + // Returns a callback that calls OnEncoderStatusChange(). The callback is + // canceled by invalidating its bound weak pointer just before a replacement + // encoder is instantiated. In this scheme, OnEncoderStatusChange() can only + // be called by the most-recent encoder. + StatusChangeCallback CreateEncoderStatusChangeCallback(); + + // Overridden by subclasses to create a new encoder instance that handles + // frames of the size specified by |frame_size()|. + virtual scoped_ptr<VideoEncoder> CreateEncoder() = 0; + + // Overridden by subclasses to perform additional steps when + // |replacement_encoder| becomes the active encoder. + virtual void OnEncoderReplaced(VideoEncoder* replacement_encoder); + + // Overridden by subclasses to perform additional steps before/after the + // current encoder is destroyed. + virtual void DestroyEncoder(); + + private: + // Create and initialize a replacement video encoder, if this not already + // in-progress. The replacement will call back to OnEncoderStatusChange() + // with success/fail status. + void TrySpawningReplacementEncoder(const gfx::Size& size_needed); + + // Called when a status change is received from an encoder. + void OnEncoderStatusChange(OperationalStatus status); + + // Called by the |encoder_| with the next EncodedFrame. + void OnEncodedVideoFrame(const FrameEncodedCallback& frame_encoded_callback, + scoped_ptr<EncodedFrame> encoded_frame); + + const scoped_refptr<CastEnvironment> cast_environment_; + + // This is not const since |video_config_.starting_bitrate| is modified by + // SetBitRate(), for when a replacement encoder is spawned. + VideoSenderConfig video_config_; + + // Run whenever the underlying encoder reports a status change. + const StatusChangeCallback status_change_cb_; + + // The underlying platform video encoder and the frame size it expects. + scoped_ptr<VideoEncoder> encoder_; + gfx::Size frame_size_; + + // The number of frames in |encoder_|'s pipeline. If this is set to + // kEncoderIsInitializing, |encoder_| is not yet ready to accept frames. + enum { kEncoderIsInitializing = -1 }; + int frames_in_encoder_; + + // The ID of the last frame that was emitted from |encoder_|. + uint32 last_frame_id_; + + // NOTE: Weak pointers must be invalidated before all other member variables. + base::WeakPtrFactory<SizeAdaptableVideoEncoderBase> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(SizeAdaptableVideoEncoderBase); +}; + +} // namespace cast +} // namespace media + +#endif // MEDIA_CAST_SENDER_SIZE_ADAPTABLE_VIDEO_ENCODER_BASE_H_ diff --git a/media/cast/sender/video_encoder.cc b/media/cast/sender/video_encoder.cc index d6b62a8..d65bf3c 100644 --- a/media/cast/sender/video_encoder.cc +++ b/media/cast/sender/video_encoder.cc @@ -3,11 +3,61 @@ // found in the LICENSE file. #include "media/cast/sender/video_encoder.h" -#include "media/cast/sender/video_frame_factory.h" + +#include "media/cast/sender/external_video_encoder.h" +#include "media/cast/sender/video_encoder_impl.h" + +#if defined(OS_MACOSX) +#include "media/cast/sender/h264_vt_encoder.h" +#endif namespace media { namespace cast { +// static +scoped_ptr<VideoEncoder> VideoEncoder::Create( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb, + const CreateVideoEncodeAcceleratorCallback& create_vea_cb, + const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb) { + // On MacOS or IOS, attempt to use the system VideoToolbox library to + // perform optimized H.264 encoding. +#if defined(OS_MACOSX) || defined(OS_IOS) + if (!video_config.use_external_encoder && + H264VideoToolboxEncoder::IsSupported(video_config)) { + return scoped_ptr<VideoEncoder>( + new SizeAdaptableH264VideoToolboxVideoEncoder( + cast_environment, + video_config, + status_change_cb)); + } +#endif // defined(OS_MACOSX) + + // If the system provides a hardware-accelerated encoder, use it. +#if !defined(OS_IOS) + if (ExternalVideoEncoder::IsSupported(video_config)) { + return scoped_ptr<VideoEncoder>(new SizeAdaptableExternalVideoEncoder( + cast_environment, + video_config, + status_change_cb, + create_vea_cb, + create_video_encode_memory_cb)); + } +#endif // !defined(OS_IOS) + + // Attempt to use the software encoder implementation. + if (VideoEncoderImpl::IsSupported(video_config)) { + return scoped_ptr<VideoEncoder>(new VideoEncoderImpl( + cast_environment, + video_config, + status_change_cb)); + } + + // No encoder implementation will suffice. + return nullptr; +} + scoped_ptr<VideoFrameFactory> VideoEncoder::CreateVideoFrameFactory() { return nullptr; } diff --git a/media/cast/sender/video_encoder.h b/media/cast/sender/video_encoder.h index 92e8547..94b0bbf 100644 --- a/media/cast/sender/video_encoder.h +++ b/media/cast/sender/video_encoder.h @@ -8,27 +8,37 @@ #include "base/callback.h" #include "base/memory/ref_counted.h" #include "base/memory/scoped_ptr.h" -#include "base/memory/weak_ptr.h" #include "base/time/time.h" #include "media/base/video_frame.h" #include "media/cast/cast_config.h" #include "media/cast/cast_environment.h" +#include "media/cast/sender/video_frame_factory.h" namespace media { namespace cast { -class VideoFrameFactory; - // All these functions are called from the main cast thread. class VideoEncoder { public: typedef base::Callback<void(scoped_ptr<EncodedFrame>)> FrameEncodedCallback; - virtual ~VideoEncoder() {} + // Creates a VideoEncoder instance from the given |video_config| and based on + // the current platform's hardware/library support; or null if no + // implementation will suffice. The instance will run |status_change_cb| at + // some point in the future to indicate initialization success/failure. + // + // All VideoEncoder instances returned by this function support encoding + // sequences of differently-size VideoFrames. + // + // TODO(miu): Remove the CreateVEA callbacks. http://crbug.com/454029 + static scoped_ptr<VideoEncoder> Create( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb, + const CreateVideoEncodeAcceleratorCallback& create_vea_cb, + const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb); - // Returns true if the size of video frames passed in successive calls to - // EncodedVideoFrame() can vary. - virtual bool CanEncodeVariedFrameSizes() const = 0; + virtual ~VideoEncoder() {} // If true is returned, the Encoder has accepted the request and will process // it asynchronously, running |frame_encoded_callback| on the MAIN diff --git a/media/cast/sender/video_encoder_impl.cc b/media/cast/sender/video_encoder_impl.cc index a2b0da9..84f700e 100644 --- a/media/cast/sender/video_encoder_impl.cc +++ b/media/cast/sender/video_encoder_impl.cc @@ -52,6 +52,15 @@ void EncodeVideoFrameOnEncoderThread( } } // namespace +// static +bool VideoEncoderImpl::IsSupported(const VideoSenderConfig& video_config) { +#ifndef OFFICIAL_BUILD + if (video_config.codec == CODEC_VIDEO_FAKE) + return true; +#endif + return video_config.codec == CODEC_VIDEO_VP8; +} + VideoEncoderImpl::VideoEncoderImpl( scoped_refptr<CastEnvironment> cast_environment, const VideoSenderConfig& video_config, @@ -98,12 +107,6 @@ VideoEncoderImpl::~VideoEncoderImpl() { } } -bool VideoEncoderImpl::CanEncodeVariedFrameSizes() const { - // Both the VP8Encoder and FakeSoftwareVideoEncoder support calls to - // EncodeVideoFrame() with different frame sizes. - return true; -} - bool VideoEncoderImpl::EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, diff --git a/media/cast/sender/video_encoder_impl.h b/media/cast/sender/video_encoder_impl.h index 07ed999..7572c4a 100644 --- a/media/cast/sender/video_encoder_impl.h +++ b/media/cast/sender/video_encoder_impl.h @@ -29,6 +29,9 @@ class VideoEncoderImpl : public VideoEncoder { typedef base::Callback<void(scoped_ptr<EncodedFrame>)> FrameEncodedCallback; + // Returns true if VideoEncoderImpl can be used with the given |video_config|. + static bool IsSupported(const VideoSenderConfig& video_config); + VideoEncoderImpl(scoped_refptr<CastEnvironment> cast_environment, const VideoSenderConfig& video_config, const StatusChangeCallback& status_change_cb); @@ -36,7 +39,6 @@ class VideoEncoderImpl : public VideoEncoder { ~VideoEncoderImpl() override; // VideoEncoder implementation. - bool CanEncodeVariedFrameSizes() const override; bool EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, diff --git a/media/cast/sender/video_encoder_impl_unittest.cc b/media/cast/sender/video_encoder_impl_unittest.cc deleted file mode 100644 index dab16ea..0000000 --- a/media/cast/sender/video_encoder_impl_unittest.cc +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <vector> - -#include "base/bind.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "media/base/video_frame.h" -#include "media/cast/cast_defines.h" -#include "media/cast/cast_environment.h" -#include "media/cast/sender/video_encoder_impl.h" -#include "media/cast/test/fake_single_thread_task_runner.h" -#include "media/cast/test/utility/default_config.h" -#include "media/cast/test/utility/video_utility.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace media { -namespace cast { - -class VideoEncoderImplTest : public ::testing::TestWithParam<Codec> { - protected: - VideoEncoderImplTest() - : testing_clock_(new base::SimpleTestTickClock()), - task_runner_(new test::FakeSingleThreadTaskRunner(testing_clock_)), - cast_environment_(new CastEnvironment( - scoped_ptr<base::TickClock>(testing_clock_).Pass(), - task_runner_, - task_runner_, - task_runner_)), - video_config_(GetDefaultVideoSenderConfig()), - operational_status_(STATUS_UNINITIALIZED), - count_frames_delivered_(0) { - testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); - first_frame_time_ = testing_clock_->NowTicks(); - } - - ~VideoEncoderImplTest() override {} - - void SetUp() override { - video_config_.codec = GetParam(); - } - - void TearDown() override { - video_encoder_.reset(); - task_runner_->RunTasks(); - } - - void CreateEncoder(bool three_buffer_mode) { - ASSERT_EQ(STATUS_UNINITIALIZED, operational_status_); - video_config_.max_number_of_video_buffers_used = - (three_buffer_mode ? 3 : 1); - video_encoder_.reset(new VideoEncoderImpl( - cast_environment_, - video_config_, - base::Bind(&VideoEncoderImplTest::OnOperationalStatusChange, - base::Unretained(this)))); - task_runner_->RunTasks(); - ASSERT_EQ(STATUS_INITIALIZED, operational_status_); - } - - VideoEncoder* video_encoder() const { - return video_encoder_.get(); - } - - void AdvanceClock() { - testing_clock_->Advance(base::TimeDelta::FromMilliseconds(33)); - } - - base::TimeTicks Now() const { - return testing_clock_->NowTicks(); - } - - void RunTasks() const { - return task_runner_->RunTasks(); - } - - int count_frames_delivered() const { - return count_frames_delivered_; - } - - // Return a callback that, when run, expects the EncodedFrame to have the - // given properties. - VideoEncoder::FrameEncodedCallback CreateFrameDeliverCallback( - uint32 expected_frame_id, - uint32 expected_last_referenced_frame_id, - uint32 expected_rtp_timestamp, - const base::TimeTicks& expected_reference_time) { - return base::Bind(&VideoEncoderImplTest::DeliverEncodedVideoFrame, - base::Unretained(this), - expected_frame_id, - expected_last_referenced_frame_id, - expected_rtp_timestamp, - expected_reference_time); - } - - // Creates a new VideoFrame of the given |size|, filled with a test pattern. - scoped_refptr<media::VideoFrame> CreateTestVideoFrame( - const gfx::Size& size) const { - const scoped_refptr<media::VideoFrame> frame = - media::VideoFrame::CreateFrame( - VideoFrame::I420, size, gfx::Rect(size), size, - testing_clock_->NowTicks() - first_frame_time_); - PopulateVideoFrame(frame.get(), 123); - return frame; - } - - private: - void OnOperationalStatusChange(OperationalStatus status) { - operational_status_ = status; - } - - // Checks that |encoded_frame| matches expected values. This is the method - // bound in the callback returned from CreateFrameDeliverCallback(). - void DeliverEncodedVideoFrame( - uint32 expected_frame_id, - uint32 expected_last_referenced_frame_id, - uint32 expected_rtp_timestamp, - const base::TimeTicks& expected_reference_time, - scoped_ptr<EncodedFrame> encoded_frame) { - if (expected_frame_id != expected_last_referenced_frame_id) { - EXPECT_EQ(EncodedFrame::DEPENDENT, encoded_frame->dependency); - } else if (video_config_.max_number_of_video_buffers_used == 1) { - EXPECT_EQ(EncodedFrame::KEY, encoded_frame->dependency); - } - EXPECT_EQ(expected_frame_id, encoded_frame->frame_id); - EXPECT_EQ(expected_last_referenced_frame_id, - encoded_frame->referenced_frame_id) - << "frame id: " << expected_frame_id; - EXPECT_EQ(expected_rtp_timestamp, encoded_frame->rtp_timestamp); - EXPECT_EQ(expected_reference_time, encoded_frame->reference_time); - EXPECT_FALSE(encoded_frame->data.empty()); - ++count_frames_delivered_; - } - - base::SimpleTestTickClock* const testing_clock_; // Owned by CastEnvironment. - const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; - const scoped_refptr<CastEnvironment> cast_environment_; - VideoSenderConfig video_config_; - base::TimeTicks first_frame_time_; - OperationalStatus operational_status_; - scoped_ptr<VideoEncoder> video_encoder_; - - int count_frames_delivered_; - - DISALLOW_COPY_AND_ASSIGN(VideoEncoderImplTest); -}; - -// A simple test to encode ten frames of video, expecting to see one key frame -// followed by nine delta frames. -TEST_P(VideoEncoderImplTest, GeneratesKeyFrameThenOnlyDeltaFrames) { - CreateEncoder(false); - - EXPECT_EQ(0, count_frames_delivered()); - - scoped_refptr<media::VideoFrame> video_frame = - CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - 0, 0, TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - - for (uint32 frame_id = 1; frame_id < 10; ++frame_id) { - AdvanceClock(); - video_frame = CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - frame_id, frame_id - 1, - TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - } - - EXPECT_EQ(10, count_frames_delivered()); -} - -// Tests basic frame dependency rules when using the VP8 encoder in multi-buffer -// mode. -TEST_P(VideoEncoderImplTest, - FramesDoNotDependOnUnackedFramesInMultiBufferMode) { - if (GetParam() != CODEC_VIDEO_VP8) - return; // Only test multibuffer mode for the VP8 encoder. - CreateEncoder(true); - - EXPECT_EQ(0, count_frames_delivered()); - - scoped_refptr<media::VideoFrame> video_frame = - CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - 0, 0, TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - - AdvanceClock(); - video_encoder()->LatestFrameIdToReference(0); - video_frame = CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - 1, 0, TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - - AdvanceClock(); - video_encoder()->LatestFrameIdToReference(1); - video_frame = CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - 2, 1, TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - - video_encoder()->LatestFrameIdToReference(2); - - for (uint32 frame_id = 3; frame_id < 10; ++frame_id) { - AdvanceClock(); - video_frame = CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - frame_id, 2, - TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - } - - EXPECT_EQ(10, count_frames_delivered()); -} - -// Tests that the encoder continues to output EncodedFrames as the frame size -// changes. See media/cast/receiver/video_decoder_unittest.cc for a complete -// encode/decode cycle of varied frame sizes that actually checks the frame -// content. -TEST_P(VideoEncoderImplTest, EncodesVariedFrameSizes) { - CreateEncoder(false); - ASSERT_TRUE(video_encoder()->CanEncodeVariedFrameSizes()); - - EXPECT_EQ(0, count_frames_delivered()); - - std::vector<gfx::Size> frame_sizes; - frame_sizes.push_back(gfx::Size(1280, 720)); - frame_sizes.push_back(gfx::Size(640, 360)); // Shrink both dimensions. - frame_sizes.push_back(gfx::Size(300, 200)); // Shrink both dimensions again. - frame_sizes.push_back(gfx::Size(200, 300)); // Same area. - frame_sizes.push_back(gfx::Size(600, 400)); // Grow both dimensions. - frame_sizes.push_back(gfx::Size(638, 400)); // Shrink only one dimension. - frame_sizes.push_back(gfx::Size(638, 398)); // Shrink the other dimension. - frame_sizes.push_back(gfx::Size(320, 180)); // Shrink both dimensions again. - frame_sizes.push_back(gfx::Size(322, 180)); // Grow only one dimension. - frame_sizes.push_back(gfx::Size(322, 182)); // Grow the other dimension. - frame_sizes.push_back(gfx::Size(1920, 1080)); // Grow both dimensions again. - - uint32 frame_id = 0; - - // Encode one frame at each size. Expect nothing but key frames to come out. - for (const auto& frame_size : frame_sizes) { - AdvanceClock(); - const scoped_refptr<media::VideoFrame> video_frame = - CreateTestVideoFrame(frame_size); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - frame_id, - frame_id, - TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - ++frame_id; - } - - // Encode 10 frames at each size. Expect one key frame followed by nine delta - // frames for each frame size. - for (const auto& frame_size : frame_sizes) { - for (int i = 0; i < 10; ++i) { - AdvanceClock(); - const scoped_refptr<media::VideoFrame> video_frame = - CreateTestVideoFrame(frame_size); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - frame_id, - i == 0 ? frame_id : frame_id - 1, - TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - ++frame_id; - } - } - - EXPECT_EQ(static_cast<int>(frame_id), count_frames_delivered()); -} - -INSTANTIATE_TEST_CASE_P(, - VideoEncoderImplTest, - ::testing::Values(CODEC_VIDEO_FAKE, CODEC_VIDEO_VP8)); - -} // namespace cast -} // namespace media diff --git a/media/cast/sender/video_encoder_unittest.cc b/media/cast/sender/video_encoder_unittest.cc new file mode 100644 index 0000000..a528e28 --- /dev/null +++ b/media/cast/sender/video_encoder_unittest.cc @@ -0,0 +1,454 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <vector> +#include <utility> + +#include "base/bind.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "media/base/video_frame.h" +#include "media/cast/cast_defines.h" +#include "media/cast/cast_environment.h" +#include "media/cast/sender/fake_video_encode_accelerator_factory.h" +#include "media/cast/sender/video_frame_factory.h" +#include "media/cast/sender/video_encoder.h" +#include "media/cast/test/fake_single_thread_task_runner.h" +#include "media/cast/test/utility/default_config.h" +#include "media/cast/test/utility/video_utility.h" +#include "testing/gtest/include/gtest/gtest.h" + +#if defined(OS_MACOSX) +#include "media/cast/sender/h264_vt_encoder.h" +#endif + +namespace media { +namespace cast { + +class VideoEncoderTest + : public ::testing::TestWithParam<std::pair<Codec, bool>> { + protected: + VideoEncoderTest() + : testing_clock_(new base::SimpleTestTickClock()), + task_runner_(new test::FakeSingleThreadTaskRunner(testing_clock_)), + cast_environment_(new CastEnvironment( + scoped_ptr<base::TickClock>(testing_clock_).Pass(), + task_runner_, + task_runner_, + task_runner_)), + video_config_(GetDefaultVideoSenderConfig()), + operational_status_(STATUS_UNINITIALIZED), + count_frames_delivered_(0) { + testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); + first_frame_time_ = testing_clock_->NowTicks(); + } + + ~VideoEncoderTest() override {} + + void SetUp() override { + video_config_.codec = GetParam().first; + video_config_.use_external_encoder = GetParam().second; + + if (video_config_.use_external_encoder) + vea_factory_.reset(new FakeVideoEncodeAcceleratorFactory(task_runner_)); + } + + void TearDown() override { + video_encoder_.reset(); + RunTasksAndAdvanceClock(); + } + + void CreateEncoder(bool three_buffer_mode) { + ASSERT_EQ(STATUS_UNINITIALIZED, operational_status_); + video_config_.max_number_of_video_buffers_used = + (three_buffer_mode ? 3 : 1); + video_encoder_ = VideoEncoder::Create( + cast_environment_, + video_config_, + base::Bind(&VideoEncoderTest::OnOperationalStatusChange, + base::Unretained(this)), + base::Bind( + &FakeVideoEncodeAcceleratorFactory::CreateVideoEncodeAccelerator, + base::Unretained(vea_factory_.get())), + base::Bind(&FakeVideoEncodeAcceleratorFactory::CreateSharedMemory, + base::Unretained(vea_factory_.get()))).Pass(); + RunTasksAndAdvanceClock(); + if (is_encoder_present()) + ASSERT_EQ(STATUS_INITIALIZED, operational_status_); + } + + bool is_encoder_present() const { + return !!video_encoder_; + } + + bool is_testing_software_vp8_encoder() const { + return video_config_.codec == CODEC_VIDEO_VP8 && + !video_config_.use_external_encoder; + } + + bool is_testing_video_toolbox_encoder() const { + return +#if defined(OS_MACOSX) + (!video_config_.use_external_encoder && + H264VideoToolboxEncoder::IsSupported(video_config_)) || +#endif + false; + } + + bool is_testing_platform_encoder() const { + return video_config_.use_external_encoder || + is_testing_video_toolbox_encoder(); + } + + VideoEncoder* video_encoder() const { + return video_encoder_.get(); + } + + void DestroyEncoder() { + video_encoder_.reset(); + } + + base::TimeTicks Now() const { + return testing_clock_->NowTicks(); + } + + void RunTasksAndAdvanceClock() const { + const base::TimeDelta frame_duration = base::TimeDelta::FromMicroseconds( + 1000000.0 / video_config_.max_frame_rate); +#if defined(OS_MACOSX) + if (is_testing_video_toolbox_encoder()) { + // The H264VideoToolboxEncoder (on MAC_OSX and IOS) is not a faked + // implementation in these tests, and performs its encoding asynchronously + // on an unknown set of threads. Therefore, sleep the current thread for + // the real amount of time to avoid excessively spinning the CPU while + // waiting for something to happen. + base::PlatformThread::Sleep(frame_duration); + } +#endif + task_runner_->RunTasks(); + testing_clock_->Advance(frame_duration); + } + + int count_frames_delivered() const { + return count_frames_delivered_; + } + + void WaitForAllFramesToBeDelivered(int total_expected) const { + video_encoder_->EmitFrames(); + while (count_frames_delivered_ < total_expected) + RunTasksAndAdvanceClock(); + } + + // Creates a new VideoFrame of the given |size|, filled with a test pattern. + // When available, it attempts to use the VideoFrameFactory provided by the + // encoder. + scoped_refptr<media::VideoFrame> CreateTestVideoFrame(const gfx::Size& size) { + const base::TimeDelta timestamp = + testing_clock_->NowTicks() - first_frame_time_; + scoped_refptr<media::VideoFrame> frame; + if (video_frame_factory_) + frame = video_frame_factory_->MaybeCreateFrame(size, timestamp); + if (!frame) { + frame = media::VideoFrame::CreateFrame( + VideoFrame::I420, size, gfx::Rect(size), size, timestamp); + } + PopulateVideoFrame(frame.get(), 123); + return frame; + } + + // Requests encoding the |video_frame| and has the resulting frame delivered + // via a callback that checks for expected results. Returns false if the + // encoder rejected the request. + bool EncodeAndCheckDelivery( + const scoped_refptr<media::VideoFrame>& video_frame, + uint32 frame_id, + uint32 reference_frame_id) { + return video_encoder_->EncodeVideoFrame( + video_frame, + Now(), + base::Bind(&VideoEncoderTest::DeliverEncodedVideoFrame, + base::Unretained(this), + frame_id, + reference_frame_id, + TimeDeltaToRtpDelta(video_frame->timestamp(), + kVideoFrequency), + Now())); + } + + // If the implementation of |video_encoder_| is ExternalVideoEncoder, check + // that the VEA factory has responded (by running the callbacks) a specific + // number of times. Otherwise, check that the VEA factory is inactive. + void ExpectVEAResponsesForExternalVideoEncoder( + int vea_response_count, + int shm_response_count) const { + if (!vea_factory_) + return; + EXPECT_EQ(vea_response_count, vea_factory_->vea_response_count()); + EXPECT_EQ(shm_response_count, vea_factory_->shm_response_count()); + } + + void SetVEAFactoryAutoRespond(bool auto_respond) { + if (vea_factory_) + vea_factory_->SetAutoRespond(auto_respond); + } + + private: + void OnOperationalStatusChange(OperationalStatus status) { + DVLOG(1) << "OnOperationalStatusChange: from " << operational_status_ + << " to " << status; + operational_status_ = status; + + EXPECT_TRUE(operational_status_ == STATUS_CODEC_REINIT_PENDING || + operational_status_ == STATUS_INITIALIZED); + + // Create the VideoFrameFactory the first time status changes to + // STATUS_INITIALIZED. + if (operational_status_ == STATUS_INITIALIZED && !video_frame_factory_) + video_frame_factory_ = video_encoder_->CreateVideoFrameFactory().Pass(); + } + + // Checks that |encoded_frame| matches expected values. This is the method + // bound in the callback returned from EncodeAndCheckDelivery(). + void DeliverEncodedVideoFrame( + uint32 expected_frame_id, + uint32 expected_last_referenced_frame_id, + uint32 expected_rtp_timestamp, + const base::TimeTicks& expected_reference_time, + scoped_ptr<EncodedFrame> encoded_frame) { + EXPECT_TRUE(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + + EXPECT_EQ(expected_frame_id, encoded_frame->frame_id); + EXPECT_EQ(expected_rtp_timestamp, encoded_frame->rtp_timestamp); + EXPECT_EQ(expected_reference_time, encoded_frame->reference_time); + + // The platform encoders are "black boxes" and may choose to vend key frames + // and/or empty data at any time. The software encoders, however, should + // strictly adhere to expected behavior. + if (is_testing_platform_encoder()) { + const bool expected_key_frame = + expected_frame_id == expected_last_referenced_frame_id; + const bool have_key_frame = + encoded_frame->dependency == EncodedFrame::KEY; + EXPECT_EQ(have_key_frame, + encoded_frame->frame_id == encoded_frame->referenced_frame_id); + LOG_IF(WARNING, expected_key_frame != have_key_frame) + << "Platform encoder chose to emit a " + << (have_key_frame ? "key" : "delta") + << " frame instead of the expected kind @ frame_id=" + << encoded_frame->frame_id; + LOG_IF(WARNING, encoded_frame->data.empty()) + << "Platform encoder returned an empty frame @ frame_id=" + << encoded_frame->frame_id; + } else { + if (expected_frame_id != expected_last_referenced_frame_id) { + EXPECT_EQ(EncodedFrame::DEPENDENT, encoded_frame->dependency); + } else if (video_config_.max_number_of_video_buffers_used == 1) { + EXPECT_EQ(EncodedFrame::KEY, encoded_frame->dependency); + } + EXPECT_EQ(expected_last_referenced_frame_id, + encoded_frame->referenced_frame_id); + EXPECT_FALSE(encoded_frame->data.empty()); + } + + ++count_frames_delivered_; + } + + base::SimpleTestTickClock* const testing_clock_; // Owned by CastEnvironment. + const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; + const scoped_refptr<CastEnvironment> cast_environment_; + VideoSenderConfig video_config_; + scoped_ptr<FakeVideoEncodeAcceleratorFactory> vea_factory_; + base::TimeTicks first_frame_time_; + OperationalStatus operational_status_; + scoped_ptr<VideoEncoder> video_encoder_; + scoped_ptr<VideoFrameFactory> video_frame_factory_; + + int count_frames_delivered_; + + DISALLOW_COPY_AND_ASSIGN(VideoEncoderTest); +}; + +// A simple test to encode ten frames of video, expecting to see one key frame +// followed by nine delta frames. +TEST_P(VideoEncoderTest, GeneratesKeyFrameThenOnlyDeltaFrames) { + CreateEncoder(false); + SetVEAFactoryAutoRespond(true); + + EXPECT_EQ(0, count_frames_delivered()); + ExpectVEAResponsesForExternalVideoEncoder(0, 0); + + uint32 frame_id = 0; + uint32 reference_frame_id = 0; + const gfx::Size frame_size(1280, 720); + + // For the platform encoders, the first one or more frames is dropped while + // the encoder initializes. Then, for all encoders, expect one key frame is + // delivered. + bool accepted_first_frame = false; + do { + accepted_first_frame = EncodeAndCheckDelivery( + CreateTestVideoFrame(frame_size), frame_id, reference_frame_id); + if (!is_testing_platform_encoder()) + EXPECT_TRUE(accepted_first_frame); + RunTasksAndAdvanceClock(); + } while (!accepted_first_frame); + ExpectVEAResponsesForExternalVideoEncoder(1, 3); + + // Expect the remaining frames are encoded as delta frames. + for (++frame_id; frame_id < 10; ++frame_id, ++reference_frame_id) { + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), + frame_id, + reference_frame_id)); + RunTasksAndAdvanceClock(); + } + + WaitForAllFramesToBeDelivered(10); + ExpectVEAResponsesForExternalVideoEncoder(1, 3); +} + +// Tests basic frame dependency rules when using the VP8 encoder in multi-buffer +// mode. +TEST_P(VideoEncoderTest, FramesDoNotDependOnUnackedFramesInMultiBufferMode) { + if (!is_testing_software_vp8_encoder()) + return; // Only test multibuffer mode for the software VP8 encoder. + CreateEncoder(true); + + EXPECT_EQ(0, count_frames_delivered()); + + const gfx::Size frame_size(1280, 720); + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 0, 0)); + RunTasksAndAdvanceClock(); + + video_encoder()->LatestFrameIdToReference(0); + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 1, 0)); + RunTasksAndAdvanceClock(); + + video_encoder()->LatestFrameIdToReference(1); + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 2, 1)); + RunTasksAndAdvanceClock(); + + video_encoder()->LatestFrameIdToReference(2); + + for (uint32 frame_id = 3; frame_id < 10; ++frame_id) { + EXPECT_TRUE(EncodeAndCheckDelivery( + CreateTestVideoFrame(frame_size), frame_id, 2)); + RunTasksAndAdvanceClock(); + } + + EXPECT_EQ(10, count_frames_delivered()); +} + +// Tests that the encoder continues to output EncodedFrames as the frame size +// changes. See media/cast/receiver/video_decoder_unittest.cc for a complete +// encode/decode cycle of varied frame sizes that actually checks the frame +// content. +TEST_P(VideoEncoderTest, EncodesVariedFrameSizes) { + CreateEncoder(false); + SetVEAFactoryAutoRespond(true); + + EXPECT_EQ(0, count_frames_delivered()); + ExpectVEAResponsesForExternalVideoEncoder(0, 0); + + std::vector<gfx::Size> frame_sizes; + frame_sizes.push_back(gfx::Size(1280, 720)); + frame_sizes.push_back(gfx::Size(640, 360)); // Shrink both dimensions. + frame_sizes.push_back(gfx::Size(300, 200)); // Shrink both dimensions again. + frame_sizes.push_back(gfx::Size(200, 300)); // Same area. + frame_sizes.push_back(gfx::Size(600, 400)); // Grow both dimensions. + frame_sizes.push_back(gfx::Size(638, 400)); // Shrink only one dimension. + frame_sizes.push_back(gfx::Size(638, 398)); // Shrink the other dimension. + frame_sizes.push_back(gfx::Size(320, 180)); // Shrink both dimensions again. + frame_sizes.push_back(gfx::Size(322, 180)); // Grow only one dimension. + frame_sizes.push_back(gfx::Size(322, 182)); // Grow the other dimension. + frame_sizes.push_back(gfx::Size(1920, 1080)); // Grow both dimensions again. + + uint32 frame_id = 0; + + // Encode one frame at each size. For the platform encoders, expect no frames + // to be delivered since each frame size change will sprun re-initialization + // of the underlying encoder. Otherwise, expect all key frames to come out. + for (const auto& frame_size : frame_sizes) { + EXPECT_EQ(!is_testing_platform_encoder(), + EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), + frame_id, + frame_id)); + RunTasksAndAdvanceClock(); + if (!is_testing_platform_encoder()) + ++frame_id; + } + + // Encode 10+ frames at each size. For the platform decoders, expect the + // first one or more frames are dropped while the encoder re-inits. Then, for + // all encoders, expect one key frame followed by all delta frames. + for (const auto& frame_size : frame_sizes) { + bool accepted_first_frame = false; + do { + accepted_first_frame = EncodeAndCheckDelivery( + CreateTestVideoFrame(frame_size), frame_id, frame_id); + if (!is_testing_platform_encoder()) + EXPECT_TRUE(accepted_first_frame); + RunTasksAndAdvanceClock(); + } while (!accepted_first_frame); + ++frame_id; + for (int i = 1; i < 10; ++i, ++frame_id) { + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), + frame_id, + frame_id - 1)); + RunTasksAndAdvanceClock(); + } + } + + WaitForAllFramesToBeDelivered(10 * frame_sizes.size()); + ExpectVEAResponsesForExternalVideoEncoder( + 2 * frame_sizes.size(), 6 * frame_sizes.size()); +} + +// Verify that everything goes well even if ExternalVideoEncoder is destroyed +// before it has a chance to receive the VEA creation callback. For all other +// encoders, this tests that the encoder can be safely destroyed before the task +// is run that delivers the first EncodedFrame. +TEST_P(VideoEncoderTest, CanBeDestroyedBeforeVEAIsCreated) { + CreateEncoder(false); + + // Send a frame to spawn creation of the ExternalVideoEncoder instance. + EncodeAndCheckDelivery(CreateTestVideoFrame(gfx::Size(1280, 720)), 0, 0); + + // Destroy the encoder, and confirm the VEA Factory did not respond yet. + DestroyEncoder(); + ExpectVEAResponsesForExternalVideoEncoder(0, 0); + + // Allow the VEA Factory to respond by running the creation callback. When + // the task runs, it will be a no-op since the weak pointers to the + // ExternalVideoEncoder were invalidated. + SetVEAFactoryAutoRespond(true); + RunTasksAndAdvanceClock(); + ExpectVEAResponsesForExternalVideoEncoder(1, 0); +} + +namespace { +std::vector<std::pair<Codec, bool>> DetermineEncodersToTest() { + std::vector<std::pair<Codec, bool>> values; + // Fake encoder. + values.push_back(std::make_pair(CODEC_VIDEO_FAKE, false)); + // Software VP8 encoder. + values.push_back(std::make_pair(CODEC_VIDEO_VP8, false)); + // Hardware-accelerated encoder (faked). + values.push_back(std::make_pair(CODEC_VIDEO_VP8, true)); +#if defined(OS_MACOSX) + // VideoToolbox encoder (when VideoToolbox is present). + VideoSenderConfig video_config = GetDefaultVideoSenderConfig(); + video_config.use_external_encoder = false; + video_config.codec = CODEC_VIDEO_H264; + if (H264VideoToolboxEncoder::IsSupported(video_config)) + values.push_back(std::make_pair(CODEC_VIDEO_H264, false)); +#endif + return values; +} +} // namespace + +INSTANTIATE_TEST_CASE_P( + , VideoEncoderTest, ::testing::ValuesIn(DetermineEncodersToTest())); + +} // namespace cast +} // namespace media diff --git a/media/cast/sender/video_frame_factory.h b/media/cast/sender/video_frame_factory.h index 3c23f88..d55deb7 100644 --- a/media/cast/sender/video_frame_factory.h +++ b/media/cast/sender/video_frame_factory.h @@ -8,6 +8,10 @@ #include "base/memory/scoped_ptr.h" #include "base/time/time.h" +namespace gfx { +class Size; +} + namespace media { class VideoFrame; @@ -33,7 +37,11 @@ class VideoFrameFactory { // Creates a |VideoFrame| suitable for input via |InsertRawVideoFrame|. Frames // obtained in this manner may provide benefits such memory reuse and affinity // with the encoder. The format is guaranteed to be I420 or NV12. - virtual scoped_refptr<VideoFrame> CreateFrame(base::TimeDelta timestamp) = 0; + // + // This can transiently return null if the encoder is not yet initialized or + // is re-initializing. + virtual scoped_refptr<VideoFrame> MaybeCreateFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) = 0; }; } // namespace cast diff --git a/media/cast/sender/video_sender.cc b/media/cast/sender/video_sender.cc index 54e474c..2389e9f 100644 --- a/media/cast/sender/video_sender.cc +++ b/media/cast/sender/video_sender.cc @@ -12,13 +12,7 @@ #include "base/trace_event/trace_event.h" #include "media/cast/cast_defines.h" #include "media/cast/net/cast_transport_config.h" -#include "media/cast/sender/external_video_encoder.h" -#include "media/cast/sender/video_encoder_impl.h" -#include "media/cast/sender/video_frame_factory.h" - -#if defined(OS_MACOSX) -#include "media/cast/sender/h264_vt_encoder.h" -#endif +#include "media/cast/sender/video_encoder.h" namespace media { namespace cast { @@ -71,34 +65,12 @@ VideoSender::VideoSender( last_bitrate_(0), playout_delay_change_cb_(playout_delay_change_cb), weak_factory_(this) { -#if defined(OS_MACOSX) - // On Apple platforms, use the hardware H.264 encoder if possible. It is the - // only reasonable option for iOS. - if (!video_config.use_external_encoder && - video_config.codec == CODEC_VIDEO_H264) { - video_encoder_.reset(new H264VideoToolboxEncoder( - cast_environment, - video_config, - gfx::Size(video_config.width, video_config.height), - status_change_cb)); - } -#endif // defined(OS_MACOSX) -#if !defined(OS_IOS) - if (video_config.use_external_encoder) { - video_encoder_.reset(new ExternalVideoEncoder( - cast_environment, - video_config, - gfx::Size(video_config.width, video_config.height), - status_change_cb, - create_vea_cb, - create_video_encode_mem_cb)); - } else if (!video_encoder_) { - // Software encoder is initialized immediately. - video_encoder_.reset(new VideoEncoderImpl( - cast_environment, video_config, status_change_cb)); - } -#endif // !defined(OS_IOS) - + video_encoder_ = VideoEncoder::Create( + cast_environment_, + video_config, + status_change_cb, + create_vea_cb, + create_video_encode_mem_cb); if (!video_encoder_) { cast_environment_->PostTask( CastEnvironment::MAIN, @@ -202,6 +174,11 @@ void VideoSender::InsertRawVideoFrame( last_bitrate_ = bitrate; } + if (video_frame->visible_rect().IsEmpty()) { + VLOG(1) << "Rejecting empty video frame."; + return; + } + if (video_encoder_->EncodeVideoFrame( video_frame, reference_time, diff --git a/media/cast/sender/video_sender_unittest.cc b/media/cast/sender/video_sender_unittest.cc index 6c0ba98..89a9c4f 100644 --- a/media/cast/sender/video_sender_unittest.cc +++ b/media/cast/sender/video_sender_unittest.cc @@ -15,6 +15,7 @@ #include "media/cast/net/cast_transport_config.h" #include "media/cast/net/cast_transport_sender_impl.h" #include "media/cast/net/pacing/paced_sender.h" +#include "media/cast/sender/fake_video_encode_accelerator_factory.h" #include "media/cast/sender/video_frame_factory.h" #include "media/cast/sender/video_sender.h" #include "media/cast/test/fake_single_thread_task_runner.h" @@ -35,22 +36,6 @@ static const int kHeight = 240; using testing::_; using testing::AtLeast; -void CreateVideoEncodeAccelerator( - const scoped_refptr<base::SingleThreadTaskRunner>& task_runner, - scoped_ptr<VideoEncodeAccelerator> fake_vea, - const ReceiveVideoEncodeAcceleratorCallback& callback) { - callback.Run(task_runner, fake_vea.Pass()); -} - -void CreateSharedMemory( - size_t size, const ReceiveVideoEncodeMemoryCallback& callback) { - scoped_ptr<base::SharedMemory> shm(new base::SharedMemory()); - if (!shm->CreateAndMapAnonymous(size)) { - NOTREACHED(); - return; - } - callback.Run(shm.Pass()); -} void SaveOperationalStatus(OperationalStatus* out_status, OperationalStatus in_status) { @@ -145,8 +130,9 @@ class VideoSenderTest : public ::testing::Test { task_runner_, task_runner_)), operational_status_(STATUS_UNINITIALIZED), - stored_bitrates_(NULL) { + vea_factory_(task_runner_) { testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); + vea_factory_.SetAutoRespond(true); last_pixel_value_ = kPixelValue; net::IPEndPoint dummy_endpoint; transport_sender_.reset(new CastTransportSenderImpl( @@ -182,8 +168,6 @@ class VideoSenderTest : public ::testing::Test { video_config.receiver_ssrc = 2; video_config.rtp_payload_type = 127; video_config.use_external_encoder = external; - video_config.width = kWidth; - video_config.height = kHeight; video_config.max_bitrate = 5000000; video_config.min_bitrate = 1000000; video_config.start_bitrate = 1000000; @@ -196,30 +180,25 @@ class VideoSenderTest : public ::testing::Test { ASSERT_EQ(operational_status_, STATUS_UNINITIALIZED); if (external) { - media::FakeVideoEncodeAccelerator* fake_vea = - new media::FakeVideoEncodeAccelerator(task_runner_); - stored_bitrates_ = &fake_vea->stored_bitrates(); - fake_vea->SetWillInitializationSucceed(expect_init_success); - scoped_ptr<VideoEncodeAccelerator> fake_vea_owner(fake_vea); - video_sender_.reset( - new PeerVideoSender(cast_environment_, - video_config, - base::Bind(&SaveOperationalStatus, - &operational_status_), - base::Bind(&CreateVideoEncodeAccelerator, - task_runner_, - base::Passed(&fake_vea_owner)), - base::Bind(&CreateSharedMemory), - transport_sender_.get())); + vea_factory_.SetInitializationWillSucceed(expect_init_success); + video_sender_.reset(new PeerVideoSender( + cast_environment_, + video_config, + base::Bind(&SaveOperationalStatus, &operational_status_), + base::Bind( + &FakeVideoEncodeAcceleratorFactory::CreateVideoEncodeAccelerator, + base::Unretained(&vea_factory_)), + base::Bind(&FakeVideoEncodeAcceleratorFactory::CreateSharedMemory, + base::Unretained(&vea_factory_)), + transport_sender_.get())); } else { - video_sender_.reset( - new PeerVideoSender(cast_environment_, - video_config, - base::Bind(&SaveOperationalStatus, - &operational_status_), - CreateDefaultVideoEncodeAcceleratorCallback(), - CreateDefaultVideoEncodeMemoryCallback(), - transport_sender_.get())); + video_sender_.reset(new PeerVideoSender( + cast_environment_, + video_config, + base::Bind(&SaveOperationalStatus, &operational_status_), + CreateDefaultVideoEncodeAcceleratorCallback(), + CreateDefaultVideoEncodeMemoryCallback(), + transport_sender_.get())); } task_runner_->RunTasks(); } @@ -256,10 +235,10 @@ class VideoSenderTest : public ::testing::Test { const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; const scoped_refptr<CastEnvironment> cast_environment_; OperationalStatus operational_status_; + FakeVideoEncodeAcceleratorFactory vea_factory_; TestPacketSender transport_; scoped_ptr<CastTransportSenderImpl> transport_sender_; scoped_ptr<PeerVideoSender> video_sender_; - const std::vector<uint32>* stored_bitrates_; // Owned by |video_sender_|. int last_pixel_value_; base::TimeTicks first_frame_timestamp_; @@ -284,25 +263,53 @@ TEST_F(VideoSenderTest, ExternalEncoder) { InitEncoder(true, true); ASSERT_EQ(STATUS_INITIALIZED, operational_status_); + // The SizeAdaptableExternalVideoEncoder initally reports STATUS_INITIALIZED + // so that frames will be sent to it. Therefore, no encoder activity should + // have occurred at this point. Send a frame to spurn creation of the + // underlying ExternalVideoEncoder instance. + if (vea_factory_.vea_response_count() == 0) { + video_sender_->InsertRawVideoFrame(GetNewVideoFrame(), + testing_clock_->NowTicks()); + task_runner_->RunTasks(); + } + ASSERT_EQ(STATUS_INITIALIZED, operational_status_); + RunTasks(33); + + // VideoSender created an encoder for 1280x720 frames, in order to provide the + // INITIALIZED status. + EXPECT_EQ(1, vea_factory_.vea_response_count()); + EXPECT_EQ(3, vea_factory_.shm_response_count()); + scoped_refptr<media::VideoFrame> video_frame = GetNewVideoFrame(); - const base::TimeTicks reference_time = testing_clock_->NowTicks(); - video_sender_->InsertRawVideoFrame(video_frame, reference_time); - task_runner_->RunTasks(); - video_sender_->InsertRawVideoFrame(video_frame, reference_time); - task_runner_->RunTasks(); - video_sender_->InsertRawVideoFrame(video_frame, reference_time); - task_runner_->RunTasks(); + for (int i = 0; i < 3; ++i) { + const base::TimeTicks reference_time = testing_clock_->NowTicks(); + video_sender_->InsertRawVideoFrame(video_frame, reference_time); + RunTasks(33); + // VideoSender re-created the encoder for the 320x240 frames we're + // providing. + EXPECT_EQ(1, vea_factory_.vea_response_count()); + EXPECT_EQ(3, vea_factory_.shm_response_count()); + } - // Fixed bitrate is used for external encoder. Bitrate is only once - // to the encoder. - EXPECT_EQ(1u, stored_bitrates_->size()); video_sender_.reset(NULL); task_runner_->RunTasks(); + EXPECT_EQ(1, vea_factory_.vea_response_count()); + EXPECT_EQ(3, vea_factory_.shm_response_count()); } TEST_F(VideoSenderTest, ExternalEncoderInitFails) { InitEncoder(true, false); + + // The SizeAdaptableExternalVideoEncoder initally reports STATUS_INITIALIZED + // so that frames will be sent to it. Send a frame to spurn creation of the + // underlying ExternalVideoEncoder instance, which should result in failure. + if (operational_status_ == STATUS_INITIALIZED || + operational_status_ == STATUS_CODEC_REINIT_PENDING) { + video_sender_->InsertRawVideoFrame(GetNewVideoFrame(), + testing_clock_->NowTicks()); + task_runner_->RunTasks(); + } EXPECT_EQ(STATUS_CODEC_INIT_FAILED, operational_status_); video_sender_.reset(NULL); diff --git a/media/cast/test/cast_benchmarks.cc b/media/cast/test/cast_benchmarks.cc index aa89e94..064c5db 100644 --- a/media/cast/test/cast_benchmarks.cc +++ b/media/cast/test/cast_benchmarks.cc @@ -69,8 +69,6 @@ namespace { static const int64 kStartMillisecond = INT64_C(1245); static const int kAudioChannels = 2; -static const int kVideoHdWidth = 1280; -static const int kVideoHdHeight = 720; static const int kTargetPlayoutDelayMs = 300; // The tests are commonly implemented with |kFrameTimerMs| RunTask function; @@ -266,8 +264,6 @@ class RunOneBenchmark { base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs); video_sender_config_.rtp_payload_type = 97; video_sender_config_.use_external_encoder = false; - video_sender_config_.width = kVideoHdWidth; - video_sender_config_.height = kVideoHdHeight; #if 0 video_sender_config_.max_bitrate = 10000000; // 10Mbit max video_sender_config_.min_bitrate = 1000000; // 1Mbit min diff --git a/media/cast/test/end2end_unittest.cc b/media/cast/test/end2end_unittest.cc index e7a0c14..9b36b59 100644 --- a/media/cast/test/end2end_unittest.cc +++ b/media/cast/test/end2end_unittest.cc @@ -53,8 +53,6 @@ static const double kSoundFrequency = 314.15926535897; // Freq of sine wave. static const float kSoundVolume = 0.5f; static const int kVideoHdWidth = 1280; static const int kVideoHdHeight = 720; -static const int kVideoQcifWidth = 176; -static const int kVideoQcifHeight = 144; // Since the video encoded and decoded an error will be introduced; when // comparing individual pixels the error can be quite large; we allow a PSNR of @@ -356,8 +354,6 @@ class TestReceiverVideoCallback public: struct ExpectedVideoFrame { int start_value; - int width; - int height; base::TimeTicks playout_time; bool should_be_continuous; }; @@ -365,14 +361,10 @@ class TestReceiverVideoCallback TestReceiverVideoCallback() : num_called_(0) {} void AddExpectedResult(int start_value, - int width, - int height, const base::TimeTicks& playout_time, bool should_be_continuous) { ExpectedVideoFrame expected_video_frame; expected_video_frame.start_value = start_value; - expected_video_frame.width = width; - expected_video_frame.height = height; expected_video_frame.playout_time = playout_time; expected_video_frame.should_be_continuous = should_be_continuous; expected_frame_.push_back(expected_video_frame); @@ -388,11 +380,10 @@ class TestReceiverVideoCallback ExpectedVideoFrame expected_video_frame = expected_frame_.front(); expected_frame_.pop_front(); - EXPECT_EQ(expected_video_frame.width, video_frame->visible_rect().width()); - EXPECT_EQ(expected_video_frame.height, - video_frame->visible_rect().height()); + EXPECT_EQ(kVideoHdWidth, video_frame->visible_rect().width()); + EXPECT_EQ(kVideoHdHeight, video_frame->visible_rect().height()); - gfx::Size size(expected_video_frame.width, expected_video_frame.height); + const gfx::Size size(kVideoHdWidth, kVideoHdHeight); scoped_refptr<media::VideoFrame> expected_I420_frame = media::VideoFrame::CreateFrame( VideoFrame::I420, size, gfx::Rect(size), size, base::TimeDelta()); @@ -496,8 +487,6 @@ class End2EndTest : public ::testing::Test { base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs); video_sender_config_.rtp_payload_type = 97; video_sender_config_.use_external_encoder = false; - video_sender_config_.width = kVideoHdWidth; - video_sender_config_.height = kVideoHdHeight; video_sender_config_.max_bitrate = 50000; video_sender_config_.min_bitrate = 10000; video_sender_config_.start_bitrate = 10000; @@ -666,7 +655,7 @@ class End2EndTest : public ::testing::Test { // TODO(miu): Consider using a slightly skewed clock for the media timestamp // since the video clock may not be the same as the reference clock. const base::TimeDelta time_diff = reference_time - start_time_; - gfx::Size size(video_sender_config_.width, video_sender_config_.height); + const gfx::Size size(kVideoHdWidth, kVideoHdHeight); EXPECT_TRUE(VideoFrame::IsValidConfig( VideoFrame::I420, size, gfx::Rect(size), size)); scoped_refptr<media::VideoFrame> video_frame = @@ -834,10 +823,6 @@ class End2EndTest : public ::testing::Test { TEST_F(End2EndTest, LoopNoLossPcm16) { Configure(CODEC_VIDEO_VP8, CODEC_AUDIO_PCM16, 32000, 1); - // Reduce video resolution to allow processing multiple frames within a - // reasonable time frame. - video_sender_config_.width = kVideoQcifWidth; - video_sender_config_.height = kVideoQcifHeight; Create(); const int kNumIterations = 50; @@ -853,8 +838,6 @@ TEST_F(End2EndTest, LoopNoLossPcm16) { test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, testing_clock_sender_->NowTicks() + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); @@ -959,8 +942,6 @@ TEST_F(End2EndTest, DISABLED_StartSenderBeforeReceiver) { // packets, and specifically no RTCP packets were sent. test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, initial_send_time + expected_delay + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); @@ -989,8 +970,6 @@ TEST_F(End2EndTest, DISABLED_StartSenderBeforeReceiver) { test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, testing_clock_sender_->NowTicks() + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); @@ -1040,8 +1019,6 @@ TEST_F(End2EndTest, DropEveryOtherFrame3Buffers) { if (i % 2 == 0) { test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, reference_time + base::TimeDelta::FromMilliseconds(target_delay), i == 0); @@ -1081,8 +1058,6 @@ TEST_F(End2EndTest, CryptoVideo) { test_receiver_video_callback_->AddExpectedResult( frames_counter, - video_sender_config_.width, - video_sender_config_.height, reference_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); @@ -1137,8 +1112,6 @@ TEST_F(End2EndTest, VideoLogging) { base::TimeTicks reference_time = testing_clock_sender_->NowTicks(); test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, reference_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); diff --git a/media/cast/test/utility/default_config.cc b/media/cast/test/utility/default_config.cc index 39b9ae6..3d4a471 100644 --- a/media/cast/test/utility/default_config.cc +++ b/media/cast/test/utility/default_config.cc @@ -74,8 +74,6 @@ VideoSenderConfig GetDefaultVideoSenderConfig() { config.receiver_ssrc = recv_config.feedback_ssrc; config.rtp_payload_type = recv_config.rtp_payload_type; config.use_external_encoder = false; - config.width = 1280; - config.height = 720; config.max_bitrate = 4000000; config.min_bitrate = 2000000; config.start_bitrate = 4000000; diff --git a/media/cast/test/utility/video_utility.cc b/media/cast/test/utility/video_utility.cc index ff050da..46469563 100644 --- a/media/cast/test/utility/video_utility.cc +++ b/media/cast/test/utility/video_utility.cc @@ -63,16 +63,10 @@ void PopulateVideoFrame(VideoFrame* frame, int start_value) { const int stripe_size = std::max(32, std::min(frame_size.width(), frame_size.height()) / 8) & -2; - int height = frame_size.height(); - int stride_y = frame->stride(VideoFrame::kYPlane); - int stride_u = frame->stride(VideoFrame::kUPlane); - int stride_v = frame->stride(VideoFrame::kVPlane); - int half_height = (height + 1) / 2; - uint8* y_plane = frame->data(VideoFrame::kYPlane); - uint8* u_plane = frame->data(VideoFrame::kUPlane); - uint8* v_plane = frame->data(VideoFrame::kVPlane); - // Set Y. + const int height = frame_size.height(); + const int stride_y = frame->stride(VideoFrame::kYPlane); + uint8* y_plane = frame->data(VideoFrame::kYPlane); for (int j = 0; j < height; ++j) { const int stripe_j = (j / stripe_size) * stripe_size; for (int i = 0; i < stride_y; ++i) { @@ -82,23 +76,45 @@ void PopulateVideoFrame(VideoFrame* frame, int start_value) { } } - // Set U. - for (int j = 0; j < half_height; ++j) { - const int stripe_j = (j / stripe_size) * stripe_size; - for (int i = 0; i < stride_u; ++i) { - const int stripe_i = (i / stripe_size) * stripe_size; - *u_plane = static_cast<uint8>(start_value + stripe_i + stripe_j); - ++u_plane; + const int half_height = (height + 1) / 2; + if (frame->format() == VideoFrame::NV12) { + const int stride_uv = frame->stride(VideoFrame::kUVPlane); + uint8* uv_plane = frame->data(VideoFrame::kUVPlane); + + // Set U and V. + for (int j = 0; j < half_height; ++j) { + const int stripe_j = (j / stripe_size) * stripe_size; + for (int i = 0; i < stride_uv; i += 2) { + const int stripe_i = (i / stripe_size) * stripe_size; + *uv_plane = *(uv_plane + 1) = + static_cast<uint8>(start_value + stripe_i + stripe_j); + uv_plane += 2; + } + } + } else { // I420, YV12, etc. + const int stride_u = frame->stride(VideoFrame::kUPlane); + const int stride_v = frame->stride(VideoFrame::kVPlane); + uint8* u_plane = frame->data(VideoFrame::kUPlane); + uint8* v_plane = frame->data(VideoFrame::kVPlane); + + // Set U. + for (int j = 0; j < half_height; ++j) { + const int stripe_j = (j / stripe_size) * stripe_size; + for (int i = 0; i < stride_u; ++i) { + const int stripe_i = (i / stripe_size) * stripe_size; + *u_plane = static_cast<uint8>(start_value + stripe_i + stripe_j); + ++u_plane; + } } - } - // Set V. - for (int j = 0; j < half_height; ++j) { - const int stripe_j = (j / stripe_size) * stripe_size; - for (int i = 0; i < stride_v; ++i) { - const int stripe_i = (i / stripe_size) * stripe_size; - *u_plane = static_cast<uint8>(start_value + stripe_i + stripe_j); - ++v_plane; + // Set V. + for (int j = 0; j < half_height; ++j) { + const int stripe_j = (j / stripe_size) * stripe_size; + for (int i = 0; i < stride_v; ++i) { + const int stripe_i = (i / stripe_size) * stripe_size; + *u_plane = static_cast<uint8>(start_value + stripe_i + stripe_j); + ++v_plane; + } } } } |