diff options
Diffstat (limited to 'media')
28 files changed, 1343 insertions, 833 deletions
diff --git a/media/cast/BUILD.gn b/media/cast/BUILD.gn index 6a250d0..99c3332 100644 --- a/media/cast/BUILD.gn +++ b/media/cast/BUILD.gn @@ -121,6 +121,8 @@ source_set("sender") { "sender/fake_software_video_encoder.cc", "sender/frame_sender.cc", "sender/frame_sender.h", + "sender/size_adaptable_video_encoder_base.cc", + "sender/size_adaptable_video_encoder_base.h", "sender/software_video_encoder.h", "sender/video_encoder.h", "sender/video_encoder.cc", @@ -296,10 +298,9 @@ test("cast_unittests") { "sender/audio_encoder_unittest.cc", "sender/audio_sender_unittest.cc", "sender/congestion_control_unittest.cc", - "sender/external_video_encoder_unittest.cc", "sender/fake_video_encode_accelerator_factory.cc", "sender/fake_video_encode_accelerator_factory.h", - "sender/video_encoder_impl_unittest.cc", + "sender/video_encoder_unittest.cc", "sender/video_sender_unittest.cc", "test/end2end_unittest.cc", "test/fake_receiver_time_offset_estimator.cc", diff --git a/media/cast/cast.gyp b/media/cast/cast.gyp index 0a7c619..82ba1f4 100644 --- a/media/cast/cast.gyp +++ b/media/cast/cast.gyp @@ -162,6 +162,8 @@ 'sender/fake_software_video_encoder.cc', 'sender/frame_sender.cc', 'sender/frame_sender.h', + 'sender/size_adaptable_video_encoder_base.cc', + 'sender/size_adaptable_video_encoder_base.h', 'sender/software_video_encoder.h', 'sender/video_encoder.h', 'sender/video_encoder.cc', diff --git a/media/cast/cast_config.cc b/media/cast/cast_config.cc index 9dda608..7c8a195 100644 --- a/media/cast/cast_config.cc +++ b/media/cast/cast_config.cc @@ -29,8 +29,6 @@ VideoSenderConfig::VideoSenderConfig() base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs)), rtp_payload_type(0), use_external_encoder(false), - width(0), - height(0), congestion_control_back_off(kDefaultCongestionControlBackOff), max_bitrate(5000000), min_bitrate(1000000), diff --git a/media/cast/cast_config.h b/media/cast/cast_config.h index 89af430..9deb678 100644 --- a/media/cast/cast_config.h +++ b/media/cast/cast_config.h @@ -87,8 +87,6 @@ struct VideoSenderConfig { int rtp_payload_type; bool use_external_encoder; - int width; // Incoming frames will be scaled to this size. - int height; float congestion_control_back_off; int max_bitrate; diff --git a/media/cast/cast_sender.h b/media/cast/cast_sender.h index 1ccebd1..7c68913 100644 --- a/media/cast/cast_sender.h +++ b/media/cast/cast_sender.h @@ -21,6 +21,10 @@ #include "media/cast/cast_environment.h" #include "media/cast/net/cast_transport_sender.h" +namespace gfx { +class Size; +} + namespace media { class VideoFrame; @@ -40,14 +44,18 @@ class VideoFrameInput : public base::RefCountedThreadSafe<VideoFrameInput> { // frames offer performance benefits, such as memory copy elimination. The // format is guaranteed to be I420 or NV12. // - // Not every encoder supports this method. Use |ShouldCreateOptimizedFrame| - // to determine if you can and should use this method. Calling - // this method when |ShouldCreateOptimizedFrame| is false will CHECK. - virtual scoped_refptr<VideoFrame> CreateOptimizedFrame( - base::TimeDelta timestamp) = 0; + // Not every encoder supports this method. Use |CanCreateOptimizedFrames| to + // determine if you can and should use this method. + // + // Even if |CanCreateOptimizedFrames| indicates support, there are transient + // conditions during a session where optimized frames cannot be provided. In + // this case, the caller must be able to account for a nullptr return value + // and instantiate its own media::VideoFrames. + virtual scoped_refptr<VideoFrame> MaybeCreateOptimizedFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) = 0; // Returns true if the encoder supports creating optimized frames. - virtual bool SupportsCreateOptimizedFrame() const = 0; + virtual bool CanCreateOptimizedFrames() const = 0; protected: virtual ~VideoFrameInput() {} diff --git a/media/cast/cast_sender_impl.cc b/media/cast/cast_sender_impl.cc index b47ae9c..23e8a23 100644 --- a/media/cast/cast_sender_impl.cc +++ b/media/cast/cast_sender_impl.cc @@ -19,11 +19,12 @@ namespace cast { class LocalVideoFrameInput : public VideoFrameInput { public: LocalVideoFrameInput(scoped_refptr<CastEnvironment> cast_environment, - base::WeakPtr<VideoSender> video_sender, - scoped_ptr<VideoFrameFactory> video_frame_factory) + base::WeakPtr<VideoSender> video_sender) : cast_environment_(cast_environment), video_sender_(video_sender), - video_frame_factory_(video_frame_factory.Pass()) {} + video_frame_factory_( + video_sender.get() ? + video_sender->CreateVideoFrameFactory().release() : nullptr) {} void InsertRawVideoFrame(const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& capture_time) override { @@ -35,13 +36,14 @@ class LocalVideoFrameInput : public VideoFrameInput { capture_time)); } - scoped_refptr<VideoFrame> CreateOptimizedFrame( + scoped_refptr<VideoFrame> MaybeCreateOptimizedFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) override { - DCHECK(video_frame_factory_.get()); - return video_frame_factory_->CreateFrame(timestamp); + return video_frame_factory_ ? + video_frame_factory_->MaybeCreateFrame(frame_size, timestamp) : nullptr; } - bool SupportsCreateOptimizedFrame() const override { + bool CanCreateOptimizedFrames() const override { return video_frame_factory_.get() != nullptr; } @@ -51,9 +53,9 @@ class LocalVideoFrameInput : public VideoFrameInput { private: friend class base::RefCountedThreadSafe<LocalVideoFrameInput>; - scoped_refptr<CastEnvironment> cast_environment_; - base::WeakPtr<VideoSender> video_sender_; - scoped_ptr<VideoFrameFactory> video_frame_factory_; + const scoped_refptr<CastEnvironment> cast_environment_; + const base::WeakPtr<VideoSender> video_sender_; + const scoped_ptr<VideoFrameFactory> video_frame_factory_; DISALLOW_COPY_AND_ASSIGN(LocalVideoFrameInput); }; @@ -194,8 +196,7 @@ void CastSenderImpl::OnVideoStatusChange( DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); if (status == STATUS_INITIALIZED && !video_frame_input_) { video_frame_input_ = - new LocalVideoFrameInput(cast_environment_, video_sender_->AsWeakPtr(), - video_sender_->CreateVideoFrameFactory()); + new LocalVideoFrameInput(cast_environment_, video_sender_->AsWeakPtr()); } status_change_cb.Run(status); } diff --git a/media/cast/cast_testing.gypi b/media/cast/cast_testing.gypi index 67a0f7d..22a2182 100644 --- a/media/cast/cast_testing.gypi +++ b/media/cast/cast_testing.gypi @@ -115,10 +115,9 @@ 'sender/audio_encoder_unittest.cc', 'sender/audio_sender_unittest.cc', 'sender/congestion_control_unittest.cc', - 'sender/external_video_encoder_unittest.cc', 'sender/fake_video_encode_accelerator_factory.cc', 'sender/fake_video_encode_accelerator_factory.h', - 'sender/video_encoder_impl_unittest.cc', + 'sender/video_encoder_unittest.cc', 'sender/video_sender_unittest.cc', 'test/end2end_unittest.cc', 'test/fake_receiver_time_offset_estimator.cc', diff --git a/media/cast/sender/external_video_encoder.cc b/media/cast/sender/external_video_encoder.cc index c5685e5..fcabe5c 100644 --- a/media/cast/sender/external_video_encoder.cc +++ b/media/cast/sender/external_video_encoder.cc @@ -72,7 +72,7 @@ class ExternalVideoEncoder::VEAClientImpl create_video_encode_memory_cb_(create_video_encode_memory_cb), video_encode_accelerator_(vea.Pass()), encoder_active_(false), - last_encoded_frame_id_(kStartFrameId), + next_frame_id_(0u), key_frame_encountered_(false) { } @@ -82,9 +82,9 @@ class ExternalVideoEncoder::VEAClientImpl void Initialize(const gfx::Size& frame_size, VideoCodecProfile codec_profile, - int start_bit_rate) { + int start_bit_rate, + uint32 first_frame_id) { DCHECK(task_runner_->RunsTasksOnCurrentThread()); - DCHECK(!frame_size.IsEmpty()); encoder_active_ = video_encode_accelerator_->Initialize( media::VideoFrame::I420, @@ -92,6 +92,7 @@ class ExternalVideoEncoder::VEAClientImpl codec_profile, start_bit_rate, this); + next_frame_id_ = first_frame_id; UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess", encoder_active_); @@ -203,7 +204,7 @@ class ExternalVideoEncoder::VEAClientImpl scoped_ptr<EncodedFrame> encoded_frame(new EncodedFrame()); encoded_frame->dependency = key_frame ? EncodedFrame::KEY : EncodedFrame::DEPENDENT; - encoded_frame->frame_id = ++last_encoded_frame_id_; + encoded_frame->frame_id = next_frame_id_++; if (key_frame) encoded_frame->referenced_frame_id = encoded_frame->frame_id; else @@ -290,7 +291,7 @@ class ExternalVideoEncoder::VEAClientImpl const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_; scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_; bool encoder_active_; - uint32 last_encoded_frame_id_; + uint32 next_frame_id_; bool key_frame_encountered_; std::string stream_header_; @@ -303,71 +304,61 @@ class ExternalVideoEncoder::VEAClientImpl DISALLOW_COPY_AND_ASSIGN(VEAClientImpl); }; +// static +bool ExternalVideoEncoder::IsSupported(const VideoSenderConfig& video_config) { + if (video_config.codec != CODEC_VIDEO_VP8 && + video_config.codec != CODEC_VIDEO_H264) + return false; + + // TODO(miu): "Layering hooks" are needed to be able to query outside of + // libmedia, to determine whether the system provides a hardware encoder. For + // now, assume that this was already checked by this point. + // http://crbug.com/454029 + return video_config.use_external_encoder; +} + ExternalVideoEncoder::ExternalVideoEncoder( const scoped_refptr<CastEnvironment>& cast_environment, const VideoSenderConfig& video_config, const gfx::Size& frame_size, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb, const CreateVideoEncodeAcceleratorCallback& create_vea_cb, const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb) : cast_environment_(cast_environment), create_video_encode_memory_cb_(create_video_encode_memory_cb), + frame_size_(frame_size), bit_rate_(video_config.start_bitrate), key_frame_requested_(false), weak_factory_(this) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); DCHECK_GT(video_config.max_frame_rate, 0); - DCHECK(!frame_size.IsEmpty()); + DCHECK(!frame_size_.IsEmpty()); DCHECK(!status_change_cb.is_null()); DCHECK(!create_vea_cb.is_null()); DCHECK(!create_video_encode_memory_cb_.is_null()); DCHECK_GT(bit_rate_, 0); - VideoCodecProfile codec_profile; - switch (video_config.codec) { - case CODEC_VIDEO_VP8: - codec_profile = media::VP8PROFILE_ANY; - break; - case CODEC_VIDEO_H264: - codec_profile = media::H264PROFILE_MAIN; - break; - case CODEC_VIDEO_FAKE: - NOTREACHED() << "Fake software video encoder cannot be external"; - // ...flow through to next case... - default: - cast_environment_->PostTask( - CastEnvironment::MAIN, - FROM_HERE, - base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC)); - return; - } - create_vea_cb.Run( base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator, weak_factory_.GetWeakPtr(), - frame_size, - codec_profile, - video_config.max_frame_rate, + video_config, + first_frame_id, status_change_cb)); } ExternalVideoEncoder::~ExternalVideoEncoder() { } -bool ExternalVideoEncoder::CanEncodeVariedFrameSizes() const { - return false; -} - bool ExternalVideoEncoder::EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, const FrameEncodedCallback& frame_encoded_callback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); - DCHECK(!video_frame->visible_rect().IsEmpty()); DCHECK(!frame_encoded_callback.is_null()); - if (!client_) - return false; // Not ready. + if (!client_ || video_frame->visible_rect().size() != frame_size_) + return false; client_->task_runner()->PostTask(FROM_HERE, base::Bind(&VEAClientImpl::EncodeVideoFrame, @@ -401,9 +392,8 @@ void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) { } void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator( - const gfx::Size& frame_size, - VideoCodecProfile codec_profile, - int max_frame_rate, + const VideoSenderConfig& video_config, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb, scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner, scoped_ptr<media::VideoEncodeAccelerator> vea) { @@ -420,19 +410,64 @@ void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator( return; } + VideoCodecProfile codec_profile; + switch (video_config.codec) { + case CODEC_VIDEO_VP8: + codec_profile = media::VP8PROFILE_ANY; + break; + case CODEC_VIDEO_H264: + codec_profile = media::H264PROFILE_MAIN; + break; + case CODEC_VIDEO_FAKE: + NOTREACHED() << "Fake software video encoder cannot be external"; + // ...flow through to next case... + default: + cast_environment_->PostTask( + CastEnvironment::MAIN, + FROM_HERE, + base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC)); + return; + } + DCHECK(!client_); client_ = new VEAClientImpl(cast_environment_, encoder_task_runner, vea.Pass(), - max_frame_rate, + video_config.max_frame_rate, status_change_cb, create_video_encode_memory_cb_); client_->task_runner()->PostTask(FROM_HERE, base::Bind(&VEAClientImpl::Initialize, client_, - frame_size, + frame_size_, codec_profile, - bit_rate_)); + bit_rate_, + first_frame_id)); +} + +SizeAdaptableExternalVideoEncoder::SizeAdaptableExternalVideoEncoder( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb, + const CreateVideoEncodeAcceleratorCallback& create_vea_cb, + const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb) + : SizeAdaptableVideoEncoderBase(cast_environment, + video_config, + status_change_cb), + create_vea_cb_(create_vea_cb), + create_video_encode_memory_cb_(create_video_encode_memory_cb) {} + +SizeAdaptableExternalVideoEncoder::~SizeAdaptableExternalVideoEncoder() {} + +scoped_ptr<VideoEncoder> SizeAdaptableExternalVideoEncoder::CreateEncoder() { + return scoped_ptr<VideoEncoder>(new ExternalVideoEncoder( + cast_environment(), + video_config(), + frame_size(), + last_frame_id() + 1, + CreateEncoderStatusChangeCallback(), + create_vea_cb_, + create_video_encode_memory_cb_)); } } // namespace cast diff --git a/media/cast/sender/external_video_encoder.h b/media/cast/sender/external_video_encoder.h index e560c57..ee8b7f4 100644 --- a/media/cast/sender/external_video_encoder.h +++ b/media/cast/sender/external_video_encoder.h @@ -9,6 +9,7 @@ #include "base/memory/weak_ptr.h" #include "media/cast/cast_config.h" #include "media/cast/cast_environment.h" +#include "media/cast/sender/size_adaptable_video_encoder_base.h" #include "media/cast/sender/video_encoder.h" #include "media/video/video_encode_accelerator.h" #include "ui/gfx/geometry/size.h" @@ -21,10 +22,15 @@ namespace cast { // emits media::cast::EncodedFrames. class ExternalVideoEncoder : public VideoEncoder { public: + // Returns true if the current platform and system configuration supports + // using ExternalVideoEncoder with the given |video_config|. + static bool IsSupported(const VideoSenderConfig& video_config); + ExternalVideoEncoder( const scoped_refptr<CastEnvironment>& cast_environment, const VideoSenderConfig& video_config, const gfx::Size& frame_size, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb, const CreateVideoEncodeAcceleratorCallback& create_vea_cb, const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb); @@ -32,7 +38,6 @@ class ExternalVideoEncoder : public VideoEncoder { ~ExternalVideoEncoder() override; // VideoEncoder implementation. - bool CanEncodeVariedFrameSizes() const override; bool EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, @@ -48,15 +53,18 @@ class ExternalVideoEncoder : public VideoEncoder { // VEAClientImpl to own and interface with a new |vea|. Upon return, // |client_| holds a reference to the new VEAClientImpl. void OnCreateVideoEncodeAccelerator( - const gfx::Size& frame_size, - VideoCodecProfile codec_profile, - int max_frame_rate, + const VideoSenderConfig& video_config, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb, scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner, scoped_ptr<media::VideoEncodeAccelerator> vea); const scoped_refptr<CastEnvironment> cast_environment_; const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_; + + // The size of the visible region of the video frames to be encoded. + const gfx::Size frame_size_; + int bit_rate_; bool key_frame_requested_; @@ -69,6 +77,31 @@ class ExternalVideoEncoder : public VideoEncoder { DISALLOW_COPY_AND_ASSIGN(ExternalVideoEncoder); }; +// An implementation of SizeAdaptableVideoEncoderBase to proxy for +// ExternalVideoEncoder instances. +class SizeAdaptableExternalVideoEncoder : public SizeAdaptableVideoEncoderBase { + public: + SizeAdaptableExternalVideoEncoder( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb, + const CreateVideoEncodeAcceleratorCallback& create_vea_cb, + const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb); + + ~SizeAdaptableExternalVideoEncoder() override; + + protected: + scoped_ptr<VideoEncoder> CreateEncoder() override; + + private: + // Special callbacks needed by media::cast::ExternalVideoEncoder. + // TODO(miu): Remove these. http://crbug.com/454029 + const CreateVideoEncodeAcceleratorCallback create_vea_cb_; + const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_; + + DISALLOW_COPY_AND_ASSIGN(SizeAdaptableExternalVideoEncoder); +}; + } // namespace cast } // namespace media diff --git a/media/cast/sender/external_video_encoder_unittest.cc b/media/cast/sender/external_video_encoder_unittest.cc deleted file mode 100644 index 0ade1ed..0000000 --- a/media/cast/sender/external_video_encoder_unittest.cc +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <vector> - -#include "base/bind.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "media/base/video_frame.h" -#include "media/cast/cast_defines.h" -#include "media/cast/cast_environment.h" -#include "media/cast/sender/external_video_encoder.h" -#include "media/cast/sender/fake_video_encode_accelerator_factory.h" -#include "media/cast/test/fake_single_thread_task_runner.h" -#include "media/cast/test/utility/video_utility.h" -#include "media/video/fake_video_encode_accelerator.h" -#include "testing/gmock/include/gmock/gmock.h" - -namespace media { -namespace cast { - -using testing::_; - -namespace { - -class TestVideoEncoderCallback - : public base::RefCountedThreadSafe<TestVideoEncoderCallback> { - public: - TestVideoEncoderCallback() {} - - void SetExpectedResult(uint32 expected_frame_id, - uint32 expected_last_referenced_frame_id, - uint32 expected_rtp_timestamp, - const base::TimeTicks& expected_reference_time) { - expected_frame_id_ = expected_frame_id; - expected_last_referenced_frame_id_ = expected_last_referenced_frame_id; - expected_rtp_timestamp_ = expected_rtp_timestamp; - expected_reference_time_ = expected_reference_time; - } - - void DeliverEncodedVideoFrame( - scoped_ptr<EncodedFrame> encoded_frame) { - if (expected_frame_id_ == expected_last_referenced_frame_id_) { - EXPECT_EQ(EncodedFrame::KEY, encoded_frame->dependency); - } else { - EXPECT_EQ(EncodedFrame::DEPENDENT, - encoded_frame->dependency); - } - EXPECT_EQ(expected_frame_id_, encoded_frame->frame_id); - EXPECT_EQ(expected_last_referenced_frame_id_, - encoded_frame->referenced_frame_id); - EXPECT_EQ(expected_rtp_timestamp_, encoded_frame->rtp_timestamp); - EXPECT_EQ(expected_reference_time_, encoded_frame->reference_time); - } - - protected: - virtual ~TestVideoEncoderCallback() {} - - private: - friend class base::RefCountedThreadSafe<TestVideoEncoderCallback>; - - uint32 expected_frame_id_; - uint32 expected_last_referenced_frame_id_; - uint32 expected_rtp_timestamp_; - base::TimeTicks expected_reference_time_; - - DISALLOW_COPY_AND_ASSIGN(TestVideoEncoderCallback); -}; -} // namespace - -class ExternalVideoEncoderTest : public ::testing::Test { - protected: - ExternalVideoEncoderTest() - : testing_clock_(new base::SimpleTestTickClock()), - task_runner_(new test::FakeSingleThreadTaskRunner(testing_clock_)), - cast_environment_(new CastEnvironment( - scoped_ptr<base::TickClock>(testing_clock_).Pass(), - task_runner_, - task_runner_, - task_runner_)), - vea_factory_(task_runner_), - operational_status_(STATUS_UNINITIALIZED), - test_video_encoder_callback_(new TestVideoEncoderCallback()) { - testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); - - video_config_.ssrc = 1; - video_config_.receiver_ssrc = 2; - video_config_.rtp_payload_type = 127; - video_config_.use_external_encoder = true; - video_config_.max_bitrate = 5000000; - video_config_.min_bitrate = 1000000; - video_config_.start_bitrate = 2000000; - video_config_.max_qp = 56; - video_config_.min_qp = 0; - video_config_.max_frame_rate = 30; - video_config_.max_number_of_video_buffers_used = 3; - video_config_.codec = CODEC_VIDEO_VP8; - const gfx::Size size(320, 240); - video_frame_ = media::VideoFrame::CreateFrame( - VideoFrame::I420, size, gfx::Rect(size), size, base::TimeDelta()); - PopulateVideoFrame(video_frame_.get(), 123); - - video_encoder_.reset(new ExternalVideoEncoder( - cast_environment_, - video_config_, - size, - base::Bind(&ExternalVideoEncoderTest::SaveOperationalStatus, - base::Unretained(this)), - base::Bind( - &FakeVideoEncodeAcceleratorFactory::CreateVideoEncodeAccelerator, - base::Unretained(&vea_factory_)), - base::Bind(&FakeVideoEncodeAcceleratorFactory::CreateSharedMemory, - base::Unretained(&vea_factory_)))); - } - - ~ExternalVideoEncoderTest() override {} - - void AdvanceClockAndVideoFrameTimestamp() { - testing_clock_->Advance(base::TimeDelta::FromMilliseconds(33)); - video_frame_->set_timestamp( - video_frame_->timestamp() + base::TimeDelta::FromMilliseconds(33)); - } - - void SaveOperationalStatus(OperationalStatus result) { - EXPECT_EQ(STATUS_UNINITIALIZED, operational_status_); - operational_status_ = result; - } - - base::SimpleTestTickClock* const testing_clock_; // Owned by CastEnvironment. - const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; - const scoped_refptr<CastEnvironment> cast_environment_; - FakeVideoEncodeAcceleratorFactory vea_factory_; - OperationalStatus operational_status_; - scoped_refptr<TestVideoEncoderCallback> test_video_encoder_callback_; - VideoSenderConfig video_config_; - scoped_ptr<VideoEncoder> video_encoder_; - scoped_refptr<media::VideoFrame> video_frame_; - - DISALLOW_COPY_AND_ASSIGN(ExternalVideoEncoderTest); -}; - -TEST_F(ExternalVideoEncoderTest, EncodePattern30fpsRunningOutOfAck) { - vea_factory_.SetAutoRespond(true); - task_runner_->RunTasks(); // Run the initializer on the correct thread. - EXPECT_EQ(STATUS_INITIALIZED, operational_status_); - - VideoEncoder::FrameEncodedCallback frame_encoded_callback = - base::Bind(&TestVideoEncoderCallback::DeliverEncodedVideoFrame, - test_video_encoder_callback_.get()); - - test_video_encoder_callback_->SetExpectedResult( - 0, 0, TimeDeltaToRtpDelta(video_frame_->timestamp(), kVideoFrequency), - testing_clock_->NowTicks()); - video_encoder_->SetBitRate(2000); - EXPECT_TRUE(video_encoder_->EncodeVideoFrame( - video_frame_, testing_clock_->NowTicks(), frame_encoded_callback)); - task_runner_->RunTasks(); - - for (int i = 0; i < 6; ++i) { - AdvanceClockAndVideoFrameTimestamp(); - test_video_encoder_callback_->SetExpectedResult( - i + 1, - i, - TimeDeltaToRtpDelta(video_frame_->timestamp(), kVideoFrequency), - testing_clock_->NowTicks()); - EXPECT_TRUE(video_encoder_->EncodeVideoFrame( - video_frame_, testing_clock_->NowTicks(), frame_encoded_callback)); - task_runner_->RunTasks(); - } - - ASSERT_EQ(1u, vea_factory_.last_response_vea()->stored_bitrates().size()); - EXPECT_EQ(2000u, vea_factory_.last_response_vea()->stored_bitrates()[0]); - - // We need to run the task to cleanup the GPU instance. - video_encoder_.reset(NULL); - task_runner_->RunTasks(); - - EXPECT_EQ(1, vea_factory_.vea_response_count()); - EXPECT_EQ(3, vea_factory_.shm_response_count()); -} - -TEST_F(ExternalVideoEncoderTest, StreamHeader) { - vea_factory_.SetAutoRespond(true); - task_runner_->RunTasks(); // Run the initializer on the correct thread. - EXPECT_EQ(STATUS_INITIALIZED, operational_status_); - - VideoEncoder::FrameEncodedCallback frame_encoded_callback = - base::Bind(&TestVideoEncoderCallback::DeliverEncodedVideoFrame, - test_video_encoder_callback_.get()); - - // Force the FakeVideoEncodeAccelerator to return a dummy non-key frame first. - vea_factory_.last_response_vea()->SendDummyFrameForTesting(false); - - // Verify the first returned bitstream buffer is still a key frame. - test_video_encoder_callback_->SetExpectedResult( - 0, 0, TimeDeltaToRtpDelta(video_frame_->timestamp(), kVideoFrequency), - testing_clock_->NowTicks()); - EXPECT_TRUE(video_encoder_->EncodeVideoFrame( - video_frame_, testing_clock_->NowTicks(), frame_encoded_callback)); - task_runner_->RunTasks(); - - // We need to run the task to cleanup the GPU instance. - video_encoder_.reset(NULL); - task_runner_->RunTasks(); - - EXPECT_EQ(1, vea_factory_.vea_response_count()); - EXPECT_EQ(3, vea_factory_.shm_response_count()); -} - -// Verify that everything goes well even if ExternalVideoEncoder is destroyed -// before it has a chance to receive the VEA creation callback. -TEST_F(ExternalVideoEncoderTest, DestroyBeforeVEACreatedCallback) { - video_encoder_.reset(); - EXPECT_EQ(0, vea_factory_.vea_response_count()); - vea_factory_.SetAutoRespond(true); - task_runner_->RunTasks(); - EXPECT_EQ(1, vea_factory_.vea_response_count()); - EXPECT_EQ(STATUS_UNINITIALIZED, operational_status_); -} - -} // namespace cast -} // namespace media diff --git a/media/cast/sender/h264_vt_encoder.cc b/media/cast/sender/h264_vt_encoder.cc index 364a416..c8ae82b 100644 --- a/media/cast/sender/h264_vt_encoder.cc +++ b/media/cast/sender/h264_vt_encoder.cc @@ -13,6 +13,7 @@ #include "base/location.h" #include "base/logging.h" #include "base/macros.h" +#include "base/synchronization/lock.h" #include "media/base/mac/corevideo_glue.h" #include "media/base/mac/video_frame_mac.h" #include "media/cast/sender/video_frame_factory.h" @@ -36,25 +37,35 @@ struct InProgressFrameEncode { frame_encoded_callback(callback) {} }; +base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeysAndValues( + CFTypeRef* keys, + CFTypeRef* values, + size_t size) { + return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate( + kCFAllocatorDefault, + keys, + values, + size, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks)); +} + base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key, CFTypeRef value) { CFTypeRef keys[1] = {key}; CFTypeRef values[1] = {value}; - return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate( - kCFAllocatorDefault, keys, values, 1, &kCFTypeDictionaryKeyCallBacks, - &kCFTypeDictionaryValueCallBacks)); + return DictionaryWithKeysAndValues(keys, values, 1); } -base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const std::vector<int>& v) { +base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, size_t size) { std::vector<CFNumberRef> numbers; - numbers.reserve(v.size()); - for (const int i : v) { - numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, &i)); - } + numbers.reserve(size); + for (const int* end = v + size; v < end; ++v) + numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v)); base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate( kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]), numbers.size(), &kCFTypeArrayCallBacks)); - for (CFNumberRef number : numbers) { + for (auto& number : numbers) { CFRelease(number); } return array; @@ -202,42 +213,61 @@ void CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf, class VideoFrameFactoryCVPixelBufferPoolImpl : public VideoFrameFactory { public: VideoFrameFactoryCVPixelBufferPoolImpl( - const base::ScopedCFTypeRef<CVPixelBufferPoolRef>& pool) - : pool_(pool) {} + const base::ScopedCFTypeRef<CVPixelBufferPoolRef>& pool, + const gfx::Size& frame_size) + : pool_(pool), + frame_size_(frame_size) {} ~VideoFrameFactoryCVPixelBufferPoolImpl() override {} - scoped_refptr<VideoFrame> CreateFrame(base::TimeDelta timestamp) override { + scoped_refptr<VideoFrame> MaybeCreateFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) override { + if (frame_size != frame_size_) + return nullptr; // Buffer pool is not a match for requested frame size. + base::ScopedCFTypeRef<CVPixelBufferRef> buffer; - CHECK_EQ(kCVReturnSuccess, - CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool_, - buffer.InitializeInto())); + if (CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool_, + buffer.InitializeInto()) != + kCVReturnSuccess) + return nullptr; // Buffer pool has run out of pixel buffers. + DCHECK(buffer); + return VideoFrame::WrapCVPixelBuffer(buffer, timestamp); } private: - base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool_; + const base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool_; + const gfx::Size frame_size_; DISALLOW_COPY_AND_ASSIGN(VideoFrameFactoryCVPixelBufferPoolImpl); }; } // namespace +// static +bool H264VideoToolboxEncoder::IsSupported( + const VideoSenderConfig& video_config) { + return video_config.codec == CODEC_VIDEO_H264 && VideoToolboxGlue::Get(); +} + H264VideoToolboxEncoder::H264VideoToolboxEncoder( const scoped_refptr<CastEnvironment>& cast_environment, const VideoSenderConfig& video_config, const gfx::Size& frame_size, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb) : cast_environment_(cast_environment), videotoolbox_glue_(VideoToolboxGlue::Get()), - frame_id_(kStartFrameId), + frame_size_(frame_size), + status_change_cb_(status_change_cb), + next_frame_id_(first_frame_id), encode_next_frame_as_keyframe_(false) { - DCHECK(!frame_size.IsEmpty()); - DCHECK(!status_change_cb.is_null()); + DCHECK(!frame_size_.IsEmpty()); + DCHECK(!status_change_cb_.is_null()); OperationalStatus operational_status; if (video_config.codec == CODEC_VIDEO_H264 && videotoolbox_glue_) { - operational_status = Initialize(video_config, frame_size) ? + operational_status = Initialize(video_config) ? STATUS_INITIALIZED : STATUS_INVALID_CONFIGURATION; } else { operational_status = STATUS_UNSUPPORTED_CODEC; @@ -245,7 +275,7 @@ H264VideoToolboxEncoder::H264VideoToolboxEncoder( cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, - base::Bind(status_change_cb, operational_status)); + base::Bind(status_change_cb_, operational_status)); } H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { @@ -253,8 +283,7 @@ H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { } bool H264VideoToolboxEncoder::Initialize( - const VideoSenderConfig& video_config, - const gfx::Size& frame_size) { + const VideoSenderConfig& video_config) { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(!compression_session_); @@ -280,16 +309,40 @@ bool H264VideoToolboxEncoder::Initialize( // Certain encoders prefer kCVPixelFormatType_422YpCbCr8, which is not // supported through VideoFrame. We can force 420 formats to be used instead. const int formats[] = { - kCVPixelFormatType_420YpCbCr8Planar, - CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange}; - base::ScopedCFTypeRef<CFArrayRef> formats_array = ArrayWithIntegers( - std::vector<int>(formats, formats + arraysize(formats))); - base::ScopedCFTypeRef<CFDictionaryRef> buffer_attributes = - DictionaryWithKeyValue(kCVPixelBufferPixelFormatTypeKey, formats_array); + kCVPixelFormatType_420YpCbCr8Planar, + CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + }; + // Keep these attachment settings in-sync with those in ConfigureSession(). + CFTypeRef attachments_keys[] = { + kCVImageBufferColorPrimariesKey, + kCVImageBufferTransferFunctionKey, + kCVImageBufferYCbCrMatrixKey + }; + CFTypeRef attachments_values[] = { + kCVImageBufferColorPrimaries_ITU_R_709_2, + kCVImageBufferTransferFunction_ITU_R_709_2, + kCVImageBufferYCbCrMatrix_ITU_R_709_2 + }; + CFTypeRef buffer_attributes_keys[] = { + kCVPixelBufferPixelFormatTypeKey, + kCVBufferPropagatedAttachmentsKey + }; + CFTypeRef buffer_attributes_values[] = { + ArrayWithIntegers(formats, arraysize(formats)).release(), + DictionaryWithKeysAndValues(attachments_keys, + attachments_values, + arraysize(attachments_keys)).release() + }; + const base::ScopedCFTypeRef<CFDictionaryRef> buffer_attributes = + DictionaryWithKeysAndValues(buffer_attributes_keys, + buffer_attributes_values, + arraysize(buffer_attributes_keys)); + for (auto& v : buffer_attributes_values) + CFRelease(v); VTCompressionSessionRef session; OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate( - kCFAllocatorDefault, frame_size.width(), frame_size.height(), + kCFAllocatorDefault, frame_size_.width(), frame_size_.height(), CoreMediaGlue::kCMVideoCodecType_H264, encoder_spec, buffer_attributes, nullptr /* compressedDataAllocator */, &H264VideoToolboxEncoder::CompressionCallback, @@ -329,6 +382,7 @@ void H264VideoToolboxEncoder::ConfigureSession( SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(), video_config.max_frame_rate); + // Keep these attachment settings in-sync with those in Initialize(). SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(), kCVImageBufferColorPrimaries_ITU_R_709_2); @@ -357,16 +411,11 @@ void H264VideoToolboxEncoder::Teardown() { } } -bool H264VideoToolboxEncoder::CanEncodeVariedFrameSizes() const { - return false; -} - bool H264VideoToolboxEncoder::EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, const FrameEncodedCallback& frame_encoded_callback) { DCHECK(thread_checker_.CalledOnValidThread()); - DCHECK(!video_frame->visible_rect().IsEmpty()); DCHECK(!frame_encoded_callback.is_null()); if (!compression_session_) { @@ -374,6 +423,9 @@ bool H264VideoToolboxEncoder::EncodeVideoFrame( return false; } + if (video_frame->visible_rect().size() != frame_size_) + return false; + // Wrap the VideoFrame in a CVPixelBuffer. In all cases, no data will be // copied. If the VideoFrame was created by this encoder's video frame // factory, then the returned CVPixelBuffer will have been obtained from the @@ -435,12 +487,14 @@ void H264VideoToolboxEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) { scoped_ptr<VideoFrameFactory> H264VideoToolboxEncoder::CreateVideoFrameFactory() { + if (!videotoolbox_glue_ || !compression_session_) + return nullptr; base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool( videotoolbox_glue_->VTCompressionSessionGetPixelBufferPool( compression_session_), base::scoped_policy::RETAIN); return scoped_ptr<VideoFrameFactory>( - new VideoFrameFactoryCVPixelBufferPoolImpl(pool)); + new VideoFrameFactoryCVPixelBufferPoolImpl(pool, frame_size_)); } void H264VideoToolboxEncoder::EmitFrames() { @@ -483,31 +537,38 @@ void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, OSStatus status, VTEncodeInfoFlags info, CMSampleBufferRef sbuf) { - if (status != noErr) { - DLOG(ERROR) << " encoding failed: " << status; - return; - } - if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) { - DVLOG(2) << " frame dropped"; - return; - } - auto encoder = reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque); const scoped_ptr<InProgressFrameEncode> request( reinterpret_cast<InProgressFrameEncode*>(request_opaque)); - auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex( - CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0)); + bool keyframe = false; + bool has_frame_data = false; - // If the NotSync key is not present, it implies Sync, which indicates a - // keyframe (at least I think, VT documentation is, erm, sparse). Could - // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false. - bool keyframe = - !CFDictionaryContainsKey(sample_attachments, - CoreMediaGlue::kCMSampleAttachmentKey_NotSync()); + if (status != noErr) { + DLOG(ERROR) << " encoding failed: " << status; + encoder->cast_environment_->PostTask( + CastEnvironment::MAIN, + FROM_HERE, + base::Bind(encoder->status_change_cb_, STATUS_CODEC_RUNTIME_ERROR)); + } else if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) { + DVLOG(2) << " frame dropped"; + } else { + auto sample_attachments = static_cast<CFDictionaryRef>( + CFArrayGetValueAtIndex( + CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), + 0)); + + // If the NotSync key is not present, it implies Sync, which indicates a + // keyframe (at least I think, VT documentation is, erm, sparse). Could + // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false. + keyframe = !CFDictionaryContainsKey( + sample_attachments, + CoreMediaGlue::kCMSampleAttachmentKey_NotSync()); + has_frame_data = true; + } // Increment the encoder-scoped frame id and assign the new value to this // frame. VideoToolbox calls the output callback serially, so this is safe. - uint32 frame_id = ++encoder->frame_id_; + const uint32 frame_id = encoder->next_frame_id_++; scoped_ptr<EncodedFrame> encoded_frame(new EncodedFrame()); encoded_frame->frame_id = frame_id; @@ -530,7 +591,8 @@ void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, encoded_frame->referenced_frame_id = frame_id - 1; } - CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe); + if (has_frame_data) + CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe); encoder->cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, @@ -538,5 +600,89 @@ void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, base::Passed(&encoded_frame))); } +// A ref-counted structure that is shared to provide concurrent access to the +// VideoFrameFactory instance for the current encoder. OnEncoderReplaced() can +// change |factory| whenever an encoder instance has been replaced, while users +// of CreateVideoFrameFactory() may attempt to read/use |factory| by any thread +// at any time. +struct SizeAdaptableH264VideoToolboxVideoEncoder::FactoryHolder + : public base::RefCountedThreadSafe<FactoryHolder> { + base::Lock lock; + scoped_ptr<VideoFrameFactory> factory; + + private: + friend class base::RefCountedThreadSafe<FactoryHolder>; + ~FactoryHolder() {} +}; + +SizeAdaptableH264VideoToolboxVideoEncoder:: + SizeAdaptableH264VideoToolboxVideoEncoder( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb) + : SizeAdaptableVideoEncoderBase(cast_environment, + video_config, + status_change_cb), + holder_(new FactoryHolder()) {} + +SizeAdaptableH264VideoToolboxVideoEncoder:: + ~SizeAdaptableH264VideoToolboxVideoEncoder() {} + +// A proxy allowing SizeAdaptableH264VideoToolboxVideoEncoder to swap out the +// VideoFrameFactory instance to match one appropriate for the current encoder +// instance. +class SizeAdaptableH264VideoToolboxVideoEncoder::VideoFrameFactoryProxy + : public VideoFrameFactory { + public: + explicit VideoFrameFactoryProxy(const scoped_refptr<FactoryHolder>& holder) + : holder_(holder) {} + + ~VideoFrameFactoryProxy() override {} + + scoped_refptr<VideoFrame> MaybeCreateFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) override { + base::AutoLock auto_lock(holder_->lock); + return holder_->factory ? + holder_->factory->MaybeCreateFrame(frame_size, timestamp) : nullptr; + } + + private: + const scoped_refptr<FactoryHolder> holder_; + + DISALLOW_COPY_AND_ASSIGN(VideoFrameFactoryProxy); +}; + +scoped_ptr<VideoFrameFactory> + SizeAdaptableH264VideoToolboxVideoEncoder::CreateVideoFrameFactory() { + return scoped_ptr<VideoFrameFactory>(new VideoFrameFactoryProxy(holder_)); +} + +scoped_ptr<VideoEncoder> + SizeAdaptableH264VideoToolboxVideoEncoder::CreateEncoder() { + return scoped_ptr<VideoEncoder>(new H264VideoToolboxEncoder( + cast_environment(), + video_config(), + frame_size(), + last_frame_id() + 1, + CreateEncoderStatusChangeCallback())); +} + +void SizeAdaptableH264VideoToolboxVideoEncoder::OnEncoderReplaced( + VideoEncoder* replacement_encoder) { + scoped_ptr<VideoFrameFactory> current_factory( + replacement_encoder->CreateVideoFrameFactory()); + base::AutoLock auto_lock(holder_->lock); + holder_->factory = current_factory.Pass(); + SizeAdaptableVideoEncoderBase::OnEncoderReplaced(replacement_encoder); +} + +void SizeAdaptableH264VideoToolboxVideoEncoder::DestroyEncoder() { + { + base::AutoLock auto_lock(holder_->lock); + holder_->factory.reset(); + } + SizeAdaptableVideoEncoderBase::DestroyEncoder(); +} + } // namespace cast } // namespace media diff --git a/media/cast/sender/h264_vt_encoder.h b/media/cast/sender/h264_vt_encoder.h index 349075a..ece6ab3 100644 --- a/media/cast/sender/h264_vt_encoder.h +++ b/media/cast/sender/h264_vt_encoder.h @@ -8,6 +8,7 @@ #include "base/mac/scoped_cftyperef.h" #include "base/threading/thread_checker.h" #include "media/base/mac/videotoolbox_glue.h" +#include "media/cast/sender/size_adaptable_video_encoder_base.h" #include "media/cast/sender/video_encoder.h" namespace media { @@ -22,15 +23,19 @@ class H264VideoToolboxEncoder : public VideoEncoder { typedef VideoToolboxGlue::VTEncodeInfoFlags VTEncodeInfoFlags; public: + // Returns true if the current platform and system configuration supports + // using H264VideoToolboxEncoder with the given |video_config|. + static bool IsSupported(const VideoSenderConfig& video_config); + H264VideoToolboxEncoder( const scoped_refptr<CastEnvironment>& cast_environment, const VideoSenderConfig& video_config, const gfx::Size& frame_size, + uint32 first_frame_id, const StatusChangeCallback& status_change_cb); ~H264VideoToolboxEncoder() override; // media::cast::VideoEncoder implementation - bool CanEncodeVariedFrameSizes() const override; bool EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, @@ -43,8 +48,7 @@ class H264VideoToolboxEncoder : public VideoEncoder { private: // Initialize the compression session. - bool Initialize(const VideoSenderConfig& video_config, - const gfx::Size& frame_size); + bool Initialize(const VideoSenderConfig& video_config); // Configure the compression session. void ConfigureSession(const VideoSenderConfig& video_config); @@ -68,7 +72,13 @@ class H264VideoToolboxEncoder : public VideoEncoder { const scoped_refptr<CastEnvironment> cast_environment_; // VideoToolboxGlue provides access to VideoToolbox at runtime. - const VideoToolboxGlue* videotoolbox_glue_; + const VideoToolboxGlue* const videotoolbox_glue_; + + // The size of the visible region of the video frames to be encoded. + const gfx::Size frame_size_; + + // Callback used to report initialization status and runtime errors. + const StatusChangeCallback status_change_cb_; // Thread checker to enforce that this object is used on a specific thread. base::ThreadChecker thread_checker_; @@ -77,7 +87,7 @@ class H264VideoToolboxEncoder : public VideoEncoder { base::ScopedCFTypeRef<VTCompressionSessionRef> compression_session_; // Frame identifier counter. - uint32 frame_id_; + uint32 next_frame_id_; // Force next frame to be a keyframe. bool encode_next_frame_as_keyframe_; @@ -85,6 +95,34 @@ class H264VideoToolboxEncoder : public VideoEncoder { DISALLOW_COPY_AND_ASSIGN(H264VideoToolboxEncoder); }; +// An implementation of SizeAdaptableVideoEncoderBase to proxy for +// H264VideoToolboxEncoder instances. +class SizeAdaptableH264VideoToolboxVideoEncoder + : public SizeAdaptableVideoEncoderBase { + public: + SizeAdaptableH264VideoToolboxVideoEncoder( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb); + + ~SizeAdaptableH264VideoToolboxVideoEncoder() override; + + scoped_ptr<VideoFrameFactory> CreateVideoFrameFactory() override; + + protected: + scoped_ptr<VideoEncoder> CreateEncoder() override; + void OnEncoderReplaced(VideoEncoder* replacement_encoder) override; + void DestroyEncoder() override; + + private: + struct FactoryHolder; + class VideoFrameFactoryProxy; + + const scoped_refptr<FactoryHolder> holder_; + + DISALLOW_COPY_AND_ASSIGN(SizeAdaptableH264VideoToolboxVideoEncoder); +}; + } // namespace cast } // namespace media diff --git a/media/cast/sender/h264_vt_encoder_unittest.cc b/media/cast/sender/h264_vt_encoder_unittest.cc index 78012eb..04f3d629 100644 --- a/media/cast/sender/h264_vt_encoder_unittest.cc +++ b/media/cast/sender/h264_vt_encoder_unittest.cc @@ -176,7 +176,9 @@ class EndToEndFrameChecker }; void CreateFrameAndMemsetPlane(VideoFrameFactory* const video_frame_factory) { - auto video_frame = video_frame_factory->CreateFrame(base::TimeDelta()); + const scoped_refptr<media::VideoFrame> video_frame = + video_frame_factory->MaybeCreateFrame( + gfx::Size(kVideoWidth, kVideoHeight), base::TimeDelta()); ASSERT_TRUE(video_frame.get()); auto cv_pixel_buffer = video_frame->cv_pixel_buffer(); ASSERT_TRUE(cv_pixel_buffer); @@ -207,6 +209,7 @@ class H264VideoToolboxEncoderTest : public ::testing::Test { cast_environment_, video_sender_config_, gfx::Size(kVideoWidth, kVideoHeight), + 0u, base::Bind(&SaveOperationalStatus, &operational_status_))); message_loop_.RunUntilIdle(); EXPECT_EQ(STATUS_INITIALIZED, operational_status_); @@ -304,6 +307,8 @@ TEST_F(H264VideoToolboxEncoderTest, CheckVideoFrameFactory) { auto video_frame_factory = encoder_->CreateVideoFrameFactory(); ASSERT_TRUE(video_frame_factory.get()); CreateFrameAndMemsetPlane(video_frame_factory.get()); + // TODO(jfroy): Need to test that the encoder can encode VideoFrames provided + // by the VideoFrameFactory. encoder_.reset(); message_loop_.RunUntilIdle(); CreateFrameAndMemsetPlane(video_frame_factory.get()); diff --git a/media/cast/sender/size_adaptable_video_encoder_base.cc b/media/cast/sender/size_adaptable_video_encoder_base.cc new file mode 100644 index 0000000..96c65a0 --- /dev/null +++ b/media/cast/sender/size_adaptable_video_encoder_base.cc @@ -0,0 +1,168 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/cast/sender/size_adaptable_video_encoder_base.h" + +#include "base/bind.h" +#include "base/location.h" +#include "media/base/video_frame.h" + +namespace media { +namespace cast { + +SizeAdaptableVideoEncoderBase::SizeAdaptableVideoEncoderBase( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb) + : cast_environment_(cast_environment), + video_config_(video_config), + status_change_cb_(status_change_cb), + frames_in_encoder_(0), + last_frame_id_(kStartFrameId), + weak_factory_(this) { + cast_environment_->PostTask( + CastEnvironment::MAIN, + FROM_HERE, + base::Bind(status_change_cb_, STATUS_INITIALIZED)); +} + +SizeAdaptableVideoEncoderBase::~SizeAdaptableVideoEncoderBase() { + DestroyEncoder(); +} + +bool SizeAdaptableVideoEncoderBase::EncodeVideoFrame( + const scoped_refptr<media::VideoFrame>& video_frame, + const base::TimeTicks& reference_time, + const FrameEncodedCallback& frame_encoded_callback) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + + const gfx::Size frame_size = video_frame->visible_rect().size(); + if (frame_size.IsEmpty()) { + DVLOG(1) << "Rejecting empty video frame."; + return false; + } + if (frames_in_encoder_ == kEncoderIsInitializing) { + VLOG(1) << "Dropping frame since encoder initialization is in-progress."; + return false; + } + if (frame_size != frame_size_ || !encoder_) { + VLOG(1) << "Dropping this frame, and future frames until a replacement " + "encoder is spun-up to handle size " << frame_size.ToString(); + TrySpawningReplacementEncoder(frame_size); + return false; + } + + const bool is_frame_accepted = encoder_->EncodeVideoFrame( + video_frame, + reference_time, + base::Bind(&SizeAdaptableVideoEncoderBase::OnEncodedVideoFrame, + weak_factory_.GetWeakPtr(), + frame_encoded_callback)); + if (is_frame_accepted) + ++frames_in_encoder_; + return is_frame_accepted; +} + +void SizeAdaptableVideoEncoderBase::SetBitRate(int new_bit_rate) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + video_config_.start_bitrate = new_bit_rate; + if (encoder_) + encoder_->SetBitRate(new_bit_rate); +} + +void SizeAdaptableVideoEncoderBase::GenerateKeyFrame() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (encoder_) + encoder_->GenerateKeyFrame(); +} + +void SizeAdaptableVideoEncoderBase::LatestFrameIdToReference(uint32 frame_id) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (encoder_) + encoder_->LatestFrameIdToReference(frame_id); +} + +scoped_ptr<VideoFrameFactory> + SizeAdaptableVideoEncoderBase::CreateVideoFrameFactory() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + return nullptr; +} + +void SizeAdaptableVideoEncoderBase::EmitFrames() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (encoder_) + encoder_->EmitFrames(); +} + +StatusChangeCallback + SizeAdaptableVideoEncoderBase::CreateEncoderStatusChangeCallback() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + return base::Bind(&SizeAdaptableVideoEncoderBase::OnEncoderStatusChange, + weak_factory_.GetWeakPtr()); +} + +void SizeAdaptableVideoEncoderBase::OnEncoderReplaced( + VideoEncoder* replacement_encoder) {} + +void SizeAdaptableVideoEncoderBase::DestroyEncoder() { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + // The weak pointers are invalidated to prevent future calls back to |this|. + // This effectively cancels any of |encoder_|'s posted tasks that have not yet + // run. + weak_factory_.InvalidateWeakPtrs(); + encoder_.reset(); +} + +void SizeAdaptableVideoEncoderBase::TrySpawningReplacementEncoder( + const gfx::Size& size_needed) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + + // If prior frames are still encoding in the current encoder, let them finish + // first. + if (frames_in_encoder_ > 0) { + encoder_->EmitFrames(); + // Check again, since EmitFrames() is a synchronous operation for some + // encoders. + if (frames_in_encoder_ > 0) + return; + } + + if (frames_in_encoder_ == kEncoderIsInitializing) + return; // Already spawned. + + DestroyEncoder(); + frames_in_encoder_ = kEncoderIsInitializing; + OnEncoderStatusChange(STATUS_CODEC_REINIT_PENDING); + VLOG(1) << "Creating replacement video encoder (for frame size change from " + << frame_size_.ToString() << " to " + << size_needed.ToString() << ")."; + frame_size_ = size_needed; + encoder_ = CreateEncoder().Pass(); + DCHECK(encoder_); +} + +void SizeAdaptableVideoEncoderBase::OnEncoderStatusChange( + OperationalStatus status) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + if (frames_in_encoder_ == kEncoderIsInitializing && + status == STATUS_INITIALIZED) { + // Begin using the replacement encoder. + frames_in_encoder_ = 0; + OnEncoderReplaced(encoder_.get()); + } + status_change_cb_.Run(status); +} + +void SizeAdaptableVideoEncoderBase::OnEncodedVideoFrame( + const FrameEncodedCallback& frame_encoded_callback, + scoped_ptr<EncodedFrame> encoded_frame) { + DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + --frames_in_encoder_; + DCHECK_GE(frames_in_encoder_, 0); + last_frame_id_ = encoded_frame->frame_id; + frame_encoded_callback.Run(encoded_frame.Pass()); +} + +} // namespace cast +} // namespace media diff --git a/media/cast/sender/size_adaptable_video_encoder_base.h b/media/cast/sender/size_adaptable_video_encoder_base.h new file mode 100644 index 0000000..293a684 --- /dev/null +++ b/media/cast/sender/size_adaptable_video_encoder_base.h @@ -0,0 +1,120 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_CAST_SENDER_SIZE_ADAPTABLE_VIDEO_ENCODER_BASE_H_ +#define MEDIA_CAST_SENDER_SIZE_ADAPTABLE_VIDEO_ENCODER_BASE_H_ + +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/weak_ptr.h" +#include "media/cast/cast_config.h" +#include "media/cast/cast_environment.h" +#include "media/cast/sender/video_encoder.h" +#include "ui/gfx/geometry/size.h" + +namespace media { +namespace cast { + +// Creates and owns a VideoEncoder instance. The owned instance is an +// implementation that does not support changing frame sizes, and so +// SizeAdaptableVideoEncoderBase acts as a proxy to automatically detect when +// the owned instance should be replaced with one that can handle the new frame +// size. +class SizeAdaptableVideoEncoderBase : public VideoEncoder { + public: + SizeAdaptableVideoEncoderBase( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb); + + ~SizeAdaptableVideoEncoderBase() override; + + // VideoEncoder implementation. + bool EncodeVideoFrame( + const scoped_refptr<media::VideoFrame>& video_frame, + const base::TimeTicks& reference_time, + const FrameEncodedCallback& frame_encoded_callback) override; + void SetBitRate(int new_bit_rate) override; + void GenerateKeyFrame() override; + void LatestFrameIdToReference(uint32 frame_id) override; + scoped_ptr<VideoFrameFactory> CreateVideoFrameFactory() override; + void EmitFrames() override; + + protected: + // Accessors for subclasses. + CastEnvironment* cast_environment() const { + return cast_environment_.get(); + } + const VideoSenderConfig& video_config() const { + return video_config_; + } + const gfx::Size& frame_size() const { + return frame_size_; + } + uint32 last_frame_id() const { + return last_frame_id_; + } + + // Returns a callback that calls OnEncoderStatusChange(). The callback is + // canceled by invalidating its bound weak pointer just before a replacement + // encoder is instantiated. In this scheme, OnEncoderStatusChange() can only + // be called by the most-recent encoder. + StatusChangeCallback CreateEncoderStatusChangeCallback(); + + // Overridden by subclasses to create a new encoder instance that handles + // frames of the size specified by |frame_size()|. + virtual scoped_ptr<VideoEncoder> CreateEncoder() = 0; + + // Overridden by subclasses to perform additional steps when + // |replacement_encoder| becomes the active encoder. + virtual void OnEncoderReplaced(VideoEncoder* replacement_encoder); + + // Overridden by subclasses to perform additional steps before/after the + // current encoder is destroyed. + virtual void DestroyEncoder(); + + private: + // Create and initialize a replacement video encoder, if this not already + // in-progress. The replacement will call back to OnEncoderStatusChange() + // with success/fail status. + void TrySpawningReplacementEncoder(const gfx::Size& size_needed); + + // Called when a status change is received from an encoder. + void OnEncoderStatusChange(OperationalStatus status); + + // Called by the |encoder_| with the next EncodedFrame. + void OnEncodedVideoFrame(const FrameEncodedCallback& frame_encoded_callback, + scoped_ptr<EncodedFrame> encoded_frame); + + const scoped_refptr<CastEnvironment> cast_environment_; + + // This is not const since |video_config_.starting_bitrate| is modified by + // SetBitRate(), for when a replacement encoder is spawned. + VideoSenderConfig video_config_; + + // Run whenever the underlying encoder reports a status change. + const StatusChangeCallback status_change_cb_; + + // The underlying platform video encoder and the frame size it expects. + scoped_ptr<VideoEncoder> encoder_; + gfx::Size frame_size_; + + // The number of frames in |encoder_|'s pipeline. If this is set to + // kEncoderIsInitializing, |encoder_| is not yet ready to accept frames. + enum { kEncoderIsInitializing = -1 }; + int frames_in_encoder_; + + // The ID of the last frame that was emitted from |encoder_|. + uint32 last_frame_id_; + + // NOTE: Weak pointers must be invalidated before all other member variables. + base::WeakPtrFactory<SizeAdaptableVideoEncoderBase> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(SizeAdaptableVideoEncoderBase); +}; + +} // namespace cast +} // namespace media + +#endif // MEDIA_CAST_SENDER_SIZE_ADAPTABLE_VIDEO_ENCODER_BASE_H_ diff --git a/media/cast/sender/video_encoder.cc b/media/cast/sender/video_encoder.cc index d6b62a8..d65bf3c 100644 --- a/media/cast/sender/video_encoder.cc +++ b/media/cast/sender/video_encoder.cc @@ -3,11 +3,61 @@ // found in the LICENSE file. #include "media/cast/sender/video_encoder.h" -#include "media/cast/sender/video_frame_factory.h" + +#include "media/cast/sender/external_video_encoder.h" +#include "media/cast/sender/video_encoder_impl.h" + +#if defined(OS_MACOSX) +#include "media/cast/sender/h264_vt_encoder.h" +#endif namespace media { namespace cast { +// static +scoped_ptr<VideoEncoder> VideoEncoder::Create( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb, + const CreateVideoEncodeAcceleratorCallback& create_vea_cb, + const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb) { + // On MacOS or IOS, attempt to use the system VideoToolbox library to + // perform optimized H.264 encoding. +#if defined(OS_MACOSX) || defined(OS_IOS) + if (!video_config.use_external_encoder && + H264VideoToolboxEncoder::IsSupported(video_config)) { + return scoped_ptr<VideoEncoder>( + new SizeAdaptableH264VideoToolboxVideoEncoder( + cast_environment, + video_config, + status_change_cb)); + } +#endif // defined(OS_MACOSX) + + // If the system provides a hardware-accelerated encoder, use it. +#if !defined(OS_IOS) + if (ExternalVideoEncoder::IsSupported(video_config)) { + return scoped_ptr<VideoEncoder>(new SizeAdaptableExternalVideoEncoder( + cast_environment, + video_config, + status_change_cb, + create_vea_cb, + create_video_encode_memory_cb)); + } +#endif // !defined(OS_IOS) + + // Attempt to use the software encoder implementation. + if (VideoEncoderImpl::IsSupported(video_config)) { + return scoped_ptr<VideoEncoder>(new VideoEncoderImpl( + cast_environment, + video_config, + status_change_cb)); + } + + // No encoder implementation will suffice. + return nullptr; +} + scoped_ptr<VideoFrameFactory> VideoEncoder::CreateVideoFrameFactory() { return nullptr; } diff --git a/media/cast/sender/video_encoder.h b/media/cast/sender/video_encoder.h index 92e8547..94b0bbf 100644 --- a/media/cast/sender/video_encoder.h +++ b/media/cast/sender/video_encoder.h @@ -8,27 +8,37 @@ #include "base/callback.h" #include "base/memory/ref_counted.h" #include "base/memory/scoped_ptr.h" -#include "base/memory/weak_ptr.h" #include "base/time/time.h" #include "media/base/video_frame.h" #include "media/cast/cast_config.h" #include "media/cast/cast_environment.h" +#include "media/cast/sender/video_frame_factory.h" namespace media { namespace cast { -class VideoFrameFactory; - // All these functions are called from the main cast thread. class VideoEncoder { public: typedef base::Callback<void(scoped_ptr<EncodedFrame>)> FrameEncodedCallback; - virtual ~VideoEncoder() {} + // Creates a VideoEncoder instance from the given |video_config| and based on + // the current platform's hardware/library support; or null if no + // implementation will suffice. The instance will run |status_change_cb| at + // some point in the future to indicate initialization success/failure. + // + // All VideoEncoder instances returned by this function support encoding + // sequences of differently-size VideoFrames. + // + // TODO(miu): Remove the CreateVEA callbacks. http://crbug.com/454029 + static scoped_ptr<VideoEncoder> Create( + const scoped_refptr<CastEnvironment>& cast_environment, + const VideoSenderConfig& video_config, + const StatusChangeCallback& status_change_cb, + const CreateVideoEncodeAcceleratorCallback& create_vea_cb, + const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb); - // Returns true if the size of video frames passed in successive calls to - // EncodedVideoFrame() can vary. - virtual bool CanEncodeVariedFrameSizes() const = 0; + virtual ~VideoEncoder() {} // If true is returned, the Encoder has accepted the request and will process // it asynchronously, running |frame_encoded_callback| on the MAIN diff --git a/media/cast/sender/video_encoder_impl.cc b/media/cast/sender/video_encoder_impl.cc index a2b0da9..84f700e 100644 --- a/media/cast/sender/video_encoder_impl.cc +++ b/media/cast/sender/video_encoder_impl.cc @@ -52,6 +52,15 @@ void EncodeVideoFrameOnEncoderThread( } } // namespace +// static +bool VideoEncoderImpl::IsSupported(const VideoSenderConfig& video_config) { +#ifndef OFFICIAL_BUILD + if (video_config.codec == CODEC_VIDEO_FAKE) + return true; +#endif + return video_config.codec == CODEC_VIDEO_VP8; +} + VideoEncoderImpl::VideoEncoderImpl( scoped_refptr<CastEnvironment> cast_environment, const VideoSenderConfig& video_config, @@ -98,12 +107,6 @@ VideoEncoderImpl::~VideoEncoderImpl() { } } -bool VideoEncoderImpl::CanEncodeVariedFrameSizes() const { - // Both the VP8Encoder and FakeSoftwareVideoEncoder support calls to - // EncodeVideoFrame() with different frame sizes. - return true; -} - bool VideoEncoderImpl::EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, diff --git a/media/cast/sender/video_encoder_impl.h b/media/cast/sender/video_encoder_impl.h index 07ed999..7572c4a 100644 --- a/media/cast/sender/video_encoder_impl.h +++ b/media/cast/sender/video_encoder_impl.h @@ -29,6 +29,9 @@ class VideoEncoderImpl : public VideoEncoder { typedef base::Callback<void(scoped_ptr<EncodedFrame>)> FrameEncodedCallback; + // Returns true if VideoEncoderImpl can be used with the given |video_config|. + static bool IsSupported(const VideoSenderConfig& video_config); + VideoEncoderImpl(scoped_refptr<CastEnvironment> cast_environment, const VideoSenderConfig& video_config, const StatusChangeCallback& status_change_cb); @@ -36,7 +39,6 @@ class VideoEncoderImpl : public VideoEncoder { ~VideoEncoderImpl() override; // VideoEncoder implementation. - bool CanEncodeVariedFrameSizes() const override; bool EncodeVideoFrame( const scoped_refptr<media::VideoFrame>& video_frame, const base::TimeTicks& reference_time, diff --git a/media/cast/sender/video_encoder_impl_unittest.cc b/media/cast/sender/video_encoder_impl_unittest.cc deleted file mode 100644 index dab16ea..0000000 --- a/media/cast/sender/video_encoder_impl_unittest.cc +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <vector> - -#include "base/bind.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "media/base/video_frame.h" -#include "media/cast/cast_defines.h" -#include "media/cast/cast_environment.h" -#include "media/cast/sender/video_encoder_impl.h" -#include "media/cast/test/fake_single_thread_task_runner.h" -#include "media/cast/test/utility/default_config.h" -#include "media/cast/test/utility/video_utility.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace media { -namespace cast { - -class VideoEncoderImplTest : public ::testing::TestWithParam<Codec> { - protected: - VideoEncoderImplTest() - : testing_clock_(new base::SimpleTestTickClock()), - task_runner_(new test::FakeSingleThreadTaskRunner(testing_clock_)), - cast_environment_(new CastEnvironment( - scoped_ptr<base::TickClock>(testing_clock_).Pass(), - task_runner_, - task_runner_, - task_runner_)), - video_config_(GetDefaultVideoSenderConfig()), - operational_status_(STATUS_UNINITIALIZED), - count_frames_delivered_(0) { - testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); - first_frame_time_ = testing_clock_->NowTicks(); - } - - ~VideoEncoderImplTest() override {} - - void SetUp() override { - video_config_.codec = GetParam(); - } - - void TearDown() override { - video_encoder_.reset(); - task_runner_->RunTasks(); - } - - void CreateEncoder(bool three_buffer_mode) { - ASSERT_EQ(STATUS_UNINITIALIZED, operational_status_); - video_config_.max_number_of_video_buffers_used = - (three_buffer_mode ? 3 : 1); - video_encoder_.reset(new VideoEncoderImpl( - cast_environment_, - video_config_, - base::Bind(&VideoEncoderImplTest::OnOperationalStatusChange, - base::Unretained(this)))); - task_runner_->RunTasks(); - ASSERT_EQ(STATUS_INITIALIZED, operational_status_); - } - - VideoEncoder* video_encoder() const { - return video_encoder_.get(); - } - - void AdvanceClock() { - testing_clock_->Advance(base::TimeDelta::FromMilliseconds(33)); - } - - base::TimeTicks Now() const { - return testing_clock_->NowTicks(); - } - - void RunTasks() const { - return task_runner_->RunTasks(); - } - - int count_frames_delivered() const { - return count_frames_delivered_; - } - - // Return a callback that, when run, expects the EncodedFrame to have the - // given properties. - VideoEncoder::FrameEncodedCallback CreateFrameDeliverCallback( - uint32 expected_frame_id, - uint32 expected_last_referenced_frame_id, - uint32 expected_rtp_timestamp, - const base::TimeTicks& expected_reference_time) { - return base::Bind(&VideoEncoderImplTest::DeliverEncodedVideoFrame, - base::Unretained(this), - expected_frame_id, - expected_last_referenced_frame_id, - expected_rtp_timestamp, - expected_reference_time); - } - - // Creates a new VideoFrame of the given |size|, filled with a test pattern. - scoped_refptr<media::VideoFrame> CreateTestVideoFrame( - const gfx::Size& size) const { - const scoped_refptr<media::VideoFrame> frame = - media::VideoFrame::CreateFrame( - VideoFrame::I420, size, gfx::Rect(size), size, - testing_clock_->NowTicks() - first_frame_time_); - PopulateVideoFrame(frame.get(), 123); - return frame; - } - - private: - void OnOperationalStatusChange(OperationalStatus status) { - operational_status_ = status; - } - - // Checks that |encoded_frame| matches expected values. This is the method - // bound in the callback returned from CreateFrameDeliverCallback(). - void DeliverEncodedVideoFrame( - uint32 expected_frame_id, - uint32 expected_last_referenced_frame_id, - uint32 expected_rtp_timestamp, - const base::TimeTicks& expected_reference_time, - scoped_ptr<EncodedFrame> encoded_frame) { - if (expected_frame_id != expected_last_referenced_frame_id) { - EXPECT_EQ(EncodedFrame::DEPENDENT, encoded_frame->dependency); - } else if (video_config_.max_number_of_video_buffers_used == 1) { - EXPECT_EQ(EncodedFrame::KEY, encoded_frame->dependency); - } - EXPECT_EQ(expected_frame_id, encoded_frame->frame_id); - EXPECT_EQ(expected_last_referenced_frame_id, - encoded_frame->referenced_frame_id) - << "frame id: " << expected_frame_id; - EXPECT_EQ(expected_rtp_timestamp, encoded_frame->rtp_timestamp); - EXPECT_EQ(expected_reference_time, encoded_frame->reference_time); - EXPECT_FALSE(encoded_frame->data.empty()); - ++count_frames_delivered_; - } - - base::SimpleTestTickClock* const testing_clock_; // Owned by CastEnvironment. - const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; - const scoped_refptr<CastEnvironment> cast_environment_; - VideoSenderConfig video_config_; - base::TimeTicks first_frame_time_; - OperationalStatus operational_status_; - scoped_ptr<VideoEncoder> video_encoder_; - - int count_frames_delivered_; - - DISALLOW_COPY_AND_ASSIGN(VideoEncoderImplTest); -}; - -// A simple test to encode ten frames of video, expecting to see one key frame -// followed by nine delta frames. -TEST_P(VideoEncoderImplTest, GeneratesKeyFrameThenOnlyDeltaFrames) { - CreateEncoder(false); - - EXPECT_EQ(0, count_frames_delivered()); - - scoped_refptr<media::VideoFrame> video_frame = - CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - 0, 0, TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - - for (uint32 frame_id = 1; frame_id < 10; ++frame_id) { - AdvanceClock(); - video_frame = CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - frame_id, frame_id - 1, - TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - } - - EXPECT_EQ(10, count_frames_delivered()); -} - -// Tests basic frame dependency rules when using the VP8 encoder in multi-buffer -// mode. -TEST_P(VideoEncoderImplTest, - FramesDoNotDependOnUnackedFramesInMultiBufferMode) { - if (GetParam() != CODEC_VIDEO_VP8) - return; // Only test multibuffer mode for the VP8 encoder. - CreateEncoder(true); - - EXPECT_EQ(0, count_frames_delivered()); - - scoped_refptr<media::VideoFrame> video_frame = - CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - 0, 0, TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - - AdvanceClock(); - video_encoder()->LatestFrameIdToReference(0); - video_frame = CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - 1, 0, TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - - AdvanceClock(); - video_encoder()->LatestFrameIdToReference(1); - video_frame = CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - 2, 1, TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - - video_encoder()->LatestFrameIdToReference(2); - - for (uint32 frame_id = 3; frame_id < 10; ++frame_id) { - AdvanceClock(); - video_frame = CreateTestVideoFrame(gfx::Size(1280, 720)); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - frame_id, 2, - TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - } - - EXPECT_EQ(10, count_frames_delivered()); -} - -// Tests that the encoder continues to output EncodedFrames as the frame size -// changes. See media/cast/receiver/video_decoder_unittest.cc for a complete -// encode/decode cycle of varied frame sizes that actually checks the frame -// content. -TEST_P(VideoEncoderImplTest, EncodesVariedFrameSizes) { - CreateEncoder(false); - ASSERT_TRUE(video_encoder()->CanEncodeVariedFrameSizes()); - - EXPECT_EQ(0, count_frames_delivered()); - - std::vector<gfx::Size> frame_sizes; - frame_sizes.push_back(gfx::Size(1280, 720)); - frame_sizes.push_back(gfx::Size(640, 360)); // Shrink both dimensions. - frame_sizes.push_back(gfx::Size(300, 200)); // Shrink both dimensions again. - frame_sizes.push_back(gfx::Size(200, 300)); // Same area. - frame_sizes.push_back(gfx::Size(600, 400)); // Grow both dimensions. - frame_sizes.push_back(gfx::Size(638, 400)); // Shrink only one dimension. - frame_sizes.push_back(gfx::Size(638, 398)); // Shrink the other dimension. - frame_sizes.push_back(gfx::Size(320, 180)); // Shrink both dimensions again. - frame_sizes.push_back(gfx::Size(322, 180)); // Grow only one dimension. - frame_sizes.push_back(gfx::Size(322, 182)); // Grow the other dimension. - frame_sizes.push_back(gfx::Size(1920, 1080)); // Grow both dimensions again. - - uint32 frame_id = 0; - - // Encode one frame at each size. Expect nothing but key frames to come out. - for (const auto& frame_size : frame_sizes) { - AdvanceClock(); - const scoped_refptr<media::VideoFrame> video_frame = - CreateTestVideoFrame(frame_size); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - frame_id, - frame_id, - TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - ++frame_id; - } - - // Encode 10 frames at each size. Expect one key frame followed by nine delta - // frames for each frame size. - for (const auto& frame_size : frame_sizes) { - for (int i = 0; i < 10; ++i) { - AdvanceClock(); - const scoped_refptr<media::VideoFrame> video_frame = - CreateTestVideoFrame(frame_size); - EXPECT_TRUE(video_encoder()->EncodeVideoFrame( - video_frame, - Now(), - CreateFrameDeliverCallback( - frame_id, - i == 0 ? frame_id : frame_id - 1, - TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency), - Now()))); - RunTasks(); - ++frame_id; - } - } - - EXPECT_EQ(static_cast<int>(frame_id), count_frames_delivered()); -} - -INSTANTIATE_TEST_CASE_P(, - VideoEncoderImplTest, - ::testing::Values(CODEC_VIDEO_FAKE, CODEC_VIDEO_VP8)); - -} // namespace cast -} // namespace media diff --git a/media/cast/sender/video_encoder_unittest.cc b/media/cast/sender/video_encoder_unittest.cc new file mode 100644 index 0000000..a528e28 --- /dev/null +++ b/media/cast/sender/video_encoder_unittest.cc @@ -0,0 +1,454 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <vector> +#include <utility> + +#include "base/bind.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "media/base/video_frame.h" +#include "media/cast/cast_defines.h" +#include "media/cast/cast_environment.h" +#include "media/cast/sender/fake_video_encode_accelerator_factory.h" +#include "media/cast/sender/video_frame_factory.h" +#include "media/cast/sender/video_encoder.h" +#include "media/cast/test/fake_single_thread_task_runner.h" +#include "media/cast/test/utility/default_config.h" +#include "media/cast/test/utility/video_utility.h" +#include "testing/gtest/include/gtest/gtest.h" + +#if defined(OS_MACOSX) +#include "media/cast/sender/h264_vt_encoder.h" +#endif + +namespace media { +namespace cast { + +class VideoEncoderTest + : public ::testing::TestWithParam<std::pair<Codec, bool>> { + protected: + VideoEncoderTest() + : testing_clock_(new base::SimpleTestTickClock()), + task_runner_(new test::FakeSingleThreadTaskRunner(testing_clock_)), + cast_environment_(new CastEnvironment( + scoped_ptr<base::TickClock>(testing_clock_).Pass(), + task_runner_, + task_runner_, + task_runner_)), + video_config_(GetDefaultVideoSenderConfig()), + operational_status_(STATUS_UNINITIALIZED), + count_frames_delivered_(0) { + testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); + first_frame_time_ = testing_clock_->NowTicks(); + } + + ~VideoEncoderTest() override {} + + void SetUp() override { + video_config_.codec = GetParam().first; + video_config_.use_external_encoder = GetParam().second; + + if (video_config_.use_external_encoder) + vea_factory_.reset(new FakeVideoEncodeAcceleratorFactory(task_runner_)); + } + + void TearDown() override { + video_encoder_.reset(); + RunTasksAndAdvanceClock(); + } + + void CreateEncoder(bool three_buffer_mode) { + ASSERT_EQ(STATUS_UNINITIALIZED, operational_status_); + video_config_.max_number_of_video_buffers_used = + (three_buffer_mode ? 3 : 1); + video_encoder_ = VideoEncoder::Create( + cast_environment_, + video_config_, + base::Bind(&VideoEncoderTest::OnOperationalStatusChange, + base::Unretained(this)), + base::Bind( + &FakeVideoEncodeAcceleratorFactory::CreateVideoEncodeAccelerator, + base::Unretained(vea_factory_.get())), + base::Bind(&FakeVideoEncodeAcceleratorFactory::CreateSharedMemory, + base::Unretained(vea_factory_.get()))).Pass(); + RunTasksAndAdvanceClock(); + if (is_encoder_present()) + ASSERT_EQ(STATUS_INITIALIZED, operational_status_); + } + + bool is_encoder_present() const { + return !!video_encoder_; + } + + bool is_testing_software_vp8_encoder() const { + return video_config_.codec == CODEC_VIDEO_VP8 && + !video_config_.use_external_encoder; + } + + bool is_testing_video_toolbox_encoder() const { + return +#if defined(OS_MACOSX) + (!video_config_.use_external_encoder && + H264VideoToolboxEncoder::IsSupported(video_config_)) || +#endif + false; + } + + bool is_testing_platform_encoder() const { + return video_config_.use_external_encoder || + is_testing_video_toolbox_encoder(); + } + + VideoEncoder* video_encoder() const { + return video_encoder_.get(); + } + + void DestroyEncoder() { + video_encoder_.reset(); + } + + base::TimeTicks Now() const { + return testing_clock_->NowTicks(); + } + + void RunTasksAndAdvanceClock() const { + const base::TimeDelta frame_duration = base::TimeDelta::FromMicroseconds( + 1000000.0 / video_config_.max_frame_rate); +#if defined(OS_MACOSX) + if (is_testing_video_toolbox_encoder()) { + // The H264VideoToolboxEncoder (on MAC_OSX and IOS) is not a faked + // implementation in these tests, and performs its encoding asynchronously + // on an unknown set of threads. Therefore, sleep the current thread for + // the real amount of time to avoid excessively spinning the CPU while + // waiting for something to happen. + base::PlatformThread::Sleep(frame_duration); + } +#endif + task_runner_->RunTasks(); + testing_clock_->Advance(frame_duration); + } + + int count_frames_delivered() const { + return count_frames_delivered_; + } + + void WaitForAllFramesToBeDelivered(int total_expected) const { + video_encoder_->EmitFrames(); + while (count_frames_delivered_ < total_expected) + RunTasksAndAdvanceClock(); + } + + // Creates a new VideoFrame of the given |size|, filled with a test pattern. + // When available, it attempts to use the VideoFrameFactory provided by the + // encoder. + scoped_refptr<media::VideoFrame> CreateTestVideoFrame(const gfx::Size& size) { + const base::TimeDelta timestamp = + testing_clock_->NowTicks() - first_frame_time_; + scoped_refptr<media::VideoFrame> frame; + if (video_frame_factory_) + frame = video_frame_factory_->MaybeCreateFrame(size, timestamp); + if (!frame) { + frame = media::VideoFrame::CreateFrame( + VideoFrame::I420, size, gfx::Rect(size), size, timestamp); + } + PopulateVideoFrame(frame.get(), 123); + return frame; + } + + // Requests encoding the |video_frame| and has the resulting frame delivered + // via a callback that checks for expected results. Returns false if the + // encoder rejected the request. + bool EncodeAndCheckDelivery( + const scoped_refptr<media::VideoFrame>& video_frame, + uint32 frame_id, + uint32 reference_frame_id) { + return video_encoder_->EncodeVideoFrame( + video_frame, + Now(), + base::Bind(&VideoEncoderTest::DeliverEncodedVideoFrame, + base::Unretained(this), + frame_id, + reference_frame_id, + TimeDeltaToRtpDelta(video_frame->timestamp(), + kVideoFrequency), + Now())); + } + + // If the implementation of |video_encoder_| is ExternalVideoEncoder, check + // that the VEA factory has responded (by running the callbacks) a specific + // number of times. Otherwise, check that the VEA factory is inactive. + void ExpectVEAResponsesForExternalVideoEncoder( + int vea_response_count, + int shm_response_count) const { + if (!vea_factory_) + return; + EXPECT_EQ(vea_response_count, vea_factory_->vea_response_count()); + EXPECT_EQ(shm_response_count, vea_factory_->shm_response_count()); + } + + void SetVEAFactoryAutoRespond(bool auto_respond) { + if (vea_factory_) + vea_factory_->SetAutoRespond(auto_respond); + } + + private: + void OnOperationalStatusChange(OperationalStatus status) { + DVLOG(1) << "OnOperationalStatusChange: from " << operational_status_ + << " to " << status; + operational_status_ = status; + + EXPECT_TRUE(operational_status_ == STATUS_CODEC_REINIT_PENDING || + operational_status_ == STATUS_INITIALIZED); + + // Create the VideoFrameFactory the first time status changes to + // STATUS_INITIALIZED. + if (operational_status_ == STATUS_INITIALIZED && !video_frame_factory_) + video_frame_factory_ = video_encoder_->CreateVideoFrameFactory().Pass(); + } + + // Checks that |encoded_frame| matches expected values. This is the method + // bound in the callback returned from EncodeAndCheckDelivery(). + void DeliverEncodedVideoFrame( + uint32 expected_frame_id, + uint32 expected_last_referenced_frame_id, + uint32 expected_rtp_timestamp, + const base::TimeTicks& expected_reference_time, + scoped_ptr<EncodedFrame> encoded_frame) { + EXPECT_TRUE(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); + + EXPECT_EQ(expected_frame_id, encoded_frame->frame_id); + EXPECT_EQ(expected_rtp_timestamp, encoded_frame->rtp_timestamp); + EXPECT_EQ(expected_reference_time, encoded_frame->reference_time); + + // The platform encoders are "black boxes" and may choose to vend key frames + // and/or empty data at any time. The software encoders, however, should + // strictly adhere to expected behavior. + if (is_testing_platform_encoder()) { + const bool expected_key_frame = + expected_frame_id == expected_last_referenced_frame_id; + const bool have_key_frame = + encoded_frame->dependency == EncodedFrame::KEY; + EXPECT_EQ(have_key_frame, + encoded_frame->frame_id == encoded_frame->referenced_frame_id); + LOG_IF(WARNING, expected_key_frame != have_key_frame) + << "Platform encoder chose to emit a " + << (have_key_frame ? "key" : "delta") + << " frame instead of the expected kind @ frame_id=" + << encoded_frame->frame_id; + LOG_IF(WARNING, encoded_frame->data.empty()) + << "Platform encoder returned an empty frame @ frame_id=" + << encoded_frame->frame_id; + } else { + if (expected_frame_id != expected_last_referenced_frame_id) { + EXPECT_EQ(EncodedFrame::DEPENDENT, encoded_frame->dependency); + } else if (video_config_.max_number_of_video_buffers_used == 1) { + EXPECT_EQ(EncodedFrame::KEY, encoded_frame->dependency); + } + EXPECT_EQ(expected_last_referenced_frame_id, + encoded_frame->referenced_frame_id); + EXPECT_FALSE(encoded_frame->data.empty()); + } + + ++count_frames_delivered_; + } + + base::SimpleTestTickClock* const testing_clock_; // Owned by CastEnvironment. + const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; + const scoped_refptr<CastEnvironment> cast_environment_; + VideoSenderConfig video_config_; + scoped_ptr<FakeVideoEncodeAcceleratorFactory> vea_factory_; + base::TimeTicks first_frame_time_; + OperationalStatus operational_status_; + scoped_ptr<VideoEncoder> video_encoder_; + scoped_ptr<VideoFrameFactory> video_frame_factory_; + + int count_frames_delivered_; + + DISALLOW_COPY_AND_ASSIGN(VideoEncoderTest); +}; + +// A simple test to encode ten frames of video, expecting to see one key frame +// followed by nine delta frames. +TEST_P(VideoEncoderTest, GeneratesKeyFrameThenOnlyDeltaFrames) { + CreateEncoder(false); + SetVEAFactoryAutoRespond(true); + + EXPECT_EQ(0, count_frames_delivered()); + ExpectVEAResponsesForExternalVideoEncoder(0, 0); + + uint32 frame_id = 0; + uint32 reference_frame_id = 0; + const gfx::Size frame_size(1280, 720); + + // For the platform encoders, the first one or more frames is dropped while + // the encoder initializes. Then, for all encoders, expect one key frame is + // delivered. + bool accepted_first_frame = false; + do { + accepted_first_frame = EncodeAndCheckDelivery( + CreateTestVideoFrame(frame_size), frame_id, reference_frame_id); + if (!is_testing_platform_encoder()) + EXPECT_TRUE(accepted_first_frame); + RunTasksAndAdvanceClock(); + } while (!accepted_first_frame); + ExpectVEAResponsesForExternalVideoEncoder(1, 3); + + // Expect the remaining frames are encoded as delta frames. + for (++frame_id; frame_id < 10; ++frame_id, ++reference_frame_id) { + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), + frame_id, + reference_frame_id)); + RunTasksAndAdvanceClock(); + } + + WaitForAllFramesToBeDelivered(10); + ExpectVEAResponsesForExternalVideoEncoder(1, 3); +} + +// Tests basic frame dependency rules when using the VP8 encoder in multi-buffer +// mode. +TEST_P(VideoEncoderTest, FramesDoNotDependOnUnackedFramesInMultiBufferMode) { + if (!is_testing_software_vp8_encoder()) + return; // Only test multibuffer mode for the software VP8 encoder. + CreateEncoder(true); + + EXPECT_EQ(0, count_frames_delivered()); + + const gfx::Size frame_size(1280, 720); + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 0, 0)); + RunTasksAndAdvanceClock(); + + video_encoder()->LatestFrameIdToReference(0); + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 1, 0)); + RunTasksAndAdvanceClock(); + + video_encoder()->LatestFrameIdToReference(1); + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), 2, 1)); + RunTasksAndAdvanceClock(); + + video_encoder()->LatestFrameIdToReference(2); + + for (uint32 frame_id = 3; frame_id < 10; ++frame_id) { + EXPECT_TRUE(EncodeAndCheckDelivery( + CreateTestVideoFrame(frame_size), frame_id, 2)); + RunTasksAndAdvanceClock(); + } + + EXPECT_EQ(10, count_frames_delivered()); +} + +// Tests that the encoder continues to output EncodedFrames as the frame size +// changes. See media/cast/receiver/video_decoder_unittest.cc for a complete +// encode/decode cycle of varied frame sizes that actually checks the frame +// content. +TEST_P(VideoEncoderTest, EncodesVariedFrameSizes) { + CreateEncoder(false); + SetVEAFactoryAutoRespond(true); + + EXPECT_EQ(0, count_frames_delivered()); + ExpectVEAResponsesForExternalVideoEncoder(0, 0); + + std::vector<gfx::Size> frame_sizes; + frame_sizes.push_back(gfx::Size(1280, 720)); + frame_sizes.push_back(gfx::Size(640, 360)); // Shrink both dimensions. + frame_sizes.push_back(gfx::Size(300, 200)); // Shrink both dimensions again. + frame_sizes.push_back(gfx::Size(200, 300)); // Same area. + frame_sizes.push_back(gfx::Size(600, 400)); // Grow both dimensions. + frame_sizes.push_back(gfx::Size(638, 400)); // Shrink only one dimension. + frame_sizes.push_back(gfx::Size(638, 398)); // Shrink the other dimension. + frame_sizes.push_back(gfx::Size(320, 180)); // Shrink both dimensions again. + frame_sizes.push_back(gfx::Size(322, 180)); // Grow only one dimension. + frame_sizes.push_back(gfx::Size(322, 182)); // Grow the other dimension. + frame_sizes.push_back(gfx::Size(1920, 1080)); // Grow both dimensions again. + + uint32 frame_id = 0; + + // Encode one frame at each size. For the platform encoders, expect no frames + // to be delivered since each frame size change will sprun re-initialization + // of the underlying encoder. Otherwise, expect all key frames to come out. + for (const auto& frame_size : frame_sizes) { + EXPECT_EQ(!is_testing_platform_encoder(), + EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), + frame_id, + frame_id)); + RunTasksAndAdvanceClock(); + if (!is_testing_platform_encoder()) + ++frame_id; + } + + // Encode 10+ frames at each size. For the platform decoders, expect the + // first one or more frames are dropped while the encoder re-inits. Then, for + // all encoders, expect one key frame followed by all delta frames. + for (const auto& frame_size : frame_sizes) { + bool accepted_first_frame = false; + do { + accepted_first_frame = EncodeAndCheckDelivery( + CreateTestVideoFrame(frame_size), frame_id, frame_id); + if (!is_testing_platform_encoder()) + EXPECT_TRUE(accepted_first_frame); + RunTasksAndAdvanceClock(); + } while (!accepted_first_frame); + ++frame_id; + for (int i = 1; i < 10; ++i, ++frame_id) { + EXPECT_TRUE(EncodeAndCheckDelivery(CreateTestVideoFrame(frame_size), + frame_id, + frame_id - 1)); + RunTasksAndAdvanceClock(); + } + } + + WaitForAllFramesToBeDelivered(10 * frame_sizes.size()); + ExpectVEAResponsesForExternalVideoEncoder( + 2 * frame_sizes.size(), 6 * frame_sizes.size()); +} + +// Verify that everything goes well even if ExternalVideoEncoder is destroyed +// before it has a chance to receive the VEA creation callback. For all other +// encoders, this tests that the encoder can be safely destroyed before the task +// is run that delivers the first EncodedFrame. +TEST_P(VideoEncoderTest, CanBeDestroyedBeforeVEAIsCreated) { + CreateEncoder(false); + + // Send a frame to spawn creation of the ExternalVideoEncoder instance. + EncodeAndCheckDelivery(CreateTestVideoFrame(gfx::Size(1280, 720)), 0, 0); + + // Destroy the encoder, and confirm the VEA Factory did not respond yet. + DestroyEncoder(); + ExpectVEAResponsesForExternalVideoEncoder(0, 0); + + // Allow the VEA Factory to respond by running the creation callback. When + // the task runs, it will be a no-op since the weak pointers to the + // ExternalVideoEncoder were invalidated. + SetVEAFactoryAutoRespond(true); + RunTasksAndAdvanceClock(); + ExpectVEAResponsesForExternalVideoEncoder(1, 0); +} + +namespace { +std::vector<std::pair<Codec, bool>> DetermineEncodersToTest() { + std::vector<std::pair<Codec, bool>> values; + // Fake encoder. + values.push_back(std::make_pair(CODEC_VIDEO_FAKE, false)); + // Software VP8 encoder. + values.push_back(std::make_pair(CODEC_VIDEO_VP8, false)); + // Hardware-accelerated encoder (faked). + values.push_back(std::make_pair(CODEC_VIDEO_VP8, true)); +#if defined(OS_MACOSX) + // VideoToolbox encoder (when VideoToolbox is present). + VideoSenderConfig video_config = GetDefaultVideoSenderConfig(); + video_config.use_external_encoder = false; + video_config.codec = CODEC_VIDEO_H264; + if (H264VideoToolboxEncoder::IsSupported(video_config)) + values.push_back(std::make_pair(CODEC_VIDEO_H264, false)); +#endif + return values; +} +} // namespace + +INSTANTIATE_TEST_CASE_P( + , VideoEncoderTest, ::testing::ValuesIn(DetermineEncodersToTest())); + +} // namespace cast +} // namespace media diff --git a/media/cast/sender/video_frame_factory.h b/media/cast/sender/video_frame_factory.h index 3c23f88..d55deb7 100644 --- a/media/cast/sender/video_frame_factory.h +++ b/media/cast/sender/video_frame_factory.h @@ -8,6 +8,10 @@ #include "base/memory/scoped_ptr.h" #include "base/time/time.h" +namespace gfx { +class Size; +} + namespace media { class VideoFrame; @@ -33,7 +37,11 @@ class VideoFrameFactory { // Creates a |VideoFrame| suitable for input via |InsertRawVideoFrame|. Frames // obtained in this manner may provide benefits such memory reuse and affinity // with the encoder. The format is guaranteed to be I420 or NV12. - virtual scoped_refptr<VideoFrame> CreateFrame(base::TimeDelta timestamp) = 0; + // + // This can transiently return null if the encoder is not yet initialized or + // is re-initializing. + virtual scoped_refptr<VideoFrame> MaybeCreateFrame( + const gfx::Size& frame_size, base::TimeDelta timestamp) = 0; }; } // namespace cast diff --git a/media/cast/sender/video_sender.cc b/media/cast/sender/video_sender.cc index 54e474c..2389e9f 100644 --- a/media/cast/sender/video_sender.cc +++ b/media/cast/sender/video_sender.cc @@ -12,13 +12,7 @@ #include "base/trace_event/trace_event.h" #include "media/cast/cast_defines.h" #include "media/cast/net/cast_transport_config.h" -#include "media/cast/sender/external_video_encoder.h" -#include "media/cast/sender/video_encoder_impl.h" -#include "media/cast/sender/video_frame_factory.h" - -#if defined(OS_MACOSX) -#include "media/cast/sender/h264_vt_encoder.h" -#endif +#include "media/cast/sender/video_encoder.h" namespace media { namespace cast { @@ -71,34 +65,12 @@ VideoSender::VideoSender( last_bitrate_(0), playout_delay_change_cb_(playout_delay_change_cb), weak_factory_(this) { -#if defined(OS_MACOSX) - // On Apple platforms, use the hardware H.264 encoder if possible. It is the - // only reasonable option for iOS. - if (!video_config.use_external_encoder && - video_config.codec == CODEC_VIDEO_H264) { - video_encoder_.reset(new H264VideoToolboxEncoder( - cast_environment, - video_config, - gfx::Size(video_config.width, video_config.height), - status_change_cb)); - } -#endif // defined(OS_MACOSX) -#if !defined(OS_IOS) - if (video_config.use_external_encoder) { - video_encoder_.reset(new ExternalVideoEncoder( - cast_environment, - video_config, - gfx::Size(video_config.width, video_config.height), - status_change_cb, - create_vea_cb, - create_video_encode_mem_cb)); - } else if (!video_encoder_) { - // Software encoder is initialized immediately. - video_encoder_.reset(new VideoEncoderImpl( - cast_environment, video_config, status_change_cb)); - } -#endif // !defined(OS_IOS) - + video_encoder_ = VideoEncoder::Create( + cast_environment_, + video_config, + status_change_cb, + create_vea_cb, + create_video_encode_mem_cb); if (!video_encoder_) { cast_environment_->PostTask( CastEnvironment::MAIN, @@ -202,6 +174,11 @@ void VideoSender::InsertRawVideoFrame( last_bitrate_ = bitrate; } + if (video_frame->visible_rect().IsEmpty()) { + VLOG(1) << "Rejecting empty video frame."; + return; + } + if (video_encoder_->EncodeVideoFrame( video_frame, reference_time, diff --git a/media/cast/sender/video_sender_unittest.cc b/media/cast/sender/video_sender_unittest.cc index 6c0ba98..89a9c4f 100644 --- a/media/cast/sender/video_sender_unittest.cc +++ b/media/cast/sender/video_sender_unittest.cc @@ -15,6 +15,7 @@ #include "media/cast/net/cast_transport_config.h" #include "media/cast/net/cast_transport_sender_impl.h" #include "media/cast/net/pacing/paced_sender.h" +#include "media/cast/sender/fake_video_encode_accelerator_factory.h" #include "media/cast/sender/video_frame_factory.h" #include "media/cast/sender/video_sender.h" #include "media/cast/test/fake_single_thread_task_runner.h" @@ -35,22 +36,6 @@ static const int kHeight = 240; using testing::_; using testing::AtLeast; -void CreateVideoEncodeAccelerator( - const scoped_refptr<base::SingleThreadTaskRunner>& task_runner, - scoped_ptr<VideoEncodeAccelerator> fake_vea, - const ReceiveVideoEncodeAcceleratorCallback& callback) { - callback.Run(task_runner, fake_vea.Pass()); -} - -void CreateSharedMemory( - size_t size, const ReceiveVideoEncodeMemoryCallback& callback) { - scoped_ptr<base::SharedMemory> shm(new base::SharedMemory()); - if (!shm->CreateAndMapAnonymous(size)) { - NOTREACHED(); - return; - } - callback.Run(shm.Pass()); -} void SaveOperationalStatus(OperationalStatus* out_status, OperationalStatus in_status) { @@ -145,8 +130,9 @@ class VideoSenderTest : public ::testing::Test { task_runner_, task_runner_)), operational_status_(STATUS_UNINITIALIZED), - stored_bitrates_(NULL) { + vea_factory_(task_runner_) { testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks()); + vea_factory_.SetAutoRespond(true); last_pixel_value_ = kPixelValue; net::IPEndPoint dummy_endpoint; transport_sender_.reset(new CastTransportSenderImpl( @@ -182,8 +168,6 @@ class VideoSenderTest : public ::testing::Test { video_config.receiver_ssrc = 2; video_config.rtp_payload_type = 127; video_config.use_external_encoder = external; - video_config.width = kWidth; - video_config.height = kHeight; video_config.max_bitrate = 5000000; video_config.min_bitrate = 1000000; video_config.start_bitrate = 1000000; @@ -196,30 +180,25 @@ class VideoSenderTest : public ::testing::Test { ASSERT_EQ(operational_status_, STATUS_UNINITIALIZED); if (external) { - media::FakeVideoEncodeAccelerator* fake_vea = - new media::FakeVideoEncodeAccelerator(task_runner_); - stored_bitrates_ = &fake_vea->stored_bitrates(); - fake_vea->SetWillInitializationSucceed(expect_init_success); - scoped_ptr<VideoEncodeAccelerator> fake_vea_owner(fake_vea); - video_sender_.reset( - new PeerVideoSender(cast_environment_, - video_config, - base::Bind(&SaveOperationalStatus, - &operational_status_), - base::Bind(&CreateVideoEncodeAccelerator, - task_runner_, - base::Passed(&fake_vea_owner)), - base::Bind(&CreateSharedMemory), - transport_sender_.get())); + vea_factory_.SetInitializationWillSucceed(expect_init_success); + video_sender_.reset(new PeerVideoSender( + cast_environment_, + video_config, + base::Bind(&SaveOperationalStatus, &operational_status_), + base::Bind( + &FakeVideoEncodeAcceleratorFactory::CreateVideoEncodeAccelerator, + base::Unretained(&vea_factory_)), + base::Bind(&FakeVideoEncodeAcceleratorFactory::CreateSharedMemory, + base::Unretained(&vea_factory_)), + transport_sender_.get())); } else { - video_sender_.reset( - new PeerVideoSender(cast_environment_, - video_config, - base::Bind(&SaveOperationalStatus, - &operational_status_), - CreateDefaultVideoEncodeAcceleratorCallback(), - CreateDefaultVideoEncodeMemoryCallback(), - transport_sender_.get())); + video_sender_.reset(new PeerVideoSender( + cast_environment_, + video_config, + base::Bind(&SaveOperationalStatus, &operational_status_), + CreateDefaultVideoEncodeAcceleratorCallback(), + CreateDefaultVideoEncodeMemoryCallback(), + transport_sender_.get())); } task_runner_->RunTasks(); } @@ -256,10 +235,10 @@ class VideoSenderTest : public ::testing::Test { const scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_; const scoped_refptr<CastEnvironment> cast_environment_; OperationalStatus operational_status_; + FakeVideoEncodeAcceleratorFactory vea_factory_; TestPacketSender transport_; scoped_ptr<CastTransportSenderImpl> transport_sender_; scoped_ptr<PeerVideoSender> video_sender_; - const std::vector<uint32>* stored_bitrates_; // Owned by |video_sender_|. int last_pixel_value_; base::TimeTicks first_frame_timestamp_; @@ -284,25 +263,53 @@ TEST_F(VideoSenderTest, ExternalEncoder) { InitEncoder(true, true); ASSERT_EQ(STATUS_INITIALIZED, operational_status_); + // The SizeAdaptableExternalVideoEncoder initally reports STATUS_INITIALIZED + // so that frames will be sent to it. Therefore, no encoder activity should + // have occurred at this point. Send a frame to spurn creation of the + // underlying ExternalVideoEncoder instance. + if (vea_factory_.vea_response_count() == 0) { + video_sender_->InsertRawVideoFrame(GetNewVideoFrame(), + testing_clock_->NowTicks()); + task_runner_->RunTasks(); + } + ASSERT_EQ(STATUS_INITIALIZED, operational_status_); + RunTasks(33); + + // VideoSender created an encoder for 1280x720 frames, in order to provide the + // INITIALIZED status. + EXPECT_EQ(1, vea_factory_.vea_response_count()); + EXPECT_EQ(3, vea_factory_.shm_response_count()); + scoped_refptr<media::VideoFrame> video_frame = GetNewVideoFrame(); - const base::TimeTicks reference_time = testing_clock_->NowTicks(); - video_sender_->InsertRawVideoFrame(video_frame, reference_time); - task_runner_->RunTasks(); - video_sender_->InsertRawVideoFrame(video_frame, reference_time); - task_runner_->RunTasks(); - video_sender_->InsertRawVideoFrame(video_frame, reference_time); - task_runner_->RunTasks(); + for (int i = 0; i < 3; ++i) { + const base::TimeTicks reference_time = testing_clock_->NowTicks(); + video_sender_->InsertRawVideoFrame(video_frame, reference_time); + RunTasks(33); + // VideoSender re-created the encoder for the 320x240 frames we're + // providing. + EXPECT_EQ(1, vea_factory_.vea_response_count()); + EXPECT_EQ(3, vea_factory_.shm_response_count()); + } - // Fixed bitrate is used for external encoder. Bitrate is only once - // to the encoder. - EXPECT_EQ(1u, stored_bitrates_->size()); video_sender_.reset(NULL); task_runner_->RunTasks(); + EXPECT_EQ(1, vea_factory_.vea_response_count()); + EXPECT_EQ(3, vea_factory_.shm_response_count()); } TEST_F(VideoSenderTest, ExternalEncoderInitFails) { InitEncoder(true, false); + + // The SizeAdaptableExternalVideoEncoder initally reports STATUS_INITIALIZED + // so that frames will be sent to it. Send a frame to spurn creation of the + // underlying ExternalVideoEncoder instance, which should result in failure. + if (operational_status_ == STATUS_INITIALIZED || + operational_status_ == STATUS_CODEC_REINIT_PENDING) { + video_sender_->InsertRawVideoFrame(GetNewVideoFrame(), + testing_clock_->NowTicks()); + task_runner_->RunTasks(); + } EXPECT_EQ(STATUS_CODEC_INIT_FAILED, operational_status_); video_sender_.reset(NULL); diff --git a/media/cast/test/cast_benchmarks.cc b/media/cast/test/cast_benchmarks.cc index aa89e94..064c5db 100644 --- a/media/cast/test/cast_benchmarks.cc +++ b/media/cast/test/cast_benchmarks.cc @@ -69,8 +69,6 @@ namespace { static const int64 kStartMillisecond = INT64_C(1245); static const int kAudioChannels = 2; -static const int kVideoHdWidth = 1280; -static const int kVideoHdHeight = 720; static const int kTargetPlayoutDelayMs = 300; // The tests are commonly implemented with |kFrameTimerMs| RunTask function; @@ -266,8 +264,6 @@ class RunOneBenchmark { base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs); video_sender_config_.rtp_payload_type = 97; video_sender_config_.use_external_encoder = false; - video_sender_config_.width = kVideoHdWidth; - video_sender_config_.height = kVideoHdHeight; #if 0 video_sender_config_.max_bitrate = 10000000; // 10Mbit max video_sender_config_.min_bitrate = 1000000; // 1Mbit min diff --git a/media/cast/test/end2end_unittest.cc b/media/cast/test/end2end_unittest.cc index e7a0c14..9b36b59 100644 --- a/media/cast/test/end2end_unittest.cc +++ b/media/cast/test/end2end_unittest.cc @@ -53,8 +53,6 @@ static const double kSoundFrequency = 314.15926535897; // Freq of sine wave. static const float kSoundVolume = 0.5f; static const int kVideoHdWidth = 1280; static const int kVideoHdHeight = 720; -static const int kVideoQcifWidth = 176; -static const int kVideoQcifHeight = 144; // Since the video encoded and decoded an error will be introduced; when // comparing individual pixels the error can be quite large; we allow a PSNR of @@ -356,8 +354,6 @@ class TestReceiverVideoCallback public: struct ExpectedVideoFrame { int start_value; - int width; - int height; base::TimeTicks playout_time; bool should_be_continuous; }; @@ -365,14 +361,10 @@ class TestReceiverVideoCallback TestReceiverVideoCallback() : num_called_(0) {} void AddExpectedResult(int start_value, - int width, - int height, const base::TimeTicks& playout_time, bool should_be_continuous) { ExpectedVideoFrame expected_video_frame; expected_video_frame.start_value = start_value; - expected_video_frame.width = width; - expected_video_frame.height = height; expected_video_frame.playout_time = playout_time; expected_video_frame.should_be_continuous = should_be_continuous; expected_frame_.push_back(expected_video_frame); @@ -388,11 +380,10 @@ class TestReceiverVideoCallback ExpectedVideoFrame expected_video_frame = expected_frame_.front(); expected_frame_.pop_front(); - EXPECT_EQ(expected_video_frame.width, video_frame->visible_rect().width()); - EXPECT_EQ(expected_video_frame.height, - video_frame->visible_rect().height()); + EXPECT_EQ(kVideoHdWidth, video_frame->visible_rect().width()); + EXPECT_EQ(kVideoHdHeight, video_frame->visible_rect().height()); - gfx::Size size(expected_video_frame.width, expected_video_frame.height); + const gfx::Size size(kVideoHdWidth, kVideoHdHeight); scoped_refptr<media::VideoFrame> expected_I420_frame = media::VideoFrame::CreateFrame( VideoFrame::I420, size, gfx::Rect(size), size, base::TimeDelta()); @@ -496,8 +487,6 @@ class End2EndTest : public ::testing::Test { base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs); video_sender_config_.rtp_payload_type = 97; video_sender_config_.use_external_encoder = false; - video_sender_config_.width = kVideoHdWidth; - video_sender_config_.height = kVideoHdHeight; video_sender_config_.max_bitrate = 50000; video_sender_config_.min_bitrate = 10000; video_sender_config_.start_bitrate = 10000; @@ -666,7 +655,7 @@ class End2EndTest : public ::testing::Test { // TODO(miu): Consider using a slightly skewed clock for the media timestamp // since the video clock may not be the same as the reference clock. const base::TimeDelta time_diff = reference_time - start_time_; - gfx::Size size(video_sender_config_.width, video_sender_config_.height); + const gfx::Size size(kVideoHdWidth, kVideoHdHeight); EXPECT_TRUE(VideoFrame::IsValidConfig( VideoFrame::I420, size, gfx::Rect(size), size)); scoped_refptr<media::VideoFrame> video_frame = @@ -834,10 +823,6 @@ class End2EndTest : public ::testing::Test { TEST_F(End2EndTest, LoopNoLossPcm16) { Configure(CODEC_VIDEO_VP8, CODEC_AUDIO_PCM16, 32000, 1); - // Reduce video resolution to allow processing multiple frames within a - // reasonable time frame. - video_sender_config_.width = kVideoQcifWidth; - video_sender_config_.height = kVideoQcifHeight; Create(); const int kNumIterations = 50; @@ -853,8 +838,6 @@ TEST_F(End2EndTest, LoopNoLossPcm16) { test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, testing_clock_sender_->NowTicks() + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); @@ -959,8 +942,6 @@ TEST_F(End2EndTest, DISABLED_StartSenderBeforeReceiver) { // packets, and specifically no RTCP packets were sent. test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, initial_send_time + expected_delay + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); @@ -989,8 +970,6 @@ TEST_F(End2EndTest, DISABLED_StartSenderBeforeReceiver) { test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, testing_clock_sender_->NowTicks() + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); @@ -1040,8 +1019,6 @@ TEST_F(End2EndTest, DropEveryOtherFrame3Buffers) { if (i % 2 == 0) { test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, reference_time + base::TimeDelta::FromMilliseconds(target_delay), i == 0); @@ -1081,8 +1058,6 @@ TEST_F(End2EndTest, CryptoVideo) { test_receiver_video_callback_->AddExpectedResult( frames_counter, - video_sender_config_.width, - video_sender_config_.height, reference_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); @@ -1137,8 +1112,6 @@ TEST_F(End2EndTest, VideoLogging) { base::TimeTicks reference_time = testing_clock_sender_->NowTicks(); test_receiver_video_callback_->AddExpectedResult( video_start, - video_sender_config_.width, - video_sender_config_.height, reference_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs), true); diff --git a/media/cast/test/utility/default_config.cc b/media/cast/test/utility/default_config.cc index 39b9ae6..3d4a471 100644 --- a/media/cast/test/utility/default_config.cc +++ b/media/cast/test/utility/default_config.cc @@ -74,8 +74,6 @@ VideoSenderConfig GetDefaultVideoSenderConfig() { config.receiver_ssrc = recv_config.feedback_ssrc; config.rtp_payload_type = recv_config.rtp_payload_type; config.use_external_encoder = false; - config.width = 1280; - config.height = 720; config.max_bitrate = 4000000; config.min_bitrate = 2000000; config.start_bitrate = 4000000; diff --git a/media/cast/test/utility/video_utility.cc b/media/cast/test/utility/video_utility.cc index ff050da..46469563 100644 --- a/media/cast/test/utility/video_utility.cc +++ b/media/cast/test/utility/video_utility.cc @@ -63,16 +63,10 @@ void PopulateVideoFrame(VideoFrame* frame, int start_value) { const int stripe_size = std::max(32, std::min(frame_size.width(), frame_size.height()) / 8) & -2; - int height = frame_size.height(); - int stride_y = frame->stride(VideoFrame::kYPlane); - int stride_u = frame->stride(VideoFrame::kUPlane); - int stride_v = frame->stride(VideoFrame::kVPlane); - int half_height = (height + 1) / 2; - uint8* y_plane = frame->data(VideoFrame::kYPlane); - uint8* u_plane = frame->data(VideoFrame::kUPlane); - uint8* v_plane = frame->data(VideoFrame::kVPlane); - // Set Y. + const int height = frame_size.height(); + const int stride_y = frame->stride(VideoFrame::kYPlane); + uint8* y_plane = frame->data(VideoFrame::kYPlane); for (int j = 0; j < height; ++j) { const int stripe_j = (j / stripe_size) * stripe_size; for (int i = 0; i < stride_y; ++i) { @@ -82,23 +76,45 @@ void PopulateVideoFrame(VideoFrame* frame, int start_value) { } } - // Set U. - for (int j = 0; j < half_height; ++j) { - const int stripe_j = (j / stripe_size) * stripe_size; - for (int i = 0; i < stride_u; ++i) { - const int stripe_i = (i / stripe_size) * stripe_size; - *u_plane = static_cast<uint8>(start_value + stripe_i + stripe_j); - ++u_plane; + const int half_height = (height + 1) / 2; + if (frame->format() == VideoFrame::NV12) { + const int stride_uv = frame->stride(VideoFrame::kUVPlane); + uint8* uv_plane = frame->data(VideoFrame::kUVPlane); + + // Set U and V. + for (int j = 0; j < half_height; ++j) { + const int stripe_j = (j / stripe_size) * stripe_size; + for (int i = 0; i < stride_uv; i += 2) { + const int stripe_i = (i / stripe_size) * stripe_size; + *uv_plane = *(uv_plane + 1) = + static_cast<uint8>(start_value + stripe_i + stripe_j); + uv_plane += 2; + } + } + } else { // I420, YV12, etc. + const int stride_u = frame->stride(VideoFrame::kUPlane); + const int stride_v = frame->stride(VideoFrame::kVPlane); + uint8* u_plane = frame->data(VideoFrame::kUPlane); + uint8* v_plane = frame->data(VideoFrame::kVPlane); + + // Set U. + for (int j = 0; j < half_height; ++j) { + const int stripe_j = (j / stripe_size) * stripe_size; + for (int i = 0; i < stride_u; ++i) { + const int stripe_i = (i / stripe_size) * stripe_size; + *u_plane = static_cast<uint8>(start_value + stripe_i + stripe_j); + ++u_plane; + } } - } - // Set V. - for (int j = 0; j < half_height; ++j) { - const int stripe_j = (j / stripe_size) * stripe_size; - for (int i = 0; i < stride_v; ++i) { - const int stripe_i = (i / stripe_size) * stripe_size; - *u_plane = static_cast<uint8>(start_value + stripe_i + stripe_j); - ++v_plane; + // Set V. + for (int j = 0; j < half_height; ++j) { + const int stripe_j = (j / stripe_size) * stripe_size; + for (int i = 0; i < stride_v; ++i) { + const int stripe_i = (i / stripe_size) * stripe_size; + *u_plane = static_cast<uint8>(start_value + stripe_i + stripe_j); + ++v_plane; + } } } } |