diff options
author | mcasas <mcasas@chromium.org> | 2015-09-09 16:32:06 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-09-09 23:32:47 +0000 |
commit | 1567842287b4097350d33c14e5f795559fa008b5 (patch) | |
tree | f50a9f48f649abf87a84590681bf9516026ee812 | |
parent | 7606f64ba73cbc175f2dbffe2d20714bab299de2 (diff) | |
download | chromium_src-1567842287b4097350d33c14e5f795559fa008b5.zip chromium_src-1567842287b4097350d33c14e5f795559fa008b5.tar.gz chromium_src-1567842287b4097350d33c14e5f795559fa008b5.tar.bz2 |
MediaRecorderHandler (video part) and unittests
MediaRecorderHandler is Blink-orchestrated class implementing
MediaRecorder API (see below). It plugs together an existing
MediaStreamVideoTrack to a new VideoTrackRecorder-WebmMuxer
pair. When MSVTrack passes frames, these get encoded, muxed,
and the result is sent to Blink.
A *note on threading*: As is customary in MediaStream* and
derived classes, all configuration happens on Main Render thread
while frame manipulation and forwarding happens on Render IO
thread [1]. Moreover, all objects can be, and often are, destroyed in
asynchronous and unexpected ways from Blink. This forces
ref-counting for VideoTrackRecorder::VpxEncoder. This is the
also the reason behind the change in WebmMuxer to 2-threaded.
See DD @ https://goo.gl/vSjzC5 (*) for the plan.
(*) Used to be https://goo.gl/kreaQj
[1] https://code.google.com/p/chromium/codesearch#chromium/src/content/renderer/media/media_stream_video_track.cc&sq=package:chromium&type=cs&l=155&rcl=1440530828
BUG=262211
Review URL: https://codereview.chromium.org/1313603004
Cr-Commit-Position: refs/heads/master@{#348037}
-rw-r--r-- | content/content_renderer.gypi | 6 | ||||
-rw-r--r-- | content/content_tests.gypi | 3 | ||||
-rw-r--r-- | content/renderer/media/media_recorder_handler.cc | 131 | ||||
-rw-r--r-- | content/renderer/media/media_recorder_handler.h | 82 | ||||
-rw-r--r-- | content/renderer/media/media_recorder_handler_unittest.cc | 155 | ||||
-rw-r--r-- | content/renderer/media/video_track_recorder.cc | 131 | ||||
-rw-r--r-- | content/renderer/media/video_track_recorder.h | 13 | ||||
-rw-r--r-- | content/renderer/media/video_track_recorder_unittest.cc | 8 | ||||
-rw-r--r-- | media/capture/webm_muxer.cc | 25 | ||||
-rw-r--r-- | media/capture/webm_muxer.h | 10 | ||||
-rw-r--r-- | media/capture/webm_muxer_unittest.cc | 8 | ||||
-rw-r--r-- | media/media.gyp | 11 |
12 files changed, 501 insertions, 82 deletions
diff --git a/content/content_renderer.gypi b/content/content_renderer.gypi index 3f31165..cee3e1e 100644 --- a/content/content_renderer.gypi +++ b/content/content_renderer.gypi @@ -596,6 +596,8 @@ # WebRTC-specific sources. Put WebRTC plugin-related stuff in the # Plugin+WebRTC section below. 'private_renderer_webrtc_sources': [ + 'renderer/media/media_recorder_handler.cc', + 'renderer/media/media_recorder_handler.h', 'renderer/media/media_stream.cc', 'renderer/media/media_stream.h', 'renderer/media/media_stream_audio_processor.cc', @@ -756,6 +758,10 @@ ['OS=="android"', { 'sources!': [ 'renderer/media/audio_decoder.cc', + 'renderer/media/video_track_recorder.cc', + 'renderer/media/video_track_recorder.h', + 'renderer/media/media_recorder_handler.cc', + 'renderer/media/media_recorder_handler.h', 'renderer/usb/type_converters.cc', 'renderer/usb/type_converters.h', 'renderer/usb/web_usb_client_impl.cc', diff --git a/content/content_tests.gypi b/content/content_tests.gypi index 299fbc5..6677a79 100644 --- a/content/content_tests.gypi +++ b/content/content_tests.gypi @@ -728,6 +728,7 @@ 'browser/renderer_host/p2p/socket_host_test_utils.h', 'browser/renderer_host/p2p/socket_host_udp_unittest.cc', 'browser/renderer_host/p2p/socket_host_unittest.cc', + 'renderer/media/media_recorder_handler_unittest.cc', 'renderer/media/media_stream_audio_processor_unittest.cc', 'renderer/media/media_stream_constraints_util_unittest.cc', 'renderer/media/media_stream_dispatcher_unittest.cc', @@ -1170,6 +1171,8 @@ 'browser/power_usage_monitor_impl_unittest.cc', 'browser/renderer_host/begin_frame_observer_proxy_unittest.cc', 'browser/webui/url_data_manager_backend_unittest.cc', + 'renderer/media/media_recorder_handler_unittest.cc', + 'renderer/media/video_track_recorder_unittest.cc', ], 'dependencies': [ '../testing/android/native_test.gyp:native_test_native_code', diff --git a/content/renderer/media/media_recorder_handler.cc b/content/renderer/media/media_recorder_handler.cc new file mode 100644 index 0000000..c15a262 --- /dev/null +++ b/content/renderer/media/media_recorder_handler.cc @@ -0,0 +1,131 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/renderer/media/media_recorder_handler.h" + +#include "base/bind.h" +#include "base/location.h" +#include "base/logging.h" +#include "content/renderer/media/video_track_recorder.h" +#include "media/base/bind_to_current_loop.h" +#include "media/capture/webm_muxer.h" +#include "third_party/WebKit/public/platform/WebString.h" + +namespace content { + +MediaRecorderHandler::MediaRecorderHandler() + : recording_(false), client_(nullptr), weak_factory_(this) { + DVLOG(3) << __FUNCTION__; +} + +MediaRecorderHandler::~MediaRecorderHandler() { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + // Send a |last_in_slice| to our |client_|. + if (client_) + client_->writeData(nullptr, 0u, true); +} + +bool MediaRecorderHandler::canSupportMimeType( + const blink::WebString& mimeType) { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + // TODO(mcasas): For the time being only "video/vp8" is supported. + return mimeType.utf8().compare("video/vp8") == 0; +} + +bool MediaRecorderHandler::initialize( + blink::WebMediaRecorderHandlerClient* client, + const blink::WebMediaStream& media_stream, + const blink::WebString& mimeType) { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + if (!canSupportMimeType(mimeType)) { + DLOG(ERROR) << "Can't support type " << mimeType.utf8(); + return false; + } + media_stream_ = media_stream; + DCHECK(client); + client_ = client; + + return true; +} + +bool MediaRecorderHandler::start() { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + DCHECK(!recording_); + return start(0); +} + +bool MediaRecorderHandler::start(int timeslice) { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + DCHECK(!recording_); + DCHECK(!media_stream_.isNull()); + + webm_muxer_.reset(new media::WebmMuxer(media::BindToCurrentLoop(base::Bind( + &MediaRecorderHandler::WriteData, weak_factory_.GetWeakPtr())))); + DCHECK(webm_muxer_); + + blink::WebVector<blink::WebMediaStreamTrack> video_tracks; + media_stream_.videoTracks(video_tracks); + + if (video_tracks.isEmpty()) { + // TODO(mcasas): Add audio_tracks and update the code in this function + // correspondingly, see http://crbug.com/528519. As of now, only video + // tracks are supported. + LOG(WARNING) << "Recording no video tracks is not implemented"; + return false; + } + // TODO(mcasas): The muxer API supports only one video track. Extend it to + // several video tracks, see http://crbug.com/528523. + LOG_IF(WARNING, video_tracks.size() > 1u) << "Recording multiple video" + << " tracks is not implemented. Only recording first video track."; + const blink::WebMediaStreamTrack& video_track = video_tracks[0]; + if (video_track.isNull()) + return false; + + const VideoTrackRecorder::OnEncodedVideoCB on_encoded_video_cb = + base::Bind(&media::WebmMuxer::OnEncodedVideo, + base::Unretained(webm_muxer_.get())); + + video_recorders_.push_back(new VideoTrackRecorder(video_track, + on_encoded_video_cb)); + + recording_ = true; + return true; +} + +void MediaRecorderHandler::stop() { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + DCHECK(recording_); + + recording_ = false; + video_recorders_.clear(); + webm_muxer_.reset(NULL); +} + +void MediaRecorderHandler::pause() { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + DCHECK(recording_); + recording_ = false; + NOTIMPLEMENTED(); +} + +void MediaRecorderHandler::resume() { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + DCHECK(!recording_); + recording_ = true; + NOTIMPLEMENTED(); +} + +void MediaRecorderHandler::WriteData(const base::StringPiece& data) { + DCHECK(main_render_thread_checker_.CalledOnValidThread()); + client_->writeData(data.data(), data.size(), false /* lastInSlice */); +} + +void MediaRecorderHandler::OnVideoFrameForTesting( + const scoped_refptr<media::VideoFrame>& frame, + const base::TimeTicks& timestamp) { + for (auto* recorder : video_recorders_) + recorder->OnVideoFrameForTesting(frame, timestamp); +} + +} // namespace content diff --git a/content/renderer/media/media_recorder_handler.h b/content/renderer/media/media_recorder_handler.h new file mode 100644 index 0000000..e4f0597 --- /dev/null +++ b/content/renderer/media/media_recorder_handler.h @@ -0,0 +1,82 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_RENDERER_MEDIA_MEDIA_RECORDER_HANDLER_H_ +#define CONTENT_RENDERER_MEDIA_MEDIA_RECORDER_HANDLER_H_ + +#include "base/memory/scoped_vector.h" +#include "base/memory/weak_ptr.h" +#include "base/strings/string_piece.h" +#include "base/threading/thread_checker.h" +#include "content/common/content_export.h" +#include "third_party/WebKit/public/platform/WebMediaRecorderHandlerClient.h" +#include "third_party/WebKit/public/platform/WebMediaStream.h" + +namespace blink { +class WebString; +} // namespace blink + +namespace media { +class VideoFrame; +class WebmMuxer; +} // namespace media + +namespace content { + +class VideoTrackRecorder; + +// MediaRecorderHandler orchestrates the creation, lifetime mgmt and mapping +// between: +// - MediaStreamTrack(s) providing data, +// - {Audio,Video}TrackRecorders encoding that data, +// - a WebmMuxer class multiplexing encoded data into a WebM container, and +// - a single recorder client receiving this contained data. +// All methods are called on the same thread as construction and destruction, +// i.e. the Main Render thread. +// TODO(mcasas): Implement audio recording. +class CONTENT_EXPORT MediaRecorderHandler final { + public: + MediaRecorderHandler(); + ~MediaRecorderHandler(); + + // See above, these methods should override blink::WebMediaRecorderHandler. + bool canSupportMimeType(const blink::WebString& mimeType); + bool initialize(blink::WebMediaRecorderHandlerClient* client, + const blink::WebMediaStream& media_stream, + const blink::WebString& mimeType); + bool start(); + bool start(int timeslice); + void stop(); + void pause(); + void resume(); + + private: + friend class MediaRecorderHandlerTest; + + void WriteData(const base::StringPiece& data); + + void OnVideoFrameForTesting(const scoped_refptr<media::VideoFrame>& frame, + const base::TimeTicks& timestamp); + + // Bound to the main render thread. + base::ThreadChecker main_render_thread_checker_; + + bool recording_; + blink::WebMediaStream media_stream_; // The MediaStream being recorded. + + // |client_| is a weak pointer, and is valid for the lifetime of this object. + blink::WebMediaRecorderHandlerClient* client_; + + ScopedVector<VideoTrackRecorder> video_recorders_; + + // Worker class doing the actual Webm Muxing work. + scoped_ptr<media::WebmMuxer> webm_muxer_; + + base::WeakPtrFactory<MediaRecorderHandler> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(MediaRecorderHandler); +}; + +} // namespace content +#endif // CONTENT_RENDERER_MEDIA_MEDIA_RECORDER_HANDLER_H_ diff --git a/content/renderer/media/media_recorder_handler_unittest.cc b/content/renderer/media/media_recorder_handler_unittest.cc new file mode 100644 index 0000000..f435855 --- /dev/null +++ b/content/renderer/media/media_recorder_handler_unittest.cc @@ -0,0 +1,155 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/run_loop.h" +#include "base/strings/utf_string_conversions.h" +#include "content/child/child_process.h" +#include "content/renderer/media/media_recorder_handler.h" +#include "content/renderer/media/mock_media_stream_registry.h" +#include "media/base/video_frame.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/WebKit/public/platform/WebMediaRecorderHandlerClient.h" +#include "third_party/WebKit/public/platform/WebString.h" + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::InSequence; +using ::testing::Lt; +using ::testing::Mock; + +using blink::WebString; + +namespace content { + +ACTION_P(RunClosure, closure) { + closure.Run(); +} + +static const std::string kTestStreamUrl = "stream_url"; +static const std::string kTestVideoTrackId = "video_track_id"; + +class MediaRecorderHandlerTest + : public testing::Test + , public blink::WebMediaRecorderHandlerClient { + public: + MediaRecorderHandlerTest() + : media_recorder_handler_(new MediaRecorderHandler()) { + EXPECT_FALSE(media_recorder_handler_->recording_); + + registry_.Init(kTestStreamUrl); + registry_.AddVideoTrack(kTestVideoTrackId); + } + + MOCK_METHOD3(writeData, void(const char*, size_t, bool)); + MOCK_METHOD1(failOutOfMemory, void(const WebString& message)); + MOCK_METHOD1(failIllegalStreamModification, void(const WebString& message)); + MOCK_METHOD1(failOtherRecordingError, void(const WebString& message)); + + bool recording() const { return media_recorder_handler_->recording_; } + bool hasVideoRecorders() const { + return !media_recorder_handler_->video_recorders_.empty(); + } + + void OnVideoFrameForTesting(const scoped_refptr<media::VideoFrame>& frame) { + media_recorder_handler_->OnVideoFrameForTesting(frame, + base::TimeTicks::Now()); + } + + // The Class under test. Needs to be scoped_ptr to force its destruction. + scoped_ptr<MediaRecorderHandler> media_recorder_handler_; + + // A ChildProcess and a MessageLoopForUI are both needed to fool the Tracks + // and Sources in |registry_| into believing they are on the right threads. + const base::MessageLoopForUI message_loop_; + const ChildProcess child_process_; + MockMediaStreamRegistry registry_; + + private: + DISALLOW_COPY_AND_ASSIGN(MediaRecorderHandlerTest); +}; + +// Checks that canSupportMimeType() works as expected. +// TODO(mcasas): revisit this when canSupportMimeType() is fully implemented. +TEST_F(MediaRecorderHandlerTest, CanSupportMimeType) { + const WebString good_mime_type(base::UTF8ToUTF16("video/vp8")); + EXPECT_TRUE(media_recorder_handler_->canSupportMimeType(good_mime_type)); + + const WebString bad_mime_type(base::UTF8ToUTF16("video/unsupportedcodec")); + EXPECT_FALSE(media_recorder_handler_->canSupportMimeType(bad_mime_type)); +} + +// Checks that the initialization-destruction sequence works fine. +TEST_F(MediaRecorderHandlerTest, InitializeStartStop) { + const WebString mime_type(base::UTF8ToUTF16("video/vp8")); + EXPECT_TRUE(media_recorder_handler_->initialize(this, + registry_.test_stream(), + mime_type)); + EXPECT_FALSE(recording()); + EXPECT_FALSE(hasVideoRecorders()); + + EXPECT_TRUE(media_recorder_handler_->start()); + EXPECT_TRUE(recording()); + EXPECT_TRUE(hasVideoRecorders()); + + media_recorder_handler_->stop(); + EXPECT_FALSE(recording()); + EXPECT_FALSE(hasVideoRecorders()); + + // Expect a last call on destruction. + EXPECT_CALL(*this, writeData(_, _, true)).Times(1); + media_recorder_handler_.reset(); +} + +// Sends 2 frames and expect them as WebM contained encoded data in writeData(). +TEST_F(MediaRecorderHandlerTest, EncodeVideoFrames) { + const WebString mime_type(base::UTF8ToUTF16("video/vp8")); + EXPECT_TRUE(media_recorder_handler_->initialize(this, registry_.test_stream(), + mime_type)); + EXPECT_TRUE(media_recorder_handler_->start()); + + InSequence s; + const scoped_refptr<media::VideoFrame> video_frame = + media::VideoFrame::CreateBlackFrame(gfx::Size(160, 80)); + + { + base::RunLoop run_loop; + base::Closure quit_closure = run_loop.QuitClosure(); + // writeData() is pinged a number of times as the WebM header is written; + // the last time it is called it has the encoded data. + const size_t kEncodedDataSize = 52; + EXPECT_CALL(*this, writeData(_, Lt(kEncodedDataSize), false)) + .Times(AtLeast(1)); + EXPECT_CALL(*this, writeData(_, kEncodedDataSize, false)) + .Times(1) + .WillOnce(RunClosure(quit_closure)); + + OnVideoFrameForTesting(video_frame); + run_loop.Run(); + } + + { + base::RunLoop run_loop; + base::Closure quit_closure = run_loop.QuitClosure(); + // The second time around writeData() is called a number of times to write + // the WebM frame header, and then is pinged with the encoded data. + const size_t kSecondEncodedDataSize = 32; + EXPECT_CALL(*this, writeData(_, Lt(kSecondEncodedDataSize), false)) + .Times(AtLeast(1)); + EXPECT_CALL(*this, writeData(_, kSecondEncodedDataSize, false)) + .Times(1) + .WillOnce(RunClosure(quit_closure)); + + OnVideoFrameForTesting(video_frame); + run_loop.Run(); + } + + media_recorder_handler_->stop(); + + // Expect a last call on destruction, with size 0 and |lastInSlice| true. + EXPECT_CALL(*this, writeData(nullptr, 0, true)).Times(1); + media_recorder_handler_.reset(); +} + +} // namespace content diff --git a/content/renderer/media/video_track_recorder.cc b/content/renderer/media/video_track_recorder.cc index b5aaa72..84ce844 100644 --- a/content/renderer/media/video_track_recorder.cc +++ b/content/renderer/media/video_track_recorder.cc @@ -10,8 +10,6 @@ #include "base/threading/thread.h" #include "base/time/time.h" #include "base/trace_event/trace_event.h" -#include "content/child/child_process.h" -#include "media/base/bind_to_current_loop.h" #include "media/base/video_frame.h" extern "C" { @@ -28,16 +26,31 @@ using media::VideoFrameMetadata; namespace content { namespace { + const vpx_codec_flags_t kNoFlags = 0; +// Originally from remoting/codec/scoped_vpx_codec.h. +// TODO(mcasas): Refactor into a common location. +struct VpxCodecDeleter { + void operator()(vpx_codec_ctx_t* codec) { + if (!codec) + return; + vpx_codec_err_t ret = vpx_codec_destroy(codec); + CHECK_EQ(ret, VPX_CODEC_OK); + delete codec; + } +}; + +typedef scoped_ptr<vpx_codec_ctx_t, VpxCodecDeleter> ScopedVpxCodecCtxPtr; + void OnFrameEncodeCompleted( - const content::VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb, + const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb, const scoped_refptr<VideoFrame>& frame, scoped_ptr<std::string> data, base::TimeTicks capture_timestamp, bool keyframe) { - DVLOG(1) << (keyframe ? "" : "non ") << "keyframe " - << capture_timestamp << " ms - " << data->length() << "B "; + DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "<< data->length() << "B, " + << capture_timestamp << " ms"; on_encoded_video_cb.Run(frame, base::StringPiece(*data), capture_timestamp, keyframe); } @@ -45,23 +58,32 @@ void OnFrameEncodeCompleted( } // anonymous namespace // Inner class encapsulating all libvpx interactions and the encoding+delivery -// of received frames. This class is: -// - created and destroyed on its parent's thread (usually the main render -// thread), -// - receives VideoFrames and Run()s the callbacks on another thread (supposedly -// the render IO thread), which is cached on first frame arrival, +// of received frames. Limitation: Only VP8 is supported for the time being. +// This class must be ref-counted because the MediaStreamVideoTrack will hold a +// reference to it, via the callback MediaStreamVideoSink passes along, and it's +// unknown when exactly it will release that reference. This class: +// - is created and destroyed on its parent's thread (usually the main Render +// thread); +// - receives VideoFrames and Run()s the callbacks on |origin_task_runner_|, +// which is cached on first frame arrival, and is supposed to be the render IO +// thread, but this is not enforced; // - uses an internal |encoding_thread_| for libvpx interactions, notably for // encoding (which might take some time). -// Only VP8 is supported for the time being. -class VideoTrackRecorder::VpxEncoder final { +class VideoTrackRecorder::VpxEncoder final + : public base::RefCountedThreadSafe<VpxEncoder> { public: + static void ShutdownEncoder(scoped_ptr<base::Thread> encoding_thread, + ScopedVpxCodecCtxPtr encoder); + explicit VpxEncoder(const OnEncodedVideoCB& on_encoded_video_callback); - ~VpxEncoder(); void StartFrameEncode(const scoped_refptr<VideoFrame>& frame, base::TimeTicks capture_timestamp); private: + friend class base::RefCountedThreadSafe<VpxEncoder>; + ~VpxEncoder(); + void EncodeOnEncodingThread(const scoped_refptr<VideoFrame>& frame, base::TimeTicks capture_timestamp); @@ -74,8 +96,8 @@ class VideoTrackRecorder::VpxEncoder final { base::TimeDelta CalculateFrameDuration( const scoped_refptr<VideoFrame>& frame); - // Used to check that we are destroyed on the same thread we were created. - base::ThreadChecker main_render_thread_checker_; + // Used to shutdown properly on the same thread we were created. + const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; // Task runner where frames to encode and reply callbacks must happen. scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_; @@ -83,12 +105,14 @@ class VideoTrackRecorder::VpxEncoder final { // This callback should be exercised on IO thread. const OnEncodedVideoCB on_encoded_video_callback_; - // Thread for encoding. Active as long as VpxEncoder exists. All variables + // Thread for encoding. Active for the lifetime of VpxEncoder. All variables // below this are used in this thread. - base::Thread encoding_thread_; - // VP8 internal objects: configuration, encoder and Vpx Image wrapper. + scoped_ptr<base::Thread> encoding_thread_; + // VP8 internal objects: configuration and encoder. vpx_codec_enc_cfg_t codec_config_; - vpx_codec_ctx_t encoder_; + // |encoder_| is a special scoped pointer to guarantee proper destruction. + // Again, it should only be accessed on |encoding_thread_|. + ScopedVpxCodecCtxPtr encoder_; // The |VideoFrame::timestamp()| of the last encoded frame. This is used to // predict the duration of the next frame. @@ -97,23 +121,33 @@ class VideoTrackRecorder::VpxEncoder final { DISALLOW_COPY_AND_ASSIGN(VpxEncoder); }; +// static +void VideoTrackRecorder::VpxEncoder::ShutdownEncoder( + scoped_ptr<base::Thread> encoding_thread, + ScopedVpxCodecCtxPtr encoder) { + DCHECK(encoding_thread->IsRunning()); + encoding_thread->Stop(); + // Both |encoding_thread| and |encoder| will be destroyed at end-of-scope. +} + VideoTrackRecorder::VpxEncoder::VpxEncoder( const OnEncodedVideoCB& on_encoded_video_callback) - : on_encoded_video_callback_(on_encoded_video_callback), - encoding_thread_("EncodingThread") { + : main_task_runner_(base::MessageLoop::current()->task_runner()), + on_encoded_video_callback_(on_encoded_video_callback), + encoding_thread_(new base::Thread("EncodingThread")) { DCHECK(!on_encoded_video_callback_.is_null()); codec_config_.g_timebase.den = 0; // Not initialized. - DCHECK(!encoding_thread_.IsRunning()); - encoding_thread_.Start(); + DCHECK(!encoding_thread_->IsRunning()); + encoding_thread_->Start(); } VideoTrackRecorder::VpxEncoder::~VpxEncoder() { - DCHECK(main_render_thread_checker_.CalledOnValidThread()); - DCHECK(encoding_thread_.IsRunning()); - encoding_thread_.Stop(); - vpx_codec_destroy(&encoder_); + main_task_runner_->PostTask(FROM_HERE, + base::Bind(&VpxEncoder::ShutdownEncoder, + base::Passed(&encoding_thread_), + base::Passed(&encoder_))); } void VideoTrackRecorder::VpxEncoder::StartFrameEncode( @@ -124,9 +158,9 @@ void VideoTrackRecorder::VpxEncoder::StartFrameEncode( origin_task_runner_ = base::MessageLoop::current()->task_runner(); DCHECK(origin_task_runner_->BelongsToCurrentThread()); - encoding_thread_.task_runner()->PostTask( + encoding_thread_->task_runner()->PostTask( FROM_HERE, base::Bind(&VpxEncoder::EncodeOnEncodingThread, - base::Unretained(this), frame, capture_timestamp)); + this, frame, capture_timestamp)); } void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread( @@ -134,7 +168,7 @@ void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread( base::TimeTicks capture_timestamp) { TRACE_EVENT0("video", "VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread"); - DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread()); + DCHECK(encoding_thread_->task_runner()->BelongsToCurrentThread()); const gfx::Size frame_size = frame->visible_rect().size(); if (!IsInitialized() || @@ -161,21 +195,21 @@ void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread( // Encode the frame. The presentation time stamp argument here is fixed to // zero to force the encoder to base its single-frame bandwidth calculations // entirely on |predicted_frame_duration|. - const vpx_codec_err_t ret = vpx_codec_encode(&encoder_, + const vpx_codec_err_t ret = vpx_codec_encode(encoder_.get(), &vpx_image, 0 /* pts */, duration.InMicroseconds(), kNoFlags, VPX_DL_REALTIME); DCHECK_EQ(ret, VPX_CODEC_OK) << vpx_codec_err_to_string(ret) << ", #" - << vpx_codec_error(&encoder_) << " -" - << vpx_codec_error_detail(&encoder_); + << vpx_codec_error(encoder_.get()) << " -" + << vpx_codec_error_detail(encoder_.get()); scoped_ptr<std::string> data(new std::string); bool keyframe = false; vpx_codec_iter_t iter = NULL; const vpx_codec_cx_pkt_t* pkt = NULL; - while ((pkt = vpx_codec_get_cx_data(&encoder_, &iter)) != NULL) { + while ((pkt = vpx_codec_get_cx_data(encoder_.get(), &iter)) != NULL) { if (pkt->kind != VPX_CODEC_CX_FRAME_PKT) continue; data->assign(static_cast<char*>(pkt->data.frame.buf), pkt->data.frame.sz); @@ -183,7 +217,7 @@ void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread( break; } origin_task_runner_->PostTask(FROM_HERE, - base::Bind(&OnFrameEncodeCompleted, + base::Bind(OnFrameEncodeCompleted, on_encoded_video_callback_, frame, base::Passed(&data), @@ -200,7 +234,7 @@ void VideoTrackRecorder::VpxEncoder::ConfigureVp8Encoding( DVLOG(1) << "Destroying/Re-Creating encoder for new frame size: " << gfx::Size(codec_config_.g_w, codec_config_.g_h).ToString() << " --> " << size.ToString(); - vpx_codec_destroy(&encoder_); + encoder_.reset(); } const vpx_codec_iface_t* interface = vpx_codec_vp8_cx(); @@ -246,19 +280,21 @@ void VideoTrackRecorder::VpxEncoder::ConfigureVp8Encoding( // Number of frames to consume before producing output. codec_config_.g_lag_in_frames = 0; - const vpx_codec_err_t ret = vpx_codec_enc_init(&encoder_, interface, + DCHECK(!encoder_); + encoder_.reset(new vpx_codec_ctx_t); + const vpx_codec_err_t ret = vpx_codec_enc_init(encoder_.get(), interface, &codec_config_, kNoFlags); DCHECK_EQ(VPX_CODEC_OK, ret); } bool VideoTrackRecorder::VpxEncoder::IsInitialized() const { - DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread()); + DCHECK(encoding_thread_->task_runner()->BelongsToCurrentThread()); return codec_config_.g_timebase.den != 0; } base::TimeDelta VideoTrackRecorder::VpxEncoder::CalculateFrameDuration( const scoped_refptr<VideoFrame>& frame) { - DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread()); + DCHECK(encoding_thread_->task_runner()->BelongsToCurrentThread()); using base::TimeDelta; TimeDelta predicted_frame_duration; @@ -285,28 +321,27 @@ VideoTrackRecorder::VideoTrackRecorder( const blink::WebMediaStreamTrack& track, const OnEncodedVideoCB& on_encoded_video_callback) : track_(track), - encoder_(new VpxEncoder(on_encoded_video_callback)), - weak_factory_(this) { + encoder_(new VpxEncoder(on_encoded_video_callback)) { DCHECK(main_render_thread_checker_.CalledOnValidThread()); DCHECK(!track_.isNull()); DCHECK(track.extraData()); + + // StartFrameEncode() will be called on Render IO thread. AddToVideoTrack(this, - media::BindToCurrentLoop( - base::Bind(&VideoTrackRecorder::OnVideoFrame, - weak_factory_.GetWeakPtr())), + base::Bind(&VideoTrackRecorder::VpxEncoder::StartFrameEncode, + encoder_), track_); } VideoTrackRecorder::~VideoTrackRecorder() { DCHECK(main_render_thread_checker_.CalledOnValidThread()); RemoveFromVideoTrack(this, track_); - weak_factory_.InvalidateWeakPtrs(); track_.reset(); } -void VideoTrackRecorder::OnVideoFrame(const scoped_refptr<VideoFrame>& frame, - base::TimeTicks timestamp) { - DCHECK(main_render_thread_checker_.CalledOnValidThread()); +void VideoTrackRecorder::OnVideoFrameForTesting( + const scoped_refptr<media::VideoFrame>& frame, + base::TimeTicks timestamp) { encoder_->StartFrameEncode(frame, timestamp); } diff --git a/content/renderer/media/video_track_recorder.h b/content/renderer/media/video_track_recorder.h index 99a6acb..bc877d6 100644 --- a/content/renderer/media/video_track_recorder.h +++ b/content/renderer/media/video_track_recorder.h @@ -7,7 +7,6 @@ #include "base/macros.h" #include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" #include "base/strings/string_piece.h" #include "base/threading/thread_checker.h" #include "content/public/renderer/media_stream_video_sink.h" @@ -23,7 +22,9 @@ namespace content { // VideoTrackRecorder is a MediaStreamVideoSink that encodes the video frames // received from a Stream Video Track. The class is constructed and used on a // single thread, namely the main Render thread. It has an internal VpxEncoder -// that uses a worker thread for encoding. +// with its own threading subtleties, see the implementation file. This mirrors +// the other MediaStreamVideo* classes that are constructed/configured on Main +// Render thread but that pass frames on Render IO thread. class CONTENT_EXPORT VideoTrackRecorder : NON_EXPORTED_BASE(public MediaStreamVideoSink) { public: @@ -37,8 +38,8 @@ class CONTENT_EXPORT VideoTrackRecorder const OnEncodedVideoCB& on_encoded_video_cb); ~VideoTrackRecorder() override; - void OnVideoFrame(const scoped_refptr<media::VideoFrame>& frame, - base::TimeTicks capture_time); + void OnVideoFrameForTesting(const scoped_refptr<media::VideoFrame>& frame, + base::TimeTicks capture_time); private: friend class VideoTrackRecorderTest; @@ -51,9 +52,7 @@ class CONTENT_EXPORT VideoTrackRecorder // Forward declaration and member of an inner class to encode using VPx. class VpxEncoder; - const scoped_ptr<VpxEncoder> encoder_; - - base::WeakPtrFactory<VideoTrackRecorder> weak_factory_; + const scoped_refptr<VpxEncoder> encoder_; DISALLOW_COPY_AND_ASSIGN(VideoTrackRecorder); }; diff --git a/content/renderer/media/video_track_recorder_unittest.cc b/content/renderer/media/video_track_recorder_unittest.cc index 34af5fc7..75ccc04 100644 --- a/content/renderer/media/video_track_recorder_unittest.cc +++ b/content/renderer/media/video_track_recorder_unittest.cc @@ -81,13 +81,13 @@ class VideoTrackRecorderTest : public testing::Test, void Encode(const scoped_refptr<media::VideoFrame>& frame, base::TimeTicks capture_time) { EXPECT_TRUE(message_loop_.IsCurrent()); - video_track_recorder_->OnVideoFrame(frame, capture_time); + video_track_recorder_->OnVideoFrameForTesting(frame, capture_time); } // A ChildProcess and a MessageLoopForUI are both needed to fool the Tracks // and Sources below into believing they are on the right threads. const base::MessageLoopForUI message_loop_; - ChildProcess child_process_; + const ChildProcess child_process_; // All members are non-const due to the series of initialize() calls needed. // |mock_source_| is owned by |blink_source_|, |track_| by |blink_track_|. @@ -102,6 +102,10 @@ class VideoTrackRecorderTest : public testing::Test, DISALLOW_COPY_AND_ASSIGN(VideoTrackRecorderTest); }; +// Construct and destruct all objects, in particular |video_track_recorder_| and +// its inner object(s). This is a non trivial sequence. +TEST_F(VideoTrackRecorderTest, ConstructAndDestruct) {} + // Creates the encoder and encodes 2 frames of the same size; the encoder should // be initialised and produce a keyframe, then a non-keyframe. Finally a frame // of larger size is sent and is expected to be encoded as a keyframe. diff --git a/media/capture/webm_muxer.cc b/media/capture/webm_muxer.cc index b3f8e74..b913fa7 100644 --- a/media/capture/webm_muxer.cc +++ b/media/capture/webm_muxer.cc @@ -28,19 +28,14 @@ WebmMuxer::WebmMuxer(const WriteDataCB& write_data_callback) : track_index_(0), write_data_callback_(write_data_callback), position_(0) { - DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(!write_data_callback_.is_null()); - segment_.Init(this); - segment_.set_mode(mkvmuxer::Segment::kLive); - segment_.OutputCues(false); - - mkvmuxer::SegmentInfo* const info = segment_.GetSegmentInfo(); - info->set_writing_app("Chrome"); - info->set_muxing_app("Chrome"); + // Creation is done on a different thread than main activities. + thread_checker_.DetachFromThread(); } WebmMuxer::~WebmMuxer() { - DCHECK(thread_checker_.CalledOnValidThread()); + // No need to segment_.Finalize() since is not Seekable(), i.e. a live stream, + // but is good practice. segment_.Finalize(); } @@ -48,6 +43,7 @@ void WebmMuxer::OnEncodedVideo(const scoped_refptr<VideoFrame>& video_frame, const base::StringPiece& encoded_data, base::TimeTicks timestamp, bool is_key_frame) { + DVLOG(1) << __FUNCTION__ << " - " << encoded_data.size() << "B"; DCHECK(thread_checker_.CalledOnValidThread()); if (!track_index_) { // |track_index_|, cannot be zero (!), initialize WebmMuxer in that case. @@ -66,7 +62,16 @@ void WebmMuxer::OnEncodedVideo(const scoped_refptr<VideoFrame>& video_frame, void WebmMuxer::AddVideoTrack(const gfx::Size& frame_size, double frame_rate) { DCHECK(thread_checker_.CalledOnValidThread()); - DCHECK_EQ(track_index_, 0u); + DCHECK_EQ(track_index_, 0u) << "WebmMuxer can only be initialised once."; + + segment_.Init(this); + segment_.set_mode(mkvmuxer::Segment::kLive); + segment_.OutputCues(false); + + mkvmuxer::SegmentInfo* const info = segment_.GetSegmentInfo(); + info->set_writing_app("Chrome"); + info->set_muxing_app("Chrome"); + track_index_ = segment_.AddVideoTrack(frame_size.width(), frame_size.height(), 0); DCHECK_GT(track_index_, 0u); diff --git a/media/capture/webm_muxer.h b/media/capture/webm_muxer.h index 153e14f..3fec647 100644 --- a/media/capture/webm_muxer.h +++ b/media/capture/webm_muxer.h @@ -27,8 +27,9 @@ class VideoFrame; // containing a single encoded video frame. WebM container has no Trailer. // Clients will push encoded VPx video frames one by one via the // OnEncodedVideo(). libwebm will eventually ping the WriteDataCB passed on -// contructor with the wrapped encoded data. All operations must happen in a -// single thread, where WebmMuxer is created and destroyed. +// contructor with the wrapped encoded data. +// WebmMuxer is created/destroyed on a thread, usually the Main Render thread, +// and receives OnEncodedVideo()s on another thread, usually Render IO. // [1] http://www.webmproject.org/docs/container/ // [2] http://www.matroska.org/technical/specs/index.html // TODO(mcasas): Add support for Audio muxing. @@ -56,8 +57,7 @@ class MEDIA_EXPORT WebmMuxer : public NON_EXPORTED_BASE(mkvmuxer::IMkvWriter) { // Creates and adds a new video track. Called upon receiving the first // frame of a given Track, adds |frame_size| and |frame_rate| to the Segment // info, although individual frames passed to OnEncodedVideo() can have any - // frame size. Returns OnEncodedVideo() callback with a |track_number| bound - // in the callback. + // frame size. void AddVideoTrack(const gfx::Size& frame_size, double frame_rate); // IMkvWriter interface. @@ -68,7 +68,7 @@ class MEDIA_EXPORT WebmMuxer : public NON_EXPORTED_BASE(mkvmuxer::IMkvWriter) { void ElementStartNotify(mkvmuxer::uint64 element_id, mkvmuxer::int64 position) override; - // Used to DCHECK that we are called on the correct thread. + // Used to DCHECK that we are called on the correct thread (usually IO) base::ThreadChecker thread_checker_; // A caller-side identifier to interact with |segment_|, initialised upon diff --git a/media/capture/webm_muxer_unittest.cc b/media/capture/webm_muxer_unittest.cc index e3fd4d4..bde9b4f 100644 --- a/media/capture/webm_muxer_unittest.cc +++ b/media/capture/webm_muxer_unittest.cc @@ -34,8 +34,9 @@ class WebmMuxerTest : public testing::Test, public EventHandlerInterface { last_encoded_length_(0), accumulated_position_(0) { EXPECT_EQ(webm_muxer_.Position(), 0); + const mkvmuxer::int64 kRandomNewPosition = 333; + EXPECT_EQ(webm_muxer_.Position(kRandomNewPosition), -1); EXPECT_FALSE(webm_muxer_.Seekable()); - EXPECT_EQ(webm_muxer_.segment_.mode(), mkvmuxer::Segment::kLive); } MOCK_METHOD1(WriteCallback, void(const base::StringPiece&)); @@ -49,6 +50,10 @@ class WebmMuxerTest : public testing::Test, public EventHandlerInterface { return webm_muxer_.Position(); } + mkvmuxer::Segment::Mode GetWebmSegmentMode() const { + return webm_muxer_.segment_.mode(); + } + mkvmuxer::int32 WebmMuxerWrite(const void* buf, mkvmuxer::uint32 len) { return webm_muxer_.Write(buf, len); } @@ -95,6 +100,7 @@ TEST_F(WebmMuxerTest, OnEncodedVideoTwoFrames) { EXPECT_EQ(last_encoded_length_, encoded_data.size()); EXPECT_EQ(GetWebmMuxerPosition(), accumulated_position_); EXPECT_GE(GetWebmMuxerPosition(), static_cast<int64_t>(last_encoded_length_)); + EXPECT_EQ(GetWebmSegmentMode(), mkvmuxer::Segment::kLive); const int64_t begin_of_second_block = accumulated_position_; EXPECT_CALL(*this, WriteCallback(_)) diff --git a/media/media.gyp b/media/media.gyp index 3d48e59..102117e 100644 --- a/media/media.gyp +++ b/media/media.gyp @@ -485,8 +485,6 @@ 'capture/video/win/video_capture_device_mf_win.h', 'capture/video/win/video_capture_device_win.cc', 'capture/video/win/video_capture_device_win.h', - 'capture/webm_muxer.cc', - 'capture/webm_muxer.h', 'cdm/aes_decryptor.cc', 'cdm/aes_decryptor.h', 'cdm/default_cdm_factory.cc', @@ -707,9 +705,7 @@ 'dependencies': [ '<(DEPTH)/third_party/libwebm/libwebm.gyp:libwebm', ], - }, { # media_use_libwebm==0 - # Exclude the sources that depend on libwebm. - 'sources!': [ + 'sources': [ 'capture/webm_muxer.cc', 'capture/webm_muxer.h', ], @@ -1297,7 +1293,6 @@ 'filters/vp8_parser_unittest.cc', 'filters/vp9_parser_unittest.cc', 'filters/vp9_raw_bits_reader_unittest.cc', - 'capture/webm_muxer_unittest.cc', 'formats/common/offset_byte_queue_unittest.cc', 'formats/webm/cluster_builder.cc', 'formats/webm/cluster_builder.h', @@ -1369,9 +1364,7 @@ 'dependencies': [ '<(DEPTH)/third_party/libwebm/libwebm.gyp:libwebm', ], - }, { # media_use_libwebm==0 - # Exclude the sources that depend on libwebm. - 'sources!': [ + 'sources': [ 'capture/webm_muxer_unittest.cc', ], }], |