diff options
author | hclam@chromium.org <hclam@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-05-08 02:45:00 +0000 |
---|---|---|
committer | hclam@chromium.org <hclam@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-05-08 02:45:00 +0000 |
commit | 42d81e5594fc1db20e972a989e0a96d70d4c66f3 (patch) | |
tree | f79313a2f5599b22707bb562468aada97122f3ef /content/renderer | |
parent | 3d88b9e84ca02f88be061842f20eed774ee78eb1 (diff) | |
download | chromium_src-42d81e5594fc1db20e972a989e0a96d70d4c66f3.zip chromium_src-42d81e5594fc1db20e972a989e0a96d70d4c66f3.tar.gz chromium_src-42d81e5594fc1db20e972a989e0a96d70d4c66f3.tar.bz2 |
Change MediaStreamVideoTrack and MediaStreamVideoSource to be able to receive frames on the IO-thread.
Add MediaStreamVideoTrack::AddSink(MediaStreamSink* sink, const VideoFrameCB& callback) where callback is bound to the IO-thread.
The purpose of the cl is to avoid posting frames to the main renderthread unless its needed.
Note that MediaStreamVideoSink still receive its frames on the main render thread - if a sink want to receive frames on the IO thread, it should use the new MediaStreamVideoTrack::AddSink(MediaStreamSink* sink, const VideoFrameCB& callback) method.
TEST= test on apprtc.appspot.com between two tabs.
BUG=335327
R=dmichael@chromium.org, hclam@chromium.org, ronghuawu@chromium.org, tommi@chromium.org
Review URL: https://codereview.chromium.org/252393006
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@269020 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'content/renderer')
29 files changed, 1048 insertions, 400 deletions
diff --git a/content/renderer/media/media_stream_impl_unittest.cc b/content/renderer/media/media_stream_impl_unittest.cc index bc0fbaf..2d15fb1 100644 --- a/content/renderer/media/media_stream_impl_unittest.cc +++ b/content/renderer/media/media_stream_impl_unittest.cc @@ -4,6 +4,7 @@ #include "base/memory/scoped_ptr.h" #include "base/strings/utf_string_conversions.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream.h" #include "content/renderer/media/media_stream_impl.h" #include "content/renderer/media/media_stream_track.h" @@ -106,6 +107,7 @@ class MediaStreamImplTest : public ::testing::Test { public: virtual void SetUp() { // Create our test object. + child_process_.reset(new ChildProcess()); ms_dispatcher_.reset(new MockMediaStreamDispatcher()); dependency_factory_.reset(new MockMediaStreamDependencyFactory()); ms_impl_.reset(new MediaStreamImplUnderTest(ms_dispatcher_.get(), @@ -165,6 +167,7 @@ class MediaStreamImplTest : public ::testing::Test { } protected: + scoped_ptr<ChildProcess> child_process_; scoped_ptr<MockMediaStreamDispatcher> ms_dispatcher_; scoped_ptr<MediaStreamImplUnderTest> ms_impl_; scoped_ptr<MockMediaStreamDependencyFactory> dependency_factory_; diff --git a/content/renderer/media/media_stream_video_capture_source_unittest.cc b/content/renderer/media/media_stream_video_capture_source_unittest.cc index f721070..a204f1e 100644 --- a/content/renderer/media/media_stream_video_capture_source_unittest.cc +++ b/content/renderer/media/media_stream_video_capture_source_unittest.cc @@ -4,6 +4,7 @@ #include "base/bind.h" #include "base/strings/utf_string_conversions.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream_video_capturer_source.h" #include "content/renderer/media/media_stream_video_track.h" #include "content/renderer/media/mock_media_constraint_factory.h" @@ -29,6 +30,11 @@ class MockVideoCapturerDelegate : public VideoCapturerDelegate { class MediaStreamVideoCapturerSourceTest : public testing::Test { public: + MediaStreamVideoCapturerSourceTest() + : child_process_(new ChildProcess()), + source_(NULL) { + } + void InitWithDeviceInfo(const StreamDeviceInfo& device_info) { delegate_ = new MockVideoCapturerDelegate(device_info); source_ = new MediaStreamVideoCapturerSource( @@ -58,6 +64,7 @@ class MediaStreamVideoCapturerSourceTest : public testing::Test { void OnConstraintsApplied(MediaStreamSource* source, bool success) { } + scoped_ptr<ChildProcess> child_process_; blink::WebMediaStreamSource webkit_source_; MediaStreamVideoCapturerSource* source_; // owned by webkit_source. scoped_refptr<MockVideoCapturerDelegate> delegate_; diff --git a/content/renderer/media/media_stream_video_capturer_source.cc b/content/renderer/media/media_stream_video_capturer_source.cc index 0eb3104..4bcab47 100644 --- a/content/renderer/media/media_stream_video_capturer_source.cc +++ b/content/renderer/media/media_stream_video_capturer_source.cc @@ -52,7 +52,6 @@ VideoCapturerDelegate::VideoCapturerDelegate( VideoCapturerDelegate::~VideoCapturerDelegate() { DVLOG(3) << "VideoCapturerDelegate::dtor"; - DCHECK(new_frame_callback_.is_null()); if (!release_device_cb_.is_null()) release_device_cb_.Run(); } @@ -102,7 +101,6 @@ void VideoCapturerDelegate::StartCapture( const StartedCallback& started_callback) { DCHECK(params.requested_format.IsValid()); DCHECK(thread_checker_.CalledOnValidThread()); - new_frame_callback_ = new_frame_callback; started_callback_ = started_callback; got_first_frame_ = false; @@ -119,8 +117,7 @@ void VideoCapturerDelegate::StartCapture( params, media::BindToCurrentLoop(base::Bind( &VideoCapturerDelegate::OnStateUpdateOnRenderThread, this)), - media::BindToCurrentLoop(base::Bind( - &VideoCapturerDelegate::OnFrameReadyOnRenderThread, this))); + new_frame_callback); } void VideoCapturerDelegate::StopCapture() { @@ -130,29 +127,17 @@ void VideoCapturerDelegate::StopCapture() { if (!stop_capture_cb_.is_null()) { base::ResetAndReturn(&stop_capture_cb_).Run(); } - new_frame_callback_.Reset(); started_callback_.Reset(); source_formats_callback_.Reset(); } -void VideoCapturerDelegate::OnFrameReadyOnRenderThread( - const scoped_refptr<media::VideoFrame>& frame, - const media::VideoCaptureFormat& format) { - if (!got_first_frame_) { - got_first_frame_ = true; - if (!started_callback_.is_null()) - started_callback_.Run(true); - } - - if (!new_frame_callback_.is_null()) { - new_frame_callback_.Run(frame, format); - } -} - void VideoCapturerDelegate::OnStateUpdateOnRenderThread( VideoCaptureState state) { - if (state == VIDEO_CAPTURE_STATE_ERROR && !started_callback_.is_null()) { - started_callback_.Run(false); + DCHECK(thread_checker_.CalledOnValidThread()); + DVLOG(3) << "OnStateUpdateOnRenderThread state = " << state; + if (state > VIDEO_CAPTURE_STATE_STARTING && !started_callback_.is_null()) { + base::ResetAndReturn(&started_callback_).Run( + state == VIDEO_CAPTURE_STATE_STARTED); } } @@ -217,8 +202,7 @@ MediaStreamVideoCapturerSource::MediaStreamVideoCapturerSource( const StreamDeviceInfo& device_info, const SourceStoppedCallback& stop_callback, const scoped_refptr<VideoCapturerDelegate>& delegate) - : MediaStreamVideoSource(), - delegate_(delegate) { + : delegate_(delegate) { SetDeviceInfo(device_info); SetStopCallback(stop_callback); } @@ -228,16 +212,17 @@ MediaStreamVideoCapturerSource::~MediaStreamVideoCapturerSource() { void MediaStreamVideoCapturerSource::GetCurrentSupportedFormats( int max_requested_width, - int max_requested_height) { + int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) { delegate_->GetCurrentSupportedFormats( max_requested_width, max_requested_height, - base::Bind(&MediaStreamVideoCapturerSource::OnSupportedFormats, - base::Unretained(this))); + callback); } void MediaStreamVideoCapturerSource::StartSourceImpl( - const media::VideoCaptureParams& params) { + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) { media::VideoCaptureParams new_params(params); if (device_info().device.type == MEDIA_TAB_VIDEO_CAPTURE || device_info().device.type == MEDIA_DESKTOP_VIDEO_CAPTURE) { @@ -245,8 +230,7 @@ void MediaStreamVideoCapturerSource::StartSourceImpl( } delegate_->StartCapture( new_params, - base::Bind(&MediaStreamVideoCapturerSource::DeliverVideoFrame, - base::Unretained(this)), + frame_callback, base::Bind(&MediaStreamVideoCapturerSource::OnStartDone, base::Unretained(this))); } diff --git a/content/renderer/media/media_stream_video_capturer_source.h b/content/renderer/media/media_stream_video_capturer_source.h index 0fbef5d..dcae91a7 100644 --- a/content/renderer/media/media_stream_video_capturer_source.h +++ b/content/renderer/media/media_stream_video_capturer_source.h @@ -54,9 +54,6 @@ class CONTENT_EXPORT VideoCapturerDelegate virtual ~VideoCapturerDelegate(); - void OnFrameReadyOnRenderThread( - const scoped_refptr<media::VideoFrame>& frame, - const media::VideoCaptureFormat& format); void OnStateUpdateOnRenderThread(VideoCaptureState state); void OnDeviceFormatsInUseReceived(const media::VideoCaptureFormats& formats); void OnDeviceSupportedFormatsEnumerated( @@ -71,11 +68,8 @@ class CONTENT_EXPORT VideoCapturerDelegate bool is_screen_cast_; bool got_first_frame_; - // |new_frame_callback_| is provided to this class in StartToDeliver and must - // be valid until StopDeliver is called. - VideoCaptureDeliverFrameCB new_frame_callback_; - // |started_callback| is provided to this class in StartToDeliver and must be - // valid until StopDeliver is called. + // |started_callback| is provided to this class in StartCapture and must be + // valid until StopCapture is called. StartedCallback started_callback_; VideoCaptureDeviceFormatsCB source_formats_callback_; @@ -104,10 +98,12 @@ class CONTENT_EXPORT MediaStreamVideoCapturerSource // Implements MediaStreamVideoSource. virtual void GetCurrentSupportedFormats( int max_requested_width, - int max_requested_height) OVERRIDE; + int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) OVERRIDE; virtual void StartSourceImpl( - const media::VideoCaptureParams& params) OVERRIDE; + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) OVERRIDE; virtual void StopSourceImpl() OVERRIDE; diff --git a/content/renderer/media/media_stream_video_source.cc b/content/renderer/media/media_stream_video_source.cc index 491f6f8..2db0ef0 100644 --- a/content/renderer/media/media_stream_video_source.cc +++ b/content/renderer/media/media_stream_video_source.cc @@ -8,10 +8,13 @@ #include <limits> #include <string> +#include "base/debug/trace_event.h" #include "base/logging.h" #include "base/strings/string_number_conversions.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream_dependency_factory.h" #include "content/renderer/media/media_stream_video_track.h" +#include "content/renderer/media/video_frame_deliverer.h" #include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h" namespace content { @@ -215,6 +218,13 @@ bool GetConstraintValue(const blink::WebMediaConstraints& constraints, return ret; } +// Returns true if |constraint| has mandatory constraints. +bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) { + blink::WebVector<blink::WebMediaConstraint> mandatory_constraints; + constraints.getMandatoryConstraints(mandatory_constraints); + return !mandatory_constraints.isEmpty(); +} + // Retrieve the desired max width and height from |constraints|. void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints, int* desired_width, int* desired_height) { @@ -277,14 +287,90 @@ void GetBestCaptureFormat( } // Empty method used for keeping a reference to the original media::VideoFrame -// in MediaStreamVideoSource::DeliverVideoFrame if cropping is needed. -// The reference to |frame| is kept in the closure that calls this method. +// in MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO if cropping is +// needed. The reference to |frame| is kept in the closure that calls this +// method. void ReleaseOriginalFrame( const scoped_refptr<media::VideoFrame>& frame) { } } // anonymous namespace +// Helper class used for delivering video frames to all registered tracks +// on the IO-thread. +class MediaStreamVideoSource::FrameDeliverer : public VideoFrameDeliverer { + public: + FrameDeliverer( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop) + : VideoFrameDeliverer(io_message_loop) { + } + + // Register |callback| to receive video frames of max size + // |max_frame_output_size| on the IO thread. + // TODO(perkj): Currently |max_frame_output_size| must be the same for all + // |callbacks|. + void AddCallback(void* id, + const VideoCaptureDeliverFrameCB& callback, + const gfx::Size& max_frame_output_size) { + DCHECK(thread_checker().CalledOnValidThread()); + io_message_loop()->PostTask( + FROM_HERE, + base::Bind( + &FrameDeliverer::AddCallbackWithResolutionOnIO, + this, id, callback, max_frame_output_size)); + } + + virtual void DeliverFrameOnIO( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) OVERRIDE { + DCHECK(io_message_loop()->BelongsToCurrentThread()); + TRACE_EVENT0("video", "MediaStreamVideoSource::DeliverFrameOnIO"); + if (max_output_size_.IsEmpty()) + return; // Frame received before the output has been decided. + + scoped_refptr<media::VideoFrame> video_frame(frame); + const gfx::Size& visible_size = frame->visible_rect().size(); + if (visible_size.width() > max_output_size_.width() || + visible_size.height() > max_output_size_.height()) { + // If |frame| is not the size that is expected, we need to crop it by + // providing a new |visible_rect|. The new visible rect must be within the + // original |visible_rect|. + gfx::Rect output_rect = frame->visible_rect(); + output_rect.ClampToCenteredSize(max_output_size_); + // TODO(perkj): Allow cropping of textures once http://crbug/362521 is + // fixed. + if (frame->format() != media::VideoFrame::NATIVE_TEXTURE) { + video_frame = media::VideoFrame::WrapVideoFrame( + frame, + output_rect, + output_rect.size(), + base::Bind(&ReleaseOriginalFrame, frame)); + } + } + VideoFrameDeliverer::DeliverFrameOnIO(video_frame, format); + } + + protected: + virtual ~FrameDeliverer() { + } + + void AddCallbackWithResolutionOnIO( + void* id, + const VideoCaptureDeliverFrameCB& callback, + const gfx::Size& max_frame_output_size) { + DCHECK(io_message_loop()->BelongsToCurrentThread()); + // Currently we only support one frame output size. + DCHECK(!max_frame_output_size.IsEmpty() && + (max_output_size_.IsEmpty() || + max_output_size_ == max_frame_output_size)); + max_output_size_ = max_frame_output_size; + VideoFrameDeliverer::AddCallbackOnIO(id, callback); + } + + private: + gfx::Size max_output_size_; +}; + // static MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource( const blink::WebMediaStreamSource& source) { @@ -301,14 +387,20 @@ bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) { } MediaStreamVideoSource::MediaStreamVideoSource() - : state_(NEW) { + : state_(NEW), + frame_deliverer_( + new MediaStreamVideoSource::FrameDeliverer( + ChildProcess::current()->io_message_loop_proxy())), + weak_factory_(this) { } MediaStreamVideoSource::~MediaStreamVideoSource() { + DVLOG(3) << "~MediaStreamVideoSource()"; } void MediaStreamVideoSource::AddTrack( MediaStreamVideoTrack* track, + const VideoCaptureDeliverFrameCB& frame_callback, const blink::WebMediaConstraints& constraints, const ConstraintsCallback& callback) { DCHECK(CalledOnValidThread()); @@ -317,7 +409,7 @@ void MediaStreamVideoSource::AddTrack( tracks_.push_back(track); requested_constraints_.push_back( - RequestedConstraints(constraints, callback)); + RequestedConstraints(track, frame_callback, constraints, callback)); switch (state_) { case NEW: { @@ -330,8 +422,11 @@ void MediaStreamVideoSource::AddTrack( GetConstraintValue(constraints, true, kMaxHeight, &max_requested_height); state_ = RETRIEVING_CAPABILITIES; - GetCurrentSupportedFormats(max_requested_width, - max_requested_height); + GetCurrentSupportedFormats( + max_requested_width, + max_requested_height, + base::Bind(&MediaStreamVideoSource::OnSupportedFormats, + weak_factory_.GetWeakPtr())); break; } @@ -355,10 +450,19 @@ void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) { std::find(tracks_.begin(), tracks_.end(), video_track); DCHECK(it != tracks_.end()); tracks_.erase(it); + // Call |RemoveCallback| here even if adding the track has failed and + // frame_deliverer_->AddCallback has not been called. + frame_deliverer_->RemoveCallback(video_track); + if (tracks_.empty()) StopSource(); } +const scoped_refptr<base::MessageLoopProxy>& +MediaStreamVideoSource::io_message_loop() const { + return frame_deliverer_->io_message_loop(); +} + void MediaStreamVideoSource::DoStopSource() { DCHECK(CalledOnValidThread()); DVLOG(3) << "DoStopSource()"; @@ -369,36 +473,6 @@ void MediaStreamVideoSource::DoStopSource() { SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded); } -void MediaStreamVideoSource::DeliverVideoFrame( - const scoped_refptr<media::VideoFrame>& frame, - const media::VideoCaptureFormat& format) { - DCHECK(CalledOnValidThread()); - scoped_refptr<media::VideoFrame> video_frame(frame); - - if (frame->visible_rect().size().width() > max_frame_output_size_.width() || - frame->visible_rect().size().height() > max_frame_output_size_.height()) { - // If |frame| is not the size that is expected, we need to crop it by - // providing a new |visible_rect|. The new visible rect must be within the - // original |visible_rect|. - gfx::Rect output_rect = frame->visible_rect(); - output_rect.ClampToCenteredSize(max_frame_output_size_); - // TODO(perkj): Allow cropping of textures once http://crbug/362521 is - // fixed. - if (frame->format() != media::VideoFrame::NATIVE_TEXTURE) { - video_frame = media::VideoFrame::WrapVideoFrame( - frame, - output_rect, - output_rect.size(), - base::Bind(&ReleaseOriginalFrame, frame)); - } - } - - for (std::vector<MediaStreamVideoTrack*>::iterator it = tracks_.begin(); - it != tracks_.end(); ++it) { - (*it)->OnVideoFrame(video_frame); - } -} - void MediaStreamVideoSource::OnSupportedFormats( const media::VideoCaptureFormats& formats) { DCHECK(CalledOnValidThread()); @@ -424,7 +498,10 @@ void MediaStreamVideoSource::OnSupportedFormats( media::VideoCaptureParams params; params.requested_format = current_format_; - StartSourceImpl(params); + StartSourceImpl( + params, + base::Bind(&MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO, + frame_deliverer_)); } bool MediaStreamVideoSource::FindBestFormatWithConstraints( @@ -439,6 +516,16 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints( const blink::WebMediaConstraints& requested_constraints = request_it->constraints; + // If the source doesn't support capability enumeration it is still ok if + // no mandatory constraints have been specified. That just means that + // we will start with whatever format is native to the source. + if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) { + *best_format = media::VideoCaptureFormat(); + *resulting_constraints = requested_constraints; + *max_frame_output_size = gfx::Size(std::numeric_limits<int>::max(), + std::numeric_limits<int>::max()); + return true; + } media::VideoCaptureFormats filtered_formats = FilterFormats(requested_constraints, formats); if (filtered_formats.size() > 0) { @@ -480,8 +567,19 @@ void MediaStreamVideoSource::FinalizeAddTrack() { callbacks.swap(requested_constraints_); for (std::vector<RequestedConstraints>::iterator it = callbacks.begin(); it != callbacks.end(); ++it) { - bool success = state_ == STARTED && - !FilterFormats(it->constraints, formats).empty(); + // The track has been added successfully if the source has started and + // there are either no mandatory constraints and the source doesn't expose + // its format capabilities, or the constraints and the format match. + // For example, a remote source doesn't expose its format capabilities. + bool success = + state_ == STARTED && + ((!current_format_.IsValid() && !HasMandatoryConstraints( + it->constraints)) || + !FilterFormats(it->constraints, formats).empty()); + if (success) { + frame_deliverer_->AddCallback(it->track, it->frame_callback, + max_frame_output_size_); + } DVLOG(3) << "FinalizeAddTrack() success " << success; if (!it->callback.is_null()) it->callback.Run(this, success); @@ -500,9 +598,14 @@ void MediaStreamVideoSource::SetReadyState( } MediaStreamVideoSource::RequestedConstraints::RequestedConstraints( + MediaStreamVideoTrack* track, + const VideoCaptureDeliverFrameCB& frame_callback, const blink::WebMediaConstraints& constraints, const ConstraintsCallback& callback) - : constraints(constraints), callback(callback) { + : track(track), + frame_callback(frame_callback), + constraints(constraints), + callback(callback) { } MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() { diff --git a/content/renderer/media/media_stream_video_source.h b/content/renderer/media/media_stream_video_source.h index 933017c..34b9170 100644 --- a/content/renderer/media/media_stream_video_source.h +++ b/content/renderer/media/media_stream_video_source.h @@ -5,12 +5,16 @@ #ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_VIDEO_SOURCE_H_ #define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_VIDEO_SOURCE_H_ +#include <string> #include <vector> #include "base/compiler_specific.h" +#include "base/memory/weak_ptr.h" #include "base/threading/non_thread_safe.h" #include "content/common/content_export.h" +#include "content/common/media/video_capture.h" #include "content/renderer/media/media_stream_source.h" +#include "content/renderer/media/video_frame_deliverer.h" #include "media/base/video_frame.h" #include "media/video/capture/video_capture_types.h" #include "third_party/WebKit/public/platform/WebMediaConstraints.h" @@ -49,6 +53,7 @@ class CONTENT_EXPORT MediaStreamVideoSource // Puts |track| in the registered tracks list. void AddTrack(MediaStreamVideoTrack* track, + const VideoCaptureDeliverFrameCB& frame_callback, const blink::WebMediaConstraints& constraints, const ConstraintsCallback& callback); void RemoveTrack(MediaStreamVideoTrack* track); @@ -56,6 +61,9 @@ class CONTENT_EXPORT MediaStreamVideoSource // Return true if |name| is a constraint supported by MediaStreamVideoSource. static bool IsConstraintSupported(const std::string& name); + // Returns the MessageLoopProxy where video frames will be delivered on. + const scoped_refptr<base::MessageLoopProxy>& io_message_loop() const; + // Constraint keys used by a video source. // Specified by draft-alvestrand-constraints-resolution-00b static const char kMinAspectRatio[]; // minAspectRatio @@ -79,30 +87,26 @@ class CONTENT_EXPORT MediaStreamVideoSource // Sets ready state and notifies the ready state to all registered tracks. virtual void SetReadyState(blink::WebMediaStreamSource::ReadyState state); - // Delivers |frame| to registered tracks according to their constraints. - // Note: current implementation assumes |frame| be contiguous layout of image - // planes and I420. - // |format| contains extra information like the frame rate of this source. - virtual void DeliverVideoFrame(const scoped_refptr<media::VideoFrame>& frame, - const media::VideoCaptureFormat& format); - // An implementation must fetch the formats that can currently be used by // the source and call OnSupportedFormats when done. // |max_requested_height| and |max_requested_width| is the max height and // width set as a mandatory constraint if set when calling // MediaStreamVideoSource::AddTrack. If max height and max width is not set // |max_requested_height| and |max_requested_width| are 0. - virtual void GetCurrentSupportedFormats(int max_requested_width, - int max_requested_height) = 0; - void OnSupportedFormats(const media::VideoCaptureFormats& formats); + virtual void GetCurrentSupportedFormats( + int max_requested_width, + int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) = 0; // An implementation must start capture frames using the resolution in // |params|. When the source has started or the source failed to start // OnStartDone must be called. An implementation must call - // DeliverVideoFrame with the captured frames. + // invoke |frame_callback| on the IO thread with the captured frames. // TODO(perkj): pass a VideoCaptureFormats instead of VideoCaptureParams for // subclasses to customize. - virtual void StartSourceImpl(const media::VideoCaptureParams& params) = 0; + virtual void StartSourceImpl( + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) = 0; void OnStartDone(bool success); // An implementation must immediately stop capture video frames and must not @@ -120,8 +124,7 @@ class CONTENT_EXPORT MediaStreamVideoSource State state() const { return state_; } private: - // Creates a webrtc::VideoSourceInterface used by libjingle. - void InitAdapter(); + void OnSupportedFormats(const media::VideoCaptureFormats& formats); // Finds the first constraints in |requested_constraints_| that can be // fulfilled. |best_format| is set to the video resolution that can be @@ -152,10 +155,14 @@ class CONTENT_EXPORT MediaStreamVideoSource gfx::Size max_frame_output_size_; struct RequestedConstraints { - RequestedConstraints(const blink::WebMediaConstraints& constraints, + RequestedConstraints(MediaStreamVideoTrack* track, + const VideoCaptureDeliverFrameCB& frame_callback, + const blink::WebMediaConstraints& constraints, const ConstraintsCallback& callback); ~RequestedConstraints(); + MediaStreamVideoTrack* track; + VideoCaptureDeliverFrameCB frame_callback; blink::WebMediaConstraints constraints; ConstraintsCallback callback; }; @@ -163,9 +170,17 @@ class CONTENT_EXPORT MediaStreamVideoSource media::VideoCaptureFormats supported_formats_; + // |FrameDeliverer| is an internal helper object used for delivering video + // frames using callbacks to all registered tracks on the IO thread. + class FrameDeliverer; + scoped_refptr<FrameDeliverer> frame_deliverer_; + // Tracks that currently are receiving video frames. std::vector<MediaStreamVideoTrack*> tracks_; + // NOTE: Weak pointers must be invalidated before all other member variables. + base::WeakPtrFactory<MediaStreamVideoSource> weak_factory_; + DISALLOW_COPY_AND_ASSIGN(MediaStreamVideoSource); }; diff --git a/content/renderer/media/media_stream_video_source_unittest.cc b/content/renderer/media/media_stream_video_source_unittest.cc index 1274cc5..1b1f204 100644 --- a/content/renderer/media/media_stream_video_source_unittest.cc +++ b/content/renderer/media/media_stream_video_source_unittest.cc @@ -6,24 +6,31 @@ #include <vector> #include "base/bind.h" +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" #include "base/strings/string_number_conversions.h" #include "base/strings/utf_string_conversions.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream_video_source.h" #include "content/renderer/media/media_stream_video_track.h" #include "content/renderer/media/mock_media_constraint_factory.h" #include "content/renderer/media/mock_media_stream_video_sink.h" -#include "content/renderer/media/mock_media_stream_video_sink.h" #include "content/renderer/media/mock_media_stream_video_source.h" #include "media/base/video_frame.h" #include "testing/gtest/include/gtest/gtest.h" namespace content { +ACTION_P(RunClosure, closure) { + closure.Run(); +} + class MediaStreamVideoSourceTest : public ::testing::Test { public: MediaStreamVideoSourceTest() - : number_of_successful_constraints_applied_(0), + : child_process_(new ChildProcess()), + number_of_successful_constraints_applied_(0), number_of_failed_constraints_applied_(0), mock_source_(new MockMediaStreamVideoSource(true)) { media::VideoCaptureFormats formats; @@ -96,17 +103,13 @@ class MediaStreamVideoSourceTest int expected_height) { // Expect the source to start capture with the supported resolution. blink::WebMediaStreamTrack track = - CreateTrackAndStartSource(constraints, capture_width, capture_height , + CreateTrackAndStartSource(constraints, capture_width, capture_height, 30); MockMediaStreamVideoSink sink; MediaStreamVideoSink::AddToVideoTrack(&sink, track); - EXPECT_EQ(0, sink.number_of_frames()); - scoped_refptr<media::VideoFrame> frame = - media::VideoFrame::CreateBlackFrame(gfx::Size(capture_width, - capture_height)); - mock_source()->DeliverVideoFrame(frame, media::VideoCaptureFormat()); + DeliverVideoFrameAndWaitForRenderer(capture_width, capture_height, &sink); EXPECT_EQ(1, sink.number_of_frames()); // Expect the delivered frame to be cropped. @@ -115,6 +118,19 @@ class MediaStreamVideoSourceTest MediaStreamVideoSink::RemoveFromVideoTrack(&sink, track); } + void DeliverVideoFrameAndWaitForRenderer(int width, int height, + MockMediaStreamVideoSink* sink) { + base::RunLoop run_loop; + base::Closure quit_closure = run_loop.QuitClosure(); + EXPECT_CALL(*sink, OnVideoFrame()).WillOnce( + RunClosure(quit_closure)); + scoped_refptr<media::VideoFrame> frame = + media::VideoFrame::CreateBlackFrame(gfx::Size(width, height)); + mock_source()->DeliverVideoFrame(frame); + run_loop.Run(); + } + + void ReleaseTrackAndSourceOnAddTrackCallback( const blink::WebMediaStreamTrack& track_to_release) { track_to_release_ = track_to_release; @@ -135,7 +151,8 @@ class MediaStreamVideoSourceTest track_to_release_.reset(); } } - + scoped_ptr<ChildProcess> child_process_; + base::MessageLoopForUI message_loop_; blink::WebMediaStreamTrack track_to_release_; int number_of_successful_constraints_applied_; int number_of_failed_constraints_applied_; @@ -414,37 +431,22 @@ TEST_F(MediaStreamVideoSourceTest, SourceChangeFrameSize) { MockMediaStreamVideoSink sink; MediaStreamVideoSink::AddToVideoTrack(&sink, track); EXPECT_EQ(0, sink.number_of_frames()); - - { - scoped_refptr<media::VideoFrame> frame1 = - media::VideoFrame::CreateBlackFrame(gfx::Size(320, 240)); - mock_source()->DeliverVideoFrame(frame1, - media::VideoCaptureFormat()); - } + DeliverVideoFrameAndWaitForRenderer(320, 240, &sink); EXPECT_EQ(1, sink.number_of_frames()); // Expect the delivered frame to be passed unchanged since its smaller than // max requested. EXPECT_EQ(320, sink.frame_size().width()); EXPECT_EQ(240, sink.frame_size().height()); - { - scoped_refptr<media::VideoFrame> frame2 = - media::VideoFrame::CreateBlackFrame(gfx::Size(640, 480)); - mock_source()->DeliverVideoFrame(frame2, - media::VideoCaptureFormat()); - } + DeliverVideoFrameAndWaitForRenderer(640, 480, &sink); EXPECT_EQ(2, sink.number_of_frames()); // Expect the delivered frame to be passed unchanged since its smaller than // max requested. EXPECT_EQ(640, sink.frame_size().width()); EXPECT_EQ(480, sink.frame_size().height()); - { - scoped_refptr<media::VideoFrame> frame3 = - media::VideoFrame::CreateBlackFrame(gfx::Size(1280, 720)); - mock_source()->DeliverVideoFrame(frame3, - media::VideoCaptureFormat()); - } + DeliverVideoFrameAndWaitForRenderer(1280, 720, &sink); + EXPECT_EQ(3, sink.number_of_frames()); // Expect a frame to be cropped since its larger than max requested. EXPECT_EQ(800, sink.frame_size().width()); diff --git a/content/renderer/media/media_stream_video_track.cc b/content/renderer/media/media_stream_video_track.cc index f5cea67..577c7dc 100644 --- a/content/renderer/media/media_stream_video_track.cc +++ b/content/renderer/media/media_stream_video_track.cc @@ -4,9 +4,104 @@ #include "content/renderer/media/media_stream_video_track.h" +#include "base/bind.h" +#include "content/renderer/media/video_frame_deliverer.h" +#include "media/base/bind_to_current_loop.h" + namespace content { -//static +// Helper class used for delivering video frames to MediaStreamSinks on the +// IO-thread and MediaStreamVideoSinks on the main render thread. +// Frames are delivered to an instance of this class from a +// MediaStreamVideoSource on the IO-thread to the method DeliverFrameOnIO. +// Frames are only delivered to the sinks if the track is enabled. +class MediaStreamVideoTrack::FrameDeliverer : public VideoFrameDeliverer { + public: + FrameDeliverer( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy, + bool enabled) + : VideoFrameDeliverer(io_message_loop_proxy), + enabled_(enabled) { + } + + // Add |sink| to receive frames and state changes on the main render thread. + void AddSink(MediaStreamVideoSink* sink) { + DCHECK(thread_checker().CalledOnValidThread()); + DCHECK(std::find(sinks_.begin(), sinks_.end(), sink) == sinks_.end()); + if (sinks_.empty()) { + VideoCaptureDeliverFrameCB frame_callback = media::BindToCurrentLoop( + base::Bind( + &MediaStreamVideoTrack::FrameDeliverer::OnVideoFrameOnMainThread, + this)); + AddCallback(this, frame_callback); + } + + sinks_.push_back(sink); + } + + void RemoveSink(MediaStreamVideoSink* sink) { + DCHECK(thread_checker().CalledOnValidThread()); + std::vector<MediaStreamVideoSink*>::iterator it = + std::find(sinks_.begin(), sinks_.end(), sink); + DCHECK(it != sinks_.end()); + sinks_.erase(it); + if (sinks_.empty()) { + RemoveCallback(this); + } + } + + // Called when a video frame is received on the main render thread. + // It delivers the received frames to the registered MediaStreamVideo sinks. + void OnVideoFrameOnMainThread( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) { + DCHECK(thread_checker().CalledOnValidThread()); + for (std::vector<MediaStreamVideoSink*>::iterator it = sinks_.begin(); + it != sinks_.end(); ++it) { + (*it)->OnVideoFrame(frame); + } + } + + void SetEnabled(bool enabled) { + DCHECK(thread_checker().CalledOnValidThread()); + io_message_loop()->PostTask( + FROM_HERE, + base::Bind(&MediaStreamVideoTrack::FrameDeliverer::SetEnabledOnIO, + this, enabled)); + } + + virtual void DeliverFrameOnIO( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) OVERRIDE { + DCHECK(io_message_loop()->BelongsToCurrentThread()); + if (!enabled_) + return; + VideoFrameDeliverer::DeliverFrameOnIO(frame, format); + } + + const std::vector<MediaStreamVideoSink*>& sinks() const { return sinks_; } + + protected: + virtual ~FrameDeliverer() { + DCHECK(sinks_.empty()); + } + + void SetEnabledOnIO(bool enabled) { + DCHECK(io_message_loop()->BelongsToCurrentThread()); + enabled_ = enabled; + } + + private: + // The below members are used on the main render thread. + std::vector<MediaStreamVideoSink*> sinks_; + + // The below parameters are used on the IO-thread. + bool enabled_; + + DISALLOW_COPY_AND_ASSIGN(FrameDeliverer); +}; + +// static blink::WebMediaStreamTrack MediaStreamVideoTrack::CreateVideoTrack( MediaStreamVideoSource* source, const blink::WebMediaConstraints& constraints, @@ -33,37 +128,52 @@ MediaStreamVideoTrack::MediaStreamVideoTrack( const MediaStreamVideoSource::ConstraintsCallback& callback, bool enabled) : MediaStreamTrack(NULL, true), - enabled_(enabled), + frame_deliverer_( + new MediaStreamVideoTrack::FrameDeliverer(source->io_message_loop(), + enabled)), constraints_(constraints), source_(source) { - source->AddTrack(this, constraints, callback); + source->AddTrack(this, + base::Bind( + &MediaStreamVideoTrack::FrameDeliverer::DeliverFrameOnIO, + frame_deliverer_), + constraints, callback); } MediaStreamVideoTrack::~MediaStreamVideoTrack() { - DCHECK(sinks_.empty()); Stop(); + DVLOG(3) << "~MediaStreamVideoTrack()"; } void MediaStreamVideoTrack::AddSink(MediaStreamVideoSink* sink) { DCHECK(thread_checker_.CalledOnValidThread()); - DCHECK(std::find(sinks_.begin(), sinks_.end(), sink) == sinks_.end()); - sinks_.push_back(sink); + frame_deliverer_->AddSink(sink); } void MediaStreamVideoTrack::RemoveSink(MediaStreamVideoSink* sink) { DCHECK(thread_checker_.CalledOnValidThread()); - std::vector<MediaStreamVideoSink*>::iterator it = - std::find(sinks_.begin(), sinks_.end(), sink); - DCHECK(it != sinks_.end()); - sinks_.erase(it); + frame_deliverer_->RemoveSink(sink); +} + +void MediaStreamVideoTrack::AddSink( + MediaStreamSink* sink, const VideoCaptureDeliverFrameCB& callback) { + DCHECK(thread_checker_.CalledOnValidThread()); + frame_deliverer_->AddCallback(sink, callback); +} + +void MediaStreamVideoTrack::RemoveSink(MediaStreamSink* sink) { + DCHECK(thread_checker_.CalledOnValidThread()); + frame_deliverer_->RemoveCallback(sink); } void MediaStreamVideoTrack::SetEnabled(bool enabled) { DCHECK(thread_checker_.CalledOnValidThread()); - enabled_ = enabled; MediaStreamTrack::SetEnabled(enabled); - for (std::vector<MediaStreamVideoSink*>::iterator it = sinks_.begin(); - it != sinks_.end(); ++it) { + + frame_deliverer_->SetEnabled(enabled); + const std::vector<MediaStreamVideoSink*>& sinks = frame_deliverer_->sinks(); + for (std::vector<MediaStreamVideoSink*>::const_iterator it = sinks.begin(); + it != sinks.end(); ++it) { (*it)->OnEnabledChanged(enabled); } } @@ -77,23 +187,12 @@ void MediaStreamVideoTrack::Stop() { OnReadyStateChanged(blink::WebMediaStreamSource::ReadyStateEnded); } -void MediaStreamVideoTrack::OnVideoFrame( - const scoped_refptr<media::VideoFrame>& frame) { - DCHECK(thread_checker_.CalledOnValidThread()); - if (!enabled_) - return; - - for (std::vector<MediaStreamVideoSink*>::iterator it = sinks_.begin(); - it != sinks_.end(); ++it) { - (*it)->OnVideoFrame(frame); - } -} - void MediaStreamVideoTrack::OnReadyStateChanged( blink::WebMediaStreamSource::ReadyState state) { DCHECK(thread_checker_.CalledOnValidThread()); - for (std::vector<MediaStreamVideoSink*>::iterator it = sinks_.begin(); - it != sinks_.end(); ++it) { + const std::vector<MediaStreamVideoSink*>& sinks = frame_deliverer_->sinks(); + for (std::vector<MediaStreamVideoSink*>::const_iterator it = sinks.begin(); + it != sinks.end(); ++it) { (*it)->OnReadyStateChanged(state); } } diff --git a/content/renderer/media/media_stream_video_track.h b/content/renderer/media/media_stream_video_track.h index c28ebb1..9fa8125 100644 --- a/content/renderer/media/media_stream_video_track.h +++ b/content/renderer/media/media_stream_video_track.h @@ -47,13 +47,21 @@ class CONTENT_EXPORT MediaStreamVideoTrack : public MediaStreamTrack { const MediaStreamVideoSource::ConstraintsCallback& callback, bool enabled); virtual ~MediaStreamVideoTrack(); + + // Add |sink| to receive state changes and video frames on the main render + // thread. virtual void AddSink(MediaStreamVideoSink* sink); virtual void RemoveSink(MediaStreamVideoSink* sink); + // Add |sink| to receive state changes on the main render thread and video + // frames in the |callback| method on the IO-thread. + virtual void AddSink(MediaStreamSink* sink, + const VideoCaptureDeliverFrameCB& callback); + virtual void RemoveSink(MediaStreamSink* sink); + virtual void SetEnabled(bool enabled) OVERRIDE; virtual void Stop() OVERRIDE; - void OnVideoFrame(const scoped_refptr<media::VideoFrame>& frame); void OnReadyStateChanged(blink::WebMediaStreamSource::ReadyState state); const blink::WebMediaConstraints& constraints() const { @@ -65,8 +73,11 @@ class CONTENT_EXPORT MediaStreamVideoTrack : public MediaStreamTrack { base::ThreadChecker thread_checker_; private: - bool enabled_; - std::vector<MediaStreamVideoSink*> sinks_; + // |FrameDeliverer| is an internal helper object used for delivering video + // frames on the IO-thread using callbacks to all registered tracks. + class FrameDeliverer; + scoped_refptr<FrameDeliverer> frame_deliverer_; + blink::WebMediaConstraints constraints_; // Weak ref to the source this tracks is connected to. |source_| is owned diff --git a/content/renderer/media/media_stream_video_track_unittest.cc b/content/renderer/media/media_stream_video_track_unittest.cc index c10fba2..758c568 100644 --- a/content/renderer/media/media_stream_video_track_unittest.cc +++ b/content/renderer/media/media_stream_video_track_unittest.cc @@ -2,7 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" #include "base/strings/utf_string_conversions.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream_video_track.h" #include "content/renderer/media/mock_media_stream_video_sink.h" #include "content/renderer/media/mock_media_stream_video_source.h" @@ -11,10 +14,15 @@ namespace content { +ACTION_P(RunClosure, closure) { + closure.Run(); +} + class MediaStreamVideoTrackTest : public ::testing::Test { public: MediaStreamVideoTrackTest() - : mock_source_(new MockMediaStreamVideoSource(false)), + : child_process_(new ChildProcess()), + mock_source_(new MockMediaStreamVideoSource(false)), source_started_(false) { blink_source_.initialize(base::UTF8ToUTF16("dummy_source_id"), blink::WebMediaStreamSource::TypeVideo, @@ -22,7 +30,27 @@ class MediaStreamVideoTrackTest : public ::testing::Test { blink_source_.setExtraData(mock_source_); } + virtual ~MediaStreamVideoTrackTest() { + } + + void DeliverVideoFrameAndWaitForRenderer(MockMediaStreamVideoSink* sink) { + base::RunLoop run_loop; + base::Closure quit_closure = run_loop.QuitClosure(); + EXPECT_CALL(*sink, OnVideoFrame()).WillOnce( + RunClosure(quit_closure)); + scoped_refptr<media::VideoFrame> frame = + media::VideoFrame::CreateBlackFrame( + gfx::Size(MediaStreamVideoSource::kDefaultWidth, + MediaStreamVideoSource::kDefaultHeight)); + mock_source()->DeliverVideoFrame(frame); + run_loop.Run(); + } + protected: + base::MessageLoop* io_message_loop() const { + return child_process_->io_message_loop(); + } + // Create a track that's associated with |mock_source_|. blink::WebMediaStreamTrack CreateTrack() { blink::WebMediaConstraints constraints; @@ -45,6 +73,8 @@ class MediaStreamVideoTrackTest : public ::testing::Test { } private: + scoped_ptr<ChildProcess> child_process_; + base::MessageLoopForUI message_loop_; blink::WebMediaStreamSource blink_source_; // |mock_source_| is owned by |webkit_source_|. MockMediaStreamVideoSource* mock_source_; @@ -56,19 +86,20 @@ TEST_F(MediaStreamVideoTrackTest, AddAndRemoveSink) { blink::WebMediaStreamTrack track = CreateTrack(); MediaStreamVideoSink::AddToVideoTrack(&sink, track); - MediaStreamVideoTrack* video_track = - MediaStreamVideoTrack::GetVideoTrack(track); + DeliverVideoFrameAndWaitForRenderer(&sink); + EXPECT_EQ(1, sink.number_of_frames()); + + DeliverVideoFrameAndWaitForRenderer(&sink); + + MediaStreamVideoSink::RemoveFromVideoTrack(&sink, track); + scoped_refptr<media::VideoFrame> frame = media::VideoFrame::CreateBlackFrame( gfx::Size(MediaStreamVideoSource::kDefaultWidth, MediaStreamVideoSource::kDefaultHeight)); - video_track->OnVideoFrame(frame); - EXPECT_EQ(1, sink.number_of_frames()); - video_track->OnVideoFrame(frame); - EXPECT_EQ(2, sink.number_of_frames()); - - MediaStreamVideoSink::RemoveFromVideoTrack(&sink, track); - video_track->OnVideoFrame(frame); + mock_source()->DeliverVideoFrame(frame); + // Wait for the IO thread to complete delivering frames. + io_message_loop()->RunUntilIdle(); EXPECT_EQ(2, sink.number_of_frames()); } @@ -79,21 +110,26 @@ TEST_F(MediaStreamVideoTrackTest, SetEnabled) { MediaStreamVideoTrack* video_track = MediaStreamVideoTrack::GetVideoTrack(track); - scoped_refptr<media::VideoFrame> frame = - media::VideoFrame::CreateBlackFrame( - gfx::Size(MediaStreamVideoSource::kDefaultWidth, - MediaStreamVideoSource::kDefaultHeight)); - video_track->OnVideoFrame(frame); + + DeliverVideoFrameAndWaitForRenderer(&sink); EXPECT_EQ(1, sink.number_of_frames()); video_track->SetEnabled(false); EXPECT_FALSE(sink.enabled()); - video_track->OnVideoFrame(frame); + + scoped_refptr<media::VideoFrame> frame = + media::VideoFrame::CreateBlackFrame( + gfx::Size(MediaStreamVideoSource::kDefaultWidth, + MediaStreamVideoSource::kDefaultHeight)); + mock_source()->DeliverVideoFrame(frame); + // Wait for the IO thread to complete delivering frames. + io_message_loop()->RunUntilIdle(); EXPECT_EQ(1, sink.number_of_frames()); video_track->SetEnabled(true); EXPECT_TRUE(sink.enabled()); - video_track->OnVideoFrame(frame); + mock_source()->DeliverVideoFrame(frame); + DeliverVideoFrameAndWaitForRenderer(&sink); EXPECT_EQ(2, sink.number_of_frames()); MediaStreamVideoSink::RemoveFromVideoTrack(&sink, track); } diff --git a/content/renderer/media/mock_media_stream_video_sink.cc b/content/renderer/media/mock_media_stream_video_sink.cc index ce9222f..4246a39 100644 --- a/content/renderer/media/mock_media_stream_video_sink.cc +++ b/content/renderer/media/mock_media_stream_video_sink.cc @@ -13,11 +13,15 @@ MockMediaStreamVideoSink::MockMediaStreamVideoSink() state_(blink::WebMediaStreamSource::ReadyStateLive) { } +MockMediaStreamVideoSink::~MockMediaStreamVideoSink() { +} + void MockMediaStreamVideoSink::OnVideoFrame( const scoped_refptr<media::VideoFrame>& frame) { ++number_of_frames_; format_ = frame->format(); frame_size_ = frame->natural_size(); + OnVideoFrame(); } void MockMediaStreamVideoSink::OnReadyStateChanged( diff --git a/content/renderer/media/mock_media_stream_video_sink.h b/content/renderer/media/mock_media_stream_video_sink.h index 9511a0a..07ec43f 100644 --- a/content/renderer/media/mock_media_stream_video_sink.h +++ b/content/renderer/media/mock_media_stream_video_sink.h @@ -8,12 +8,14 @@ #include "content/public/renderer/media_stream_video_sink.h" #include "media/base/video_frame.h" +#include "testing/gmock/include/gmock/gmock.h" namespace content { class MockMediaStreamVideoSink : public MediaStreamVideoSink { public: MockMediaStreamVideoSink(); + virtual ~MockMediaStreamVideoSink(); virtual void OnVideoFrame( const scoped_refptr<media::VideoFrame>& frame) OVERRIDE; @@ -21,6 +23,10 @@ class MockMediaStreamVideoSink : public MediaStreamVideoSink { blink::WebMediaStreamSource::ReadyState state) OVERRIDE; virtual void OnEnabledChanged(bool enabled) OVERRIDE; + // Triggered when OnVideoFrame(const scoped_refptr<media::VideoFrame>& frame) + // is called. + MOCK_METHOD0(OnVideoFrame, void()); + int number_of_frames() const { return number_of_frames_; } media::VideoFrame::Format format() const { return format_; } gfx::Size frame_size() const { return frame_size_; } diff --git a/content/renderer/media/mock_media_stream_video_source.cc b/content/renderer/media/mock_media_stream_video_source.cc index 84a4479..12bddb4 100644 --- a/content/renderer/media/mock_media_stream_video_source.cc +++ b/content/renderer/media/mock_media_stream_video_source.cc @@ -4,21 +4,24 @@ #include "content/renderer/media/mock_media_stream_video_source.h" +#include "base/bind.h" +#include "base/callback_helpers.h" +#include "base/location.h" + namespace content { MockMediaStreamVideoSource::MockMediaStreamVideoSource( bool manual_get_supported_formats) - : MediaStreamVideoSource(), - manual_get_supported_formats_(manual_get_supported_formats), + : manual_get_supported_formats_(manual_get_supported_formats), max_requested_height_(0), max_requested_width_(0), attempted_to_start_(false) { supported_formats_.push_back( - media::VideoCaptureFormat( - gfx::Size(MediaStreamVideoSource::kDefaultWidth, - MediaStreamVideoSource::kDefaultHeight), - MediaStreamVideoSource::kDefaultFrameRate, - media::PIXEL_FORMAT_I420)); + media::VideoCaptureFormat( + gfx::Size(MediaStreamVideoSource::kDefaultWidth, + MediaStreamVideoSource::kDefaultHeight), + MediaStreamVideoSource::kDefaultFrameRate, + media::PIXEL_FORMAT_I420)); } MockMediaStreamVideoSource::~MockMediaStreamVideoSource() {} @@ -36,30 +39,53 @@ void MockMediaStreamVideoSource::FailToStartMockedSource() { } void MockMediaStreamVideoSource::CompleteGetSupportedFormats() { - OnSupportedFormats(supported_formats_); + DCHECK(!formats_callback_.is_null()); + base::ResetAndReturn(&formats_callback_).Run(supported_formats_); } void MockMediaStreamVideoSource::GetCurrentSupportedFormats( int max_requested_height, - int max_requested_width) { + int max_requested_width, + const VideoCaptureDeviceFormatsCB& callback) { + DCHECK(formats_callback_.is_null()); max_requested_height_ = max_requested_height; max_requested_width_ = max_requested_width; - if (!manual_get_supported_formats_) - OnSupportedFormats(supported_formats_); + if (manual_get_supported_formats_) { + formats_callback_ = callback; + return; + } + callback.Run(supported_formats_); } void MockMediaStreamVideoSource::StartSourceImpl( - const media::VideoCaptureParams& params) { + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) { + DCHECK(frame_callback_.is_null()); params_ = params; attempted_to_start_ = true; + frame_callback_ = frame_callback; } void MockMediaStreamVideoSource::StopSourceImpl() { } -} // namespace content - - +void MockMediaStreamVideoSource::DeliverVideoFrame( + const scoped_refptr<media::VideoFrame>& frame) { + DCHECK(!frame_callback_.is_null()); + io_message_loop()->PostTask( + FROM_HERE, + base::Bind(&MockMediaStreamVideoSource::DeliverVideoFrameOnIO, + base::Unretained(this), frame, params_.requested_format, + frame_callback_)); +} +void MockMediaStreamVideoSource::DeliverVideoFrameOnIO( + const scoped_refptr<media::VideoFrame>& frame, + media::VideoCaptureFormat format, + const VideoCaptureDeliverFrameCB& frame_callback) { + DCHECK(io_message_loop()->BelongsToCurrentThread()); + frame_callback.Run(frame, format); +} +} // namespace content diff --git a/content/renderer/media/mock_media_stream_video_source.h b/content/renderer/media/mock_media_stream_video_source.h index 33d3f70c..d0d82e6 100644 --- a/content/renderer/media/mock_media_stream_video_source.h +++ b/content/renderer/media/mock_media_stream_video_source.h @@ -9,10 +9,9 @@ namespace content { -class MockMediaStreamVideoSource - : public MediaStreamVideoSource { +class MockMediaStreamVideoSource : public MediaStreamVideoSource { public: - MockMediaStreamVideoSource(bool manual_get_supported_formats); + explicit MockMediaStreamVideoSource(bool manual_get_supported_formats); virtual ~MockMediaStreamVideoSource(); // Simulate that the underlying source start successfully. @@ -29,21 +28,30 @@ class MockMediaStreamVideoSource supported_formats_ = formats; } + // Delivers |frame| to all registered tracks on the IO thread. Its up to the + // call to make sure MockMediaStreamVideoSource is not destroyed before the + // frame has been delivered. + void DeliverVideoFrame(const scoped_refptr<media::VideoFrame>& frame); + void CompleteGetSupportedFormats(); const media::VideoCaptureParams& start_params() const { return params_; } int max_requested_height() const { return max_requested_height_; } int max_requested_width() const { return max_requested_width_; } - using MediaStreamVideoSource::DeliverVideoFrame; - protected: + void DeliverVideoFrameOnIO(const scoped_refptr<media::VideoFrame>& frame, + media::VideoCaptureFormat format, + const VideoCaptureDeliverFrameCB& frame_callback); + // Implements MediaStreamVideoSource. virtual void GetCurrentSupportedFormats( int max_requested_height, - int max_requested_width) OVERRIDE; + int max_requested_width, + const VideoCaptureDeviceFormatsCB& callback) OVERRIDE; virtual void StartSourceImpl( - const media::VideoCaptureParams& params) OVERRIDE; + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) OVERRIDE; virtual void StopSourceImpl() OVERRIDE; private: @@ -53,6 +61,10 @@ class MockMediaStreamVideoSource int max_requested_height_; int max_requested_width_; bool attempted_to_start_; + VideoCaptureDeviceFormatsCB formats_callback_; + VideoCaptureDeliverFrameCB frame_callback_; + + DISALLOW_COPY_AND_ASSIGN(MockMediaStreamVideoSource); }; } // namespace content diff --git a/content/renderer/media/rtc_peer_connection_handler_unittest.cc b/content/renderer/media/rtc_peer_connection_handler_unittest.cc index 1544020..93e5b23 100644 --- a/content/renderer/media/rtc_peer_connection_handler_unittest.cc +++ b/content/renderer/media/rtc_peer_connection_handler_unittest.cc @@ -6,8 +6,10 @@ #include <vector> #include "base/memory/scoped_ptr.h" +#include "base/message_loop/message_loop.h" #include "base/strings/utf_string_conversions.h" #include "base/values.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream.h" #include "content/renderer/media/media_stream_audio_source.h" #include "content/renderer/media/media_stream_source.h" @@ -198,6 +200,7 @@ class RTCPeerConnectionHandlerUnderTest : public RTCPeerConnectionHandler { class RTCPeerConnectionHandlerTest : public ::testing::Test { public: RTCPeerConnectionHandlerTest() : mock_peer_connection_(NULL) { + child_process_.reset(new ChildProcess()); } virtual void SetUp() { @@ -284,7 +287,7 @@ class RTCPeerConnectionHandlerTest : public ::testing::Test { return stream; } - base::MessageLoop message_loop_; + scoped_ptr<ChildProcess> child_process_; scoped_ptr<MockWebRTCPeerConnectionHandlerClient> mock_client_; scoped_ptr<MockMediaStreamDependencyFactory> mock_dependency_factory_; scoped_ptr<NiceMock<MockPeerConnectionTracker> > mock_tracker_; diff --git a/content/renderer/media/video_frame_deliverer.cc b/content/renderer/media/video_frame_deliverer.cc new file mode 100644 index 0000000..67b11c4 --- /dev/null +++ b/content/renderer/media/video_frame_deliverer.cc @@ -0,0 +1,68 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/renderer/media/video_frame_deliverer.h" + +#include "base/bind.h" +#include "base/location.h" + +namespace content { + +VideoFrameDeliverer::VideoFrameDeliverer( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop) + : io_message_loop_(io_message_loop) { + DCHECK(io_message_loop_); +} + +VideoFrameDeliverer::~VideoFrameDeliverer() { + DCHECK(callbacks_.empty()); +} + +void VideoFrameDeliverer::AddCallback( + void* id, + const VideoCaptureDeliverFrameCB& callback) { + DCHECK(thread_checker_.CalledOnValidThread()); + io_message_loop_->PostTask( + FROM_HERE, + base::Bind(&VideoFrameDeliverer::AddCallbackOnIO, + this, id, callback)); +} + +void VideoFrameDeliverer::AddCallbackOnIO( + void* id, + const VideoCaptureDeliverFrameCB& callback) { + DCHECK(io_message_loop_->BelongsToCurrentThread()); + callbacks_.push_back(std::make_pair(id, callback)); +} + +void VideoFrameDeliverer::RemoveCallback(void* id) { + DCHECK(thread_checker_.CalledOnValidThread()); + io_message_loop_->PostTask( + FROM_HERE, + base::Bind(&VideoFrameDeliverer::RemoveCallbackOnIO, + this, id)); +} + +void VideoFrameDeliverer::RemoveCallbackOnIO(void* id) { + DCHECK(io_message_loop_->BelongsToCurrentThread()); + std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin(); + for (; it != callbacks_.end(); ++it) { + if (it->first == id) { + callbacks_.erase(it); + return; + } + } +} + +void VideoFrameDeliverer::DeliverFrameOnIO( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) { + DCHECK(io_message_loop_->BelongsToCurrentThread()); + for (std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin(); + it != callbacks_.end(); ++it) { + it->second.Run(frame, format); + } +} + +} // namespace content diff --git a/content/renderer/media/video_frame_deliverer.h b/content/renderer/media/video_frame_deliverer.h new file mode 100644 index 0000000..f17c588 --- /dev/null +++ b/content/renderer/media/video_frame_deliverer.h @@ -0,0 +1,76 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_RENDERER_MEDIA_VIDEO_FRAME_DELIVERER_H_ +#define CONTENT_RENDERER_MEDIA_VIDEO_FRAME_DELIVERER_H_ + +#include <utility> +#include <vector> + +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/message_loop/message_loop_proxy.h" +#include "base/threading/thread_checker.h" +#include "content/common/media/video_capture.h" +#include "media/base/video_frame.h" + +namespace content { + +// VideoFrameDeliverer is a helper class used for registering +// VideoCaptureDeliverFrameCB on the main render thread to receive video frames +// on the IO-thread. +// Its used by MediaStreamVideoTrack and MediaStreamVideoSource. +class VideoFrameDeliverer + : public base::RefCountedThreadSafe<VideoFrameDeliverer> { + public: + explicit VideoFrameDeliverer( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop); + + // Add |callback| to receive video frames on the IO-thread. + // Must be called on the main render thread. + void AddCallback(void* id, const VideoCaptureDeliverFrameCB& callback); + // Removes |callback| associated with |id| from receiving video frames if |id| + // has been added. It is ok to call RemoveCallback even if the |id| has not + // been added. + // Must be called on the main render thread. + void RemoveCallback(void* id); + + // Triggers all registered callbacks with |frame| and |format| as parameters. + // Must be called on the IO-thread. + virtual void DeliverFrameOnIO( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format); + + const scoped_refptr<base::MessageLoopProxy>& io_message_loop() const { + return io_message_loop_; + } + + protected: + void AddCallbackOnIO(void* id, const VideoCaptureDeliverFrameCB& callback); + // It is ok to call RemoveCallback even if |id| has not been added. + void RemoveCallbackOnIO(void* id); + + protected: + virtual ~VideoFrameDeliverer(); + const base::ThreadChecker& thread_checker() const { + return thread_checker_; + } + + private: + friend class base::RefCountedThreadSafe<VideoFrameDeliverer>; + + // Used to DCHECK that AddCallback and RemoveCallback are called on the main + // render thread. + base::ThreadChecker thread_checker_; + scoped_refptr<base::MessageLoopProxy> io_message_loop_; + + typedef std::pair<void*, VideoCaptureDeliverFrameCB> VideoIdCallbackPair; + std::vector<VideoIdCallbackPair> callbacks_; + + DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliverer); +}; + +} // namespace content + +#endif // CONTENT_RENDERER_MEDIA_VIDEO_FRAME_DELIVERER_H_ diff --git a/content/renderer/media/video_source_handler_unittest.cc b/content/renderer/media/video_source_handler_unittest.cc index 59b803d..1d66ca0 100644 --- a/content/renderer/media/video_source_handler_unittest.cc +++ b/content/renderer/media/video_source_handler_unittest.cc @@ -5,6 +5,7 @@ #include <string> #include "base/strings/utf_string_conversions.h" +#include "content/child/child_process.h" #include "content/public/renderer/media_stream_video_sink.h" #include "content/renderer/media/media_stream.h" #include "content/renderer/media/media_stream_registry_interface.h" @@ -15,7 +16,6 @@ #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" #include "third_party/WebKit/public/platform/WebString.h" - namespace content { static const std::string kTestStreamUrl = "stream_url"; @@ -40,13 +40,16 @@ class FakeFrameReader : public FrameReaderInterface { class VideoSourceHandlerTest : public ::testing::Test { public: - VideoSourceHandlerTest() : registry_() { + VideoSourceHandlerTest() + : child_process_(new ChildProcess()), + registry_() { handler_.reset(new VideoSourceHandler(®istry_)); registry_.Init(kTestStreamUrl); registry_.AddVideoTrack(kTestVideoTrackId); } protected: + scoped_ptr<ChildProcess> child_process_; scoped_ptr<VideoSourceHandler> handler_; MockMediaStreamRegistry registry_; }; diff --git a/content/renderer/media/webrtc/media_stream_remote_video_source.cc b/content/renderer/media/webrtc/media_stream_remote_video_source.cc index 97c810a..d2984f5 100644 --- a/content/renderer/media/webrtc/media_stream_remote_video_source.cc +++ b/content/renderer/media/webrtc/media_stream_remote_video_source.cc @@ -5,59 +5,71 @@ #include "content/renderer/media/webrtc/media_stream_remote_video_source.h" #include "base/bind.h" +#include "base/callback_helpers.h" #include "base/location.h" #include "base/message_loop/message_loop_proxy.h" #include "content/renderer/media/native_handle_impl.h" +#include "media/base/bind_to_current_loop.h" #include "media/base/video_frame.h" +#include "media/base/video_frame_pool.h" #include "media/base/video_util.h" #include "third_party/libjingle/source/talk/media/base/videoframe.h" namespace content { -MediaStreamRemoteVideoSource::MediaStreamRemoteVideoSource( - webrtc::VideoTrackInterface* remote_track) - : MediaStreamVideoSource(), - message_loop_proxy_(base::MessageLoopProxy::current()), - remote_track_(remote_track), - last_state_(remote_track_->state()), - first_frame_received_(false) { - remote_track_->AddRenderer(this); - remote_track_->RegisterObserver(this); +// Internal class used for receiving frames from the webrtc track on a +// libjingle thread and forward it to the IO-thread. +class MediaStreamRemoteVideoSource::RemoteVideoSourceDelegate + : public base::RefCountedThreadSafe<RemoteVideoSourceDelegate>, + public webrtc::VideoRendererInterface { + public: + RemoteVideoSourceDelegate( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop, + const VideoCaptureDeliverFrameCB& new_frame_callback); + + protected: + friend class base::RefCountedThreadSafe<RemoteVideoSourceDelegate>; + virtual ~RemoteVideoSourceDelegate(); + + // Implements webrtc::VideoRendererInterface used for receiving video frames + // from the PeerConnection video track. May be called on a libjingle internal + // thread. + virtual void SetSize(int width, int height) OVERRIDE; + virtual void RenderFrame(const cricket::VideoFrame* frame) OVERRIDE; + + void DoRenderFrameOnIOThread(scoped_refptr<media::VideoFrame> video_frame, + const media::VideoCaptureFormat& format); + private: + // Bound to the render thread. + base::ThreadChecker thread_checker_; + + scoped_refptr<base::MessageLoopProxy> io_message_loop_; + // |frame_pool_| is only accessed on whatever + // thread webrtc::VideoRendererInterface::RenderFrame is called on. + media::VideoFramePool frame_pool_; + + // |frame_callback_| is accessed on the IO thread. + VideoCaptureDeliverFrameCB frame_callback_; +}; + +MediaStreamRemoteVideoSource:: +RemoteVideoSourceDelegate::RemoteVideoSourceDelegate( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop, + const VideoCaptureDeliverFrameCB& new_frame_callback) + : io_message_loop_(io_message_loop), + frame_callback_(new_frame_callback) { } -MediaStreamRemoteVideoSource::~MediaStreamRemoteVideoSource() { - StopSourceImpl(); +MediaStreamRemoteVideoSource:: +RemoteVideoSourceDelegate::~RemoteVideoSourceDelegate() { } -void MediaStreamRemoteVideoSource::GetCurrentSupportedFormats( - int max_requested_width, - int max_requested_height) { - DCHECK(message_loop_proxy_->BelongsToCurrentThread()); - if (format_.IsValid()) { - media::VideoCaptureFormats formats; - formats.push_back(format_); - OnSupportedFormats(formats); - } +void MediaStreamRemoteVideoSource:: +RemoteVideoSourceDelegate::SetSize(int width, int height) { } -void MediaStreamRemoteVideoSource::StartSourceImpl( - const media::VideoCaptureParams& params) { - DCHECK(message_loop_proxy_->BelongsToCurrentThread()); - OnStartDone(true); -} - -void MediaStreamRemoteVideoSource::StopSourceImpl() { - DCHECK(message_loop_proxy_->BelongsToCurrentThread()); - if (state() != MediaStreamVideoSource::ENDED) { - remote_track_->RemoveRenderer(this); - remote_track_->UnregisterObserver(this); - } -} - -void MediaStreamRemoteVideoSource::SetSize(int width, int height) { -} - -void MediaStreamRemoteVideoSource::RenderFrame( +void MediaStreamRemoteVideoSource:: +RemoteVideoSourceDelegate::RenderFrame( const cricket::VideoFrame* frame) { base::TimeDelta timestamp = base::TimeDelta::FromMilliseconds( frame->GetTimeStamp() / talk_base::kNumNanosecsPerMillisec); @@ -87,32 +99,75 @@ void MediaStreamRemoteVideoSource::RenderFrame( frame->GetVPlane(), frame->GetVPitch(), uv_rows, video_frame.get()); } - if (!first_frame_received_) { - first_frame_received_ = true; - media::VideoPixelFormat pixel_format = ( - video_frame->format() == media::VideoFrame::YV12) ? - media::PIXEL_FORMAT_YV12 : media::PIXEL_FORMAT_TEXTURE; - media::VideoCaptureFormat format( - gfx::Size(video_frame->natural_size().width(), - video_frame->natural_size().height()), - MediaStreamVideoSource::kDefaultFrameRate, - pixel_format); - message_loop_proxy_->PostTask( - FROM_HERE, - base::Bind(&MediaStreamRemoteVideoSource::FrameFormatOnMainThread, - AsWeakPtr(), format)); - } + media::VideoPixelFormat pixel_format = + (video_frame->format() == media::VideoFrame::YV12) ? + media::PIXEL_FORMAT_YV12 : media::PIXEL_FORMAT_TEXTURE; + + media::VideoCaptureFormat format( + gfx::Size(video_frame->natural_size().width(), + video_frame->natural_size().height()), + MediaStreamVideoSource::kDefaultFrameRate, + pixel_format); - // TODO(perkj): Consider delivering the frame on whatever thread the frame is - // delivered on once crbug/335327 is fixed. - message_loop_proxy_->PostTask( + io_message_loop_->PostTask( FROM_HERE, - base::Bind(&MediaStreamRemoteVideoSource::DoRenderFrameOnMainThread, - AsWeakPtr(), video_frame)); + base::Bind(&RemoteVideoSourceDelegate::DoRenderFrameOnIOThread, + this, video_frame, format)); +} + +void MediaStreamRemoteVideoSource:: +RemoteVideoSourceDelegate::DoRenderFrameOnIOThread( + scoped_refptr<media::VideoFrame> video_frame, + const media::VideoCaptureFormat& format) { + DCHECK(io_message_loop_->BelongsToCurrentThread()); + frame_callback_.Run(video_frame, format); +} + +MediaStreamRemoteVideoSource::MediaStreamRemoteVideoSource( + webrtc::VideoTrackInterface* remote_track) + : remote_track_(remote_track), + last_state_(remote_track->state()) { + remote_track_->RegisterObserver(this); +} + +MediaStreamRemoteVideoSource::~MediaStreamRemoteVideoSource() { + remote_track_->UnregisterObserver(this); +} + +void MediaStreamRemoteVideoSource::GetCurrentSupportedFormats( + int max_requested_width, + int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) { + DCHECK(thread_checker_.CalledOnValidThread()); + media::VideoCaptureFormats formats; + // Since the remote end is free to change the resolution at any point in time + // the supported formats are unknown. + callback.Run(formats); +} + +void MediaStreamRemoteVideoSource::StartSourceImpl( + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) { + DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(!delegate_); + delegate_ = new RemoteVideoSourceDelegate(io_message_loop(), frame_callback); + remote_track_->AddRenderer(delegate_); + OnStartDone(true); +} + +void MediaStreamRemoteVideoSource::StopSourceImpl() { + DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(state() != MediaStreamVideoSource::ENDED); + remote_track_->RemoveRenderer(delegate_); +} + +webrtc::VideoRendererInterface* +MediaStreamRemoteVideoSource::RenderInterfaceForTest() { + return delegate_; } void MediaStreamRemoteVideoSource::OnChanged() { - DCHECK(message_loop_proxy_->BelongsToCurrentThread()); + DCHECK(thread_checker_.CalledOnValidThread()); webrtc::MediaStreamTrackInterface::TrackState state = remote_track_->state(); if (state != last_state_) { last_state_ = state; @@ -134,22 +189,4 @@ void MediaStreamRemoteVideoSource::OnChanged() { } } -void MediaStreamRemoteVideoSource::FrameFormatOnMainThread( - const media::VideoCaptureFormat& format) { - DCHECK(message_loop_proxy_->BelongsToCurrentThread()); - format_ = format; - if (state() == RETRIEVING_CAPABILITIES) { - media::VideoCaptureFormats formats; - formats.push_back(format); - OnSupportedFormats(formats); - } -} - -void MediaStreamRemoteVideoSource::DoRenderFrameOnMainThread( - scoped_refptr<media::VideoFrame> video_frame) { - DCHECK(message_loop_proxy_->BelongsToCurrentThread()); - if (state() == STARTED) - DeliverVideoFrame(video_frame, format_); -} - } // namespace content diff --git a/content/renderer/media/webrtc/media_stream_remote_video_source.h b/content/renderer/media/webrtc/media_stream_remote_video_source.h index a409271..e7d65dd 100644 --- a/content/renderer/media/webrtc/media_stream_remote_video_source.h +++ b/content/renderer/media/webrtc/media_stream_remote_video_source.h @@ -5,10 +5,8 @@ #ifndef CONTENT_RENDERER_MEDIA_WEBRTC_MEDIA_STREAM_REMOTE_VIDEO_SOURCE_H_ #define CONTENT_RENDERER_MEDIA_WEBRTC_MEDIA_STREAM_REMOTE_VIDEO_SOURCE_H_ -#include "base/memory/weak_ptr.h" #include "content/common/content_export.h" #include "content/renderer/media/media_stream_video_source.h" -#include "media/base/video_frame_pool.h" #include "third_party/WebKit/public/platform/WebMediaStreamSource.h" #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h" @@ -20,9 +18,7 @@ namespace content { // a local source and a video track where the source is a remote video track. class CONTENT_EXPORT MediaStreamRemoteVideoSource : public MediaStreamVideoSource, - NON_EXPORTED_BASE(public webrtc::VideoRendererInterface), - NON_EXPORTED_BASE(public webrtc::ObserverInterface), - public base::SupportsWeakPtr<MediaStreamRemoteVideoSource> { + NON_EXPORTED_BASE(public webrtc::ObserverInterface) { public: explicit MediaStreamRemoteVideoSource( webrtc::VideoTrackInterface* remote_track); @@ -32,37 +28,33 @@ class CONTENT_EXPORT MediaStreamRemoteVideoSource // Implements MediaStreamVideoSource. virtual void GetCurrentSupportedFormats( int max_requested_width, - int max_requested_height) OVERRIDE; + int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) OVERRIDE; virtual void StartSourceImpl( - const media::VideoCaptureParams& params) OVERRIDE; + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) OVERRIDE; virtual void StopSourceImpl() OVERRIDE; - // Implements webrtc::VideoRendererInterface used for receiving video frames - // from the PeerConnection video track. May be called on - // a different thread. - virtual void SetSize(int width, int height) OVERRIDE; - virtual void RenderFrame(const cricket::VideoFrame* frame) OVERRIDE; + // Used by tests to test that a frame can be received and that the + // MediaStreamRemoteVideoSource behaves as expected. + webrtc::VideoRendererInterface* RenderInterfaceForTest(); + private: // webrtc::ObserverInterface implementation. virtual void OnChanged() OVERRIDE; - private: - void FrameFormatOnMainThread(const media::VideoCaptureFormat& format); - void DoRenderFrameOnMainThread(scoped_refptr<media::VideoFrame> video_frame); - - scoped_refptr<base::MessageLoopProxy> message_loop_proxy_; scoped_refptr<webrtc::VideoTrackInterface> remote_track_; webrtc::MediaStreamTrackInterface::TrackState last_state_; - // |first_frame_received_| and |frame_pool_| are only accessed on whatever - // thread webrtc::VideoRendererInterface::RenderFrame is called on. - bool first_frame_received_; - media::VideoFramePool frame_pool_; + // Internal class used for receiving frames from the webrtc track on a + // libjingle thread and forward it to the IO-thread. + class RemoteVideoSourceDelegate; + scoped_refptr<RemoteVideoSourceDelegate> delegate_; - // |format_| is the format currently received by this source. - media::VideoCaptureFormat format_; + // Bound to the render thread. + base::ThreadChecker thread_checker_; DISALLOW_COPY_AND_ASSIGN(MediaStreamRemoteVideoSource); }; diff --git a/content/renderer/media/webrtc/media_stream_remote_video_source_unittest.cc b/content/renderer/media/webrtc/media_stream_remote_video_source_unittest.cc index 0b6b327..16c3687 100644 --- a/content/renderer/media/webrtc/media_stream_remote_video_source_unittest.cc +++ b/content/renderer/media/webrtc/media_stream_remote_video_source_unittest.cc @@ -2,10 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" #include "base/strings/utf_string_conversions.h" -#include "content/public/renderer/media_stream_video_sink.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream_video_track.h" #include "content/renderer/media/mock_media_stream_dependency_factory.h" +#include "content/renderer/media/mock_media_stream_video_sink.h" #include "content/renderer/media/webrtc/media_stream_remote_video_source.h" #include "media/base/video_frame.h" #include "testing/gtest/include/gtest/gtest.h" @@ -13,30 +16,9 @@ namespace content { -class MockVideoSink : public MediaStreamVideoSink { - public: - MockVideoSink() - : number_of_frames_(0), - state_(blink::WebMediaStreamSource::ReadyStateLive) { - } - - virtual void OnVideoFrame( - const scoped_refptr<media::VideoFrame>& frame) OVERRIDE { - ++number_of_frames_; - } - - virtual void OnReadyStateChanged( - blink::WebMediaStreamSource::ReadyState state) OVERRIDE { - state_ = state; - } - - int number_of_frames() const { return number_of_frames_; } - blink::WebMediaStreamSource::ReadyState state() const { return state_; } - - private: - int number_of_frames_; - blink::WebMediaStreamSource::ReadyState state_; -}; +ACTION_P(RunClosure, closure) { + closure.Run(); +} class MediaStreamRemoteVideoSourceUnderTest : public MediaStreamRemoteVideoSource { @@ -44,14 +26,15 @@ class MediaStreamRemoteVideoSourceUnderTest MediaStreamRemoteVideoSourceUnderTest(webrtc::VideoTrackInterface* track) : MediaStreamRemoteVideoSource(track) { } - using MediaStreamRemoteVideoSource::RenderFrame; + using MediaStreamRemoteVideoSource::RenderInterfaceForTest; }; class MediaStreamRemoteVideoSourceTest : public ::testing::Test { public: MediaStreamRemoteVideoSourceTest() - : mock_factory_(new MockMediaStreamDependencyFactory()), + : child_process_(new ChildProcess()), + mock_factory_(new MockMediaStreamDependencyFactory()), webrtc_video_track_( mock_factory_->CreateLocalVideoTrack( "test", @@ -65,7 +48,6 @@ class MediaStreamRemoteVideoSourceTest base::UTF8ToUTF16("dummy_source_name")); webkit_source_.setExtraData(remote_source_); } - virtual ~MediaStreamRemoteVideoSourceTest() {} MediaStreamRemoteVideoSourceUnderTest* source() { return remote_source_; @@ -97,8 +79,6 @@ class MediaStreamRemoteVideoSourceTest webrtc::MediaStreamTrackInterface::kEnded); } - base::MessageLoop* message_loop() { return &message_loop_; } - const blink::WebMediaStreamSource& webkit_source() const { return webkit_source_; } @@ -112,7 +92,8 @@ class MediaStreamRemoteVideoSourceTest ++number_of_failed_constraints_applied_; } - base::MessageLoop message_loop_; + scoped_ptr<ChildProcess> child_process_; + base::MessageLoopForUI message_loop_; scoped_ptr<MockMediaStreamDependencyFactory> mock_factory_; scoped_refptr<webrtc::VideoTrackInterface> webrtc_video_track_; // |remote_source_| is owned by |webkit_source_|. @@ -124,17 +105,22 @@ class MediaStreamRemoteVideoSourceTest TEST_F(MediaStreamRemoteVideoSourceTest, StartTrack) { scoped_ptr<MediaStreamVideoTrack> track(CreateTrack()); - EXPECT_EQ(0, NumberOfSuccessConstraintsCallbacks()); + EXPECT_EQ(1, NumberOfSuccessConstraintsCallbacks()); - MockVideoSink sink; + MockMediaStreamVideoSink sink; track->AddSink(&sink); + + + base::RunLoop run_loop; + base::Closure quit_closure = run_loop.QuitClosure(); + EXPECT_CALL(sink, OnVideoFrame()).WillOnce( + RunClosure(quit_closure)); cricket::WebRtcVideoFrame webrtc_frame; webrtc_frame.InitToBlack(320, 240, 1, 1, 0, 1); - source()->RenderFrame(&webrtc_frame); - message_loop()->RunUntilIdle(); + source()->RenderInterfaceForTest()->RenderFrame(&webrtc_frame); + run_loop.Run(); - EXPECT_EQ(1, NumberOfSuccessConstraintsCallbacks()); EXPECT_EQ(1, sink.number_of_frames()); track->RemoveSink(&sink); } @@ -142,7 +128,7 @@ TEST_F(MediaStreamRemoteVideoSourceTest, StartTrack) { TEST_F(MediaStreamRemoteVideoSourceTest, RemoteTrackStop) { scoped_ptr<MediaStreamVideoTrack> track(CreateTrack()); - MockVideoSink sink; + MockMediaStreamVideoSink sink; track->AddSink(&sink); EXPECT_EQ(blink::WebMediaStreamSource::ReadyStateLive, sink.state()); diff --git a/content/renderer/media/webrtc/video_destination_handler.cc b/content/renderer/media/webrtc/video_destination_handler.cc index 3335293..c8eff87 100644 --- a/content/renderer/media/webrtc/video_destination_handler.cc +++ b/content/renderer/media/webrtc/video_destination_handler.cc @@ -24,8 +24,53 @@ namespace content { -PpFrameWriter::PpFrameWriter() - : MediaStreamVideoSource(), first_frame_received_(false) { +class PpFrameWriter::FrameWriterDelegate + : public base::RefCountedThreadSafe<FrameWriterDelegate> { + public: + FrameWriterDelegate( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy, + const VideoCaptureDeliverFrameCB& new_frame_callback); + + void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format); + private: + friend class base::RefCountedThreadSafe<FrameWriterDelegate>; + virtual ~FrameWriterDelegate(); + + void DeliverFrameOnIO(const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format); + + scoped_refptr<base::MessageLoopProxy> io_message_loop_; + VideoCaptureDeliverFrameCB new_frame_callback_; +}; + +PpFrameWriter::FrameWriterDelegate::FrameWriterDelegate( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy, + const VideoCaptureDeliverFrameCB& new_frame_callback) + : io_message_loop_(io_message_loop_proxy), + new_frame_callback_(new_frame_callback) { +} + +PpFrameWriter::FrameWriterDelegate::~FrameWriterDelegate() { +} + +void PpFrameWriter::FrameWriterDelegate::DeliverFrame( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) { + io_message_loop_->PostTask( + FROM_HERE, + base::Bind(&FrameWriterDelegate::DeliverFrameOnIO, + this, frame, format)); +} + +void PpFrameWriter::FrameWriterDelegate::DeliverFrameOnIO( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) { + DCHECK(io_message_loop_->BelongsToCurrentThread()); + new_frame_callback_.Run(frame, format); +} + +PpFrameWriter::PpFrameWriter() { DVLOG(3) << "PpFrameWriter ctor"; } @@ -33,21 +78,25 @@ PpFrameWriter::~PpFrameWriter() { DVLOG(3) << "PpFrameWriter dtor"; } -void PpFrameWriter::GetCurrentSupportedFormats(int max_requested_width, - int max_requested_height) { +void PpFrameWriter::GetCurrentSupportedFormats( + int max_requested_width, + int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) { DCHECK(CalledOnValidThread()); DVLOG(3) << "PpFrameWriter::GetCurrentSupportedFormats()"; - if (format_.IsValid()) { - media::VideoCaptureFormats formats; - formats.push_back(format_); - OnSupportedFormats(formats); - } + // Since the input is free to change the resolution at any point in time + // the supported formats are unknown. + media::VideoCaptureFormats formats; + callback.Run(formats); } void PpFrameWriter::StartSourceImpl( - const media::VideoCaptureParams& params) { + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) { DCHECK(CalledOnValidThread()); + DCHECK(!delegate_); DVLOG(3) << "PpFrameWriter::StartSourceImpl()"; + delegate_ = new FrameWriterDelegate(io_message_loop(), frame_callback); OnStartDone(true); } @@ -79,19 +128,6 @@ void PpFrameWriter::PutFrame(PPB_ImageData_Impl* image_data, const gfx::Size frame_size(bitmap->width(), bitmap->height()); - if (!first_frame_received_) { - first_frame_received_ = true; - format_ = media::VideoCaptureFormat( - frame_size, - MediaStreamVideoSource::kDefaultFrameRate, - media::PIXEL_FORMAT_YV12); - if (state() == MediaStreamVideoSource::RETRIEVING_CAPABILITIES) { - media::VideoCaptureFormats formats; - formats.push_back(format_); - OnSupportedFormats(formats); - } - } - if (state() != MediaStreamVideoSource::STARTED) return; @@ -105,6 +141,10 @@ void PpFrameWriter::PutFrame(PPB_ImageData_Impl* image_data, scoped_refptr<media::VideoFrame> new_frame = frame_pool_.CreateFrame(media::VideoFrame::YV12, frame_size, gfx::Rect(frame_size), frame_size, timestamp); + media::VideoCaptureFormat format( + frame_size, + MediaStreamVideoSource::kDefaultFrameRate, + media::PIXEL_FORMAT_YV12); libyuv::BGRAToI420(reinterpret_cast<uint8*>(bitmap->getPixels()), bitmap->rowBytes(), @@ -116,7 +156,7 @@ void PpFrameWriter::PutFrame(PPB_ImageData_Impl* image_data, new_frame->stride(media::VideoFrame::kVPlane), frame_size.width(), frame_size.height()); - DeliverVideoFrame(new_frame, format_); + delegate_->DeliverFrame(new_frame, format); } // PpFrameWriterProxy is a helper class to make sure the user won't use @@ -167,6 +207,7 @@ bool VideoDestinationHandler::Open( // theoretically it's possible we can get an id that's duplicated with the // existing sources. base::Base64Encode(base::RandBytesAsString(64), &track_id); + PpFrameWriter* writer = new PpFrameWriter(); // Create a new webkit video track. diff --git a/content/renderer/media/webrtc/video_destination_handler.h b/content/renderer/media/webrtc/video_destination_handler.h index b13d26d..0b6641c 100644 --- a/content/renderer/media/webrtc/video_destination_handler.h +++ b/content/renderer/media/webrtc/video_destination_handler.h @@ -49,18 +49,21 @@ class CONTENT_EXPORT PpFrameWriter int64 time_stamp_ns) OVERRIDE; protected: // MediaStreamVideoSource implementation. - virtual void GetCurrentSupportedFormats(int max_requested_width, - int max_requested_height) OVERRIDE; + virtual void GetCurrentSupportedFormats( + int max_requested_width, + int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) OVERRIDE; virtual void StartSourceImpl( - const media::VideoCaptureParams& params) OVERRIDE; + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) OVERRIDE; virtual void StopSourceImpl() OVERRIDE; private: - // |format_| is the format currently received by this source. - media::VideoCaptureFormat format_; - bool first_frame_received_; media::VideoFramePool frame_pool_; + class FrameWriterDelegate; + scoped_refptr<FrameWriterDelegate> delegate_; + DISALLOW_COPY_AND_ASSIGN(PpFrameWriter); }; diff --git a/content/renderer/media/webrtc/video_destination_handler_unittest.cc b/content/renderer/media/webrtc/video_destination_handler_unittest.cc index fb3d863..062c277 100644 --- a/content/renderer/media/webrtc/video_destination_handler_unittest.cc +++ b/content/renderer/media/webrtc/video_destination_handler_unittest.cc @@ -4,7 +4,10 @@ #include <string> +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" #include "base/strings/utf_string_conversions.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream.h" #include "content/renderer/media/media_stream_video_track.h" #include "content/renderer/media/mock_media_stream_registry.h" @@ -13,22 +16,36 @@ #include "content/renderer/pepper/pepper_plugin_instance_impl.h" #include "content/renderer/pepper/ppb_image_data_impl.h" #include "content/test/ppapi_unittest.h" +#include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" #include "third_party/WebKit/public/platform/WebString.h" +using ::testing::_; + namespace content { +ACTION_P(RunClosure, closure) { + closure.Run(); +} + static const std::string kTestStreamUrl = "stream_url"; static const std::string kUnknownStreamUrl = "unknown_stream_url"; class VideoDestinationHandlerTest : public PpapiUnittest { public: - VideoDestinationHandlerTest() : registry_() { + VideoDestinationHandlerTest() + : child_process_(new ChildProcess()), + registry_(MockMediaStreamRegistry()) { registry_.Init(kTestStreamUrl); } + base::MessageLoop* io_message_loop() const { + return child_process_->io_message_loop(); + } + protected: + scoped_ptr<ChildProcess> child_process_; MockMediaStreamRegistry registry_; }; @@ -39,7 +56,7 @@ TEST_F(VideoDestinationHandlerTest, Open) { kUnknownStreamUrl, &frame_writer)); EXPECT_TRUE(VideoDestinationHandler::Open(®istry_, kTestStreamUrl, &frame_writer)); - // The |frame_writer| is a proxy and is owned by who call Open. + // The |frame_writer| is a proxy and is owned by whoever call Open. delete frame_writer; } @@ -67,14 +84,21 @@ TEST_F(VideoDestinationHandlerTest, PutFrame) { new PPB_ImageData_Impl(instance()->pp_instance(), PPB_ImageData_Impl::ForTest())); image->Init(PP_IMAGEDATAFORMAT_BGRA_PREMUL, 640, 360, true); - frame_writer->PutFrame(image, 10); - EXPECT_EQ(1, sink.number_of_frames()); + { + base::RunLoop run_loop; + base::Closure quit_closure = run_loop.QuitClosure(); + + EXPECT_CALL(sink, OnVideoFrame()).WillOnce( + RunClosure(quit_closure)); + frame_writer->PutFrame(image, 10); + run_loop.Run(); + } // TODO(perkj): Verify that the track output I420 when // https://codereview.chromium.org/213423006/ is landed. - + EXPECT_EQ(1, sink.number_of_frames()); native_track->RemoveSink(&sink); - // The |frame_writer| is a proxy and is owned by who call Open. + // The |frame_writer| is a proxy and is owned by whoever call Open. delete frame_writer; } diff --git a/content/renderer/media/webrtc/webrtc_media_stream_adapter_unittest.cc b/content/renderer/media/webrtc/webrtc_media_stream_adapter_unittest.cc index 8cbceff..5a5b9bf 100644 --- a/content/renderer/media/webrtc/webrtc_media_stream_adapter_unittest.cc +++ b/content/renderer/media/webrtc/webrtc_media_stream_adapter_unittest.cc @@ -3,6 +3,7 @@ // found in the LICENSE file. #include "base/memory/scoped_ptr.h" +#include "content/child/child_process.h" #include "content/renderer/media/media_stream.h" #include "content/renderer/media/media_stream_audio_source.h" #include "content/renderer/media/media_stream_video_source.h" @@ -22,6 +23,7 @@ namespace content { class WebRtcMediaStreamAdapterTest : public ::testing::Test { public: virtual void SetUp() { + child_process_.reset(new ChildProcess()); dependency_factory_.reset(new MockMediaStreamDependencyFactory()); } @@ -89,6 +91,7 @@ class WebRtcMediaStreamAdapterTest : public ::testing::Test { } protected: + scoped_ptr<ChildProcess> child_process_; scoped_ptr<MockMediaStreamDependencyFactory> dependency_factory_; scoped_ptr<WebRtcMediaStreamAdapter> adapter_; }; diff --git a/content/renderer/media/webrtc/webrtc_video_track_adapter.cc b/content/renderer/media/webrtc/webrtc_video_track_adapter.cc index 023aba1..0fdef64 100644 --- a/content/renderer/media/webrtc/webrtc_video_track_adapter.cc +++ b/content/renderer/media/webrtc/webrtc_video_track_adapter.cc @@ -22,27 +22,77 @@ bool ConstraintKeyExists(const blink::WebMediaConstraints& constraints, namespace content { +// Simple help class used for receiving video frames on the IO-thread from +// a MediaStreamVideoTrack and forward the frames to a +// WebRtcVideoCapturerAdapter that implements a video capturer for libjingle. +class WebRtcVideoTrackAdapter::WebRtcVideoSourceAdapter + : public base::RefCountedThreadSafe<WebRtcVideoSourceAdapter> { + public: + WebRtcVideoSourceAdapter( + const scoped_refptr<webrtc::VideoSourceInterface>& source, + WebRtcVideoCapturerAdapter* capture_adapter); + + void OnVideoFrameOnIO(const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format); + + private: + friend class base::RefCountedThreadSafe<WebRtcVideoSourceAdapter>; + virtual ~WebRtcVideoSourceAdapter(); + + // Used to DCHECK that frames are called on the IO-thread. + base::ThreadChecker io_thread_checker_; + scoped_refptr<webrtc::VideoSourceInterface> video_source_; + // |capture_adapter_| is owned by |video_source_| + WebRtcVideoCapturerAdapter* capture_adapter_; +}; + +WebRtcVideoTrackAdapter::WebRtcVideoSourceAdapter::WebRtcVideoSourceAdapter( + const scoped_refptr<webrtc::VideoSourceInterface>& source, + WebRtcVideoCapturerAdapter* capture_adapter) + : video_source_(source), + capture_adapter_(capture_adapter) { + io_thread_checker_.DetachFromThread(); +} + +WebRtcVideoTrackAdapter::WebRtcVideoSourceAdapter::~WebRtcVideoSourceAdapter() { +} + +void WebRtcVideoTrackAdapter::WebRtcVideoSourceAdapter::OnVideoFrameOnIO( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) { + DCHECK(io_thread_checker_.CalledOnValidThread()); + capture_adapter_->OnFrameCaptured(frame); +} + WebRtcVideoTrackAdapter::WebRtcVideoTrackAdapter( const blink::WebMediaStreamTrack& track, MediaStreamDependencyFactory* factory) : web_track_(track) { - MediaStreamVideoSink::AddToVideoTrack(this, web_track_); - const blink::WebMediaConstraints& constraints = MediaStreamVideoTrack::GetVideoTrack(track)->constraints(); bool is_screencast = ConstraintKeyExists( constraints, base::UTF8ToUTF16(kMediaStreamSource)); - capture_adapter_ = factory->CreateVideoCapturer(is_screencast); + WebRtcVideoCapturerAdapter* capture_adapter = + factory->CreateVideoCapturer(is_screencast); // |video_source| owns |capture_adapter| - video_source_ = factory->CreateVideoSource(capture_adapter_, - track.source().constraints()); + scoped_refptr<webrtc::VideoSourceInterface> video_source( + factory->CreateVideoSource(capture_adapter, + track.source().constraints())); video_track_ = factory->CreateLocalVideoTrack(web_track_.id().utf8(), - video_source_); + video_source.get()); + video_track_->set_enabled(web_track_.isEnabled()); + source_adapter_ = new WebRtcVideoSourceAdapter(video_source, + capture_adapter); + + MediaStreamVideoTrack::GetVideoTrack(track)->AddSink( + this, base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnIO, + source_adapter_)); + DVLOG(3) << "WebRtcVideoTrackAdapter ctor() : is_screencast " << is_screencast; } @@ -50,7 +100,7 @@ WebRtcVideoTrackAdapter::WebRtcVideoTrackAdapter( WebRtcVideoTrackAdapter::~WebRtcVideoTrackAdapter() { DCHECK(thread_checker_.CalledOnValidThread()); DVLOG(3) << "WebRtcVideoTrackAdapter dtor()."; - MediaStreamVideoSink::RemoveFromVideoTrack(this, web_track_); + MediaStreamVideoTrack::GetVideoTrack(web_track_)->RemoveSink(this); } void WebRtcVideoTrackAdapter::OnEnabledChanged(bool enabled) { @@ -58,11 +108,5 @@ void WebRtcVideoTrackAdapter::OnEnabledChanged(bool enabled) { video_track_->set_enabled(enabled); } -void WebRtcVideoTrackAdapter::OnVideoFrame( - const scoped_refptr<media::VideoFrame>& frame) { - DCHECK(thread_checker_.CalledOnValidThread()); - capture_adapter_->OnFrameCaptured(frame); -} - } // namespace content diff --git a/content/renderer/media/webrtc/webrtc_video_track_adapter.h b/content/renderer/media/webrtc/webrtc_video_track_adapter.h index b2cff0a..6b649ba 100644 --- a/content/renderer/media/webrtc/webrtc_video_track_adapter.h +++ b/content/renderer/media/webrtc/webrtc_video_track_adapter.h @@ -7,7 +7,7 @@ #include "base/macros.h" #include "base/threading/thread_checker.h" -#include "content/public/renderer/media_stream_video_sink.h" +#include "content/public/renderer/media_stream_sink.h" #include "content/renderer/media/media_stream_dependency_factory.h" #include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h" #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" @@ -16,6 +16,8 @@ namespace content { +class MediaStreamVideoTrack; + // WebRtcVideoTrackAdapter is an adapter between a // content::MediaStreamVideoTrack object and a webrtc VideoTrack that is // currently sent on a PeerConnection. @@ -25,7 +27,7 @@ namespace content { // added to an RTCPeerConnection object. // Instances of this class is owned by the WebRtcMediaStreamAdapter object that // created it. -class WebRtcVideoTrackAdapter : public MediaStreamVideoSink { +class WebRtcVideoTrackAdapter : public MediaStreamSink { public: WebRtcVideoTrackAdapter(const blink::WebMediaStreamTrack& track, MediaStreamDependencyFactory* factory); @@ -36,24 +38,20 @@ class WebRtcVideoTrackAdapter : public MediaStreamVideoSink { } protected: - // Implements MediaStreamVideoSink - virtual void OnVideoFrame( - const scoped_refptr<media::VideoFrame>& frame) OVERRIDE; + // Implementation of MediaStreamSink. virtual void OnEnabledChanged(bool enabled) OVERRIDE; private: // Used to DCHECK that we are called on the correct thread. base::ThreadChecker thread_checker_; - scoped_refptr<webrtc::VideoSourceInterface> video_source_; scoped_refptr<webrtc::VideoTrackInterface> video_track_; - blink::WebMediaStreamTrack web_track_; - // |capture_adapter_| is owned by |video_source_| - WebRtcVideoCapturerAdapter* capture_adapter_; + class WebRtcVideoSourceAdapter; + scoped_refptr<WebRtcVideoSourceAdapter> source_adapter_; - DISALLOW_COPY_AND_ASSIGN (WebRtcVideoTrackAdapter); + DISALLOW_COPY_AND_ASSIGN(WebRtcVideoTrackAdapter); }; } // namespace content diff --git a/content/renderer/pepper/pepper_media_stream_video_track_host.cc b/content/renderer/pepper/pepper_media_stream_video_track_host.cc index 854ed90..7d98e71 100644 --- a/content/renderer/pepper/pepper_media_stream_video_track_host.cc +++ b/content/renderer/pepper/pepper_media_stream_video_track_host.cc @@ -176,6 +176,57 @@ void ConvertFromMediaVideoFrame(const scoped_refptr<media::VideoFrame>& src, namespace content { +// Internal class used for delivering video frames on the IO-thread to +// the MediaStreamVideoSource implementation. +class PepperMediaStreamVideoTrackHost::FrameDeliverer + : public base::RefCountedThreadSafe<FrameDeliverer> { + public: + FrameDeliverer( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy, + const VideoCaptureDeliverFrameCB& new_frame_callback); + + void DeliverVideoFrame(const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format); + + private: + friend class base::RefCountedThreadSafe<FrameDeliverer>; + virtual ~FrameDeliverer(); + + void DeliverFrameOnIO(const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format); + + scoped_refptr<base::MessageLoopProxy> io_message_loop_; + VideoCaptureDeliverFrameCB new_frame_callback_; + + DISALLOW_COPY_AND_ASSIGN(FrameDeliverer); +}; + +PepperMediaStreamVideoTrackHost::FrameDeliverer::FrameDeliverer( + const scoped_refptr<base::MessageLoopProxy>& io_message_loop_proxy, + const VideoCaptureDeliverFrameCB& new_frame_callback) + : io_message_loop_(io_message_loop_proxy), + new_frame_callback_(new_frame_callback) { +} + +PepperMediaStreamVideoTrackHost::FrameDeliverer::~FrameDeliverer() { +} + +void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverVideoFrame( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) { + io_message_loop_->PostTask( + FROM_HERE, + base::Bind(&FrameDeliverer::DeliverFrameOnIO, + this, frame, format)); +} + +void PepperMediaStreamVideoTrackHost::FrameDeliverer::DeliverFrameOnIO( + const scoped_refptr<media::VideoFrame>& frame, + const media::VideoCaptureFormat& format) { + DCHECK(io_message_loop_->BelongsToCurrentThread()); + new_frame_callback_.Run(frame, format); +} + PepperMediaStreamVideoTrackHost::PepperMediaStreamVideoTrackHost( RendererPpapiHost* host, PP_Instance instance, @@ -312,10 +363,11 @@ int32_t PepperMediaStreamVideoTrackHost::SendFrameToTrack(int32_t index) { base::TimeDelta::FromMilliseconds(ts_ms), base::Closure()); - DeliverVideoFrame(frame, media::VideoCaptureFormat( - plugin_frame_size_, - kDefaultOutputFrameRate, - ToPixelFormat(plugin_frame_format_))); + frame_deliverer_->DeliverVideoFrame( + frame, + media::VideoCaptureFormat(plugin_frame_size_, + kDefaultOutputFrameRate, + ToPixelFormat(plugin_frame_format_))); } // Makes the frame available again for plugin. @@ -365,9 +417,11 @@ void PepperMediaStreamVideoTrackHost::OnVideoFrame( } void PepperMediaStreamVideoTrackHost::GetCurrentSupportedFormats( - int max_requested_width, int max_requested_height) { + int max_requested_width, int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) { if (type_ != kWrite) { DVLOG(1) << "GetCurrentSupportedFormats is only supported in output mode."; + callback.Run(media::VideoCaptureFormats()); return; } @@ -376,16 +430,19 @@ void PepperMediaStreamVideoTrackHost::GetCurrentSupportedFormats( media::VideoCaptureFormat(plugin_frame_size_, kDefaultOutputFrameRate, ToPixelFormat(plugin_frame_format_))); - OnSupportedFormats(formats); + callback.Run(formats); } void PepperMediaStreamVideoTrackHost::StartSourceImpl( - const media::VideoCaptureParams& params) { + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) { output_started_ = true; + frame_deliverer_ = new FrameDeliverer(io_message_loop(), frame_callback); } void PepperMediaStreamVideoTrackHost::StopSourceImpl() { output_started_ = false; + frame_deliverer_ = NULL; } void PepperMediaStreamVideoTrackHost::DidConnectPendingHostToResource() { diff --git a/content/renderer/pepper/pepper_media_stream_video_track_host.h b/content/renderer/pepper/pepper_media_stream_video_track_host.h index 0b8218d..3658706 100644 --- a/content/renderer/pepper/pepper_media_stream_video_track_host.h +++ b/content/renderer/pepper/pepper_media_stream_video_track_host.h @@ -59,10 +59,14 @@ class PepperMediaStreamVideoTrackHost : public PepperMediaStreamTrackHostBase, OVERRIDE; // MediaStreamVideoSource overrides: - virtual void GetCurrentSupportedFormats(int max_requested_width, - int max_requested_height) OVERRIDE; + virtual void GetCurrentSupportedFormats( + int max_requested_width, + int max_requested_height, + const VideoCaptureDeviceFormatsCB& callback) OVERRIDE; + virtual void StartSourceImpl( - const media::VideoCaptureParams& params) OVERRIDE; + const media::VideoCaptureParams& params, + const VideoCaptureDeliverFrameCB& frame_callback) OVERRIDE; virtual void StopSourceImpl() OVERRIDE; @@ -110,6 +114,11 @@ class PepperMediaStreamVideoTrackHost : public PepperMediaStreamTrackHostBase, TrackType type_; bool output_started_; + // Internal class used for delivering video frames on the IO-thread to + // the MediaStreamVideoSource implementation. + class FrameDeliverer; + scoped_refptr<FrameDeliverer> frame_deliverer_; + DISALLOW_COPY_AND_ASSIGN(PepperMediaStreamVideoTrackHost); }; |