Change MediaStreamVideoTrack and MediaStreamVideoSource to be able to receive frames on the IO-thread.

Add MediaStreamVideoTrack::AddSink(MediaStreamSink* sink, const VideoFrameCB& callback) where callback is bound to the IO-thread.

The purpose of the cl is to avoid posting frames to the main renderthread unless its needed.
Note that MediaStreamVideoSink still receive its frames on the main render thread - if a sink want to receive frames on the IO thread, it should use the new MediaStreamVideoTrack::AddSink(MediaStreamSink* sink, const VideoFrameCB& callback) method.

TEST= test on apprtc.appspot.com between two tabs.
BUG=335327
R=dmichael@chromium.org, hclam@chromium.org, ronghuawu@chromium.org, tommi@chromium.org

Review URL: https://codereview.chromium.org/252393006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@269020 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 143 insertions, 40 deletions

diff --git a/content/renderer/media/media_stream_video_source.cc b/content/renderer/media/media_stream_video_source.cc
index 491f6f8..2db0ef0 100644
--- a/content/renderer/media/media_stream_video_source.cc
+++ b/content/renderer/media/media_stream_video_source.cc
@@ -8,10 +8,13 @@
 #include <limits>
 #include <string>
 
+#include "base/debug/trace_event.h"
 #include "base/logging.h"
 #include "base/strings/string_number_conversions.h"
+#include "content/child/child_process.h"
 #include "content/renderer/media/media_stream_dependency_factory.h"
 #include "content/renderer/media/media_stream_video_track.h"
+#include "content/renderer/media/video_frame_deliverer.h"
 #include "content/renderer/media/webrtc/webrtc_video_capturer_adapter.h"
 
 namespace content {
@@ -215,6 +218,13 @@ bool GetConstraintValue(const blink::WebMediaConstraints& constraints,
   return ret;
 }
 
+// Returns true if |constraint| has mandatory constraints.
+bool HasMandatoryConstraints(const blink::WebMediaConstraints& constraints) {
+  blink::WebVector<blink::WebMediaConstraint> mandatory_constraints;
+  constraints.getMandatoryConstraints(mandatory_constraints);
+  return !mandatory_constraints.isEmpty();
+}
+
 // Retrieve the desired max width and height from |constraints|.
 void GetDesiredMaxWidthAndHeight(const blink::WebMediaConstraints& constraints,
                                  int* desired_width, int* desired_height) {
@@ -277,14 +287,90 @@ void GetBestCaptureFormat(
 }
 
 // Empty method used for keeping a reference to the original media::VideoFrame
-// in MediaStreamVideoSource::DeliverVideoFrame if cropping is needed.
-// The reference to |frame| is kept in the closure that calls this method.
+// in MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO if cropping is
+// needed. The reference to |frame| is kept in the closure that calls this
+// method.
 void ReleaseOriginalFrame(
     const scoped_refptr<media::VideoFrame>& frame) {
 }
 
 }  // anonymous namespace
 
+// Helper class used for delivering video frames to all registered tracks
+// on the IO-thread.
+class MediaStreamVideoSource::FrameDeliverer : public VideoFrameDeliverer {
+ public:
+  FrameDeliverer(
+      const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
+      : VideoFrameDeliverer(io_message_loop)  {
+  }
+
+  // Register |callback| to receive video frames of max size
+  // |max_frame_output_size| on the IO thread.
+  // TODO(perkj): Currently |max_frame_output_size| must be the same for all
+  // |callbacks|.
+  void AddCallback(void* id,
+                   const VideoCaptureDeliverFrameCB& callback,
+                   const gfx::Size& max_frame_output_size) {
+    DCHECK(thread_checker().CalledOnValidThread());
+    io_message_loop()->PostTask(
+        FROM_HERE,
+        base::Bind(
+            &FrameDeliverer::AddCallbackWithResolutionOnIO,
+            this, id, callback, max_frame_output_size));
+  }
+
+  virtual void DeliverFrameOnIO(
+      const scoped_refptr<media::VideoFrame>& frame,
+      const media::VideoCaptureFormat& format) OVERRIDE {
+    DCHECK(io_message_loop()->BelongsToCurrentThread());
+    TRACE_EVENT0("video", "MediaStreamVideoSource::DeliverFrameOnIO");
+    if (max_output_size_.IsEmpty())
+      return;  // Frame received before the output has been decided.
+
+    scoped_refptr<media::VideoFrame> video_frame(frame);
+    const gfx::Size& visible_size = frame->visible_rect().size();
+    if (visible_size.width() > max_output_size_.width() ||
+        visible_size.height() > max_output_size_.height()) {
+      // If |frame| is not the size that is expected, we need to crop it by
+      // providing a new |visible_rect|. The new visible rect must be within the
+      // original |visible_rect|.
+      gfx::Rect output_rect = frame->visible_rect();
+      output_rect.ClampToCenteredSize(max_output_size_);
+      // TODO(perkj): Allow cropping of textures once http://crbug/362521 is
+      // fixed.
+      if (frame->format() != media::VideoFrame::NATIVE_TEXTURE) {
+        video_frame = media::VideoFrame::WrapVideoFrame(
+            frame,
+            output_rect,
+            output_rect.size(),
+            base::Bind(&ReleaseOriginalFrame, frame));
+      }
+    }
+    VideoFrameDeliverer::DeliverFrameOnIO(video_frame, format);
+  }
+
+ protected:
+  virtual ~FrameDeliverer() {
+  }
+
+  void AddCallbackWithResolutionOnIO(
+      void* id,
+     const VideoCaptureDeliverFrameCB& callback,
+      const gfx::Size& max_frame_output_size) {
+    DCHECK(io_message_loop()->BelongsToCurrentThread());
+    // Currently we only support one frame output size.
+    DCHECK(!max_frame_output_size.IsEmpty() &&
+           (max_output_size_.IsEmpty() ||
+            max_output_size_ == max_frame_output_size));
+    max_output_size_ = max_frame_output_size;
+    VideoFrameDeliverer::AddCallbackOnIO(id, callback);
+  }
+
+ private:
+  gfx::Size max_output_size_;
+};
+
 // static
 MediaStreamVideoSource* MediaStreamVideoSource::GetVideoSource(
     const blink::WebMediaStreamSource& source) {
@@ -301,14 +387,20 @@ bool MediaStreamVideoSource::IsConstraintSupported(const std::string& name) {
 }
 
 MediaStreamVideoSource::MediaStreamVideoSource()
-    : state_(NEW) {
+    : state_(NEW),
+      frame_deliverer_(
+          new MediaStreamVideoSource::FrameDeliverer(
+              ChildProcess::current()->io_message_loop_proxy())),
+      weak_factory_(this) {
 }
 
 MediaStreamVideoSource::~MediaStreamVideoSource() {
+  DVLOG(3) << "~MediaStreamVideoSource()";
 }
 
 void MediaStreamVideoSource::AddTrack(
     MediaStreamVideoTrack* track,
+    const VideoCaptureDeliverFrameCB& frame_callback,
     const blink::WebMediaConstraints& constraints,
     const ConstraintsCallback& callback) {
   DCHECK(CalledOnValidThread());
@@ -317,7 +409,7 @@ void MediaStreamVideoSource::AddTrack(
   tracks_.push_back(track);
 
   requested_constraints_.push_back(
-      RequestedConstraints(constraints, callback));
+      RequestedConstraints(track, frame_callback, constraints, callback));
 
   switch (state_) {
     case NEW: {
@@ -330,8 +422,11 @@ void MediaStreamVideoSource::AddTrack(
       GetConstraintValue(constraints, true, kMaxHeight, &max_requested_height);
 
       state_ = RETRIEVING_CAPABILITIES;
-      GetCurrentSupportedFormats(max_requested_width,
-                                 max_requested_height);
+      GetCurrentSupportedFormats(
+          max_requested_width,
+          max_requested_height,
+          base::Bind(&MediaStreamVideoSource::OnSupportedFormats,
+                     weak_factory_.GetWeakPtr()));
 
       break;
     }
@@ -355,10 +450,19 @@ void MediaStreamVideoSource::RemoveTrack(MediaStreamVideoTrack* video_track) {
       std::find(tracks_.begin(), tracks_.end(), video_track);
   DCHECK(it != tracks_.end());
   tracks_.erase(it);
+  // Call |RemoveCallback| here even if adding the track has failed and
+  // frame_deliverer_->AddCallback has not been called.
+  frame_deliverer_->RemoveCallback(video_track);
+
   if (tracks_.empty())
     StopSource();
 }
 
+const scoped_refptr<base::MessageLoopProxy>&
+MediaStreamVideoSource::io_message_loop() const {
+  return frame_deliverer_->io_message_loop();
+}
+
 void MediaStreamVideoSource::DoStopSource() {
   DCHECK(CalledOnValidThread());
   DVLOG(3) << "DoStopSource()";
@@ -369,36 +473,6 @@ void MediaStreamVideoSource::DoStopSource() {
   SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
 }
 
-void MediaStreamVideoSource::DeliverVideoFrame(
-    const scoped_refptr<media::VideoFrame>& frame,
-    const media::VideoCaptureFormat& format) {
-  DCHECK(CalledOnValidThread());
-  scoped_refptr<media::VideoFrame> video_frame(frame);
-
-  if (frame->visible_rect().size().width() > max_frame_output_size_.width() ||
-      frame->visible_rect().size().height() > max_frame_output_size_.height()) {
-    // If |frame| is not the size that is expected, we need to crop it by
-    // providing a new |visible_rect|. The new visible rect must be within the
-    // original |visible_rect|.
-    gfx::Rect output_rect = frame->visible_rect();
-    output_rect.ClampToCenteredSize(max_frame_output_size_);
-    // TODO(perkj): Allow cropping of textures once http://crbug/362521 is
-    // fixed.
-    if (frame->format() != media::VideoFrame::NATIVE_TEXTURE) {
-      video_frame = media::VideoFrame::WrapVideoFrame(
-          frame,
-          output_rect,
-          output_rect.size(),
-          base::Bind(&ReleaseOriginalFrame, frame));
-    }
-  }
-
-  for (std::vector<MediaStreamVideoTrack*>::iterator it = tracks_.begin();
-       it != tracks_.end(); ++it) {
-    (*it)->OnVideoFrame(video_frame);
-  }
-}
-
 void MediaStreamVideoSource::OnSupportedFormats(
     const media::VideoCaptureFormats& formats) {
   DCHECK(CalledOnValidThread());
@@ -424,7 +498,10 @@ void MediaStreamVideoSource::OnSupportedFormats(
 
   media::VideoCaptureParams params;
   params.requested_format = current_format_;
-  StartSourceImpl(params);
+  StartSourceImpl(
+      params,
+      base::Bind(&MediaStreamVideoSource::FrameDeliverer::DeliverFrameOnIO,
+                 frame_deliverer_));
 }
 
 bool MediaStreamVideoSource::FindBestFormatWithConstraints(
@@ -439,6 +516,16 @@ bool MediaStreamVideoSource::FindBestFormatWithConstraints(
     const blink::WebMediaConstraints& requested_constraints =
         request_it->constraints;
 
+    // If the source doesn't support capability enumeration it is still ok if
+    // no mandatory constraints have been specified. That just means that
+    // we will start with whatever format is native to the source.
+    if (formats.empty() && !HasMandatoryConstraints(requested_constraints)) {
+      *best_format = media::VideoCaptureFormat();
+      *resulting_constraints = requested_constraints;
+      *max_frame_output_size = gfx::Size(std::numeric_limits<int>::max(),
+                                         std::numeric_limits<int>::max());
+      return true;
+    }
     media::VideoCaptureFormats filtered_formats =
         FilterFormats(requested_constraints, formats);
     if (filtered_formats.size() > 0) {
@@ -480,8 +567,19 @@ void MediaStreamVideoSource::FinalizeAddTrack() {
   callbacks.swap(requested_constraints_);
   for (std::vector<RequestedConstraints>::iterator it = callbacks.begin();
        it != callbacks.end(); ++it) {
-    bool success = state_ == STARTED &&
-        !FilterFormats(it->constraints, formats).empty();
+    // The track has been added successfully if the source has started and
+    // there are either no mandatory constraints and the source doesn't expose
+    // its format capabilities, or the constraints and the format match.
+    // For example, a remote source doesn't expose its format capabilities.
+    bool success =
+        state_ == STARTED &&
+        ((!current_format_.IsValid() && !HasMandatoryConstraints(
+            it->constraints)) ||
+         !FilterFormats(it->constraints, formats).empty());
+    if (success) {
+      frame_deliverer_->AddCallback(it->track, it->frame_callback,
+                                    max_frame_output_size_);
+    }
     DVLOG(3) << "FinalizeAddTrack() success " << success;
     if (!it->callback.is_null())
       it->callback.Run(this, success);
@@ -500,9 +598,14 @@ void MediaStreamVideoSource::SetReadyState(
 }
 
 MediaStreamVideoSource::RequestedConstraints::RequestedConstraints(
+    MediaStreamVideoTrack* track,
+    const VideoCaptureDeliverFrameCB& frame_callback,
     const blink::WebMediaConstraints& constraints,
     const ConstraintsCallback& callback)
-    : constraints(constraints), callback(callback) {
+    : track(track),
+      frame_callback(frame_callback),
+      constraints(constraints),
+      callback(callback) {
 }
 
 MediaStreamVideoSource::RequestedConstraints::~RequestedConstraints() {