Add metadata to media::VideoFrame and plumb it through IPC/MediaStream.

This change has three main goals: First, to be able to pass extra
information associated with each VideoFrame from the capture source to
the downstream consumers (see bugs for details).  Second, to eliminate
redundancies in the current MediaStreamVideoSink API; specifically,
media::VideoFrame contains most of the same properties as
media::VideoCaptureFormat.  Third, to fully support the separate
VideoFrame concepts of "coded size" versus "visible size" in the capture
pipeline, rather than force all producers/consumers to deal with packed
data.  (Using packed frame sizes can be suboptimal for performance in
some use cases.)

The metadata is stored in a base::DictionaryValue owned by
media::VideoFrame to allow for structured data passing.  DictionaryValue
is a great choice since an efficient IPC (de)serialization
implementation already exists, and the metadata can be easily
pretty-printed for logging where needed.  Also, it's logical for
VideoFrame to own the metadata, as both need to be passed/shared
together across threads without copying.

Finally, this change includes one new use of the metadata as a
motivation for its existence: The tab/desktop capture code now includes
capture timing information, which will allow Cast streaming sessions to
be analyzed for user experience improvements.  In the future, some of
the special-use-case data members in VideoFrame should be moved into
the metadata.

BUG=461116,462101

Review URL: https://codereview.chromium.org/955253002

Cr-Commit-Position: refs/heads/master@{#318954}

11 files changed, 323 insertions, 17 deletions

diff --git a/media/base/BUILD.gn b/media/base/BUILD.gn
index 11d87f2..235ef60 100644
--- a/media/base/BUILD.gn
+++ b/media/base/BUILD.gn
@@ -172,6 +172,8 @@ source_set("base") {
     "video_decoder_config.h",
     "video_frame.cc",
     "video_frame.h",
+    "video_frame_metadata.cc",
+    "video_frame_metadata.h",
     "video_frame_pool.cc",
     "video_frame_pool.h",
     "video_renderer.cc",
@@ -272,6 +274,8 @@ source_set("base_for_cast_ios") {
   sources = [
     "video_frame.cc",
     "video_frame.h",
+    "video_frame_metadata.cc",
+    "video_frame_metadata.h",
   ]
   configs += [
     "//build/config/compiler:no_size_t_to_int_warning",
diff --git a/media/base/video_capturer_source.h b/media/base/video_capturer_source.h
index 56ec729..774f28d 100644
--- a/media/base/video_capturer_source.h
+++ b/media/base/video_capturer_source.h
@@ -42,7 +42,6 @@ class MEDIA_EXPORT VideoCapturerSource {
   // the first video frame delivered may not have timestamp equal to 0.
   typedef base::Callback<
    void(const scoped_refptr<media::VideoFrame>& video_frame,
-        const media::VideoCaptureFormat& format,
         const base::TimeTicks& estimated_capture_time)>
       VideoCaptureDeliverFrameCB;
 
diff --git a/media/base/video_frame.h b/media/base/video_frame.h
index 366a356..cebed0a 100644
--- a/media/base/video_frame.h
+++ b/media/base/video_frame.h
@@ -12,6 +12,7 @@
 #include "base/memory/shared_memory.h"
 #include "base/synchronization/lock.h"
 #include "media/base/buffers.h"
+#include "media/base/video_frame_metadata.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/size.h"
 
@@ -288,6 +289,15 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
   // Returns the offset into the shared memory where the frame data begins.
   size_t shared_memory_offset() const;
 
+  // Returns a dictionary of optional metadata.  This contains information
+  // associated with the frame that downstream clients might use for frame-level
+  // logging, quality/performance optimizations, signaling, etc.
+  //
+  // TODO(miu): Move some of the "extra" members of VideoFrame (below) into
+  // here as a later clean-up step.
+  const VideoFrameMetadata* metadata() const { return &metadata_; }
+  VideoFrameMetadata* metadata() { return &metadata_; }
+
   bool allow_overlay() const { return allow_overlay_; }
 
 #if defined(OS_POSIX)
@@ -403,6 +413,8 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
 
   const bool end_of_stream_;
 
+  VideoFrameMetadata metadata_;
+
   bool allow_overlay_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
diff --git a/media/base/video_frame_metadata.cc b/media/base/video_frame_metadata.cc
new file mode 100644
index 0000000..d14bbe9
--- /dev/null
+++ b/media/base/video_frame_metadata.cc
@@ -0,0 +1,125 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/logging.h"
+#include "base/strings/string_number_conversions.h"
+#include "media/base/video_frame_metadata.h"
+
+namespace media {
+
+namespace {
+
+// Map enum key to internal std::string key used by base::DictionaryValue.
+inline std::string ToInternalKey(VideoFrameMetadata::Key key) {
+  DCHECK_LT(key, VideoFrameMetadata::NUM_KEYS);
+  return base::IntToString(static_cast<int>(key));
+}
+
+}  // namespace
+
+VideoFrameMetadata::VideoFrameMetadata() {}
+
+VideoFrameMetadata::~VideoFrameMetadata() {}
+
+bool VideoFrameMetadata::HasKey(Key key) const {
+  return dictionary_.HasKey(ToInternalKey(key));
+}
+
+void VideoFrameMetadata::SetBoolean(Key key, bool value) {
+  dictionary_.SetBooleanWithoutPathExpansion(ToInternalKey(key), value);
+}
+
+void VideoFrameMetadata::SetInteger(Key key, int value) {
+  dictionary_.SetIntegerWithoutPathExpansion(ToInternalKey(key), value);
+}
+
+void VideoFrameMetadata::SetDouble(Key key, double value) {
+  dictionary_.SetDoubleWithoutPathExpansion(ToInternalKey(key), value);
+}
+
+void VideoFrameMetadata::SetString(Key key, const std::string& value) {
+  dictionary_.SetWithoutPathExpansion(
+      ToInternalKey(key),
+      // Using BinaryValue since we don't want the |value| interpreted as having
+      // any particular character encoding (e.g., UTF-8) by
+      // base::DictionaryValue.
+      base::BinaryValue::CreateWithCopiedBuffer(value.data(), value.size()));
+}
+
+void VideoFrameMetadata::SetTimeTicks(Key key, const base::TimeTicks& value) {
+  const int64 internal_value = value.ToInternalValue();
+  dictionary_.SetWithoutPathExpansion(
+      ToInternalKey(key),
+      base::BinaryValue::CreateWithCopiedBuffer(
+          reinterpret_cast<const char*>(&internal_value),
+          sizeof(internal_value)));
+}
+
+void VideoFrameMetadata::SetValue(Key key, scoped_ptr<base::Value> value) {
+  dictionary_.SetWithoutPathExpansion(ToInternalKey(key), value.Pass());
+}
+
+bool VideoFrameMetadata::GetBoolean(Key key, bool* value) const {
+  DCHECK(value);
+  return dictionary_.GetBooleanWithoutPathExpansion(ToInternalKey(key), value);
+}
+
+bool VideoFrameMetadata::GetInteger(Key key, int* value) const {
+  DCHECK(value);
+  return dictionary_.GetIntegerWithoutPathExpansion(ToInternalKey(key), value);
+}
+
+bool VideoFrameMetadata::GetDouble(Key key, double* value) const {
+  DCHECK(value);
+  return dictionary_.GetDoubleWithoutPathExpansion(ToInternalKey(key), value);
+}
+
+bool VideoFrameMetadata::GetString(Key key, std::string* value) const {
+  DCHECK(value);
+  const base::BinaryValue* const binary_value = GetBinaryValue(key);
+  if (binary_value)
+    value->assign(binary_value->GetBuffer(), binary_value->GetSize());
+  return !!binary_value;
+}
+
+bool VideoFrameMetadata::GetTimeTicks(Key key, base::TimeTicks* value) const {
+  DCHECK(value);
+  const base::BinaryValue* const binary_value = GetBinaryValue(key);
+  if (binary_value && binary_value->GetSize() == sizeof(int64)) {
+    int64 internal_value;
+    memcpy(&internal_value, binary_value->GetBuffer(), sizeof(internal_value));
+    *value = base::TimeTicks::FromInternalValue(internal_value);
+    return true;
+  }
+  return false;
+}
+
+const base::Value* VideoFrameMetadata::GetValue(Key key) const {
+  const base::Value* result = nullptr;
+  if (!dictionary_.GetWithoutPathExpansion(ToInternalKey(key), &result))
+    return nullptr;
+  return result;
+}
+
+void VideoFrameMetadata::MergeInternalValuesInto(
+    base::DictionaryValue* out) const {
+  out->MergeDictionary(&dictionary_);
+}
+
+void VideoFrameMetadata::MergeInternalValuesFrom(
+    const base::DictionaryValue& in) {
+  dictionary_.MergeDictionary(&in);
+}
+
+const base::BinaryValue* VideoFrameMetadata::GetBinaryValue(Key key) const {
+  const base::Value* internal_value = nullptr;
+  if (dictionary_.GetWithoutPathExpansion(ToInternalKey(key),
+                                          &internal_value) &&
+      internal_value->GetType() == base::Value::TYPE_BINARY) {
+    return static_cast<const base::BinaryValue*>(internal_value);
+  }
+  return nullptr;
+}
+
+}  // namespace media
diff --git a/media/base/video_frame_metadata.h b/media/base/video_frame_metadata.h
new file mode 100644
index 0000000..31fbe74
--- /dev/null
+++ b/media/base/video_frame_metadata.h
@@ -0,0 +1,70 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_BASE_VIDEO_FRAME_METADATA_H_
+#define MEDIA_BASE_VIDEO_FRAME_METADATA_H_
+
+#include "base/compiler_specific.h"
+#include "base/time/time.h"
+#include "base/values.h"
+#include "media/base/media_export.h"
+
+namespace media {
+
+class MEDIA_EXPORT VideoFrameMetadata {
+ public:
+  enum Key {
+    // Video capture begin/end timestamps.  Consumers can use these values for
+    // dynamic optimizations, logging stats, etc.  Use Get/SetTimeTicks() for
+    // these keys.
+    CAPTURE_BEGIN_TIME,
+    CAPTURE_END_TIME,
+
+    // Represents either the fixed frame rate, or the maximum frame rate to
+    // expect from a variable-rate source.  Use Get/SetDouble() for this key.
+    FRAME_RATE,
+
+    NUM_KEYS
+  };
+
+  VideoFrameMetadata();
+  ~VideoFrameMetadata();
+
+  bool HasKey(Key key) const;
+
+  void Clear() { dictionary_.Clear(); }
+
+  // Setters.  Overwrites existing value, if present.
+  void SetBoolean(Key key, bool value);
+  void SetInteger(Key key, int value);
+  void SetDouble(Key key, double value);
+  void SetString(Key key, const std::string& value);
+  void SetTimeTicks(Key key, const base::TimeTicks& value);
+  void SetValue(Key key, scoped_ptr<base::Value> value);
+
+  // Getters.  Returns true if |key| was present and has the value has been set.
+  bool GetBoolean(Key key, bool* value) const WARN_UNUSED_RESULT;
+  bool GetInteger(Key key, int* value) const WARN_UNUSED_RESULT;
+  bool GetDouble(Key key, double* value) const WARN_UNUSED_RESULT;
+  bool GetString(Key key, std::string* value) const WARN_UNUSED_RESULT;
+  bool GetTimeTicks(Key key, base::TimeTicks* value) const WARN_UNUSED_RESULT;
+
+  // Returns null if |key| was not present.
+  const base::Value* GetValue(Key key) const WARN_UNUSED_RESULT;
+
+  // For serialization.
+  void MergeInternalValuesInto(base::DictionaryValue* out) const;
+  void MergeInternalValuesFrom(const base::DictionaryValue& in);
+
+ private:
+  const base::BinaryValue* GetBinaryValue(Key key) const;
+
+  base::DictionaryValue dictionary_;
+
+  DISALLOW_COPY_AND_ASSIGN(VideoFrameMetadata);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_BASE_VIDEO_FRAME_METADATA_H_
diff --git a/media/base/video_frame_unittest.cc b/media/base/video_frame_unittest.cc
index f2635f9..5c2159f 100644
--- a/media/base/video_frame_unittest.cc
+++ b/media/base/video_frame_unittest.cc
@@ -328,4 +328,83 @@ TEST(VideoFrame, ZeroInitialized) {
     EXPECT_EQ(0, frame->data(i)[0]);
 }
 
+TEST(VideoFrameMetadata, SetAndThenGetAllKeysForAllTypes) {
+  VideoFrameMetadata metadata;
+
+  for (int i = 0; i < VideoFrameMetadata::NUM_KEYS; ++i) {
+    const VideoFrameMetadata::Key key = static_cast<VideoFrameMetadata::Key>(i);
+
+    EXPECT_FALSE(metadata.HasKey(key));
+    metadata.SetBoolean(key, true);
+    EXPECT_TRUE(metadata.HasKey(key));
+    bool bool_value = false;
+    EXPECT_TRUE(metadata.GetBoolean(key, &bool_value));
+    EXPECT_EQ(true, bool_value);
+    metadata.Clear();
+
+    EXPECT_FALSE(metadata.HasKey(key));
+    metadata.SetInteger(key, i);
+    EXPECT_TRUE(metadata.HasKey(key));
+    int int_value = -999;
+    EXPECT_TRUE(metadata.GetInteger(key, &int_value));
+    EXPECT_EQ(i, int_value);
+    metadata.Clear();
+
+    EXPECT_FALSE(metadata.HasKey(key));
+    metadata.SetDouble(key, 3.14 * i);
+    EXPECT_TRUE(metadata.HasKey(key));
+    double double_value = -999.99;
+    EXPECT_TRUE(metadata.GetDouble(key, &double_value));
+    EXPECT_EQ(3.14 * i, double_value);
+    metadata.Clear();
+
+    EXPECT_FALSE(metadata.HasKey(key));
+    metadata.SetString(key, base::StringPrintf("\xfe%d\xff", i));
+    EXPECT_TRUE(metadata.HasKey(key));
+    std::string string_value;
+    EXPECT_TRUE(metadata.GetString(key, &string_value));
+    EXPECT_EQ(base::StringPrintf("\xfe%d\xff", i), string_value);
+    metadata.Clear();
+
+    EXPECT_FALSE(metadata.HasKey(key));
+    metadata.SetTimeTicks(key, base::TimeTicks::FromInternalValue(~(0LL) + i));
+    EXPECT_TRUE(metadata.HasKey(key));
+    base::TimeTicks ticks_value;
+    EXPECT_TRUE(metadata.GetTimeTicks(key, &ticks_value));
+    EXPECT_EQ(base::TimeTicks::FromInternalValue(~(0LL) + i), ticks_value);
+    metadata.Clear();
+
+    EXPECT_FALSE(metadata.HasKey(key));
+    metadata.SetValue(key,
+                      scoped_ptr<base::Value>(base::Value::CreateNullValue()));
+    EXPECT_TRUE(metadata.HasKey(key));
+    const base::Value* const null_value = metadata.GetValue(key);
+    EXPECT_TRUE(null_value);
+    EXPECT_EQ(base::Value::TYPE_NULL, null_value->GetType());
+    metadata.Clear();
+  }
+}
+
+TEST(VideoFrameMetadata, PassMetadataViaIntermediary) {
+  VideoFrameMetadata expected;
+  for (int i = 0; i < VideoFrameMetadata::NUM_KEYS; ++i) {
+    const VideoFrameMetadata::Key key = static_cast<VideoFrameMetadata::Key>(i);
+    expected.SetInteger(key, i);
+  }
+
+  base::DictionaryValue tmp;
+  expected.MergeInternalValuesInto(&tmp);
+  EXPECT_EQ(static_cast<size_t>(VideoFrameMetadata::NUM_KEYS), tmp.size());
+
+  VideoFrameMetadata result;
+  result.MergeInternalValuesFrom(tmp);
+
+  for (int i = 0; i < VideoFrameMetadata::NUM_KEYS; ++i) {
+    const VideoFrameMetadata::Key key = static_cast<VideoFrameMetadata::Key>(i);
+    int value = -1;
+    EXPECT_TRUE(result.GetInteger(key, &value));
+    EXPECT_EQ(i, value);
+  }
+}
+
 }  // namespace media
diff --git a/media/cast/sender/video_sender.cc b/media/cast/sender/video_sender.cc
index 2389e9f..e4e3a45 100644
--- a/media/cast/sender/video_sender.cc
+++ b/media/cast/sender/video_sender.cc
@@ -30,6 +30,30 @@ const int kRoundTripsNeeded = 4;
 // time).
 const int kConstantTimeMs = 75;
 
+// Extract capture begin/end timestamps from |video_frame|'s metadata and log
+// it.
+void LogVideoCaptureTimestamps(const CastEnvironment& cast_environment,
+                               const media::VideoFrame& video_frame,
+                               RtpTimestamp rtp_timestamp) {
+  base::TimeTicks capture_begin_time;
+  base::TimeTicks capture_end_time;
+  if (!video_frame.metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::CAPTURE_BEGIN_TIME, &capture_begin_time) ||
+      !video_frame.metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::CAPTURE_END_TIME, &capture_end_time)) {
+    // The frame capture timestamps were not provided by the video capture
+    // source.  Simply log the events as happening right now.
+    capture_begin_time = capture_end_time =
+        cast_environment.Clock()->NowTicks();
+  }
+  cast_environment.Logging()->InsertFrameEvent(
+      capture_begin_time, FRAME_CAPTURE_BEGIN, VIDEO_EVENT, rtp_timestamp,
+      kFrameIdUnknown);
+  cast_environment.Logging()->InsertFrameEvent(
+      capture_end_time, FRAME_CAPTURE_END, VIDEO_EVENT, rtp_timestamp,
+      kFrameIdUnknown);
+}
+
 }  // namespace
 
 // Note, we use a fixed bitrate value when external video encoder is used.
@@ -108,15 +132,7 @@ void VideoSender::InsertRawVideoFrame(
 
   const RtpTimestamp rtp_timestamp =
       TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency);
-  const base::TimeTicks insertion_time = cast_environment_->Clock()->NowTicks();
-  // TODO(miu): Plumb in capture timestamps.  For now, make it look like capture
-  // took zero time by setting the BEGIN and END event to the same timestamp.
-  cast_environment_->Logging()->InsertFrameEvent(
-      insertion_time, FRAME_CAPTURE_BEGIN, VIDEO_EVENT, rtp_timestamp,
-      kFrameIdUnknown);
-  cast_environment_->Logging()->InsertFrameEvent(
-      insertion_time, FRAME_CAPTURE_END, VIDEO_EVENT, rtp_timestamp,
-      kFrameIdUnknown);
+  LogVideoCaptureTimestamps(*cast_environment_, *video_frame, rtp_timestamp);
 
   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
   TRACE_EVENT_INSTANT2(
diff --git a/media/media.gyp b/media/media.gyp
index 3f95cc8..ea044b9 100644
--- a/media/media.gyp
+++ b/media/media.gyp
@@ -388,6 +388,8 @@
         'base/video_decoder_config.h',
         'base/video_frame.cc',
         'base/video_frame.h',
+        'base/video_frame_metadata.cc',
+        'base/video_frame_metadata.h',
         'base/video_frame_pool.cc',
         'base/video_frame_pool.h',
         'base/video_renderer.cc',
@@ -1889,6 +1891,8 @@
             'base/mac/videotoolbox_glue.mm',
             'base/video_frame.cc',
             'base/video_frame.h',
+            'base/video_frame_metadata.cc',
+            'base/video_frame_metadata.h',
           ],
           'link_settings': {
             'libraries': [
diff --git a/media/video/capture/fake_video_capture_device_unittest.cc b/media/video/capture/fake_video_capture_device_unittest.cc
index ace1a28..3ad349f 100644
--- a/media/video/capture/fake_video_capture_device_unittest.cc
+++ b/media/video/capture/fake_video_capture_device_unittest.cc
@@ -26,9 +26,8 @@ class MockClient : public VideoCaptureDevice::Client {
   MOCK_METHOD2(ReserveOutputBuffer,
                scoped_refptr<Buffer>(VideoFrame::Format format,
                                      const gfx::Size& dimensions));
-  MOCK_METHOD4(OnIncomingCapturedVideoFrame,
+  MOCK_METHOD3(OnIncomingCapturedVideoFrame,
                void(const scoped_refptr<Buffer>& buffer,
-                    const VideoCaptureFormat& buffer_format,
                     const scoped_refptr<media::VideoFrame>& frame,
                     const base::TimeTicks& timestamp));
   MOCK_METHOD1(OnError, void(const std::string& reason));
@@ -82,7 +81,7 @@ class FakeVideoCaptureDeviceTest : public testing::Test {
 
   void SetUp() override {
     EXPECT_CALL(*client_, ReserveOutputBuffer(_,_)).Times(0);
-    EXPECT_CALL(*client_, OnIncomingCapturedVideoFrame(_,_,_,_)).Times(0);
+    EXPECT_CALL(*client_, OnIncomingCapturedVideoFrame(_,_,_)).Times(0);
   }
 
   void OnFrameCaptured(const VideoCaptureFormat& format) {
diff --git a/media/video/capture/video_capture_device.h b/media/video/capture/video_capture_device.h
index 87de975..747f1b0 100644
--- a/media/video/capture/video_capture_device.h
+++ b/media/video/capture/video_capture_device.h
@@ -219,7 +219,6 @@ class MEDIA_EXPORT VideoCaptureDevice {
     // additional copies in the browser process.
     virtual void OnIncomingCapturedVideoFrame(
         const scoped_refptr<Buffer>& buffer,
-        const VideoCaptureFormat& buffer_format,
         const scoped_refptr<media::VideoFrame>& frame,
         const base::TimeTicks& timestamp) = 0;
 
diff --git a/media/video/capture/video_capture_device_unittest.cc b/media/video/capture/video_capture_device_unittest.cc
index c4e94a0..0c66c34 100644
--- a/media/video/capture/video_capture_device_unittest.cc
+++ b/media/video/capture/video_capture_device_unittest.cc
@@ -67,9 +67,8 @@ class MockClient : public VideoCaptureDevice::Client {
   MOCK_METHOD2(ReserveOutputBuffer,
                scoped_refptr<Buffer>(VideoFrame::Format format,
                                      const gfx::Size& dimensions));
-  MOCK_METHOD4(OnIncomingCapturedVideoFrame,
+  MOCK_METHOD3(OnIncomingCapturedVideoFrame,
                void(const scoped_refptr<Buffer>& buffer,
-                    const VideoCaptureFormat& buffer_format,
                     const scoped_refptr<VideoFrame>& frame,
                     const base::TimeTicks& timestamp));
   MOCK_METHOD1(OnError, void(const std::string& reason));
@@ -129,7 +128,7 @@ class VideoCaptureDeviceTest : public testing::Test {
         base::android::AttachCurrentThread());
 #endif
     EXPECT_CALL(*client_, ReserveOutputBuffer(_,_)).Times(0);
-    EXPECT_CALL(*client_, OnIncomingCapturedVideoFrame(_,_,_,_)).Times(0);
+    EXPECT_CALL(*client_, OnIncomingCapturedVideoFrame(_,_,_)).Times(0);
   }
 
   void ResetWithNewClient() {