diff options
author | miu <miu@chromium.org> | 2015-03-03 15:07:39 -0800 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-03-03 23:09:06 +0000 |
commit | 78807dc7968c9a397326aef113435cbca486f96c (patch) | |
tree | cb68a683017ad572baa00ea59ded2a3e7a49d7ed /media | |
parent | 937b6c98b45dda368ac6562b3522f8941da44ae6 (diff) | |
download | chromium_src-78807dc7968c9a397326aef113435cbca486f96c.zip chromium_src-78807dc7968c9a397326aef113435cbca486f96c.tar.gz chromium_src-78807dc7968c9a397326aef113435cbca486f96c.tar.bz2 |
Add metadata to media::VideoFrame and plumb it through IPC/MediaStream.
This change has three main goals: First, to be able to pass extra
information associated with each VideoFrame from the capture source to
the downstream consumers (see bugs for details). Second, to eliminate
redundancies in the current MediaStreamVideoSink API; specifically,
media::VideoFrame contains most of the same properties as
media::VideoCaptureFormat. Third, to fully support the separate
VideoFrame concepts of "coded size" versus "visible size" in the capture
pipeline, rather than force all producers/consumers to deal with packed
data. (Using packed frame sizes can be suboptimal for performance in
some use cases.)
The metadata is stored in a base::DictionaryValue owned by
media::VideoFrame to allow for structured data passing. DictionaryValue
is a great choice since an efficient IPC (de)serialization
implementation already exists, and the metadata can be easily
pretty-printed for logging where needed. Also, it's logical for
VideoFrame to own the metadata, as both need to be passed/shared
together across threads without copying.
Finally, this change includes one new use of the metadata as a
motivation for its existence: The tab/desktop capture code now includes
capture timing information, which will allow Cast streaming sessions to
be analyzed for user experience improvements. In the future, some of
the special-use-case data members in VideoFrame should be moved into
the metadata.
BUG=461116,462101
Review URL: https://codereview.chromium.org/955253002
Cr-Commit-Position: refs/heads/master@{#318954}
Diffstat (limited to 'media')
-rw-r--r-- | media/base/BUILD.gn | 4 | ||||
-rw-r--r-- | media/base/video_capturer_source.h | 1 | ||||
-rw-r--r-- | media/base/video_frame.h | 12 | ||||
-rw-r--r-- | media/base/video_frame_metadata.cc | 125 | ||||
-rw-r--r-- | media/base/video_frame_metadata.h | 70 | ||||
-rw-r--r-- | media/base/video_frame_unittest.cc | 79 | ||||
-rw-r--r-- | media/cast/sender/video_sender.cc | 34 | ||||
-rw-r--r-- | media/media.gyp | 4 | ||||
-rw-r--r-- | media/video/capture/fake_video_capture_device_unittest.cc | 5 | ||||
-rw-r--r-- | media/video/capture/video_capture_device.h | 1 | ||||
-rw-r--r-- | media/video/capture/video_capture_device_unittest.cc | 5 |
11 files changed, 323 insertions, 17 deletions
diff --git a/media/base/BUILD.gn b/media/base/BUILD.gn index 11d87f2..235ef60 100644 --- a/media/base/BUILD.gn +++ b/media/base/BUILD.gn @@ -172,6 +172,8 @@ source_set("base") { "video_decoder_config.h", "video_frame.cc", "video_frame.h", + "video_frame_metadata.cc", + "video_frame_metadata.h", "video_frame_pool.cc", "video_frame_pool.h", "video_renderer.cc", @@ -272,6 +274,8 @@ source_set("base_for_cast_ios") { sources = [ "video_frame.cc", "video_frame.h", + "video_frame_metadata.cc", + "video_frame_metadata.h", ] configs += [ "//build/config/compiler:no_size_t_to_int_warning", diff --git a/media/base/video_capturer_source.h b/media/base/video_capturer_source.h index 56ec729..774f28d 100644 --- a/media/base/video_capturer_source.h +++ b/media/base/video_capturer_source.h @@ -42,7 +42,6 @@ class MEDIA_EXPORT VideoCapturerSource { // the first video frame delivered may not have timestamp equal to 0. typedef base::Callback< void(const scoped_refptr<media::VideoFrame>& video_frame, - const media::VideoCaptureFormat& format, const base::TimeTicks& estimated_capture_time)> VideoCaptureDeliverFrameCB; diff --git a/media/base/video_frame.h b/media/base/video_frame.h index 366a356..cebed0a 100644 --- a/media/base/video_frame.h +++ b/media/base/video_frame.h @@ -12,6 +12,7 @@ #include "base/memory/shared_memory.h" #include "base/synchronization/lock.h" #include "media/base/buffers.h" +#include "media/base/video_frame_metadata.h" #include "ui/gfx/geometry/rect.h" #include "ui/gfx/geometry/size.h" @@ -288,6 +289,15 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { // Returns the offset into the shared memory where the frame data begins. size_t shared_memory_offset() const; + // Returns a dictionary of optional metadata. This contains information + // associated with the frame that downstream clients might use for frame-level + // logging, quality/performance optimizations, signaling, etc. + // + // TODO(miu): Move some of the "extra" members of VideoFrame (below) into + // here as a later clean-up step. + const VideoFrameMetadata* metadata() const { return &metadata_; } + VideoFrameMetadata* metadata() { return &metadata_; } + bool allow_overlay() const { return allow_overlay_; } #if defined(OS_POSIX) @@ -403,6 +413,8 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { const bool end_of_stream_; + VideoFrameMetadata metadata_; + bool allow_overlay_; DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame); diff --git a/media/base/video_frame_metadata.cc b/media/base/video_frame_metadata.cc new file mode 100644 index 0000000..d14bbe9 --- /dev/null +++ b/media/base/video_frame_metadata.cc @@ -0,0 +1,125 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/logging.h" +#include "base/strings/string_number_conversions.h" +#include "media/base/video_frame_metadata.h" + +namespace media { + +namespace { + +// Map enum key to internal std::string key used by base::DictionaryValue. +inline std::string ToInternalKey(VideoFrameMetadata::Key key) { + DCHECK_LT(key, VideoFrameMetadata::NUM_KEYS); + return base::IntToString(static_cast<int>(key)); +} + +} // namespace + +VideoFrameMetadata::VideoFrameMetadata() {} + +VideoFrameMetadata::~VideoFrameMetadata() {} + +bool VideoFrameMetadata::HasKey(Key key) const { + return dictionary_.HasKey(ToInternalKey(key)); +} + +void VideoFrameMetadata::SetBoolean(Key key, bool value) { + dictionary_.SetBooleanWithoutPathExpansion(ToInternalKey(key), value); +} + +void VideoFrameMetadata::SetInteger(Key key, int value) { + dictionary_.SetIntegerWithoutPathExpansion(ToInternalKey(key), value); +} + +void VideoFrameMetadata::SetDouble(Key key, double value) { + dictionary_.SetDoubleWithoutPathExpansion(ToInternalKey(key), value); +} + +void VideoFrameMetadata::SetString(Key key, const std::string& value) { + dictionary_.SetWithoutPathExpansion( + ToInternalKey(key), + // Using BinaryValue since we don't want the |value| interpreted as having + // any particular character encoding (e.g., UTF-8) by + // base::DictionaryValue. + base::BinaryValue::CreateWithCopiedBuffer(value.data(), value.size())); +} + +void VideoFrameMetadata::SetTimeTicks(Key key, const base::TimeTicks& value) { + const int64 internal_value = value.ToInternalValue(); + dictionary_.SetWithoutPathExpansion( + ToInternalKey(key), + base::BinaryValue::CreateWithCopiedBuffer( + reinterpret_cast<const char*>(&internal_value), + sizeof(internal_value))); +} + +void VideoFrameMetadata::SetValue(Key key, scoped_ptr<base::Value> value) { + dictionary_.SetWithoutPathExpansion(ToInternalKey(key), value.Pass()); +} + +bool VideoFrameMetadata::GetBoolean(Key key, bool* value) const { + DCHECK(value); + return dictionary_.GetBooleanWithoutPathExpansion(ToInternalKey(key), value); +} + +bool VideoFrameMetadata::GetInteger(Key key, int* value) const { + DCHECK(value); + return dictionary_.GetIntegerWithoutPathExpansion(ToInternalKey(key), value); +} + +bool VideoFrameMetadata::GetDouble(Key key, double* value) const { + DCHECK(value); + return dictionary_.GetDoubleWithoutPathExpansion(ToInternalKey(key), value); +} + +bool VideoFrameMetadata::GetString(Key key, std::string* value) const { + DCHECK(value); + const base::BinaryValue* const binary_value = GetBinaryValue(key); + if (binary_value) + value->assign(binary_value->GetBuffer(), binary_value->GetSize()); + return !!binary_value; +} + +bool VideoFrameMetadata::GetTimeTicks(Key key, base::TimeTicks* value) const { + DCHECK(value); + const base::BinaryValue* const binary_value = GetBinaryValue(key); + if (binary_value && binary_value->GetSize() == sizeof(int64)) { + int64 internal_value; + memcpy(&internal_value, binary_value->GetBuffer(), sizeof(internal_value)); + *value = base::TimeTicks::FromInternalValue(internal_value); + return true; + } + return false; +} + +const base::Value* VideoFrameMetadata::GetValue(Key key) const { + const base::Value* result = nullptr; + if (!dictionary_.GetWithoutPathExpansion(ToInternalKey(key), &result)) + return nullptr; + return result; +} + +void VideoFrameMetadata::MergeInternalValuesInto( + base::DictionaryValue* out) const { + out->MergeDictionary(&dictionary_); +} + +void VideoFrameMetadata::MergeInternalValuesFrom( + const base::DictionaryValue& in) { + dictionary_.MergeDictionary(&in); +} + +const base::BinaryValue* VideoFrameMetadata::GetBinaryValue(Key key) const { + const base::Value* internal_value = nullptr; + if (dictionary_.GetWithoutPathExpansion(ToInternalKey(key), + &internal_value) && + internal_value->GetType() == base::Value::TYPE_BINARY) { + return static_cast<const base::BinaryValue*>(internal_value); + } + return nullptr; +} + +} // namespace media diff --git a/media/base/video_frame_metadata.h b/media/base/video_frame_metadata.h new file mode 100644 index 0000000..31fbe74 --- /dev/null +++ b/media/base/video_frame_metadata.h @@ -0,0 +1,70 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_BASE_VIDEO_FRAME_METADATA_H_ +#define MEDIA_BASE_VIDEO_FRAME_METADATA_H_ + +#include "base/compiler_specific.h" +#include "base/time/time.h" +#include "base/values.h" +#include "media/base/media_export.h" + +namespace media { + +class MEDIA_EXPORT VideoFrameMetadata { + public: + enum Key { + // Video capture begin/end timestamps. Consumers can use these values for + // dynamic optimizations, logging stats, etc. Use Get/SetTimeTicks() for + // these keys. + CAPTURE_BEGIN_TIME, + CAPTURE_END_TIME, + + // Represents either the fixed frame rate, or the maximum frame rate to + // expect from a variable-rate source. Use Get/SetDouble() for this key. + FRAME_RATE, + + NUM_KEYS + }; + + VideoFrameMetadata(); + ~VideoFrameMetadata(); + + bool HasKey(Key key) const; + + void Clear() { dictionary_.Clear(); } + + // Setters. Overwrites existing value, if present. + void SetBoolean(Key key, bool value); + void SetInteger(Key key, int value); + void SetDouble(Key key, double value); + void SetString(Key key, const std::string& value); + void SetTimeTicks(Key key, const base::TimeTicks& value); + void SetValue(Key key, scoped_ptr<base::Value> value); + + // Getters. Returns true if |key| was present and has the value has been set. + bool GetBoolean(Key key, bool* value) const WARN_UNUSED_RESULT; + bool GetInteger(Key key, int* value) const WARN_UNUSED_RESULT; + bool GetDouble(Key key, double* value) const WARN_UNUSED_RESULT; + bool GetString(Key key, std::string* value) const WARN_UNUSED_RESULT; + bool GetTimeTicks(Key key, base::TimeTicks* value) const WARN_UNUSED_RESULT; + + // Returns null if |key| was not present. + const base::Value* GetValue(Key key) const WARN_UNUSED_RESULT; + + // For serialization. + void MergeInternalValuesInto(base::DictionaryValue* out) const; + void MergeInternalValuesFrom(const base::DictionaryValue& in); + + private: + const base::BinaryValue* GetBinaryValue(Key key) const; + + base::DictionaryValue dictionary_; + + DISALLOW_COPY_AND_ASSIGN(VideoFrameMetadata); +}; + +} // namespace media + +#endif // MEDIA_BASE_VIDEO_FRAME_METADATA_H_ diff --git a/media/base/video_frame_unittest.cc b/media/base/video_frame_unittest.cc index f2635f9..5c2159f 100644 --- a/media/base/video_frame_unittest.cc +++ b/media/base/video_frame_unittest.cc @@ -328,4 +328,83 @@ TEST(VideoFrame, ZeroInitialized) { EXPECT_EQ(0, frame->data(i)[0]); } +TEST(VideoFrameMetadata, SetAndThenGetAllKeysForAllTypes) { + VideoFrameMetadata metadata; + + for (int i = 0; i < VideoFrameMetadata::NUM_KEYS; ++i) { + const VideoFrameMetadata::Key key = static_cast<VideoFrameMetadata::Key>(i); + + EXPECT_FALSE(metadata.HasKey(key)); + metadata.SetBoolean(key, true); + EXPECT_TRUE(metadata.HasKey(key)); + bool bool_value = false; + EXPECT_TRUE(metadata.GetBoolean(key, &bool_value)); + EXPECT_EQ(true, bool_value); + metadata.Clear(); + + EXPECT_FALSE(metadata.HasKey(key)); + metadata.SetInteger(key, i); + EXPECT_TRUE(metadata.HasKey(key)); + int int_value = -999; + EXPECT_TRUE(metadata.GetInteger(key, &int_value)); + EXPECT_EQ(i, int_value); + metadata.Clear(); + + EXPECT_FALSE(metadata.HasKey(key)); + metadata.SetDouble(key, 3.14 * i); + EXPECT_TRUE(metadata.HasKey(key)); + double double_value = -999.99; + EXPECT_TRUE(metadata.GetDouble(key, &double_value)); + EXPECT_EQ(3.14 * i, double_value); + metadata.Clear(); + + EXPECT_FALSE(metadata.HasKey(key)); + metadata.SetString(key, base::StringPrintf("\xfe%d\xff", i)); + EXPECT_TRUE(metadata.HasKey(key)); + std::string string_value; + EXPECT_TRUE(metadata.GetString(key, &string_value)); + EXPECT_EQ(base::StringPrintf("\xfe%d\xff", i), string_value); + metadata.Clear(); + + EXPECT_FALSE(metadata.HasKey(key)); + metadata.SetTimeTicks(key, base::TimeTicks::FromInternalValue(~(0LL) + i)); + EXPECT_TRUE(metadata.HasKey(key)); + base::TimeTicks ticks_value; + EXPECT_TRUE(metadata.GetTimeTicks(key, &ticks_value)); + EXPECT_EQ(base::TimeTicks::FromInternalValue(~(0LL) + i), ticks_value); + metadata.Clear(); + + EXPECT_FALSE(metadata.HasKey(key)); + metadata.SetValue(key, + scoped_ptr<base::Value>(base::Value::CreateNullValue())); + EXPECT_TRUE(metadata.HasKey(key)); + const base::Value* const null_value = metadata.GetValue(key); + EXPECT_TRUE(null_value); + EXPECT_EQ(base::Value::TYPE_NULL, null_value->GetType()); + metadata.Clear(); + } +} + +TEST(VideoFrameMetadata, PassMetadataViaIntermediary) { + VideoFrameMetadata expected; + for (int i = 0; i < VideoFrameMetadata::NUM_KEYS; ++i) { + const VideoFrameMetadata::Key key = static_cast<VideoFrameMetadata::Key>(i); + expected.SetInteger(key, i); + } + + base::DictionaryValue tmp; + expected.MergeInternalValuesInto(&tmp); + EXPECT_EQ(static_cast<size_t>(VideoFrameMetadata::NUM_KEYS), tmp.size()); + + VideoFrameMetadata result; + result.MergeInternalValuesFrom(tmp); + + for (int i = 0; i < VideoFrameMetadata::NUM_KEYS; ++i) { + const VideoFrameMetadata::Key key = static_cast<VideoFrameMetadata::Key>(i); + int value = -1; + EXPECT_TRUE(result.GetInteger(key, &value)); + EXPECT_EQ(i, value); + } +} + } // namespace media diff --git a/media/cast/sender/video_sender.cc b/media/cast/sender/video_sender.cc index 2389e9f..e4e3a45 100644 --- a/media/cast/sender/video_sender.cc +++ b/media/cast/sender/video_sender.cc @@ -30,6 +30,30 @@ const int kRoundTripsNeeded = 4; // time). const int kConstantTimeMs = 75; +// Extract capture begin/end timestamps from |video_frame|'s metadata and log +// it. +void LogVideoCaptureTimestamps(const CastEnvironment& cast_environment, + const media::VideoFrame& video_frame, + RtpTimestamp rtp_timestamp) { + base::TimeTicks capture_begin_time; + base::TimeTicks capture_end_time; + if (!video_frame.metadata()->GetTimeTicks( + media::VideoFrameMetadata::CAPTURE_BEGIN_TIME, &capture_begin_time) || + !video_frame.metadata()->GetTimeTicks( + media::VideoFrameMetadata::CAPTURE_END_TIME, &capture_end_time)) { + // The frame capture timestamps were not provided by the video capture + // source. Simply log the events as happening right now. + capture_begin_time = capture_end_time = + cast_environment.Clock()->NowTicks(); + } + cast_environment.Logging()->InsertFrameEvent( + capture_begin_time, FRAME_CAPTURE_BEGIN, VIDEO_EVENT, rtp_timestamp, + kFrameIdUnknown); + cast_environment.Logging()->InsertFrameEvent( + capture_end_time, FRAME_CAPTURE_END, VIDEO_EVENT, rtp_timestamp, + kFrameIdUnknown); +} + } // namespace // Note, we use a fixed bitrate value when external video encoder is used. @@ -108,15 +132,7 @@ void VideoSender::InsertRawVideoFrame( const RtpTimestamp rtp_timestamp = TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency); - const base::TimeTicks insertion_time = cast_environment_->Clock()->NowTicks(); - // TODO(miu): Plumb in capture timestamps. For now, make it look like capture - // took zero time by setting the BEGIN and END event to the same timestamp. - cast_environment_->Logging()->InsertFrameEvent( - insertion_time, FRAME_CAPTURE_BEGIN, VIDEO_EVENT, rtp_timestamp, - kFrameIdUnknown); - cast_environment_->Logging()->InsertFrameEvent( - insertion_time, FRAME_CAPTURE_END, VIDEO_EVENT, rtp_timestamp, - kFrameIdUnknown); + LogVideoCaptureTimestamps(*cast_environment_, *video_frame, rtp_timestamp); // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc TRACE_EVENT_INSTANT2( diff --git a/media/media.gyp b/media/media.gyp index 3f95cc8..ea044b9 100644 --- a/media/media.gyp +++ b/media/media.gyp @@ -388,6 +388,8 @@ 'base/video_decoder_config.h', 'base/video_frame.cc', 'base/video_frame.h', + 'base/video_frame_metadata.cc', + 'base/video_frame_metadata.h', 'base/video_frame_pool.cc', 'base/video_frame_pool.h', 'base/video_renderer.cc', @@ -1889,6 +1891,8 @@ 'base/mac/videotoolbox_glue.mm', 'base/video_frame.cc', 'base/video_frame.h', + 'base/video_frame_metadata.cc', + 'base/video_frame_metadata.h', ], 'link_settings': { 'libraries': [ diff --git a/media/video/capture/fake_video_capture_device_unittest.cc b/media/video/capture/fake_video_capture_device_unittest.cc index ace1a28..3ad349f 100644 --- a/media/video/capture/fake_video_capture_device_unittest.cc +++ b/media/video/capture/fake_video_capture_device_unittest.cc @@ -26,9 +26,8 @@ class MockClient : public VideoCaptureDevice::Client { MOCK_METHOD2(ReserveOutputBuffer, scoped_refptr<Buffer>(VideoFrame::Format format, const gfx::Size& dimensions)); - MOCK_METHOD4(OnIncomingCapturedVideoFrame, + MOCK_METHOD3(OnIncomingCapturedVideoFrame, void(const scoped_refptr<Buffer>& buffer, - const VideoCaptureFormat& buffer_format, const scoped_refptr<media::VideoFrame>& frame, const base::TimeTicks& timestamp)); MOCK_METHOD1(OnError, void(const std::string& reason)); @@ -82,7 +81,7 @@ class FakeVideoCaptureDeviceTest : public testing::Test { void SetUp() override { EXPECT_CALL(*client_, ReserveOutputBuffer(_,_)).Times(0); - EXPECT_CALL(*client_, OnIncomingCapturedVideoFrame(_,_,_,_)).Times(0); + EXPECT_CALL(*client_, OnIncomingCapturedVideoFrame(_,_,_)).Times(0); } void OnFrameCaptured(const VideoCaptureFormat& format) { diff --git a/media/video/capture/video_capture_device.h b/media/video/capture/video_capture_device.h index 87de975..747f1b0 100644 --- a/media/video/capture/video_capture_device.h +++ b/media/video/capture/video_capture_device.h @@ -219,7 +219,6 @@ class MEDIA_EXPORT VideoCaptureDevice { // additional copies in the browser process. virtual void OnIncomingCapturedVideoFrame( const scoped_refptr<Buffer>& buffer, - const VideoCaptureFormat& buffer_format, const scoped_refptr<media::VideoFrame>& frame, const base::TimeTicks& timestamp) = 0; diff --git a/media/video/capture/video_capture_device_unittest.cc b/media/video/capture/video_capture_device_unittest.cc index c4e94a0..0c66c34 100644 --- a/media/video/capture/video_capture_device_unittest.cc +++ b/media/video/capture/video_capture_device_unittest.cc @@ -67,9 +67,8 @@ class MockClient : public VideoCaptureDevice::Client { MOCK_METHOD2(ReserveOutputBuffer, scoped_refptr<Buffer>(VideoFrame::Format format, const gfx::Size& dimensions)); - MOCK_METHOD4(OnIncomingCapturedVideoFrame, + MOCK_METHOD3(OnIncomingCapturedVideoFrame, void(const scoped_refptr<Buffer>& buffer, - const VideoCaptureFormat& buffer_format, const scoped_refptr<VideoFrame>& frame, const base::TimeTicks& timestamp)); MOCK_METHOD1(OnError, void(const std::string& reason)); @@ -129,7 +128,7 @@ class VideoCaptureDeviceTest : public testing::Test { base::android::AttachCurrentThread()); #endif EXPECT_CALL(*client_, ReserveOutputBuffer(_,_)).Times(0); - EXPECT_CALL(*client_, OnIncomingCapturedVideoFrame(_,_,_,_)).Times(0); + EXPECT_CALL(*client_, OnIncomingCapturedVideoFrame(_,_,_)).Times(0); } void ResetWithNewClient() { |