diff options
author | emircan <emircan@chromium.org> | 2016-03-15 18:02:17 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2016-03-16 01:03:37 +0000 |
commit | ecfe5eced92b99fb8dc2351cd2afd533c2bfb953 (patch) | |
tree | 787ad397495412bc33ff73de0564d8a131e1649d /media/base | |
parent | b5841b612f8c01518fc64b81ab31210d9bdc7abb (diff) | |
download | chromium_src-ecfe5eced92b99fb8dc2351cd2afd533c2bfb953.zip chromium_src-ecfe5eced92b99fb8dc2351cd2afd533c2bfb953.tar.gz chromium_src-ecfe5eced92b99fb8dc2351cd2afd533c2bfb953.tar.bz2 |
Reland: H264 HW encode using VideoToolbox
Reland CL: https://codereview.chromium.org/1636083003/
This CL adds VTVideoEncodeAccelerator which enables H264 encode support using
VideoToolbox on mac. Also, it includes a refactor of common VideoToolbox classes
under video_toolbox_helpers.*.
Note that, this is the first CL and H264 codec is still behind a flag. More
patches will follow adding additional codec profiles and support for
bitrate adaptations.
Design Doc: https://docs.google.com/document/d/1oUTyZdNh8QstKRds-8wHEF_hqKryMiUpEOW8M57sUGU/edit?usp=sharing
BUG=500605
TEST= Tested AppRTC loopback with Chrome flag
"--enable-webrtc-hw-h264-encoding" on
https://apprtc.appspot.com/?debug=loopback&vsc=h264
TBR=avi@chromium.org, jfroy@chromium.org, sandersd@chromium.org, thakis@chromium.org, posciak@chromium.org, miu@chromium.org
Review URL: https://codereview.chromium.org/1805723002
Cr-Commit-Position: refs/heads/master@{#381373}
Diffstat (limited to 'media/base')
-rw-r--r-- | media/base/mac/BUILD.gn | 2 | ||||
-rw-r--r-- | media/base/mac/videotoolbox_glue.h | 15 | ||||
-rw-r--r-- | media/base/mac/videotoolbox_glue.mm | 9 | ||||
-rw-r--r-- | media/base/mac/videotoolbox_helpers.cc | 304 | ||||
-rw-r--r-- | media/base/mac/videotoolbox_helpers.h | 69 |
5 files changed, 394 insertions, 5 deletions
diff --git a/media/base/mac/BUILD.gn b/media/base/mac/BUILD.gn index 16ab29c..95540d6 100644 --- a/media/base/mac/BUILD.gn +++ b/media/base/mac/BUILD.gn @@ -14,6 +14,8 @@ source_set("mac") { "video_frame_mac.h", "videotoolbox_glue.h", "videotoolbox_glue.mm", + "videotoolbox_helpers.cc", + "videotoolbox_helpers.h", ] if (is_mac) { sources += [ diff --git a/media/base/mac/videotoolbox_glue.h b/media/base/mac/videotoolbox_glue.h index 212722a..9a978bf 100644 --- a/media/base/mac/videotoolbox_glue.h +++ b/media/base/mac/videotoolbox_glue.h @@ -11,11 +11,13 @@ #include "media/base/mac/coremedia_glue.h" #include "media/base/media_export.h" -// VideoToolbox API is available in OS X 10.9 and iOS 8 (10.8 has support for -// software encoding, but this class exposes the 10.9 API level). Chromium -// requires OS X 10.6 or iOS 6. Linking with VideoToolbox therefore has to -// happen at runtime. This class is defined to try and load the VideoToolbox -// library. If it succeeds, clients can use VideoToolbox via this class. +// VideoToolbox API is available in and after OS X 10.9 and iOS 8 (10.8 has +// support for software encoding, but this class exposes the 10.9 API level). +// Chromium requires OS X 10.9 or iOS 9. This class is defined to try and load +// the VideoToolbox library at runtime. If it succeeds, clients can use +// VideoToolbox via this class. +// Note that this file is necessary because Chromium still targets OS X 10.6 for +// deployment. It should be deprecated soon, see crbug.com/579648. class MEDIA_EXPORT VideoToolboxGlue { public: class Loader; @@ -49,6 +51,7 @@ class MEDIA_EXPORT VideoToolboxGlue { CFStringRef kVTCompressionPropertyKey_AllowFrameReordering() const; CFStringRef kVTCompressionPropertyKey_AverageBitRate() const; CFStringRef kVTCompressionPropertyKey_ColorPrimaries() const; + CFStringRef kVTCompressionPropertyKey_DataRateLimits() const; CFStringRef kVTCompressionPropertyKey_ExpectedFrameRate() const; CFStringRef kVTCompressionPropertyKey_MaxFrameDelayCount() const; CFStringRef kVTCompressionPropertyKey_MaxKeyFrameInterval() const; @@ -68,6 +71,8 @@ class MEDIA_EXPORT VideoToolboxGlue { CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder() const; + CFStringRef + kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder() const; // Originally from VTCompressionSession.h OSStatus VTCompressionSessionCreate( diff --git a/media/base/mac/videotoolbox_glue.mm b/media/base/mac/videotoolbox_glue.mm index 010767c..234f0ee 100644 --- a/media/base/mac/videotoolbox_glue.mm +++ b/media/base/mac/videotoolbox_glue.mm @@ -53,6 +53,7 @@ struct VideoToolboxGlue::Library { CFStringRef* kVTCompressionPropertyKey_AllowFrameReordering; CFStringRef* kVTCompressionPropertyKey_AverageBitRate; CFStringRef* kVTCompressionPropertyKey_ColorPrimaries; + CFStringRef* kVTCompressionPropertyKey_DataRateLimits; CFStringRef* kVTCompressionPropertyKey_ExpectedFrameRate; CFStringRef* kVTCompressionPropertyKey_MaxFrameDelayCount; CFStringRef* kVTCompressionPropertyKey_MaxKeyFrameInterval; @@ -68,6 +69,8 @@ struct VideoToolboxGlue::Library { CFStringRef* kVTProfileLevel_H264_High_AutoLevel; CFStringRef* kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder; + CFStringRef* + kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder; }; // Lazy-instance responsible for loading VideoToolbox. @@ -98,6 +101,7 @@ class VideoToolboxGlue::Loader { LOAD_SYMBOL(kVTCompressionPropertyKey_AllowFrameReordering) LOAD_SYMBOL(kVTCompressionPropertyKey_AverageBitRate) LOAD_SYMBOL(kVTCompressionPropertyKey_ColorPrimaries) + LOAD_SYMBOL(kVTCompressionPropertyKey_DataRateLimits) LOAD_SYMBOL(kVTCompressionPropertyKey_ExpectedFrameRate) LOAD_SYMBOL(kVTCompressionPropertyKey_MaxFrameDelayCount) LOAD_SYMBOL(kVTCompressionPropertyKey_MaxKeyFrameInterval) @@ -113,6 +117,8 @@ class VideoToolboxGlue::Loader { LOAD_SYMBOL(kVTProfileLevel_H264_High_AutoLevel) LOAD_SYMBOL( kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder) + LOAD_SYMBOL( + kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder) #undef LOAD_SYMBOL @@ -216,6 +222,7 @@ OSStatus VideoToolboxGlue::VTSessionSetProperty(VTSessionRef session, KEY_ACCESSOR(kVTCompressionPropertyKey_AllowFrameReordering) KEY_ACCESSOR(kVTCompressionPropertyKey_AverageBitRate) KEY_ACCESSOR(kVTCompressionPropertyKey_ColorPrimaries) +KEY_ACCESSOR(kVTCompressionPropertyKey_DataRateLimits) KEY_ACCESSOR(kVTCompressionPropertyKey_ExpectedFrameRate) KEY_ACCESSOR(kVTCompressionPropertyKey_MaxFrameDelayCount) KEY_ACCESSOR(kVTCompressionPropertyKey_MaxKeyFrameInterval) @@ -230,5 +237,7 @@ KEY_ACCESSOR(kVTProfileLevel_H264_Main_AutoLevel) KEY_ACCESSOR(kVTProfileLevel_H264_Extended_AutoLevel) KEY_ACCESSOR(kVTProfileLevel_H264_High_AutoLevel) KEY_ACCESSOR(kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder) +KEY_ACCESSOR( + kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder) #undef KEY_ACCESSOR diff --git a/media/base/mac/videotoolbox_helpers.cc b/media/base/mac/videotoolbox_helpers.cc new file mode 100644 index 0000000..47cfb2f --- /dev/null +++ b/media/base/mac/videotoolbox_helpers.cc @@ -0,0 +1,304 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/mac/videotoolbox_helpers.h" + +#include <array> +#include <vector> + +#include "base/big_endian.h" +#include "base/memory/scoped_ptr.h" + +namespace media { + +namespace video_toolbox { + +base::ScopedCFTypeRef<CFDictionaryRef> +DictionaryWithKeysAndValues(CFTypeRef* keys, CFTypeRef* values, size_t size) { + return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate( + kCFAllocatorDefault, keys, values, size, &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks)); +} + +base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key, + CFTypeRef value) { + CFTypeRef keys[1] = {key}; + CFTypeRef values[1] = {value}; + return DictionaryWithKeysAndValues(keys, values, 1); +} + +base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, size_t size) { + std::vector<CFNumberRef> numbers; + numbers.reserve(size); + for (const int* end = v + size; v < end; ++v) + numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v)); + base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate( + kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]), + numbers.size(), &kCFTypeArrayCallBacks)); + for (auto& number : numbers) { + CFRelease(number); + } + return array; +} + +base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegerAndFloat(int int_val, + float float_val) { + std::array<CFNumberRef, 2> numbers = { + {CFNumberCreate(nullptr, kCFNumberSInt32Type, &int_val), + CFNumberCreate(nullptr, kCFNumberFloat32Type, &float_val)}}; + base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate( + kCFAllocatorDefault, reinterpret_cast<const void**>(numbers.data()), + numbers.size(), &kCFTypeArrayCallBacks)); + for (auto& number : numbers) + CFRelease(number); + return array; +} + +// Wrapper class for writing AnnexBBuffer output into. +class AnnexBBuffer { + public: + virtual bool Reserve(size_t size) = 0; + virtual void Append(const char* s, size_t n) = 0; + virtual size_t GetReservedSize() const = 0; +}; + +class RawAnnexBBuffer : public AnnexBBuffer { + public: + RawAnnexBBuffer(char* annexb_buffer, size_t annexb_buffer_size) + : annexb_buffer_(annexb_buffer), + annexb_buffer_size_(annexb_buffer_size), + annexb_buffer_offset_(0) {} + bool Reserve(size_t size) override { + reserved_size_ = size; + return size <= annexb_buffer_size_; + } + void Append(const char* s, size_t n) override { + memcpy(annexb_buffer_ + annexb_buffer_offset_, s, n); + annexb_buffer_offset_ += n; + DCHECK_GE(reserved_size_, annexb_buffer_offset_); + } + size_t GetReservedSize() const override { return reserved_size_; } + + private: + char* annexb_buffer_; + size_t annexb_buffer_size_; + size_t annexb_buffer_offset_; + size_t reserved_size_; + + DISALLOW_IMPLICIT_CONSTRUCTORS(RawAnnexBBuffer); +}; + +class StringAnnexBBuffer : public AnnexBBuffer { + public: + explicit StringAnnexBBuffer(std::string* str_annexb_buffer) + : str_annexb_buffer_(str_annexb_buffer) {} + bool Reserve(size_t size) override { + str_annexb_buffer_->reserve(size); + return true; + } + void Append(const char* s, size_t n) override { + str_annexb_buffer_->append(s, n); + } + size_t GetReservedSize() const override { return str_annexb_buffer_->size(); } + + private: + std::string* str_annexb_buffer_; + DISALLOW_IMPLICIT_CONSTRUCTORS(StringAnnexBBuffer); +}; + +template <typename NalSizeType> +void CopyNalsToAnnexB(char* avcc_buffer, + const size_t avcc_size, + AnnexBBuffer* annexb_buffer) { + static_assert(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 || + sizeof(NalSizeType) == 4, + "NAL size type has unsupported size"); + static const char startcode_3[3] = {0, 0, 1}; + DCHECK(avcc_buffer); + DCHECK(annexb_buffer); + size_t bytes_left = avcc_size; + while (bytes_left > 0) { + DCHECK_GT(bytes_left, sizeof(NalSizeType)); + NalSizeType nal_size; + base::ReadBigEndian(avcc_buffer, &nal_size); + bytes_left -= sizeof(NalSizeType); + avcc_buffer += sizeof(NalSizeType); + + DCHECK_GE(bytes_left, nal_size); + annexb_buffer->Append(startcode_3, sizeof(startcode_3)); + annexb_buffer->Append(avcc_buffer, nal_size); + bytes_left -= nal_size; + avcc_buffer += nal_size; + } +} + +bool CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf, + AnnexBBuffer* annexb_buffer, + bool keyframe) { + // Perform two pass, one to figure out the total output size, and another to + // copy the data after having performed a single output allocation. Note that + // we'll allocate a bit more because we'll count 4 bytes instead of 3 for + // video NALs. + OSStatus status; + + // Get the sample buffer's block buffer and format description. + auto bb = CoreMediaGlue::CMSampleBufferGetDataBuffer(sbuf); + DCHECK(bb); + auto fdesc = CoreMediaGlue::CMSampleBufferGetFormatDescription(sbuf); + DCHECK(fdesc); + + size_t bb_size = CoreMediaGlue::CMBlockBufferGetDataLength(bb); + size_t total_bytes = bb_size; + + size_t pset_count; + int nal_size_field_bytes; + status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + fdesc, 0, nullptr, nullptr, &pset_count, &nal_size_field_bytes); + if (status == + CoreMediaGlue::kCMFormatDescriptionBridgeError_InvalidParameter) { + DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header"; + pset_count = 2; + nal_size_field_bytes = 4; + } else if (status != noErr) { + DLOG(ERROR) + << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " + << status; + return false; + } + + if (keyframe) { + const uint8_t* pset; + size_t pset_size; + for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) { + status = + CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + fdesc, pset_i, &pset, &pset_size, nullptr, nullptr); + if (status != noErr) { + DLOG(ERROR) + << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " + << status; + return false; + } + total_bytes += pset_size + nal_size_field_bytes; + } + } + + if (!annexb_buffer->Reserve(total_bytes)) { + DLOG(ERROR) << "Cannot fit encode output into bitstream buffer. Requested:" + << total_bytes; + return false; + } + + // Copy all parameter sets before keyframes. + if (keyframe) { + const uint8_t* pset; + size_t pset_size; + for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) { + status = + CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + fdesc, pset_i, &pset, &pset_size, nullptr, nullptr); + if (status != noErr) { + DLOG(ERROR) + << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " + << status; + return false; + } + static const char startcode_4[4] = {0, 0, 0, 1}; + annexb_buffer->Append(startcode_4, sizeof(startcode_4)); + annexb_buffer->Append(reinterpret_cast<const char*>(pset), pset_size); + } + } + + // Block buffers can be composed of non-contiguous chunks. For the sake of + // keeping this code simple, flatten non-contiguous block buffers. + base::ScopedCFTypeRef<CoreMediaGlue::CMBlockBufferRef> contiguous_bb( + bb, base::scoped_policy::RETAIN); + if (!CoreMediaGlue::CMBlockBufferIsRangeContiguous(bb, 0, 0)) { + contiguous_bb.reset(); + status = CoreMediaGlue::CMBlockBufferCreateContiguous( + kCFAllocatorDefault, bb, kCFAllocatorDefault, nullptr, 0, 0, 0, + contiguous_bb.InitializeInto()); + if (status != noErr) { + DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status; + return false; + } + } + + // Copy all the NAL units. In the process convert them from AVCC format + // (length header) to AnnexB format (start code). + char* bb_data; + status = CoreMediaGlue::CMBlockBufferGetDataPointer(contiguous_bb, 0, nullptr, + nullptr, &bb_data); + if (status != noErr) { + DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status; + return false; + } + + if (nal_size_field_bytes == 1) { + CopyNalsToAnnexB<uint8_t>(bb_data, bb_size, annexb_buffer); + } else if (nal_size_field_bytes == 2) { + CopyNalsToAnnexB<uint16_t>(bb_data, bb_size, annexb_buffer); + } else if (nal_size_field_bytes == 4) { + CopyNalsToAnnexB<uint32_t>(bb_data, bb_size, annexb_buffer); + } else { + NOTREACHED(); + } + return true; +} + +bool CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf, + bool keyframe, + std::string* annexb_buffer) { + StringAnnexBBuffer buffer(annexb_buffer); + return CopySampleBufferToAnnexBBuffer(sbuf, &buffer, keyframe); +} + +bool CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf, + bool keyframe, + size_t annexb_buffer_size, + char* annexb_buffer, + size_t* used_buffer_size) { + RawAnnexBBuffer buffer(annexb_buffer, annexb_buffer_size); + const bool copy_rv = CopySampleBufferToAnnexBBuffer(sbuf, &buffer, keyframe); + *used_buffer_size = buffer.GetReservedSize(); + return copy_rv; +} + +SessionPropertySetter::SessionPropertySetter( + base::ScopedCFTypeRef<VideoToolboxGlue::VTCompressionSessionRef> session, + const VideoToolboxGlue* const glue) + : session_(session), glue_(glue) {} + +SessionPropertySetter::~SessionPropertySetter() {} + +bool SessionPropertySetter::Set(CFStringRef key, int32_t value) { + DCHECK(session_); + DCHECK(glue_); + base::ScopedCFTypeRef<CFNumberRef> cfvalue( + CFNumberCreate(nullptr, kCFNumberSInt32Type, &value)); + return glue_->VTSessionSetProperty(session_, key, cfvalue) == noErr; +} + +bool SessionPropertySetter::Set(CFStringRef key, bool value) { + DCHECK(session_); + DCHECK(glue_); + CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse; + return glue_->VTSessionSetProperty(session_, key, cfvalue) == noErr; +} + +bool SessionPropertySetter::Set(CFStringRef key, CFStringRef value) { + DCHECK(session_); + DCHECK(glue_); + return glue_->VTSessionSetProperty(session_, key, value) == noErr; +} + +bool SessionPropertySetter::Set(CFStringRef key, CFArrayRef value) { + DCHECK(session_); + DCHECK(glue_); + return glue_->VTSessionSetProperty(session_, key, value) == noErr; +} + +} // namespace video_toolbox + +} // namespace media diff --git a/media/base/mac/videotoolbox_helpers.h b/media/base/mac/videotoolbox_helpers.h new file mode 100644 index 0000000..87d769e --- /dev/null +++ b/media/base/mac/videotoolbox_helpers.h @@ -0,0 +1,69 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_BASE_MAC_VIDEOTOOLBOX_HELPERS_H_ +#define MEDIA_BASE_MAC_VIDEOTOOLBOX_HELPERS_H_ + +#include "base/mac/scoped_cftyperef.h" +#include "media/base/mac/videotoolbox_glue.h" +#include "media/base/media_export.h" + +namespace media { + +namespace video_toolbox { + +// Create a CFDictionaryRef with the given keys and values. +MEDIA_EXPORT base::ScopedCFTypeRef<CFDictionaryRef> +DictionaryWithKeysAndValues(CFTypeRef* keys, CFTypeRef* values, size_t size); + +// Create a CFDictionaryRef with the given key and value. +MEDIA_EXPORT base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue( + CFTypeRef key, + CFTypeRef value); + +// Create a CFArrayRef with the given array of integers. +MEDIA_EXPORT base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, + size_t size); + +// Create a CFArrayRef with the given int and float values. +MEDIA_EXPORT base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegerAndFloat( + int int_val, + float float_val); + +// Copy a H.264 frame stored in a CM sample buffer to an Annex B buffer. Copies +// parameter sets for keyframes before the frame data as well. +MEDIA_EXPORT bool CopySampleBufferToAnnexBBuffer( + CoreMediaGlue::CMSampleBufferRef sbuf, + bool keyframe, + std::string* annexb_buffer); +MEDIA_EXPORT bool CopySampleBufferToAnnexBBuffer( + CoreMediaGlue::CMSampleBufferRef sbuf, + bool keyframe, + size_t annexb_buffer_size, + char* annexb_buffer, + size_t* used_buffer_size); + +// Helper class to add session properties to a VTCompressionSessionRef. +class MEDIA_EXPORT SessionPropertySetter { + public: + SessionPropertySetter( + base::ScopedCFTypeRef<VideoToolboxGlue::VTCompressionSessionRef> session, + const VideoToolboxGlue* const glue); + ~SessionPropertySetter(); + + bool Set(CFStringRef key, int32_t value); + bool Set(CFStringRef key, bool value); + bool Set(CFStringRef key, CFStringRef value); + bool Set(CFStringRef key, CFArrayRef value); + + private: + base::ScopedCFTypeRef<VideoToolboxGlue::VTCompressionSessionRef> session_; + const VideoToolboxGlue* glue_; +}; + +} // namespace video_toolbox + +} // namespace media + +#endif // MEDIA_BASE_MAC_VIDEOTOOLBOX_HELPERS_H_ |