// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "media/cast/sender/h264_vt_encoder.h" #include #include #include #include "base/big_endian.h" #include "base/bind.h" #include "base/bind_helpers.h" #include "base/location.h" #include "base/logging.h" #include "base/macros.h" #include "base/power_monitor/power_monitor.h" #include "base/synchronization/lock.h" #include "build/build_config.h" #include "media/base/mac/corevideo_glue.h" #include "media/base/mac/video_frame_mac.h" #include "media/cast/common/rtp_time.h" #include "media/cast/constants.h" #include "media/cast/sender/video_frame_factory.h" namespace media { namespace cast { namespace { // Container for the associated data of a video frame being processed. struct InProgressFrameEncode { const RtpTimeTicks rtp_timestamp; const base::TimeTicks reference_time; const VideoEncoder::FrameEncodedCallback frame_encoded_callback; InProgressFrameEncode(RtpTimeTicks rtp, base::TimeTicks r_time, VideoEncoder::FrameEncodedCallback callback) : rtp_timestamp(rtp), reference_time(r_time), frame_encoded_callback(callback) {} }; base::ScopedCFTypeRef DictionaryWithKeysAndValues(CFTypeRef* keys, CFTypeRef* values, size_t size) { return base::ScopedCFTypeRef(CFDictionaryCreate( kCFAllocatorDefault, keys, values, size, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks)); } base::ScopedCFTypeRef DictionaryWithKeyValue(CFTypeRef key, CFTypeRef value) { CFTypeRef keys[1] = {key}; CFTypeRef values[1] = {value}; return DictionaryWithKeysAndValues(keys, values, 1); } base::ScopedCFTypeRef ArrayWithIntegers(const int* v, size_t size) { std::vector numbers; numbers.reserve(size); for (const int* end = v + size; v < end; ++v) numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v)); base::ScopedCFTypeRef array(CFArrayCreate( kCFAllocatorDefault, reinterpret_cast(&numbers[0]), numbers.size(), &kCFTypeArrayCallBacks)); for (auto& number : numbers) { CFRelease(number); } return array; } template void CopyNalsToAnnexB(char* avcc_buffer, const size_t avcc_size, std::string* annexb_buffer) { static_assert(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 || sizeof(NalSizeType) == 4, "NAL size type has unsupported size"); static const char startcode_3[3] = {0, 0, 1}; DCHECK(avcc_buffer); DCHECK(annexb_buffer); size_t bytes_left = avcc_size; while (bytes_left > 0) { DCHECK_GT(bytes_left, sizeof(NalSizeType)); NalSizeType nal_size; base::ReadBigEndian(avcc_buffer, &nal_size); bytes_left -= sizeof(NalSizeType); avcc_buffer += sizeof(NalSizeType); DCHECK_GE(bytes_left, nal_size); annexb_buffer->append(startcode_3, sizeof(startcode_3)); annexb_buffer->append(avcc_buffer, nal_size); bytes_left -= nal_size; avcc_buffer += nal_size; } } // Copy a H.264 frame stored in a CM sample buffer to an Annex B buffer. Copies // parameter sets for keyframes before the frame data as well. void CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf, std::string* annexb_buffer, bool keyframe) { // Perform two pass, one to figure out the total output size, and another to // copy the data after having performed a single output allocation. Note that // we'll allocate a bit more because we'll count 4 bytes instead of 3 for // video NALs. OSStatus status; // Get the sample buffer's block buffer and format description. auto bb = CoreMediaGlue::CMSampleBufferGetDataBuffer(sbuf); DCHECK(bb); auto fdesc = CoreMediaGlue::CMSampleBufferGetFormatDescription(sbuf); DCHECK(fdesc); size_t bb_size = CoreMediaGlue::CMBlockBufferGetDataLength(bb); size_t total_bytes = bb_size; size_t pset_count; int nal_size_field_bytes; status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( fdesc, 0, nullptr, nullptr, &pset_count, &nal_size_field_bytes); if (status == CoreMediaGlue::kCMFormatDescriptionBridgeError_InvalidParameter) { DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header"; pset_count = 2; nal_size_field_bytes = 4; } else if (status != noErr) { DLOG(ERROR) << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " << status; return; } if (keyframe) { const uint8_t* pset; size_t pset_size; for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) { status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( fdesc, pset_i, &pset, &pset_size, nullptr, nullptr); if (status != noErr) { DLOG(ERROR) << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " << status; return; } total_bytes += pset_size + nal_size_field_bytes; } } annexb_buffer->reserve(total_bytes); // Copy all parameter sets before keyframes. if (keyframe) { const uint8_t* pset; size_t pset_size; for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) { status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( fdesc, pset_i, &pset, &pset_size, nullptr, nullptr); if (status != noErr) { DLOG(ERROR) << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " << status; return; } static const char startcode_4[4] = {0, 0, 0, 1}; annexb_buffer->append(startcode_4, sizeof(startcode_4)); annexb_buffer->append(reinterpret_cast(pset), pset_size); } } // Block buffers can be composed of non-contiguous chunks. For the sake of // keeping this code simple, flatten non-contiguous block buffers. base::ScopedCFTypeRef contiguous_bb( bb, base::scoped_policy::RETAIN); if (!CoreMediaGlue::CMBlockBufferIsRangeContiguous(bb, 0, 0)) { contiguous_bb.reset(); status = CoreMediaGlue::CMBlockBufferCreateContiguous( kCFAllocatorDefault, bb, kCFAllocatorDefault, nullptr, 0, 0, 0, contiguous_bb.InitializeInto()); if (status != noErr) { DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status; return; } } // Copy all the NAL units. In the process convert them from AVCC format // (length header) to AnnexB format (start code). char* bb_data; status = CoreMediaGlue::CMBlockBufferGetDataPointer(contiguous_bb, 0, nullptr, nullptr, &bb_data); if (status != noErr) { DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status; return; } if (nal_size_field_bytes == 1) { CopyNalsToAnnexB(bb_data, bb_size, annexb_buffer); } else if (nal_size_field_bytes == 2) { CopyNalsToAnnexB(bb_data, bb_size, annexb_buffer); } else if (nal_size_field_bytes == 4) { CopyNalsToAnnexB(bb_data, bb_size, annexb_buffer); } else { NOTREACHED(); } } } // namespace class H264VideoToolboxEncoder::VideoFrameFactoryImpl : public base::RefCountedThreadSafe, public VideoFrameFactory { public: // Type that proxies the VideoFrameFactory interface to this class. class Proxy; VideoFrameFactoryImpl(const base::WeakPtr& encoder, const scoped_refptr& cast_environment) : encoder_(encoder), cast_environment_(cast_environment) {} scoped_refptr MaybeCreateFrame( const gfx::Size& frame_size, base::TimeDelta timestamp) final { if (frame_size.IsEmpty()) { DVLOG(1) << "Rejecting empty video frame."; return nullptr; } base::AutoLock auto_lock(lock_); // If the pool size does not match, speculatively reset the encoder to use // the new size and return null. Cache the new frame size right away and // toss away the pixel buffer pool to avoid spurious tasks until the encoder // is done resetting. if (frame_size != pool_frame_size_) { DVLOG(1) << "MaybeCreateFrame: Detected frame size change."; cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, base::Bind(&H264VideoToolboxEncoder::UpdateFrameSize, encoder_, frame_size)); pool_frame_size_ = frame_size; pool_.reset(); return nullptr; } if (!pool_) { DVLOG(1) << "MaybeCreateFrame: No pixel buffer pool."; return nullptr; } // Allocate a pixel buffer from the pool and return a wrapper VideoFrame. base::ScopedCFTypeRef buffer; auto status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool_, buffer.InitializeInto()); if (status != kCVReturnSuccess) { DLOG(ERROR) << "CVPixelBufferPoolCreatePixelBuffer failed: " << status; return nullptr; } DCHECK(buffer); return VideoFrame::WrapCVPixelBuffer(buffer, timestamp); } void Update(const base::ScopedCFTypeRef& pool, const gfx::Size& frame_size) { base::AutoLock auto_lock(lock_); pool_ = pool; pool_frame_size_ = frame_size; } private: friend class base::RefCountedThreadSafe; ~VideoFrameFactoryImpl() final {} base::Lock lock_; base::ScopedCFTypeRef pool_; gfx::Size pool_frame_size_; // Weak back reference to the encoder and the cast envrionment so we can // message the encoder when the frame size changes. const base::WeakPtr encoder_; const scoped_refptr cast_environment_; DISALLOW_COPY_AND_ASSIGN(VideoFrameFactoryImpl); }; class H264VideoToolboxEncoder::VideoFrameFactoryImpl::Proxy : public VideoFrameFactory { public: explicit Proxy( const scoped_refptr& video_frame_factory) : video_frame_factory_(video_frame_factory) { DCHECK(video_frame_factory_); } scoped_refptr MaybeCreateFrame( const gfx::Size& frame_size, base::TimeDelta timestamp) final { return video_frame_factory_->MaybeCreateFrame(frame_size, timestamp); } private: ~Proxy() final {} const scoped_refptr video_frame_factory_; DISALLOW_COPY_AND_ASSIGN(Proxy); }; // static bool H264VideoToolboxEncoder::IsSupported( const VideoSenderConfig& video_config) { return video_config.codec == CODEC_VIDEO_H264 && VideoToolboxGlue::Get(); } H264VideoToolboxEncoder::H264VideoToolboxEncoder( const scoped_refptr& cast_environment, const VideoSenderConfig& video_config, const StatusChangeCallback& status_change_cb) : cast_environment_(cast_environment), videotoolbox_glue_(VideoToolboxGlue::Get()), video_config_(video_config), status_change_cb_(status_change_cb), last_frame_id_(kFirstFrameId - 1), encode_next_frame_as_keyframe_(false), power_suspended_(false), weak_factory_(this) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); DCHECK(!status_change_cb_.is_null()); OperationalStatus operational_status = H264VideoToolboxEncoder::IsSupported(video_config) ? STATUS_INITIALIZED : STATUS_UNSUPPORTED_CODEC; cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, base::Bind(status_change_cb_, operational_status)); if (operational_status == STATUS_INITIALIZED) { // Create the shared video frame factory. It persists for the combined // lifetime of the encoder and all video frame factory proxies created by // |CreateVideoFrameFactory| that reference it. video_frame_factory_ = scoped_refptr(new VideoFrameFactoryImpl( weak_factory_.GetWeakPtr(), cast_environment_)); // Register for power state changes. auto power_monitor = base::PowerMonitor::Get(); if (power_monitor) { power_monitor->AddObserver(this); VLOG(1) << "Registered for power state changes."; } else { DLOG(WARNING) << "No power monitor. Process suspension will invalidate " "the encoder."; } } } H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { DestroyCompressionSession(); // If video_frame_factory_ is not null, the encoder registered for power state // changes in the ctor and it must now unregister. if (video_frame_factory_) { auto power_monitor = base::PowerMonitor::Get(); if (power_monitor) power_monitor->RemoveObserver(this); } } void H264VideoToolboxEncoder::ResetCompressionSession() { DCHECK(thread_checker_.CalledOnValidThread()); // Ignore reset requests while power suspended. if (power_suspended_) return; // Notify that we're resetting the encoder. cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, base::Bind(status_change_cb_, STATUS_CODEC_REINIT_PENDING)); // Destroy the current session, if any. DestroyCompressionSession(); // On OS X, allow the hardware encoder. Don't require it, it does not support // all configurations (some of which are used for testing). base::ScopedCFTypeRef encoder_spec; #if !defined(OS_IOS) encoder_spec = DictionaryWithKeyValue( videotoolbox_glue_ ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder(), kCFBooleanTrue); #endif // Force 420v so that clients can easily use these buffers as GPU textures. const int format[] = { CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange}; // Keep these attachment settings in-sync with those in ConfigureSession(). CFTypeRef attachments_keys[] = {kCVImageBufferColorPrimariesKey, kCVImageBufferTransferFunctionKey, kCVImageBufferYCbCrMatrixKey}; CFTypeRef attachments_values[] = {kCVImageBufferColorPrimaries_ITU_R_709_2, kCVImageBufferTransferFunction_ITU_R_709_2, kCVImageBufferYCbCrMatrix_ITU_R_709_2}; CFTypeRef buffer_attributes_keys[] = {kCVPixelBufferPixelFormatTypeKey, kCVBufferPropagatedAttachmentsKey}; CFTypeRef buffer_attributes_values[] = { ArrayWithIntegers(format, arraysize(format)).release(), DictionaryWithKeysAndValues(attachments_keys, attachments_values, arraysize(attachments_keys)).release()}; const base::ScopedCFTypeRef buffer_attributes = DictionaryWithKeysAndValues(buffer_attributes_keys, buffer_attributes_values, arraysize(buffer_attributes_keys)); for (auto& v : buffer_attributes_values) CFRelease(v); // Create the compression session. // Note that the encoder object is given to the compression session as the // callback context using a raw pointer. The C API does not allow us to use a // smart pointer, nor is this encoder ref counted. However, this is still // safe, because we 1) we own the compression session and 2) we tear it down // safely. When destructing the encoder, the compression session is flushed // and invalidated. Internally, VideoToolbox will join all of its threads // before returning to the client. Therefore, when control returns to us, we // are guaranteed that the output callback will not execute again. OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate( kCFAllocatorDefault, frame_size_.width(), frame_size_.height(), CoreMediaGlue::kCMVideoCodecType_H264, encoder_spec, buffer_attributes, nullptr /* compressedDataAllocator */, &H264VideoToolboxEncoder::CompressionCallback, reinterpret_cast(this), compression_session_.InitializeInto()); if (status != noErr) { DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status; // Notify that reinitialization has failed. cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, base::Bind(status_change_cb_, STATUS_CODEC_INIT_FAILED)); return; } // Configure the session (apply session properties based on the current state // of the encoder, experimental tuning and requirements). ConfigureCompressionSession(); // Update the video frame factory. base::ScopedCFTypeRef pool( videotoolbox_glue_->VTCompressionSessionGetPixelBufferPool( compression_session_), base::scoped_policy::RETAIN); video_frame_factory_->Update(pool, frame_size_); // Notify that reinitialization is done. cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, base::Bind(status_change_cb_, STATUS_INITIALIZED)); } void H264VideoToolboxEncoder::ConfigureCompressionSession() { SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(), videotoolbox_glue_->kVTProfileLevel_H264_Main_AutoLevel()); SetSessionProperty(videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), true); SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(), false); SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_MaxKeyFrameInterval(), 240); SetSessionProperty( videotoolbox_glue_ ->kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration(), 240); // TODO(jfroy): implement better bitrate control // https://crbug.com/425352 SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(), (video_config_.min_bitrate + video_config_.max_bitrate) / 2); SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(), video_config_.max_frame_rate); // Keep these attachment settings in-sync with those in Initialize(). SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(), kCVImageBufferColorPrimaries_ITU_R_709_2); SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_TransferFunction(), kCVImageBufferTransferFunction_ITU_R_709_2); SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_YCbCrMatrix(), kCVImageBufferYCbCrMatrix_ITU_R_709_2); if (video_config_.max_number_of_video_buffers_used > 0) { SetSessionProperty( videotoolbox_glue_->kVTCompressionPropertyKey_MaxFrameDelayCount(), video_config_.max_number_of_video_buffers_used); } } void H264VideoToolboxEncoder::DestroyCompressionSession() { DCHECK(thread_checker_.CalledOnValidThread()); // If the compression session exists, invalidate it. This blocks until all // pending output callbacks have returned and any internal threads have // joined, ensuring no output callback ever sees a dangling encoder pointer. // // Before destroying the compression session, the video frame factory's pool // is updated to null so that no thread will produce new video frames via the // factory until a new compression session is created. The current frame size // is passed to prevent the video frame factory from posting |UpdateFrameSize| // tasks. Indeed, |DestroyCompressionSession| is either called from // |ResetCompressionSession|, in which case a new pool and frame size will be // set, or from callsites that require that there be no compression session // (ex: the dtor). if (compression_session_) { video_frame_factory_->Update( base::ScopedCFTypeRef(nullptr), frame_size_); videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_); compression_session_.reset(); } } bool H264VideoToolboxEncoder::EncodeVideoFrame( const scoped_refptr& video_frame, const base::TimeTicks& reference_time, const FrameEncodedCallback& frame_encoded_callback) { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(!frame_encoded_callback.is_null()); // Reject empty video frames. const gfx::Size frame_size = video_frame->visible_rect().size(); if (frame_size.IsEmpty()) { DVLOG(1) << "Rejecting empty video frame."; return false; } // Handle frame size changes. This will reset the compression session. if (frame_size != frame_size_) { DVLOG(1) << "EncodeVideoFrame: Detected frame size change."; UpdateFrameSize(frame_size); } // Need a compression session to continue. if (!compression_session_) { DLOG(ERROR) << "No compression session."; return false; } // Wrap the VideoFrame in a CVPixelBuffer. In all cases, no data will be // copied. If the VideoFrame was created by this encoder's video frame // factory, then the returned CVPixelBuffer will have been obtained from the // compression session's pixel buffer pool. This will eliminate a copy of the // frame into memory visible by the hardware encoder. The VideoFrame's // lifetime is extended for the lifetime of the returned CVPixelBuffer. auto pixel_buffer = media::WrapVideoFrameInCVPixelBuffer(*video_frame); if (!pixel_buffer) { DLOG(ERROR) << "WrapVideoFrameInCVPixelBuffer failed."; return false; } // Convert the frame timestamp to CMTime. auto timestamp_cm = CoreMediaGlue::CMTimeMake( (reference_time - base::TimeTicks()).InMicroseconds(), USEC_PER_SEC); // Wrap information we'll need after the frame is encoded in a heap object. // We'll get the pointer back from the VideoToolbox completion callback. scoped_ptr request(new InProgressFrameEncode( RtpTimeTicks::FromTimeDelta(video_frame->timestamp(), kVideoFrequency), reference_time, frame_encoded_callback)); // Build a suitable frame properties dictionary for keyframes. base::ScopedCFTypeRef frame_props; if (encode_next_frame_as_keyframe_) { frame_props = DictionaryWithKeyValue( videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(), kCFBooleanTrue); encode_next_frame_as_keyframe_ = false; } // Submit the frame to the compression session. The function returns as soon // as the frame has been enqueued. OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame( compression_session_, pixel_buffer, timestamp_cm, CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props, reinterpret_cast(request.release()), nullptr); if (status != noErr) { DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status; return false; } return true; } void H264VideoToolboxEncoder::UpdateFrameSize(const gfx::Size& size_needed) { DCHECK(thread_checker_.CalledOnValidThread()); // Our video frame factory posts a task to update the frame size when its // cache of the frame size differs from what the client requested. To avoid // spurious encoder resets, check again here. if (size_needed == frame_size_) { DCHECK(compression_session_); return; } VLOG(1) << "Resetting compression session (for frame size change from " << frame_size_.ToString() << " to " << size_needed.ToString() << ")."; // If there is an existing session, finish every pending frame. if (compression_session_) { EmitFrames(); } // Store the new frame size. frame_size_ = size_needed; // Reset the compression session. ResetCompressionSession(); } void H264VideoToolboxEncoder::SetBitRate(int /*new_bit_rate*/) { DCHECK(thread_checker_.CalledOnValidThread()); // VideoToolbox does not seem to support bitrate reconfiguration. } void H264VideoToolboxEncoder::GenerateKeyFrame() { DCHECK(thread_checker_.CalledOnValidThread()); encode_next_frame_as_keyframe_ = true; } scoped_ptr H264VideoToolboxEncoder::CreateVideoFrameFactory() { DCHECK(thread_checker_.CalledOnValidThread()); return scoped_ptr( new VideoFrameFactoryImpl::Proxy(video_frame_factory_)); } void H264VideoToolboxEncoder::EmitFrames() { DCHECK(thread_checker_.CalledOnValidThread()); if (!compression_session_) return; OSStatus status = videotoolbox_glue_->VTCompressionSessionCompleteFrames( compression_session_, CoreMediaGlue::CMTime{0, 0, 0, 0}); if (status != noErr) { DLOG(ERROR) << " VTCompressionSessionCompleteFrames failed: " << status; } } void H264VideoToolboxEncoder::OnSuspend() { VLOG(1) << "OnSuspend: Emitting all frames and destroying compression session."; EmitFrames(); DestroyCompressionSession(); power_suspended_ = true; } void H264VideoToolboxEncoder::OnResume() { power_suspended_ = false; // Reset the compression session only if the frame size is not zero (which // will obviously fail). It is possible for the frame size to be zero if no // frame was submitted for encoding or requested from the video frame factory // before suspension. if (!frame_size_.IsEmpty()) { VLOG(1) << "OnResume: Resetting compression session."; ResetCompressionSession(); } } bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, int32_t value) { base::ScopedCFTypeRef cfvalue( CFNumberCreate(nullptr, kCFNumberSInt32Type, &value)); return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key, cfvalue) == noErr; } bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, bool value) { CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse; return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key, cfvalue) == noErr; } bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, CFStringRef value) { return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key, value) == noErr; } void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, void* request_opaque, OSStatus status, VTEncodeInfoFlags info, CMSampleBufferRef sbuf) { auto encoder = reinterpret_cast(encoder_opaque); const scoped_ptr request( reinterpret_cast(request_opaque)); bool keyframe = false; bool has_frame_data = false; if (status != noErr) { DLOG(ERROR) << " encoding failed: " << status; encoder->cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, base::Bind(encoder->status_change_cb_, STATUS_CODEC_RUNTIME_ERROR)); } else if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) { DVLOG(2) << " frame dropped"; } else { auto sample_attachments = static_cast(CFArrayGetValueAtIndex( CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0)); // If the NotSync key is not present, it implies Sync, which indicates a // keyframe (at least I think, VT documentation is, erm, sparse). Could // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false. keyframe = !CFDictionaryContainsKey( sample_attachments, CoreMediaGlue::kCMSampleAttachmentKey_NotSync()); has_frame_data = true; } // Increment the encoder-scoped frame id and assign the new value to this // frame. VideoToolbox calls the output callback serially, so this is safe. const uint32_t frame_id = ++encoder->last_frame_id_; scoped_ptr encoded_frame(new SenderEncodedFrame()); encoded_frame->frame_id = frame_id; encoded_frame->reference_time = request->reference_time; encoded_frame->rtp_timestamp = request->rtp_timestamp; if (keyframe) { encoded_frame->dependency = EncodedFrame::KEY; encoded_frame->referenced_frame_id = frame_id; } else { encoded_frame->dependency = EncodedFrame::DEPENDENT; // H.264 supports complex frame reference schemes (multiple reference // frames, slice references, backward and forward references, etc). Cast // doesn't support the concept of forward-referencing frame dependencies or // multiple frame dependencies; so pretend that all frames are only // decodable after their immediately preceding frame is decoded. This will // ensure a Cast receiver only attempts to decode the frames sequentially // and in order. Furthermore, the encoder is configured to never use forward // references (see |kVTCompressionPropertyKey_AllowFrameReordering|). There // is no way to prevent multiple reference frames. encoded_frame->referenced_frame_id = frame_id - 1; } if (has_frame_data) CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe); // TODO(miu): Compute and populate the |deadline_utilization| and // |lossy_utilization| performance metrics in |encoded_frame|. encoded_frame->encode_completion_time = encoder->cast_environment_->Clock()->NowTicks(); encoder->cast_environment_->PostTask( CastEnvironment::MAIN, FROM_HERE, base::Bind(request->frame_encoded_callback, base::Passed(&encoded_frame))); } } // namespace cast } // namespace media