// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "media/cast/sender/vp8_encoder.h" #include "base/logging.h" #include "media/base/video_frame.h" #include "media/cast/cast_defines.h" #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" namespace media { namespace cast { namespace { // After a pause in the video stream, what is the maximum duration amount to // pass to the encoder for the next frame (in terms of 1/max_fps sized periods)? // This essentially controls the encoded size of the first frame that follows a // pause in the video stream. const int kRestartFramePeriods = 3; } // namespace Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config) : cast_config_(video_config), use_multiple_video_buffers_( cast_config_.max_number_of_video_buffers_used == kNumberOfVp8VideoBuffers), key_frame_requested_(true), bitrate_kbit_(cast_config_.start_bitrate / 1000), last_encoded_frame_id_(kStartFrameId), last_acked_frame_id_(kStartFrameId), undroppable_frames_(0) { config_.g_timebase.den = 0; // Not initialized. for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { buffer_state_[i].frame_id = last_encoded_frame_id_; buffer_state_[i].state = kBufferStartState; } // VP8 have 3 buffers available for prediction, with // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency // however in this mode we can not skip frames in the receiver to catch up // after a temporary network outage; with max_number_of_video_buffers_used // set to 3 we allow 2 frames to be skipped by the receiver without error // propagation. DCHECK(cast_config_.max_number_of_video_buffers_used == 1 || cast_config_.max_number_of_video_buffers_used == kNumberOfVp8VideoBuffers) << "Invalid argument"; thread_checker_.DetachFromThread(); } Vp8Encoder::~Vp8Encoder() { DCHECK(thread_checker_.CalledOnValidThread()); if (is_initialized()) vpx_codec_destroy(&encoder_); } void Vp8Encoder::Initialize() { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(!is_initialized()); // The encoder will be created/configured when the first frame encode is // requested. } void Vp8Encoder::ConfigureForNewFrameSize(const gfx::Size& frame_size) { if (is_initialized()) { // Workaround for VP8 bug: If the new size is strictly less-than-or-equal to // the old size, in terms of area, the existing encoder instance can // continue. Otherwise, completely tear-down and re-create a new encoder to // avoid a shutdown crash. if (frame_size.GetArea() <= gfx::Size(config_.g_w, config_.g_h).GetArea() && !use_multiple_video_buffers_) { DVLOG(1) << "Continuing to use existing encoder at smaller frame size: " << gfx::Size(config_.g_w, config_.g_h).ToString() << " --> " << frame_size.ToString(); config_.g_w = frame_size.width(); config_.g_h = frame_size.height(); if (vpx_codec_enc_config_set(&encoder_, &config_) == VPX_CODEC_OK) return; DVLOG(1) << "libvpx rejected the attempt to use a smaller frame size in " "the current instance."; } DVLOG(1) << "Destroying/Re-Creating encoder for larger frame size: " << gfx::Size(config_.g_w, config_.g_h).ToString() << " --> " << frame_size.ToString(); vpx_codec_destroy(&encoder_); } else { DVLOG(1) << "Creating encoder for the first frame; size: " << frame_size.ToString(); } // Reset multi-buffer mode state. last_acked_frame_id_ = last_encoded_frame_id_; undroppable_frames_ = 0; for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { buffer_state_[i].frame_id = last_encoded_frame_id_; buffer_state_[i].state = kBufferStartState; } // Populate encoder configuration with default values. CHECK_EQ(vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config_, 0), VPX_CODEC_OK); config_.g_threads = cast_config_.number_of_encode_threads; config_.g_w = frame_size.width(); config_.g_h = frame_size.height(); // Set the timebase to match that of base::TimeDelta. config_.g_timebase.num = 1; config_.g_timebase.den = base::Time::kMicrosecondsPerSecond; if (use_multiple_video_buffers_) { // We must enable error resilience when we use multiple buffers, due to // codec requirements. config_.g_error_resilient = 1; } // |g_pass| and |g_lag_in_frames| must be "one pass" and zero, respectively, // in order for VP8 to support changing frame sizes during encoding: config_.g_pass = VPX_RC_ONE_PASS; config_.g_lag_in_frames = 0; // Immediate data output for each frame. // Rate control settings. config_.rc_dropframe_thresh = 0; // The encoder may not drop any frames. config_.rc_resize_allowed = 0; // TODO(miu): Why not? Investigate this. config_.rc_end_usage = VPX_CBR; config_.rc_target_bitrate = bitrate_kbit_; config_.rc_min_quantizer = cast_config_.min_qp; config_.rc_max_quantizer = cast_config_.max_qp; // TODO(miu): Revisit these now that the encoder is being successfully // micro-managed. config_.rc_undershoot_pct = 100; config_.rc_overshoot_pct = 15; // TODO(miu): Document why these rc_buf_*_sz values were chosen and/or // research for better values. Should they be computed from the target // playout delay? config_.rc_buf_initial_sz = 500; config_.rc_buf_optimal_sz = 600; config_.rc_buf_sz = 1000; config_.kf_mode = VPX_KF_DISABLED; vpx_codec_flags_t flags = 0; CHECK_EQ(vpx_codec_enc_init(&encoder_, vpx_codec_vp8_cx(), &config_, flags), VPX_CODEC_OK); // Raise the threshold for considering macroblocks as static. The default is // zero, so this setting makes the encoder less sensitive to motion. This // lowers the probability of needing to utilize more CPU to search for motion // vectors. CHECK_EQ(vpx_codec_control(&encoder_, VP8E_SET_STATIC_THRESHOLD, 1), VPX_CODEC_OK); // Improve quality by enabling sets of codec features that utilize more CPU. // The default is zero, with increasingly more CPU to be used as the value is // more negative. // TODO(miu): Document why this value was chosen and expected behaviors. // Should this be dynamic w.r.t. hardware performance? CHECK_EQ(vpx_codec_control(&encoder_, VP8E_SET_CPUUSED, -6), VPX_CODEC_OK); } void Vp8Encoder::Encode(const scoped_refptr& video_frame, const base::TimeTicks& reference_time, SenderEncodedFrame* encoded_frame) { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(encoded_frame); // Note: This is used to compute the |deadline_utilization| and so it uses the // real-world clock instead of the CastEnvironment clock, the latter of which // might be simulated. const base::TimeTicks start_time = base::TimeTicks::Now(); // Initialize on-demand. Later, if the video frame size has changed, update // the encoder configuration. const gfx::Size frame_size = video_frame->visible_rect().size(); if (!is_initialized() || gfx::Size(config_.g_w, config_.g_h) != frame_size) ConfigureForNewFrameSize(frame_size); uint32 latest_frame_id_to_reference; Vp8Buffers buffer_to_update; vpx_codec_flags_t flags = 0; if (key_frame_requested_) { flags = VPX_EFLAG_FORCE_KF; // Self reference. latest_frame_id_to_reference = last_encoded_frame_id_ + 1; // We can pick any buffer as buffer_to_update since we update // them all. buffer_to_update = kLastBuffer; } else { // Reference all acked frames (buffers). latest_frame_id_to_reference = GetCodecReferenceFlags(&flags); buffer_to_update = GetNextBufferToUpdate(); GetCodecUpdateFlags(buffer_to_update, &flags); } // Wrapper for vpx_codec_encode() to access the YUV data in the |video_frame|. // Only the VISIBLE rectangle within |video_frame| is exposed to the codec. vpx_image_t vpx_image; vpx_image_t* const result = vpx_img_wrap( &vpx_image, VPX_IMG_FMT_I420, frame_size.width(), frame_size.height(), 1, video_frame->data(VideoFrame::kYPlane)); DCHECK_EQ(result, &vpx_image); vpx_image.planes[VPX_PLANE_Y] = video_frame->visible_data(VideoFrame::kYPlane); vpx_image.planes[VPX_PLANE_U] = video_frame->visible_data(VideoFrame::kUPlane); vpx_image.planes[VPX_PLANE_V] = video_frame->visible_data(VideoFrame::kVPlane); vpx_image.stride[VPX_PLANE_Y] = video_frame->stride(VideoFrame::kYPlane); vpx_image.stride[VPX_PLANE_U] = video_frame->stride(VideoFrame::kUPlane); vpx_image.stride[VPX_PLANE_V] = video_frame->stride(VideoFrame::kVPlane); // The frame duration given to the VP8 codec affects a number of important // behaviors, including: per-frame bandwidth, CPU time spent encoding, // temporal quality trade-offs, and key/golden/alt-ref frame generation // intervals. Bound the prediction to account for the fact that the frame // rate can be highly variable, including long pauses in the video stream. const base::TimeDelta minimum_frame_duration = base::TimeDelta::FromSecondsD(1.0 / cast_config_.max_frame_rate); const base::TimeDelta maximum_frame_duration = base::TimeDelta::FromSecondsD(static_cast(kRestartFramePeriods) / cast_config_.max_frame_rate); base::TimeDelta predicted_frame_duration; if (!video_frame->metadata()->GetTimeDelta( media::VideoFrameMetadata::FRAME_DURATION, &predicted_frame_duration) || predicted_frame_duration <= base::TimeDelta()) { // The source of the video frame did not provide the frame duration. Use // the actual amount of time between the current and previous frame as a // prediction for the next frame's duration. predicted_frame_duration = video_frame->timestamp() - last_frame_timestamp_; } predicted_frame_duration = std::max(minimum_frame_duration, std::min(maximum_frame_duration, predicted_frame_duration)); last_frame_timestamp_ = video_frame->timestamp(); // Encode the frame. The presentation time stamp argument here is fixed to // zero to force the encoder to base its single-frame bandwidth calculations // entirely on |predicted_frame_duration| and the target bitrate setting being // micro-managed via calls to UpdateRates(). CHECK_EQ(vpx_codec_encode(&encoder_, &vpx_image, 0, predicted_frame_duration.InMicroseconds(), flags, VPX_DL_REALTIME), VPX_CODEC_OK) << "BUG: Invalid arguments passed to vpx_codec_encode()."; // Pull data from the encoder, populating a new EncodedFrame. encoded_frame->frame_id = ++last_encoded_frame_id_; const vpx_codec_cx_pkt_t* pkt = NULL; vpx_codec_iter_t iter = NULL; while ((pkt = vpx_codec_get_cx_data(&encoder_, &iter)) != NULL) { if (pkt->kind != VPX_CODEC_CX_FRAME_PKT) continue; if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) { // TODO(hubbe): Replace "dependency" with a "bool is_key_frame". encoded_frame->dependency = EncodedFrame::KEY; encoded_frame->referenced_frame_id = encoded_frame->frame_id; } else { encoded_frame->dependency = EncodedFrame::DEPENDENT; // Frame dependencies could theoretically be relaxed by looking for the // VPX_FRAME_IS_DROPPABLE flag, but in recent testing (Oct 2014), this // flag never seems to be set. encoded_frame->referenced_frame_id = latest_frame_id_to_reference; } encoded_frame->rtp_timestamp = TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency); encoded_frame->reference_time = reference_time; encoded_frame->data.assign( static_cast(pkt->data.frame.buf), static_cast(pkt->data.frame.buf) + pkt->data.frame.sz); break; // Done, since all data is provided in one CX_FRAME_PKT packet. } DCHECK(!encoded_frame->data.empty()) << "BUG: Encoder must provide data since lagged encoding is disabled."; // Compute deadline utilization as the real-world time elapsed divided by the // frame duration. const base::TimeDelta processing_time = base::TimeTicks::Now() - start_time; encoded_frame->deadline_utilization = processing_time.InSecondsF() / predicted_frame_duration.InSecondsF(); // Compute lossy utilization. The VP8 encoder took an estimated guess at what // quantizer value would produce an encoded frame size as close to the target // as possible. Now that the frame has been encoded and the number of bytes // is known, the perfect quantizer value (i.e., the one that should have been // used) can be determined. This perfect quantizer is then normalized and // used as the lossy utilization. const double actual_bitrate = encoded_frame->data.size() * 8.0 / predicted_frame_duration.InSecondsF(); const double target_bitrate = 1000.0 * config_.rc_target_bitrate; DCHECK_GT(target_bitrate, 0.0); const double bitrate_utilization = actual_bitrate / target_bitrate; int quantizer = -1; CHECK_EQ(vpx_codec_control(&encoder_, VP8E_GET_LAST_QUANTIZER_64, &quantizer), VPX_CODEC_OK); const double perfect_quantizer = bitrate_utilization * std::max(0, quantizer); // Side note: If it was possible for the encoder to encode within the target // number of bytes, the |perfect_quantizer| will be in the range [0.0,63.0]. // If it was never possible, the value will be greater than 63.0. encoded_frame->lossy_utilization = perfect_quantizer / 63.0; DVLOG(2) << "VP8 encoded frame_id " << encoded_frame->frame_id << ", sized: " << encoded_frame->data.size() << ", deadline_utilization: " << encoded_frame->deadline_utilization << ", lossy_utilization: " << encoded_frame->lossy_utilization << " (quantizer chosen by the encoder was " << quantizer << ')'; if (encoded_frame->dependency == EncodedFrame::KEY) { key_frame_requested_ = false; for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { buffer_state_[i].state = kBufferSent; buffer_state_[i].frame_id = encoded_frame->frame_id; } } else { if (buffer_to_update != kNoBuffer) { buffer_state_[buffer_to_update].state = kBufferSent; buffer_state_[buffer_to_update].frame_id = encoded_frame->frame_id; } } } uint32 Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) { if (!use_multiple_video_buffers_) return last_encoded_frame_id_; const uint32 kMagicFrameOffset = 512; // We set latest_frame_to_reference to an old frame so that // IsNewerFrameId will work correctly. uint32 latest_frame_to_reference = last_encoded_frame_id_ - kMagicFrameOffset; // Reference all acked frames. // TODO(hubbe): We may also want to allow references to the // last encoded frame, if that frame was assigned to a buffer. for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { if (buffer_state_[i].state == kBufferAcked) { if (IsNewerFrameId(buffer_state_[i].frame_id, latest_frame_to_reference)) { latest_frame_to_reference = buffer_state_[i].frame_id; } } else { switch (i) { case kAltRefBuffer: *flags |= VP8_EFLAG_NO_REF_ARF; break; case kGoldenBuffer: *flags |= VP8_EFLAG_NO_REF_GF; break; case kLastBuffer: *flags |= VP8_EFLAG_NO_REF_LAST; break; } } } if (latest_frame_to_reference == last_encoded_frame_id_ - kMagicFrameOffset) { // We have nothing to reference, it's kind of like a key frame, // but doesn't reset buffers. latest_frame_to_reference = last_encoded_frame_id_ + 1; } return latest_frame_to_reference; } Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() { if (!use_multiple_video_buffers_) return kNoBuffer; // The goal here is to make sure that we always keep one ACKed // buffer while trying to get an ACK for a newer buffer as we go. // Here are the rules for which buffer to select for update: // 1. If there is a buffer in state kStartState, use it. // 2. If there is a buffer other than the oldest buffer // which is Acked, use the oldest buffer. // 3. If there are Sent buffers which are older than // latest_acked_frame_, use the oldest one. // 4. If all else fails, just overwrite the newest buffer, // but no more than 3 times in a row. // TODO(hubbe): Figure out if 3 is optimal. // Note, rule 1-3 describe cases where there is a "free" buffer // that we can use. Rule 4 describes what happens when there is // no free buffer available. // Buffers, sorted from oldest frame to newest. Vp8Encoder::Vp8Buffers buffers[kNumberOfVp8VideoBuffers]; for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { Vp8Encoder::Vp8Buffers buffer = static_cast(i); // Rule 1 if (buffer_state_[buffer].state == kBufferStartState) { undroppable_frames_ = 0; return buffer; } buffers[buffer] = buffer; } // Sorting three elements with selection sort. for (int i = 0; i < kNumberOfVp8VideoBuffers - 1; i++) { for (int j = i + 1; j < kNumberOfVp8VideoBuffers; j++) { if (IsOlderFrameId(buffer_state_[buffers[j]].frame_id, buffer_state_[buffers[i]].frame_id)) { std::swap(buffers[i], buffers[j]); } } } // Rule 2 if (buffer_state_[buffers[1]].state == kBufferAcked || buffer_state_[buffers[2]].state == kBufferAcked) { undroppable_frames_ = 0; return buffers[0]; } // Rule 3 for (int i = 0; i < kNumberOfVp8VideoBuffers; i++) { if (buffer_state_[buffers[i]].state == kBufferSent && IsOlderFrameId(buffer_state_[buffers[i]].frame_id, last_acked_frame_id_)) { undroppable_frames_ = 0; return buffers[i]; } } // Rule 4 if (undroppable_frames_ >= 3) { undroppable_frames_ = 0; return kNoBuffer; } else { undroppable_frames_++; return buffers[kNumberOfVp8VideoBuffers - 1]; } } void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update, vpx_codec_flags_t* flags) { if (!use_multiple_video_buffers_) return; // Update at most one buffer, except for key-frames. switch (buffer_to_update) { case kAltRefBuffer: *flags |= VP8_EFLAG_NO_UPD_GF; *flags |= VP8_EFLAG_NO_UPD_LAST; break; case kLastBuffer: *flags |= VP8_EFLAG_NO_UPD_GF; *flags |= VP8_EFLAG_NO_UPD_ARF; break; case kGoldenBuffer: *flags |= VP8_EFLAG_NO_UPD_ARF; *flags |= VP8_EFLAG_NO_UPD_LAST; break; case kNoBuffer: *flags |= VP8_EFLAG_NO_UPD_ARF; *flags |= VP8_EFLAG_NO_UPD_GF; *flags |= VP8_EFLAG_NO_UPD_LAST; *flags |= VP8_EFLAG_NO_UPD_ENTROPY; break; } } void Vp8Encoder::UpdateRates(uint32 new_bitrate) { DCHECK(thread_checker_.CalledOnValidThread()); if (!is_initialized()) return; uint32 new_bitrate_kbit = new_bitrate / 1000; if (config_.rc_target_bitrate == new_bitrate_kbit) return; config_.rc_target_bitrate = bitrate_kbit_ = new_bitrate_kbit; // Update encoder context. if (vpx_codec_enc_config_set(&encoder_, &config_)) { NOTREACHED() << "Invalid return value"; } VLOG(1) << "VP8 new rc_target_bitrate: " << new_bitrate_kbit << " kbps"; } void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) { DCHECK(thread_checker_.CalledOnValidThread()); if (!use_multiple_video_buffers_) return; VLOG(2) << "VP8 ok to reference frame:" << static_cast(frame_id); for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { if (frame_id == buffer_state_[i].frame_id) { buffer_state_[i].state = kBufferAcked; break; } } if (IsOlderFrameId(last_acked_frame_id_, frame_id)) { last_acked_frame_id_ = frame_id; } } void Vp8Encoder::GenerateKeyFrame() { DCHECK(thread_checker_.CalledOnValidThread()); key_frame_requested_ = true; } } // namespace cast } // namespace media