diff options
author | hubbe@chromium.org <hubbe@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-05-24 12:17:52 +0000 |
---|---|---|
committer | hubbe@chromium.org <hubbe@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-05-24 12:17:52 +0000 |
commit | b6e2be31d71cbcb78d87ceae277b2bf4279afc95 (patch) | |
tree | b4853c4a0a4b9ed3c9b7e7095ddce3c05d024cac | |
parent | 8304f61a936e26b0be93e5b64dbafced34a528b4 (diff) | |
download | chromium_src-b6e2be31d71cbcb78d87ceae277b2bf4279afc95.zip chromium_src-b6e2be31d71cbcb78d87ceae277b2bf4279afc95.tar.gz chromium_src-b6e2be31d71cbcb78d87ceae277b2bf4279afc95.tar.bz2 |
Cast: Only ACK decododable frames
Also, add an optimization to not decode frames if we are behind and multiple frames are currently decodable.
Review URL: https://codereview.chromium.org/289483003
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@272709 0039d316-1c4b-4281-b951-d872f2087c98
-rw-r--r-- | media/cast/audio_receiver/audio_receiver.cc | 20 | ||||
-rw-r--r-- | media/cast/framer/cast_message_builder.cc | 73 | ||||
-rw-r--r-- | media/cast/framer/cast_message_builder.h | 7 | ||||
-rw-r--r-- | media/cast/framer/cast_message_builder_unittest.cc | 97 | ||||
-rw-r--r-- | media/cast/framer/frame_id_map.cc | 60 | ||||
-rw-r--r-- | media/cast/framer/frame_id_map.h | 6 | ||||
-rw-r--r-- | media/cast/framer/framer.cc | 46 | ||||
-rw-r--r-- | media/cast/framer/framer.h | 17 | ||||
-rw-r--r-- | media/cast/framer/framer_unittest.cc | 103 | ||||
-rw-r--r-- | media/cast/video_receiver/video_receiver.cc | 21 |
10 files changed, 194 insertions, 256 deletions
diff --git a/media/cast/audio_receiver/audio_receiver.cc b/media/cast/audio_receiver/audio_receiver.cc index ab06c53..4d6c635 100644 --- a/media/cast/audio_receiver/audio_receiver.cc +++ b/media/cast/audio_receiver/audio_receiver.cc @@ -169,19 +169,29 @@ void AudioReceiver::EmitAvailableEncodedFrames() { scoped_ptr<transport::EncodedFrame> encoded_frame( new transport::EncodedFrame()); bool is_consecutively_next_frame = false; - if (!framer_.GetEncodedAudioFrame(encoded_frame.get(), - &is_consecutively_next_frame)) { + bool have_multiple_complete_frames = false; + if (!framer_.GetEncodedFrame(encoded_frame.get(), + &is_consecutively_next_frame, + &have_multiple_complete_frames)) { VLOG(1) << "Wait for more audio packets to produce a completed frame."; return; // OnReceivedPayloadData() will invoke this method in the future. } + const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); + const base::TimeTicks playout_time = + GetPlayoutTime(now, encoded_frame->rtp_timestamp); + + // If we have multiple decodable frames, and the current frame is + // too old, then skip it and decode the next frame instead. + if (have_multiple_complete_frames && now > playout_time) { + framer_.ReleaseFrame(encoded_frame->frame_id); + continue; + } + // If |framer_| has a frame ready that is out of sequence, examine the // playout time to determine whether it's acceptable to continue, thereby // skipping one or more frames. Skip if the missing frame wouldn't complete // playing before the start of playback of the available frame. - const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); - const base::TimeTicks playout_time = - GetPlayoutTime(now, encoded_frame->rtp_timestamp); if (!is_consecutively_next_frame) { // TODO(miu): Also account for expected decode time here? const base::TimeTicks earliest_possible_end_time_of_missing_frame = diff --git a/media/cast/framer/cast_message_builder.cc b/media/cast/framer/cast_message_builder.cc index 5c31771..f3473f9 100644 --- a/media/cast/framer/cast_message_builder.cc +++ b/media/cast/framer/cast_message_builder.cc @@ -23,7 +23,6 @@ CastMessageBuilder::CastMessageBuilder( decoder_faster_than_max_frame_rate_(decoder_faster_than_max_frame_rate), max_unacked_frames_(max_unacked_frames), cast_msg_(media_ssrc), - waiting_for_key_frame_(true), slowing_down_ack_(false), acked_last_frame_(true), last_acked_frame_id_(kStartFrameId) { @@ -32,67 +31,61 @@ CastMessageBuilder::CastMessageBuilder( CastMessageBuilder::~CastMessageBuilder() {} -void CastMessageBuilder::CompleteFrameReceived(uint32 frame_id, - bool is_key_frame) { +void CastMessageBuilder::CompleteFrameReceived(uint32 frame_id) { + DCHECK_GE(static_cast<int32>(frame_id - last_acked_frame_id_), 0); + VLOG(2) << "CompleteFrameReceived: " << frame_id; if (last_update_time_.is_null()) { // Our first update. last_update_time_ = clock_->NowTicks(); } - if (waiting_for_key_frame_) { - if (!is_key_frame) { - // Ignore that we have received this complete frame since we are - // waiting on a key frame. - return; - } - waiting_for_key_frame_ = false; - cast_msg_.missing_frames_and_packets_.clear(); - cast_msg_.ack_frame_id_ = frame_id; - last_update_time_ = clock_->NowTicks(); - // We might have other complete frames waiting after we receive the last - // packet in the key-frame. - UpdateAckMessage(); - } else { - if (!UpdateAckMessage()) - return; - - BuildPacketList(); + + if (!UpdateAckMessage(frame_id)) { + return; } + BuildPacketList(); + // Send cast message. VLOG(2) << "Send cast message Ack:" << static_cast<int>(frame_id); cast_feedback_->CastFeedback(cast_msg_); } -bool CastMessageBuilder::UpdateAckMessage() { +bool CastMessageBuilder::UpdateAckMessage(uint32 frame_id) { if (!decoder_faster_than_max_frame_rate_) { int complete_frame_count = frame_id_map_->NumberOfCompleteFrames(); if (complete_frame_count > max_unacked_frames_) { // We have too many frames pending in our framer; slow down ACK. - slowing_down_ack_ = true; + if (!slowing_down_ack_) { + slowing_down_ack_ = true; + ack_queue_.push_back(last_acked_frame_id_); + } } else if (complete_frame_count <= 1) { // We are down to one or less frames in our framer; ACK normally. slowing_down_ack_ = false; + ack_queue_.clear(); } } + if (slowing_down_ack_) { // We are slowing down acknowledgment by acknowledging every other frame. - if (acked_last_frame_) { - acked_last_frame_ = false; - } else { - acked_last_frame_ = true; - last_acked_frame_id_++; - // Note: frame skipping and slowdown ACK is not supported at the same - // time; and it's not needed since we can skip frames to catch up. - } - } else { - uint32 frame_id = frame_id_map_->LastContinuousFrame(); - - // Is it a new frame? - if (last_acked_frame_id_ == frame_id) + // Note: frame skipping and slowdown ACK is not supported at the same + // time; and it's not needed since we can skip frames to catch up. + if (!ack_queue_.empty() && ack_queue_.back() == frame_id) { return false; + } + ack_queue_.push_back(frame_id); + if (!acked_last_frame_) { + ack_queue_.pop_front(); + } + frame_id = ack_queue_.front(); + } - last_acked_frame_id_ = frame_id; - acked_last_frame_ = true; + acked_last_frame_ = false; + // Is it a new frame? + if (last_acked_frame_id_ == frame_id) { + return false; } + acked_last_frame_ = true; + last_acked_frame_id_ = frame_id; cast_msg_.ack_frame_id_ = last_acked_frame_id_; cast_msg_.missing_frames_and_packets_.clear(); last_update_time_ = clock_->NowTicks(); @@ -120,7 +113,6 @@ void CastMessageBuilder::UpdateCastMessage() { } void CastMessageBuilder::Reset() { - waiting_for_key_frame_ = true; cast_msg_.ack_frame_id_ = kStartFrameId; cast_msg_.missing_frames_and_packets_.clear(); time_last_nacked_map_.clear(); @@ -142,7 +134,8 @@ bool CastMessageBuilder::UpdateCastMessageInternal(RtcpCastMessage* message) { } last_update_time_ = now; - UpdateAckMessage(); // Needed to cover when a frame is skipped. + // Needed to cover when a frame is skipped. + UpdateAckMessage(last_acked_frame_id_); BuildPacketList(); message->Copy(cast_msg_); return true; diff --git a/media/cast/framer/cast_message_builder.h b/media/cast/framer/cast_message_builder.h index b76a1961..9db88d4 100644 --- a/media/cast/framer/cast_message_builder.h +++ b/media/cast/framer/cast_message_builder.h @@ -7,6 +7,7 @@ #ifndef MEDIA_CAST_FRAMER_CAST_MESSAGE_BUILDER_H_ #define MEDIA_CAST_FRAMER_CAST_MESSAGE_BUILDER_H_ +#include <deque> #include <map> #include "media/cast/framer/frame_id_map.h" @@ -30,13 +31,13 @@ class CastMessageBuilder { int max_unacked_frames); ~CastMessageBuilder(); - void CompleteFrameReceived(uint32 frame_id, bool is_key_frame); + void CompleteFrameReceived(uint32 frame_id); bool TimeToSendNextCastMessage(base::TimeTicks* time_to_send); void UpdateCastMessage(); void Reset(); private: - bool UpdateAckMessage(); + bool UpdateAckMessage(uint32 frame_id); void BuildPacketList(); bool UpdateCastMessageInternal(RtcpCastMessage* message); @@ -51,13 +52,13 @@ class CastMessageBuilder { RtcpCastMessage cast_msg_; base::TimeTicks last_update_time_; - bool waiting_for_key_frame_; TimeLastNackMap time_last_nacked_map_; bool slowing_down_ack_; bool acked_last_frame_; uint32 last_acked_frame_id_; + std::deque<uint32> ack_queue_; DISALLOW_COPY_AND_ASSIGN(CastMessageBuilder); }; diff --git a/media/cast/framer/cast_message_builder_unittest.cc b/media/cast/framer/cast_message_builder_unittest.cc index c84a28f..ef75162 100644 --- a/media/cast/framer/cast_message_builder_unittest.cc +++ b/media/cast/framer/cast_message_builder_unittest.cc @@ -112,8 +112,7 @@ class CastMessageBuilderTest : public ::testing::Test { void InsertPacket() { PacketType packet_type = frame_id_map_.InsertPacket(rtp_header_); if (packet_type == kNewPacketCompletingFrame) { - cast_msg_builder_->CompleteFrameReceived(rtp_header_.frame_id, - rtp_header_.is_key_frame); + cast_msg_builder_->CompleteFrameReceived(rtp_header_.frame_id); } cast_msg_builder_->UpdateCastMessage(); } @@ -136,26 +135,6 @@ class CastMessageBuilderTest : public ::testing::Test { DISALLOW_COPY_AND_ASSIGN(CastMessageBuilderTest); }; -TEST_F(CastMessageBuilderTest, StartWithAKeyFrame) { - SetFrameIds(3, 2); - SetPacketId(0); - SetMaxPacketId(0); - InsertPacket(); - // Should not trigger ack. - EXPECT_FALSE(feedback_.triggered()); - SetFrameIds(5, 5); - SetPacketId(0); - SetMaxPacketId(0); - SetKeyFrame(true); - InsertPacket(); - frame_id_map_.RemoveOldFrames(5); // Simulate 5 being pulled for rendering. - testing_clock_.Advance( - base::TimeDelta::FromMilliseconds(kLongTimeIncrementMs)); - cast_msg_builder_->UpdateCastMessage(); - EXPECT_TRUE(feedback_.triggered()); - EXPECT_EQ(5u, feedback_.last_frame_acked()); -} - TEST_F(CastMessageBuilderTest, OneFrameNackList) { SetFrameIds(0, 0); SetPacketId(4); @@ -187,31 +166,6 @@ TEST_F(CastMessageBuilderTest, CompleteFrameMissing) { EXPECT_EQ(kRtcpCastAllPacketsLost, feedback_.num_missing_packets(1)); } -TEST_F(CastMessageBuilderTest, FastForwardAck) { - SetFrameIds(1, 0); - SetPacketId(0); - SetMaxPacketId(0); - InsertPacket(); - EXPECT_FALSE(feedback_.triggered()); - testing_clock_.Advance( - base::TimeDelta::FromMilliseconds(kLongTimeIncrementMs)); - SetFrameIds(2, 1); - SetPacketId(0); - SetMaxPacketId(0); - InsertPacket(); - EXPECT_TRUE(feedback_.triggered()); - EXPECT_EQ(kStartFrameId, feedback_.last_frame_acked()); - testing_clock_.Advance( - base::TimeDelta::FromMilliseconds(kLongTimeIncrementMs)); - SetFrameIds(0, 0); - SetPacketId(0); - SetMaxPacketId(0); - SetKeyFrame(true); - InsertPacket(); - EXPECT_TRUE(feedback_.triggered()); - EXPECT_EQ(2u, feedback_.last_frame_acked()); -} - TEST_F(CastMessageBuilderTest, RemoveOldFrames) { SetFrameIds(1, 0); SetPacketId(0); @@ -232,7 +186,7 @@ TEST_F(CastMessageBuilderTest, RemoveOldFrames) { SetMaxPacketId(5); InsertPacket(); EXPECT_TRUE(feedback_.triggered()); - EXPECT_EQ(kStartFrameId, feedback_.last_frame_acked()); + EXPECT_EQ(2u, feedback_.last_frame_acked()); testing_clock_.Advance( base::TimeDelta::FromMilliseconds(kLongTimeIncrementMs)); SetFrameIds(5, 5); @@ -260,42 +214,6 @@ TEST_F(CastMessageBuilderTest, RemoveOldFrames) { EXPECT_EQ(5u, feedback_.last_frame_acked()); } -TEST_F(CastMessageBuilderTest, WrapFastForward) { - SetFrameIds(254, 254); - SetPacketId(0); - SetMaxPacketId(1); - SetKeyFrame(true); - InsertPacket(); - EXPECT_FALSE(feedback_.triggered()); - testing_clock_.Advance( - base::TimeDelta::FromMilliseconds(kLongTimeIncrementMs)); - SetFrameIds(255, 254); - SetPacketId(0); - SetMaxPacketId(0); - SetKeyFrame(false); - InsertPacket(); - EXPECT_TRUE(feedback_.triggered()); - EXPECT_EQ(253u, feedback_.last_frame_acked()); - testing_clock_.Advance( - base::TimeDelta::FromMilliseconds(kLongTimeIncrementMs)); - SetFrameIds(256, 255); - SetPacketId(0); - SetMaxPacketId(0); - SetKeyFrame(false); - InsertPacket(); - EXPECT_TRUE(feedback_.triggered()); - EXPECT_EQ(253u, feedback_.last_frame_acked()); - testing_clock_.Advance( - base::TimeDelta::FromMilliseconds(kLongTimeIncrementMs)); - SetFrameIds(254, 254); - SetPacketId(1); - SetMaxPacketId(1); - SetKeyFrame(true); - InsertPacket(); - EXPECT_TRUE(feedback_.triggered()); - EXPECT_EQ(256u, feedback_.last_frame_acked()); -} - TEST_F(CastMessageBuilderTest, NackUntilMaxReceivedPacket) { SetFrameIds(0, 0); SetPacketId(0); @@ -404,7 +322,7 @@ TEST_F(CastMessageBuilderTest, BasicRps) { SetKeyFrame(false); InsertPacket(); EXPECT_TRUE(feedback_.triggered()); - EXPECT_EQ(0u, feedback_.last_frame_acked()); + EXPECT_EQ(3u, feedback_.last_frame_acked()); testing_clock_.Advance( base::TimeDelta::FromMilliseconds(kLongTimeIncrementMs)); frame_id_map_.RemoveOldFrames(3); // Simulate 3 being pulled for rendering. @@ -480,16 +398,19 @@ TEST_F(CastMessageBuilderTest, SlowDownAck) { // We should now have entered the slowdown ACK state. uint32 expected_frame_id = 1; for (; frame_id < 10; ++frame_id) { - if (frame_id % 2) + if (frame_id % 2) { ++expected_frame_id; - EXPECT_TRUE(feedback_.triggered()); + EXPECT_TRUE(feedback_.triggered()); + } else { + EXPECT_FALSE(feedback_.triggered()); + } EXPECT_EQ(expected_frame_id, feedback_.last_frame_acked()); SetFrameIds(frame_id, frame_id - 1); InsertPacket(); testing_clock_.Advance( base::TimeDelta::FromMilliseconds(kShortTimeIncrementMs)); } - EXPECT_TRUE(feedback_.triggered()); + EXPECT_FALSE(feedback_.triggered()); EXPECT_EQ(expected_frame_id, feedback_.last_frame_acked()); // Simulate frame_id being pulled for rendering. diff --git a/media/cast/framer/frame_id_map.cc b/media/cast/framer/frame_id_map.cc index f93fb85..b4389fd 100644 --- a/media/cast/framer/frame_id_map.cc +++ b/media/cast/framer/frame_id_map.cc @@ -137,6 +137,22 @@ bool FrameIdMap::NextContinuousFrame(uint32* frame_id) const { return false; } +bool FrameIdMap::HaveMultipleDecodableFrames() const { + // Find the oldest decodable frame. + FrameMap::const_iterator it; + bool found_one = false; + for (it = frame_map_.begin(); it != frame_map_.end(); ++it) { + if (it->second->Complete() && DecodableFrame(it->second.get())) { + if (found_one) { + return true; + } else { + found_one = true; + } + } + } + return false; +} + uint32 FrameIdMap::LastContinuousFrame() const { uint32 last_continuous_frame_id = last_released_frame_; uint32 next_expected_frame = last_released_frame_; @@ -157,43 +173,16 @@ uint32 FrameIdMap::LastContinuousFrame() const { return last_continuous_frame_id; } -bool FrameIdMap::NextAudioFrameAllowingMissingFrames(uint32* frame_id) const { - // First check if we have continuous frames. - if (NextContinuousFrame(frame_id)) - return true; - - // Find the oldest frame. - FrameMap::const_iterator it_best_match = frame_map_.end(); - FrameMap::const_iterator it; - - // Find first complete frame. - for (it = frame_map_.begin(); it != frame_map_.end(); ++it) { - if (it->second->Complete()) { - it_best_match = it; - break; - } - } - if (it_best_match == frame_map_.end()) - return false; // No complete frame. - - ++it; - for (; it != frame_map_.end(); ++it) { - if (it->second->Complete() && - IsOlderFrameId(it->first, it_best_match->first)) { - it_best_match = it; - } - } - *frame_id = it_best_match->first; - return true; -} - -bool FrameIdMap::NextVideoFrameAllowingSkippingFrames(uint32* frame_id) const { +bool FrameIdMap::NextFrameAllowingSkippingFrames(uint32* frame_id) const { // Find the oldest decodable frame. FrameMap::const_iterator it_best_match = frame_map_.end(); FrameMap::const_iterator it; for (it = frame_map_.begin(); it != frame_map_.end(); ++it) { - if (it->second->Complete() && DecodableVideoFrame(it->second.get())) { - it_best_match = it; + if (it->second->Complete() && DecodableFrame(it->second.get())) { + if (it_best_match == frame_map_.end() || + IsOlderFrameId(it->first, it_best_match->first)) { + it_best_match = it; + } } } if (it_best_match == frame_map_.end()) @@ -237,11 +226,14 @@ bool FrameIdMap::ContinuousFrame(FrameInfo* frame) const { return static_cast<uint32>(last_released_frame_ + 1) == frame->frame_id(); } -bool FrameIdMap::DecodableVideoFrame(FrameInfo* frame) const { +bool FrameIdMap::DecodableFrame(FrameInfo* frame) const { if (frame->is_key_frame()) return true; if (waiting_for_key_ && !frame->is_key_frame()) return false; + // Self-reference? + if (frame->referenced_frame_id() == frame->frame_id()) + return true; // Current frame is not necessarily referencing the last frame. // Do we have the reference frame? diff --git a/media/cast/framer/frame_id_map.h b/media/cast/framer/frame_id_map.h index f61e251..66e306f 100644 --- a/media/cast/framer/frame_id_map.h +++ b/media/cast/framer/frame_id_map.h @@ -64,8 +64,8 @@ class FrameIdMap { bool NextContinuousFrame(uint32* frame_id) const; uint32 LastContinuousFrame() const; - bool NextAudioFrameAllowingMissingFrames(uint32* frame_id) const; - bool NextVideoFrameAllowingSkippingFrames(uint32* frame_id) const; + bool NextFrameAllowingSkippingFrames(uint32* frame_id) const; + bool HaveMultipleDecodableFrames() const; int NumberOfCompleteFrames() const; void GetMissingPackets(uint32 frame_id, @@ -74,7 +74,7 @@ class FrameIdMap { private: bool ContinuousFrame(FrameInfo* frame) const; - bool DecodableVideoFrame(FrameInfo* frame) const; + bool DecodableFrame(FrameInfo* frame) const; FrameMap frame_map_; bool waiting_for_key_; diff --git a/media/cast/framer/framer.cc b/media/cast/framer/framer.cc index ee5e6aa..de4451a 100644 --- a/media/cast/framer/framer.cc +++ b/media/cast/framer/framer.cc @@ -58,42 +58,15 @@ bool Framer::InsertPacket(const uint8* payload_data, it->second->InsertPacket(payload_data, payload_size, rtp_header); } - bool complete = (packet_type == kNewPacketCompletingFrame); - if (complete) { - // ACK as soon as possible. - VLOG(2) << "Complete frame " << static_cast<int>(rtp_header.frame_id); - cast_msg_builder_->CompleteFrameReceived(rtp_header.frame_id, - rtp_header.is_key_frame); - } - return complete; + return packet_type == kNewPacketCompletingFrame; } // This does not release the frame. -bool Framer::GetEncodedAudioFrame(transport::EncodedFrame* audio_frame, - bool* next_frame) { - uint32 frame_id; - // Find frame id. - if (frame_id_map_.NextContinuousFrame(&frame_id)) { - // We have our next frame. - *next_frame = true; - } else { - if (!frame_id_map_.NextAudioFrameAllowingMissingFrames(&frame_id)) { - return false; - } - *next_frame = false; - } +bool Framer::GetEncodedFrame(transport::EncodedFrame* frame, + bool* next_frame, + bool* have_multiple_decodable_frames) { + *have_multiple_decodable_frames = frame_id_map_.HaveMultipleDecodableFrames(); - ConstFrameIterator it = frames_.find(frame_id); - DCHECK(it != frames_.end()); - if (it == frames_.end()) - return false; - - return it->second->AssembleEncodedFrame(audio_frame); -} - -// This does not release the frame. -bool Framer::GetEncodedVideoFrame(transport::EncodedFrame* video_frame, - bool* next_frame) { uint32 frame_id; // Find frame id. if (frame_id_map_.NextContinuousFrame(&frame_id)) { @@ -104,18 +77,23 @@ bool Framer::GetEncodedVideoFrame(transport::EncodedFrame* video_frame, if (!decoder_faster_than_max_frame_rate_) return false; - if (!frame_id_map_.NextVideoFrameAllowingSkippingFrames(&frame_id)) { + if (!frame_id_map_.NextFrameAllowingSkippingFrames(&frame_id)) { return false; } *next_frame = false; } + if (*next_frame) { + VLOG(2) << "ACK frame " << frame_id; + cast_msg_builder_->CompleteFrameReceived(frame_id); + } + ConstFrameIterator it = frames_.find(frame_id); DCHECK(it != frames_.end()); if (it == frames_.end()) return false; - return it->second->AssembleEncodedFrame(video_frame); + return it->second->AssembleEncodedFrame(frame); } void Framer::Reset() { diff --git a/media/cast/framer/framer.h b/media/cast/framer/framer.h index 0c7397b..0b7249e 100644 --- a/media/cast/framer/framer.h +++ b/media/cast/framer/framer.h @@ -40,15 +40,14 @@ class Framer { const RtpCastHeader& rtp_header, bool* duplicate); - // Extracts a complete encoded frame - will only return a complete continuous - // frame. - // Returns false if the frame does not exist or if the frame is not complete - // within the given time frame. - bool GetEncodedVideoFrame(transport::EncodedFrame* video_frame, - bool* next_frame); - - bool GetEncodedAudioFrame(transport::EncodedFrame* audio_frame, - bool* next_frame); + // Extracts a complete encoded frame - will only return a complete and + // decodable frame. Returns false if no such frames exist. + // |next_frame| will be set to true if the returned frame is the very + // next frame. |have_multiple_complete_frames| will be set to true + // if there are more decodadble frames available. + bool GetEncodedFrame(transport::EncodedFrame* video_frame, + bool* next_frame, + bool* have_multiple_complete_frames); void ReleaseFrame(uint32 frame_id); diff --git a/media/cast/framer/framer_unittest.cc b/media/cast/framer/framer_unittest.cc index 5cb12ce..ad53ef0 100644 --- a/media/cast/framer/framer_unittest.cc +++ b/media/cast/framer/framer_unittest.cc @@ -35,28 +35,31 @@ class FramerTest : public ::testing::Test { TEST_F(FramerTest, EmptyState) { transport::EncodedFrame frame; bool next_frame = false; - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + bool multiple = false; + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); } TEST_F(FramerTest, AlwaysStartWithKey) { transport::EncodedFrame frame; bool next_frame = false; bool complete = false; + bool multiple = false; bool duplicate = false; // Insert non key first frame. complete = framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); rtp_header_.frame_id = 1; rtp_header_.reference_frame_id = 1; rtp_header_.is_key_frame = true; complete = framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_TRUE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(1u, frame.frame_id); EXPECT_EQ(1u, frame.referenced_frame_id); @@ -67,6 +70,7 @@ TEST_F(FramerTest, CompleteFrame) { transport::EncodedFrame frame; bool next_frame = false; bool complete = false; + bool multiple = false; bool duplicate = false; // Start with a complete key frame. @@ -74,8 +78,9 @@ TEST_F(FramerTest, CompleteFrame) { complete = framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(0u, frame.frame_id); EXPECT_EQ(0u, frame.referenced_frame_id); @@ -89,7 +94,7 @@ TEST_F(FramerTest, CompleteFrame) { complete = framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_FALSE(complete); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); // Complete delta - can't skip, as incomplete sequence. ++rtp_header_.frame_id; @@ -98,13 +103,14 @@ TEST_F(FramerTest, CompleteFrame) { complete = framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); } TEST_F(FramerTest, DuplicatePackets) { transport::EncodedFrame frame; bool next_frame = false; bool complete = false; + bool multiple = false; bool duplicate = false; // Start with an incomplete key frame. @@ -115,7 +121,7 @@ TEST_F(FramerTest, DuplicatePackets) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_FALSE(complete); EXPECT_FALSE(duplicate); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); // Add same packet again in incomplete key frame. duplicate = false; @@ -123,7 +129,7 @@ TEST_F(FramerTest, DuplicatePackets) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_FALSE(complete); EXPECT_TRUE(duplicate); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); // Complete key frame. rtp_header_.packet_id = 1; @@ -132,9 +138,9 @@ TEST_F(FramerTest, DuplicatePackets) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); EXPECT_FALSE(duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); - EXPECT_EQ(0u, frame.frame_id); + EXPECT_FALSE(multiple); EXPECT_EQ(0u, frame.referenced_frame_id); // Add same packet again in complete key frame. @@ -143,9 +149,10 @@ TEST_F(FramerTest, DuplicatePackets) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_FALSE(complete); EXPECT_TRUE(duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(0u, frame.frame_id); + EXPECT_FALSE(multiple); EXPECT_EQ(0u, frame.referenced_frame_id); framer_.ReleaseFrame(frame.frame_id); @@ -159,7 +166,7 @@ TEST_F(FramerTest, DuplicatePackets) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_FALSE(complete); EXPECT_FALSE(duplicate); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); // Add same packet again in incomplete delta frame. duplicate = false; @@ -167,7 +174,7 @@ TEST_F(FramerTest, DuplicatePackets) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_FALSE(complete); EXPECT_TRUE(duplicate); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); // Complete delta frame. rtp_header_.packet_id = 1; @@ -176,10 +183,11 @@ TEST_F(FramerTest, DuplicatePackets) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); EXPECT_FALSE(duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_EQ(transport::EncodedFrame::DEPENDENT, frame.dependency); EXPECT_EQ(1u, frame.frame_id); EXPECT_EQ(0u, frame.referenced_frame_id); + EXPECT_FALSE(multiple); // Add same packet again in complete delta frame. duplicate = false; @@ -187,16 +195,18 @@ TEST_F(FramerTest, DuplicatePackets) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_FALSE(complete); EXPECT_TRUE(duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_EQ(transport::EncodedFrame::DEPENDENT, frame.dependency); EXPECT_EQ(1u, frame.frame_id); EXPECT_EQ(0u, frame.referenced_frame_id); + EXPECT_FALSE(multiple); } TEST_F(FramerTest, ContinuousSequence) { transport::EncodedFrame frame; bool next_frame = false; bool complete = false; + bool multiple = false; bool duplicate = false; // Start with a complete key frame. @@ -204,8 +214,9 @@ TEST_F(FramerTest, ContinuousSequence) { complete = framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(0u, frame.frame_id); EXPECT_EQ(0u, frame.referenced_frame_id); @@ -218,13 +229,14 @@ TEST_F(FramerTest, ContinuousSequence) { complete = framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); } TEST_F(FramerTest, Wrap) { // Insert key frame, frame_id = 255 (will jump to that) transport::EncodedFrame frame; bool next_frame = false; + bool multiple = true; bool duplicate = false; // Start with a complete key frame. @@ -233,8 +245,9 @@ TEST_F(FramerTest, Wrap) { rtp_header_.reference_frame_id = 255; framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(255u, frame.frame_id); EXPECT_EQ(255u, frame.referenced_frame_id); @@ -245,8 +258,9 @@ TEST_F(FramerTest, Wrap) { rtp_header_.frame_id = 256; framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::DEPENDENT, frame.dependency); EXPECT_EQ(256u, frame.frame_id); EXPECT_EQ(255u, frame.referenced_frame_id); @@ -257,6 +271,7 @@ TEST_F(FramerTest, Reset) { transport::EncodedFrame frame; bool next_frame = false; bool complete = false; + bool multiple = true; bool duplicate = false; // Start with a complete key frame. @@ -265,12 +280,13 @@ TEST_F(FramerTest, Reset) { payload_.data(), payload_.size(), rtp_header_, &duplicate); EXPECT_TRUE(complete); framer_.Reset(); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); } TEST_F(FramerTest, RequireKeyAfterReset) { transport::EncodedFrame frame; bool next_frame = false; + bool multiple = false; bool duplicate = false; framer_.Reset(); @@ -280,19 +296,21 @@ TEST_F(FramerTest, RequireKeyAfterReset) { rtp_header_.frame_id = 0; framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); rtp_header_.frame_id = 1; rtp_header_.reference_frame_id = 1; rtp_header_.is_key_frame = true; framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_TRUE(multiple); } TEST_F(FramerTest, BasicNonLastReferenceId) { transport::EncodedFrame frame; bool next_frame = false; + bool multiple = false; bool duplicate = false; rtp_header_.is_key_frame = true; @@ -300,7 +318,8 @@ TEST_F(FramerTest, BasicNonLastReferenceId) { framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); + EXPECT_FALSE(multiple); framer_.ReleaseFrame(frame.frame_id); rtp_header_.is_key_frame = false; @@ -309,14 +328,16 @@ TEST_F(FramerTest, BasicNonLastReferenceId) { framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_FALSE(next_frame); + EXPECT_FALSE(multiple); } TEST_F(FramerTest, InOrderReferenceFrameSelection) { // Create pattern: 0, 1, 4, 5. transport::EncodedFrame frame; bool next_frame = false; + bool multiple = false; bool duplicate = false; rtp_header_.is_key_frame = true; @@ -338,19 +359,22 @@ TEST_F(FramerTest, InOrderReferenceFrameSelection) { rtp_header_.reference_frame_id = 0; framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(0u, frame.frame_id); EXPECT_EQ(0u, frame.referenced_frame_id); + EXPECT_FALSE(multiple); framer_.ReleaseFrame(frame.frame_id); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_TRUE(multiple); EXPECT_EQ(transport::EncodedFrame::DEPENDENT, frame.dependency); EXPECT_EQ(1u, frame.frame_id); EXPECT_EQ(0u, frame.referenced_frame_id); framer_.ReleaseFrame(frame.frame_id); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_FALSE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::DEPENDENT, frame.dependency); EXPECT_EQ(4u, frame.frame_id); EXPECT_EQ(0u, frame.referenced_frame_id); @@ -360,15 +384,16 @@ TEST_F(FramerTest, InOrderReferenceFrameSelection) { rtp_header_.packet_id = 1; framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_FALSE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_FALSE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); rtp_header_.frame_id = 5; rtp_header_.reference_frame_id = rtp_header_.frame_id - 1; rtp_header_.packet_id = 0; rtp_header_.max_packet_id = 0; framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedVideoFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::DEPENDENT, frame.dependency); EXPECT_EQ(5u, frame.frame_id); EXPECT_EQ(4u, frame.referenced_frame_id); @@ -378,6 +403,7 @@ TEST_F(FramerTest, AudioWrap) { // All audio frames are marked as key frames. transport::EncodedFrame frame; bool next_frame = false; + bool multiple = false; bool duplicate = false; rtp_header_.is_key_frame = true; @@ -386,8 +412,9 @@ TEST_F(FramerTest, AudioWrap) { framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedAudioFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(254u, frame.frame_id); EXPECT_EQ(254u, frame.referenced_frame_id); @@ -404,15 +431,17 @@ TEST_F(FramerTest, AudioWrap) { framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedAudioFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_TRUE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(255u, frame.frame_id); EXPECT_EQ(255u, frame.referenced_frame_id); framer_.ReleaseFrame(frame.frame_id); - EXPECT_TRUE(framer_.GetEncodedAudioFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(256u, frame.frame_id); EXPECT_EQ(256u, frame.referenced_frame_id); @@ -423,6 +452,7 @@ TEST_F(FramerTest, AudioWrapWithMissingFrame) { // All audio frames are marked as key frames. transport::EncodedFrame frame; bool next_frame = false; + bool multiple = true; bool duplicate = false; // Insert and get first packet. @@ -431,8 +461,9 @@ TEST_F(FramerTest, AudioWrapWithMissingFrame) { rtp_header_.reference_frame_id = 253; framer_.InsertPacket( payload_.data(), payload_.size(), rtp_header_, &duplicate); - EXPECT_TRUE(framer_.GetEncodedAudioFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(253u, frame.frame_id); EXPECT_EQ(253u, frame.referenced_frame_id); @@ -449,14 +480,16 @@ TEST_F(FramerTest, AudioWrapWithMissingFrame) { payload_.data(), payload_.size(), rtp_header_, &duplicate); // Get third and fourth packets. - EXPECT_TRUE(framer_.GetEncodedAudioFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_FALSE(next_frame); + EXPECT_TRUE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(255u, frame.frame_id); EXPECT_EQ(255u, frame.referenced_frame_id); framer_.ReleaseFrame(frame.frame_id); - EXPECT_TRUE(framer_.GetEncodedAudioFrame(&frame, &next_frame)); + EXPECT_TRUE(framer_.GetEncodedFrame(&frame, &next_frame, &multiple)); EXPECT_TRUE(next_frame); + EXPECT_FALSE(multiple); EXPECT_EQ(transport::EncodedFrame::KEY, frame.dependency); EXPECT_EQ(256u, frame.frame_id); EXPECT_EQ(256u, frame.referenced_frame_id); diff --git a/media/cast/video_receiver/video_receiver.cc b/media/cast/video_receiver/video_receiver.cc index df17a53..cbd62ae 100644 --- a/media/cast/video_receiver/video_receiver.cc +++ b/media/cast/video_receiver/video_receiver.cc @@ -155,19 +155,30 @@ void VideoReceiver::EmitAvailableEncodedFrames() { scoped_ptr<transport::EncodedFrame> encoded_frame( new transport::EncodedFrame()); bool is_consecutively_next_frame = false; - if (!framer_.GetEncodedVideoFrame(encoded_frame.get(), - &is_consecutively_next_frame)) { + bool have_multiple_complete_frames = false; + + if (!framer_.GetEncodedFrame(encoded_frame.get(), + &is_consecutively_next_frame, + &have_multiple_complete_frames)) { VLOG(1) << "Wait for more video packets to produce a completed frame."; return; // OnReceivedPayloadData() will invoke this method in the future. } + const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); + const base::TimeTicks playout_time = + GetPlayoutTime(now, encoded_frame->rtp_timestamp); + + // If we have multiple decodable frames, and the current frame is + // too old, then skip it and decode the next frame instead. + if (have_multiple_complete_frames && now > playout_time) { + framer_.ReleaseFrame(encoded_frame->frame_id); + continue; + } + // If |framer_| has a frame ready that is out of sequence, examine the // playout time to determine whether it's acceptable to continue, thereby // skipping one or more frames. Skip if the missing frame wouldn't complete // playing before the start of playback of the available frame. - const base::TimeTicks now = cast_environment_->Clock()->NowTicks(); - const base::TimeTicks playout_time = - GetPlayoutTime(now, encoded_frame->rtp_timestamp); if (!is_consecutively_next_frame) { // TODO(miu): Also account for expected decode time here? const base::TimeTicks earliest_possible_end_time_of_missing_frame = |