summaryrefslogtreecommitdiffstats
path: root/media/cast
diff options
context:
space:
mode:
authormiu@chromium.org <miu@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2014-05-27 00:03:27 +0000
committermiu@chromium.org <miu@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2014-05-27 00:03:27 +0000
commit31a5bc2eb44e79c7eaabd463cdb121a0ed9e5b35 (patch)
tree5da659c44164af09b9b06b776daf7691f82629d3 /media/cast
parent51f82b0c9e3755e7fa068f99fc34e54c36d5066c (diff)
downloadchromium_src-31a5bc2eb44e79c7eaabd463cdb121a0ed9e5b35.zip
chromium_src-31a5bc2eb44e79c7eaabd463cdb121a0ed9e5b35.tar.gz
chromium_src-31a5bc2eb44e79c7eaabd463cdb121a0ed9e5b35.tar.bz2
[Cast] Repair receiver playout time calculations and frame skip logic.
Fixed playout time calculations by relying on the sender reports in order to compute reliable capture timestamps in terms of the local clock. Added a ClockDriftSmoother to smooth out jitter/skew in the "NTP to local clock TimeTicks" conversions. Used the same to provide a gradual timeline shift in the case where a receiver had to "hack up" playout times before the first sender report was processed. As proof-of-concept, also added playout_time "smoothness" testing to the End2EndTest's. Both first and second order effects are tested. Miscellaneous clean-ups and fixes in timing code throughout media/cast. BUG=356942 Review URL: https://codereview.chromium.org/280993002 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@272893 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'media/cast')
-rw-r--r--media/cast/audio_receiver/audio_receiver.cc136
-rw-r--r--media/cast/audio_receiver/audio_receiver.h56
-rw-r--r--media/cast/audio_receiver/audio_receiver_unittest.cc121
-rw-r--r--media/cast/audio_sender/audio_sender_unittest.cc5
-rw-r--r--media/cast/base/clock_drift_smoother.cc58
-rw-r--r--media/cast/base/clock_drift_smoother.h52
-rw-r--r--media/cast/cast.gyp2
-rw-r--r--media/cast/cast_defines.h22
-rw-r--r--media/cast/rtcp/rtcp.cc131
-rw-r--r--media/cast/rtcp/rtcp.h58
-rw-r--r--media/cast/rtcp/rtcp_unittest.cc117
-rw-r--r--media/cast/test/end2end_unittest.cc248
-rw-r--r--media/cast/test/receiver.cc24
-rw-r--r--media/cast/test/skewed_tick_clock.cc3
-rw-r--r--media/cast/transport/cast_transport_config.h3
-rw-r--r--media/cast/video_receiver/video_receiver.cc143
-rw-r--r--media/cast/video_receiver/video_receiver.h55
-rw-r--r--media/cast/video_receiver/video_receiver_unittest.cc144
-rw-r--r--media/cast/video_sender/video_sender_unittest.cc4
19 files changed, 783 insertions, 599 deletions
diff --git a/media/cast/audio_receiver/audio_receiver.cc b/media/cast/audio_receiver/audio_receiver.cc
index 4d6c635..4b0803f 100644
--- a/media/cast/audio_receiver/audio_receiver.cc
+++ b/media/cast/audio_receiver/audio_receiver.cc
@@ -29,8 +29,9 @@ AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment,
event_subscriber_(kReceiverRtcpEventHistorySize, AUDIO_EVENT),
codec_(audio_config.codec),
frequency_(audio_config.frequency),
- target_delay_delta_(
+ target_playout_delay_(
base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms)),
+ reports_are_scheduled_(false),
framer_(cast_environment->Clock(),
this,
audio_config.incoming_ssrc,
@@ -48,11 +49,12 @@ AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment,
audio_config.rtcp_c_name,
true),
is_waiting_for_consecutive_frame_(false),
+ lip_sync_drift_(ClockDriftSmoother::GetDefaultTimeConstant()),
weak_factory_(this) {
if (!audio_config.use_external_decoder)
audio_decoder_.reset(new AudioDecoder(cast_environment, audio_config));
decryptor_.Initialize(audio_config.aes_key, audio_config.aes_iv_mask);
- rtcp_.SetTargetDelay(target_delay_delta_);
+ rtcp_.SetTargetDelay(target_playout_delay_);
cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_);
memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_));
}
@@ -62,24 +64,12 @@ AudioReceiver::~AudioReceiver() {
cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_);
}
-void AudioReceiver::InitializeTimers() {
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- ScheduleNextRtcpReport();
- ScheduleNextCastMessage();
-}
-
void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data,
size_t payload_size,
const RtpCastHeader& rtp_header) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- base::TimeTicks now = cast_environment_->Clock()->NowTicks();
- // TODO(pwestin): update this as video to refresh over time.
- if (time_first_incoming_packet_.is_null()) {
- InitializeTimers();
- first_incoming_rtp_timestamp_ = rtp_header.rtp_timestamp;
- time_first_incoming_packet_ = now;
- }
+ const base::TimeTicks now = cast_environment_->Clock()->NowTicks();
frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] =
rtp_header.rtp_timestamp;
@@ -93,7 +83,41 @@ void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data,
framer_.InsertPacket(payload_data, payload_size, rtp_header, &duplicate);
// Duplicate packets are ignored.
- if (duplicate || !complete)
+ if (duplicate)
+ return;
+
+ // Update lip-sync values upon receiving the first packet of each frame, or if
+ // they have never been set yet.
+ if (rtp_header.packet_id == 0 || lip_sync_reference_time_.is_null()) {
+ RtpTimestamp fresh_sync_rtp;
+ base::TimeTicks fresh_sync_reference;
+ if (!rtcp_.GetLatestLipSyncTimes(&fresh_sync_rtp, &fresh_sync_reference)) {
+ // HACK: The sender should have provided Sender Reports before the first
+ // frame was sent. However, the spec does not currently require this.
+ // Therefore, when the data is missing, the local clock is used to
+ // generate reference timestamps.
+ VLOG(2) << "Lip sync info missing. Falling-back to local clock.";
+ fresh_sync_rtp = rtp_header.rtp_timestamp;
+ fresh_sync_reference = now;
+ }
+ // |lip_sync_reference_time_| is always incremented according to the time
+ // delta computed from the difference in RTP timestamps. Then,
+ // |lip_sync_drift_| accounts for clock drift and also smoothes-out any
+ // sudden/discontinuous shifts in the series of reference time values.
+ if (lip_sync_reference_time_.is_null()) {
+ lip_sync_reference_time_ = fresh_sync_reference;
+ } else {
+ lip_sync_reference_time_ += RtpDeltaToTimeDelta(
+ static_cast<int32>(fresh_sync_rtp - lip_sync_rtp_timestamp_),
+ frequency_);
+ }
+ lip_sync_rtp_timestamp_ = fresh_sync_rtp;
+ lip_sync_drift_.Update(
+ now, fresh_sync_reference - lip_sync_reference_time_);
+ }
+
+ // Frame not complete; wait for more packets.
+ if (!complete)
return;
EmitAvailableEncodedFrames();
@@ -179,7 +203,7 @@ void AudioReceiver::EmitAvailableEncodedFrames() {
const base::TimeTicks now = cast_environment_->Clock()->NowTicks();
const base::TimeTicks playout_time =
- GetPlayoutTime(now, encoded_frame->rtp_timestamp);
+ GetPlayoutTime(encoded_frame->rtp_timestamp);
// If we have multiple decodable frames, and the current frame is
// too old, then skip it and decode the next frame instead.
@@ -243,6 +267,15 @@ void AudioReceiver::EmitAvailableEncodedFramesAfterWaiting() {
EmitAvailableEncodedFrames();
}
+base::TimeTicks AudioReceiver::GetPlayoutTime(uint32 rtp_timestamp) const {
+ return lip_sync_reference_time_ +
+ lip_sync_drift_.Current() +
+ RtpDeltaToTimeDelta(
+ static_cast<int32>(rtp_timestamp - lip_sync_rtp_timestamp_),
+ frequency_) +
+ target_playout_delay_;
+}
+
void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
if (Rtcp::IsRtcpPacket(&packet->front(), packet->size())) {
@@ -250,12 +283,11 @@ void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) {
} else {
ReceivedPacket(&packet->front(), packet->size());
}
-}
-
-void AudioReceiver::SetTargetDelay(base::TimeDelta target_delay) {
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- target_delay_delta_ = target_delay;
- rtcp_.SetTargetDelay(target_delay_delta_);
+ if (!reports_are_scheduled_) {
+ ScheduleNextRtcpReport();
+ ScheduleNextCastMessage();
+ reports_are_scheduled_ = true;
+ }
}
void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
@@ -272,64 +304,6 @@ void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
rtcp_.SendRtcpFromRtpReceiver(&cast_message, &rtcp_events);
}
-base::TimeTicks AudioReceiver::GetPlayoutTime(base::TimeTicks now,
- uint32 rtp_timestamp) {
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- // Senders time in ms when this frame was recorded.
- // Note: the senders clock and our local clock might not be synced.
- base::TimeTicks rtp_timestamp_in_ticks;
- base::TimeTicks playout_time;
- if (time_offset_ == base::TimeDelta()) {
- if (rtcp_.RtpTimestampInSenderTime(frequency_,
- first_incoming_rtp_timestamp_,
- &rtp_timestamp_in_ticks)) {
- time_offset_ = time_first_incoming_packet_ - rtp_timestamp_in_ticks;
- // TODO(miu): As clocks drift w.r.t. each other, and other factors take
- // effect, |time_offset_| should be updated. Otherwise, we might as well
- // always compute the time offsets agnostic of RTCP's time data.
- } else {
- // We have not received any RTCP to sync the stream play it out as soon as
- // possible.
-
- // BUG: This means we're literally switching to a different timeline a
- // short time after a cast receiver has been running. Re-enable
- // End2EndTest.StartSenderBeforeReceiver once this is fixed.
- // http://crbug.com/356942
- uint32 rtp_timestamp_diff = rtp_timestamp - first_incoming_rtp_timestamp_;
-
- int frequency_khz = frequency_ / 1000;
- base::TimeDelta rtp_time_diff_delta =
- base::TimeDelta::FromMilliseconds(rtp_timestamp_diff / frequency_khz);
- base::TimeDelta time_diff_delta = now - time_first_incoming_packet_;
-
- playout_time = now + std::max(rtp_time_diff_delta - time_diff_delta,
- base::TimeDelta());
- }
- }
- if (playout_time.is_null()) {
- // This can fail if we have not received any RTCP packets in a long time.
- if (rtcp_.RtpTimestampInSenderTime(frequency_, rtp_timestamp,
- &rtp_timestamp_in_ticks)) {
- playout_time =
- rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_;
- } else {
- playout_time = now;
- }
- }
-
- // TODO(miu): This is broken since we literally switch timelines once |rtcp_|
- // can provide us the |time_offset_|. Furthermore, this "getter" method may
- // be called on frames received out-of-order, which means the playout times
- // for earlier frames will be computed incorrectly.
-#if 0
- // Don't allow the playout time to go backwards.
- if (last_playout_time_ > playout_time) playout_time = last_playout_time_;
- last_playout_time_ = playout_time;
-#endif
-
- return playout_time;
-}
-
void AudioReceiver::ScheduleNextRtcpReport() {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
base::TimeDelta time_to_send = rtcp_.TimeToSendNextRtcpReport() -
diff --git a/media/cast/audio_receiver/audio_receiver.h b/media/cast/audio_receiver/audio_receiver.h
index 6ce01f6..454a62f 100644
--- a/media/cast/audio_receiver/audio_receiver.h
+++ b/media/cast/audio_receiver/audio_receiver.h
@@ -14,6 +14,7 @@
#include "base/threading/non_thread_safe.h"
#include "base/time/tick_clock.h"
#include "base/time/time.h"
+#include "media/cast/base/clock_drift_smoother.h"
#include "media/cast/cast_config.h"
#include "media/cast/cast_environment.h"
#include "media/cast/cast_receiver.h"
@@ -39,10 +40,6 @@ class AudioDecoder;
// each step of the pipeline (i.e., encode frame, then transmit/retransmit from
// the sender, then receive and re-order packets on the receiver, then decode
// frame) can vary in duration and is typically very hard to predict.
-// Heuristics will determine when the targeted playout delay is insufficient in
-// the current environment; and the receiver can then increase the playout
-// delay, notifying the sender, to account for the extra variance.
-// TODO(miu): Make the last sentence true. http://crbug.com/360111
//
// Two types of frames can be requested: 1) A frame of decoded audio data; or 2)
// a frame of still-encoded audio data, to be passed into an external audio
@@ -81,10 +78,6 @@ class AudioReceiver : public RtpReceiver,
// Deliver another packet, possibly a duplicate, and possibly out-of-order.
void IncomingPacket(scoped_ptr<Packet> packet);
- // Update target audio delay used to compute the playout time. Rtcp
- // will also be updated (will be included in all outgoing reports).
- void SetTargetDelay(base::TimeDelta target_delay);
-
protected:
friend class AudioReceiverTest; // Invokes OnReceivedPayloadData().
@@ -112,10 +105,10 @@ class AudioReceiver : public RtpReceiver,
const AudioFrameDecodedCallback& callback,
scoped_ptr<transport::EncodedFrame> encoded_frame);
- // Return the playout time based on the current time and rtp timestamp.
- base::TimeTicks GetPlayoutTime(base::TimeTicks now, uint32 rtp_timestamp);
-
- void InitializeTimers();
+ // Computes the playout time for a frame with the given |rtp_timestamp|.
+ // Because lip-sync info is refreshed regularly, calling this method with the
+ // same argument may return different results.
+ base::TimeTicks GetPlayoutTime(uint32 rtp_timestamp) const;
// Schedule the next RTCP report.
void ScheduleNextRtcpReport();
@@ -149,15 +142,39 @@ class AudioReceiver : public RtpReceiver,
// Processes raw audio events to be sent over to the cast sender via RTCP.
ReceiverRtcpEventSubscriber event_subscriber_;
+ // Configured audio codec.
const transport::AudioCodec codec_;
+
+ // RTP timebase: The number of RTP units advanced per one second. For audio,
+ // this is the sampling rate.
const int frequency_;
- base::TimeDelta target_delay_delta_;
+
+ // The total amount of time between a frame's capture/recording on the sender
+ // and its playback on the receiver (i.e., shown to a user). This is fixed as
+ // a value large enough to give the system sufficient time to encode,
+ // transmit/retransmit, receive, decode, and render; given its run-time
+ // environment (sender/receiver hardware performance, network conditions,
+ // etc.).
+ const base::TimeDelta target_playout_delay_;
+
+ // Set to false initially, then set to true after scheduling the periodic
+ // sending of reports back to the sender. Reports are first scheduled just
+ // after receiving a first packet (since the first packet identifies the
+ // sender for the remainder of the session).
+ bool reports_are_scheduled_;
+
+ // Assembles packets into frames, providing this receiver with complete,
+ // decodable EncodedFrames.
Framer framer_;
+
+ // Decodes frames into raw audio for playback.
scoped_ptr<AudioDecoder> audio_decoder_;
+
+ // Manages sending/receiving of RTCP packets, including sender/receiver
+ // reports.
Rtcp rtcp_;
- base::TimeDelta time_offset_;
- base::TimeTicks time_first_incoming_packet_;
- uint32 first_incoming_rtp_timestamp_;
+
+ // Decrypts encrypted frames.
transport::TransportEncryptionHandler decryptor_;
// Outstanding callbacks to run to deliver on client requests for frames.
@@ -171,6 +188,13 @@ class AudioReceiver : public RtpReceiver,
// it allows the event to be transmitted via RTCP.
RtpTimestamp frame_id_to_rtp_timestamp_[256];
+ // Lip-sync values used to compute the playout time of each frame from its RTP
+ // timestamp. These are updated each time the first packet of a frame is
+ // received.
+ RtpTimestamp lip_sync_rtp_timestamp_;
+ base::TimeTicks lip_sync_reference_time_;
+ ClockDriftSmoother lip_sync_drift_;
+
// NOTE: Weak pointers must be invalidated before all other member variables.
base::WeakPtrFactory<AudioReceiver> weak_factory_;
diff --git a/media/cast/audio_receiver/audio_receiver_unittest.cc b/media/cast/audio_receiver/audio_receiver_unittest.cc
index bcb7e34..9e2b1c21 100644
--- a/media/cast/audio_receiver/audio_receiver_unittest.cc
+++ b/media/cast/audio_receiver/audio_receiver_unittest.cc
@@ -2,6 +2,9 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+#include <deque>
+#include <utility>
+
#include "base/bind.h"
#include "base/memory/ref_counted.h"
#include "base/memory/scoped_ptr.h"
@@ -15,42 +18,44 @@
#include "media/cast/transport/pacing/mock_paced_packet_sender.h"
#include "testing/gmock/include/gmock/gmock.h"
+using ::testing::_;
+
namespace media {
namespace cast {
-using ::testing::_;
-
namespace {
-const int64 kStartMillisecond = INT64_C(12345678900000);
const uint32 kFirstFrameId = 1234;
+const int kPlayoutDelayMillis = 300;
class FakeAudioClient {
public:
FakeAudioClient() : num_called_(0) {}
virtual ~FakeAudioClient() {}
- void SetNextExpectedResult(uint32 expected_frame_id,
- const base::TimeTicks& expected_playout_time) {
- expected_frame_id_ = expected_frame_id;
- expected_playout_time_ = expected_playout_time;
+ void AddExpectedResult(uint32 expected_frame_id,
+ const base::TimeTicks& expected_playout_time) {
+ expected_results_.push_back(
+ std::make_pair(expected_frame_id, expected_playout_time));
}
void DeliverEncodedAudioFrame(
scoped_ptr<transport::EncodedFrame> audio_frame) {
+ SCOPED_TRACE(::testing::Message() << "num_called_ is " << num_called_);
ASSERT_FALSE(!audio_frame)
<< "If at shutdown: There were unsatisfied requests enqueued.";
- EXPECT_EQ(expected_frame_id_, audio_frame->frame_id);
- EXPECT_EQ(expected_playout_time_, audio_frame->reference_time);
+ ASSERT_FALSE(expected_results_.empty());
+ EXPECT_EQ(expected_results_.front().first, audio_frame->frame_id);
+ EXPECT_EQ(expected_results_.front().second, audio_frame->reference_time);
+ expected_results_.pop_front();
num_called_++;
}
int number_times_called() const { return num_called_; }
private:
+ std::deque<std::pair<uint32, base::TimeTicks> > expected_results_;
int num_called_;
- uint32 expected_frame_id_;
- base::TimeTicks expected_playout_time_;
DISALLOW_COPY_AND_ASSIGN(FakeAudioClient);
};
@@ -67,9 +72,11 @@ class AudioReceiverTest : public ::testing::Test {
audio_config_.codec = transport::kPcm16;
audio_config_.use_external_decoder = true;
audio_config_.feedback_ssrc = 1234;
+ audio_config_.incoming_ssrc = 5678;
+ audio_config_.rtp_max_delay_ms = kPlayoutDelayMillis;
testing_clock_ = new base::SimpleTestTickClock();
- testing_clock_->Advance(
- base::TimeDelta::FromMilliseconds(kStartMillisecond));
+ testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks());
+ start_time_ = testing_clock_->NowTicks();
task_runner_ = new test::FakeSingleThreadTaskRunner(testing_clock_);
cast_environment_ = new CastEnvironment(
@@ -99,10 +106,26 @@ class AudioReceiverTest : public ::testing::Test {
payload_.data(), payload_.size(), rtp_header_);
}
+ void FeedLipSyncInfoIntoReceiver() {
+ const base::TimeTicks now = testing_clock_->NowTicks();
+ const int64 rtp_timestamp = (now - start_time_) *
+ audio_config_.frequency / base::TimeDelta::FromSeconds(1);
+ CHECK_LE(0, rtp_timestamp);
+ uint32 ntp_seconds;
+ uint32 ntp_fraction;
+ ConvertTimeTicksToNtp(now, &ntp_seconds, &ntp_fraction);
+ TestRtcpPacketBuilder rtcp_packet;
+ rtcp_packet.AddSrWithNtp(audio_config_.incoming_ssrc,
+ ntp_seconds, ntp_fraction,
+ static_cast<uint32>(rtp_timestamp));
+ receiver_->IncomingPacket(rtcp_packet.GetPacket().Pass());
+ }
+
AudioReceiverConfig audio_config_;
std::vector<uint8> payload_;
RtpCastHeader rtp_header_;
base::SimpleTestTickClock* testing_clock_; // Owned by CastEnvironment.
+ base::TimeTicks start_time_;
transport::MockPacedPacketSender mock_transport_;
scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_;
scoped_refptr<CastEnvironment> cast_environment_;
@@ -113,11 +136,15 @@ class AudioReceiverTest : public ::testing::Test {
scoped_ptr<AudioReceiver> receiver_;
};
-TEST_F(AudioReceiverTest, GetOnePacketEncodedFrame) {
+TEST_F(AudioReceiverTest, ReceivesOneFrame) {
SimpleEventSubscriber event_subscriber;
cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber);
- EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _)).Times(1);
+ EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _))
+ .WillRepeatedly(testing::Return(true));
+
+ FeedLipSyncInfoIntoReceiver();
+ task_runner_->RunTasks();
// Enqueue a request for an audio frame.
receiver_->GetEncodedAudioFrame(
@@ -129,8 +156,10 @@ TEST_F(AudioReceiverTest, GetOnePacketEncodedFrame) {
EXPECT_EQ(0, fake_audio_client_.number_times_called());
// Deliver one audio frame to the receiver and expect to get one frame back.
- fake_audio_client_.SetNextExpectedResult(kFirstFrameId,
- testing_clock_->NowTicks());
+ const base::TimeDelta target_playout_delay =
+ base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis);
+ fake_audio_client_.AddExpectedResult(
+ kFirstFrameId, testing_clock_->NowTicks() + target_playout_delay);
FeedOneFrameIntoReceiver();
task_runner_->RunTasks();
EXPECT_EQ(1, fake_audio_client_.number_times_called());
@@ -147,10 +176,18 @@ TEST_F(AudioReceiverTest, GetOnePacketEncodedFrame) {
cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber);
}
-TEST_F(AudioReceiverTest, MultiplePendingGetCalls) {
+TEST_F(AudioReceiverTest, ReceivesFramesSkippingWhenAppropriate) {
EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _))
.WillRepeatedly(testing::Return(true));
+ const uint32 rtp_advance_per_frame = audio_config_.frequency / 100;
+ const base::TimeDelta time_advance_per_frame =
+ base::TimeDelta::FromMilliseconds(10);
+
+ FeedLipSyncInfoIntoReceiver();
+ task_runner_->RunTasks();
+ const base::TimeTicks first_frame_capture_time = testing_clock_->NowTicks();
+
// Enqueue a request for an audio frame.
const FrameEncodedCallback frame_encoded_callback =
base::Bind(&FakeAudioClient::DeliverEncodedAudioFrame,
@@ -160,24 +197,15 @@ TEST_F(AudioReceiverTest, MultiplePendingGetCalls) {
EXPECT_EQ(0, fake_audio_client_.number_times_called());
// Receive one audio frame and expect to see the first request satisfied.
- fake_audio_client_.SetNextExpectedResult(kFirstFrameId,
- testing_clock_->NowTicks());
+ const base::TimeDelta target_playout_delay =
+ base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis);
+ fake_audio_client_.AddExpectedResult(
+ kFirstFrameId, first_frame_capture_time + target_playout_delay);
+ rtp_header_.rtp_timestamp = 0;
FeedOneFrameIntoReceiver();
task_runner_->RunTasks();
EXPECT_EQ(1, fake_audio_client_.number_times_called());
- TestRtcpPacketBuilder rtcp_packet;
-
- uint32 ntp_high;
- uint32 ntp_low;
- ConvertTimeTicksToNtp(testing_clock_->NowTicks(), &ntp_high, &ntp_low);
- rtcp_packet.AddSrWithNtp(audio_config_.feedback_ssrc, ntp_high, ntp_low,
- rtp_header_.rtp_timestamp);
-
- testing_clock_->Advance(base::TimeDelta::FromMilliseconds(20));
-
- receiver_->IncomingPacket(rtcp_packet.GetPacket().Pass());
-
// Enqueue a second request for an audio frame, but it should not be
// fulfilled yet.
receiver_->GetEncodedAudioFrame(frame_encoded_callback);
@@ -189,10 +217,11 @@ TEST_F(AudioReceiverTest, MultiplePendingGetCalls) {
rtp_header_.is_key_frame = false;
rtp_header_.frame_id = kFirstFrameId + 2;
rtp_header_.reference_frame_id = 0;
- rtp_header_.rtp_timestamp = 960;
- fake_audio_client_.SetNextExpectedResult(
+ rtp_header_.rtp_timestamp += 2 * rtp_advance_per_frame;
+ fake_audio_client_.AddExpectedResult(
kFirstFrameId + 2,
- testing_clock_->NowTicks() + base::TimeDelta::FromMilliseconds(100));
+ first_frame_capture_time + 2 * time_advance_per_frame +
+ target_playout_delay);
FeedOneFrameIntoReceiver();
// Frame 2 should not come out at this point in time.
@@ -204,25 +233,27 @@ TEST_F(AudioReceiverTest, MultiplePendingGetCalls) {
task_runner_->RunTasks();
EXPECT_EQ(1, fake_audio_client_.number_times_called());
- // After 100 ms has elapsed, Frame 2 is emitted (to satisfy the second
- // request) because a decision was made to skip over the no-show Frame 1.
- testing_clock_->Advance(base::TimeDelta::FromMilliseconds(100));
+ // Now, advance time forward such that the receiver is convinced it should
+ // skip Frame 2. Frame 3 is emitted (to satisfy the second request) because a
+ // decision was made to skip over the no-show Frame 2.
+ testing_clock_->Advance(2 * time_advance_per_frame + target_playout_delay);
task_runner_->RunTasks();
EXPECT_EQ(2, fake_audio_client_.number_times_called());
- // Receive Frame 3 and expect it to fulfill the third request immediately.
+ // Receive Frame 4 and expect it to fulfill the third request immediately.
rtp_header_.frame_id = kFirstFrameId + 3;
rtp_header_.reference_frame_id = rtp_header_.frame_id - 1;
- rtp_header_.rtp_timestamp = 1280;
- fake_audio_client_.SetNextExpectedResult(kFirstFrameId + 3,
- testing_clock_->NowTicks());
+ rtp_header_.rtp_timestamp += rtp_advance_per_frame;
+ fake_audio_client_.AddExpectedResult(
+ kFirstFrameId + 3, first_frame_capture_time + 3 * time_advance_per_frame +
+ target_playout_delay);
FeedOneFrameIntoReceiver();
task_runner_->RunTasks();
EXPECT_EQ(3, fake_audio_client_.number_times_called());
- // Move forward another 100 ms and run any pending tasks (there should be
- // none). Expect no additional frames where emitted.
- testing_clock_->Advance(base::TimeDelta::FromMilliseconds(100));
+ // Move forward to the playout time of an unreceived Frame 5. Expect no
+ // additional frames were emitted.
+ testing_clock_->Advance(3 * time_advance_per_frame);
task_runner_->RunTasks();
EXPECT_EQ(3, fake_audio_client_.number_times_called());
}
diff --git a/media/cast/audio_sender/audio_sender_unittest.cc b/media/cast/audio_sender/audio_sender_unittest.cc
index 7c9b154..51edd49 100644
--- a/media/cast/audio_sender/audio_sender_unittest.cc
+++ b/media/cast/audio_sender/audio_sender_unittest.cc
@@ -22,8 +22,6 @@
namespace media {
namespace cast {
-static const int64 kStartMillisecond = INT64_C(12345678900000);
-
class TestPacketSender : public transport::PacketSender {
public:
TestPacketSender() : number_of_rtp_packets_(0), number_of_rtcp_packets_(0) {}
@@ -60,8 +58,7 @@ class AudioSenderTest : public ::testing::Test {
AudioSenderTest() {
InitializeMediaLibraryForTesting();
testing_clock_ = new base::SimpleTestTickClock();
- testing_clock_->Advance(
- base::TimeDelta::FromMilliseconds(kStartMillisecond));
+ testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks());
task_runner_ = new test::FakeSingleThreadTaskRunner(testing_clock_);
cast_environment_ =
new CastEnvironment(scoped_ptr<base::TickClock>(testing_clock_).Pass(),
diff --git a/media/cast/base/clock_drift_smoother.cc b/media/cast/base/clock_drift_smoother.cc
new file mode 100644
index 0000000..ca03805
--- /dev/null
+++ b/media/cast/base/clock_drift_smoother.cc
@@ -0,0 +1,58 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/cast/base/clock_drift_smoother.h"
+
+#include "base/logging.h"
+
+namespace media {
+namespace cast {
+
+ClockDriftSmoother::ClockDriftSmoother(base::TimeDelta time_constant)
+ : time_constant_(time_constant),
+ estimate_us_(0.0) {
+ DCHECK(time_constant_ > base::TimeDelta());
+}
+
+ClockDriftSmoother::~ClockDriftSmoother() {}
+
+base::TimeDelta ClockDriftSmoother::Current() const {
+ DCHECK(!last_update_time_.is_null());
+ return base::TimeDelta::FromMicroseconds(
+ static_cast<int64>(estimate_us_ + 0.5)); // Round to nearest microsecond.
+}
+
+void ClockDriftSmoother::Reset(base::TimeTicks now,
+ base::TimeDelta measured_offset) {
+ DCHECK(!now.is_null());
+ last_update_time_ = now;
+ estimate_us_ = measured_offset.InMicroseconds();
+}
+
+void ClockDriftSmoother::Update(base::TimeTicks now,
+ base::TimeDelta measured_offset) {
+ DCHECK(!now.is_null());
+ if (last_update_time_.is_null()) {
+ Reset(now, measured_offset);
+ } else if (now < last_update_time_) {
+ // |now| is not monotonically non-decreasing.
+ NOTREACHED();
+ } else {
+ const double elapsed_us = (now - last_update_time_).InMicroseconds();
+ last_update_time_ = now;
+ const double weight =
+ elapsed_us / (elapsed_us + time_constant_.InMicroseconds());
+ estimate_us_ = weight * measured_offset.InMicroseconds() +
+ (1.0 - weight) * estimate_us_;
+ }
+}
+
+// static
+base::TimeDelta ClockDriftSmoother::GetDefaultTimeConstant() {
+ static const int kDefaultTimeConstantInSeconds = 30;
+ return base::TimeDelta::FromSeconds(kDefaultTimeConstantInSeconds);
+}
+
+} // namespace cast
+} // namespace media
diff --git a/media/cast/base/clock_drift_smoother.h b/media/cast/base/clock_drift_smoother.h
new file mode 100644
index 0000000..67de4cb
--- /dev/null
+++ b/media/cast/base/clock_drift_smoother.h
@@ -0,0 +1,52 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_CAST_BASE_CLOCK_DRIFT_SMOOTHER_H_
+#define MEDIA_CAST_BASE_CLOCK_DRIFT_SMOOTHER_H_
+
+#include "base/time/time.h"
+
+namespace media {
+namespace cast {
+
+// Tracks the jitter and drift between clocks, providing a smoothed offset.
+// Internally, a Simple IIR filter is used to maintain a running average that
+// moves at a rate based on the passage of time.
+class ClockDriftSmoother {
+ public:
+ // |time_constant| is the amount of time an impulse signal takes to decay by
+ // ~62.6%. Interpretation: If the value passed to several Update() calls is
+ // held constant for T seconds, then the running average will have moved
+ // towards the value by ~62.6% from where it started.
+ explicit ClockDriftSmoother(base::TimeDelta time_constant);
+ ~ClockDriftSmoother();
+
+ // Returns the current offset.
+ base::TimeDelta Current() const;
+
+ // Discard all history and reset to exactly |offset|, measured |now|.
+ void Reset(base::TimeTicks now, base::TimeDelta offset);
+
+ // Update the current offset, which was measured |now|. The weighting that
+ // |measured_offset| will have on the running average is influenced by how
+ // much time has passed since the last call to this method (or Reset()).
+ // |now| should be monotonically non-decreasing over successive calls of this
+ // method.
+ void Update(base::TimeTicks now, base::TimeDelta measured_offset);
+
+ // Returns a time constant suitable for most use cases, where the clocks
+ // are expected to drift very little with respect to each other, and the
+ // jitter caused by clock imprecision is effectively canceled out.
+ static base::TimeDelta GetDefaultTimeConstant();
+
+ private:
+ const base::TimeDelta time_constant_;
+ base::TimeTicks last_update_time_;
+ double estimate_us_;
+};
+
+} // namespace cast
+} // namespace media
+
+#endif // MEDIA_CAST_BASE_CLOCK_DRIFT_SMOOTHER_H_
diff --git a/media/cast/cast.gyp b/media/cast/cast.gyp
index ea44430..4f0c2dd 100644
--- a/media/cast/cast.gyp
+++ b/media/cast/cast.gyp
@@ -33,6 +33,8 @@
'cast_defines.h',
'cast_environment.cc',
'cast_environment.h',
+ 'base/clock_drift_smoother.cc',
+ 'base/clock_drift_smoother.h',
'logging/encoding_event_subscriber.cc',
'logging/encoding_event_subscriber.h',
'logging/log_deserializer.cc',
diff --git a/media/cast/cast_defines.h b/media/cast/cast_defines.h
index 7f68d67..afb50e0 100644
--- a/media/cast/cast_defines.h
+++ b/media/cast/cast_defines.h
@@ -135,13 +135,22 @@ inline base::TimeDelta ConvertFromNtpDiff(uint32 ntp_delay) {
return base::TimeDelta::FromMilliseconds(delay_ms);
}
-inline void ConvertTimeToFractions(int64 time_us,
+inline void ConvertTimeToFractions(int64 ntp_time_us,
uint32* seconds,
uint32* fractions) {
- DCHECK_GE(time_us, 0) << "Time must NOT be negative";
- *seconds = static_cast<uint32>(time_us / base::Time::kMicrosecondsPerSecond);
+ DCHECK_GE(ntp_time_us, 0) << "Time must NOT be negative";
+ const int64 seconds_component =
+ ntp_time_us / base::Time::kMicrosecondsPerSecond;
+ // NTP time will overflow in the year 2036. Also, make sure unit tests don't
+ // regress and use an origin past the year 2036. If this overflows here, the
+ // inverse calculation fails to compute the correct TimeTicks value, throwing
+ // off the entire system.
+ DCHECK_LT(seconds_component, INT64_C(4263431296))
+ << "One year left to fix the NTP year 2036 wrap-around issue!";
+ *seconds = static_cast<uint32>(seconds_component);
*fractions = static_cast<uint32>(
- (time_us % base::Time::kMicrosecondsPerSecond) * kMagicFractionalUnit);
+ (ntp_time_us % base::Time::kMicrosecondsPerSecond) *
+ kMagicFractionalUnit);
}
inline void ConvertTimeTicksToNtp(const base::TimeTicks& time,
@@ -169,6 +178,11 @@ inline base::TimeTicks ConvertNtpToTimeTicks(uint32 ntp_seconds,
return base::TimeTicks::UnixEpoch() + elapsed_since_unix_epoch;
}
+inline base::TimeDelta RtpDeltaToTimeDelta(int64 rtp_delta, int rtp_timebase) {
+ DCHECK_GT(rtp_timebase, 0);
+ return rtp_delta * base::TimeDelta::FromSeconds(1) / rtp_timebase;
+}
+
inline uint32 GetVideoRtpTimestamp(const base::TimeTicks& time_ticks) {
base::TimeTicks zero_time;
base::TimeDelta recorded_delta = time_ticks - zero_time;
diff --git a/media/cast/rtcp/rtcp.cc b/media/cast/rtcp/rtcp.cc
index 40d2961..3aa936b 100644
--- a/media/cast/rtcp/rtcp.cc
+++ b/media/cast/rtcp/rtcp.cc
@@ -19,10 +19,7 @@ namespace media {
namespace cast {
static const int kMaxRttMs = 10000; // 10 seconds.
-static const uint16 kMaxDelay = 2000;
-
-// Time limit for received RTCP messages when we stop using it for lip-sync.
-static const int64 kMaxDiffSinceReceivedRtcpMs = 100000; // 100 seconds.
+static const int kMaxDelay = 2000;
class LocalRtcpRttFeedback : public RtcpRttFeedback {
public:
@@ -95,10 +92,10 @@ Rtcp::Rtcp(scoped_refptr<CastEnvironment> cast_environment,
receiver_feedback_(new LocalRtcpReceiverFeedback(this, cast_environment)),
rtcp_sender_(new RtcpSender(cast_environment, paced_packet_sender,
local_ssrc, c_name)),
- last_report_received_(0),
- last_received_rtp_timestamp_(0),
- last_received_ntp_seconds_(0),
- last_received_ntp_fraction_(0),
+ last_report_truncated_ntp_(0),
+ local_clock_ahead_by_(ClockDriftSmoother::GetDefaultTimeConstant()),
+ lip_sync_rtp_timestamp_(0),
+ lip_sync_ntp_timestamp_(0),
min_rtt_(base::TimeDelta::FromMilliseconds(kMaxRttMs)),
number_of_rtt_in_avg_(0),
is_audio_(is_audio) {
@@ -184,7 +181,7 @@ void Rtcp::SendRtcpFromRtpReceiver(
&report_block.extended_high_sequence_number, &report_block.jitter);
}
- report_block.last_sr = last_report_received_;
+ report_block.last_sr = last_report_truncated_ntp_;
if (!time_last_report_received_.is_null()) {
uint32 delay_seconds = 0;
uint32 delay_fraction = 0;
@@ -220,7 +217,7 @@ void Rtcp::SendRtcpFromRtpSender(base::TimeTicks current_time,
transport::RtcpDlrrReportBlock dlrr;
if (!time_last_report_received_.is_null()) {
packet_type_flags |= transport::kRtcpDlrr;
- dlrr.last_rr = last_report_received_;
+ dlrr.last_rr = last_report_truncated_ntp_;
uint32 delay_seconds = 0;
uint32 delay_fraction = 0;
base::TimeDelta delta = current_time - time_last_report_received_;
@@ -237,58 +234,66 @@ void Rtcp::SendRtcpFromRtpSender(base::TimeTicks current_time,
}
void Rtcp::OnReceivedNtp(uint32 ntp_seconds, uint32 ntp_fraction) {
- last_report_received_ = (ntp_seconds << 16) + (ntp_fraction >> 16);
+ last_report_truncated_ntp_ = ConvertToNtpDiff(ntp_seconds, ntp_fraction);
- base::TimeTicks now = cast_environment_->Clock()->NowTicks();
+ const base::TimeTicks now = cast_environment_->Clock()->NowTicks();
time_last_report_received_ = now;
+
+ // TODO(miu): This clock offset calculation does not account for packet
+ // transit time over the network. End2EndTest.EvilNetwork confirms that this
+ // contributes a very significant source of error here. Fix this along with
+ // the RTT clean-up.
+ const base::TimeDelta measured_offset =
+ now - ConvertNtpToTimeTicks(ntp_seconds, ntp_fraction);
+ local_clock_ahead_by_.Update(now, measured_offset);
+ if (measured_offset < local_clock_ahead_by_.Current()) {
+ // Logically, the minimum offset between the clocks has to be the correct
+ // one. For example, the time it took to transmit the current report may
+ // have been lower than usual, and so some of the error introduced by the
+ // transmission time can be eliminated.
+ local_clock_ahead_by_.Reset(now, measured_offset);
+ }
+ VLOG(1) << "Local clock is ahead of the remote clock by: "
+ << "measured=" << measured_offset.InMicroseconds() << " usec, "
+ << "filtered=" << local_clock_ahead_by_.Current().InMicroseconds()
+ << " usec.";
}
void Rtcp::OnReceivedLipSyncInfo(uint32 rtp_timestamp, uint32 ntp_seconds,
uint32 ntp_fraction) {
- last_received_rtp_timestamp_ = rtp_timestamp;
- last_received_ntp_seconds_ = ntp_seconds;
- last_received_ntp_fraction_ = ntp_fraction;
-}
-
-void Rtcp::OnReceivedSendReportRequest() {
- base::TimeTicks now = cast_environment_->Clock()->NowTicks();
-
- // Trigger a new RTCP report at next timer.
- next_time_to_send_rtcp_ = now;
+ if (ntp_seconds == 0) {
+ NOTREACHED();
+ return;
+ }
+ lip_sync_rtp_timestamp_ = rtp_timestamp;
+ lip_sync_ntp_timestamp_ =
+ (static_cast<uint64>(ntp_seconds) << 32) | ntp_fraction;
}
-bool Rtcp::RtpTimestampInSenderTime(int frequency, uint32 rtp_timestamp,
- base::TimeTicks* rtp_timestamp_in_ticks)
- const {
- if (last_received_ntp_seconds_ == 0)
+bool Rtcp::GetLatestLipSyncTimes(uint32* rtp_timestamp,
+ base::TimeTicks* reference_time) const {
+ if (!lip_sync_ntp_timestamp_)
return false;
- int wrap = CheckForWrapAround(rtp_timestamp, last_received_rtp_timestamp_);
- int64 rtp_timestamp_int64 = rtp_timestamp;
- int64 last_received_rtp_timestamp_int64 = last_received_rtp_timestamp_;
+ const base::TimeTicks local_reference_time =
+ ConvertNtpToTimeTicks(static_cast<uint32>(lip_sync_ntp_timestamp_ >> 32),
+ static_cast<uint32>(lip_sync_ntp_timestamp_)) +
+ local_clock_ahead_by_.Current();
- if (wrap == 1) {
- rtp_timestamp_int64 += (1LL << 32);
- } else if (wrap == -1) {
- last_received_rtp_timestamp_int64 += (1LL << 32);
- }
- // Time since the last RTCP message.
- // Note that this can be negative since we can compare a rtp timestamp from
- // a frame older than the last received RTCP message.
- int64 rtp_timestamp_diff =
- rtp_timestamp_int64 - last_received_rtp_timestamp_int64;
+ // Sanity-check: Getting regular lip sync updates?
+ DCHECK((cast_environment_->Clock()->NowTicks() - local_reference_time) <
+ base::TimeDelta::FromMinutes(1));
- int frequency_khz = frequency / 1000;
- int64 rtp_time_diff_ms = rtp_timestamp_diff / frequency_khz;
+ *rtp_timestamp = lip_sync_rtp_timestamp_;
+ *reference_time = local_reference_time;
+ return true;
+}
- // Sanity check.
- if (std::abs(rtp_time_diff_ms) > kMaxDiffSinceReceivedRtcpMs)
- return false;
+void Rtcp::OnReceivedSendReportRequest() {
+ base::TimeTicks now = cast_environment_->Clock()->NowTicks();
- *rtp_timestamp_in_ticks = ConvertNtpToTimeTicks(last_received_ntp_seconds_,
- last_received_ntp_fraction_) +
- base::TimeDelta::FromMilliseconds(rtp_time_diff_ms);
- return true;
+ // Trigger a new RTCP report at next timer.
+ next_time_to_send_rtcp_ = now;
}
void Rtcp::SetCastReceiverEventHistorySize(size_t size) {
@@ -296,8 +301,8 @@ void Rtcp::SetCastReceiverEventHistorySize(size_t size) {
}
void Rtcp::SetTargetDelay(base::TimeDelta target_delay) {
+ DCHECK(target_delay.InMilliseconds() < kMaxDelay);
target_delay_ms_ = static_cast<uint16>(target_delay.InMilliseconds());
- DCHECK(target_delay_ms_ < kMaxDelay);
}
void Rtcp::OnReceivedDelaySinceLastReport(uint32 receivers_ssrc,
@@ -342,17 +347,21 @@ void Rtcp::SaveLastSentNtpTime(const base::TimeTicks& now,
void Rtcp::UpdateRtt(const base::TimeDelta& sender_delay,
const base::TimeDelta& receiver_delay) {
base::TimeDelta rtt = sender_delay - receiver_delay;
+ // TODO(miu): Find out why this must be >= 1 ms, and remove the fudge if it's
+ // bogus.
rtt = std::max(rtt, base::TimeDelta::FromMilliseconds(1));
rtt_ = rtt;
min_rtt_ = std::min(min_rtt_, rtt);
max_rtt_ = std::max(max_rtt_, rtt);
+ // TODO(miu): Replace "average for all time" with an EWMA, or suitable
+ // "average over recent past" mechanism.
if (number_of_rtt_in_avg_ != 0) {
- float ac = static_cast<float>(number_of_rtt_in_avg_);
+ const double ac = static_cast<double>(number_of_rtt_in_avg_);
avg_rtt_ms_ = ((ac / (ac + 1.0)) * avg_rtt_ms_) +
- ((1.0 / (ac + 1.0)) * rtt.InMilliseconds());
+ ((1.0 / (ac + 1.0)) * rtt.InMillisecondsF());
} else {
- avg_rtt_ms_ = rtt.InMilliseconds();
+ avg_rtt_ms_ = rtt.InMillisecondsF();
}
number_of_rtt_in_avg_++;
}
@@ -367,28 +376,12 @@ bool Rtcp::Rtt(base::TimeDelta* rtt, base::TimeDelta* avg_rtt,
if (number_of_rtt_in_avg_ == 0) return false;
*rtt = rtt_;
- *avg_rtt = base::TimeDelta::FromMilliseconds(avg_rtt_ms_);
+ *avg_rtt = base::TimeDelta::FromMillisecondsD(avg_rtt_ms_);
*min_rtt = min_rtt_;
*max_rtt = max_rtt_;
return true;
}
-int Rtcp::CheckForWrapAround(uint32 new_timestamp, uint32 old_timestamp) const {
- if (new_timestamp < old_timestamp) {
- // This difference should be less than -2^31 if we have had a wrap around
- // (e.g. |new_timestamp| = 1, |rtcp_rtp_timestamp| = 2^32 - 1). Since it is
- // cast to a int32_t, it should be positive.
- if (static_cast<int32>(new_timestamp - old_timestamp) > 0) {
- return 1; // Forward wrap around.
- }
- } else if (static_cast<int32>(old_timestamp - new_timestamp) > 0) {
- // This difference should be less than -2^31 if we have had a backward wrap
- // around. Since it is cast to a int32, it should be positive.
- return -1;
- }
- return 0;
-}
-
void Rtcp::UpdateNextTimeToSendRtcp() {
int random = base::RandInt(0, 999);
base::TimeDelta time_to_next =
diff --git a/media/cast/rtcp/rtcp.h b/media/cast/rtcp/rtcp.h
index 9d152f0..ff81bb9 100644
--- a/media/cast/rtcp/rtcp.h
+++ b/media/cast/rtcp/rtcp.h
@@ -5,16 +5,15 @@
#ifndef MEDIA_CAST_RTCP_RTCP_H_
#define MEDIA_CAST_RTCP_RTCP_H_
-#include <list>
#include <map>
#include <queue>
-#include <set>
#include <string>
#include "base/basictypes.h"
#include "base/memory/scoped_ptr.h"
#include "base/time/tick_clock.h"
#include "base/time/time.h"
+#include "media/cast/base/clock_drift_smoother.h"
#include "media/cast/cast_config.h"
#include "media/cast/cast_defines.h"
#include "media/cast/cast_environment.h"
@@ -94,27 +93,39 @@ class Rtcp {
const ReceiverRtcpEventSubscriber::RtcpEventMultiMap* rtcp_events);
void IncomingRtcpPacket(const uint8* rtcp_buffer, size_t length);
+
+ // TODO(miu): Clean up this method and downstream code: Only VideoSender uses
+ // this (for congestion control), and only the |rtt| and |avg_rtt| values, and
+ // it's not clear that any of the downstream code is doing the right thing
+ // with this data.
bool Rtt(base::TimeDelta* rtt,
base::TimeDelta* avg_rtt,
base::TimeDelta* min_rtt,
base::TimeDelta* max_rtt) const;
+
bool is_rtt_available() const { return number_of_rtt_in_avg_ > 0; }
- bool RtpTimestampInSenderTime(int frequency,
- uint32 rtp_timestamp,
- base::TimeTicks* rtp_timestamp_in_ticks) const;
+
+ // If available, returns true and sets the output arguments to the latest
+ // lip-sync timestamps gleaned from the sender reports. While the sender
+ // provides reference NTP times relative to its own wall clock, the
+ // |reference_time| returned here has been translated to the local
+ // CastEnvironment clock.
+ bool GetLatestLipSyncTimes(uint32* rtp_timestamp,
+ base::TimeTicks* reference_time) const;
// Set the history size to record Cast receiver events. The event history is
// used to remove duplicates. The history will store at most |size| events.
void SetCastReceiverEventHistorySize(size_t size);
- // Update the target delay. Will be added to every sender report.
+ // Update the target delay. Will be added to every report sent back to the
+ // sender.
+ // TODO(miu): Remove this deprecated functionality. The sender ignores this.
void SetTargetDelay(base::TimeDelta target_delay);
void OnReceivedReceiverLog(const RtcpReceiverLogMessage& receiver_log);
protected:
- int CheckForWrapAround(uint32 new_timestamp, uint32 old_timestamp) const;
-
+ void OnReceivedNtp(uint32 ntp_seconds, uint32 ntp_fraction);
void OnReceivedLipSyncInfo(uint32 rtp_timestamp,
uint32 ntp_seconds,
uint32 ntp_fraction);
@@ -123,13 +134,6 @@ class Rtcp {
friend class LocalRtcpRttFeedback;
friend class LocalRtcpReceiverFeedback;
- void SendRtcp(const base::TimeTicks& now,
- uint32 packet_type_flags,
- uint32 media_ssrc,
- const RtcpCastMessage* cast_message);
-
- void OnReceivedNtp(uint32 ntp_seconds, uint32 ntp_fraction);
-
void OnReceivedDelaySinceLastReport(uint32 receivers_ssrc,
uint32 last_report,
uint32 delay_since_last_report);
@@ -164,18 +168,32 @@ class Rtcp {
base::TimeTicks next_time_to_send_rtcp_;
RtcpSendTimeMap last_reports_sent_map_;
RtcpSendTimeQueue last_reports_sent_queue_;
+
+ // The truncated (i.e., 64-->32-bit) NTP timestamp provided in the last report
+ // from the remote peer, along with the local time at which the report was
+ // received. These values are used for ping-pong'ing NTP timestamps between
+ // the peers so that they can estimate the network's round-trip time.
+ uint32 last_report_truncated_ntp_;
base::TimeTicks time_last_report_received_;
- uint32 last_report_received_;
- uint32 last_received_rtp_timestamp_;
- uint32 last_received_ntp_seconds_;
- uint32 last_received_ntp_fraction_;
+ // Maintains a smoothed offset between the local clock and the remote clock.
+ // Calling this member's Current() method is only valid if
+ // |time_last_report_received_| is not "null."
+ ClockDriftSmoother local_clock_ahead_by_;
+
+ // Latest "lip sync" info from the sender. The sender provides the RTP
+ // timestamp of some frame of its choosing and also a corresponding reference
+ // NTP timestamp sampled from a clock common to all media streams. It is
+ // expected that the sender will update this data regularly and in a timely
+ // manner (e.g., about once per second).
+ uint32 lip_sync_rtp_timestamp_;
+ uint64 lip_sync_ntp_timestamp_;
base::TimeDelta rtt_;
base::TimeDelta min_rtt_;
base::TimeDelta max_rtt_;
int number_of_rtt_in_avg_;
- float avg_rtt_ms_;
+ double avg_rtt_ms_;
uint16 target_delay_ms_;
bool is_audio_;
diff --git a/media/cast/rtcp/rtcp_unittest.cc b/media/cast/rtcp/rtcp_unittest.cc
index b8f5388..f864a00 100644
--- a/media/cast/rtcp/rtcp_unittest.cc
+++ b/media/cast/rtcp/rtcp_unittest.cc
@@ -142,7 +142,7 @@ class RtcpPeer : public Rtcp {
c_name,
true) {}
- using Rtcp::CheckForWrapAround;
+ using Rtcp::OnReceivedNtp;
using Rtcp::OnReceivedLipSyncInfo;
};
@@ -204,7 +204,7 @@ class RtcpTest : public ::testing::Test {
};
TEST_F(RtcpTest, TimeToSend) {
- base::TimeTicks start_time = testing_clock_->NowTicks();
+ const base::TimeTicks start_time = testing_clock_->NowTicks();
Rtcp rtcp(cast_environment_,
&mock_sender_feedback_,
transport_sender_.get(),
@@ -498,136 +498,45 @@ TEST_F(RtcpTest, NtpAndTime) {
const int64 kSecondsbetweenYear1900and2030 = INT64_C(47481 * 24 * 60 * 60);
uint32 ntp_seconds_1 = 0;
- uint32 ntp_fractions_1 = 0;
+ uint32 ntp_fraction_1 = 0;
base::TimeTicks input_time = base::TimeTicks::Now();
- ConvertTimeTicksToNtp(input_time, &ntp_seconds_1, &ntp_fractions_1);
+ ConvertTimeTicksToNtp(input_time, &ntp_seconds_1, &ntp_fraction_1);
// Verify absolute value.
EXPECT_GT(ntp_seconds_1, kSecondsbetweenYear1900and2010);
EXPECT_LT(ntp_seconds_1, kSecondsbetweenYear1900and2030);
- base::TimeTicks out_1 = ConvertNtpToTimeTicks(ntp_seconds_1, ntp_fractions_1);
+ base::TimeTicks out_1 = ConvertNtpToTimeTicks(ntp_seconds_1, ntp_fraction_1);
EXPECT_EQ(input_time, out_1); // Verify inverse.
base::TimeDelta time_delta = base::TimeDelta::FromMilliseconds(1000);
input_time += time_delta;
uint32 ntp_seconds_2 = 0;
- uint32 ntp_fractions_2 = 0;
+ uint32 ntp_fraction_2 = 0;
- ConvertTimeTicksToNtp(input_time, &ntp_seconds_2, &ntp_fractions_2);
- base::TimeTicks out_2 = ConvertNtpToTimeTicks(ntp_seconds_2, ntp_fractions_2);
+ ConvertTimeTicksToNtp(input_time, &ntp_seconds_2, &ntp_fraction_2);
+ base::TimeTicks out_2 = ConvertNtpToTimeTicks(ntp_seconds_2, ntp_fraction_2);
EXPECT_EQ(input_time, out_2); // Verify inverse.
// Verify delta.
EXPECT_EQ((out_2 - out_1), time_delta);
EXPECT_EQ((ntp_seconds_2 - ntp_seconds_1), UINT32_C(1));
- EXPECT_NEAR(ntp_fractions_2, ntp_fractions_1, 1);
+ EXPECT_NEAR(ntp_fraction_2, ntp_fraction_1, 1);
time_delta = base::TimeDelta::FromMilliseconds(500);
input_time += time_delta;
uint32 ntp_seconds_3 = 0;
- uint32 ntp_fractions_3 = 0;
+ uint32 ntp_fraction_3 = 0;
- ConvertTimeTicksToNtp(input_time, &ntp_seconds_3, &ntp_fractions_3);
- base::TimeTicks out_3 = ConvertNtpToTimeTicks(ntp_seconds_3, ntp_fractions_3);
+ ConvertTimeTicksToNtp(input_time, &ntp_seconds_3, &ntp_fraction_3);
+ base::TimeTicks out_3 = ConvertNtpToTimeTicks(ntp_seconds_3, ntp_fraction_3);
EXPECT_EQ(input_time, out_3); // Verify inverse.
// Verify delta.
EXPECT_EQ((out_3 - out_2), time_delta);
- EXPECT_NEAR((ntp_fractions_3 - ntp_fractions_2), 0xffffffff / 2, 1);
-}
-
-TEST_F(RtcpTest, WrapAround) {
- RtcpPeer rtcp_peer(cast_environment_,
- &mock_sender_feedback_,
- transport_sender_.get(),
- NULL,
- NULL,
- kRtcpReducedSize,
- base::TimeDelta::FromMilliseconds(kRtcpIntervalMs),
- kReceiverSsrc,
- kSenderSsrc,
- kCName);
- uint32 new_timestamp = 0;
- uint32 old_timestamp = 0;
- EXPECT_EQ(0, rtcp_peer.CheckForWrapAround(new_timestamp, old_timestamp));
- new_timestamp = 1234567890;
- old_timestamp = 1234567000;
- EXPECT_EQ(0, rtcp_peer.CheckForWrapAround(new_timestamp, old_timestamp));
- new_timestamp = 1234567000;
- old_timestamp = 1234567890;
- EXPECT_EQ(0, rtcp_peer.CheckForWrapAround(new_timestamp, old_timestamp));
- new_timestamp = 123;
- old_timestamp = 4234567890u;
- EXPECT_EQ(1, rtcp_peer.CheckForWrapAround(new_timestamp, old_timestamp));
- new_timestamp = 4234567890u;
- old_timestamp = 123;
- EXPECT_EQ(-1, rtcp_peer.CheckForWrapAround(new_timestamp, old_timestamp));
-}
-
-TEST_F(RtcpTest, RtpTimestampInSenderTime) {
- RtcpPeer rtcp_peer(cast_environment_,
- &mock_sender_feedback_,
- transport_sender_.get(),
- &receiver_to_sender_,
- NULL,
- kRtcpReducedSize,
- base::TimeDelta::FromMilliseconds(kRtcpIntervalMs),
- kReceiverSsrc,
- kSenderSsrc,
- kCName);
- int frequency = 32000;
- uint32 rtp_timestamp = 64000;
- base::TimeTicks rtp_timestamp_in_ticks;
-
- // Test fail before we get a OnReceivedLipSyncInfo.
- EXPECT_FALSE(rtcp_peer.RtpTimestampInSenderTime(
- frequency, rtp_timestamp, &rtp_timestamp_in_ticks));
-
- uint32 ntp_seconds = 0;
- uint32 ntp_fractions = 0;
- uint64 input_time_us = 12345678901000LL;
- base::TimeTicks input_time;
- input_time += base::TimeDelta::FromMicroseconds(input_time_us);
-
- // Test exact match.
- ConvertTimeTicksToNtp(input_time, &ntp_seconds, &ntp_fractions);
- rtcp_peer.OnReceivedLipSyncInfo(rtp_timestamp, ntp_seconds, ntp_fractions);
- EXPECT_TRUE(rtcp_peer.RtpTimestampInSenderTime(
- frequency, rtp_timestamp, &rtp_timestamp_in_ticks));
- EXPECT_EQ(input_time, rtp_timestamp_in_ticks);
-
- // Test older rtp_timestamp.
- rtp_timestamp = 32000;
- EXPECT_TRUE(rtcp_peer.RtpTimestampInSenderTime(
- frequency, rtp_timestamp, &rtp_timestamp_in_ticks));
- EXPECT_EQ(input_time - base::TimeDelta::FromMilliseconds(1000),
- rtp_timestamp_in_ticks);
-
- // Test older rtp_timestamp with wrap.
- rtp_timestamp = 4294903296u;
- EXPECT_TRUE(rtcp_peer.RtpTimestampInSenderTime(
- frequency, rtp_timestamp, &rtp_timestamp_in_ticks));
- EXPECT_EQ(input_time - base::TimeDelta::FromMilliseconds(4000),
- rtp_timestamp_in_ticks);
-
- // Test newer rtp_timestamp.
- rtp_timestamp = 128000;
- EXPECT_TRUE(rtcp_peer.RtpTimestampInSenderTime(
- frequency, rtp_timestamp, &rtp_timestamp_in_ticks));
- EXPECT_EQ(input_time + base::TimeDelta::FromMilliseconds(2000),
- rtp_timestamp_in_ticks);
-
- // Test newer rtp_timestamp with wrap.
- rtp_timestamp = 4294903296u;
- rtcp_peer.OnReceivedLipSyncInfo(rtp_timestamp, ntp_seconds, ntp_fractions);
- rtp_timestamp = 64000;
- EXPECT_TRUE(rtcp_peer.RtpTimestampInSenderTime(
- frequency, rtp_timestamp, &rtp_timestamp_in_ticks));
- EXPECT_EQ(input_time + base::TimeDelta::FromMilliseconds(4000),
- rtp_timestamp_in_ticks);
+ EXPECT_NEAR((ntp_fraction_3 - ntp_fraction_2), 0xffffffff / 2, 1);
}
} // namespace cast
diff --git a/media/cast/test/end2end_unittest.cc b/media/cast/test/end2end_unittest.cc
index f46d526..ae55fae 100644
--- a/media/cast/test/end2end_unittest.cc
+++ b/media/cast/test/end2end_unittest.cc
@@ -63,16 +63,11 @@ static const double kVideoAcceptedPSNR = 38.0;
// The tests are commonly implemented with |kFrameTimerMs| RunTask function;
// a normal video is 30 fps hence the 33 ms between frames.
+//
+// TODO(miu): The errors in timing will add up significantly. Find an
+// alternative approach that eliminates use of this constant.
static const int kFrameTimerMs = 33;
-// The packets pass through the pacer which can delay the beginning of the
-// frame by 10 ms if there is packets belonging to the previous frame being
-// retransmitted.
-// In addition, audio packets are sent in 10mS intervals in audio_encoder.cc,
-// although we send an audio frame every 33mS, which adds an extra delay.
-// A TODO was added in the code to resolve this.
-static const int kTimerErrorMs = 20;
-
// Start the video synthetic start value to medium range value, to avoid edge
// effects cause by encoding and quantization.
static const int kVideoStart = 100;
@@ -81,6 +76,14 @@ static const int kVideoStart = 100;
// chunks of this size.
static const int kAudioFrameDurationMs = 10;
+// The amount of time between frame capture on the sender and playout on the
+// receiver.
+static const int kTargetPlayoutDelayMs = 100;
+
+// The maximum amount of deviation expected in the playout times emitted by the
+// receiver.
+static const int kMaxAllowedPlayoutErrorMs = 30;
+
std::string ConvertFromBase16String(const std::string base_16) {
std::string compressed;
DCHECK_EQ(base_16.size() % 2, 0u) << "Must be a multiple of 2";
@@ -244,7 +247,7 @@ class TestReceiverAudioCallback
public:
struct ExpectedAudioFrame {
scoped_ptr<AudioBus> audio_bus;
- base::TimeTicks record_time;
+ base::TimeTicks playout_time;
};
TestReceiverAudioCallback() : num_called_(0) {}
@@ -254,13 +257,13 @@ class TestReceiverAudioCallback
}
void AddExpectedResult(const AudioBus& audio_bus,
- const base::TimeTicks& record_time) {
+ const base::TimeTicks& playout_time) {
scoped_ptr<ExpectedAudioFrame> expected_audio_frame(
new ExpectedAudioFrame());
expected_audio_frame->audio_bus =
AudioBus::Create(audio_bus.channels(), audio_bus.frames()).Pass();
audio_bus.CopyTo(expected_audio_frame->audio_bus.get());
- expected_audio_frame->record_time = record_time;
+ expected_audio_frame->playout_time = playout_time;
expected_frames_.push_back(expected_audio_frame.release());
}
@@ -292,15 +295,14 @@ class TestReceiverAudioCallback
1);
}
- // TODO(miu): This is a "fuzzy" way to check the timestamps. We should be
- // able to compute exact offsets with "omnipotent" knowledge of the system.
- const base::TimeTicks upper_bound =
- expected_audio_frame->record_time +
- base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs +
- kTimerErrorMs);
- EXPECT_GE(upper_bound, playout_time)
- << "playout_time - upper_bound == "
- << (playout_time - upper_bound).InMicroseconds() << " usec";
+ EXPECT_NEAR(
+ (playout_time - expected_audio_frame->playout_time).InMillisecondsF(),
+ 0.0,
+ kMaxAllowedPlayoutErrorMs);
+ VLOG_IF(1, !last_playout_time_.is_null())
+ << "Audio frame playout time delta (compared to last frame) is "
+ << (playout_time - last_playout_time_).InMicroseconds() << " usec.";
+ last_playout_time_ = playout_time;
EXPECT_TRUE(is_continuous);
}
@@ -345,6 +347,7 @@ class TestReceiverAudioCallback
int num_called_;
int expected_sampling_frequency_;
std::list<ExpectedAudioFrame*> expected_frames_;
+ base::TimeTicks last_playout_time_;
};
// Class that verifies the video frames coming out of the receiver.
@@ -355,7 +358,7 @@ class TestReceiverVideoCallback
int start_value;
int width;
int height;
- base::TimeTicks capture_time;
+ base::TimeTicks playout_time;
bool should_be_continuous;
};
@@ -364,19 +367,19 @@ class TestReceiverVideoCallback
void AddExpectedResult(int start_value,
int width,
int height,
- const base::TimeTicks& capture_time,
+ const base::TimeTicks& playout_time,
bool should_be_continuous) {
ExpectedVideoFrame expected_video_frame;
expected_video_frame.start_value = start_value;
expected_video_frame.width = width;
expected_video_frame.height = height;
- expected_video_frame.capture_time = capture_time;
+ expected_video_frame.playout_time = playout_time;
expected_video_frame.should_be_continuous = should_be_continuous;
expected_frame_.push_back(expected_video_frame);
}
void CheckVideoFrame(const scoped_refptr<media::VideoFrame>& video_frame,
- const base::TimeTicks& render_time,
+ const base::TimeTicks& playout_time,
bool is_continuous) {
++num_called_;
@@ -385,21 +388,6 @@ class TestReceiverVideoCallback
ExpectedVideoFrame expected_video_frame = expected_frame_.front();
expected_frame_.pop_front();
- base::TimeDelta time_since_capture =
- render_time - expected_video_frame.capture_time;
- const base::TimeDelta upper_bound = base::TimeDelta::FromMilliseconds(
- kDefaultRtpMaxDelayMs + kTimerErrorMs);
-
- // TODO(miu): This is a "fuzzy" way to check the timestamps. We should be
- // able to compute exact offsets with "omnipotent" knowledge of the system.
- EXPECT_GE(upper_bound, time_since_capture)
- << "time_since_capture - upper_bound == "
- << (time_since_capture - upper_bound).InMicroseconds() << " usec";
- // TODO(miu): I broke the concept of 100 ms target delay timing on the
- // receiver side, but the logic for computing playout time really isn't any
- // more broken than it was. This only affects the receiver, and is to be
- // rectified in an soon-upcoming change. http://crbug.com/356942
- // EXPECT_LE(expected_video_frame.capture_time, render_time);
EXPECT_EQ(expected_video_frame.width, video_frame->visible_rect().width());
EXPECT_EQ(expected_video_frame.height,
video_frame->visible_rect().height());
@@ -412,6 +400,15 @@ class TestReceiverVideoCallback
EXPECT_GE(I420PSNR(expected_I420_frame, video_frame), kVideoAcceptedPSNR);
+ EXPECT_NEAR(
+ (playout_time - expected_video_frame.playout_time).InMillisecondsF(),
+ 0.0,
+ kMaxAllowedPlayoutErrorMs);
+ VLOG_IF(1, !last_playout_time_.is_null())
+ << "Video frame playout time delta (compared to last frame) is "
+ << (playout_time - last_playout_time_).InMicroseconds() << " usec.";
+ last_playout_time_ = playout_time;
+
EXPECT_EQ(expected_video_frame.should_be_continuous, is_continuous);
}
@@ -425,6 +422,7 @@ class TestReceiverVideoCallback
int num_called_;
std::list<ExpectedVideoFrame> expected_frame_;
+ base::TimeTicks last_playout_time_;
};
// The actual test class, generate synthetic data for both audio and video and
@@ -466,6 +464,7 @@ class End2EndTest : public ::testing::Test {
bool external_audio_decoder,
int max_number_of_video_buffers_used) {
audio_sender_config_.rtp_config.ssrc = 1;
+ audio_sender_config_.rtp_config.max_delay_ms = kTargetPlayoutDelayMs;
audio_sender_config_.incoming_feedback_ssrc = 2;
audio_sender_config_.rtp_config.payload_type = 96;
audio_sender_config_.use_external_encoder = false;
@@ -477,6 +476,7 @@ class End2EndTest : public ::testing::Test {
audio_receiver_config_.feedback_ssrc =
audio_sender_config_.incoming_feedback_ssrc;
audio_receiver_config_.incoming_ssrc = audio_sender_config_.rtp_config.ssrc;
+ audio_receiver_config_.rtp_max_delay_ms = kTargetPlayoutDelayMs;
audio_receiver_config_.rtp_payload_type =
audio_sender_config_.rtp_config.payload_type;
audio_receiver_config_.use_external_decoder = external_audio_decoder;
@@ -488,6 +488,7 @@ class End2EndTest : public ::testing::Test {
audio_receiver_config_.frequency);
video_sender_config_.rtp_config.ssrc = 3;
+ video_sender_config_.rtp_config.max_delay_ms = kTargetPlayoutDelayMs;
video_sender_config_.incoming_feedback_ssrc = 4;
video_sender_config_.rtp_config.payload_type = 97;
video_sender_config_.use_external_encoder = false;
@@ -506,32 +507,43 @@ class End2EndTest : public ::testing::Test {
video_receiver_config_.feedback_ssrc =
video_sender_config_.incoming_feedback_ssrc;
video_receiver_config_.incoming_ssrc = video_sender_config_.rtp_config.ssrc;
+ video_receiver_config_.rtp_max_delay_ms = kTargetPlayoutDelayMs;
video_receiver_config_.rtp_payload_type =
video_sender_config_.rtp_config.payload_type;
video_receiver_config_.use_external_decoder = false;
video_receiver_config_.codec = video_sender_config_.codec;
}
- void SetSenderSkew(double skew, base::TimeDelta offset) {
- testing_clock_sender_->SetSkew(skew, offset);
- task_runner_sender_->SetSkew(1.0 / skew);
- }
-
void SetReceiverSkew(double skew, base::TimeDelta offset) {
testing_clock_receiver_->SetSkew(skew, offset);
task_runner_receiver_->SetSkew(1.0 / skew);
}
+ // Specify the minimum/maximum difference in playout times between two
+ // consecutive frames. Also, specify the maximum absolute rate of change over
+ // each three consecutive frames.
+ void SetExpectedVideoPlayoutSmoothness(base::TimeDelta min_delta,
+ base::TimeDelta max_delta,
+ base::TimeDelta max_curvature) {
+ min_video_playout_delta_ = min_delta;
+ max_video_playout_delta_ = max_delta;
+ max_video_playout_curvature_ = max_curvature;
+ }
+
void FeedAudioFrames(int count, bool will_be_checked) {
for (int i = 0; i < count; ++i) {
scoped_ptr<AudioBus> audio_bus(audio_bus_factory_->NextAudioBus(
base::TimeDelta::FromMilliseconds(kAudioFrameDurationMs)));
- const base::TimeTicks send_time =
+ const base::TimeTicks capture_time =
testing_clock_sender_->NowTicks() +
i * base::TimeDelta::FromMilliseconds(kAudioFrameDurationMs);
- if (will_be_checked)
- test_receiver_audio_callback_->AddExpectedResult(*audio_bus, send_time);
- audio_frame_input_->InsertAudio(audio_bus.Pass(), send_time);
+ if (will_be_checked) {
+ test_receiver_audio_callback_->AddExpectedResult(
+ *audio_bus,
+ capture_time +
+ base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs));
+ }
+ audio_frame_input_->InsertAudio(audio_bus.Pass(), capture_time);
}
}
@@ -540,12 +552,14 @@ class End2EndTest : public ::testing::Test {
for (int i = 0; i < count; ++i) {
scoped_ptr<AudioBus> audio_bus(audio_bus_factory_->NextAudioBus(
base::TimeDelta::FromMilliseconds(kAudioFrameDurationMs)));
- const base::TimeTicks send_time =
+ const base::TimeTicks capture_time =
testing_clock_sender_->NowTicks() +
i * base::TimeDelta::FromMilliseconds(kAudioFrameDurationMs);
- test_receiver_audio_callback_->AddExpectedResult(*audio_bus,
- send_time + delay);
- audio_frame_input_->InsertAudio(audio_bus.Pass(), send_time);
+ test_receiver_audio_callback_->AddExpectedResult(
+ *audio_bus,
+ capture_time + delay +
+ base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs));
+ audio_frame_input_->InsertAudio(audio_bus.Pass(), capture_time);
}
}
@@ -563,6 +577,7 @@ class End2EndTest : public ::testing::Test {
audio_receiver_config_,
video_receiver_config_,
&receiver_to_sender_);
+
net::IPEndPoint dummy_endpoint;
transport_sender_.reset(new transport::CastTransportSenderImpl(
NULL,
@@ -640,10 +655,34 @@ class End2EndTest : public ::testing::Test {
void BasicPlayerGotVideoFrame(
const scoped_refptr<media::VideoFrame>& video_frame,
- const base::TimeTicks& render_time, bool continuous) {
+ const base::TimeTicks& playout_time, bool continuous) {
+ // The following tests that the sender and receiver clocks can be
+ // out-of-sync, drift, and jitter with respect to one another; and depsite
+ // this, the receiver will produce smoothly-progressing playout times.
+ // Both first-order and second-order effects are tested.
+ if (!last_video_playout_time_.is_null() &&
+ min_video_playout_delta_ > base::TimeDelta()) {
+ const base::TimeDelta delta = playout_time - last_video_playout_time_;
+ VLOG(1) << "Video frame playout time delta (compared to last frame) is "
+ << delta.InMicroseconds() << " usec.";
+ EXPECT_LE(min_video_playout_delta_.InMicroseconds(),
+ delta.InMicroseconds());
+ EXPECT_GE(max_video_playout_delta_.InMicroseconds(),
+ delta.InMicroseconds());
+ if (last_video_playout_delta_ > base::TimeDelta()) {
+ base::TimeDelta abs_curvature = delta - last_video_playout_delta_;
+ if (abs_curvature < base::TimeDelta())
+ abs_curvature = -abs_curvature;
+ EXPECT_GE(max_video_playout_curvature_.InMicroseconds(),
+ abs_curvature.InMicroseconds());
+ }
+ last_video_playout_delta_ = delta;
+ }
+ last_video_playout_time_ = playout_time;
+
video_ticks_.push_back(std::make_pair(
testing_clock_receiver_->NowTicks(),
- render_time));
+ playout_time));
frame_receiver_->GetRawVideoFrame(
base::Bind(&End2EndTest::BasicPlayerGotVideoFrame,
base::Unretained(this)));
@@ -652,6 +691,12 @@ class End2EndTest : public ::testing::Test {
void BasicPlayerGotAudioFrame(scoped_ptr<AudioBus> audio_bus,
const base::TimeTicks& playout_time,
bool is_continuous) {
+ VLOG_IF(1, !last_audio_playout_time_.is_null())
+ << "Audio frame playout time delta (compared to last frame) is "
+ << (playout_time - last_audio_playout_time_).InMicroseconds()
+ << " usec.";
+ last_audio_playout_time_ = playout_time;
+
audio_ticks_.push_back(std::make_pair(
testing_clock_receiver_->NowTicks(),
playout_time));
@@ -704,6 +749,12 @@ class End2EndTest : public ::testing::Test {
// These run on the receiver timeline.
test::SkewedTickClock* testing_clock_receiver_;
scoped_refptr<test::SkewedSingleThreadTaskRunner> task_runner_receiver_;
+ base::TimeDelta min_video_playout_delta_;
+ base::TimeDelta max_video_playout_delta_;
+ base::TimeDelta max_video_playout_curvature_;
+ base::TimeTicks last_video_playout_time_;
+ base::TimeDelta last_video_playout_delta_;
+ base::TimeTicks last_audio_playout_time_;
scoped_refptr<CastEnvironment> cast_environment_sender_;
scoped_refptr<CastEnvironment> cast_environment_receiver_;
@@ -755,7 +806,8 @@ TEST_F(End2EndTest, LoopNoLossPcm16) {
video_start,
video_sender_config_.width,
video_sender_config_.height,
- testing_clock_sender_->NowTicks(),
+ testing_clock_sender_->NowTicks() +
+ base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs),
true);
SendVideoFrame(video_start, testing_clock_sender_->NowTicks());
@@ -860,7 +912,8 @@ TEST_F(End2EndTest, DISABLED_StartSenderBeforeReceiver) {
video_start,
video_sender_config_.width,
video_sender_config_.height,
- initial_send_time + expected_delay,
+ initial_send_time + expected_delay +
+ base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs),
true);
SendVideoFrame(video_start, testing_clock_sender_->NowTicks());
@@ -889,7 +942,8 @@ TEST_F(End2EndTest, DISABLED_StartSenderBeforeReceiver) {
video_start,
video_sender_config_.width,
video_sender_config_.height,
- testing_clock_sender_->NowTicks(),
+ testing_clock_sender_->NowTicks() +
+ base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs),
true);
SendVideoFrame(video_start, testing_clock_sender_->NowTicks());
@@ -925,18 +979,18 @@ TEST_F(End2EndTest, DISABLED_GlitchWith3Buffers) {
Create();
int video_start = kVideoStart;
- base::TimeTicks send_time;
+ base::TimeTicks capture_time;
// Frames will rendered on completion until the render time stabilizes, i.e.
// we got enough data.
const int frames_before_glitch = 20;
for (int i = 0; i < frames_before_glitch; ++i) {
- send_time = testing_clock_sender_->NowTicks();
- SendVideoFrame(video_start, send_time);
+ capture_time = testing_clock_sender_->NowTicks();
+ SendVideoFrame(video_start, capture_time);
test_receiver_video_callback_->AddExpectedResult(
video_start,
video_sender_config_.width,
video_sender_config_.height,
- send_time,
+ capture_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs),
true);
frame_receiver_->GetRawVideoFrame(
base::Bind(&TestReceiverVideoCallback::CheckVideoFrame,
@@ -948,27 +1002,28 @@ TEST_F(End2EndTest, DISABLED_GlitchWith3Buffers) {
// Introduce a glitch lasting for 10 frames.
sender_to_receiver_.SetSendPackets(false);
for (int i = 0; i < 10; ++i) {
- send_time = testing_clock_sender_->NowTicks();
+ capture_time = testing_clock_sender_->NowTicks();
// First 3 will be sent and lost.
- SendVideoFrame(video_start, send_time);
+ SendVideoFrame(video_start, capture_time);
RunTasks(kFrameTimerMs);
video_start++;
}
sender_to_receiver_.SetSendPackets(true);
RunTasks(100);
- send_time = testing_clock_sender_->NowTicks();
+ capture_time = testing_clock_sender_->NowTicks();
// Frame 1 should be acked by now and we should have an opening to send 4.
- SendVideoFrame(video_start, send_time);
+ SendVideoFrame(video_start, capture_time);
RunTasks(kFrameTimerMs);
// Frames 1-3 are old frames by now, and therefore should be decoded, but
// not rendered. The next frame we expect to render is frame #4.
- test_receiver_video_callback_->AddExpectedResult(video_start,
- video_sender_config_.width,
- video_sender_config_.height,
- send_time,
- true);
+ test_receiver_video_callback_->AddExpectedResult(
+ video_start,
+ video_sender_config_.width,
+ video_sender_config_.height,
+ capture_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs),
+ true);
frame_receiver_->GetRawVideoFrame(
base::Bind(&TestReceiverVideoCallback::CheckVideoFrame,
@@ -989,19 +1044,20 @@ TEST_F(End2EndTest, DISABLED_DropEveryOtherFrame3Buffers) {
sender_to_receiver_.DropAllPacketsBelongingToOddFrames();
int video_start = kVideoStart;
- base::TimeTicks send_time;
+ base::TimeTicks capture_time;
int i = 0;
for (; i < 20; ++i) {
- send_time = testing_clock_sender_->NowTicks();
- SendVideoFrame(video_start, send_time);
+ capture_time = testing_clock_sender_->NowTicks();
+ SendVideoFrame(video_start, capture_time);
if (i % 2 == 0) {
test_receiver_video_callback_->AddExpectedResult(
video_start,
video_sender_config_.width,
video_sender_config_.height,
- send_time,
+ capture_time +
+ base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs),
i == 0);
// GetRawVideoFrame will not return the frame until we are close in
@@ -1035,14 +1091,14 @@ TEST_F(End2EndTest, CryptoVideo) {
int frames_counter = 0;
for (; frames_counter < 3; ++frames_counter) {
- const base::TimeTicks send_time = testing_clock_sender_->NowTicks();
- SendVideoFrame(frames_counter, send_time);
+ const base::TimeTicks capture_time = testing_clock_sender_->NowTicks();
+ SendVideoFrame(frames_counter, capture_time);
test_receiver_video_callback_->AddExpectedResult(
frames_counter,
video_sender_config_.width,
video_sender_config_.height,
- send_time,
+ capture_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs),
true);
RunTasks(kFrameTimerMs);
@@ -1092,15 +1148,15 @@ TEST_F(End2EndTest, VideoLogging) {
int video_start = kVideoStart;
const int num_frames = 5;
for (int i = 0; i < num_frames; ++i) {
- base::TimeTicks send_time = testing_clock_sender_->NowTicks();
+ base::TimeTicks capture_time = testing_clock_sender_->NowTicks();
test_receiver_video_callback_->AddExpectedResult(
video_start,
video_sender_config_.width,
video_sender_config_.height,
- send_time,
+ capture_time + base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs),
true);
- SendVideoFrame(video_start, send_time);
+ SendVideoFrame(video_start, capture_time);
RunTasks(kFrameTimerMs);
frame_receiver_->GetRawVideoFrame(
@@ -1291,6 +1347,13 @@ TEST_F(End2EndTest, BasicFakeSoftwareVideo) {
Configure(transport::kFakeSoftwareVideo, transport::kPcm16, 32000, false, 1);
Create();
StartBasicPlayer();
+ SetReceiverSkew(1.0, base::TimeDelta::FromMilliseconds(1));
+
+ // Expect very smooth playout when there is no clock skew.
+ SetExpectedVideoPlayoutSmoothness(
+ base::TimeDelta::FromMilliseconds(kFrameTimerMs) * 99 / 100,
+ base::TimeDelta::FromMilliseconds(kFrameTimerMs) * 101 / 100,
+ base::TimeDelta::FromMilliseconds(kFrameTimerMs) / 100);
int frames_counter = 0;
for (; frames_counter < 1000; ++frames_counter) {
@@ -1305,7 +1368,7 @@ TEST_F(End2EndTest, ReceiverClockFast) {
Configure(transport::kFakeSoftwareVideo, transport::kPcm16, 32000, false, 1);
Create();
StartBasicPlayer();
- SetReceiverSkew(2.0, base::TimeDelta());
+ SetReceiverSkew(2.0, base::TimeDelta::FromMicroseconds(1234567));
int frames_counter = 0;
for (; frames_counter < 10000; ++frames_counter) {
@@ -1320,7 +1383,28 @@ TEST_F(End2EndTest, ReceiverClockSlow) {
Configure(transport::kFakeSoftwareVideo, transport::kPcm16, 32000, false, 1);
Create();
StartBasicPlayer();
- SetReceiverSkew(0.5, base::TimeDelta());
+ SetReceiverSkew(0.5, base::TimeDelta::FromMicroseconds(-765432));
+
+ int frames_counter = 0;
+ for (; frames_counter < 10000; ++frames_counter) {
+ SendFakeVideoFrame(testing_clock_sender_->NowTicks());
+ RunTasks(kFrameTimerMs);
+ }
+ RunTasks(2 * kFrameTimerMs + 1); // Empty the pipeline.
+ EXPECT_EQ(10000ul, video_ticks_.size());
+}
+
+TEST_F(End2EndTest, SmoothPlayoutWithFivePercentClockRateSkew) {
+ Configure(transport::kFakeSoftwareVideo, transport::kPcm16, 32000, false, 1);
+ Create();
+ StartBasicPlayer();
+ SetReceiverSkew(1.05, base::TimeDelta::FromMilliseconds(-42));
+
+ // Expect smooth playout when there is 5% skew.
+ SetExpectedVideoPlayoutSmoothness(
+ base::TimeDelta::FromMilliseconds(kFrameTimerMs) * 90 / 100,
+ base::TimeDelta::FromMilliseconds(kFrameTimerMs) * 110 / 100,
+ base::TimeDelta::FromMilliseconds(kFrameTimerMs) / 10);
int frames_counter = 0;
for (; frames_counter < 10000; ++frames_counter) {
diff --git a/media/cast/test/receiver.cc b/media/cast/test/receiver.cc
index f15d17d..c32ad8e 100644
--- a/media/cast/test/receiver.cc
+++ b/media/cast/test/receiver.cc
@@ -424,14 +424,21 @@ class NaivePlayer : public InProcessReceiver,
DCHECK(cast_env()->CurrentlyOn(CastEnvironment::MAIN));
if (is_being_skipped) {
- VLOG(1) << "VideoFrame[" << num_video_frames_processed_ << "]: Skipped.";
+ VLOG(1) << "VideoFrame[" << num_video_frames_processed_
+ << " (dt=" << (video_playout_queue_.front().first -
+ last_popped_video_playout_time_).InMicroseconds()
+ << " usec)]: Skipped.";
} else {
- VLOG(1) << "VideoFrame[" << num_video_frames_processed_ << "]: Playing "
+ VLOG(1) << "VideoFrame[" << num_video_frames_processed_
+ << " (dt=" << (video_playout_queue_.front().first -
+ last_popped_video_playout_time_).InMicroseconds()
+ << " usec)]: Playing "
<< (cast_env()->Clock()->NowTicks() -
video_playout_queue_.front().first).InMicroseconds()
<< " usec later than intended.";
}
+ last_popped_video_playout_time_ = video_playout_queue_.front().first;
const scoped_refptr<VideoFrame> ret = video_playout_queue_.front().second;
video_playout_queue_.pop_front();
++num_video_frames_processed_;
@@ -442,14 +449,21 @@ class NaivePlayer : public InProcessReceiver,
audio_lock_.AssertAcquired();
if (was_skipped) {
- VLOG(1) << "AudioFrame[" << num_audio_frames_processed_ << "]: Skipped";
+ VLOG(1) << "AudioFrame[" << num_audio_frames_processed_
+ << " (dt=" << (audio_playout_queue_.front().first -
+ last_popped_audio_playout_time_).InMicroseconds()
+ << " usec)]: Skipped.";
} else {
- VLOG(1) << "AudioFrame[" << num_audio_frames_processed_ << "]: Playing "
+ VLOG(1) << "AudioFrame[" << num_audio_frames_processed_
+ << " (dt=" << (audio_playout_queue_.front().first -
+ last_popped_audio_playout_time_).InMicroseconds()
+ << " usec)]: Playing "
<< (cast_env()->Clock()->NowTicks() -
audio_playout_queue_.front().first).InMicroseconds()
<< " usec later than intended.";
}
+ last_popped_audio_playout_time_ = audio_playout_queue_.front().first;
scoped_ptr<AudioBus> ret(audio_playout_queue_.front().second);
audio_playout_queue_.pop_front();
++num_audio_frames_processed_;
@@ -501,6 +515,7 @@ class NaivePlayer : public InProcessReceiver,
typedef std::pair<base::TimeTicks, scoped_refptr<VideoFrame> >
VideoQueueEntry;
std::deque<VideoQueueEntry> video_playout_queue_;
+ base::TimeTicks last_popped_video_playout_time_;
int64 num_video_frames_processed_;
base::OneShotTimer<NaivePlayer> video_playout_timer_;
@@ -509,6 +524,7 @@ class NaivePlayer : public InProcessReceiver,
base::Lock audio_lock_;
typedef std::pair<base::TimeTicks, AudioBus*> AudioQueueEntry;
std::deque<AudioQueueEntry> audio_playout_queue_;
+ base::TimeTicks last_popped_audio_playout_time_;
int64 num_audio_frames_processed_;
// These must only be used on the audio thread calling OnMoreData().
diff --git a/media/cast/test/skewed_tick_clock.cc b/media/cast/test/skewed_tick_clock.cc
index fd3fb3c..272c61e 100644
--- a/media/cast/test/skewed_tick_clock.cc
+++ b/media/cast/test/skewed_tick_clock.cc
@@ -25,8 +25,9 @@ base::TimeTicks SkewedTickClock::SkewTicks(base::TimeTicks now) {
void SkewedTickClock::SetSkew(double skew, base::TimeDelta offset) {
base::TimeTicks now = clock_->NowTicks();
- last_skew_set_time_ = now;
skew_clock_at_last_set_ = SkewTicks(now) + offset;
+ skew_ = skew;
+ last_skew_set_time_ = now;
}
base::TimeTicks SkewedTickClock::NowTicks() {
diff --git a/media/cast/transport/cast_transport_config.h b/media/cast/transport/cast_transport_config.h
index b25ce4e..d3136e0 100644
--- a/media/cast/transport/cast_transport_config.h
+++ b/media/cast/transport/cast_transport_config.h
@@ -31,8 +31,7 @@ enum AudioCodec {
kFakeSoftwareAudio,
kOpus,
kPcm16,
- kExternalAudio,
- kAudioCodecLast = kExternalAudio
+ kAudioCodecLast = kPcm16
};
struct RtpConfig {
diff --git a/media/cast/video_receiver/video_receiver.cc b/media/cast/video_receiver/video_receiver.cc
index cbd62ae..9988eac 100644
--- a/media/cast/video_receiver/video_receiver.cc
+++ b/media/cast/video_receiver/video_receiver.cc
@@ -17,8 +17,6 @@
namespace {
const int kMinSchedulingDelayMs = 1;
-const int kMinTimeBetweenOffsetUpdatesMs = 1000;
-const int kTimeOffsetMaxCounter = 10;
} // namespace
namespace media {
@@ -31,10 +29,11 @@ VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment,
cast_environment_(cast_environment),
event_subscriber_(kReceiverRtcpEventHistorySize, VIDEO_EVENT),
codec_(video_config.codec),
- target_delay_delta_(
+ target_playout_delay_(
base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
expected_frame_duration_(
base::TimeDelta::FromSeconds(1) / video_config.max_frame_rate),
+ reports_are_scheduled_(false),
framer_(cast_environment->Clock(),
this,
video_config.incoming_ssrc,
@@ -52,10 +51,8 @@ VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment,
video_config.incoming_ssrc,
video_config.rtcp_c_name,
false),
- time_offset_counter_(0),
- time_incoming_packet_updated_(false),
- incoming_rtp_timestamp_(0),
is_waiting_for_consecutive_frame_(false),
+ lip_sync_drift_(ClockDriftSmoother::GetDefaultTimeConstant()),
weak_factory_(this) {
DCHECK_GT(video_config.rtp_max_delay_ms, 0);
DCHECK_GT(video_config.max_frame_rate, 0);
@@ -63,7 +60,7 @@ VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment,
video_decoder_.reset(new VideoDecoder(cast_environment, video_config));
}
decryptor_.Initialize(video_config.aes_key, video_config.aes_iv_mask);
- rtcp_.SetTargetDelay(target_delay_delta_);
+ rtcp_.SetTargetDelay(target_playout_delay_);
cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_);
memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_));
}
@@ -73,12 +70,6 @@ VideoReceiver::~VideoReceiver() {
cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_);
}
-void VideoReceiver::InitializeTimers() {
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- ScheduleNextRtcpReport();
- ScheduleNextCastMessage();
-}
-
void VideoReceiver::GetRawVideoFrame(
const VideoFrameDecodedCallback& callback) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
@@ -166,7 +157,7 @@ void VideoReceiver::EmitAvailableEncodedFrames() {
const base::TimeTicks now = cast_environment_->Clock()->NowTicks();
const base::TimeTicks playout_time =
- GetPlayoutTime(now, encoded_frame->rtp_timestamp);
+ GetPlayoutTime(encoded_frame->rtp_timestamp);
// If we have multiple decodable frames, and the current frame is
// too old, then skip it and decode the next frame instead.
@@ -237,72 +228,13 @@ void VideoReceiver::EmitAvailableEncodedFramesAfterWaiting() {
EmitAvailableEncodedFrames();
}
-base::TimeTicks VideoReceiver::GetPlayoutTime(base::TimeTicks now,
- uint32 rtp_timestamp) {
- // TODO(miu): This and AudioReceiver::GetPlayoutTime() need to be reconciled!
-
- DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- // Senders time in ms when this frame was captured.
- // Note: the senders clock and our local clock might not be synced.
- base::TimeTicks rtp_timestamp_in_ticks;
-
- // Compute the time offset_in_ticks based on the incoming_rtp_timestamp_.
- if (time_offset_counter_ == 0) {
- // Check for received RTCP to sync the stream play it out asap.
- if (rtcp_.RtpTimestampInSenderTime(kVideoFrequency,
- incoming_rtp_timestamp_,
- &rtp_timestamp_in_ticks)) {
- ++time_offset_counter_;
- }
- } else if (time_incoming_packet_updated_) {
- if (rtcp_.RtpTimestampInSenderTime(kVideoFrequency,
- incoming_rtp_timestamp_,
- &rtp_timestamp_in_ticks)) {
- // Time to update the time_offset.
- base::TimeDelta time_offset =
- time_incoming_packet_ - rtp_timestamp_in_ticks;
- // Taking the minimum of the first kTimeOffsetMaxCounter values. We are
- // assuming that we are looking for the minimum offset, which will occur
- // when network conditions are the best. This should occur at least once
- // within the first kTimeOffsetMaxCounter samples. Any drift should be
- // very slow, and negligible for this use case.
- if (time_offset_counter_ == 1)
- time_offset_ = time_offset;
- else if (time_offset_counter_ < kTimeOffsetMaxCounter) {
- time_offset_ = std::min(time_offset_, time_offset);
- }
- if (time_offset_counter_ < kTimeOffsetMaxCounter)
- ++time_offset_counter_;
- }
- }
- // Reset |time_incoming_packet_updated_| to enable a future measurement.
- time_incoming_packet_updated_ = false;
- // Compute the actual rtp_timestamp_in_ticks based on the current timestamp.
- if (!rtcp_.RtpTimestampInSenderTime(
- kVideoFrequency, rtp_timestamp, &rtp_timestamp_in_ticks)) {
- // This can fail if we have not received any RTCP packets in a long time.
- // BUG: These calculations are a placeholder, and to be revisited in a
- // soon-upcoming change. http://crbug.com/356942
- const int frequency_khz = kVideoFrequency / 1000;
- const base::TimeDelta delta_based_on_rtp_timestamps =
- base::TimeDelta::FromMilliseconds(
- static_cast<int32>(rtp_timestamp - incoming_rtp_timestamp_) /
- frequency_khz);
- return time_incoming_packet_ + delta_based_on_rtp_timestamps;
- }
-
- base::TimeTicks render_time =
- rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_;
- // TODO(miu): This is broken since this "getter" method may be called on
- // frames received out-of-order, which means the playout times for earlier
- // frames will be computed incorrectly.
-#if 0
- if (last_render_time_ > render_time)
- render_time = last_render_time_;
- last_render_time_ = render_time;
-#endif
-
- return render_time;
+base::TimeTicks VideoReceiver::GetPlayoutTime(uint32 rtp_timestamp) const {
+ return lip_sync_reference_time_ +
+ lip_sync_drift_.Current() +
+ RtpDeltaToTimeDelta(
+ static_cast<int32>(rtp_timestamp - lip_sync_rtp_timestamp_),
+ kVideoFrequency) +
+ target_playout_delay_;
}
void VideoReceiver::IncomingPacket(scoped_ptr<Packet> packet) {
@@ -312,6 +244,11 @@ void VideoReceiver::IncomingPacket(scoped_ptr<Packet> packet) {
} else {
ReceivedPacket(&packet->front(), packet->size());
}
+ if (!reports_are_scheduled_) {
+ ScheduleNextRtcpReport();
+ ScheduleNextCastMessage();
+ reports_are_scheduled_ = true;
+ }
}
void VideoReceiver::OnReceivedPayloadData(const uint8* payload_data,
@@ -319,21 +256,7 @@ void VideoReceiver::OnReceivedPayloadData(const uint8* payload_data,
const RtpCastHeader& rtp_header) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
- base::TimeTicks now = cast_environment_->Clock()->NowTicks();
- if (time_incoming_packet_.is_null() ||
- now - time_incoming_packet_ >
- base::TimeDelta::FromMilliseconds(kMinTimeBetweenOffsetUpdatesMs)) {
- if (time_incoming_packet_.is_null())
- InitializeTimers();
- incoming_rtp_timestamp_ = rtp_header.rtp_timestamp;
- // The following incoming packet info is used for syncing sender and
- // receiver clock. Use only the first packet of every frame to obtain a
- // minimal value.
- if (rtp_header.packet_id == 0) {
- time_incoming_packet_ = now;
- time_incoming_packet_updated_ = true;
- }
- }
+ const base::TimeTicks now = cast_environment_->Clock()->NowTicks();
frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] =
rtp_header.rtp_timestamp;
@@ -355,6 +278,36 @@ void VideoReceiver::OnReceivedPayloadData(const uint8* payload_data,
if (duplicate)
return;
+ // Update lip-sync values upon receiving the first packet of each frame, or if
+ // they have never been set yet.
+ if (rtp_header.packet_id == 0 || lip_sync_reference_time_.is_null()) {
+ RtpTimestamp fresh_sync_rtp;
+ base::TimeTicks fresh_sync_reference;
+ if (!rtcp_.GetLatestLipSyncTimes(&fresh_sync_rtp, &fresh_sync_reference)) {
+ // HACK: The sender should have provided Sender Reports before the first
+ // frame was sent. However, the spec does not currently require this.
+ // Therefore, when the data is missing, the local clock is used to
+ // generate reference timestamps.
+ VLOG(2) << "Lip sync info missing. Falling-back to local clock.";
+ fresh_sync_rtp = rtp_header.rtp_timestamp;
+ fresh_sync_reference = now;
+ }
+ // |lip_sync_reference_time_| is always incremented according to the time
+ // delta computed from the difference in RTP timestamps. Then,
+ // |lip_sync_drift_| accounts for clock drift and also smoothes-out any
+ // sudden/discontinuous shifts in the series of reference time values.
+ if (lip_sync_reference_time_.is_null()) {
+ lip_sync_reference_time_ = fresh_sync_reference;
+ } else {
+ lip_sync_reference_time_ += RtpDeltaToTimeDelta(
+ static_cast<int32>(fresh_sync_rtp - lip_sync_rtp_timestamp_),
+ kVideoFrequency);
+ }
+ lip_sync_rtp_timestamp_ = fresh_sync_rtp;
+ lip_sync_drift_.Update(
+ now, fresh_sync_reference - lip_sync_reference_time_);
+ }
+
// Video frame not complete; wait for more packets.
if (!complete)
return;
diff --git a/media/cast/video_receiver/video_receiver.h b/media/cast/video_receiver/video_receiver.h
index ff7ccda..360e746 100644
--- a/media/cast/video_receiver/video_receiver.h
+++ b/media/cast/video_receiver/video_receiver.h
@@ -13,6 +13,7 @@
#include "base/threading/non_thread_safe.h"
#include "base/time/tick_clock.h"
#include "base/time/time.h"
+#include "media/cast/base/clock_drift_smoother.h"
#include "media/cast/cast_config.h"
#include "media/cast/cast_environment.h"
#include "media/cast/cast_receiver.h"
@@ -41,10 +42,6 @@ class VideoDecoder;
// each step of the pipeline (i.e., encode frame, then transmit/retransmit from
// the sender, then receive and re-order packets on the receiver, then decode
// frame) can vary in duration and is typically very hard to predict.
-// Heuristics will determine when the targeted playout delay is insufficient in
-// the current environment; and the receiver can then increase the playout
-// delay, notifying the sender, to account for the extra variance.
-// TODO(miu): Make the last sentence true. http://crbug.com/360111
//
// Two types of frames can be requested: 1) A frame of decoded video data; or 2)
// a frame of still-encoded video data, to be passed into an external video
@@ -82,7 +79,7 @@ class VideoReceiver : public RtpReceiver,
void IncomingPacket(scoped_ptr<Packet> packet);
protected:
- friend class VideoReceiverTest; // Invoked OnReceivedPayloadData().
+ friend class VideoReceiverTest; // Invokes OnReceivedPayloadData().
virtual void OnReceivedPayloadData(const uint8* payload_data,
size_t payload_size,
@@ -108,10 +105,10 @@ class VideoReceiver : public RtpReceiver,
const VideoFrameDecodedCallback& callback,
scoped_ptr<transport::EncodedFrame> encoded_frame);
- // Return the playout time based on the current time and rtp timestamp.
- base::TimeTicks GetPlayoutTime(base::TimeTicks now, uint32 rtp_timestamp);
-
- void InitializeTimers();
+ // Computes the playout time for a frame with the given |rtp_timestamp|.
+ // Because lip-sync info is refreshed regularly, calling this method with the
+ // same argument may return different results.
+ base::TimeTicks GetPlayoutTime(uint32 rtp_timestamp) const;
// Schedule timing for the next cast message.
void ScheduleNextCastMessage();
@@ -145,17 +142,38 @@ class VideoReceiver : public RtpReceiver,
// Processes raw audio events to be sent over to the cast sender via RTCP.
ReceiverRtcpEventSubscriber event_subscriber_;
+ // Configured audio codec.
const transport::VideoCodec codec_;
- const base::TimeDelta target_delay_delta_;
+
+ // The total amount of time between a frame's capture/recording on the sender
+ // and its playback on the receiver (i.e., shown to a user). This is fixed as
+ // a value large enough to give the system sufficient time to encode,
+ // transmit/retransmit, receive, decode, and render; given its run-time
+ // environment (sender/receiver hardware performance, network conditions,
+ // etc.).
+ const base::TimeDelta target_playout_delay_;
+
+ // Hack: This is used in logic that determines whether to skip frames.
const base::TimeDelta expected_frame_duration_;
+
+ // Set to false initially, then set to true after scheduling the periodic
+ // sending of reports back to the sender. Reports are first scheduled just
+ // after receiving a first packet (since the first packet identifies the
+ // sender for the remainder of the session).
+ bool reports_are_scheduled_;
+
+ // Assembles packets into frames, providing this receiver with complete,
+ // decodable EncodedFrames.
Framer framer_;
+
+ // Decodes frames into media::VideoFrame images for playback.
scoped_ptr<VideoDecoder> video_decoder_;
+
+ // Manages sending/receiving of RTCP packets, including sender/receiver
+ // reports.
Rtcp rtcp_;
- base::TimeDelta time_offset_; // Sender-receiver offset estimation.
- int time_offset_counter_;
- bool time_incoming_packet_updated_;
- base::TimeTicks time_incoming_packet_;
- uint32 incoming_rtp_timestamp_;
+
+ // Decrypts encrypted frames.
transport::TransportEncryptionHandler decryptor_;
// Outstanding callbacks to run to deliver on client requests for frames.
@@ -169,6 +187,13 @@ class VideoReceiver : public RtpReceiver,
// it allows the event to be transmitted via RTCP.
RtpTimestamp frame_id_to_rtp_timestamp_[256];
+ // Lip-sync values used to compute the playout time of each frame from its RTP
+ // timestamp. These are updated each time the first packet of a frame is
+ // received.
+ RtpTimestamp lip_sync_rtp_timestamp_;
+ base::TimeTicks lip_sync_reference_time_;
+ ClockDriftSmoother lip_sync_drift_;
+
// NOTE: Weak pointers must be invalidated before all other member variables.
base::WeakPtrFactory<VideoReceiver> weak_factory_;
diff --git a/media/cast/video_receiver/video_receiver_unittest.cc b/media/cast/video_receiver/video_receiver_unittest.cc
index 4cc7dd2..2f65cde 100644
--- a/media/cast/video_receiver/video_receiver_unittest.cc
+++ b/media/cast/video_receiver/video_receiver_unittest.cc
@@ -2,6 +2,9 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+#include <deque>
+#include <utility>
+
#include "base/bind.h"
#include "base/memory/ref_counted.h"
#include "base/memory/scoped_ptr.h"
@@ -9,48 +12,51 @@
#include "media/cast/cast_defines.h"
#include "media/cast/cast_environment.h"
#include "media/cast/logging/simple_event_subscriber.h"
+#include "media/cast/rtcp/test_rtcp_packet_builder.h"
#include "media/cast/test/fake_single_thread_task_runner.h"
#include "media/cast/transport/pacing/mock_paced_packet_sender.h"
#include "media/cast/video_receiver/video_receiver.h"
#include "testing/gmock/include/gmock/gmock.h"
+using ::testing::_;
+
namespace media {
namespace cast {
-using ::testing::_;
-
namespace {
const int kPacketSize = 1500;
-const int64 kStartMillisecond = INT64_C(12345678900000);
const uint32 kFirstFrameId = 1234;
+const int kPlayoutDelayMillis = 100;
class FakeVideoClient {
public:
FakeVideoClient() : num_called_(0) {}
virtual ~FakeVideoClient() {}
- void SetNextExpectedResult(uint32 expected_frame_id,
- const base::TimeTicks& expected_playout_time) {
- expected_frame_id_ = expected_frame_id;
- expected_playout_time_ = expected_playout_time;
+ void AddExpectedResult(uint32 expected_frame_id,
+ const base::TimeTicks& expected_playout_time) {
+ expected_results_.push_back(
+ std::make_pair(expected_frame_id, expected_playout_time));
}
void DeliverEncodedVideoFrame(
scoped_ptr<transport::EncodedFrame> video_frame) {
+ SCOPED_TRACE(::testing::Message() << "num_called_ is " << num_called_);
ASSERT_FALSE(!video_frame)
<< "If at shutdown: There were unsatisfied requests enqueued.";
- EXPECT_EQ(expected_frame_id_, video_frame->frame_id);
- EXPECT_EQ(expected_playout_time_, video_frame->reference_time);
+ ASSERT_FALSE(expected_results_.empty());
+ EXPECT_EQ(expected_results_.front().first, video_frame->frame_id);
+ EXPECT_EQ(expected_results_.front().second, video_frame->reference_time);
+ expected_results_.pop_front();
++num_called_;
}
int number_times_called() const { return num_called_; }
private:
+ std::deque<std::pair<uint32, base::TimeTicks> > expected_results_;
int num_called_;
- uint32 expected_frame_id_;
- base::TimeTicks expected_playout_time_;
DISALLOW_COPY_AND_ASSIGN(FakeVideoClient);
};
@@ -59,16 +65,17 @@ class FakeVideoClient {
class VideoReceiverTest : public ::testing::Test {
protected:
VideoReceiverTest() {
- // Configure to use vp8 software implementation.
- config_.rtp_max_delay_ms = 100;
+ config_.rtp_max_delay_ms = kPlayoutDelayMillis;
config_.use_external_decoder = false;
// Note: Frame rate must divide 1000 without remainder so the test code
// doesn't have to account for rounding errors.
config_.max_frame_rate = 25;
- config_.codec = transport::kVp8;
+ config_.codec = transport::kVp8; // Frame skipping not allowed.
+ config_.feedback_ssrc = 1234;
+ config_.incoming_ssrc = 5678;
testing_clock_ = new base::SimpleTestTickClock();
- testing_clock_->Advance(
- base::TimeDelta::FromMilliseconds(kStartMillisecond));
+ testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks());
+ start_time_ = testing_clock_->NowTicks();
task_runner_ = new test::FakeSingleThreadTaskRunner(testing_clock_);
cast_environment_ =
@@ -99,10 +106,26 @@ class VideoReceiverTest : public ::testing::Test {
payload_.data(), payload_.size(), rtp_header_);
}
+ void FeedLipSyncInfoIntoReceiver() {
+ const base::TimeTicks now = testing_clock_->NowTicks();
+ const int64 rtp_timestamp = (now - start_time_) *
+ kVideoFrequency / base::TimeDelta::FromSeconds(1);
+ CHECK_LE(0, rtp_timestamp);
+ uint32 ntp_seconds;
+ uint32 ntp_fraction;
+ ConvertTimeTicksToNtp(now, &ntp_seconds, &ntp_fraction);
+ TestRtcpPacketBuilder rtcp_packet;
+ rtcp_packet.AddSrWithNtp(config_.incoming_ssrc,
+ ntp_seconds, ntp_fraction,
+ static_cast<uint32>(rtp_timestamp));
+ receiver_->IncomingPacket(rtcp_packet.GetPacket().Pass());
+ }
+
VideoReceiverConfig config_;
std::vector<uint8> payload_;
RtpCastHeader rtp_header_;
base::SimpleTestTickClock* testing_clock_; // Owned by CastEnvironment.
+ base::TimeTicks start_time_;
transport::MockPacedPacketSender mock_transport_;
scoped_refptr<test::FakeSingleThreadTaskRunner> task_runner_;
scoped_refptr<CastEnvironment> cast_environment_;
@@ -115,13 +138,16 @@ class VideoReceiverTest : public ::testing::Test {
DISALLOW_COPY_AND_ASSIGN(VideoReceiverTest);
};
-TEST_F(VideoReceiverTest, GetOnePacketEncodedFrame) {
+TEST_F(VideoReceiverTest, ReceivesOneFrame) {
SimpleEventSubscriber event_subscriber;
cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber);
EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _))
.WillRepeatedly(testing::Return(true));
+ FeedLipSyncInfoIntoReceiver();
+ task_runner_->RunTasks();
+
// Enqueue a request for a video frame.
receiver_->GetEncodedVideoFrame(
base::Bind(&FakeVideoClient::DeliverEncodedVideoFrame,
@@ -132,8 +158,10 @@ TEST_F(VideoReceiverTest, GetOnePacketEncodedFrame) {
EXPECT_EQ(0, fake_video_client_.number_times_called());
// Deliver one video frame to the receiver and expect to get one frame back.
- fake_video_client_.SetNextExpectedResult(kFirstFrameId,
- testing_clock_->NowTicks());
+ const base::TimeDelta target_playout_delay =
+ base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis);
+ fake_video_client_.AddExpectedResult(
+ kFirstFrameId, testing_clock_->NowTicks() + target_playout_delay);
FeedOneFrameIntoReceiver();
task_runner_->RunTasks();
EXPECT_EQ(1, fake_video_client_.number_times_called());
@@ -149,11 +177,20 @@ TEST_F(VideoReceiverTest, GetOnePacketEncodedFrame) {
cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber);
}
-TEST_F(VideoReceiverTest, MultiplePendingGetCalls) {
+TEST_F(VideoReceiverTest, ReceivesFramesRefusingToSkipAny) {
EXPECT_CALL(mock_transport_, SendRtcpPacket(_, _))
.WillRepeatedly(testing::Return(true));
- // Enqueue a request for an video frame.
+ const uint32 rtp_advance_per_frame = kVideoFrequency / config_.max_frame_rate;
+ const base::TimeDelta time_advance_per_frame =
+ base::TimeDelta::FromSeconds(1) / config_.max_frame_rate;
+
+ // Feed and process lip sync in receiver.
+ FeedLipSyncInfoIntoReceiver();
+ task_runner_->RunTasks();
+ const base::TimeTicks first_frame_capture_time = testing_clock_->NowTicks();
+
+ // Enqueue a request for a video frame.
const FrameEncodedCallback frame_encoded_callback =
base::Bind(&FakeVideoClient::DeliverEncodedVideoFrame,
base::Unretained(&fake_video_client_));
@@ -162,17 +199,16 @@ TEST_F(VideoReceiverTest, MultiplePendingGetCalls) {
EXPECT_EQ(0, fake_video_client_.number_times_called());
// Receive one video frame and expect to see the first request satisfied.
- fake_video_client_.SetNextExpectedResult(kFirstFrameId,
- testing_clock_->NowTicks());
- const base::TimeTicks time_at_first_frame_feed = testing_clock_->NowTicks();
+ const base::TimeDelta target_playout_delay =
+ base::TimeDelta::FromMilliseconds(kPlayoutDelayMillis);
+ fake_video_client_.AddExpectedResult(
+ kFirstFrameId, first_frame_capture_time + target_playout_delay);
+ rtp_header_.rtp_timestamp = 0;
FeedOneFrameIntoReceiver();
task_runner_->RunTasks();
EXPECT_EQ(1, fake_video_client_.number_times_called());
- testing_clock_->Advance(
- base::TimeDelta::FromSeconds(1) / config_.max_frame_rate);
-
- // Enqueue a second request for an video frame, but it should not be
+ // Enqueue a second request for a video frame, but it should not be
// fulfilled yet.
receiver_->GetEncodedVideoFrame(frame_encoded_callback);
task_runner_->RunTasks();
@@ -181,47 +217,47 @@ TEST_F(VideoReceiverTest, MultiplePendingGetCalls) {
// Receive one video frame out-of-order: Make sure that we are not continuous
// and that the RTP timestamp represents a time in the future.
rtp_header_.is_key_frame = false;
- rtp_header_.frame_id = kFirstFrameId + 2;
- rtp_header_.reference_frame_id = 0;
- rtp_header_.rtp_timestamp +=
- config_.rtp_max_delay_ms * kVideoFrequency / 1000;
- fake_video_client_.SetNextExpectedResult(
- kFirstFrameId + 2,
- time_at_first_frame_feed +
- base::TimeDelta::FromMilliseconds(config_.rtp_max_delay_ms));
+ rtp_header_.frame_id = kFirstFrameId + 2; // "Frame 3"
+ rtp_header_.reference_frame_id = kFirstFrameId + 1; // "Frame 2"
+ rtp_header_.rtp_timestamp += 2 * rtp_advance_per_frame;
FeedOneFrameIntoReceiver();
// Frame 2 should not come out at this point in time.
task_runner_->RunTasks();
EXPECT_EQ(1, fake_video_client_.number_times_called());
- // Enqueue a third request for an video frame.
+ // Enqueue a third request for a video frame.
receiver_->GetEncodedVideoFrame(frame_encoded_callback);
task_runner_->RunTasks();
EXPECT_EQ(1, fake_video_client_.number_times_called());
- // After |rtp_max_delay_ms| has elapsed, Frame 2 is emitted (to satisfy the
- // second request) because a decision was made to skip over the no-show Frame
- // 1.
- testing_clock_->Advance(
- base::TimeDelta::FromMilliseconds(config_.rtp_max_delay_ms));
+ // Now, advance time forward such that Frame 2 is now too late for playback.
+ // Regardless, the receiver must NOT emit Frame 3 yet because it is not
+ // allowed to skip frames for VP8.
+ testing_clock_->Advance(2 * time_advance_per_frame + target_playout_delay);
task_runner_->RunTasks();
- EXPECT_EQ(2, fake_video_client_.number_times_called());
-
- // Receive Frame 3 and expect it to fulfill the third request immediately.
- rtp_header_.frame_id = kFirstFrameId + 3;
- rtp_header_.reference_frame_id = rtp_header_.frame_id - 1;
- rtp_header_.rtp_timestamp += kVideoFrequency / config_.max_frame_rate;
- fake_video_client_.SetNextExpectedResult(kFirstFrameId + 3,
- testing_clock_->NowTicks());
+ EXPECT_EQ(1, fake_video_client_.number_times_called());
+
+ // Now receive Frame 2 and expect both the second and third requests to be
+ // fulfilled immediately.
+ fake_video_client_.AddExpectedResult(
+ kFirstFrameId + 1, // "Frame 2"
+ first_frame_capture_time + 1 * time_advance_per_frame +
+ target_playout_delay);
+ fake_video_client_.AddExpectedResult(
+ kFirstFrameId + 2, // "Frame 3"
+ first_frame_capture_time + 2 * time_advance_per_frame +
+ target_playout_delay);
+ --rtp_header_.frame_id; // "Frame 2"
+ --rtp_header_.reference_frame_id; // "Frame 1"
+ rtp_header_.rtp_timestamp -= rtp_advance_per_frame;
FeedOneFrameIntoReceiver();
task_runner_->RunTasks();
EXPECT_EQ(3, fake_video_client_.number_times_called());
- // Move forward another |rtp_max_delay_ms| and run any pending tasks (there
- // should be none). Expect no additional frames where emitted.
- testing_clock_->Advance(
- base::TimeDelta::FromMilliseconds(config_.rtp_max_delay_ms));
+ // Move forward to the playout time of an unreceived Frame 5. Expect no
+ // additional frames were emitted.
+ testing_clock_->Advance(3 * time_advance_per_frame);
task_runner_->RunTasks();
EXPECT_EQ(3, fake_video_client_.number_times_called());
}
diff --git a/media/cast/video_sender/video_sender_unittest.cc b/media/cast/video_sender/video_sender_unittest.cc
index 87d25d1..a7987a2 100644
--- a/media/cast/video_sender/video_sender_unittest.cc
+++ b/media/cast/video_sender/video_sender_unittest.cc
@@ -27,7 +27,6 @@ namespace media {
namespace cast {
namespace {
-static const int64 kStartMillisecond = INT64_C(12345678900000);
static const uint8 kPixelValue = 123;
static const int kWidth = 320;
static const int kHeight = 240;
@@ -107,8 +106,7 @@ class VideoSenderTest : public ::testing::Test {
protected:
VideoSenderTest() {
testing_clock_ = new base::SimpleTestTickClock();
- testing_clock_->Advance(
- base::TimeDelta::FromMilliseconds(kStartMillisecond));
+ testing_clock_->Advance(base::TimeTicks::Now() - base::TimeTicks());
task_runner_ = new test::FakeSingleThreadTaskRunner(testing_clock_);
cast_environment_ =
new CastEnvironment(scoped_ptr<base::TickClock>(testing_clock_).Pass(),
generated by cgit v1.1 at 2025-03-03 12:17:04 +0000