Since the introduction of PushSource, there are two buffering layers in the

browser process, the hardware buffer used in AudioOutputStream and
transportation buffer in PushSource. Together with the latency in the IPC
audio layer we have a serious AV sync problem.

To compensate the delay and latency introduced by these three factors
two parameters are added in RequestAudioPacket message that include
the buffer fill level and timestamp of the request. These two parameters
are used to determine the playback delay to be used by the audio
renderer to update the pipeline with the time delta.

So we have three parameters we need to care about:
1. Hardware buffer in AudioOutputStream
2. Buffered data in PushSource
3. IPC latency

We have accurate values for 2 and 3 but not 1. We currently don't have the
API in AudioOutputStream to query the remaining buffer in the hardware
buffer. But usually there is a large amount of data in it, e.g. on Windows
400ms worth of data. Since we now detached the hardware buffer request of
OnMoreData() from the actual packet request of IPC (by the introduction of
PushSource), it is really critical to know the buffer level in the hardware.
I made a guess of this buffer level by using the amount of last buffer copy.

Review URL: http://codereview.chromium.org/122020

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@18536 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 80 insertions, 14 deletions

diff --git a/chrome/renderer/media/audio_renderer_impl.cc b/chrome/renderer/media/audio_renderer_impl.cc
index 9dd365c..73f2320 100644
--- a/chrome/renderer/media/audio_renderer_impl.cc
+++ b/chrome/renderer/media/audio_renderer_impl.cc
@@ -28,14 +28,18 @@ const int kMillisecondsPreroll = 400;
 
 AudioRendererImpl::AudioRendererImpl(AudioMessageFilter* filter)
     : AudioRendererBase(kDefaultMaxQueueSize),
+      channels_(0),
+      sample_rate_(0),
+      sample_bits_(0),
+      bytes_per_second_(0),
       filter_(filter),
       stream_id_(0),
       shared_memory_(NULL),
       shared_memory_size_(0),
       io_loop_(filter->message_loop()),
       stopped_(false),
-      pending_request_(false),
       playback_rate_(0.0f),
+      pending_request_(false),
       prerolling_(true),
       preroll_bytes_(0) {
   DCHECK(io_loop_);
@@ -44,6 +48,14 @@ AudioRendererImpl::AudioRendererImpl(AudioMessageFilter* filter)
 AudioRendererImpl::~AudioRendererImpl() {
 }
 
+base::TimeDelta AudioRendererImpl::ConvertToDuration(int bytes) {
+  if (bytes_per_second_) {
+    return base::TimeDelta::FromMicroseconds(
+        base::Time::kMicrosecondsPerSecond * bytes / bytes_per_second_);
+  }
+  return base::TimeDelta();
+}
+
 bool AudioRendererImpl::IsMediaFormatSupported(
     const media::MediaFormat& media_format) {
   int channels;
@@ -54,24 +66,24 @@ bool AudioRendererImpl::IsMediaFormatSupported(
 
 bool AudioRendererImpl::OnInitialize(const media::MediaFormat& media_format) {
   // Parse integer values in MediaFormat.
-  int channels;
-  int sample_rate;
-  int sample_bits;
-  if (!ParseMediaFormat(media_format, &channels, &sample_rate, &sample_bits)) {
+  if (!ParseMediaFormat(media_format,
+                        &channels_,
+                        &sample_rate_,
+                        &sample_bits_)) {
     return false;
   }
 
   // Create the audio output stream in browser process.
-  size_t bytes_per_second = sample_rate * channels * sample_bits / 8;
-  size_t packet_size = bytes_per_second * kMillisecondsPerPacket / 1000;
+  bytes_per_second_ = sample_rate_ * channels_ * sample_bits_ / 8;
+  size_t packet_size = bytes_per_second_ * kMillisecondsPerPacket / 1000;
   size_t buffer_capacity = packet_size * kPacketsInBuffer;
 
   // Calculate the amount for prerolling.
-  preroll_bytes_ = bytes_per_second * kMillisecondsPreroll / 1000;
+  preroll_bytes_ = bytes_per_second_ * kMillisecondsPreroll / 1000;
 
   io_loop_->PostTask(FROM_HERE,
       NewRunnableMethod(this, &AudioRendererImpl::OnCreateStream,
-          AudioManager::AUDIO_PCM_LINEAR, channels, sample_rate, sample_bits,
+          AudioManager::AUDIO_PCM_LINEAR, channels_, sample_rate_, sample_bits_,
           packet_size, buffer_capacity));
   return true;
 }
@@ -153,13 +165,19 @@ void AudioRendererImpl::OnCreated(base::SharedMemoryHandle handle,
   shared_memory_size_ = length;
 }
 
-void AudioRendererImpl::OnRequestPacket() {
+void AudioRendererImpl::OnRequestPacket(size_t bytes_in_buffer,
+                                        const base::Time& message_timestamp) {
   DCHECK(MessageLoop::current() == io_loop_);
 
   {
     AutoLock auto_lock(lock_);
     DCHECK(!pending_request_);
     pending_request_ = true;
+
+    // Use the information provided by the IPC message to adjust the playback
+    // delay.
+    request_timestamp_ = message_timestamp;
+    request_delay_ = ConvertToDuration(bytes_in_buffer);
   }
 
   // Try to fill in the fulfil the packet request.
@@ -253,12 +271,32 @@ void AudioRendererImpl::OnNotifyPacketReady() {
     return;
   if (pending_request_ && playback_rate_ > 0.0f) {
     DCHECK(shared_memory_.get());
+
+    // Adjust the playback delay.
+    base::Time current_time = base::Time::Now();
+
+    // Save a local copy of the request delay.
+    base::TimeDelta request_delay = request_delay_;
+    if (current_time > request_timestamp_) {
+      base::TimeDelta receive_latency = current_time - request_timestamp_;
+
+      // If the receive latency is too much it may offset all the delay.
+      if (receive_latency >= request_delay) {
+        request_delay = base::TimeDelta();
+      } else {
+        request_delay -= receive_latency;
+      }
+    }
+
     size_t filled = FillBuffer(static_cast<uint8*>(shared_memory_->memory()),
                                shared_memory_size_,
-                               playback_rate_);
+                               playback_rate_,
+                               request_delay);
     // TODO(hclam): we should try to fill in the buffer as much as possible.
     if (filled > 0) {
       pending_request_ = false;
+      request_delay_ = base::TimeDelta();
+      request_timestamp_ = base::Time();
       // Then tell browser process we are done filling into the buffer.
       filter_->Send(
           new ViewHostMsg_NotifyAudioPacketReady(0, stream_id_, filled));
diff --git a/chrome/renderer/media/audio_renderer_impl.h b/chrome/renderer/media/audio_renderer_impl.h
index 6a24dfc..162cf09 100644
--- a/chrome/renderer/media/audio_renderer_impl.h
+++ b/chrome/renderer/media/audio_renderer_impl.h
@@ -116,7 +116,8 @@ class AudioRendererImpl : public media::AudioRendererBase,
 
   // Methods called on IO thread ----------------------------------------------
   // AudioMessageFilter::Delegate methods, called by AudioMessageFilter.
-  void OnRequestPacket();
+  void OnRequestPacket(size_t bytes_in_buffer,
+                       const base::Time& message_timestamp);
   void OnStateChanged(AudioOutputStream::State state, int info);
   void OnCreated(base::SharedMemoryHandle handle, size_t length);
   void OnVolume(double left, double right);
@@ -144,6 +145,11 @@ class AudioRendererImpl : public media::AudioRendererBase,
   explicit AudioRendererImpl(AudioMessageFilter* filter);
   virtual ~AudioRendererImpl();
 
+  // Helper methods.
+  // Convert number of bytes to duration of time using information about the
+  // number of channels, sample rate and sample bits.
+  base::TimeDelta ConvertToDuration(int bytes);
+
   // Methods call on IO thread ------------------------------------------------
   // The following methods are tasks posted on the IO thread that needs to
   // be executed on that thread. They interact with AudioMessageFilter and
@@ -157,6 +163,12 @@ class AudioRendererImpl : public media::AudioRendererBase,
   void OnNotifyPacketReady();
   void OnDestroy();
 
+  // Information about the audio stream.
+  int channels_;
+  int sample_rate_;
+  int sample_bits_;
+  size_t bytes_per_second_;
+
   scoped_refptr<AudioMessageFilter> filter_;
 
   // ID of the stream created in the browser process.
@@ -170,16 +182,32 @@ class AudioRendererImpl : public media::AudioRendererBase,
   MessageLoop* io_loop_;
 
   // Protects:
-  // - |playback_rate_|
   // - |stopped_|
+  // - |playback_rate_|
   // - |pending_request_|
+  // - |request_timestamp_|
+  // - |request_delay_|
   Lock lock_;
+
+  // A flag that indicates this filter is called to stop.
   bool stopped_;
-  bool pending_request_;
+
+  // Keeps the current playback rate.
   float playback_rate_;
 
+  // A flag that indicates an outstanding packet request.
+  bool pending_request_;
+
+  // The time when a request is made.
+  base::Time request_timestamp_;
+
+  // The delay for the requested packet to be played.
+  base::TimeDelta request_delay_;
+
   // State variables for prerolling.
   bool prerolling_;
+
+  // Remaining bytes for prerolling to complete.
   size_t preroll_bytes_;
 
   DISALLOW_COPY_AND_ASSIGN(AudioRendererImpl);