Avoids irregular OnMoreData callbacks on Windows using Core Audio.

Browser changes:

- Improves how native audio buffer sizes are derived on Windows.
- Forces user to always open up at native audio paramters.
- Improved internal scheme to set up the actial endpoint buffer based on input size.
- Refactored WSAPI output implementation and introduced CoreAudioUtil methods.
- Harmonized WSAPI output implementation with exusting unified implementation (to prepare for future merge).
- Changed GetAudioHardwareBufferSize() in audio_util.

Render changes for WebRTC:

- WebRTC now always asks for an output stream using native parameters to avoid rebuffering in the audio converter.
- Any buffer-size mismatch is now taken care of in WebRtcAudioRendrer using a pull FIFO. Delay estimates are also compensated if FIFO is used.
- Added DCHECKs to verify that methods are called on the expected threads.

BUG=170498

TEST=media_unittests, content_unittests, HTML5 audio tests in Chrome, WebAudio and Flash tests in Chrome, WebRTC tests in Chrome.

Review URL: https://codereview.chromium.org/12049070

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@180936 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 170 insertions, 108 deletions

diff --git a/content/renderer/media/webrtc_audio_capturer.cc b/content/renderer/media/webrtc_audio_capturer.cc
index a3c6ebf..a7e0b4f 100644
--- a/content/renderer/media/webrtc_audio_capturer.cc
+++ b/content/renderer/media/webrtc_audio_capturer.cc
@@ -373,7 +373,7 @@ void WebRtcAudioCapturer::Capture(media::AudioBus* audio_source,
           loopback_fifo_->max_frames()) {
         loopback_fifo_->Push(audio_source);
       } else {
-        DLOG(WARNING) << "FIFO is full";
+        DVLOG(1) << "FIFO is full";
       }
     }
   }
diff --git a/content/renderer/media/webrtc_audio_device_unittest.cc b/content/renderer/media/webrtc_audio_device_unittest.cc
index e988aa2..53d1c76 100644
--- a/content/renderer/media/webrtc_audio_device_unittest.cc
+++ b/content/renderer/media/webrtc_audio_device_unittest.cc
@@ -41,7 +41,7 @@ scoped_ptr<media::AudioHardwareConfig> CreateRealHardwareConfig() {
 }
 
 // Return true if at least one element in the array matches |value|.
-bool FindElementInArray(int* array, int size, int value) {
+bool FindElementInArray(const int* array, int size, int value) {
   return (std::find(&array[0], &array[0] + size, value) != &array[size]);
 }
 
@@ -56,7 +56,7 @@ bool HardwareSampleRatesAreValid() {
   // The actual WebRTC client can limit these ranges further depending on
   // platform but this is the maximum range we support today.
   int valid_input_rates[] = {16000, 32000, 44100, 48000, 96000};
-  int valid_output_rates[] = {44100, 48000, 96000};
+  int valid_output_rates[] = {16000, 32000, 44100, 48000, 96000};
 
   media::AudioHardwareConfig* hardware_config =
       RenderThreadImpl::current()->GetAudioHardwareConfig();
@@ -448,7 +448,7 @@ TEST_F(WebRTCAudioDeviceTest, DISABLED_PlayLocalFile) {
   // Play 2 seconds worth of audio and then quit.
   message_loop_.PostDelayedTask(FROM_HERE,
                                 MessageLoop::QuitClosure(),
-                                base::TimeDelta::FromSeconds(2));
+                                base::TimeDelta::FromSeconds(6));
   message_loop_.Run();
 
   renderer->Stop();
diff --git a/content/renderer/media/webrtc_audio_renderer.cc b/content/renderer/media/webrtc_audio_renderer.cc
index 0d33713..1b66b4d 100644
--- a/content/renderer/media/webrtc_audio_renderer.cc
+++ b/content/renderer/media/webrtc_audio_renderer.cc
@@ -11,7 +11,7 @@
 #include "content/renderer/media/renderer_audio_output_device.h"
 #include "content/renderer/media/webrtc_audio_device_impl.h"
 #include "content/renderer/render_thread_impl.h"
-#include "media/audio/audio_util.h"
+#include "media/audio/audio_parameters.h"
 #include "media/audio/sample_rates.h"
 #include "media/base/audio_hardware_config.h"
 
@@ -30,14 +30,14 @@ namespace {
 // current sample rate (set by the user) on Windows and Mac OS X.  The listed
 // rates below adds restrictions and Initialize() will fail if the user selects
 // any rate outside these ranges.
-int kValidOutputRates[] = {96000, 48000, 44100};
+const int kValidOutputRates[] = {96000, 48000, 44100, 32000, 16000};
 #elif defined(OS_LINUX) || defined(OS_OPENBSD)
-int kValidOutputRates[] = {48000, 44100};
+const int kValidOutputRates[] = {48000, 44100};
 #elif defined(OS_ANDROID)
 // On Android, the most popular sampling rate is 16000.
-int kValidOutputRates[] = {48000, 44100, 16000};
+const int kValidOutputRates[] = {48000, 44100, 16000};
 #else
-int kValidOutputRates[] = {44100};
+const int kValidOutputRates[] = {44100};
 #endif
 
 // TODO(xians): Merge the following code to WebRtcAudioCapturer, or remove.
@@ -88,15 +88,21 @@ WebRtcAudioRenderer::WebRtcAudioRenderer(int source_render_view_id)
     : state_(UNINITIALIZED),
       source_render_view_id_(source_render_view_id),
       source_(NULL),
-      play_ref_count_(0) {
+      play_ref_count_(0),
+      audio_delay_milliseconds_(0),
+      frame_duration_milliseconds_(0),
+      fifo_io_ratio_(1) {
 }
 
 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
+  DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK_EQ(state_, UNINITIALIZED);
   buffer_.reset();
 }
 
 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
+  DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
+  DCHECK(thread_checker_.CalledOnValidThread());
   base::AutoLock auto_lock(lock_);
   DCHECK_EQ(state_, UNINITIALIZED);
   DCHECK(source);
@@ -106,6 +112,13 @@ bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
   sink_ = AudioDeviceFactory::NewOutputDevice();
   DCHECK(sink_);
 
+  // Use mono on all platforms but Windows for now.
+  // TODO(henrika): Tracking at http://crbug.com/166771.
+  media::ChannelLayout channel_layout = media::CHANNEL_LAYOUT_MONO;
+#if defined(OS_WIN)
+  channel_layout = media::CHANNEL_LAYOUT_STEREO;
+#endif
+
   // Ask the renderer for the default audio output hardware sample-rate.
   media::AudioHardwareConfig* hardware_config =
       RenderThreadImpl::current()->GetAudioHardwareConfig();
@@ -124,102 +137,87 @@ bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
     return false;
   }
 
-  media::ChannelLayout channel_layout = media::CHANNEL_LAYOUT_STEREO;
+  // Set up audio parameters for the source, i.e., the WebRTC client.
+  // The WebRTC client only supports multiples of 10ms as buffer size where
+  // 10ms is preferred for lowest possible delay.
 
+  media::AudioParameters source_params;
   int buffer_size = 0;
 
-  // Windows
-#if defined(OS_WIN)
-  // Always use stereo rendering on Windows.
-  channel_layout = media::CHANNEL_LAYOUT_STEREO;
-
-  // Render side: AUDIO_PCM_LOW_LATENCY is based on the Core Audio (WASAPI)
-  // API which was introduced in Windows Vista. For lower Windows versions,
-  // a callback-driven Wave implementation is used instead. An output buffer
-  // size of 10ms works well for WASAPI but 30ms is needed for Wave.
-
-  // Use different buffer sizes depending on the current hardware sample rate.
-  if (sample_rate == 96000 || sample_rate == 48000) {
+  if (sample_rate % 8000 == 0) {
     buffer_size = (sample_rate / 100);
+  } else if (sample_rate == 44100) {
+    // The resampler in WebRTC does not support 441 as input. We hard code
+    // the size to 440 (~0.9977ms) instead and rely on the internal jitter
+    // buffer in WebRTC to deal with the resulting drift.
+    // TODO(henrika): ensure that WebRTC supports 44100Hz and use 441 instead.
+    buffer_size = 440;
   } else {
-    // We do run at 44.1kHz at the actual audio layer, but ask for frames
-    // at 44.0kHz to ensure that we can feed them to the webrtc::VoiceEngine.
-    // TODO(henrika): figure out why we seem to need 20ms here for glitch-
-    // free audio.
-    buffer_size = 2 * 440;
+    return false;
   }
 
-  // Windows XP and lower can't cope with 10 ms output buffer size.
-  // It must be extended to 30 ms (60 ms will be used internally by WaveOut).
-  // Note that we can't use media::CoreAudioUtil::IsSupported() here since it
-  // tries to load the Audioses.dll and it will always fail in the render
-  // process.
-  if (base::win::GetVersion() < base::win::VERSION_VISTA) {
-    buffer_size = 3 * buffer_size;
-    DLOG(WARNING) << "Extending the output buffer size by a factor of three "
-                  << "since Windows XP has been detected.";
+  source_params.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+                      channel_layout, sample_rate, 16, buffer_size);
+
+  // Set up audio parameters for the sink, i.e., the native audio output stream.
+  // We strive to open up using native parameters to achieve best possible
+  // performance and to ensure that no FIFO is needed on the browser side to
+  // match the client request. Any mismatch between the source and the sink is
+  // taken care of in this class instead using a pull FIFO.
+
+  media::AudioParameters sink_params;
+
+  buffer_size = hardware_config->GetOutputBufferSize();
+  sink_params.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+                    channel_layout, sample_rate, 16, buffer_size);
+
+  // Create a FIFO if re-buffering is required to match the source input with
+  // the sink request. The source acts as provider here and the sink as
+  // consumer.
+  if (source_params.frames_per_buffer() != sink_params.frames_per_buffer()) {
+    DVLOG(1) << "Rebuffering from " << source_params.frames_per_buffer()
+             << " to " << sink_params.frames_per_buffer();
+    audio_fifo_.reset(new media::AudioPullFifo(
+        source_params.channels(),
+        source_params.frames_per_buffer(),
+        base::Bind(
+            &WebRtcAudioRenderer::SourceCallback,
+            base::Unretained(this))));
+
+    // The I/O ratio is used in delay calculations where one scheme is used
+    // for |fifo_io_ratio_| > 1 and another scheme for < 1.0.
+    fifo_io_ratio_ = static_cast<double>(source_params.frames_per_buffer()) /
+        sink_params.frames_per_buffer();
   }
-#elif defined(OS_MACOSX)
-  channel_layout = media::CHANNEL_LAYOUT_MONO;
-
-  // Render side: AUDIO_PCM_LOW_LATENCY on Mac OS X is based on a callback-
-  // driven Core Audio implementation. Tests have shown that 10ms is a suitable
-  // frame size to use for 96kHz, 48kHz and 44.1kHz.
-
-  // Use different buffer sizes depending on the current hardware sample rate.
-  if (sample_rate == 96000 || sample_rate == 48000) {
-    buffer_size = (sample_rate / 100);
-  } else {
-    // We do run at 44.1kHz at the actual audio layer, but ask for frames
-    // at 44.0kHz to ensure that we can feed them to the webrtc::VoiceEngine.
-    buffer_size = 440;
-  }
-#elif defined(OS_LINUX) || defined(OS_OPENBSD)
-  channel_layout = media::CHANNEL_LAYOUT_MONO;
-
-  // Based on tests using the current ALSA implementation in Chrome, we have
-  // found that 10ms buffer size on the output side works fine.
-  buffer_size = 480;
-#elif defined(OS_ANDROID)
-  channel_layout = media::CHANNEL_LAYOUT_MONO;
-
-  // The buffer size lower than GetAudioHardwareBufferSize() will lead to
-  // choppy sound because AudioOutputResampler will read the buffer multiple
-  // times in a row without allowing the client to re-fill the buffer.
-  // TODO(dwkang): check if 2048 - GetAudioHardwareBufferSize() is the right
-  //               value for Android and do further tuning.
-  buffer_size = 2048;
-#else
-  DLOG(ERROR) << "Unsupported platform";
-  return false;
-#endif
 
-  // Store utilized parameters to ensure that we can check them
-  // after a successful initialization.
-  params_.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY, channel_layout,
-                sample_rate, 16, buffer_size);
+  frame_duration_milliseconds_ = base::Time::kMillisecondsPerSecond /
+      static_cast<double>(source_params.sample_rate());
 
   // Allocate local audio buffers based on the parameters above.
   // It is assumed that each audio sample contains 16 bits and each
   // audio frame contains one or two audio samples depending on the
   // number of channels.
-  buffer_.reset(new int16[params_.frames_per_buffer() * params_.channels()]);
+  buffer_.reset(
+      new int16[source_params.frames_per_buffer() * source_params.channels()]);
 
   source_ = source;
-  source->SetRenderFormat(params_);
+  source->SetRenderFormat(source_params);
 
-  // Configure the audio rendering client and start the rendering.
-  sink_->Initialize(params_, this);
+  // Configure the audio rendering client and start rendering.
+  sink_->Initialize(sink_params, this);
   sink_->SetSourceRenderView(source_render_view_id_);
   sink_->Start();
 
+  // User must call Play() before any audio can be heard.
   state_ = PAUSED;
 
   UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioOutputChannelLayout",
-                            channel_layout, media::CHANNEL_LAYOUT_MAX);
+                            source_params.channel_layout(),
+                            media::CHANNEL_LAYOUT_MAX);
   UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioOutputFramesPerBuffer",
-                            buffer_size, kUnexpectedAudioBufferSize);
-  AddHistogramFramesPerBuffer(buffer_size);
+                            source_params.frames_per_buffer(),
+                            kUnexpectedAudioBufferSize);
+  AddHistogramFramesPerBuffer(source_params.frames_per_buffer());
 
   return true;
 }
@@ -230,6 +228,8 @@ void WebRtcAudioRenderer::Start() {
 }
 
 void WebRtcAudioRenderer::Play() {
+  DVLOG(1) << "WebRtcAudioRenderer::Play()";
+  DCHECK(thread_checker_.CalledOnValidThread());
   base::AutoLock auto_lock(lock_);
   if (state_ == UNINITIALIZED)
     return;
@@ -237,9 +237,16 @@ void WebRtcAudioRenderer::Play() {
   DCHECK(play_ref_count_ == 0 || state_ == PLAYING);
   ++play_ref_count_;
   state_ = PLAYING;
+
+  if (audio_fifo_) {
+    audio_delay_milliseconds_ = 0;
+    audio_fifo_->Clear();
+  }
 }
 
 void WebRtcAudioRenderer::Pause() {
+  DVLOG(1) << "WebRtcAudioRenderer::Pause()";
+  DCHECK(thread_checker_.CalledOnValidThread());
   base::AutoLock auto_lock(lock_);
   if (state_ == UNINITIALIZED)
     return;
@@ -251,6 +258,8 @@ void WebRtcAudioRenderer::Pause() {
 }
 
 void WebRtcAudioRenderer::Stop() {
+  DVLOG(1) << "WebRtcAudioRenderer::Stop()";
+  DCHECK(thread_checker_.CalledOnValidThread());
   base::AutoLock auto_lock(lock_);
   if (state_ == UNINITIALIZED)
     return;
@@ -262,6 +271,7 @@ void WebRtcAudioRenderer::Stop() {
 }
 
 void WebRtcAudioRenderer::SetVolume(float volume) {
+  DCHECK(thread_checker_.CalledOnValidThread());
   base::AutoLock auto_lock(lock_);
   if (state_ == UNINITIALIZED)
     return;
@@ -279,26 +289,24 @@ bool WebRtcAudioRenderer::IsLocalRenderer() const {
 
 int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus,
                                 int audio_delay_milliseconds) {
-  {
-    base::AutoLock auto_lock(lock_);
-    if (!source_)
-      return 0;
-    // We need to keep render data for the |source_| reglardless of |state_|,
-    // otherwise the data will be buffered up inside |source_|.
-    source_->RenderData(reinterpret_cast<uint8*>(buffer_.get()),
-                        audio_bus->channels(), audio_bus->frames(),
-                        audio_delay_milliseconds);
-
-    // Return 0 frames to play out silence if |state_| is not PLAYING.
-    if (state_ != PLAYING)
-      return 0;
-  }
+  base::AutoLock auto_lock(lock_);
+  if (!source_)
+    return 0;
 
-  // Deinterleave each channel and convert to 32-bit floating-point
-  // with nominal range -1.0 -> +1.0 to match the callback format.
-  audio_bus->FromInterleaved(buffer_.get(), audio_bus->frames(),
-                             params_.bits_per_sample() / 8);
-  return audio_bus->frames();
+  DVLOG(2) << "WebRtcAudioRenderer::Render()";
+  DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds;
+
+  if (fifo_io_ratio_ > 1.0)
+    audio_delay_milliseconds_ += audio_delay_milliseconds;
+  else
+    audio_delay_milliseconds_ = audio_delay_milliseconds;
+
+  if (audio_fifo_)
+    audio_fifo_->Consume(audio_bus, audio_bus->frames());
+  else
+    SourceCallback(0, audio_bus);
+
+  return (state_ == PLAYING) ? audio_bus->frames() : 0;
 }
 
 void WebRtcAudioRenderer::OnRenderError() {
@@ -306,4 +314,38 @@ void WebRtcAudioRenderer::OnRenderError() {
   LOG(ERROR) << "OnRenderError()";
 }
 
+// Called by AudioPullFifo when more data is necessary.
+void WebRtcAudioRenderer::SourceCallback(
+    int fifo_frame_delay, media::AudioBus* audio_bus) {
+  DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
+           << fifo_frame_delay << ", "
+           << audio_bus->frames() << ")";
+
+  int output_delay_milliseconds = audio_delay_milliseconds_;
+  output_delay_milliseconds += frame_duration_milliseconds_ * fifo_frame_delay;
+  DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds;
+
+  // We need to keep render data for the |source_| regardless of |state_|,
+  // otherwise the data will be buffered up inside |source_|.
+  source_->RenderData(reinterpret_cast<uint8*>(buffer_.get()),
+                      audio_bus->channels(), audio_bus->frames(),
+                      output_delay_milliseconds);
+
+  if (fifo_io_ratio_ > 1.0)
+    audio_delay_milliseconds_ = 0;
+
+  // Avoid filling up the audio bus if we are not playing; instead
+  // return here and ensure that the returned value in Render() is 0.
+  if (state_ != PLAYING) {
+    audio_bus->Zero();
+    return;
+  }
+
+  // De-interleave each channel and convert to 32-bit floating-point
+  // with nominal range -1.0 -> +1.0 to match the callback format.
+  audio_bus->FromInterleaved(buffer_.get(),
+                             audio_bus->frames(),
+                             sizeof(buffer_[0]));
+}
+
 }  // namespace content
diff --git a/content/renderer/media/webrtc_audio_renderer.h b/content/renderer/media/webrtc_audio_renderer.h
index 09cccf0..e0b19c7 100644
--- a/content/renderer/media/webrtc_audio_renderer.h
+++ b/content/renderer/media/webrtc_audio_renderer.h
@@ -7,8 +7,10 @@
 
 #include "base/memory/ref_counted.h"
 #include "base/synchronization/lock.h"
+#include "base/threading/thread_checker.h"
 #include "content/renderer/media/webrtc_audio_device_impl.h"
 #include "media/base/audio_decoder.h"
+#include "media/base/audio_pull_fifo.h"
 #include "media/base/audio_renderer_sink.h"
 #include "webkit/media/media_stream_audio_renderer.h"
 
@@ -18,16 +20,15 @@ class RendererAudioOutputDevice;
 class WebRtcAudioRendererSource;
 
 // This renderer handles calls from the pipeline and WebRtc ADM. It is used
-// for connecting WebRtc MediaStream with pipeline.
+// for connecting WebRtc MediaStream with the audio pipeline.
 class CONTENT_EXPORT WebRtcAudioRenderer
     : NON_EXPORTED_BASE(public media::AudioRendererSink::RenderCallback),
       NON_EXPORTED_BASE(public webkit_media::MediaStreamAudioRenderer) {
  public:
   explicit WebRtcAudioRenderer(int source_render_view_id);
 
-  // Initialize function called by clients like WebRtcAudioDeviceImpl. Note,
+  // Initialize function called by clients like WebRtcAudioDeviceImpl.
   // Stop() has to be called before |source| is deleted.
-  // Returns false if Initialize() fails.
   bool Initialize(WebRtcAudioRendererSource* source);
 
   // Methods called by WebMediaPlayerMS and WebRtcAudioDeviceImpl.
@@ -49,14 +50,23 @@ class CONTENT_EXPORT WebRtcAudioRenderer
     PLAYING,
     PAUSED,
   };
+
+  // Used to DCHECK that we are called on the correct thread.
+  base::ThreadChecker thread_checker_;
+
   // Flag to keep track the state of the renderer.
   State state_;
 
   // media::AudioRendererSink::RenderCallback implementation.
+  // These two methods are called on the AudioOutputDevice worker thread.
   virtual int Render(media::AudioBus* audio_bus,
                      int audio_delay_milliseconds) OVERRIDE;
   virtual void OnRenderError() OVERRIDE;
 
+  // Called by AudioPullFifo when more data is necessary.
+  // This method is called on the AudioOutputDevice worker thread.
+  void SourceCallback(int fifo_frame_delay, media::AudioBus* audio_bus);
+
   // The render view in which the audio is rendered into |sink_|.
   const int source_render_view_id_;
 
@@ -66,19 +76,29 @@ class CONTENT_EXPORT WebRtcAudioRenderer
   // Audio data source from the browser process.
   WebRtcAudioRendererSource* source_;
 
-  // Cached values of utilized audio parameters. Platform dependent.
-  media::AudioParameters params_;
-
   // Buffers used for temporary storage during render callbacks.
   // Allocated during initialization.
   scoped_array<int16> buffer_;
 
-  // Protect access to |state_|.
+  // Protects access to |state_|, |source_| and |sink_|.
   base::Lock lock_;
 
   // Ref count for the MediaPlayers which are playing audio.
   int play_ref_count_;
 
+  // Used to buffer data between the client and the output device in cases where
+  // the client buffer size is not the same as the output device buffer size.
+  scoped_ptr<media::AudioPullFifo> audio_fifo_;
+
+  // Contains the accumulated delay estimate which is provided to the WebRTC
+  // AEC.
+  int audio_delay_milliseconds_;
+
+  // Lengh of an audio frame in milliseconds.
+  double frame_duration_milliseconds_;
+
+  double fifo_io_ratio_;
+
   DISALLOW_IMPLICIT_CONSTRUCTORS(WebRtcAudioRenderer);
 };