Attempt to fix audio wedges by restarting all streams on OSX.

Introduces two new methods to AudioOutputDispatcher:
CloseStreamsForWedgeFix() and RestartStreamsForWedgeFix().

Respectively, each method closes or restarts all active
streams owned by a given dispatcher.  The process is
completely transparent to upstream clients.

A new method on AudioManager, FixWedgedAudio() calls
CloseStreamsForWedgeFix() for all dispatchers and then
calls RestartStreamsForWedgeFix() afterward.

FixWedgedAudio() is called by each AudioOutputController
when a wedge is detected.  Multiple in flight wedge checks
are serialized by the audio thread.  The hope is that wedges
will be fixed before the next WedgeCheck() fires.

While the methods are available on all platforms, FixWedgedAudio()
is only wired up on OSX.

BUG=160920
TEST=unittest. fake wedge and observe stream recreation.
R=scherkus@chromium.org

Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=238325

Review URL: https://codereview.chromium.org/61203008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@238501 0039d316-1c4b-4281-b951-d872f2087c98

12 files changed, 163 insertions, 2 deletions

diff --git a/media/audio/audio_manager.h b/media/audio/audio_manager.h
index ab9c82b..a6cb705d 100644
--- a/media/audio/audio_manager.h
+++ b/media/audio/audio_manager.h
@@ -175,6 +175,11 @@ class MEDIA_EXPORT AudioManager {
   virtual std::string GetAssociatedOutputDeviceID(
       const std::string& input_device_id) = 0;
 
+  // Called when a component has detected a OS level audio wedge.  Shuts down
+  // all active audio streams and then restarts them transparently.  See
+  // http://crbug.com/160920
+  virtual void FixWedgedAudio() = 0;
+
  protected:
   AudioManager();
 
diff --git a/media/audio/audio_manager_base.cc b/media/audio/audio_manager_base.cc
index 52d545e..fc6574c 100644
--- a/media/audio/audio_manager_base.cc
+++ b/media/audio/audio_manager_base.cc
@@ -418,4 +418,32 @@ int AudioManagerBase::GetUserBufferSize() {
   return 0;
 }
 
+void AudioManagerBase::FixWedgedAudio() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+#if defined(OS_MACOSX)
+  // Through trial and error, we've found that one way to restore audio after a
+  // hang is to close all outstanding audio streams.  Once all streams have been
+  // closed, new streams appear to work correctly.
+  //
+  // In Chrome terms, this means we need to ask all AudioOutputDispatchers to
+  // close all Open()'d streams.  Once all streams across all dispatchers have
+  // been closed, we ask for all previously Start()'d streams to be recreated
+  // using the same AudioSourceCallback they had before.
+  //
+  // Since this operation takes place on the audio thread we can be sure that no
+  // other state-changing stream operations will take place while the fix is in
+  // progress.
+  //
+  // See http://crbug.com/160920 for additional details.
+  for (AudioOutputDispatchers::iterator it = output_dispatchers_.begin();
+       it != output_dispatchers_.end(); ++it) {
+    (*it)->dispatcher->CloseStreamsForWedgeFix();
+  }
+  for (AudioOutputDispatchers::iterator it = output_dispatchers_.begin();
+       it != output_dispatchers_.end(); ++it) {
+    (*it)->dispatcher->RestartStreamsForWedgeFix();
+  }
+#endif
+}
+
 }  // namespace media
diff --git a/media/audio/audio_manager_base.h b/media/audio/audio_manager_base.h
index 860ed5c..2fe71b7 100644
--- a/media/audio/audio_manager_base.h
+++ b/media/audio/audio_manager_base.h
@@ -115,6 +115,8 @@ class MEDIA_EXPORT AudioManagerBase : public AudioManager {
   virtual std::string GetAssociatedOutputDeviceID(
       const std::string& input_device_id) OVERRIDE;
 
+  virtual void FixWedgedAudio() OVERRIDE;
+
  protected:
   AudioManagerBase();
 
diff --git a/media/audio/audio_output_controller.cc b/media/audio/audio_output_controller.cc
index 994e8f4..92f9f25 100644
--- a/media/audio/audio_output_controller.cc
+++ b/media/audio/audio_output_controller.cc
@@ -478,8 +478,15 @@ void AudioOutputController::WedgeCheck() {
 
   // If we should be playing and we haven't, that's a wedge.
   if (state_ == kPlaying) {
-    UMA_HISTOGRAM_BOOLEAN("Media.AudioOutputControllerPlaybackStartupSuccess",
-                          base::AtomicRefCountIsOne(&on_more_io_data_called_));
+    const bool playback_success =
+        base::AtomicRefCountIsOne(&on_more_io_data_called_);
+
+    UMA_HISTOGRAM_BOOLEAN(
+        "Media.AudioOutputControllerPlaybackStartupSuccess", playback_success);
+
+    // Let the AudioManager try and fix it.
+    if (!playback_success)
+      audio_manager_->FixWedgedAudio();
   }
 }
 
diff --git a/media/audio/audio_output_dispatcher.h b/media/audio/audio_output_dispatcher.h
index 5004fa0..ac69634 100644
--- a/media/audio/audio_output_dispatcher.h
+++ b/media/audio/audio_output_dispatcher.h
@@ -66,6 +66,13 @@ class MEDIA_EXPORT AudioOutputDispatcher
   // Called on the audio thread when the AudioManager is shutting down.
   virtual void Shutdown() = 0;
 
+  // Called by the AudioManager to restart streams when a wedge is detected.  A
+  // wedge means the OS failed to request any audio after StartStream().  When a
+  // wedge is detected all streams across all dispatchers must be closed.  After
+  // all streams are closed, streams are restarted.  See http://crbug.com/160920
+  virtual void CloseStreamsForWedgeFix() = 0;
+  virtual void RestartStreamsForWedgeFix() = 0;
+
   // Accessor to the input device id used by unified IO.
   const std::string& input_device_id() const { return input_device_id_; }
 
diff --git a/media/audio/audio_output_dispatcher_impl.cc b/media/audio/audio_output_dispatcher_impl.cc
index eaac8a4..92c7da4 100644
--- a/media/audio/audio_output_dispatcher_impl.cc
+++ b/media/audio/audio_output_dispatcher_impl.cc
@@ -59,6 +59,8 @@ bool AudioOutputDispatcherImpl::StartStream(
     AudioOutputStream::AudioSourceCallback* callback,
     AudioOutputProxy* stream_proxy) {
   DCHECK(message_loop_->BelongsToCurrentThread());
+  DCHECK(proxy_to_physical_map_.find(stream_proxy) ==
+         proxy_to_physical_map_.end());
 
   if (idle_streams_.empty() && !CreateAndOpenStream())
     return false;
@@ -186,4 +188,21 @@ void AudioOutputDispatcherImpl::ClosePendingStreams() {
   }
 }
 
+void AudioOutputDispatcherImpl::CloseStreamsForWedgeFix() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+  while (!pausing_streams_.empty()) {
+    idle_streams_.push_back(pausing_streams_.back());
+    pausing_streams_.pop_back();
+  }
+  ClosePendingStreams();
+}
+
+void AudioOutputDispatcherImpl::RestartStreamsForWedgeFix() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+
+  // Should only be called when the dispatcher is used with fake streams which
+  // don't need to be shutdown or restarted.
+  CHECK_EQ(params_.format(), AudioParameters::AUDIO_FAKE);
+}
+
 }  // namespace media
diff --git a/media/audio/audio_output_dispatcher_impl.h b/media/audio/audio_output_dispatcher_impl.h
index f17f3a6..ca30b3a 100644
--- a/media/audio/audio_output_dispatcher_impl.h
+++ b/media/audio/audio_output_dispatcher_impl.h
@@ -59,6 +59,9 @@ class MEDIA_EXPORT AudioOutputDispatcherImpl : public AudioOutputDispatcher {
 
   virtual void Shutdown() OVERRIDE;
 
+  virtual void CloseStreamsForWedgeFix() OVERRIDE;
+  virtual void RestartStreamsForWedgeFix() OVERRIDE;
+
  private:
   typedef std::map<AudioOutputProxy*, AudioOutputStream*> AudioStreamMap;
   friend class base::RefCountedThreadSafe<AudioOutputDispatcherImpl>;
diff --git a/media/audio/audio_output_proxy_unittest.cc b/media/audio/audio_output_proxy_unittest.cc
index 3cffcc9..74e93eb 100644
--- a/media/audio/audio_output_proxy_unittest.cc
+++ b/media/audio/audio_output_proxy_unittest.cc
@@ -748,4 +748,64 @@ TEST_F(AudioOutputResamplerTest, LowLatencyOpenEventuallyFails) {
   EXPECT_FALSE(stream3.start_called());
 }
 
+// Ensures the methods used to fix audio output wedges are working correctly.
+TEST_F(AudioOutputResamplerTest, WedgeFix) {
+  MockAudioOutputStream stream1(&manager_, params_);
+  MockAudioOutputStream stream2(&manager_, params_);
+  MockAudioOutputStream stream3(&manager_, params_);
+
+  // Setup the mock such that all three streams are successfully created.
+  EXPECT_CALL(manager(), MakeAudioOutputStream(_, _, _))
+      .WillOnce(Return(&stream1))
+      .WillOnce(Return(&stream2))
+      .WillOnce(Return(&stream3));
+
+  // Stream1 should be able to successfully open and start.
+  EXPECT_CALL(stream1, Open())
+      .WillOnce(Return(true));
+  EXPECT_CALL(stream1, Close());
+  EXPECT_CALL(stream1, SetVolume(_));
+  EXPECT_CALL(stream2, Open())
+      .WillOnce(Return(true));
+  EXPECT_CALL(stream2, Close());
+
+  // Open and start the first proxy and stream.
+  AudioOutputProxy* proxy1 = new AudioOutputProxy(resampler_.get());
+  EXPECT_TRUE(proxy1->Open());
+  proxy1->Start(&callback_);
+  OnStart();
+
+  // Open, but do not start the second proxy.
+  AudioOutputProxy* proxy2 = new AudioOutputProxy(resampler_.get());
+  EXPECT_TRUE(proxy2->Open());
+
+  // Wait for stream to timeout and shutdown.
+  WaitForCloseTimer(kTestCloseDelayMs);
+
+  resampler_->CloseStreamsForWedgeFix();
+
+  // Stream3 should take Stream1's place after RestartStreamsForWedgeFix().
+  EXPECT_CALL(stream3, Open())
+      .WillOnce(Return(true));
+  EXPECT_CALL(stream3, Close());
+  EXPECT_CALL(stream3, SetVolume(_));
+
+  resampler_->RestartStreamsForWedgeFix();
+  OnStart();
+
+  // Perform the required Stop()/Close() shutdown dance for each proxy.
+  proxy2->Close();
+  proxy1->Stop();
+  proxy1->Close();
+
+  // Wait for all of the messages to fly and then verify stream behavior.
+  WaitForCloseTimer(kTestCloseDelayMs);
+  EXPECT_TRUE(stream1.stop_called());
+  EXPECT_TRUE(stream1.start_called());
+  EXPECT_FALSE(stream2.stop_called());
+  EXPECT_FALSE(stream2.start_called());
+  EXPECT_TRUE(stream3.stop_called());
+  EXPECT_TRUE(stream3.start_called());
+}
+
 }  // namespace media
diff --git a/media/audio/audio_output_resampler.cc b/media/audio/audio_output_resampler.cc
index 86b40dd..3179ddb 100644
--- a/media/audio/audio_output_resampler.cc
+++ b/media/audio/audio_output_resampler.cc
@@ -291,6 +291,30 @@ void AudioOutputResampler::Shutdown() {
   DCHECK(callbacks_.empty());
 }
 
+void AudioOutputResampler::CloseStreamsForWedgeFix() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+
+  // Stop and close all active streams.  Once all streams across all dispatchers
+  // have been closed the AudioManager will call RestartStreamsForWedgeFix().
+  for (CallbackMap::iterator it = callbacks_.begin(); it != callbacks_.end();
+       ++it) {
+    dispatcher_->StopStream(it->first);
+    dispatcher_->CloseStream(it->first);
+  }
+
+  // Close all idle streams as well.
+  dispatcher_->CloseStreamsForWedgeFix();
+}
+
+void AudioOutputResampler::RestartStreamsForWedgeFix() {
+  DCHECK(message_loop_->BelongsToCurrentThread());
+  for (CallbackMap::iterator it = callbacks_.begin(); it != callbacks_.end();
+       ++it) {
+    dispatcher_->OpenStream();
+    dispatcher_->StartStream(it->second, it->first);
+  }
+}
+
 OnMoreDataConverter::OnMoreDataConverter(const AudioParameters& input_params,
                                          const AudioParameters& output_params)
     : io_ratio_(static_cast<double>(input_params.GetBytesPerSecond()) /
diff --git a/media/audio/audio_output_resampler.h b/media/audio/audio_output_resampler.h
index 297d4e3..a8fca23 100644
--- a/media/audio/audio_output_resampler.h
+++ b/media/audio/audio_output_resampler.h
@@ -53,6 +53,8 @@ class MEDIA_EXPORT AudioOutputResampler : public AudioOutputDispatcher {
                                double volume) OVERRIDE;
   virtual void CloseStream(AudioOutputProxy* stream_proxy) OVERRIDE;
   virtual void Shutdown() OVERRIDE;
+  virtual void CloseStreamsForWedgeFix() OVERRIDE;
+  virtual void RestartStreamsForWedgeFix() OVERRIDE;
 
  private:
   friend class base::RefCountedThreadSafe<AudioOutputResampler>;
diff --git a/media/audio/mock_audio_manager.cc b/media/audio/mock_audio_manager.cc
index 6b54be2..31a0a4d 100644
--- a/media/audio/mock_audio_manager.cc
+++ b/media/audio/mock_audio_manager.cc
@@ -103,4 +103,6 @@ std::string MockAudioManager::GetAssociatedOutputDeviceID(
   return std::string();
 }
 
+void MockAudioManager::FixWedgedAudio() {}
+
 }  // namespace media.
diff --git a/media/audio/mock_audio_manager.h b/media/audio/mock_audio_manager.h
index 6b2b4c3..48f1d79 100644
--- a/media/audio/mock_audio_manager.h
+++ b/media/audio/mock_audio_manager.h
@@ -67,6 +67,8 @@ class MockAudioManager : public media::AudioManager {
   virtual std::string GetAssociatedOutputDeviceID(
       const std::string& input_device_id) OVERRIDE;
 
+  virtual void FixWedgedAudio() OVERRIDE;
+
  protected:
   virtual ~MockAudioManager();