// Copyright (c) 2012 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "media/audio/audio_output_resampler.h" #include "base/bind.h" #include "base/bind_helpers.h" #include "base/command_line.h" #include "base/compiler_specific.h" #include "base/message_loop.h" #include "base/metrics/histogram.h" #include "base/time.h" #include "media/audio/audio_io.h" #include "media/audio/audio_output_dispatcher_impl.h" #include "media/audio/audio_output_proxy.h" #include "media/audio/audio_util.h" #include "media/audio/sample_rates.h" #include "media/base/audio_pull_fifo.h" #include "media/base/channel_mixer.h" #include "media/base/limits.h" #include "media/base/media_switches.h" #include "media/base/multi_channel_resampler.h" namespace media { class OnMoreDataResampler : public AudioOutputStream::AudioSourceCallback { public: OnMoreDataResampler(double io_ratio, const AudioParameters& input_params, const AudioParameters& output_params); virtual ~OnMoreDataResampler(); // AudioSourceCallback interface. virtual int OnMoreData(AudioBus* dest, AudioBuffersState buffers_state) OVERRIDE; virtual int OnMoreIOData(AudioBus* source, AudioBus* dest, AudioBuffersState buffers_state) OVERRIDE; virtual void OnError(AudioOutputStream* stream, int code) OVERRIDE; virtual void WaitTillDataReady() OVERRIDE; // Sets |source_callback_|. If this is not a new object, then Stop() must be // called before Start(). void Start(AudioOutputStream::AudioSourceCallback* callback); // Clears |source_callback_| and flushes the resampler. void Stop(); private: // Called by MultiChannelResampler when more data is necessary. void ProvideInput(AudioBus* audio_bus); // Called by AudioPullFifo when more data is necessary. Requires // |source_lock_| to have been acquired. void SourceCallback_Locked(AudioBus* audio_bus); // Passes through |source| to the |source_callback_| OnMoreIOData() call. void SourceIOCallback_Locked(AudioBus* source, AudioBus* dest); // Ratio of input bytes to output bytes used to correct playback delay with // regard to buffering and resampling. double io_ratio_; // Source callback and associated lock. base::Lock source_lock_; AudioOutputStream::AudioSourceCallback* source_callback_; // Last AudioBuffersState object received via OnMoreData(), used to correct // playback delay by ProvideInput() and passed on to |source_callback_|. AudioBuffersState current_buffers_state_; // Total number of bytes (in terms of output parameters) stored in resampler // or FIFO buffers which have not been sent to the audio device. int outstanding_audio_bytes_; // Used to buffer data between the client and the output device in cases where // the client buffer size is not the same as the output device buffer size. // Bound to SourceCallback_Locked() so must only be used when |source_lock_| // has already been acquired. scoped_ptr audio_fifo_; // Handles resampling. scoped_ptr resampler_; // Handles channel transforms. |unmixed_audio_| is a temporary destination // for audio data before it goes into the channel mixer. scoped_ptr channel_mixer_; scoped_ptr unmixed_audio_; int output_bytes_per_frame_; int input_bytes_per_frame_; // Since resampling is expensive, figure out if we should downmix channels // before resampling. bool downmix_early_; DISALLOW_COPY_AND_ASSIGN(OnMoreDataResampler); }; // Record UMA statistics for hardware output configuration. static void RecordStats(const AudioParameters& output_params) { UMA_HISTOGRAM_ENUMERATION( "Media.HardwareAudioBitsPerChannel", output_params.bits_per_sample(), limits::kMaxBitsPerSample); UMA_HISTOGRAM_ENUMERATION( "Media.HardwareAudioChannelLayout", output_params.channel_layout(), CHANNEL_LAYOUT_MAX); UMA_HISTOGRAM_ENUMERATION( "Media.HardwareAudioChannelCount", output_params.channels(), limits::kMaxChannels); AudioSampleRate asr = media::AsAudioSampleRate(output_params.sample_rate()); if (asr != kUnexpectedAudioSampleRate) { UMA_HISTOGRAM_ENUMERATION( "Media.HardwareAudioSamplesPerSecond", asr, kUnexpectedAudioSampleRate); } else { UMA_HISTOGRAM_COUNTS( "Media.HardwareAudioSamplesPerSecondUnexpected", output_params.sample_rate()); } } // Record UMA statistics for hardware output configuration after fallback. static void RecordFallbackStats(const AudioParameters& output_params) { UMA_HISTOGRAM_BOOLEAN("Media.FallbackToHighLatencyAudioPath", true); UMA_HISTOGRAM_ENUMERATION( "Media.FallbackHardwareAudioBitsPerChannel", output_params.bits_per_sample(), limits::kMaxBitsPerSample); UMA_HISTOGRAM_ENUMERATION( "Media.FallbackHardwareAudioChannelLayout", output_params.channel_layout(), CHANNEL_LAYOUT_MAX); UMA_HISTOGRAM_ENUMERATION( "Media.FallbackHardwareAudioChannelCount", output_params.channels(), limits::kMaxChannels); AudioSampleRate asr = media::AsAudioSampleRate(output_params.sample_rate()); if (asr != kUnexpectedAudioSampleRate) { UMA_HISTOGRAM_ENUMERATION( "Media.FallbackHardwareAudioSamplesPerSecond", asr, kUnexpectedAudioSampleRate); } else { UMA_HISTOGRAM_COUNTS( "Media.FallbackHardwareAudioSamplesPerSecondUnexpected", output_params.sample_rate()); } } // Converts low latency based |output_params| into high latency appropriate // output parameters in error situations. static AudioParameters SetupFallbackParams( const AudioParameters& input_params, const AudioParameters& output_params) { // Choose AudioParameters appropriate for opening the device in high latency // mode. |kMinLowLatencyFrameSize| is arbitrarily based on Pepper Flash's // MAXIMUM frame size for low latency. static const int kMinLowLatencyFrameSize = 2048; int frames_per_buffer = std::min( std::max(input_params.frames_per_buffer(), kMinLowLatencyFrameSize), static_cast( GetHighLatencyOutputBufferSize(input_params.sample_rate()))); return AudioParameters( AudioParameters::AUDIO_PCM_LINEAR, input_params.channel_layout(), input_params.sample_rate(), input_params.bits_per_sample(), frames_per_buffer); } AudioOutputResampler::AudioOutputResampler(AudioManager* audio_manager, const AudioParameters& input_params, const AudioParameters& output_params, const base::TimeDelta& close_delay) : AudioOutputDispatcher(audio_manager, input_params), io_ratio_(1), close_delay_(close_delay), output_params_(output_params), streams_opened_(false) { DCHECK(input_params.IsValid()); DCHECK(output_params.IsValid()); DCHECK_EQ(output_params_.format(), AudioParameters::AUDIO_PCM_LOW_LATENCY); // Record UMA statistics for the hardware configuration. RecordStats(output_params); Initialize(); } AudioOutputResampler::~AudioOutputResampler() { DCHECK(callbacks_.empty()); } void AudioOutputResampler::Initialize() { DCHECK(!streams_opened_); DCHECK(callbacks_.empty()); io_ratio_ = 1; // Only resample or rebuffer if the input parameters don't match the output // parameters to avoid any unnecessary work. if (params_.channels() != output_params_.channels() || params_.sample_rate() != output_params_.sample_rate() || params_.bits_per_sample() != output_params_.bits_per_sample() || params_.frames_per_buffer() != output_params_.frames_per_buffer()) { if (params_.sample_rate() != output_params_.sample_rate()) { double io_sample_rate_ratio = params_.sample_rate() / static_cast(output_params_.sample_rate()); // Include the I/O resampling ratio in our global I/O ratio. io_ratio_ *= io_sample_rate_ratio; } // Include bits per channel differences. io_ratio_ *= static_cast(params_.bits_per_sample()) / output_params_.bits_per_sample(); // Include channel count differences. io_ratio_ *= static_cast(params_.channels()) / output_params_.channels(); DVLOG(1) << "I/O ratio is " << io_ratio_; } else { DVLOG(1) << "Input and output params are the same; in pass-through mode."; } // TODO(dalecurtis): All this code should be merged into AudioOutputMixer once // we've stabilized the issues there. dispatcher_ = new AudioOutputDispatcherImpl( audio_manager_, output_params_, close_delay_); } bool AudioOutputResampler::OpenStream() { DCHECK_EQ(MessageLoop::current(), message_loop_); if (dispatcher_->OpenStream()) { // Only record the UMA statistic if we didn't fallback during construction // and only for the first stream we open. if (!streams_opened_ && output_params_.format() == AudioParameters::AUDIO_PCM_LOW_LATENCY) { UMA_HISTOGRAM_BOOLEAN("Media.FallbackToHighLatencyAudioPath", false); } streams_opened_ = true; return true; } // If we've already tried to open the stream in high latency mode or we've // successfully opened a stream previously, there's nothing more to be done. if (output_params_.format() == AudioParameters::AUDIO_PCM_LINEAR || streams_opened_ || !callbacks_.empty()) { return false; } DCHECK_EQ(output_params_.format(), AudioParameters::AUDIO_PCM_LOW_LATENCY); if (CommandLine::ForCurrentProcess()->HasSwitch( switches::kDisableAudioFallback)) { LOG(ERROR) << "Open failed and automatic fallback to high latency audio " << "path is disabled, aborting."; return false; } DLOG(ERROR) << "Unable to open audio device in low latency mode. Falling " << "back to high latency audio output."; // Record UMA statistics about the hardware which triggered the failure so // we can debug and triage later. RecordFallbackStats(output_params_); output_params_ = SetupFallbackParams(params_, output_params_); Initialize(); // Retry, if this fails, there's nothing left to do but report the error back. return dispatcher_->OpenStream(); } bool AudioOutputResampler::StartStream( AudioOutputStream::AudioSourceCallback* callback, AudioOutputProxy* stream_proxy) { DCHECK_EQ(MessageLoop::current(), message_loop_); OnMoreDataResampler* resampler_callback = NULL; CallbackMap::iterator it = callbacks_.find(stream_proxy); if (it == callbacks_.end()) { resampler_callback = new OnMoreDataResampler( io_ratio_, params_, output_params_); callbacks_[stream_proxy] = resampler_callback; } else { resampler_callback = it->second; } resampler_callback->Start(callback); return dispatcher_->StartStream(resampler_callback, stream_proxy); } void AudioOutputResampler::StreamVolumeSet(AudioOutputProxy* stream_proxy, double volume) { DCHECK_EQ(MessageLoop::current(), message_loop_); dispatcher_->StreamVolumeSet(stream_proxy, volume); } void AudioOutputResampler::StopStream(AudioOutputProxy* stream_proxy) { DCHECK_EQ(MessageLoop::current(), message_loop_); dispatcher_->StopStream(stream_proxy); // Now that StopStream() has completed the underlying physical stream should // be stopped and no longer calling OnMoreData(), making it safe to Stop() the // OnMoreDataResampler. CallbackMap::iterator it = callbacks_.find(stream_proxy); if (it != callbacks_.end()) it->second->Stop(); } void AudioOutputResampler::CloseStream(AudioOutputProxy* stream_proxy) { DCHECK_EQ(MessageLoop::current(), message_loop_); dispatcher_->CloseStream(stream_proxy); // We assume that StopStream() is always called prior to CloseStream(), so // that it is safe to delete the OnMoreDataResampler here. CallbackMap::iterator it = callbacks_.find(stream_proxy); if (it != callbacks_.end()) { delete it->second; callbacks_.erase(it); } } void AudioOutputResampler::Shutdown() { DCHECK_EQ(MessageLoop::current(), message_loop_); // No AudioOutputProxy objects should hold a reference to us when we get // to this stage. DCHECK(HasOneRef()) << "Only the AudioManager should hold a reference"; dispatcher_->Shutdown(); DCHECK(callbacks_.empty()); } OnMoreDataResampler::OnMoreDataResampler( double io_ratio, const AudioParameters& input_params, const AudioParameters& output_params) : io_ratio_(io_ratio), source_callback_(NULL), outstanding_audio_bytes_(0), output_bytes_per_frame_(output_params.GetBytesPerFrame()), input_bytes_per_frame_(input_params.GetBytesPerFrame()), downmix_early_(false) { // Handle different input and output channel layouts. if (input_params.channel_layout() != output_params.channel_layout()) { DVLOG(1) << "Remixing channel layout from " << input_params.channel_layout() << " to " << output_params.channel_layout() << "; from " << input_params.channels() << " channels to " << output_params.channels() << " channels."; channel_mixer_.reset(new ChannelMixer( input_params.channel_layout(), output_params.channel_layout())); // Pare off data as early as we can for efficiency. downmix_early_ = input_params.channels() > output_params.channels(); if (downmix_early_) { DVLOG(1) << "Remixing channel layout prior to resampling."; // If we're downmixing early we need a temporary AudioBus which matches // the the input channel count and input frame size since we're passing // |unmixed_audio_| directly to the |source_callback_|. unmixed_audio_ = AudioBus::Create(input_params); } else { // Instead, if we're not downmixing early we need a temporary AudioBus // which matches the input channel count but uses the output frame size // since we'll mix into the AudioBus from the output stream. unmixed_audio_ = AudioBus::Create( input_params.channels(), output_params.frames_per_buffer()); } } // Only resample if necessary since it's expensive. if (input_params.sample_rate() != output_params.sample_rate()) { DVLOG(1) << "Resampling from " << input_params.sample_rate() << " to " << output_params.sample_rate(); double io_sample_rate_ratio = input_params.sample_rate() / static_cast(output_params.sample_rate()); resampler_.reset(new MultiChannelResampler( downmix_early_ ? output_params.channels() : input_params.channels(), io_sample_rate_ratio, base::Bind( &OnMoreDataResampler::ProvideInput, base::Unretained(this)))); } // Since the resampler / output device may want a different buffer size than // the caller asked for, we need to use a FIFO to ensure that both sides // read in chunk sizes they're configured for. if (input_params.sample_rate() != output_params.sample_rate() || input_params.frames_per_buffer() != output_params.frames_per_buffer()) { DVLOG(1) << "Rebuffering from " << input_params.frames_per_buffer() << " to " << output_params.frames_per_buffer(); audio_fifo_.reset(new AudioPullFifo( downmix_early_ ? output_params.channels() : input_params.channels(), input_params.frames_per_buffer(), base::Bind( &OnMoreDataResampler::SourceCallback_Locked, base::Unretained(this)))); } } OnMoreDataResampler::~OnMoreDataResampler() {} void OnMoreDataResampler::Start( AudioOutputStream::AudioSourceCallback* callback) { base::AutoLock auto_lock(source_lock_); DCHECK(!source_callback_); source_callback_ = callback; } void OnMoreDataResampler::Stop() { base::AutoLock auto_lock(source_lock_); source_callback_ = NULL; outstanding_audio_bytes_ = 0; if (audio_fifo_) audio_fifo_->Clear(); if (resampler_) resampler_->Flush(); } int OnMoreDataResampler::OnMoreData(AudioBus* dest, AudioBuffersState buffers_state) { return OnMoreIOData(NULL, dest, buffers_state); } int OnMoreDataResampler::OnMoreIOData(AudioBus* source, AudioBus* dest, AudioBuffersState buffers_state) { base::AutoLock auto_lock(source_lock_); // While we waited for |source_lock_| the callback might have been cleared. if (!source_callback_) { dest->Zero(); return dest->frames(); } current_buffers_state_ = buffers_state; bool needs_mixing = channel_mixer_ && !downmix_early_; AudioBus* temp_dest = needs_mixing ? unmixed_audio_.get() : dest; if (!resampler_ && !audio_fifo_) { // We have no internal buffers, so clear any outstanding audio data. outstanding_audio_bytes_ = 0; SourceIOCallback_Locked(source, temp_dest); } else { if (resampler_) resampler_->Resample(temp_dest, temp_dest->frames()); else ProvideInput(temp_dest); // Calculate how much data is left in the internal FIFO and resampler. outstanding_audio_bytes_ -= temp_dest->frames() * output_bytes_per_frame_; } if (needs_mixing) { DCHECK_EQ(temp_dest->frames(), dest->frames()); channel_mixer_->Transform(temp_dest, dest); } // Due to rounding errors while multiplying against |io_ratio_|, // |outstanding_audio_bytes_| might (rarely) slip below zero. if (outstanding_audio_bytes_ < 0) { DLOG(ERROR) << "Outstanding audio bytes went negative! Value: " << outstanding_audio_bytes_; outstanding_audio_bytes_ = 0; } // Always return the full number of frames requested, ProvideInput() will pad // with silence if it wasn't able to acquire enough data. return dest->frames(); } void OnMoreDataResampler::SourceCallback_Locked(AudioBus* dest) { SourceIOCallback_Locked(NULL, dest); } void OnMoreDataResampler::SourceIOCallback_Locked(AudioBus* source, AudioBus* dest) { source_lock_.AssertAcquired(); // Adjust playback delay to include the state of the internal buffers used by // the resampler and/or the FIFO. Since the sample rate and bits per channel // may be different, we need to scale this value appropriately. AudioBuffersState new_buffers_state; new_buffers_state.pending_bytes = io_ratio_ * (current_buffers_state_.total_bytes() + outstanding_audio_bytes_); bool needs_downmix = channel_mixer_ && downmix_early_; AudioBus* temp_dest = needs_downmix ? unmixed_audio_.get() : dest; // Retrieve data from the original callback. Zero any unfilled frames. int frames = source_callback_->OnMoreIOData( source, temp_dest, new_buffers_state); if (frames < temp_dest->frames()) temp_dest->ZeroFramesPartial(frames, temp_dest->frames() - frames); // Scale the number of frames we got back in terms of input bytes to output // bytes accordingly. outstanding_audio_bytes_ += (temp_dest->frames() * input_bytes_per_frame_) / io_ratio_; if (needs_downmix) { DCHECK_EQ(temp_dest->frames(), dest->frames()); channel_mixer_->Transform(temp_dest, dest); } } void OnMoreDataResampler::ProvideInput(AudioBus* audio_bus) { audio_fifo_->Consume(audio_bus, audio_bus->frames()); } void OnMoreDataResampler::OnError(AudioOutputStream* stream, int code) { base::AutoLock auto_lock(source_lock_); if (source_callback_) source_callback_->OnError(stream, code); } void OnMoreDataResampler::WaitTillDataReady() { base::AutoLock auto_lock(source_lock_); if (source_callback_ && !outstanding_audio_bytes_) source_callback_->WaitTillDataReady(); } } // namespace media