diff options
20 files changed, 108 insertions, 56 deletions
diff --git a/content/renderer/media/renderer_webaudiodevice_impl.cc b/content/renderer/media/renderer_webaudiodevice_impl.cc index 6c366c1..6088a06 100644 --- a/content/renderer/media/renderer_webaudiodevice_impl.cc +++ b/content/renderer/media/renderer_webaudiodevice_impl.cc @@ -100,7 +100,7 @@ double RendererWebAudioDeviceImpl::sampleRate() { } int RendererWebAudioDeviceImpl::Render(media::AudioBus* dest, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) { #if defined(OS_ANDROID) if (is_first_buffer_after_silence_) { diff --git a/content/renderer/media/renderer_webaudiodevice_impl.h b/content/renderer/media/renderer_webaudiodevice_impl.h index db9f86c..20160a9 100644 --- a/content/renderer/media/renderer_webaudiodevice_impl.h +++ b/content/renderer/media/renderer_webaudiodevice_impl.h @@ -45,7 +45,7 @@ class RendererWebAudioDeviceImpl // AudioRendererSink::RenderCallback implementation. int Render(media::AudioBus* dest, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) override; void OnRenderError() override; diff --git a/content/renderer/media/track_audio_renderer.h b/content/renderer/media/track_audio_renderer.h index 0703746..1404b58 100644 --- a/content/renderer/media/track_audio_renderer.h +++ b/content/renderer/media/track_audio_renderer.h @@ -105,7 +105,7 @@ class CONTENT_EXPORT TrackAudioRenderer // Render() is called on the AudioOutputDevice thread and OnRenderError() // on the IO thread. int Render(media::AudioBus* audio_bus, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) override; void OnRenderError() override; diff --git a/content/renderer/media/webrtc_audio_renderer.cc b/content/renderer/media/webrtc_audio_renderer.cc index 49fc185..5f859c3 100644 --- a/content/renderer/media/webrtc_audio_renderer.cc +++ b/content/renderer/media/webrtc_audio_renderer.cc @@ -423,13 +423,21 @@ media::OutputDeviceStatus WebRtcAudioRenderer::GetDeviceStatus() { } int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) { DCHECK(audio_renderer_thread_checker_.CalledOnValidThread()); base::AutoLock auto_lock(lock_); if (!source_) return 0; + // TODO(grunell): Converting from frames to milliseconds will potentially lose + // hundreds of microseconds which may cause audio video drift. Update + // this class and all usage of render delay msec -> frames (possibly even + // using a double type for frames). See http://crbug.com/586540 + uint32_t audio_delay_milliseconds = static_cast<double>(frames_delayed) * + base::Time::kMillisecondsPerSecond / + sink_params_.sample_rate(); + DVLOG(2) << "WebRtcAudioRenderer::Render()"; DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds; @@ -481,6 +489,8 @@ void WebRtcAudioRenderer::SourceCallback( << audio_bus->frames() << ")"; int output_delay_milliseconds = audio_delay_milliseconds_; + // TODO(grunell): This integer division by sample_rate will cause loss of + // partial milliseconds, and may cause avsync drift. http://crbug.com/586540 output_delay_milliseconds += fifo_frame_delay * base::Time::kMillisecondsPerSecond / sink_params_.sample_rate(); diff --git a/content/renderer/media/webrtc_audio_renderer.h b/content/renderer/media/webrtc_audio_renderer.h index 449e42c..ae8686c 100644 --- a/content/renderer/media/webrtc_audio_renderer.h +++ b/content/renderer/media/webrtc_audio_renderer.h @@ -172,7 +172,7 @@ class CONTENT_EXPORT WebRtcAudioRenderer // media::AudioRendererSink::RenderCallback implementation. // These two methods are called on the AudioOutputDevice worker thread. int Render(media::AudioBus* audio_bus, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) override; void OnRenderError() override; diff --git a/media/audio/audio_device_thread.cc b/media/audio/audio_device_thread.cc index f4e00d4..2087459 100644 --- a/media/audio/audio_device_thread.cc +++ b/media/audio/audio_device_thread.cc @@ -210,21 +210,23 @@ void AudioDeviceThread::Thread::Run() { // AudioDeviceThread::Callback implementation -AudioDeviceThread::Callback::Callback( - const AudioParameters& audio_parameters, - base::SharedMemoryHandle memory, - int memory_length, - int total_segments) +AudioDeviceThread::Callback::Callback(const AudioParameters& audio_parameters, + base::SharedMemoryHandle memory, + int memory_length, + int total_segments) : audio_parameters_(audio_parameters), - samples_per_ms_(audio_parameters.sample_rate() / 1000), + samples_per_ms_(static_cast<double>(audio_parameters.sample_rate()) / + base::Time::kMillisecondsPerSecond), bytes_per_ms_(audio_parameters.channels() * (audio_parameters_.bits_per_sample() / 8) * samples_per_ms_), + bytes_per_frame_(audio_parameters.GetBytesPerFrame()), shared_memory_(memory, false), memory_length_(memory_length), total_segments_(total_segments) { - CHECK_NE(bytes_per_ms_, 0); // Catch division by zero early. - CHECK_NE(samples_per_ms_, 0); + CHECK_NE(bytes_per_ms_, 0.0); // Catch division by zero early. + CHECK_NE(samples_per_ms_, 0.0); + CHECK_NE(bytes_per_frame_, 0u); CHECK_GT(total_segments_, 0); CHECK_EQ(memory_length_ % total_segments_, 0); segment_length_ = memory_length_ / total_segments_; diff --git a/media/audio/audio_device_thread.h b/media/audio/audio_device_thread.h index 7ec9d6e..3cd673c 100644 --- a/media/audio/audio_device_thread.h +++ b/media/audio/audio_device_thread.h @@ -60,8 +60,9 @@ class MEDIA_EXPORT AudioDeviceThread { // The variables are 'const' since values are calculated/set in the // constructor and must never change. const AudioParameters audio_parameters_; - const int samples_per_ms_; - const int bytes_per_ms_; + const double samples_per_ms_; + const double bytes_per_ms_; + const uint32_t bytes_per_frame_; base::SharedMemory shared_memory_; const int memory_length_; diff --git a/media/audio/audio_input_device.cc b/media/audio/audio_input_device.cc index 52017f2..f5e9ae6 100644 --- a/media/audio/audio_input_device.cc +++ b/media/audio/audio_input_device.cc @@ -340,8 +340,7 @@ void AudioInputDevice::AudioThreadCallback::Process(uint32_t pending_data) { capture_callback_->Capture( audio_bus, buffer->params.hardware_delay_bytes / bytes_per_ms_, // Delay in ms - buffer->params.volume, - buffer->params.key_pressed); + buffer->params.volume, buffer->params.key_pressed); if (++current_segment_id_ >= total_segments_) current_segment_id_ = 0; diff --git a/media/audio/audio_output_device.cc b/media/audio/audio_output_device.cc index 3af32b7..7044e9a 100644 --- a/media/audio/audio_output_device.cc +++ b/media/audio/audio_output_device.cc @@ -6,6 +6,8 @@ #include <stddef.h> #include <stdint.h> + +#include <cmath> #include <utility> #include "base/callback_helpers.h" @@ -413,8 +415,8 @@ void AudioOutputDevice::AudioThreadCallback::MapSharedMemory() { // Called whenever we receive notifications about pending data. void AudioOutputDevice::AudioThreadCallback::Process(uint32_t pending_data) { - // Convert the number of pending bytes in the render buffer into milliseconds. - uint32_t audio_delay_milliseconds = pending_data / bytes_per_ms_; + // Convert the number of pending bytes in the render buffer into frames. + double frames_delayed = static_cast<double>(pending_data) / bytes_per_frame_; callback_num_++; TRACE_EVENT1("audio", "AudioOutputDevice::FireRenderCallback", @@ -433,11 +435,15 @@ void AudioOutputDevice::AudioThreadCallback::Process(uint32_t pending_data) { uint32_t frames_skipped = buffer->params.frames_skipped; buffer->params.frames_skipped = 0; + DVLOG(4) << __FUNCTION__ << " pending_data:" << pending_data + << " frames_delayed(pre-round):" << frames_delayed + << " frames_skipped:" << frames_skipped; + // Update the audio-delay measurement, inform about the number of skipped // frames, and ask client to render audio. Since |output_bus_| is wrapping // the shared memory the Render() call is writing directly into the shared // memory. - render_callback_->Render(output_bus_.get(), audio_delay_milliseconds, + render_callback_->Render(output_bus_.get(), std::round(frames_delayed), frames_skipped); } diff --git a/media/audio/audio_output_stream_sink.cc b/media/audio/audio_output_stream_sink.cc index 92739b3..460dedec 100644 --- a/media/audio/audio_output_stream_sink.cc +++ b/media/audio/audio_output_stream_sink.cc @@ -4,6 +4,8 @@ #include "media/audio/audio_output_stream_sink.h" +#include <cmath> + #include "base/bind.h" #include "base/bind_helpers.h" #include "base/location.h" @@ -83,8 +85,10 @@ int AudioOutputStreamSink::OnMoreData(AudioBus* dest, if (!active_render_callback_) return 0; - return active_render_callback_->Render( - dest, total_bytes_delay * 1000.0 / active_params_.GetBytesPerSecond(), 0); + uint32_t frames_delayed = std::round(static_cast<double>(total_bytes_delay) / + active_params_.GetBytesPerFrame()); + + return active_render_callback_->Render(dest, frames_delayed, frames_skipped); } void AudioOutputStreamSink::OnError(AudioOutputStream* stream) { diff --git a/media/audio/audio_parameters.cc b/media/audio/audio_parameters.cc index 08e5825..10f41d5 100644 --- a/media/audio/audio_parameters.cc +++ b/media/audio/audio_parameters.cc @@ -77,6 +77,10 @@ int AudioParameters::GetBytesPerFrame() const { return channels_ * bits_per_sample_ / 8; } +double AudioParameters::GetMicrosecondsPerFrame() const { + return static_cast<double>(base::Time::kMicrosecondsPerSecond) / sample_rate_; +} + base::TimeDelta AudioParameters::GetBufferDuration() const { return base::TimeDelta::FromMicroseconds(static_cast<int64_t>( frames_per_buffer_ * base::Time::kMicrosecondsPerSecond / diff --git a/media/audio/audio_parameters.h b/media/audio/audio_parameters.h index 3e6603b..2735d79 100644 --- a/media/audio/audio_parameters.h +++ b/media/audio/audio_parameters.h @@ -125,6 +125,12 @@ class MEDIA_EXPORT AudioParameters { // Returns the number of bytes representing a frame of audio. int GetBytesPerFrame() const; + // Returns the number of microseconds per frame of audio. Intentionally + // reported as a double to surface of partial microseconds per frame, which + // is common for many sample rates. Failing to account for these nanoseconds + // can lead to audio/video sync drift. + double GetMicrosecondsPerFrame() const; + // Returns the duration of this buffer as calculated from frames_per_buffer() // and sample_rate(). base::TimeDelta GetBufferDuration() const; diff --git a/media/base/audio_capturer_source.h b/media/base/audio_capturer_source.h index 31b39bb..fd79db3 100644 --- a/media/base/audio_capturer_source.h +++ b/media/base/audio_capturer_source.h @@ -23,6 +23,8 @@ class AudioCapturerSource class CaptureCallback { public: // Callback to deliver the captured data from the OS. + // TODO(chcunningham): Update delay argument to use frames instead of + // milliseconds to prevent loss of precision. See http://crbug.com/587291. virtual void Capture(const AudioBus* audio_source, int audio_delay_milliseconds, double volume, diff --git a/media/base/audio_renderer_mixer.cc b/media/base/audio_renderer_mixer.cc index 218bb9e..a2b5e89 100644 --- a/media/base/audio_renderer_mixer.cc +++ b/media/base/audio_renderer_mixer.cc @@ -4,6 +4,8 @@ #include "media/base/audio_renderer_mixer.h" +#include <cmath> + #include "base/bind.h" #include "base/bind_helpers.h" #include "base/logging.h" @@ -117,7 +119,7 @@ OutputDevice* AudioRendererMixer::GetOutputDevice() { } int AudioRendererMixer::Render(AudioBus* audio_bus, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) { base::AutoLock auto_lock(lock_); @@ -132,8 +134,13 @@ int AudioRendererMixer::Render(AudioBus* audio_bus, playing_ = false; } - master_converter_.ConvertWithDelay( - base::TimeDelta::FromMilliseconds(audio_delay_milliseconds), audio_bus); + // TODO(chcunningham): Delete this conversion and change ConvertWith delay to + // expect a count of frames delayed instead of TimeDelta (less precise). + // See http://crbug.com/587522. + base::TimeDelta audio_delay = base::TimeDelta::FromMicroseconds( + std::round(frames_delayed * output_params_.GetMicrosecondsPerFrame())); + + master_converter_.ConvertWithDelay(audio_delay, audio_bus); return audio_bus->frames(); } diff --git a/media/base/audio_renderer_mixer.h b/media/base/audio_renderer_mixer.h index dae8b7c..042c4be 100644 --- a/media/base/audio_renderer_mixer.h +++ b/media/base/audio_renderer_mixer.h @@ -58,7 +58,7 @@ class MEDIA_EXPORT AudioRendererMixer // AudioRendererSink::RenderCallback implementation. int Render(AudioBus* audio_bus, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) override; void OnRenderError() override; diff --git a/media/base/audio_renderer_mixer_input.cc b/media/base/audio_renderer_mixer_input.cc index d0d38d3..7c6ba26 100644 --- a/media/base/audio_renderer_mixer_input.cc +++ b/media/base/audio_renderer_mixer_input.cc @@ -4,6 +4,8 @@ #include "media/base/audio_renderer_mixer_input.h" +#include <cmath> + #include "base/bind.h" #include "base/callback_helpers.h" #include "media/base/audio_renderer_mixer.h" @@ -165,8 +167,13 @@ OutputDeviceStatus AudioRendererMixerInput::GetDeviceStatus() { double AudioRendererMixerInput::ProvideInput(AudioBus* audio_bus, base::TimeDelta buffer_delay) { - int frames_filled = callback_->Render( - audio_bus, static_cast<int>(buffer_delay.InMillisecondsF() + 0.5), 0); + // TODO(chcunningham): Delete this conversion and change ProvideInput to more + // precisely describe delay as a count of frames delayed instead of TimeDelta. + // See http://crbug.com/587522. + uint32_t frames_delayed = std::round(buffer_delay.InMicroseconds() / + params_.GetMicrosecondsPerFrame()); + + int frames_filled = callback_->Render(audio_bus, frames_delayed, 0); // AudioConverter expects unfilled frames to be zeroed. if (frames_filled < audio_bus->frames()) { diff --git a/media/base/audio_renderer_sink.h b/media/base/audio_renderer_sink.h index 9da243b..ac845f3 100644 --- a/media/base/audio_renderer_sink.h +++ b/media/base/audio_renderer_sink.h @@ -39,7 +39,7 @@ class AudioRendererSink // number of frames filled. |frames_skipped| contains the number of frames // the consumer has skipped, if any. virtual int Render(AudioBus* dest, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) = 0; // Signals an error has occurred. diff --git a/media/renderers/audio_renderer_impl.cc b/media/renderers/audio_renderer_impl.cc index e37ffc0..335dd2e 100644 --- a/media/renderers/audio_renderer_impl.cc +++ b/media/renderers/audio_renderer_impl.cc @@ -634,13 +634,13 @@ bool AudioRendererImpl::IsBeforeStartTime( } int AudioRendererImpl::Render(AudioBus* audio_bus, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) { - const int requested_frames = audio_bus->frames(); - base::TimeDelta playback_delay = base::TimeDelta::FromMilliseconds( - audio_delay_milliseconds); - const int delay_frames = static_cast<int>(playback_delay.InSecondsF() * - audio_parameters_.sample_rate()); + const int frames_requested = audio_bus->frames(); + DVLOG(4) << __FUNCTION__ << " frames_delayed:" << frames_delayed + << " frames_skipped:" << frames_skipped + << " frames_requested:" << frames_requested; + int frames_written = 0; { base::AutoLock auto_lock(lock_); @@ -648,27 +648,27 @@ int AudioRendererImpl::Render(AudioBus* audio_bus, if (!stop_rendering_time_.is_null()) { audio_clock_->CompensateForSuspendedWrites( - last_render_time_ - stop_rendering_time_, delay_frames); + last_render_time_ - stop_rendering_time_, frames_delayed); stop_rendering_time_ = base::TimeTicks(); } // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread. if (!algorithm_) { - audio_clock_->WroteAudio( - 0, requested_frames, delay_frames, playback_rate_); + audio_clock_->WroteAudio(0, frames_requested, frames_delayed, + playback_rate_); return 0; } if (playback_rate_ == 0) { - audio_clock_->WroteAudio( - 0, requested_frames, delay_frames, playback_rate_); + audio_clock_->WroteAudio(0, frames_requested, frames_delayed, + playback_rate_); return 0; } // Mute audio by returning 0 when not playing. if (state_ != kPlaying) { - audio_clock_->WroteAudio( - 0, requested_frames, delay_frames, playback_rate_); + audio_clock_->WroteAudio(0, frames_requested, frames_delayed, + playback_rate_); return 0; } @@ -688,15 +688,15 @@ int AudioRendererImpl::Render(AudioBus* audio_bus, frames_written = std::min(static_cast<int>(play_delay.InSecondsF() * audio_parameters_.sample_rate()), - requested_frames); + frames_requested); audio_bus->ZeroFramesPartial(0, frames_written); } // If there's any space left, actually render the audio; this is where the // aural magic happens. - if (frames_written < requested_frames) { + if (frames_written < frames_requested) { frames_written += algorithm_->FillBuffer( - audio_bus, frames_written, requested_frames - frames_written, + audio_bus, frames_written, frames_requested - frames_written, playback_rate_); } } @@ -727,7 +727,7 @@ int AudioRendererImpl::Render(AudioBus* audio_bus, if (received_end_of_stream_) { if (ended_timestamp_ == kInfiniteDuration()) ended_timestamp_ = audio_clock_->back_timestamp(); - frames_after_end_of_stream = requested_frames; + frames_after_end_of_stream = frames_requested; } else if (state_ == kPlaying && buffering_state_ != BUFFERING_HAVE_NOTHING) { algorithm_->IncreaseQueueCapacity(); @@ -736,9 +736,7 @@ int AudioRendererImpl::Render(AudioBus* audio_bus, } audio_clock_->WroteAudio(frames_written + frames_after_end_of_stream, - requested_frames, - delay_frames, - playback_rate_); + frames_requested, frames_delayed, playback_rate_); if (CanRead_Locked()) { task_runner_->PostTask(FROM_HERE, @@ -753,7 +751,7 @@ int AudioRendererImpl::Render(AudioBus* audio_bus, } } - DCHECK_LE(frames_written, requested_frames); + DCHECK_LE(frames_written, frames_requested); return frames_written; } diff --git a/media/renderers/audio_renderer_impl.h b/media/renderers/audio_renderer_impl.h index 450018c..63a6579 100644 --- a/media/renderers/audio_renderer_impl.h +++ b/media/renderers/audio_renderer_impl.h @@ -155,7 +155,7 @@ class MEDIA_EXPORT AudioRendererImpl // this case |audio_delay_milliseconds| should be used to indicate when in the // future should the filled buffer be played. int Render(AudioBus* audio_bus, - uint32_t audio_delay_milliseconds, + uint32_t frames_delayed, uint32_t frames_skipped) override; void OnRenderError() override; diff --git a/media/renderers/audio_renderer_impl_unittest.cc b/media/renderers/audio_renderer_impl_unittest.cc index 9440664..1f7c2cc 100644 --- a/media/renderers/audio_renderer_impl_unittest.cc +++ b/media/renderers/audio_renderer_impl_unittest.cc @@ -56,6 +56,9 @@ static int kChannels = ChannelLayoutToChannelCount(kChannelLayout); // Use a different output sample rate so the AudioBufferConverter is invoked. static int kInputSamplesPerSecond = 5000; static int kOutputSamplesPerSecond = 10000; +static double kOutputMicrosPerFrame = + static_cast<double>(base::Time::kMicrosecondsPerSecond) / + kOutputSamplesPerSecond; ACTION_P(EnterPendingDecoderInitStateAction, test) { test->EnterPendingDecoderInitState(arg2); @@ -298,16 +301,16 @@ class AudioRendererImplTest : public ::testing::Test { // buffer. Returns true if and only if all of |requested_frames| were able // to be consumed. bool ConsumeBufferedData(OutputFrames requested_frames, - base::TimeDelta delay) { + uint32_t delay_frames) { scoped_ptr<AudioBus> bus = AudioBus::Create(kChannels, requested_frames.value); int frames_read = 0; - EXPECT_TRUE(sink_->Render(bus.get(), delay.InMilliseconds(), &frames_read)); + EXPECT_TRUE(sink_->Render(bus.get(), delay_frames, &frames_read)); return frames_read == requested_frames.value; } bool ConsumeBufferedData(OutputFrames requested_frames) { - return ConsumeBufferedData(requested_frames, base::TimeDelta()); + return ConsumeBufferedData(requested_frames, 0); } base::TimeTicks ConvertMediaTime(base::TimeDelta timestamp, @@ -870,9 +873,12 @@ TEST_F(AudioRendererImplTest, TimeSourceBehavior) { EXPECT_FALSE(is_time_moving); // Consume some buffered data with a small delay. - base::TimeDelta delay_time = base::TimeDelta::FromMilliseconds(50); + uint32_t delay_frames = 500; + base::TimeDelta delay_time = base::TimeDelta::FromMicroseconds( + std::round(delay_frames * kOutputMicrosPerFrame)); + frames_to_consume.value = frames_buffered().value / 16; - EXPECT_TRUE(ConsumeBufferedData(frames_to_consume, delay_time)); + EXPECT_TRUE(ConsumeBufferedData(frames_to_consume, delay_frames)); // Verify time is adjusted for the current delay. current_time = tick_clock_->NowTicks() + delay_time; |