diff options
Diffstat (limited to 'content')
11 files changed, 310 insertions, 99 deletions
diff --git a/content/browser/renderer_host/media/audio_input_renderer_host.cc b/content/browser/renderer_host/media/audio_input_renderer_host.cc index 85d269f..121a5ac 100644 --- a/content/browser/renderer_host/media/audio_input_renderer_host.cc +++ b/content/browser/renderer_host/media/audio_input_renderer_host.cc @@ -174,7 +174,6 @@ bool AudioInputRendererHost::OnMessageReceived(const IPC::Message& message, IPC_MESSAGE_HANDLER(AudioInputHostMsg_CreateStream, OnCreateStream) IPC_MESSAGE_HANDLER(AudioInputHostMsg_RecordStream, OnRecordStream) IPC_MESSAGE_HANDLER(AudioInputHostMsg_CloseStream, OnCloseStream) - IPC_MESSAGE_HANDLER(AudioInputHostMsg_GetVolume, OnGetVolume) IPC_MESSAGE_HANDLER(AudioInputHostMsg_SetVolume, OnSetVolume) IPC_MESSAGE_UNHANDLED(handled = false) IPC_END_MESSAGE_MAP_EX() @@ -200,7 +199,8 @@ void AudioInputRendererHost::OnStartDevice(int stream_id, int session_id) { void AudioInputRendererHost::OnCreateStream(int stream_id, const AudioParameters& params, - const std::string& device_id) { + const std::string& device_id, + bool automatic_gain_control) { VLOG(1) << "AudioInputRendererHost::OnCreateStream(stream_id=" << stream_id << ")"; DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); @@ -214,9 +214,11 @@ void AudioInputRendererHost::OnCreateStream(int stream_id, // Create a new AudioEntry structure. scoped_ptr<AudioEntry> entry(new AudioEntry()); + uint32 mem_size = sizeof(AudioInputBufferParameters) + buffer_size; + // Create the shared memory and share it with the renderer process // using a new SyncWriter object. - if (!entry->shared_memory.CreateAndMapAnonymous(buffer_size)) { + if (!entry->shared_memory.CreateAndMapAnonymous(mem_size)) { // If creation of shared memory failed then send an error message. SendErrorMessage(stream_id); return; @@ -248,6 +250,9 @@ void AudioInputRendererHost::OnCreateStream(int stream_id, return; } + // Set the initial AGC state for the audio input stream. + entry->controller->SetAutomaticGainControl(automatic_gain_control); + // If we have created the controller successfully create a entry and add it // to the map. entry->stream_id = stream_id; @@ -290,21 +295,7 @@ void AudioInputRendererHost::OnSetVolume(int stream_id, double volume) { return; } - // TODO(henrika): TBI. - NOTIMPLEMENTED(); -} - -void AudioInputRendererHost::OnGetVolume(int stream_id) { - DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); - - AudioEntry* entry = LookupById(stream_id); - if (!entry) { - SendErrorMessage(stream_id); - return; - } - - // TODO(henrika): TBI. - NOTIMPLEMENTED(); + entry->controller->SetVolume(volume); } void AudioInputRendererHost::SendErrorMessage(int stream_id) { diff --git a/content/browser/renderer_host/media/audio_input_renderer_host.h b/content/browser/renderer_host/media/audio_input_renderer_host.h index 6d2a8b8..5a9cc6b 100644 --- a/content/browser/renderer_host/media/audio_input_renderer_host.h +++ b/content/browser/renderer_host/media/audio_input_renderer_host.h @@ -140,7 +140,8 @@ class CONTENT_EXPORT AudioInputRendererHost // required properties. void OnCreateStream(int stream_id, const AudioParameters& params, - const std::string& device_id); + const std::string& device_id, + bool automatic_gain_control); // Record the audio input stream referenced by |stream_id|. void OnRecordStream(int stream_id); @@ -151,9 +152,6 @@ class CONTENT_EXPORT AudioInputRendererHost // Set the volume of the audio stream referenced by |stream_id|. void OnSetVolume(int stream_id, double volume); - // Get the volume of the audio stream referenced by |stream_id|. - void OnGetVolume(int stream_id); - // Complete the process of creating an audio input stream. This will set up // the shared memory or shared socket in low latency mode. void DoCompleteCreation(media::AudioInputController* controller); diff --git a/content/browser/renderer_host/media/audio_input_sync_writer.cc b/content/browser/renderer_host/media/audio_input_sync_writer.cc index 9524ad0..d720017 100644 --- a/content/browser/renderer_host/media/audio_input_sync_writer.cc +++ b/content/browser/renderer_host/media/audio_input_sync_writer.cc @@ -15,15 +15,20 @@ AudioInputSyncWriter::AudioInputSyncWriter(base::SharedMemory* shared_memory) AudioInputSyncWriter::~AudioInputSyncWriter() {} +// TODO(henrika): Combine into one method (including Write). void AudioInputSyncWriter::UpdateRecordedBytes(uint32 bytes) { socket_->Send(&bytes, sizeof(bytes)); } -uint32 AudioInputSyncWriter::Write(const void* data, uint32 size) { - uint32 write_size = std::min(size, shared_memory_->created_size()); - // Copy audio input samples from recorded data to shared memory. - memcpy(shared_memory_->memory(), data, write_size); - return write_size; +uint32 AudioInputSyncWriter::Write(const void* data, uint32 size, + double volume) { + AudioInputBuffer* buffer = + reinterpret_cast<AudioInputBuffer*>(shared_memory_->memory()); + buffer->params.volume = volume; + buffer->params.size = size; + memcpy(buffer->audio, data, size); + + return size; } void AudioInputSyncWriter::Close() { diff --git a/content/browser/renderer_host/media/audio_input_sync_writer.h b/content/browser/renderer_host/media/audio_input_sync_writer.h index 3114c35..77acd50 100644 --- a/content/browser/renderer_host/media/audio_input_sync_writer.h +++ b/content/browser/renderer_host/media/audio_input_sync_writer.h @@ -27,7 +27,7 @@ class AudioInputSyncWriter : public media::AudioInputController::SyncWriter { // media::AudioOutputController::SyncWriter implementation. virtual void UpdateRecordedBytes(uint32 bytes) OVERRIDE; - virtual uint32 Write(const void* data, uint32 size) OVERRIDE; + virtual uint32 Write(const void* data, uint32 size, double volume) OVERRIDE; virtual void Close() OVERRIDE; bool Init(); diff --git a/content/common/media/audio_messages.h b/content/common/media/audio_messages.h index bac2537..0acb35c 100644 --- a/content/common/media/audio_messages.h +++ b/content/common/media/audio_messages.h @@ -5,6 +5,8 @@ // IPC messages for the audio. // Multiply-included message file, hence no include guard. +#include <string> + #include "base/basictypes.h" #include "base/shared_memory.h" #include "base/sync_socket.h" @@ -91,10 +93,11 @@ IPC_MESSAGE_CONTROL2(AudioHostMsg_CreateStream, AudioParameters /* params */) // Request that got sent to browser for creating an audio input stream -IPC_MESSAGE_CONTROL3(AudioInputHostMsg_CreateStream, +IPC_MESSAGE_CONTROL4(AudioInputHostMsg_CreateStream, int /* stream_id */, AudioParameters /* params */, - std::string /* device_id */) + std::string /* device_id */, + bool /* automatic_gain_control */) // Start buffering and play the audio stream specified by stream_id. IPC_MESSAGE_CONTROL1(AudioHostMsg_PlayStream, @@ -120,11 +123,6 @@ IPC_MESSAGE_CONTROL1(AudioHostMsg_CloseStream, IPC_MESSAGE_CONTROL1(AudioInputHostMsg_CloseStream, int /* stream_id */) -// Get audio volume of the input stream specified by -// (render_view_id, stream_id). -IPC_MESSAGE_CONTROL1(AudioInputHostMsg_GetVolume, - int /* stream_id */) - // Set audio volume of the stream specified by stream_id. // TODO(hclam): change this to vector if we have channel numbers other than 2. IPC_MESSAGE_CONTROL2(AudioHostMsg_SetVolume, diff --git a/content/renderer/media/audio_input_device.cc b/content/renderer/media/audio_input_device.cc index 24e9da9..11a3938 100644 --- a/content/renderer/media/audio_input_device.cc +++ b/content/renderer/media/audio_input_device.cc @@ -47,7 +47,8 @@ AudioInputDevice::AudioInputDevice(const AudioParameters& params, volume_(1.0), stream_id_(0), session_id_(0), - pending_device_ready_(false) { + pending_device_ready_(false), + agc_is_enabled_(false) { filter_ = RenderThreadImpl::current()->audio_input_message_filter(); } @@ -82,15 +83,27 @@ void AudioInputDevice::Stop() { } bool AudioInputDevice::SetVolume(double volume) { - NOTIMPLEMENTED(); - return false; + if (volume < 0 || volume > 1.0) + return false; + + message_loop()->PostTask(FROM_HERE, + base::Bind(&AudioInputDevice::SetVolumeOnIOThread, this, volume)); + + return true; } bool AudioInputDevice::GetVolume(double* volume) { - NOTIMPLEMENTED(); + NOTREACHED(); return false; } +void AudioInputDevice::SetAutomaticGainControl(bool enabled) { + DVLOG(1) << "SetAutomaticGainControl(enabled=" << enabled << ")"; + message_loop()->PostTask(FROM_HERE, + base::Bind(&AudioInputDevice::SetAutomaticGainControlOnIOThread, + this, enabled)); +} + void AudioInputDevice::InitializeOnIOThread() { DCHECK(message_loop()->BelongsToCurrentThread()); // Make sure we don't call Start() more than once. @@ -104,7 +117,8 @@ void AudioInputDevice::InitializeOnIOThread() { // and create the stream when getting a OnDeviceReady() callback. if (!session_id_) { Send(new AudioInputHostMsg_CreateStream( - stream_id_, audio_parameters_, AudioManagerBase::kDefaultDeviceId)); + stream_id_, audio_parameters_, AudioManagerBase::kDefaultDeviceId, + agc_is_enabled_)); } else { Send(new AudioInputHostMsg_StartDevice(stream_id_, session_id_)); pending_device_ready_ = true; @@ -133,6 +147,7 @@ void AudioInputDevice::ShutDownOnIOThread() { stream_id_ = 0; session_id_ = 0; pending_device_ready_ = false; + agc_is_enabled_ = false; } // We can run into an issue where ShutDownOnIOThread is called right after @@ -153,6 +168,18 @@ void AudioInputDevice::SetVolumeOnIOThread(double volume) { Send(new AudioInputHostMsg_SetVolume(stream_id_, volume)); } +void AudioInputDevice::SetAutomaticGainControlOnIOThread(bool enabled) { + DCHECK(message_loop()->BelongsToCurrentThread()); + DCHECK_EQ(0, stream_id_) << + "The AGC state can not be modified while capturing is active."; + if (stream_id_) + return; + + // We simply store the new AGC setting here. This value will be used when + // a new stream is initialized and by GetAutomaticGainControl(). + agc_is_enabled_ = enabled; +} + void AudioInputDevice::OnStreamCreated( base::SharedMemoryHandle handle, base::SyncSocket::Handle socket_handle, @@ -250,7 +277,7 @@ void AudioInputDevice::OnDeviceReady(const std::string& device_id) { stream_id_ = 0; } else { Send(new AudioInputHostMsg_CreateStream(stream_id_, audio_parameters_, - device_id)); + device_id, agc_is_enabled_)); } pending_device_ready_ = false; @@ -286,8 +313,17 @@ void AudioInputDevice::AudioThreadCallback::MapSharedMemory() { } void AudioInputDevice::AudioThreadCallback::Process(int pending_data) { + // The shared memory represents parameters, size of the data buffer and the + // actual data buffer containing audio data. Map the memory into this + // structure and parse out parameters and the data area. + AudioInputBuffer* buffer = + reinterpret_cast<AudioInputBuffer*>(shared_memory_.memory()); + uint32 size = buffer->params.size; + DCHECK_EQ(size, memory_length_ - sizeof(AudioInputBufferParameters)); + double volume = buffer->params.volume; + int audio_delay_milliseconds = pending_data / bytes_per_ms_; - int16* memory = reinterpret_cast<int16*>(shared_memory_.memory()); + int16* memory = reinterpret_cast<int16*>(&buffer->audio[0]); const size_t number_of_frames = audio_parameters_.frames_per_buffer(); const int bytes_per_sample = sizeof(memory[0]); @@ -306,5 +342,5 @@ void AudioInputDevice::AudioThreadCallback::Process(int pending_data) { // Deliver captured data to the client in floating point format // and update the audio-delay measurement. capture_callback_->Capture(audio_data_, number_of_frames, - audio_delay_milliseconds); + audio_delay_milliseconds, volume); } diff --git a/content/renderer/media/audio_input_device.h b/content/renderer/media/audio_input_device.h index a91ee16..99cd03c 100644 --- a/content/renderer/media/audio_input_device.h +++ b/content/renderer/media/audio_input_device.h @@ -69,6 +69,7 @@ #define CONTENT_RENDERER_MEDIA_AUDIO_INPUT_DEVICE_H_ #pragma once +#include <string> #include <vector> #include "base/basictypes.h" @@ -95,7 +96,8 @@ class CONTENT_EXPORT AudioInputDevice public: virtual void Capture(const std::vector<float*>& audio_data, size_t number_of_frames, - size_t audio_delay_milliseconds) = 0; + size_t audio_delay_milliseconds, + double volume) = 0; virtual void OnCaptureError() = 0; protected: virtual ~CaptureCallback() {} @@ -149,8 +151,13 @@ class CONTENT_EXPORT AudioInputDevice return audio_parameters_.frames_per_buffer(); } + // Sets the Automatic Gain Control state to on or off. + // This method must be called before Start(). It will not have any effect + // if it is called while capturing has already started. + void SetAutomaticGainControl(bool enabled); + // Methods called on IO thread ---------------------------------------------- - // AudioInputMessageFilter::Delegate impl., called by AudioInputMessageFilter + // AudioInputMessageFilter::Delegate impl., called by AudioInputMessageFilter. virtual void OnStreamCreated(base::SharedMemoryHandle handle, base::SyncSocket::Handle socket_handle, uint32 length) OVERRIDE; @@ -168,6 +175,7 @@ class CONTENT_EXPORT AudioInputDevice void StartOnIOThread(); void ShutDownOnIOThread(); void SetVolumeOnIOThread(double volume); + void SetAutomaticGainControlOnIOThread(bool enabled); void Send(IPC::Message* message); @@ -198,6 +206,10 @@ class CONTENT_EXPORT AudioInputDevice // callback. Only modified on the IO thread. bool pending_device_ready_; + // Stores the Automatic Gain Control state. Default is false. + // Only modified on the IO thread. + bool agc_is_enabled_; + // Our audio thread callback class. See source file for details. class AudioThreadCallback; diff --git a/content/renderer/media/webrtc_audio_device_impl.cc b/content/renderer/media/webrtc_audio_device_impl.cc index fefb10e..a25bdca 100644 --- a/content/renderer/media/webrtc_audio_device_impl.cc +++ b/content/renderer/media/webrtc_audio_device_impl.cc @@ -12,6 +12,7 @@ #include "media/audio/audio_util.h" static const int64 kMillisecondsBetweenProcessCalls = 5000; +static const double kMaxVolumeLevel = 255.0; // Supported hardware sample rates for input and output sides. #if defined(OS_WIN) || defined(OS_MACOSX) @@ -40,7 +41,8 @@ WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl() bytes_per_sample_(0), initialized_(false), playing_(false), - recording_(false) { + recording_(false), + agc_is_enabled_(false) { DVLOG(1) << "WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()"; DCHECK(RenderThreadImpl::current()) << "WebRtcAudioDeviceImpl must be constructed on the render thread"; @@ -132,11 +134,21 @@ void WebRtcAudioDeviceImpl::OnRenderError() { LOG(ERROR) << "OnRenderError()"; } -void WebRtcAudioDeviceImpl::Capture( - const std::vector<float*>& audio_data, - size_t number_of_frames, - size_t audio_delay_milliseconds) { +void WebRtcAudioDeviceImpl::Capture(const std::vector<float*>& audio_data, + size_t number_of_frames, + size_t audio_delay_milliseconds, + double volume) { DCHECK_LE(number_of_frames, input_buffer_size()); +#if defined(OS_WIN) || defined(OS_MACOSX) + DCHECK_LE(volume, 1.0); +#elif defined(OS_LINUX) || defined(OS_OPENBSD) + // We have a special situation on Linux where the microphone volume can be + // "higher than maximum". The input volume slider in the sound preference + // allows the user to set a scaling that is higher than 100%. It means that + // even if the reported maximum levels is N, the actual microphone level can + // go up to 1.5*N and that corresponds to a normalized |volume| of 1.5. + DCHECK_LE(volume, 1.5); +#endif int output_delay_ms = 0; { @@ -165,15 +177,17 @@ void WebRtcAudioDeviceImpl::Capture( const int bytes_per_10_msec = channels * samples_per_10_msec * bytes_per_sample_; size_t accumulated_audio_samples = 0; - char* audio_byte_buffer = reinterpret_cast<char*>(input_buffer_.get()); + // Map internal volume range of [0.0, 1.0] into [0, 255] used by the + // webrtc::VoiceEngine. + uint32_t current_mic_level = static_cast<uint32_t>(volume * kMaxVolumeLevel); + // Write audio samples in blocks of 10 milliseconds to the registered // webrtc::AudioTransport sink. Keep writing until our internal byte // buffer is empty. while (accumulated_audio_samples < number_of_frames) { - // Deliver 10ms of recorded PCM audio. - // TODO(henrika): add support for analog AGC? + // Deliver 10ms of recorded 16-bit linear PCM audio. audio_transport_callback_->RecordedDataIsAvailable( audio_byte_buffer, samples_per_10_msec, @@ -181,12 +195,24 @@ void WebRtcAudioDeviceImpl::Capture( channels, samples_per_sec, input_delay_ms_ + output_delay_ms, - 0, // clock_drift - 0, // current_mic_level - new_mic_level); // not used + 0, // TODO(henrika): |clock_drift| parameter is not utilized today. + current_mic_level, + new_mic_level); + accumulated_audio_samples += samples_per_10_msec; audio_byte_buffer += bytes_per_10_msec; } + + // The AGC returns a non-zero microphone level if it has been decided + // that a new level should be set. + if (new_mic_level != 0) { + // Use IPC and set the new level. Note that, it will take some time + // before the new level is effective due to the IPC scheme. + // During this time, |current_mic_level| will contain "non-valid" values + // and it might reduce the AGC performance. Measurements on Windows 7 have + // shown that we might receive old volume levels for one or two callbacks. + SetMicrophoneVolume(new_mic_level); + } } void WebRtcAudioDeviceImpl::OnCaptureError() { @@ -331,8 +357,8 @@ int32_t WebRtcAudioDeviceImpl::Init() { ChannelLayout out_channel_layout = CHANNEL_LAYOUT_MONO; AudioParameters::Format in_format = AudioParameters::AUDIO_PCM_LINEAR; - size_t in_buffer_size = 0; - size_t out_buffer_size = 0; + int in_buffer_size = 0; + int out_buffer_size = 0; // TODO(henrika): factor out all platform specific parts in separate // functions. Code is a bit messy right now. @@ -357,6 +383,8 @@ int32_t WebRtcAudioDeviceImpl::Init() { in_buffer_size = 440; } else { in_buffer_size = (in_sample_rate / 100); + DCHECK_EQ(in_buffer_size * 100, in_sample_rate) << + "Sample rate not supported. Should have been caught in Init()."; } // Render side: AUDIO_PCM_LOW_LATENCY is based on the Core Audio (WASAPI) @@ -401,6 +429,8 @@ int32_t WebRtcAudioDeviceImpl::Init() { in_buffer_size = 440; } else { in_buffer_size = (in_sample_rate / 100); + DCHECK_EQ(in_buffer_size * 100, in_sample_rate) << + "Sample rate not supported. Should have been caught in Init()."; } // Render side: AUDIO_PCM_LOW_LATENCY on Mac OS X is based on a callback- @@ -594,8 +624,8 @@ bool WebRtcAudioDeviceImpl::RecordingIsInitialized() const { int32_t WebRtcAudioDeviceImpl::StartPlayout() { DVLOG(1) << "StartPlayout()"; + LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing"; if (!audio_transport_callback_) { - LOG(ERROR) << "Audio transport is missing"; return -1; } if (playing_) { @@ -627,7 +657,6 @@ int32_t WebRtcAudioDeviceImpl::StartRecording() { DVLOG(1) << "StartRecording()"; LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing"; if (!audio_transport_callback_) { - LOG(ERROR) << "Audio transport is missing"; return -1; } @@ -675,13 +704,25 @@ bool WebRtcAudioDeviceImpl::Recording() const { } int32_t WebRtcAudioDeviceImpl::SetAGC(bool enable) { - DVLOG(2) << "WARNING: WebRtcAudioDeviceImpl::SetAGC() " << "NOT IMPLEMENTED"; - return -1; + DVLOG(1) << "SetAGC(enable=" << enable << ")"; + // The current implementation does not support changing the AGC state while + // recording. Using this approach simplifies the design and it is also + // inline with the latest WebRTC standard. + DCHECK(initialized_); + DCHECK(!recording_) << "Unable to set AGC state while recording is active."; + if (recording_) { + return -1; + } + + audio_input_device_->SetAutomaticGainControl(enable); + agc_is_enabled_ = enable; + return 0; } bool WebRtcAudioDeviceImpl::AGC() const { - DVLOG(2) << "WARNING: WebRtcAudioDeviceImpl::AGC() " << "NOT IMPLEMENTED"; - return false; + // To reduce the usage of IPC messages, an internal AGC state is used. + // TODO(henrika): investigate if there is a need for a "deeper" getter. + return agc_is_enabled_; } int32_t WebRtcAudioDeviceImpl::SetWaveOutVolume(uint16_t volume_left, @@ -754,8 +795,7 @@ int32_t WebRtcAudioDeviceImpl::MaxSpeakerVolume(uint32_t* max_volume) const { return -1; } -int32_t WebRtcAudioDeviceImpl::MinSpeakerVolume( - uint32_t* min_volume) const { +int32_t WebRtcAudioDeviceImpl::MinSpeakerVolume(uint32_t* min_volume) const { NOTIMPLEMENTED(); return -1; } @@ -772,32 +812,39 @@ int32_t WebRtcAudioDeviceImpl::MicrophoneVolumeIsAvailable(bool* available) { } int32_t WebRtcAudioDeviceImpl::SetMicrophoneVolume(uint32_t volume) { - NOTIMPLEMENTED(); - return -1; + DVLOG(1) << "SetMicrophoneVolume(" << volume << ")"; + if (volume > kMaxVolumeLevel) + return -1; + + // WebRTC uses a range of [0, 255] to represent the level of the microphone + // volume. The IPC channel between the renderer and browser process works + // with doubles in the [0.0, 1.0] range and we have to compensate for that. + double normalized_volume = static_cast<double>(volume / kMaxVolumeLevel); + audio_input_device_->SetVolume(normalized_volume); + return 0; } int32_t WebRtcAudioDeviceImpl::MicrophoneVolume(uint32_t* volume) const { - NOTIMPLEMENTED(); + // The microphone level is fed to this class using the Capture() callback + // and this external API should not be used. Additional IPC messages are + // required if support for this API is ever needed. + NOTREACHED(); return -1; } -int32_t WebRtcAudioDeviceImpl::MaxMicrophoneVolume( - uint32_t* max_volume) const { - DVLOG(2) << "WARNING: WebRtcAudioDeviceImpl::MaxMicrophoneVolume() " - << "NOT IMPLEMENTED"; - return -1; +int32_t WebRtcAudioDeviceImpl::MaxMicrophoneVolume(uint32_t* max_volume) const { + *max_volume = kMaxVolumeLevel; + return 0; } -int32_t WebRtcAudioDeviceImpl::MinMicrophoneVolume( - uint32_t* min_volume) const { - DVLOG(2) << "WARNING: WebRtcAudioDeviceImpl::MinMicrophoneVolume() " - << "NOT IMPLEMENTED"; - return -1; +int32_t WebRtcAudioDeviceImpl::MinMicrophoneVolume(uint32_t* min_volume) const { + *min_volume = 0; + return 0; } int32_t WebRtcAudioDeviceImpl::MicrophoneVolumeStepSize( uint16_t* step_size) const { - NOTIMPLEMENTED(); + NOTREACHED(); return -1; } diff --git a/content/renderer/media/webrtc_audio_device_impl.h b/content/renderer/media/webrtc_audio_device_impl.h index b3f3534..c00e129 100644 --- a/content/renderer/media/webrtc_audio_device_impl.h +++ b/content/renderer/media/webrtc_audio_device_impl.h @@ -49,7 +49,7 @@ // base->StartPlayout(ch); // base->StartSending(ch); // ... -// <== full-duplex audio session ==> +// <== full-duplex audio session with AGC enabled ==> // ... // base->DeleteChannel(ch); // base->Terminate(); @@ -57,15 +57,30 @@ // VoiceEngine::Delete(voe); // } // -// Note that, WebRtcAudioDeviceImpl::RegisterAudioCallback() will -// be called by the webrtc::VoiceEngine::Init() call and the -// webrtc::VoiceEngine is an webrtc::AudioTransport implementation. -// Hence, when the underlying audio layer wants data samples to be played out, -// the AudioDevice::RenderCallback() will be called, which in turn uses the -// registered webrtc::AudioTransport callback and feeds the data to the -// webrtc::VoiceEngine. +// webrtc::VoiceEngine::Init() calls these ADM methods (in this order): // -// The picture below illustrates the media flow on the capture side: +// RegisterAudioCallback(this) +// webrtc::VoiceEngine is an webrtc::AudioTransport implementation and +// implements the RecordedDataIsAvailable() and NeedMorePlayData() callbacks. +// +// Init() +// Creates and initializes the AudioDevice and AudioInputDevice objects. +// +// SetAGC(true) +// Enables the adaptive analog mode of the AGC which ensures that a +// suitable microphone volume level will be set. This scheme will affect +// the actual microphone control slider. +// +// Media example: +// +// When the underlying audio layer wants data samples to be played out, the +// AudioDevice::RenderCallback() will be called, which in turn uses the +// registered webrtc::AudioTransport callback and gets the data to be played +// out from the webrtc::VoiceEngine. +// +// The picture below illustrates the media flow on the capture side where the +// AudioInputDevice client acts as link between the renderer and browser +// process: // // .------------------. .----------------------. // (Native audio) => | AudioInputStream |-> OnData ->| AudioInputController |-. @@ -87,11 +102,106 @@ // The actual data is transferred via SharedMemory. IPC is not involved // in the actual media transfer. // +// AGC overview: +// +// It aims to maintain a constant speech loudness level from the microphone. +// This is done by both controlling the analog microphone gain and applying +// digital gain. The microphone gain on the sound card is slowly +// increased/decreased during speech only. By observing the microphone control +// slider you can see it move when you speak. If you scream, the slider moves +// downwards and then upwards again when you return to normal. It is not +// uncommon that the slider hits the maximum. This means that the maximum +// analog gain is not large enough to give the desired loudness. Nevertheless, +// we can in general still attain the desired loudness. If the microphone +// control slider is moved manually, the gain adaptation restarts and returns +// to roughly the same position as before the change if the circumstances are +// still the same. When the input microphone signal causes saturation, the +// level is decreased dramatically and has to re-adapt towards the old level. +// The adaptation is a slowly varying process and at the beginning of capture +// this is noticed by a slow increase in volume. Smaller changes in microphone +// input level is leveled out by the built-in digital control. For larger +// differences we need to rely on the slow adaptation. +// See http://en.wikipedia.org/wiki/Automatic_gain_control for more details. +// +// AGC implementation details: +// +// The adaptive analog mode of the AGC is always enabled for desktop platforms +// in WebRTC. +// +// Before recording starts, the ADM sets an AGC state in the +// AudioInputDevice by calling AudioInputDevice::SetAutomaticGainControl(true). +// +// A capture session with AGC is started up as follows (simplified): +// +// [renderer] +// | +// ADM::StartRecording() +// AudioInputDevice::InitializeOnIOThread() +// AudioInputHostMsg_CreateStream(..., agc=true) [IPC] +// | +// [IPC to the browser] +// | +// AudioInputRendererHost::OnCreateStream() +// AudioInputController::CreateLowLatency() +// AudioInputController::DoSetAutomaticGainControl(true) +// AudioInputStream::SetAutomaticGainControl(true) +// | +// AGC is now enabled in the media layer and streaming starts (details omitted). +// The figure below illustrates the AGC scheme which is active in combination +// with the default media flow explained earlier. +// | +// [browser] +// | +// AudioInputStream::(Capture thread loop) +// AudioInputStreamImpl::QueryAgcVolume() => new volume once per second +// AudioInputData::OnData(..., volume) +// AudioInputController::OnData(..., volume) +// AudioInputSyncWriter::Write(..., volume) +// | +// [volume | size | data] is sent to the renderer [shared memory] +// | +// [renderer] +// | +// AudioInputDevice::AudioThreadCallback::Process() +// WebRtcAudioDeviceImpl::Capture(..., volume) +// AudioTransport::RecordedDataIsAvailable(...,volume, new_volume) +// | +// The AGC now uses the current volume input and computes a suitable new +// level given by the |new_level| output. This value is only non-zero if the +// AGC has take a decision that the microphone level should change. +// | +// if (new_volume != 0) +// AudioInputDevice::SetVolume(new_volume) +// AudioInputHostMsg_SetVolume(new_volume) [IPC] +// | +// [IPC to the browser] +// | +// AudioInputRendererHost::OnSetVolume() +// AudioInputController::SetVolume() +// AudioInputStream::SetVolume(scaled_volume) +// | +// Here we set the new microphone level in the media layer and at the same time +// read the new setting (we might not get exactly what is set). +// | +// AudioInputData::OnData(..., updated_volume) +// AudioInputController::OnData(..., updated_volume) +// | +// | +// This process repeats until we stop capturing data. Note that, a common +// steady state is that the volume control reaches its max and the new_volume +// value from the AGC is zero. A loud voice input is required to break this +// state and start lowering the level again. +// // Implementation notes: // // - This class must be created on the main render thread. // - The webrtc::AudioDeviceModule is reference counted. -// - Recording is currently not supported on Mac OS X. +// - AGC is only supported in combination with the WASAPI-based audio layer +// on Windows, i.e., it is not supported on Windows XP. +// - All volume levels required for the AGC scheme are transfered in a +// normalized range [0.0, 1.0]. Scaling takes place in both endpoints +// (WebRTC client a media layer). This approach ensures that we can avoid +// transferring maximum levels between the renderer and the browser. // class CONTENT_EXPORT WebRtcAudioDeviceImpl : NON_EXPORTED_BASE(public webrtc::AudioDeviceModule), @@ -120,7 +230,8 @@ class CONTENT_EXPORT WebRtcAudioDeviceImpl // AudioInputDevice::CaptureCallback implementation. virtual void Capture(const std::vector<float*>& audio_data, size_t number_of_frames, - size_t audio_delay_milliseconds) OVERRIDE; + size_t audio_delay_milliseconds, + double volume) OVERRIDE; virtual void OnCaptureError() OVERRIDE; // AudioInputDevice::CaptureEventHandler implementation. @@ -336,6 +447,9 @@ class CONTENT_EXPORT WebRtcAudioDeviceImpl bool playing_; bool recording_; + // Local copy of the current Automatic Gain Control state. + bool agc_is_enabled_; + DISALLOW_COPY_AND_ASSIGN(WebRtcAudioDeviceImpl); }; diff --git a/content/renderer/media/webrtc_audio_device_unittest.cc b/content/renderer/media/webrtc_audio_device_unittest.cc index d9b291c..e9323f2 100644 --- a/content/renderer/media/webrtc_audio_device_unittest.cc +++ b/content/renderer/media/webrtc_audio_device_unittest.cc @@ -437,6 +437,7 @@ TEST_F(WebRTCAudioDeviceTest, PlayLocalFile) { EXPECT_EQ(0, base->StartPlayout(ch)); ScopedWebRTCPtr<webrtc::VoEFile> file(engine.get()); + ASSERT_TRUE(file.valid()); int duration = 0; EXPECT_EQ(0, file->GetFileDuration(file_path.c_str(), duration, webrtc::kFileFormatPcm16kHzFile)); @@ -465,8 +466,8 @@ TEST_F(WebRTCAudioDeviceTest, PlayLocalFile) { // where they are decoded and played out on the default audio output device. // Disabled when running headless since the bots don't have the required config. // TODO(henrika): improve quality by using a wideband codec, enabling noise- -// suppressions and perhaps also the digital AGC. -TEST_F(WebRTCAudioDeviceTest, FullDuplexAudio) { +// suppressions etc. +TEST_F(WebRTCAudioDeviceTest, FullDuplexAudioWithAGC) { if (IsRunningHeadless()) return; @@ -477,13 +478,13 @@ TEST_F(WebRTCAudioDeviceTest, FullDuplexAudio) { return; EXPECT_CALL(media_observer(), - OnSetAudioStreamStatus(_, 1, StrEq("created"))); + OnSetAudioStreamStatus(_, 1, StrEq("created"))); EXPECT_CALL(media_observer(), - OnSetAudioStreamPlaying(_, 1, true)); + OnSetAudioStreamPlaying(_, 1, true)); EXPECT_CALL(media_observer(), - OnSetAudioStreamStatus(_, 1, StrEq("closed"))); + OnSetAudioStreamStatus(_, 1, StrEq("closed"))); EXPECT_CALL(media_observer(), - OnDeleteAudioStream(_, 1)).Times(AnyNumber()); + OnDeleteAudioStream(_, 1)).Times(AnyNumber()); scoped_refptr<WebRtcAudioDeviceImpl> audio_device( new WebRtcAudioDeviceImpl()); @@ -496,10 +497,19 @@ TEST_F(WebRTCAudioDeviceTest, FullDuplexAudio) { int err = base->Init(audio_device); ASSERT_EQ(0, err); + ScopedWebRTCPtr<webrtc::VoEAudioProcessing> audio_processing(engine.get()); + ASSERT_TRUE(audio_processing.valid()); + bool enabled = false; + webrtc::AgcModes agc_mode = webrtc::kAgcDefault; + EXPECT_EQ(0, audio_processing->GetAgcStatus(enabled, agc_mode)); + EXPECT_TRUE(enabled); + EXPECT_EQ(agc_mode, webrtc::kAgcAdaptiveAnalog); + int ch = base->CreateChannel(); EXPECT_NE(-1, ch); ScopedWebRTCPtr<webrtc::VoENetwork> network(engine.get()); + ASSERT_TRUE(network.valid()); scoped_ptr<WebRTCTransportImpl> transport( new WebRTCTransportImpl(network.get())); EXPECT_EQ(0, network->RegisterExternalTransport(ch, *transport.get())); diff --git a/content/renderer/pepper/pepper_platform_audio_input_impl.cc b/content/renderer/pepper/pepper_platform_audio_input_impl.cc index f6c3244..6c42f75 100644 --- a/content/renderer/pepper/pepper_platform_audio_input_impl.cc +++ b/content/renderer/pepper/pepper_platform_audio_input_impl.cc @@ -127,7 +127,7 @@ void PepperPlatformAudioInputImpl::InitializeOnIOThread(int session_id) { if (!session_id) { // We will be notified by OnStreamCreated(). filter_->Send(new AudioInputHostMsg_CreateStream( - stream_id_, params_, AudioManagerBase::kDefaultDeviceId)); + stream_id_, params_, AudioManagerBase::kDefaultDeviceId, false)); } else { // We will be notified by OnDeviceReady(). filter_->Send(new AudioInputHostMsg_StartDevice(stream_id_, session_id)); @@ -229,7 +229,7 @@ void PepperPlatformAudioInputImpl::OnDeviceReady(const std::string& device_id) { } else { // We will be notified by OnStreamCreated(). filter_->Send(new AudioInputHostMsg_CreateStream(stream_id_, params_, - device_id)); + device_id, false)); } } |