diff options
| -rw-r--r-- | chrome/browser/chromeos/login/chrome_restart_request.cc | 3 | ||||
| -rw-r--r-- | content/browser/renderer_host/render_process_host_impl.cc | 1 | ||||
| -rw-r--r-- | content/content_renderer.gypi | 5 | ||||
| -rw-r--r-- | content/content_tests.gypi | 1 | ||||
| -rw-r--r-- | content/public/common/content_switches.cc | 10 | ||||
| -rw-r--r-- | content/public/common/content_switches.h | 1 | ||||
| -rw-r--r-- | content/renderer/media/media_stream_audio_processor.cc | 361 | ||||
| -rw-r--r-- | content/renderer/media/media_stream_audio_processor.h | 138 | ||||
| -rw-r--r-- | content/renderer/media/media_stream_audio_processor_options.cc | 96 | ||||
| -rw-r--r-- | content/renderer/media/media_stream_audio_processor_options.h | 53 | ||||
| -rw-r--r-- | content/renderer/media/media_stream_audio_processor_unittest.cc | 166 |
11 files changed, 831 insertions, 4 deletions
diff --git a/chrome/browser/chromeos/login/chrome_restart_request.cc b/chrome/browser/chromeos/login/chrome_restart_request.cc index 2742765..2f3258d 100644 --- a/chrome/browser/chromeos/login/chrome_restart_request.cc +++ b/chrome/browser/chromeos/login/chrome_restart_request.cc @@ -48,7 +48,7 @@ namespace chromeos { namespace { -// Increase logging level for Guest mode to avoid LOG(INFO) messages in logs. +// Increase logging level for Guest mode to avoid INFO messages in logs. const char kGuestModeLoggingLevel[] = "1"; // Format of command line switch. @@ -147,6 +147,7 @@ std::string DeriveCommandLine(const GURL& start_url, #if defined(ENABLE_WEBRTC) ::switches::kDisableWebRtcHWDecoding, ::switches::kDisableWebRtcHWEncoding, + ::switches::kEnableAudioTrackProcessing, ::switches::kEnableWebRtcHWVp8Encoding, #endif ash::switches::kAshDefaultWallpaperLarge, diff --git a/content/browser/renderer_host/render_process_host_impl.cc b/content/browser/renderer_host/render_process_host_impl.cc index c744eae..245af8c 100644 --- a/content/browser/renderer_host/render_process_host_impl.cc +++ b/content/browser/renderer_host/render_process_host_impl.cc @@ -1094,6 +1094,7 @@ void RenderProcessHostImpl::PropagateBrowserCommandLineToRenderer( switches::kDisablePepper3d, #endif #if defined(ENABLE_WEBRTC) + switches::kEnableAudioTrackProcessing, switches::kDisableDeviceEnumeration, switches::kDisableSCTPDataChannels, switches::kDisableWebRtcHWDecoding, diff --git a/content/content_renderer.gypi b/content/content_renderer.gypi index ac66122..5791a21 100644 --- a/content/content_renderer.gypi +++ b/content/content_renderer.gypi @@ -576,10 +576,15 @@ '../third_party/libjingle/libjingle.gyp:libjingle_webrtc', '../third_party/libjingle/libjingle.gyp:libpeerconnection', '../third_party/webrtc/modules/modules.gyp:audio_device', + '../third_party/webrtc/modules/modules.gyp:audio_processing', '<(DEPTH)/crypto/crypto.gyp:crypto', ], 'sources': [ 'public/renderer/webrtc_log_message_delegate.h', + 'renderer/media/media_stream_audio_processor.cc', + 'renderer/media/media_stream_audio_processor.h', + 'renderer/media/media_stream_audio_processor_options.cc', + 'renderer/media/media_stream_audio_processor_options.h', 'renderer/media/media_stream_center.cc', 'renderer/media/media_stream_dependency_factory.cc', 'renderer/media/media_stream_dispatcher.cc', diff --git a/content/content_tests.gypi b/content/content_tests.gypi index ec4fe06..f864519 100644 --- a/content/content_tests.gypi +++ b/content/content_tests.gypi @@ -681,6 +681,7 @@ 'browser/renderer_host/p2p/socket_host_tcp_unittest.cc', 'browser/renderer_host/p2p/socket_host_tcp_server_unittest.cc', 'browser/renderer_host/p2p/socket_host_udp_unittest.cc', + 'renderer/media/media_stream_audio_processor_unittest.cc', 'renderer/media/media_stream_dependency_factory_unittest.cc', 'renderer/media/media_stream_dispatcher_unittest.cc', 'renderer/media/media_stream_impl_unittest.cc', diff --git a/content/public/common/content_switches.cc b/content/public/common/content_switches.cc index ddd769f..aca77b0 100644 --- a/content/public/common/content_switches.cc +++ b/content/public/common/content_switches.cc @@ -941,10 +941,14 @@ const char kZygoteCmdPrefix[] = "zygote-cmd-prefix"; const char kZygoteProcess[] = "zygote"; #if defined(ENABLE_WEBRTC) -// Disable WebRTC device enumeration. +// Enables audio processing in a MediaStreamTrack. When this flag is on, AEC, +// NS and AGC will be done per MediaStreamTrack instead of in PeerConnection. +const char kEnableAudioTrackProcessing[] = "enable-audio-track-processing"; + +// Disables WebRTC device enumeration. const char kDisableDeviceEnumeration[] = "disable-device-enumeration"; -// Disable WebRTC DataChannels SCTP wire protocol support. +// Disables WebRTC DataChannels SCTP wire protocol support. const char kDisableSCTPDataChannels[] = "disable-sctp-data-channels"; // Disables HW decode acceleration for WebRTC. @@ -960,7 +964,7 @@ const char kDisableWebRtcHWEncoding[] = "disable-webrtc-hw-encoding"; // Enables WebRTC AEC recordings. const char kEnableWebRtcAecRecordings[] = "enable-webrtc-aec-recordings"; -// Enable WebRTC to open TCP server sockets. +// Enables WebRTC to open TCP server sockets. const char kEnableWebRtcTcpServerSocket[] = "enable-webrtc-tcp-server-socket"; // Enables VP8 HW encode acceleration for WebRTC. diff --git a/content/public/common/content_switches.h b/content/public/common/content_switches.h index ea4a74a..b294909 100644 --- a/content/public/common/content_switches.h +++ b/content/public/common/content_switches.h @@ -268,6 +268,7 @@ CONTENT_EXPORT extern const char kZygoteCmdPrefix[]; CONTENT_EXPORT extern const char kZygoteProcess[]; #if defined(ENABLE_WEBRTC) +CONTENT_EXPORT extern const char kEnableAudioTrackProcessing[]; CONTENT_EXPORT extern const char kDisableDeviceEnumeration[]; CONTENT_EXPORT extern const char kDisableSCTPDataChannels[]; CONTENT_EXPORT extern const char kDisableWebRtcHWDecoding[]; diff --git a/content/renderer/media/media_stream_audio_processor.cc b/content/renderer/media/media_stream_audio_processor.cc new file mode 100644 index 0000000..07974aa --- /dev/null +++ b/content/renderer/media/media_stream_audio_processor.cc @@ -0,0 +1,361 @@ +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/renderer/media/media_stream_audio_processor.h" + +#include "base/command_line.h" +#include "base/debug/trace_event.h" +#include "content/public/common/content_switches.h" +#include "content/renderer/media/media_stream_audio_processor_options.h" +#include "media/audio/audio_parameters.h" +#include "media/base/audio_converter.h" +#include "media/base/audio_fifo.h" +#include "media/base/channel_layout.h" + +namespace content { + +namespace { + +using webrtc::AudioProcessing; +using webrtc::MediaConstraintsInterface; + +#if defined(ANDROID) +const int kAudioProcessingSampleRate = 16000; +#else +const int kAudioProcessingSampleRate = 32000; +#endif +const int kAudioProcessingNumberOfChannel = 1; + +const int kMaxNumberOfBuffersInFifo = 2; + +} // namespace + +class MediaStreamAudioProcessor::MediaStreamAudioConverter + : public media::AudioConverter::InputCallback { + public: + MediaStreamAudioConverter(const media::AudioParameters& source_params, + const media::AudioParameters& sink_params) + : source_params_(source_params), + sink_params_(sink_params), + audio_converter_(source_params, sink_params_, false) { + audio_converter_.AddInput(this); + // Create and initialize audio fifo and audio bus wrapper. + // The size of the FIFO should be at least twice of the source buffer size + // or twice of the sink buffer size. + int buffer_size = std::max( + kMaxNumberOfBuffersInFifo * source_params_.frames_per_buffer(), + kMaxNumberOfBuffersInFifo * sink_params_.frames_per_buffer()); + fifo_.reset(new media::AudioFifo(source_params_.channels(), buffer_size)); + // TODO(xians): Use CreateWrapper to save one memcpy. + audio_wrapper_ = media::AudioBus::Create(sink_params_.channels(), + sink_params_.frames_per_buffer()); + } + + virtual ~MediaStreamAudioConverter() { + DCHECK(thread_checker_.CalledOnValidThread()); + audio_converter_.RemoveInput(this); + } + + void Push(media::AudioBus* audio_source) { + // Called on the audio thread, which is the capture audio thread for + // |MediaStreamAudioProcessor::capture_converter_|, and render audio thread + // for |MediaStreamAudioProcessor::render_converter_|. + // And it must be the same thread as calling Convert(). + DCHECK(thread_checker_.CalledOnValidThread()); + fifo_->Push(audio_source); + } + + bool Convert(webrtc::AudioFrame* out) { + // Called on the audio thread, which is the capture audio thread for + // |MediaStreamAudioProcessor::capture_converter_|, and render audio thread + // for |MediaStreamAudioProcessor::render_converter_|. + // Return false if there is no 10ms data in the FIFO. + DCHECK(thread_checker_.CalledOnValidThread()); + if (fifo_->frames() < (source_params_.sample_rate() / 100)) + return false; + + // Convert 10ms data to the output format, this will trigger ProvideInput(). + audio_converter_.Convert(audio_wrapper_.get()); + + // TODO(xians): Figure out a better way to handle the interleaved and + // deinterleaved format switching. + audio_wrapper_->ToInterleaved(audio_wrapper_->frames(), + sink_params_.bits_per_sample() / 8, + out->data_); + + out->samples_per_channel_ = sink_params_.frames_per_buffer(); + out->sample_rate_hz_ = sink_params_.sample_rate(); + out->speech_type_ = webrtc::AudioFrame::kNormalSpeech; + out->vad_activity_ = webrtc::AudioFrame::kVadUnknown; + out->num_channels_ = sink_params_.channels(); + + return true; + } + + const media::AudioParameters& source_parameters() const { + return source_params_; + } + const media::AudioParameters& sink_parameters() const { + return sink_params_; + } + + private: + // AudioConverter::InputCallback implementation. + virtual double ProvideInput(media::AudioBus* audio_bus, + base::TimeDelta buffer_delay) OVERRIDE { + // Called on realtime audio thread. + // TODO(xians): Figure out why the first Convert() triggers ProvideInput + // two times. + if (fifo_->frames() < audio_bus->frames()) + return 0; + + fifo_->Consume(audio_bus, 0, audio_bus->frames()); + + // Return 1.0 to indicate no volume scaling on the data. + return 1.0; + } + + base::ThreadChecker thread_checker_; + const media::AudioParameters source_params_; + const media::AudioParameters sink_params_; + + // TODO(xians): consider using SincResampler to save some memcpy. + // Handles mixing and resampling between input and output parameters. + media::AudioConverter audio_converter_; + scoped_ptr<media::AudioBus> audio_wrapper_; + scoped_ptr<media::AudioFifo> fifo_; +}; + +MediaStreamAudioProcessor::MediaStreamAudioProcessor( + const webrtc::MediaConstraintsInterface* constraints) + : render_delay_ms_(0) { + capture_thread_checker_.DetachFromThread(); + render_thread_checker_.DetachFromThread(); + InitializeAudioProcessingModule(constraints); +} + +MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { + DCHECK(main_thread_checker_.CalledOnValidThread()); + StopAudioProcessing(); +} + +void MediaStreamAudioProcessor::PushCaptureData(media::AudioBus* audio_source) { + DCHECK(capture_thread_checker_.CalledOnValidThread()); + capture_converter_->Push(audio_source); +} + +void MediaStreamAudioProcessor::PushRenderData( + const int16* render_audio, int sample_rate, int number_of_channels, + int number_of_frames, base::TimeDelta render_delay) { + DCHECK(render_thread_checker_.CalledOnValidThread()); + + // Return immediately if the echo cancellation is off. + if (!audio_processing_ || + !audio_processing_->echo_cancellation()->is_enabled()) { + return; + } + + TRACE_EVENT0("audio", + "MediaStreamAudioProcessor::FeedRenderDataToAudioProcessing"); + int64 new_render_delay_ms = render_delay.InMilliseconds(); + DCHECK_LT(new_render_delay_ms, + std::numeric_limits<base::subtle::Atomic32>::max()); + base::subtle::Release_Store(&render_delay_ms_, new_render_delay_ms); + + InitializeRenderConverterIfNeeded(sample_rate, number_of_channels, + number_of_frames); + + // TODO(xians): Avoid this extra interleave/deinterleave. + render_data_bus_->FromInterleaved(render_audio, + render_data_bus_->frames(), + sizeof(render_audio[0])); + render_converter_->Push(render_data_bus_.get()); + while (render_converter_->Convert(&render_frame_)) + audio_processing_->AnalyzeReverseStream(&render_frame_); +} + +bool MediaStreamAudioProcessor::ProcessAndConsumeData( + base::TimeDelta capture_delay, int volume, bool key_pressed, + int16** out) { + DCHECK(capture_thread_checker_.CalledOnValidThread()); + TRACE_EVENT0("audio", + "MediaStreamAudioProcessor::ProcessAndConsumeData"); + + if (!capture_converter_->Convert(&capture_frame_)) + return false; + + ProcessData(&capture_frame_, capture_delay, volume, key_pressed); + *out = capture_frame_.data_; + + return true; +} + +void MediaStreamAudioProcessor::SetCaptureFormat( + const media::AudioParameters& source_params) { + DCHECK(capture_thread_checker_.CalledOnValidThread()); + DCHECK(source_params.IsValid()); + + // Create and initialize audio converter for the source data. + // When the webrtc AudioProcessing is enabled, the sink format of the + // converter will be the same as the post-processed data format, which is + // 32k mono for desktops and 16k mono for Android. When the AudioProcessing + // is disabled, the sink format will be the same as the source format. + const int sink_sample_rate = audio_processing_ ? + kAudioProcessingSampleRate : source_params.sample_rate(); + const media::ChannelLayout sink_channel_layout = audio_processing_ ? + media::CHANNEL_LAYOUT_MONO : source_params.channel_layout(); + + // WebRtc is using 10ms data as its native packet size. + media::AudioParameters sink_params( + media::AudioParameters::AUDIO_PCM_LOW_LATENCY, sink_channel_layout, + sink_sample_rate, 16, sink_sample_rate / 100); + capture_converter_.reset( + new MediaStreamAudioConverter(source_params, sink_params)); +} + +const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { + return capture_converter_->sink_parameters(); +} + +void MediaStreamAudioProcessor::InitializeAudioProcessingModule( + const webrtc::MediaConstraintsInterface* constraints) { + DCHECK(!audio_processing_); + DCHECK(constraints); + if (!CommandLine::ForCurrentProcess()->HasSwitch( + switches::kEnableAudioTrackProcessing)) { + return; + } + + const bool enable_aec = GetPropertyFromConstraints( + constraints, MediaConstraintsInterface::kEchoCancellation); + const bool enable_ns = GetPropertyFromConstraints( + constraints, MediaConstraintsInterface::kNoiseSuppression); + const bool enable_high_pass_filter = GetPropertyFromConstraints( + constraints, MediaConstraintsInterface::kHighpassFilter); + const bool start_aec_dump = GetPropertyFromConstraints( + constraints, MediaConstraintsInterface::kInternalAecDump); +#if defined(IOS) || defined(ANDROID) + const bool enable_experimental_aec = false; + const bool enable_typing_detection = false; +#else + const bool enable_experimental_aec = GetPropertyFromConstraints( + constraints, MediaConstraintsInterface::kExperimentalEchoCancellation); + const bool enable_typing_detection = GetPropertyFromConstraints( + constraints, MediaConstraintsInterface::kTypingNoiseDetection); +#endif + + // Return immediately if no audio processing component is enabled. + if (!enable_aec && !enable_experimental_aec && !enable_ns && + !enable_high_pass_filter && !enable_typing_detection) { + return; + } + + // Create and configure the webrtc::AudioProcessing. + audio_processing_.reset(webrtc::AudioProcessing::Create(0)); + + // Enable the audio processing components. + if (enable_aec) { + EnableEchoCancellation(audio_processing_.get()); + if (enable_experimental_aec) + EnableExperimentalEchoCancellation(audio_processing_.get()); + } + + if (enable_ns) + EnableNoiseSuppression(audio_processing_.get()); + + if (enable_high_pass_filter) + EnableHighPassFilter(audio_processing_.get()); + + if (enable_typing_detection) + EnableTypingDetection(audio_processing_.get()); + + if (enable_aec && start_aec_dump) + StartAecDump(audio_processing_.get()); + + // Configure the audio format the audio processing is running on. This + // has to be done after all the needed components are enabled. + CHECK_EQ(audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate), + 0); + CHECK_EQ(audio_processing_->set_num_channels(kAudioProcessingNumberOfChannel, + kAudioProcessingNumberOfChannel), + 0); +} + +void MediaStreamAudioProcessor::InitializeRenderConverterIfNeeded( + int sample_rate, int number_of_channels, int frames_per_buffer) { + DCHECK(render_thread_checker_.CalledOnValidThread()); + // TODO(xians): Figure out if we need to handle the buffer size change. + if (render_converter_.get() && + render_converter_->source_parameters().sample_rate() == sample_rate && + render_converter_->source_parameters().channels() == number_of_channels) { + // Do nothing if the |render_converter_| has been setup properly. + return; + } + + // Create and initialize audio converter for the render data. + // webrtc::AudioProcessing accepts the same format as what it uses to process + // capture data, which is 32k mono for desktops and 16k mono for Android. + media::AudioParameters source_params( + media::AudioParameters::AUDIO_PCM_LOW_LATENCY, + media::GuessChannelLayout(number_of_channels), sample_rate, 16, + frames_per_buffer); + media::AudioParameters sink_params( + media::AudioParameters::AUDIO_PCM_LOW_LATENCY, + media::CHANNEL_LAYOUT_MONO, kAudioProcessingSampleRate, 16, + kAudioProcessingSampleRate / 100); + render_converter_.reset( + new MediaStreamAudioConverter(source_params, sink_params)); + render_data_bus_ = media::AudioBus::Create(number_of_channels, + frames_per_buffer); +} + +void MediaStreamAudioProcessor::ProcessData(webrtc::AudioFrame* audio_frame, + base::TimeDelta capture_delay, + int volume, + bool key_pressed) { + DCHECK(capture_thread_checker_.CalledOnValidThread()); + if (!audio_processing_) + return; + + TRACE_EVENT0("audio", "MediaStreamAudioProcessor::Process10MsData"); + DCHECK_EQ(audio_processing_->sample_rate_hz(), + capture_converter_->sink_parameters().sample_rate()); + DCHECK_EQ(audio_processing_->num_input_channels(), + capture_converter_->sink_parameters().channels()); + DCHECK_EQ(audio_processing_->num_output_channels(), + capture_converter_->sink_parameters().channels()); + + base::subtle::Atomic32 render_delay_ms = + base::subtle::Acquire_Load(&render_delay_ms_); + int64 capture_delay_ms = capture_delay.InMilliseconds(); + DCHECK_LT(capture_delay_ms, + std::numeric_limits<base::subtle::Atomic32>::max()); + int total_delay_ms = capture_delay_ms + render_delay_ms; + if (total_delay_ms > 1000) { + LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms + << "ms; render delay: " << render_delay_ms << "ms"; + } + + audio_processing_->set_stream_delay_ms(total_delay_ms); + webrtc::GainControl* agc = audio_processing_->gain_control(); + int err = agc->set_stream_analog_level(volume); + DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err; + err = audio_processing_->ProcessStream(audio_frame); + DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; + + // TODO(xians): Add support for AGC, typing detection, audio level + // calculation, stereo swapping. +} + +void MediaStreamAudioProcessor::StopAudioProcessing() { + if (!audio_processing_.get()) + return; + + // It is safe to stop the AEC dump even it is not started. + StopAecDump(audio_processing_.get()); + + audio_processing_.reset(); +} + +} // namespace content diff --git a/content/renderer/media/media_stream_audio_processor.h b/content/renderer/media/media_stream_audio_processor.h new file mode 100644 index 0000000..9c6db68 --- /dev/null +++ b/content/renderer/media/media_stream_audio_processor.h @@ -0,0 +1,138 @@ +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_ +#define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_ + +#include "base/atomicops.h" +#include "base/synchronization/lock.h" +#include "base/threading/thread_checker.h" +#include "base/time/time.h" +#include "content/common/content_export.h" +#include "media/base/audio_converter.h" +#include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface.h" +#include "third_party/webrtc/modules/audio_processing/include/audio_processing.h" +#include "third_party/webrtc/modules/interface/module_common_types.h" + +namespace media { +class AudioBus; +class AudioFifo; +class AudioParameters; +} // namespace media + +namespace webrtc { +class AudioFrame; +} + +namespace content { + +// This class owns an object of webrtc::AudioProcessing which contains signal +// processing components like AGC, AEC and NS. It enables the components based +// on the getUserMedia constraints, processes the data and outputs it in a unit +// of 10 ms data chunk. +class CONTENT_EXPORT MediaStreamAudioProcessor { + public: + explicit MediaStreamAudioProcessor( + const webrtc::MediaConstraintsInterface* constraints); + ~MediaStreamAudioProcessor(); + + // Pushes capture data in |audio_source| to the internal FIFO. + // Called on the capture audio thread. + void PushCaptureData(media::AudioBus* audio_source); + + // Push the render audio to webrtc::AudioProcessing for analysis. This is + // needed iff echo processing is enabled. + // |render_audio| is the pointer to the render audio data, its format + // is specified by |sample_rate|, |number_of_channels| and |number_of_frames|. + // Called on the render audio thread. + void PushRenderData(const int16* render_audio, + int sample_rate, + int number_of_channels, + int number_of_frames, + base::TimeDelta render_delay); + + // Processes a block of 10 ms data from the internal FIFO and outputs it via + // |out|. |out| is the address of the pointer that will be pointed to + // the post-processed data if the method is returning a true. The lifetime + // of the data represeted by |out| is guaranteed to outlive the method call. + // That also says *|out| won't change until this method is called again. + // Returns true if the internal FIFO has at least 10 ms data for processing, + // otherwise false. + // |capture_delay|, |volume| and |key_pressed| will be passed to + // webrtc::AudioProcessing to help processing the data. + // Called on the capture audio thread. + bool ProcessAndConsumeData(base::TimeDelta capture_delay, + int volume, + bool key_pressed, + int16** out); + + // Called when the format of the capture data has changed. + // This has to be called before PushCaptureData() and ProcessAndConsumeData(). + // Called on the main render thread. + void SetCaptureFormat(const media::AudioParameters& source_params); + + // The audio format of the output from the processor. + const media::AudioParameters& OutputFormat() const; + + // Accessor to check if the audio processing is enabled or not. + bool has_audio_processing() const { return audio_processing_.get() != NULL; } + + private: + class MediaStreamAudioConverter; + + // Helper to initialize the WebRtc AudioProcessing. + void InitializeAudioProcessingModule( + const webrtc::MediaConstraintsInterface* constraints); + + // Helper to initialize the render converter. + void InitializeRenderConverterIfNeeded(int sample_rate, + int number_of_channels, + int frames_per_buffer); + + // Called by ProcessAndConsumeData(). + void ProcessData(webrtc::AudioFrame* audio_frame, + base::TimeDelta capture_delay, + int volume, + bool key_pressed); + + // Called when the processor is going away. + void StopAudioProcessing(); + + // Cached value for the render delay latency. This member is accessed by + // both the capture audio thread and the render audio thread. + base::subtle::Atomic32 render_delay_ms_; + + // webrtc::AudioProcessing module which does AEC, AGC, NS, HighPass filter, + // ..etc. + scoped_ptr<webrtc::AudioProcessing> audio_processing_; + + // Converter used for the down-mixing and resampling of the capture data. + scoped_ptr<MediaStreamAudioConverter> capture_converter_; + + // AudioFrame used to hold the output of |capture_converter_|. + webrtc::AudioFrame capture_frame_; + + // Converter used for the down-mixing and resampling of the render data when + // the AEC is enabled. + scoped_ptr<MediaStreamAudioConverter> render_converter_; + + // AudioFrame used to hold the output of |render_converter_|. + webrtc::AudioFrame render_frame_; + + // Data bus to help converting interleaved data to an AudioBus. + scoped_ptr<media::AudioBus> render_data_bus_; + + // Used to DCHECK that some methods are called on the main render thread. + base::ThreadChecker main_thread_checker_; + + // Used to DCHECK that some methods are called on the capture audio thread. + base::ThreadChecker capture_thread_checker_; + + // Used to DCHECK that PushRenderData() is called on the render audio thread. + base::ThreadChecker render_thread_checker_; +}; + +} // namespace content + +#endif // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_ diff --git a/content/renderer/media/media_stream_audio_processor_options.cc b/content/renderer/media/media_stream_audio_processor_options.cc new file mode 100644 index 0000000..add7f957 --- /dev/null +++ b/content/renderer/media/media_stream_audio_processor_options.cc @@ -0,0 +1,96 @@ +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/renderer/media/media_stream_audio_processor_options.h" + +#include "base/files/file_path.h" +#include "base/logging.h" +#include "base/path_service.h" +#include "base/strings/utf_string_conversions.h" +#include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface.h" +#include "third_party/webrtc/modules/audio_processing/include/audio_processing.h" + +namespace content { + +bool GetPropertyFromConstraints(const MediaConstraintsInterface* constraints, + const std::string& key) { + bool value = false; + return webrtc::FindConstraint(constraints, key, &value, NULL) && value; +} + +void EnableEchoCancellation(AudioProcessing* audio_processing) { +#if defined(OS_IOS) + // On iOS, VPIO provides built-in EC and AGC. + return; +#elif defined(OS_ANDROID) + // Mobile devices are using AECM. + int err = audio_processing->echo_control_mobile()->Enable(true); + err |= audio_processing->echo_control_mobile()->set_routing_mode( + webrtc::EchoControlMobile::kSpeakerphone); + CHECK_EQ(err, 0); +#else + int err = audio_processing->echo_cancellation()->Enable(true); + err |= audio_processing->echo_cancellation()->set_suppression_level( + webrtc::EchoCancellation::kHighSuppression); + + // Enable the metrics for AEC. + err |= audio_processing->echo_cancellation()->enable_metrics(true); + err |= audio_processing->echo_cancellation()->enable_delay_logging(true); + CHECK_EQ(err, 0); +#endif +} + +void EnableNoiseSuppression(AudioProcessing* audio_processing) { + int err = audio_processing->noise_suppression()->set_level( + webrtc::NoiseSuppression::kHigh); + err |= audio_processing->noise_suppression()->Enable(true); + CHECK_EQ(err, 0); +} + +void EnableHighPassFilter(AudioProcessing* audio_processing) { + CHECK_EQ(audio_processing->high_pass_filter()->Enable(true), 0); +} + +// TODO(xians): stereo swapping +void EnableTypingDetection(AudioProcessing* audio_processing) { + int err = audio_processing->voice_detection()->Enable(true); + err |= audio_processing->voice_detection()->set_likelihood( + webrtc::VoiceDetection::kVeryLowLikelihood); + CHECK_EQ(err, 0); +} + +void EnableExperimentalEchoCancellation(AudioProcessing* audio_processing) { + webrtc::Config config; + config.Set<webrtc::DelayCorrection>(new webrtc::DelayCorrection(true)); + audio_processing->SetExtraOptions(config); +} + +void StartAecDump(AudioProcessing* audio_processing) { + // TODO(grunell): Figure out a more suitable directory for the audio dump + // data. + base::FilePath path; +#if defined(CHROMEOS) + PathService::Get(base::DIR_TEMP, &path); +#elif defined(ANDROID) + path = base::FilePath(FILE_PATH_LITERAL("sdcard")); +#else + PathService::Get(base::DIR_EXE, &path); +#endif + base::FilePath file = path.Append(FILE_PATH_LITERAL("audio.aecdump")); + +#if defined(OS_WIN) + const std::string file_name = WideToUTF8(file.value()); +#else + const std::string file_name = file.value(); +#endif + if (audio_processing->StartDebugRecording(file_name.c_str())) + DLOG(ERROR) << "Fail to start AEC debug recording"; +} + +void StopAecDump(AudioProcessing* audio_processing) { + if (audio_processing->StopDebugRecording()) + DLOG(ERROR) << "Fail to stop AEC debug recording"; +} + +} // namespace content diff --git a/content/renderer/media/media_stream_audio_processor_options.h b/content/renderer/media/media_stream_audio_processor_options.h new file mode 100644 index 0000000..dcdec4e --- /dev/null +++ b/content/renderer/media/media_stream_audio_processor_options.h @@ -0,0 +1,53 @@ +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_OPTIONS_H_ +#define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_OPTIONS_H_ + +#include <string> + +namespace webrtc { + +class AudioFrame; +class AudioProcessing; +class MediaConstraintsInterface; + +} + +namespace content { + +using webrtc::AudioProcessing; +using webrtc::MediaConstraintsInterface; + +// Gets the property named by |key| from the |constraints|. +// Returns true if the key is found and has a valid boolean value; Otherwise +// false. +bool GetPropertyFromConstraints( + const MediaConstraintsInterface* constraints, + const std::string& key); + +// Enables the echo cancellation in |audio_processing|. +void EnableEchoCancellation(AudioProcessing* audio_processing); + +// Enables the noise suppression in |audio_processing|. +void EnableNoiseSuppression(AudioProcessing* audio_processing); + +// Enables the high pass filter in |audio_processing|. +void EnableHighPassFilter(AudioProcessing* audio_processing); + +// Enables the typing detection in |audio_processing|. +void EnableTypingDetection(AudioProcessing* audio_processing); + +// Enables the experimental echo cancellation in |audio_processing|. +void EnableExperimentalEchoCancellation(AudioProcessing* audio_processing); + +// Starts the echo cancellation dump in |audio_processing|. +void StartAecDump(AudioProcessing* audio_processing); + +// Stops the echo cancellation dump in |audio_processing|. +void StopAecDump(AudioProcessing* audio_processing); + +} // namespace content + +#endif // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_OPTIONS_H_ diff --git a/content/renderer/media/media_stream_audio_processor_unittest.cc b/content/renderer/media/media_stream_audio_processor_unittest.cc new file mode 100644 index 0000000..2d20ded --- /dev/null +++ b/content/renderer/media/media_stream_audio_processor_unittest.cc @@ -0,0 +1,166 @@ +// Copyright 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/command_line.h" +#include "base/file_util.h" +#include "base/files/file_path.h" +#include "base/logging.h" +#include "base/path_service.h" +#include "base/time/time.h" +#include "content/public/common/content_switches.h" +#include "content/renderer/media/media_stream_audio_processor.h" +#include "content/renderer/media/rtc_media_constraints.h" +#include "media/audio/audio_parameters.h" +#include "media/base/audio_bus.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h" + +using ::testing::_; +using ::testing::AnyNumber; +using ::testing::AtLeast; +using ::testing::Return; + +namespace content { + +namespace { + +#if defined(ANDROID) +const int kAudioProcessingSampleRate = 16000; +#else +const int kAudioProcessingSampleRate = 32000; +#endif +const int kAudioProcessingNumberOfChannel = 1; + +// The number of packers used for testing. +const int kNumberOfPacketsForTest = 100; + +void ReadDataFromSpeechFile(char* data, int length) { + base::FilePath file; + CHECK(PathService::Get(base::DIR_SOURCE_ROOT, &file)); + file = file.Append(FILE_PATH_LITERAL("media")) + .Append(FILE_PATH_LITERAL("test")) + .Append(FILE_PATH_LITERAL("data")) + .Append(FILE_PATH_LITERAL("speech_16b_stereo_48kHz.raw")); + DCHECK(base::PathExists(file)); + int64 data_file_size64 = 0; + DCHECK(file_util::GetFileSize(file, &data_file_size64)); + EXPECT_EQ(length, file_util::ReadFile(file, data, length)); + DCHECK(data_file_size64 > length); +} + +void ApplyFixedAudioConstraints(RTCMediaConstraints* constraints) { + // Constant constraint keys which enables default audio constraints on + // mediastreams with audio. + struct { + const char* key; + const char* value; + } static const kDefaultAudioConstraints[] = { + { webrtc::MediaConstraintsInterface::kEchoCancellation, + webrtc::MediaConstraintsInterface::kValueTrue }, + #if defined(OS_CHROMEOS) || defined(OS_MACOSX) + // Enable the extended filter mode AEC on platforms with known echo issues. + { webrtc::MediaConstraintsInterface::kExperimentalEchoCancellation, + webrtc::MediaConstraintsInterface::kValueTrue }, + #endif + { webrtc::MediaConstraintsInterface::kAutoGainControl, + webrtc::MediaConstraintsInterface::kValueTrue }, + { webrtc::MediaConstraintsInterface::kExperimentalAutoGainControl, + webrtc::MediaConstraintsInterface::kValueTrue }, + { webrtc::MediaConstraintsInterface::kNoiseSuppression, + webrtc::MediaConstraintsInterface::kValueTrue }, + { webrtc::MediaConstraintsInterface::kHighpassFilter, + webrtc::MediaConstraintsInterface::kValueTrue }, + }; + + for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kDefaultAudioConstraints); ++i) { + constraints->AddMandatory(kDefaultAudioConstraints[i].key, + kDefaultAudioConstraints[i].value, false); + } +} + +} // namespace + +class MediaStreamAudioProcessorTest : public ::testing::Test { + public: + MediaStreamAudioProcessorTest() + : params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY, + media::CHANNEL_LAYOUT_STEREO, 48000, 16, 512) { + CommandLine::ForCurrentProcess()->AppendSwitch( + switches::kEnableAudioTrackProcessing); + } + + protected: + // Helper method to save duplicated code. + void ProcessDataAndVerifyFormat(MediaStreamAudioProcessor* audio_processor, + int expected_output_sample_rate, + int expected_output_channels, + int expected_output_buffer_size) { + // Read the audio data from a file. + const int packet_size = + params_.frames_per_buffer() * 2 * params_.channels(); + const size_t length = packet_size * kNumberOfPacketsForTest; + scoped_ptr<char[]> capture_data(new char[length]); + ReadDataFromSpeechFile(capture_data.get(), length); + const int16* data_ptr = reinterpret_cast<const int16*>(capture_data.get()); + scoped_ptr<media::AudioBus> data_bus = media::AudioBus::Create( + params_.channels(), params_.frames_per_buffer()); + for (int i = 0; i < kNumberOfPacketsForTest; ++i) { + data_bus->FromInterleaved(data_ptr, data_bus->frames(), 2); + audio_processor->PushCaptureData(data_bus.get()); + + // |audio_processor| does nothing when the audio processing is off in + // the processor. + audio_processor->PushRenderData( + data_ptr, + params_.sample_rate(), params_.channels(), + params_.frames_per_buffer(), base::TimeDelta::FromMilliseconds(10)); + + int16* output = NULL; + while(audio_processor->ProcessAndConsumeData( + base::TimeDelta::FromMilliseconds(10), 255, false, &output)) { + EXPECT_TRUE(output != NULL); + EXPECT_EQ(audio_processor->OutputFormat().sample_rate(), + expected_output_sample_rate); + EXPECT_EQ(audio_processor->OutputFormat().channels(), + expected_output_channels); + EXPECT_EQ(audio_processor->OutputFormat().frames_per_buffer(), + expected_output_buffer_size); + } + + data_ptr += params_.frames_per_buffer() * params_.channels(); + } + } + + media::AudioParameters params_; +}; + +TEST_F(MediaStreamAudioProcessorTest, WithoutAudioProcessing) { + // Setup the audio processor with empty constraint. + RTCMediaConstraints constraints; + MediaStreamAudioProcessor audio_processor(&constraints); + audio_processor.SetCaptureFormat(params_); + EXPECT_FALSE(audio_processor.has_audio_processing()); + + ProcessDataAndVerifyFormat(&audio_processor, + params_.sample_rate(), + params_.channels(), + params_.sample_rate() / 100); +} + +TEST_F(MediaStreamAudioProcessorTest, WithAudioProcessing) { + // Setup the audio processor with default constraint. + RTCMediaConstraints constraints; + ApplyFixedAudioConstraints(&constraints); + MediaStreamAudioProcessor audio_processor(&constraints); + audio_processor.SetCaptureFormat(params_); + EXPECT_TRUE(audio_processor.has_audio_processing()); + + ProcessDataAndVerifyFormat(&audio_processor, + kAudioProcessingSampleRate, + kAudioProcessingNumberOfChannel, + kAudioProcessingSampleRate / 100); +} + +} // namespace content |