// Copyright 2013 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_ #define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_ #include "base/atomicops.h" #include "base/files/file.h" #include "base/synchronization/lock.h" #include "base/threading/thread_checker.h" #include "base/time/time.h" #include "content/common/content_export.h" #include "content/renderer/media/aec_dump_message_filter.h" #include "content/renderer/media/webrtc_audio_device_impl.h" #include "media/base/audio_converter.h" #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h" #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h" #include "third_party/webrtc/modules/interface/module_common_types.h" namespace blink { class WebMediaConstraints; } namespace media { class AudioBus; class AudioFifo; class AudioParameters; } // namespace media namespace webrtc { class AudioFrame; class TypingDetection; } namespace content { class MediaStreamAudioBus; class MediaStreamAudioFifo; class RTCMediaConstraints; using webrtc::AudioProcessorInterface; // This class owns an object of webrtc::AudioProcessing which contains signal // processing components like AGC, AEC and NS. It enables the components based // on the getUserMedia constraints, processes the data and outputs it in a unit // of 10 ms data chunk. class CONTENT_EXPORT MediaStreamAudioProcessor : NON_EXPORTED_BASE(public WebRtcPlayoutDataSource::Sink), NON_EXPORTED_BASE(public AudioProcessorInterface), NON_EXPORTED_BASE(public AecDumpMessageFilter::AecDumpDelegate) { public: // Returns false if |kDisableAudioTrackProcessing| is set to true, otherwise // returns true. static bool IsAudioTrackProcessingEnabled(); // |playout_data_source| is used to register this class as a sink to the // WebRtc playout data for processing AEC. If clients do not enable AEC, // |playout_data_source| won't be used. MediaStreamAudioProcessor(const blink::WebMediaConstraints& constraints, int effects, WebRtcPlayoutDataSource* playout_data_source); // Called when the format of the capture data has changed. // Called on the main render thread. The caller is responsible for stopping // the capture thread before calling this method. // After this method, the capture thread will be changed to a new capture // thread. void OnCaptureFormatChanged(const media::AudioParameters& source_params); // Pushes capture data in |audio_source| to the internal FIFO. Each call to // this method should be followed by calls to ProcessAndConsumeData() while // it returns false, to pull out all available data. // Called on the capture audio thread. void PushCaptureData(const media::AudioBus* audio_source); // Processes a block of 10 ms data from the internal FIFO and outputs it via // |out|. |out| is the address of the pointer that will be pointed to // the post-processed data if the method is returning a true. The lifetime // of the data represeted by |out| is guaranteed until this method is called // again. // |new_volume| receives the new microphone volume from the AGC. // The new microphone volume range is [0, 255], and the value will be 0 if // the microphone volume should not be adjusted. // Returns true if the internal FIFO has at least 10 ms data for processing, // otherwise false. // Called on the capture audio thread. // // TODO(ajm): Don't we want this to output float? bool ProcessAndConsumeData(base::TimeDelta capture_delay, int volume, bool key_pressed, int* new_volume, int16** out); // Stops the audio processor, no more AEC dump or render data after calling // this method. void Stop(); // The audio formats of the capture input to and output from the processor. // Must only be called on the main render or audio capture threads. const media::AudioParameters& InputFormat() const; const media::AudioParameters& OutputFormat() const; // Accessor to check if the audio processing is enabled or not. bool has_audio_processing() const { return audio_processing_ != NULL; } // AecDumpMessageFilter::AecDumpDelegate implementation. // Called on the main render thread. virtual void OnAecDumpFile( const IPC::PlatformFileForTransit& file_handle) OVERRIDE; virtual void OnDisableAecDump() OVERRIDE; virtual void OnIpcClosing() OVERRIDE; protected: friend class base::RefCountedThreadSafe; virtual ~MediaStreamAudioProcessor(); private: friend class MediaStreamAudioProcessorTest; FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest, GetAecDumpMessageFilter); // WebRtcPlayoutDataSource::Sink implementation. virtual void OnPlayoutData(media::AudioBus* audio_bus, int sample_rate, int audio_delay_milliseconds) OVERRIDE; virtual void OnPlayoutDataSourceChanged() OVERRIDE; // webrtc::AudioProcessorInterface implementation. // This method is called on the libjingle thread. virtual void GetStats(AudioProcessorStats* stats) OVERRIDE; // Helper to initialize the WebRtc AudioProcessing. void InitializeAudioProcessingModule( const blink::WebMediaConstraints& constraints, int effects); // Helper to initialize the capture converter. void InitializeCaptureFifo(const media::AudioParameters& input_format); // Helper to initialize the render converter. void InitializeRenderFifoIfNeeded(int sample_rate, int number_of_channels, int frames_per_buffer); // Called by ProcessAndConsumeData(). // Returns the new microphone volume in the range of |0, 255]. // When the volume does not need to be updated, it returns 0. int ProcessData(const float* const* process_ptrs, int process_frames, base::TimeDelta capture_delay, int volume, bool key_pressed, float* const* output_ptrs); // Cached value for the render delay latency. This member is accessed by // both the capture audio thread and the render audio thread. base::subtle::Atomic32 render_delay_ms_; // Module to handle processing and format conversion. scoped_ptr audio_processing_; // FIFO to provide 10 ms capture chunks. scoped_ptr capture_fifo_; // Receives processing output. scoped_ptr output_bus_; // Receives interleaved int16 data for output. scoped_ptr output_data_; // FIFO to provide 10 ms render chunks when the AEC is enabled. scoped_ptr render_fifo_; // These are mutated on the main render thread in OnCaptureFormatChanged(). // The caller guarantees this does not run concurrently with accesses on the // capture audio thread. media::AudioParameters input_format_; media::AudioParameters output_format_; // Only used on the render audio thread. media::AudioParameters render_format_; // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the // lifetime of RenderThread. WebRtcPlayoutDataSource* playout_data_source_; // Used to DCHECK that some methods are called on the main render thread. base::ThreadChecker main_thread_checker_; // Used to DCHECK that some methods are called on the capture audio thread. base::ThreadChecker capture_thread_checker_; // Used to DCHECK that some methods are called on the render audio thread. base::ThreadChecker render_thread_checker_; // Flag to enable stereo channel mirroring. bool audio_mirroring_; scoped_ptr typing_detector_; // This flag is used to show the result of typing detection. // It can be accessed by the capture audio thread and by the libjingle thread // which calls GetStats(). base::subtle::Atomic32 typing_detected_; // Communication with browser for AEC dump. scoped_refptr aec_dump_message_filter_; // Flag to avoid executing Stop() more than once. bool stopped_; }; } // namespace content #endif // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_