summaryrefslogtreecommitdiffstats
path: root/content/renderer/media/media_stream_audio_processor.h
blob: 3c44f808aedc0d869b75528aec734cb52485aead (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
#define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_

#include "base/atomicops.h"
#include "base/files/file.h"
#include "base/synchronization/lock.h"
#include "base/threading/thread_checker.h"
#include "base/time/time.h"
#include "content/common/content_export.h"
#include "content/renderer/media/aec_dump_message_filter.h"
#include "content/renderer/media/webrtc_audio_device_impl.h"
#include "media/base/audio_converter.h"
#include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
#include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
#include "third_party/webrtc/modules/interface/module_common_types.h"

namespace blink {
class WebMediaConstraints;
}

namespace media {
class AudioBus;
class AudioFifo;
class AudioParameters;
}  // namespace media

namespace webrtc {
class AudioFrame;
class TypingDetection;
}

namespace content {

class MediaStreamAudioBus;
class MediaStreamAudioFifo;
class RTCMediaConstraints;

using webrtc::AudioProcessorInterface;

// This class owns an object of webrtc::AudioProcessing which contains signal
// processing components like AGC, AEC and NS. It enables the components based
// on the getUserMedia constraints, processes the data and outputs it in a unit
// of 10 ms data chunk.
class CONTENT_EXPORT MediaStreamAudioProcessor :
    NON_EXPORTED_BASE(public WebRtcPlayoutDataSource::Sink),
    NON_EXPORTED_BASE(public AudioProcessorInterface),
    NON_EXPORTED_BASE(public AecDumpMessageFilter::AecDumpDelegate) {
 public:
  // Returns false if |kDisableAudioTrackProcessing| is set to true, otherwise
  // returns true.
  static bool IsAudioTrackProcessingEnabled();

  // |playout_data_source| is used to register this class as a sink to the
  // WebRtc playout data for processing AEC. If clients do not enable AEC,
  // |playout_data_source| won't be used.
  MediaStreamAudioProcessor(const blink::WebMediaConstraints& constraints,
                            int effects,
                            WebRtcPlayoutDataSource* playout_data_source);

  // Called when the format of the capture data has changed.
  // Called on the main render thread. The caller is responsible for stopping
  // the capture thread before calling this method.
  // After this method, the capture thread will be changed to a new capture
  // thread.
  void OnCaptureFormatChanged(const media::AudioParameters& source_params);

  // Pushes capture data in |audio_source| to the internal FIFO. Each call to
  // this method should be followed by calls to ProcessAndConsumeData() while
  // it returns false, to pull out all available data.
  // Called on the capture audio thread.
  void PushCaptureData(const media::AudioBus* audio_source);

  // Processes a block of 10 ms data from the internal FIFO and outputs it via
  // |out|. |out| is the address of the pointer that will be pointed to
  // the post-processed data if the method is returning a true. The lifetime
  // of the data represeted by |out| is guaranteed until this method is called
  // again.
  // |new_volume| receives the new microphone volume from the AGC.
  // The new microphone volume range is [0, 255], and the value will be 0 if
  // the microphone volume should not be adjusted.
  // Returns true if the internal FIFO has at least 10 ms data for processing,
  // otherwise false.
  // Called on the capture audio thread.
  //
  // TODO(ajm): Don't we want this to output float?
  bool ProcessAndConsumeData(base::TimeDelta capture_delay,
                             int volume,
                             bool key_pressed,
                             int* new_volume,
                             int16** out);

  // Stops the audio processor, no more AEC dump or render data after calling
  // this method.
  void Stop();

  // The audio formats of the capture input to and output from the processor.
  // Must only be called on the main render or audio capture threads.
  const media::AudioParameters& InputFormat() const;
  const media::AudioParameters& OutputFormat() const;

  // Accessor to check if the audio processing is enabled or not.
  bool has_audio_processing() const { return audio_processing_ != NULL; }

  // AecDumpMessageFilter::AecDumpDelegate implementation.
  // Called on the main render thread.
  virtual void OnAecDumpFile(
      const IPC::PlatformFileForTransit& file_handle) OVERRIDE;
  virtual void OnDisableAecDump() OVERRIDE;
  virtual void OnIpcClosing() OVERRIDE;

 protected:
  friend class base::RefCountedThreadSafe<MediaStreamAudioProcessor>;
  virtual ~MediaStreamAudioProcessor();

 private:
  friend class MediaStreamAudioProcessorTest;
  FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest,
                           GetAecDumpMessageFilter);

  // WebRtcPlayoutDataSource::Sink implementation.
  virtual void OnPlayoutData(media::AudioBus* audio_bus,
                             int sample_rate,
                             int audio_delay_milliseconds) OVERRIDE;
  virtual void OnPlayoutDataSourceChanged() OVERRIDE;

  // webrtc::AudioProcessorInterface implementation.
  // This method is called on the libjingle thread.
  virtual void GetStats(AudioProcessorStats* stats) OVERRIDE;

  // Helper to initialize the WebRtc AudioProcessing.
  void InitializeAudioProcessingModule(
      const blink::WebMediaConstraints& constraints, int effects);

  // Helper to initialize the capture converter.
  void InitializeCaptureFifo(const media::AudioParameters& input_format);

  // Helper to initialize the render converter.
  void InitializeRenderFifoIfNeeded(int sample_rate,
                                    int number_of_channels,
                                    int frames_per_buffer);

  // Called by ProcessAndConsumeData().
  // Returns the new microphone volume in the range of |0, 255].
  // When the volume does not need to be updated, it returns 0.
  int ProcessData(const float* const* process_ptrs,
                  int process_frames,
                  base::TimeDelta capture_delay,
                  int volume,
                  bool key_pressed,
                  float* const* output_ptrs);

  // Cached value for the render delay latency. This member is accessed by
  // both the capture audio thread and the render audio thread.
  base::subtle::Atomic32 render_delay_ms_;

  // Module to handle processing and format conversion.
  scoped_ptr<webrtc::AudioProcessing> audio_processing_;

  // FIFO to provide 10 ms capture chunks.
  scoped_ptr<MediaStreamAudioFifo> capture_fifo_;
  // Receives processing output.
  scoped_ptr<MediaStreamAudioBus> output_bus_;
  // Receives interleaved int16 data for output.
  scoped_ptr<int16[]> output_data_;

  // FIFO to provide 10 ms render chunks when the AEC is enabled.
  scoped_ptr<MediaStreamAudioFifo> render_fifo_;

  // These are mutated on the main render thread in OnCaptureFormatChanged().
  // The caller guarantees this does not run concurrently with accesses on the
  // capture audio thread.
  media::AudioParameters input_format_;
  media::AudioParameters output_format_;
  // Only used on the render audio thread.
  media::AudioParameters render_format_;

  // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the
  // lifetime of RenderThread.
  WebRtcPlayoutDataSource* playout_data_source_;

  // Used to DCHECK that some methods are called on the main render thread.
  base::ThreadChecker main_thread_checker_;
  // Used to DCHECK that some methods are called on the capture audio thread.
  base::ThreadChecker capture_thread_checker_;
  // Used to DCHECK that some methods are called on the render audio thread.
  base::ThreadChecker render_thread_checker_;

  // Flag to enable stereo channel mirroring.
  bool audio_mirroring_;

  scoped_ptr<webrtc::TypingDetection> typing_detector_;
  // This flag is used to show the result of typing detection.
  // It can be accessed by the capture audio thread and by the libjingle thread
  // which calls GetStats().
  base::subtle::Atomic32 typing_detected_;

  // Communication with browser for AEC dump.
  scoped_refptr<AecDumpMessageFilter> aec_dump_message_filter_;

  // Flag to avoid executing Stop() more than once.
  bool stopped_;
};

}  // namespace content

#endif  // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_