// Copyright 2013 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "media/cast/audio_sender/audio_encoder.h" #include #include "base/bind.h" #include "base/bind_helpers.h" #include "base/logging.h" #include "base/memory/ref_counted.h" #include "base/message_loop/message_loop.h" #include "base/sys_byteorder.h" #include "base/time/time.h" #include "media/base/audio_bus.h" #include "media/cast/cast_defines.h" #include "media/cast/cast_environment.h" #include "third_party/opus/src/include/opus.h" namespace media { namespace cast { // Base class that handles the common problem of feeding one or more AudioBus' // data into a 10 ms buffer and then, once the buffer is full, encoding the // signal and emitting an EncodedAudioFrame via the FrameEncodedCallback. // // Subclasses complete the implementation by handling the actual encoding // details. class AudioEncoder::ImplBase : public base::RefCountedThreadSafe, public base::SupportsWeakPtr { public: ImplBase(const scoped_refptr& cast_environment, transport::AudioCodec codec, int num_channels, int sampling_rate, const FrameEncodedCallback& callback) : cast_environment_(cast_environment), codec_(codec), num_channels_(num_channels), samples_per_10ms_(sampling_rate / 100), callback_(callback), buffer_fill_end_(0), frame_id_(0), rtp_timestamp_(0), weak_factory_(this) { DCHECK_GT(num_channels_, 0); DCHECK_GT(samples_per_10ms_, 0); DCHECK_EQ(sampling_rate % 100, 0); DCHECK_LE(samples_per_10ms_ * num_channels_, transport::EncodedAudioFrame::kMaxNumberOfSamples); if (num_channels_ <= 0 || samples_per_10ms_ <= 0 || sampling_rate % 100 != 0 || samples_per_10ms_ * num_channels_ > transport::EncodedAudioFrame::kMaxNumberOfSamples) { cast_initialization_cb_ = STATUS_INVALID_AUDIO_CONFIGURATION; } else { cast_initialization_cb_ = STATUS_AUDIO_INITIALIZED; } } CastInitializationStatus InitializationResult() const { return cast_initialization_cb_; } void LogAudioFrameEvent(uint32 rtp_timestamp, uint32 frame_id, CastLoggingEvent type) { cast_environment_->Logging()->InsertFrameEvent( cast_environment_->Clock()->NowTicks(), type, rtp_timestamp, frame_id); } void EncodeAudio(scoped_ptr audio_bus, const base::TimeTicks& recorded_time) { int src_pos = 0; int packet_count = 0; while (src_pos < audio_bus->frames()) { const int num_samples_to_xfer = std::min( samples_per_10ms_ - buffer_fill_end_, audio_bus->frames() - src_pos); DCHECK_EQ(audio_bus->channels(), num_channels_); TransferSamplesIntoBuffer( audio_bus.get(), src_pos, buffer_fill_end_, num_samples_to_xfer); src_pos += num_samples_to_xfer; buffer_fill_end_ += num_samples_to_xfer; if (buffer_fill_end_ == samples_per_10ms_) { scoped_ptr audio_frame( new transport::EncodedAudioFrame()); audio_frame->codec = codec_; audio_frame->frame_id = frame_id_++; rtp_timestamp_ += samples_per_10ms_; audio_frame->rtp_timestamp = rtp_timestamp_; // Update logging. cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&ImplBase::LogAudioFrameEvent, weak_factory_.GetWeakPtr(), audio_frame->rtp_timestamp, audio_frame->frame_id, kAudioFrameReceived)); if (EncodeFromFilledBuffer(&audio_frame->data)) { // Update logging. cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&ImplBase::LogAudioFrameEvent, weak_factory_.GetWeakPtr(), audio_frame->rtp_timestamp, audio_frame->frame_id, kAudioFrameEncoded)); // Compute an offset to determine the recorded time for the first // audio sample in the buffer. const base::TimeDelta buffer_time_offset = (buffer_fill_end_ - src_pos) * base::TimeDelta::FromMilliseconds(10) / samples_per_10ms_; // TODO(miu): Consider batching EncodedAudioFrames so we only post a // at most one task for each call to this method. // Postpone every packet by 10mS with respect to the previous. Playout // is postponed already by 10mS, and this will better correlate with // the pacer's expectations. //TODO(mikhal): Turn this into a list of packets. // Update the end2end allowed error once this is fixed. cast_environment_->PostDelayedTask( CastEnvironment::MAIN, FROM_HERE, base::Bind(callback_, base::Passed(&audio_frame), recorded_time - buffer_time_offset), base::TimeDelta::FromMilliseconds(packet_count * 10)); ++packet_count; } buffer_fill_end_ = 0; } } } protected: friend class base::RefCountedThreadSafe; virtual ~ImplBase() {} virtual void TransferSamplesIntoBuffer(const AudioBus* audio_bus, int source_offset, int buffer_fill_offset, int num_samples) = 0; virtual bool EncodeFromFilledBuffer(std::string* out) = 0; const scoped_refptr cast_environment_; const transport::AudioCodec codec_; const int num_channels_; const int samples_per_10ms_; const FrameEncodedCallback callback_; CastInitializationStatus cast_initialization_cb_; private: // In the case where a call to EncodeAudio() cannot completely fill the // buffer, this points to the position at which to populate data in a later // call. int buffer_fill_end_; // A counter used to label EncodedAudioFrames. uint32 frame_id_; // For audio, rtp_timestamp is computed as the sum of the audio samples seen // so far. uint32 rtp_timestamp_; base::WeakPtrFactory weak_factory_; private: DISALLOW_COPY_AND_ASSIGN(ImplBase); }; class AudioEncoder::OpusImpl : public AudioEncoder::ImplBase { public: OpusImpl(const scoped_refptr& cast_environment, int num_channels, int sampling_rate, int bitrate, const FrameEncodedCallback& callback) : ImplBase(cast_environment, transport::kOpus, num_channels, sampling_rate, callback), encoder_memory_(new uint8[opus_encoder_get_size(num_channels)]), opus_encoder_(reinterpret_cast(encoder_memory_.get())), buffer_(new float[num_channels * samples_per_10ms_]) { if (ImplBase::cast_initialization_cb_ != STATUS_AUDIO_INITIALIZED) { return; } CHECK_EQ( opus_encoder_init( opus_encoder_, sampling_rate, num_channels, OPUS_APPLICATION_AUDIO), OPUS_OK); if (bitrate <= 0) { // Note: As of 2013-10-31, the encoder in "auto bitrate" mode would use a // variable bitrate up to 102kbps for 2-channel, 48 kHz audio and a 10 ms // frame size. The opus library authors may, of course, adjust this in // later versions. bitrate = OPUS_AUTO; } CHECK_EQ(opus_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(bitrate)), OPUS_OK); } private: virtual ~OpusImpl() {} virtual void TransferSamplesIntoBuffer(const AudioBus* audio_bus, int source_offset, int buffer_fill_offset, int num_samples) OVERRIDE { // Opus requires channel-interleaved samples in a single array. for (int ch = 0; ch < audio_bus->channels(); ++ch) { const float* src = audio_bus->channel(ch) + source_offset; const float* const src_end = src + num_samples; float* dest = buffer_.get() + buffer_fill_offset * num_channels_ + ch; for (; src < src_end; ++src, dest += num_channels_) *dest = *src; } } virtual bool EncodeFromFilledBuffer(std::string* out) OVERRIDE { out->resize(kOpusMaxPayloadSize); const opus_int32 result = opus_encode_float(opus_encoder_, buffer_.get(), samples_per_10ms_, reinterpret_cast(&out->at(0)), kOpusMaxPayloadSize); if (result > 1) { out->resize(result); return true; } else if (result < 0) { LOG(ERROR) << "Error code from opus_encode_float(): " << result; return false; } else { // Do nothing: The documentation says that a return value of zero or // one byte means the packet does not need to be transmitted. return false; } } const scoped_ptr encoder_memory_; OpusEncoder* const opus_encoder_; const scoped_ptr buffer_; // This is the recommended value, according to documentation in // third_party/opus/src/include/opus.h, so that the Opus encoder does not // degrade the audio due to memory constraints. // // Note: Whereas other RTP implementations do not, the cast library is // perfectly capable of transporting larger than MTU-sized audio frames. static const int kOpusMaxPayloadSize = 4000; DISALLOW_COPY_AND_ASSIGN(OpusImpl); }; class AudioEncoder::Pcm16Impl : public AudioEncoder::ImplBase { public: Pcm16Impl(const scoped_refptr& cast_environment, int num_channels, int sampling_rate, const FrameEncodedCallback& callback) : ImplBase(cast_environment, transport::kPcm16, num_channels, sampling_rate, callback), buffer_(new int16[num_channels * samples_per_10ms_]) {} private: virtual ~Pcm16Impl() {} virtual void TransferSamplesIntoBuffer(const AudioBus* audio_bus, int source_offset, int buffer_fill_offset, int num_samples) OVERRIDE { audio_bus->ToInterleavedPartial( source_offset, num_samples, sizeof(int16), buffer_.get() + buffer_fill_offset * num_channels_); } virtual bool EncodeFromFilledBuffer(std::string* out) OVERRIDE { // Output 16-bit PCM integers in big-endian byte order. out->resize(num_channels_ * samples_per_10ms_ * sizeof(int16)); const int16* src = buffer_.get(); const int16* const src_end = src + num_channels_ * samples_per_10ms_; uint16* dest = reinterpret_cast(&out->at(0)); for (; src < src_end; ++src, ++dest) *dest = base::HostToNet16(*src); return true; } private: const scoped_ptr buffer_; DISALLOW_COPY_AND_ASSIGN(Pcm16Impl); }; AudioEncoder::AudioEncoder( const scoped_refptr& cast_environment, const AudioSenderConfig& audio_config, const FrameEncodedCallback& frame_encoded_callback) : cast_environment_(cast_environment) { // Note: It doesn't matter which thread constructs AudioEncoder, just so long // as all calls to InsertAudio() are by the same thread. insert_thread_checker_.DetachFromThread(); switch (audio_config.codec) { case transport::kOpus: impl_ = new OpusImpl(cast_environment, audio_config.channels, audio_config.frequency, audio_config.bitrate, frame_encoded_callback); break; case transport::kPcm16: impl_ = new Pcm16Impl(cast_environment, audio_config.channels, audio_config.frequency, frame_encoded_callback); break; default: NOTREACHED() << "Unsupported or unspecified codec for audio encoder"; break; } } AudioEncoder::~AudioEncoder() {} CastInitializationStatus AudioEncoder::InitializationResult() const { DCHECK(insert_thread_checker_.CalledOnValidThread()); if (impl_) { return impl_->InitializationResult(); } return STATUS_UNSUPPORTED_AUDIO_CODEC; } void AudioEncoder::InsertAudio(scoped_ptr audio_bus, const base::TimeTicks& recorded_time) { DCHECK(insert_thread_checker_.CalledOnValidThread()); DCHECK(audio_bus.get()); if (!impl_) { NOTREACHED(); return; } cast_environment_->PostTask(CastEnvironment::AUDIO_ENCODER, FROM_HERE, base::Bind(&AudioEncoder::ImplBase::EncodeAudio, impl_, base::Passed(&audio_bus), recorded_time)); } } // namespace cast } // namespace media