diff options
author | wolenetz@chromium.org <wolenetz@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-05-09 20:44:18 +0000 |
---|---|---|
committer | wolenetz@chromium.org <wolenetz@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-05-09 20:44:18 +0000 |
commit | d7795c626fb6f111b73ed00edc734897346aed23 (patch) | |
tree | ec9d5a284095d26db644bfe817d9e5c6928e2f0b /media | |
parent | 5f9d195827d3119f3ae2fbcf26d80b7d0ad18730 (diff) | |
download | chromium_src-d7795c626fb6f111b73ed00edc734897346aed23.zip chromium_src-d7795c626fb6f111b73ed00edc734897346aed23.tar.gz chromium_src-d7795c626fb6f111b73ed00edc734897346aed23.tar.bz2 |
Implement core of compliant MediaSource coded frame processing
Introduces FrameProcessor and associated unit tests that implement the
core of the Coded Frame Processing algorithm in the current Media Source
Extension spec (April 1, 2014 MSE editor's draft).
R=acolwell@chromium.org
BUG=249422
TEST=All media_unittests pass and no http/tests/media/media-source layout tests regress locally on Linux with proprietary codecs enabled
Review URL: https://codereview.chromium.org/180153003
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@269397 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'media')
-rw-r--r-- | media/filters/chunk_demuxer.cc | 53 | ||||
-rw-r--r-- | media/filters/chunk_demuxer.h | 13 | ||||
-rw-r--r-- | media/filters/chunk_demuxer_unittest.cc | 31 | ||||
-rw-r--r-- | media/filters/frame_processor.cc | 378 | ||||
-rw-r--r-- | media/filters/frame_processor.h | 60 | ||||
-rw-r--r-- | media/filters/frame_processor_base.cc | 38 | ||||
-rw-r--r-- | media/filters/frame_processor_base.h | 63 | ||||
-rw-r--r-- | media/filters/frame_processor_unittest.cc | 573 | ||||
-rw-r--r-- | media/filters/legacy_frame_processor.cc | 11 | ||||
-rw-r--r-- | media/filters/pipeline_integration_test.cc | 11 | ||||
-rw-r--r-- | media/media.gyp | 3 |
11 files changed, 1184 insertions, 50 deletions
diff --git a/media/filters/chunk_demuxer.cc b/media/filters/chunk_demuxer.cc index 43aca4f..26d09b1 100644 --- a/media/filters/chunk_demuxer.cc +++ b/media/filters/chunk_demuxer.cc @@ -17,6 +17,7 @@ #include "media/base/bind_to_current_loop.h" #include "media/base/stream_parser_buffer.h" #include "media/base/video_decoder_config.h" +#include "media/filters/frame_processor.h" #include "media/filters/legacy_frame_processor.h" #include "media/filters/stream_parser_factory.h" @@ -132,6 +133,10 @@ class SourceState { // Sets |frame_processor_|'s sequence mode to |sequence_mode|. void SetSequenceMode(bool sequence_mode); + // Signals the coded frame processor to update its group start timestamp to be + // |timestamp_offset| if it is in sequence append mode. + void SetGroupStartTimestampIfInSequenceMode(base::TimeDelta timestamp_offset); + // Returns the range of buffered data in this source, capped at |duration|. // |ended| - Set to true if end of stream has been signaled and the special // end of stream range logic needs to be executed. @@ -229,6 +234,8 @@ class SourceState { // Indicates that timestampOffset should be updated automatically during // OnNewBuffers() based on the earliest end timestamp of the buffers provided. + // TODO(wolenetz): Refactor this function while integrating April 29, 2014 + // changes to MSE spec. See http://crbug.com/371499. bool auto_update_timestamp_offset_; DISALLOW_COPY_AND_ASSIGN(SourceState); @@ -286,15 +293,22 @@ void SourceState::SetSequenceMode(bool sequence_mode) { frame_processor_->SetSequenceMode(sequence_mode); } +void SourceState::SetGroupStartTimestampIfInSequenceMode( + base::TimeDelta timestamp_offset) { + DCHECK(!parsing_media_segment_); + + frame_processor_->SetGroupStartTimestampIfInSequenceMode(timestamp_offset); +} + bool SourceState::Append(const uint8* data, size_t length, TimeDelta append_window_start, TimeDelta append_window_end, TimeDelta* timestamp_offset) { DCHECK(timestamp_offset); DCHECK(!timestamp_offset_during_append_); - timestamp_offset_during_append_ = timestamp_offset; append_window_start_during_append_ = append_window_start; append_window_end_during_append_ = append_window_end; + timestamp_offset_during_append_ = timestamp_offset; // TODO(wolenetz/acolwell): Curry and pass a NewBuffersCB here bound with // append window and timestamp offset pointer. See http://crbug.com/351454. @@ -1156,14 +1170,17 @@ ChunkDemuxer::Status ChunkDemuxer::AddId( if (has_video) source_id_video_ = id; - if (!use_legacy_frame_processor) { - DLOG(WARNING) << "New frame processor is not yet supported. Using legacy."; + scoped_ptr<FrameProcessorBase> frame_processor; + if (use_legacy_frame_processor) { + frame_processor.reset(new LegacyFrameProcessor( + base::Bind(&ChunkDemuxer::IncreaseDurationIfNecessary, + base::Unretained(this)))); + } else { + frame_processor.reset(new FrameProcessor( + base::Bind(&ChunkDemuxer::IncreaseDurationIfNecessary, + base::Unretained(this)))); } - scoped_ptr<FrameProcessorBase> frame_processor(new LegacyFrameProcessor( - base::Bind(&ChunkDemuxer::IncreaseDurationIfNecessary, - base::Unretained(this)))); - scoped_ptr<SourceState> source_state( new SourceState(stream_parser.Pass(), frame_processor.Pass(), log_cb_, @@ -1389,6 +1406,20 @@ void ChunkDemuxer::SetSequenceMode(const std::string& id, source_state_map_[id]->SetSequenceMode(sequence_mode); } +void ChunkDemuxer::SetGroupStartTimestampIfInSequenceMode( + const std::string& id, + base::TimeDelta timestamp_offset) { + base::AutoLock auto_lock(lock_); + DVLOG(1) << "SetGroupStartTimestampIfInSequenceMode(" << id << ", " + << timestamp_offset.InSecondsF() << ")"; + CHECK(IsValidId(id)); + DCHECK_NE(state_, ENDED); + + source_state_map_[id]->SetGroupStartTimestampIfInSequenceMode( + timestamp_offset); +} + + void ChunkDemuxer::MarkEndOfStream(PipelineStatus status) { DVLOG(1) << "MarkEndOfStream(" << status << ")"; base::AutoLock auto_lock(lock_); @@ -1619,6 +1650,14 @@ void ChunkDemuxer::UpdateDuration(TimeDelta new_duration) { void ChunkDemuxer::IncreaseDurationIfNecessary(TimeDelta new_duration) { DCHECK(new_duration != kNoTimestamp()); + DCHECK(new_duration != kInfiniteDuration()); + + // Per April 1, 2014 MSE spec editor's draft: + // https://dvcs.w3.org/hg/html-media/raw-file/d471a4412040/media-source/ + // media-source.html#sourcebuffer-coded-frame-processing + // 5. If the media segment contains data beyond the current duration, then run + // the duration change algorithm with new duration set to the maximum of + // the current duration and the group end timestamp. if (new_duration <= duration_) return; diff --git a/media/filters/chunk_demuxer.h b/media/filters/chunk_demuxer.h index 6324d79..8078393 100644 --- a/media/filters/chunk_demuxer.h +++ b/media/filters/chunk_demuxer.h @@ -23,7 +23,7 @@ namespace media { class FFmpegURLProtocol; class SourceState; -class ChunkDemuxerStream : public DemuxerStream { +class MEDIA_EXPORT ChunkDemuxerStream : public DemuxerStream { public: typedef std::deque<scoped_refptr<StreamParserBuffer> > BufferQueue; @@ -193,10 +193,7 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer { // Registers a new |id| to use for AppendData() calls. |type| indicates // the MIME type for the data that we intend to append for this ID. // |use_legacy_frame_processor| determines which of LegacyFrameProcessor or - // a (not yet implemented) more compliant frame processor to use to process - // parsed frames from AppendData() calls. - // TODO(wolenetz): Enable usage of new frame processor based on this flag. - // See http://crbug.com/249422. + // FrameProcessor to use to process parsed frames from AppendData() calls. // kOk is returned if the demuxer has enough resources to support another ID // and supports the format indicated by |type|. // kNotSupported is returned if |type| is not a supported format. @@ -255,6 +252,12 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer { // mode. void SetSequenceMode(const std::string& id, bool sequence_mode); + // Signals the coded frame processor for the source buffer associated with + // |id| to update its group start timestamp to be |timestamp_offset| if it is + // in sequence append mode. + void SetGroupStartTimestampIfInSequenceMode(const std::string& id, + base::TimeDelta timestamp_offset); + // Called to signal changes in the "end of stream" // state. UnmarkEndOfStream() must not be called if a matching // MarkEndOfStream() has not come before it. diff --git a/media/filters/chunk_demuxer_unittest.cc b/media/filters/chunk_demuxer_unittest.cc index 6eb4eab..55815c1 100644 --- a/media/filters/chunk_demuxer_unittest.cc +++ b/media/filters/chunk_demuxer_unittest.cc @@ -134,10 +134,8 @@ static void OnSeekDone_OKExpected(bool* called, PipelineStatus status) { static void LogFunc(const std::string& str) { DVLOG(1) << str; } // Test parameter determines which coded frame processor is used to process -// appended data. If true, LegacyFrameProcessor is used. Otherwise, (not yet -// supported), a more compliant frame processor is used. -// TODO(wolenetz): Enable usage of new frame processor based on this flag. -// See http://crbug.com/249422. +// appended data. If true, LegacyFrameProcessor is used. Otherwise, the new +// FrameProcessor is used. class ChunkDemuxerTest : public ::testing::TestWithParam<bool> { protected: enum CodecsIndex { @@ -2817,12 +2815,11 @@ TEST_P(ChunkDemuxerTest, DurationChange) { // Add data beginning at the currently set duration and expect a new duration // to be signaled. Note that the last video block will have a higher end // timestamp than the last audio block. - // TODO(wolenetz): Compliant coded frame processor will emit a max of one - // duration change per each ProcessFrames(). Remove the first expectation here - // once compliant coded frame processor is used. See http://crbug.com/249422. - const int kNewStreamDurationAudio = kStreamDuration + kAudioBlockDuration; - EXPECT_CALL(host_, SetDuration( + if (use_legacy_frame_processor_) { + const int kNewStreamDurationAudio = kStreamDuration + kAudioBlockDuration; + EXPECT_CALL(host_, SetDuration( base::TimeDelta::FromMilliseconds(kNewStreamDurationAudio))); + } const int kNewStreamDurationVideo = kStreamDuration + kVideoBlockDuration; EXPECT_CALL(host_, SetDuration( base::TimeDelta::FromMilliseconds(kNewStreamDurationVideo))); @@ -2849,12 +2846,11 @@ TEST_P(ChunkDemuxerTest, DurationChangeTimestampOffset) { ASSERT_TRUE(SetTimestampOffset(kSourceId, kDefaultDuration())); - // TODO(wolenetz): Compliant coded frame processor will emit a max of one - // duration change per each ProcessFrames(). Remove the first expectation here - // once compliant coded frame processor is used. See http://crbug.com/249422. - EXPECT_CALL(host_, SetDuration( - kDefaultDuration() + base::TimeDelta::FromMilliseconds( - kAudioBlockDuration * 2))); + if (use_legacy_frame_processor_) { + EXPECT_CALL(host_, SetDuration( + kDefaultDuration() + base::TimeDelta::FromMilliseconds( + kAudioBlockDuration * 2))); + } EXPECT_CALL(host_, SetDuration( kDefaultDuration() + base::TimeDelta::FromMilliseconds( kVideoBlockDuration * 2))); @@ -3255,8 +3251,9 @@ TEST_P(ChunkDemuxerTest, SeekCompletesWithoutTextCues) { CheckExpectedBuffers(video_stream, "180 210"); } -// TODO(wolenetz): Enable testing of new frame processor based on this flag, -// once the new processor has landed. See http://crbug.com/249422. +// Generate two sets of tests: one using FrameProcessor, and one using +// LegacyFrameProcessor. +INSTANTIATE_TEST_CASE_P(NewFrameProcessor, ChunkDemuxerTest, Values(false)); INSTANTIATE_TEST_CASE_P(LegacyFrameProcessor, ChunkDemuxerTest, Values(true)); } // namespace media diff --git a/media/filters/frame_processor.cc b/media/filters/frame_processor.cc new file mode 100644 index 0000000..7f83a22 --- /dev/null +++ b/media/filters/frame_processor.cc @@ -0,0 +1,378 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/filters/frame_processor.h" + +#include "base/stl_util.h" +#include "media/base/buffers.h" +#include "media/base/stream_parser_buffer.h" + +namespace media { + +FrameProcessor::FrameProcessor(const UpdateDurationCB& update_duration_cb) + : update_duration_cb_(update_duration_cb) { + DVLOG(2) << __FUNCTION__ << "()"; + DCHECK(!update_duration_cb.is_null()); +} + +FrameProcessor::~FrameProcessor() { + DVLOG(2) << __FUNCTION__; +} + +void FrameProcessor::SetSequenceMode(bool sequence_mode) { + DVLOG(2) << __FUNCTION__ << "(" << sequence_mode << ")"; + + // Per April 1, 2014 MSE spec editor's draft: + // https://dvcs.w3.org/hg/html-media/raw-file/d471a4412040/media-source/media-source.html#widl-SourceBuffer-mode + // Step 7: If the new mode equals "sequence", then set the group start + // timestamp to the group end timestamp. + if (sequence_mode) { + DCHECK(kNoTimestamp() != group_end_timestamp_); + group_start_timestamp_ = group_end_timestamp_; + } + + // Step 8: Update the attribute to new mode. + sequence_mode_ = sequence_mode; +} + +bool FrameProcessor::ProcessFrames( + const StreamParser::BufferQueue& audio_buffers, + const StreamParser::BufferQueue& video_buffers, + const StreamParser::TextBufferQueueMap& text_map, + base::TimeDelta append_window_start, + base::TimeDelta append_window_end, + bool* new_media_segment, + base::TimeDelta* timestamp_offset) { + StreamParser::BufferQueue frames; + if (!MergeBufferQueues(audio_buffers, video_buffers, text_map, &frames)) { + DVLOG(2) << "Parse error discovered while merging parser's buffers"; + return false; + } + + DCHECK(!frames.empty()); + + // Implements the coded frame processing algorithm's outer loop for step 1. + // Note that ProcessFrame() implements an inner loop for a single frame that + // handles "jump to the Loop Top step to restart processing of the current + // coded frame" per April 1, 2014 MSE spec editor's draft: + // https://dvcs.w3.org/hg/html-media/raw-file/d471a4412040/media-source/ + // media-source.html#sourcebuffer-coded-frame-processing + // 1. For each coded frame in the media segment run the following steps: + for (StreamParser::BufferQueue::const_iterator frames_itr = frames.begin(); + frames_itr != frames.end(); ++frames_itr) { + if (!ProcessFrame(*frames_itr, append_window_start, append_window_end, + timestamp_offset, new_media_segment)) { + return false; + } + } + + // 2. - 4. Are handled by the WebMediaPlayer / Pipeline / Media Element. + + // Step 5: + update_duration_cb_.Run(group_end_timestamp_); + + return true; +} + +bool FrameProcessor::ProcessFrame(scoped_refptr<StreamParserBuffer> frame, + base::TimeDelta append_window_start, + base::TimeDelta append_window_end, + base::TimeDelta* timestamp_offset, + bool* new_media_segment) { + // Implements the loop within step 1 of the coded frame processing algorithm + // for a single input frame per April 1, 2014 MSE spec editor's draft: + // https://dvcs.w3.org/hg/html-media/raw-file/d471a4412040/media-source/ + // media-source.html#sourcebuffer-coded-frame-processing + + while (true) { + // 1. Loop Top: Let presentation timestamp be a double precision floating + // point representation of the coded frame's presentation timestamp in + // seconds. + // 2. Let decode timestamp be a double precision floating point + // representation of the coded frame's decode timestamp in seconds. + // 3. Let frame duration be a double precision floating point representation + // of the coded frame's duration in seconds. + // We use base::TimeDelta instead of double. + base::TimeDelta presentation_timestamp = frame->timestamp(); + base::TimeDelta decode_timestamp = frame->GetDecodeTimestamp(); + base::TimeDelta frame_duration = frame->duration(); + + DVLOG(3) << __FUNCTION__ << ": Processing frame " + << "Type=" << frame->type() + << ", TrackID=" << frame->track_id() + << ", PTS=" << presentation_timestamp.InSecondsF() + << ", DTS=" << decode_timestamp.InSecondsF() + << ", DUR=" << frame_duration.InSecondsF(); + + // Sanity check the timestamps. + if (presentation_timestamp < base::TimeDelta()) { + DVLOG(2) << __FUNCTION__ << ": Negative or unknown frame PTS: " + << presentation_timestamp.InSecondsF(); + return false; + } + if (decode_timestamp < base::TimeDelta()) { + DVLOG(2) << __FUNCTION__ << ": Negative or unknown frame DTS: " + << decode_timestamp.InSecondsF(); + return false; + } + if (decode_timestamp > presentation_timestamp) { + // TODO(wolenetz): Determine whether DTS>PTS should really be allowed. See + // http://crbug.com/354518. + DVLOG(2) << __FUNCTION__ << ": WARNING: Frame DTS(" + << decode_timestamp.InSecondsF() << ") > PTS(" + << presentation_timestamp.InSecondsF() << ")"; + } + + // TODO(acolwell/wolenetz): All stream parsers must emit valid (positive) + // frame durations. For now, we allow non-negative frame duration. + // See http://crbug.com/351166. + if (frame_duration == kNoTimestamp()) { + DVLOG(2) << __FUNCTION__ << ": Frame missing duration (kNoTimestamp())"; + return false; + } + if (frame_duration < base::TimeDelta()) { + DVLOG(2) << __FUNCTION__ << ": Frame duration negative: " + << frame_duration.InSecondsF(); + return false; + } + + // 4. If mode equals "sequence" and group start timestamp is set, then run + // the following steps: + if (sequence_mode_ && group_start_timestamp_ != kNoTimestamp()) { + // 4.1. Set timestampOffset equal to group start timestamp - + // presentation timestamp. + *timestamp_offset = group_start_timestamp_ - presentation_timestamp; + + DVLOG(3) << __FUNCTION__ << ": updated timestampOffset is now " + << timestamp_offset->InSecondsF(); + + // 4.2. Set group end timestamp equal to group start timestamp. + group_end_timestamp_ = group_start_timestamp_; + + // 4.3. Set the need random access point flag on all track buffers to + // true. + SetAllTrackBuffersNeedRandomAccessPoint(); + + // 4.4. Unset group start timestamp. + group_start_timestamp_ = kNoTimestamp(); + } + + // 5. If timestampOffset is not 0, then run the following steps: + if (*timestamp_offset != base::TimeDelta()) { + // 5.1. Add timestampOffset to the presentation timestamp. + // Note: |frame| PTS is only updated if it survives processing. + presentation_timestamp += *timestamp_offset; + + // 5.2. Add timestampOffset to the decode timestamp. + // Frame DTS is only updated if it survives processing. + decode_timestamp += *timestamp_offset; + } + + // 6. Let track buffer equal the track buffer that the coded frame will be + // added to. + + // Remap audio and video track types to their special singleton identifiers. + StreamParser::TrackId track_id = kAudioTrackId; + switch (frame->type()) { + case DemuxerStream::AUDIO: + break; + case DemuxerStream::VIDEO: + track_id = kVideoTrackId; + break; + case DemuxerStream::TEXT: + track_id = frame->track_id(); + break; + case DemuxerStream::UNKNOWN: + case DemuxerStream::NUM_TYPES: + DCHECK(false) << ": Invalid frame type " << frame->type(); + return false; + } + + MseTrackBuffer* track_buffer = FindTrack(track_id); + if (!track_buffer) { + DVLOG(2) << __FUNCTION__ << ": Unknown track: type=" << frame->type() + << ", frame processor track id=" << track_id + << ", parser track id=" << frame->track_id(); + return false; + } + + // 7. If last decode timestamp for track buffer is set and decode timestamp + // is less than last decode timestamp + // OR + // If last decode timestamp for track buffer is set and the difference + // between decode timestamp and last decode timestamp is greater than 2 + // times last frame duration: + base::TimeDelta last_decode_timestamp = + track_buffer->last_decode_timestamp(); + if (last_decode_timestamp != kNoTimestamp()) { + base::TimeDelta dts_delta = decode_timestamp - last_decode_timestamp; + if (dts_delta < base::TimeDelta() || + dts_delta > 2 * track_buffer->last_frame_duration()) { + // 7.1. If mode equals "segments": Set group end timestamp to + // presentation timestamp. + // If mode equals "sequence": Set group start timestamp equal to + // the group end timestamp. + if (!sequence_mode_) { + group_end_timestamp_ = presentation_timestamp; + // This triggers a discontinuity so we need to treat the next frames + // appended within the append window as if they were the beginning of + // a new segment. + *new_media_segment = true; + } else { + DVLOG(3) << __FUNCTION__ << " : Sequence mode discontinuity, GETS: " + << group_end_timestamp_.InSecondsF(); + DCHECK(kNoTimestamp() != group_end_timestamp_); + group_start_timestamp_ = group_end_timestamp_; + } + + // 7.2. - 7.5.: + Reset(); + + // 7.6. Jump to the Loop Top step above to restart processing of the + // current coded frame. + DVLOG(3) << __FUNCTION__ << ": Discontinuity: reprocessing frame"; + continue; + } + } + + // 8. If the presentation timestamp or decode timestamp is less than the + // presentation start time, then run the end of stream algorithm with the + // error parameter set to "decode", and abort these steps. + if (presentation_timestamp < base::TimeDelta() || + decode_timestamp < base::TimeDelta()) { + DVLOG(2) << __FUNCTION__ + << ": frame PTS=" << presentation_timestamp.InSecondsF() + << " or DTS=" << decode_timestamp.InSecondsF() + << " negative after applying timestampOffset and handling any " + << " discontinuity"; + return false; + } + + // 9. Let frame end timestamp equal the sum of presentation timestamp and + // frame duration. + base::TimeDelta frame_end_timestamp = presentation_timestamp + + frame_duration; + + // 10. If presentation timestamp is less than appendWindowStart, then set + // the need random access point flag to true, drop the coded frame, and + // jump to the top of the loop to start processing the next coded + // frame. + // Note: We keep the result of partial discard of a buffer that overlaps + // |append_window_start| and does not end after |append_window_end|. + // 11. If frame end timestamp is greater than appendWindowEnd, then set the + // need random access point flag to true, drop the coded frame, and jump + // to the top of the loop to start processing the next coded frame. + if (presentation_timestamp < append_window_start || + frame_end_timestamp > append_window_end) { + // See if a partial discard can be done around |append_window_start|. + // TODO(wolenetz): Refactor this into a base helper across legacy and + // new frame processors? + if (track_buffer->stream()->supports_partial_append_window_trimming() && + presentation_timestamp < append_window_start && + frame_end_timestamp > append_window_start && + frame_end_timestamp <= append_window_end) { + DCHECK(frame->IsKeyframe()); + DVLOG(1) << "Truncating buffer which overlaps append window start." + << " presentation_timestamp " + << presentation_timestamp.InSecondsF() + << " append_window_start " << append_window_start.InSecondsF(); + + // Adjust the timestamp of this frame forward to |append_window_start|, + // while decreasing the duration appropriately. + frame->set_discard_padding(std::make_pair( + append_window_start - presentation_timestamp, base::TimeDelta())); + presentation_timestamp = append_window_start; // |frame| updated below. + decode_timestamp = append_window_start; // |frame| updated below. + frame_duration = frame_end_timestamp - presentation_timestamp; + frame->set_duration(frame_duration); + + // TODO(dalecurtis): This could also be done with |append_window_end|, + // but is not necessary since splice frames covert the overlap there. + } else { + track_buffer->set_needs_random_access_point(true); + DVLOG(3) << "Dropping frame that is outside append window."; + + if (!sequence_mode_) { + // This also triggers a discontinuity so we need to treat the next + // frames appended within the append window as if they were the + // beginning of a new segment. + *new_media_segment = true; + } + + return true; + } + } + + // 12. If the need random access point flag on track buffer equals true, + // then run the following steps: + if (track_buffer->needs_random_access_point()) { + // 12.1. If the coded frame is not a random access point, then drop the + // coded frame and jump to the top of the loop to start processing + // the next coded frame. + if (!frame->IsKeyframe()) { + DVLOG(3) << __FUNCTION__ + << ": Dropping frame that is not a random access point"; + return true; + } + + // 12.2. Set the need random access point flag on track buffer to false. + track_buffer->set_needs_random_access_point(false); + } + + // We now have a processed buffer to append to the track buffer's stream. + // If it is the first in a new media segment or following a discontinuity, + // notify all the track buffers' streams that a new segment is beginning. + if (*new_media_segment) { + *new_media_segment = false; + NotifyNewMediaSegmentStarting(decode_timestamp); + } + + DVLOG(3) << __FUNCTION__ << ": Sending processed frame to stream, " + << "PTS=" << presentation_timestamp.InSecondsF() + << ", DTS=" << decode_timestamp.InSecondsF(); + frame->set_timestamp(presentation_timestamp); + frame->SetDecodeTimestamp(decode_timestamp); + + // Steps 13-18: + // TODO(wolenetz): Collect and emit more than one buffer at a time, if + // possible. Also refactor SourceBufferStream to conform to spec GC timing. + // See http://crbug.com/371197. + StreamParser::BufferQueue buffer_to_append; + buffer_to_append.push_back(frame); + track_buffer->stream()->Append(buffer_to_append); + + // 19. Set last decode timestamp for track buffer to decode timestamp. + track_buffer->set_last_decode_timestamp(decode_timestamp); + + // 20. Set last frame duration for track buffer to frame duration. + track_buffer->set_last_frame_duration(frame_duration); + + // 21. If highest presentation timestamp for track buffer is unset or frame + // end timestamp is greater than highest presentation timestamp, then + // set highest presentation timestamp for track buffer to frame end + // timestamp. + track_buffer->SetHighestPresentationTimestampIfIncreased( + frame_end_timestamp); + + // 22. If frame end timestamp is greater than group end timestamp, then set + // group end timestamp equal to frame end timestamp. + DCHECK(group_end_timestamp_ >= base::TimeDelta()); + if (frame_end_timestamp > group_end_timestamp_) + group_end_timestamp_ = frame_end_timestamp; + + return true; + } + + NOTREACHED(); + return false; +} + +void FrameProcessor::SetAllTrackBuffersNeedRandomAccessPoint() { + for (TrackBufferMap::iterator itr = track_buffers_.begin(); + itr != track_buffers_.end(); ++itr) { + itr->second->set_needs_random_access_point(true); + } +} + +} // namespace media diff --git a/media/filters/frame_processor.h b/media/filters/frame_processor.h new file mode 100644 index 0000000..54505e1 --- /dev/null +++ b/media/filters/frame_processor.h @@ -0,0 +1,60 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_FILTERS_FRAME_PROCESSOR_H_ +#define MEDIA_FILTERS_FRAME_PROCESSOR_H_ + +#include "base/basictypes.h" +#include "base/callback_forward.h" +#include "base/time/time.h" +#include "media/base/media_export.h" +#include "media/base/stream_parser.h" +#include "media/filters/frame_processor_base.h" + +namespace media { + +// Helper class that implements Media Source Extension's coded frame processing +// algorithm. +class MEDIA_EXPORT FrameProcessor : public FrameProcessorBase { + public: + typedef base::Callback<void(base::TimeDelta)> UpdateDurationCB; + explicit FrameProcessor(const UpdateDurationCB& update_duration_cb); + virtual ~FrameProcessor(); + + // FrameProcessorBase implementation + virtual void SetSequenceMode(bool sequence_mode) OVERRIDE; + virtual bool ProcessFrames(const StreamParser::BufferQueue& audio_buffers, + const StreamParser::BufferQueue& video_buffers, + const StreamParser::TextBufferQueueMap& text_map, + base::TimeDelta append_window_start, + base::TimeDelta append_window_end, + bool* new_media_segment, + base::TimeDelta* timestamp_offset) OVERRIDE; + + private: + // Helper that processes one frame with the coded frame processing algorithm. + // Returns false on error or true on success. + bool ProcessFrame(scoped_refptr<StreamParserBuffer> frame, + base::TimeDelta append_window_start, + base::TimeDelta append_window_end, + base::TimeDelta* timestamp_offset, + bool* new_media_segment); + + // Sets the need random access point flag on all track buffers to true. + void SetAllTrackBuffersNeedRandomAccessPoint(); + + // Tracks the MSE coded frame processing variable of same name. It stores the + // highest coded frame end timestamp across all coded frames in the current + // coded frame group. It is set to 0 when the SourceBuffer object is created + // and gets updated by ProcessFrames(). + base::TimeDelta group_end_timestamp_; + + UpdateDurationCB update_duration_cb_; + + DISALLOW_COPY_AND_ASSIGN(FrameProcessor); +}; + +} // namespace media + +#endif // MEDIA_FILTERS_FRAME_PROCESSOR_H_ diff --git a/media/filters/frame_processor_base.cc b/media/filters/frame_processor_base.cc index 3e088ed..09b1b86 100644 --- a/media/filters/frame_processor_base.cc +++ b/media/filters/frame_processor_base.cc @@ -10,7 +10,10 @@ namespace media { MseTrackBuffer::MseTrackBuffer(ChunkDemuxerStream* stream) - : needs_random_access_point_(true), + : last_decode_timestamp_(kNoTimestamp()), + last_frame_duration_(kNoTimestamp()), + highest_presentation_timestamp_(kNoTimestamp()), + needs_random_access_point_(true), stream_(stream) { DCHECK(stream_); } @@ -22,11 +25,23 @@ MseTrackBuffer::~MseTrackBuffer() { void MseTrackBuffer::Reset() { DVLOG(2) << __FUNCTION__ << "()"; + last_decode_timestamp_ = kNoTimestamp(); + last_frame_duration_ = kNoTimestamp(); + highest_presentation_timestamp_ = kNoTimestamp(); needs_random_access_point_ = true; } +void MseTrackBuffer::SetHighestPresentationTimestampIfIncreased( + base::TimeDelta timestamp) { + if (highest_presentation_timestamp_ == kNoTimestamp() || + timestamp > highest_presentation_timestamp_) { + highest_presentation_timestamp_ = timestamp; + } +} + FrameProcessorBase::FrameProcessorBase() - : sequence_mode_(false) {} + : sequence_mode_(false), + group_start_timestamp_(kNoTimestamp()) {} FrameProcessorBase::~FrameProcessorBase() { DVLOG(2) << __FUNCTION__ << "()"; @@ -34,6 +49,14 @@ FrameProcessorBase::~FrameProcessorBase() { STLDeleteValues(&track_buffers_); } +void FrameProcessorBase::SetGroupStartTimestampIfInSequenceMode( + base::TimeDelta timestamp_offset) { + DVLOG(2) << __FUNCTION__ << "(" << timestamp_offset.InSecondsF() << ")"; + DCHECK(kNoTimestamp() != timestamp_offset); + if (sequence_mode_) + group_start_timestamp_ = timestamp_offset; +} + bool FrameProcessorBase::AddTrack(StreamParser::TrackId id, ChunkDemuxerStream* stream) { DVLOG(2) << __FUNCTION__ << "(): id=" << id; @@ -63,4 +86,15 @@ MseTrackBuffer* FrameProcessorBase::FindTrack(StreamParser::TrackId id) { return itr->second; } +void FrameProcessorBase::NotifyNewMediaSegmentStarting( + base::TimeDelta segment_timestamp) { + DVLOG(2) << __FUNCTION__ << "(" << segment_timestamp.InSecondsF() << ")"; + + for (TrackBufferMap::iterator itr = track_buffers_.begin(); + itr != track_buffers_.end(); + ++itr) { + itr->second->stream()->OnNewMediaSegment(segment_timestamp); + } +} + } // namespace media diff --git a/media/filters/frame_processor_base.h b/media/filters/frame_processor_base.h index 8841998..cc143e2 100644 --- a/media/filters/frame_processor_base.h +++ b/media/filters/frame_processor_base.h @@ -25,6 +25,27 @@ class MseTrackBuffer { explicit MseTrackBuffer(ChunkDemuxerStream* stream); ~MseTrackBuffer(); + // Get/set |last_decode_timestamp_|. + base::TimeDelta last_decode_timestamp() const { + return last_decode_timestamp_; + } + void set_last_decode_timestamp(base::TimeDelta timestamp) { + last_decode_timestamp_ = timestamp; + } + + // Get/set |last_frame_duration_|. + base::TimeDelta last_frame_duration() const { + return last_frame_duration_; + } + void set_last_frame_duration(base::TimeDelta duration) { + last_frame_duration_ = duration; + } + + // Gets |highest_presentation_timestamp_|. + base::TimeDelta highest_presentation_timestamp() const { + return highest_presentation_timestamp_; + } + // Get/set |needs_random_access_point_|. bool needs_random_access_point() const { return needs_random_access_point_; @@ -36,12 +57,33 @@ class MseTrackBuffer { // Gets a pointer to this track's ChunkDemuxerStream. ChunkDemuxerStream* stream() const { return stream_; } - // Sets |needs_random_access_point_| to true. - // TODO(wolenetz): Add the rest of the new coded frame processing algorithm - // track buffer attributes and reset them here. See http://crbug.com/249422. + // Unsets |last_decode_timestamp_|, unsets |last_frame_duration_|, + // unsets |highest_presentation_timestamp_|, and sets + // |needs_random_access_point_| to true. void Reset(); + // If |highest_presentation_timestamp_| is unset or |timestamp| is greater + // than |highest_presentation_timestamp_|, sets + // |highest_presentation_timestamp_| to |timestamp|. Note that bidirectional + // prediction between coded frames can cause |timestamp| to not be + // monotonically increasing even though the decode timestamps are + // monotonically increasing. + void SetHighestPresentationTimestampIfIncreased(base::TimeDelta timestamp); + private: + // The decode timestamp of the last coded frame appended in the current coded + // frame group. Initially kNoTimestamp(), meaning "unset". + base::TimeDelta last_decode_timestamp_; + + // The coded frame duration of the last coded frame appended in the current + // coded frame group. Initially kNoTimestamp(), meaning "unset". + base::TimeDelta last_frame_duration_; + + // The highest presentation timestamp encountered in a coded frame appended + // in the current coded frame group. Initially kNoTimestamp(), meaning + // "unset". + base::TimeDelta highest_presentation_timestamp_; + // Keeps track of whether the track buffer is waiting for a random access // point coded frame. Initially set to true to indicate that a random access // point coded frame is needed before anything can be added to the track @@ -97,6 +139,10 @@ class MEDIA_EXPORT FrameProcessorBase { bool* new_media_segment, base::TimeDelta* timestamp_offset) = 0; + // Signals the frame processor to update its group start timestamp to be + // |timestamp_offset| if it is in sequence append mode. + void SetGroupStartTimestampIfInSequenceMode(base::TimeDelta timestamp_offset); + // Adds a new track with unique track ID |id|. // If |id| has previously been added, returns false to indicate error. // Otherwise, returns true, indicating future ProcessFrames() will emit @@ -117,6 +163,10 @@ class MEDIA_EXPORT FrameProcessorBase { // MseTrackBuffer. Otherwise, returns NULL. MseTrackBuffer* FindTrack(StreamParser::TrackId id); + // Signals all track buffers' streams that a new media segment is starting + // with timestamp |segment_timestamp|. + void NotifyNewMediaSegmentStarting(base::TimeDelta segment_timestamp); + // The AppendMode of the associated SourceBuffer. // See SetSequenceMode() for interpretation of |sequence_mode_|. // Per http://www.w3.org/TR/media-source/#widl-SourceBuffer-mode: @@ -126,6 +176,13 @@ class MEDIA_EXPORT FrameProcessorBase { // TrackId-indexed map of each track's stream. TrackBufferMap track_buffers_; + + // Tracks the MSE coded frame processing variable of same name. + // Initially kNoTimestamp(), meaning "unset". + // Note: LegacyFrameProcessor does not use this member; it's here to reduce + // short-term plumbing of SetGroupStartTimestampIfInSequenceMode() until + // LegacyFrameProcessor is removed. + base::TimeDelta group_start_timestamp_; }; } // namespace media diff --git a/media/filters/frame_processor_unittest.cc b/media/filters/frame_processor_unittest.cc new file mode 100644 index 0000000..6061bfe --- /dev/null +++ b/media/filters/frame_processor_unittest.cc @@ -0,0 +1,573 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <map> +#include <string> + +#include "base/bind.h" +#include "base/message_loop/message_loop.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_split.h" +#include "base/strings/string_util.h" +#include "base/time/time.h" +#include "media/base/mock_filters.h" +#include "media/base/test_helpers.h" +#include "media/filters/chunk_demuxer.h" +#include "media/filters/frame_processor.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::InSequence; +using ::testing::StrictMock; +using ::testing::Values; + +namespace media { + +typedef StreamParser::BufferQueue BufferQueue; +typedef StreamParser::TextBufferQueueMap TextBufferQueueMap; +typedef StreamParser::TrackId TrackId; + +static void LogFunc(const std::string& str) { DVLOG(1) << str; } + +// Used for setting expectations on callbacks. Using a StrictMock also lets us +// test for missing or extra callbacks. +class FrameProcessorTestCallbackHelper { + public: + FrameProcessorTestCallbackHelper() {} + virtual ~FrameProcessorTestCallbackHelper() {} + + MOCK_METHOD1(PossibleDurationIncrease, void(base::TimeDelta new_duration)); + + // Helper that calls the mock method as well as does basic sanity checks on + // |new_duration|. + void OnPossibleDurationIncrease(base::TimeDelta new_duration) { + PossibleDurationIncrease(new_duration); + ASSERT_NE(kNoTimestamp(), new_duration); + ASSERT_NE(kInfiniteDuration(), new_duration); + } + + private: + DISALLOW_COPY_AND_ASSIGN(FrameProcessorTestCallbackHelper); +}; + +// Test parameter determines indicates if the TEST_P instance is targeted for +// sequence mode (if true), or segments mode (if false). +class FrameProcessorTest : public testing::TestWithParam<bool> { + protected: + FrameProcessorTest() + : frame_processor_(new FrameProcessor(base::Bind( + &FrameProcessorTestCallbackHelper::OnPossibleDurationIncrease, + base::Unretained(&callbacks_)))), + append_window_end_(kInfiniteDuration()), + new_media_segment_(false), + audio_id_(FrameProcessor::kAudioTrackId), + video_id_(FrameProcessor::kVideoTrackId), + frame_duration_(base::TimeDelta::FromMilliseconds(10)) { + } + + enum StreamFlags { + HAS_AUDIO = 1 << 0, + HAS_VIDEO = 1 << 1 + }; + + void AddTestTracks(int stream_flags) { + const bool has_audio = (stream_flags & HAS_AUDIO) != 0; + const bool has_video = (stream_flags & HAS_VIDEO) != 0; + ASSERT_TRUE(has_audio || has_video); + + if (has_audio) { + CreateAndConfigureStream(DemuxerStream::AUDIO); + ASSERT_TRUE(audio_); + EXPECT_TRUE(frame_processor_->AddTrack(audio_id_, audio_.get())); + audio_->Seek(base::TimeDelta()); + audio_->StartReturningData(); + } + if (has_video) { + CreateAndConfigureStream(DemuxerStream::VIDEO); + ASSERT_TRUE(video_); + EXPECT_TRUE(frame_processor_->AddTrack(video_id_, video_.get())); + video_->Seek(base::TimeDelta()); + video_->StartReturningData(); + } + } + + void SetTimestampOffset(base::TimeDelta new_offset) { + timestamp_offset_ = new_offset; + frame_processor_->SetGroupStartTimestampIfInSequenceMode(timestamp_offset_); + } + + BufferQueue StringToBufferQueue(const std::string& buffers_to_append, + const TrackId track_id, + const DemuxerStream::Type type) { + std::vector<std::string> timestamps; + base::SplitString(buffers_to_append, ' ', ×tamps); + + BufferQueue buffers; + for (size_t i = 0; i < timestamps.size(); i++) { + bool is_keyframe = false; + if (EndsWith(timestamps[i], "K", true)) { + is_keyframe = true; + // Remove the "K" off of the token. + timestamps[i] = timestamps[i].substr(0, timestamps[i].length() - 1); + } + + int time_in_ms; + CHECK(base::StringToInt(timestamps[i], &time_in_ms)); + + // Create buffer. Encode the original time_in_ms as the buffer's data to + // enable later verification of possible buffer relocation in presentation + // timeline due to coded frame processing. + const uint8* timestamp_as_data = reinterpret_cast<uint8*>(&time_in_ms); + scoped_refptr<StreamParserBuffer> buffer = + StreamParserBuffer::CopyFrom(timestamp_as_data, sizeof(time_in_ms), + is_keyframe, type, track_id); + base::TimeDelta timestamp = + base::TimeDelta::FromMilliseconds(time_in_ms); + buffer->set_timestamp(timestamp); + buffer->SetDecodeTimestamp(timestamp); + buffer->set_duration(frame_duration_); + buffers.push_back(buffer); + } + return buffers; + } + + void ProcessFrames(const std::string& audio_timestamps, + const std::string& video_timestamps) { + ASSERT_TRUE(frame_processor_->ProcessFrames( + StringToBufferQueue(audio_timestamps, audio_id_, DemuxerStream::AUDIO), + StringToBufferQueue(video_timestamps, video_id_, DemuxerStream::VIDEO), + empty_text_buffers_, + append_window_start_, append_window_end_, + &new_media_segment_, ×tamp_offset_)); + } + + void CheckExpectedRangesByTimestamp(ChunkDemuxerStream* stream, + const std::string& expected) { + // Note, DemuxerStream::TEXT streams return [0,duration (==infinity here)) + Ranges<base::TimeDelta> r = stream->GetBufferedRanges(kInfiniteDuration()); + + std::stringstream ss; + ss << "{ "; + for (size_t i = 0; i < r.size(); ++i) { + int64 start = r.start(i).InMilliseconds(); + int64 end = r.end(i).InMilliseconds(); + ss << "[" << start << "," << end << ") "; + } + ss << "}"; + EXPECT_EQ(expected, ss.str()); + } + + void CheckReadStalls(ChunkDemuxerStream* stream) { + int loop_count = 0; + + do { + read_callback_called_ = false; + stream->Read(base::Bind(&FrameProcessorTest::StoreStatusAndBuffer, + base::Unretained(this))); + message_loop_.RunUntilIdle(); + } while (++loop_count < 2 && read_callback_called_ && + last_read_status_ == DemuxerStream::kAborted); + + ASSERT_FALSE(read_callback_called_ && + last_read_status_ == DemuxerStream::kAborted) + << "2 kAborted reads in a row. Giving up."; + EXPECT_FALSE(read_callback_called_); + } + + // Format of |expected| is a space-delimited sequence of + // timestamp_in_ms:original_timestamp_in_ms + // original_timestamp_in_ms (and the colon) must be omitted if it is the same + // as timestamp_in_ms. + void CheckReadsThenReadStalls(ChunkDemuxerStream* stream, + const std::string& expected) { + std::vector<std::string> timestamps; + base::SplitString(expected, ' ', ×tamps); + std::stringstream ss; + for (size_t i = 0; i < timestamps.size(); ++i) { + int loop_count = 0; + + do { + read_callback_called_ = false; + stream->Read(base::Bind(&FrameProcessorTest::StoreStatusAndBuffer, + base::Unretained(this))); + message_loop_.RunUntilIdle(); + EXPECT_TRUE(read_callback_called_); + } while (++loop_count < 2 && + last_read_status_ == DemuxerStream::kAborted); + + ASSERT_FALSE(last_read_status_ == DemuxerStream::kAborted) + << "2 kAborted reads in a row. Giving up."; + EXPECT_EQ(DemuxerStream::kOk, last_read_status_); + EXPECT_FALSE(last_read_buffer_->end_of_stream()); + + if (i > 0) + ss << " "; + + int time_in_ms = last_read_buffer_->timestamp().InMilliseconds(); + ss << time_in_ms; + + // Decode the original_time_in_ms from the buffer's data. + int original_time_in_ms; + ASSERT_EQ(static_cast<int>(sizeof(original_time_in_ms)), + last_read_buffer_->data_size()); + original_time_in_ms = *(reinterpret_cast<const int*>( + last_read_buffer_->data())); + if (original_time_in_ms != time_in_ms) + ss << ":" << original_time_in_ms; + } + + EXPECT_EQ(expected, ss.str()); + CheckReadStalls(stream); + } + + base::MessageLoop message_loop_; + StrictMock<FrameProcessorTestCallbackHelper> callbacks_; + + scoped_ptr<FrameProcessor> frame_processor_; + base::TimeDelta append_window_start_; + base::TimeDelta append_window_end_; + bool new_media_segment_; + base::TimeDelta timestamp_offset_; + scoped_ptr<ChunkDemuxerStream> audio_; + scoped_ptr<ChunkDemuxerStream> video_; + const TrackId audio_id_; + const TrackId video_id_; + const base::TimeDelta frame_duration_; // Currently the same for all streams. + const BufferQueue empty_queue_; + const TextBufferQueueMap empty_text_buffers_; + + // StoreStatusAndBuffer's most recent result. + DemuxerStream::Status last_read_status_; + scoped_refptr<DecoderBuffer> last_read_buffer_; + bool read_callback_called_; + + private: + void StoreStatusAndBuffer(DemuxerStream::Status status, + const scoped_refptr<DecoderBuffer>& buffer) { + if (status == DemuxerStream::kOk && buffer) { + DVLOG(3) << __FUNCTION__ << "status: " << status << " ts: " + << buffer->timestamp().InSecondsF(); + } else { + DVLOG(3) << __FUNCTION__ << "status: " << status << " ts: n/a"; + } + + read_callback_called_ = true; + last_read_status_ = status; + last_read_buffer_ = buffer; + } + + void CreateAndConfigureStream(DemuxerStream::Type type) { + // TODO(wolenetz/dalecurtis): Also test with splicing disabled? + switch (type) { + case DemuxerStream::AUDIO: + ASSERT_FALSE(audio_); + audio_.reset(new ChunkDemuxerStream(DemuxerStream::AUDIO, true)); + ASSERT_TRUE(audio_->UpdateAudioConfig( + AudioDecoderConfig(kCodecVorbis, + kSampleFormatPlanarF32, + CHANNEL_LAYOUT_STEREO, + 1000, + NULL, + 0, + false), + base::Bind(&LogFunc))); + break; + case DemuxerStream::VIDEO: + ASSERT_FALSE(video_); + video_.reset(new ChunkDemuxerStream(DemuxerStream::VIDEO, true)); + ASSERT_TRUE(video_->UpdateVideoConfig(TestVideoConfig::Normal(), + base::Bind(&LogFunc))); + break; + // TODO(wolenetz): Test text coded frame processing. + case DemuxerStream::TEXT: + case DemuxerStream::UNKNOWN: + case DemuxerStream::NUM_TYPES: + ASSERT_FALSE(true); + } + } + + DISALLOW_COPY_AND_ASSIGN(FrameProcessorTest); +}; + +TEST_F(FrameProcessorTest, WrongTypeInAppendedBuffer) { + AddTestTracks(HAS_AUDIO); + new_media_segment_ = true; + + ASSERT_FALSE(frame_processor_->ProcessFrames( + StringToBufferQueue("0K", audio_id_, DemuxerStream::VIDEO), + empty_queue_, + empty_text_buffers_, + append_window_start_, append_window_end_, + &new_media_segment_, ×tamp_offset_)); + EXPECT_TRUE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ }"); + CheckReadStalls(audio_.get()); +} + +TEST_F(FrameProcessorTest, NonMonotonicallyIncreasingTimestampInOneCall) { + AddTestTracks(HAS_AUDIO); + new_media_segment_ = true; + + ASSERT_FALSE(frame_processor_->ProcessFrames( + StringToBufferQueue("10K 0K", audio_id_, DemuxerStream::AUDIO), + empty_queue_, + empty_text_buffers_, + append_window_start_, append_window_end_, + &new_media_segment_, ×tamp_offset_)); + EXPECT_TRUE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ }"); + CheckReadStalls(audio_.get()); +} + +TEST_P(FrameProcessorTest, AudioOnly_SingleFrame) { + // Tests A: P(A) -> (a) + InSequence s; + AddTestTracks(HAS_AUDIO); + new_media_segment_ = true; + if (GetParam()) + frame_processor_->SetSequenceMode(true); + + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_)); + ProcessFrames("0K", ""); + EXPECT_FALSE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,10) }"); + CheckReadsThenReadStalls(audio_.get(), "0"); +} + +TEST_P(FrameProcessorTest, VideoOnly_SingleFrame) { + // Tests V: P(V) -> (v) + InSequence s; + AddTestTracks(HAS_VIDEO); + new_media_segment_ = true; + if (GetParam()) + frame_processor_->SetSequenceMode(true); + + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_)); + ProcessFrames("", "0K"); + EXPECT_FALSE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(video_.get(), "{ [0,10) }"); + CheckReadsThenReadStalls(video_.get(), "0"); +} + +TEST_P(FrameProcessorTest, AudioOnly_TwoFrames) { + // Tests A: P(A0, A10) -> (a0, a10) + InSequence s; + AddTestTracks(HAS_AUDIO); + new_media_segment_ = true; + if (GetParam()) + frame_processor_->SetSequenceMode(true); + + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 2)); + ProcessFrames("0K 10K", ""); + EXPECT_FALSE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,20) }"); + CheckReadsThenReadStalls(audio_.get(), "0 10"); +} + +TEST_P(FrameProcessorTest, AudioOnly_SetOffsetThenSingleFrame) { + // Tests A: STSO(50)+P(A0) -> TSO==50,(a0@50) + InSequence s; + AddTestTracks(HAS_AUDIO); + new_media_segment_ = true; + if (GetParam()) + frame_processor_->SetSequenceMode(true); + + const base::TimeDelta fifty_ms = base::TimeDelta::FromMilliseconds(50); + SetTimestampOffset(fifty_ms); + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ + fifty_ms)); + ProcessFrames("0K", ""); + EXPECT_FALSE(new_media_segment_); + EXPECT_EQ(fifty_ms, timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [50,60) }"); + + // We do not stall on reading without seeking to 50ms due to + // SourceBufferStream::kSeekToStartFudgeRoom(). + CheckReadsThenReadStalls(audio_.get(), "50:0"); +} + +TEST_P(FrameProcessorTest, AudioOnly_SetOffsetThenFrameTimestampBelowOffset) { + // Tests A: STSO(50)+P(A20) -> + // if sequence mode: TSO==30,(a20@50) + // if segments mode: TSO==50,(a20@70) + InSequence s; + AddTestTracks(HAS_AUDIO); + new_media_segment_ = true; + bool using_sequence_mode = GetParam(); + if (using_sequence_mode) + frame_processor_->SetSequenceMode(true); + + const base::TimeDelta fifty_ms = base::TimeDelta::FromMilliseconds(50); + const base::TimeDelta twenty_ms = base::TimeDelta::FromMilliseconds(20); + SetTimestampOffset(fifty_ms); + + if (using_sequence_mode) { + EXPECT_CALL(callbacks_, PossibleDurationIncrease( + fifty_ms + frame_duration_)); + } else { + EXPECT_CALL(callbacks_, PossibleDurationIncrease( + fifty_ms + twenty_ms + frame_duration_)); + } + + ProcessFrames("20K", ""); + EXPECT_FALSE(new_media_segment_); + + // We do not stall on reading without seeking to 50ms / 70ms due to + // SourceBufferStream::kSeekToStartFudgeRoom(). + if (using_sequence_mode) { + EXPECT_EQ(fifty_ms - twenty_ms, timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [50,60) }"); + CheckReadsThenReadStalls(audio_.get(), "50:20"); + } else { + EXPECT_EQ(fifty_ms, timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [70,80) }"); + CheckReadsThenReadStalls(audio_.get(), "70:20"); + } +} + +TEST_P(FrameProcessorTest, AudioOnly_SequentialProcessFrames) { + // Tests A: P(A0,A10)+P(A20,A30) -> (a0,a10,a20,a30) + InSequence s; + AddTestTracks(HAS_AUDIO); + new_media_segment_ = true; + if (GetParam()) + frame_processor_->SetSequenceMode(true); + + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 2)); + ProcessFrames("0K 10K", ""); + EXPECT_FALSE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,20) }"); + + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 4)); + ProcessFrames("20K 30K", ""); + EXPECT_FALSE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,40) }"); + + CheckReadsThenReadStalls(audio_.get(), "0 10 20 30"); +} + +TEST_P(FrameProcessorTest, AudioOnly_NonSequentialProcessFrames) { + // Tests A: P(A20,A30)+P(A0,A10) -> + // if sequence mode: TSO==-20 after first P(), 20 after second P(), and + // a(20@0,a30@10,a0@20,a10@30) + // if segments mode: TSO==0,(a0,a10,a20,a30) + InSequence s; + AddTestTracks(HAS_AUDIO); + new_media_segment_ = true; + bool using_sequence_mode = GetParam(); + if (using_sequence_mode) { + frame_processor_->SetSequenceMode(true); + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 2)); + } else { + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 4)); + } + + ProcessFrames("20K 30K", ""); + EXPECT_FALSE(new_media_segment_); + + if (using_sequence_mode) { + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,20) }"); + EXPECT_EQ(frame_duration_ * -2, timestamp_offset_); + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 4)); + } else { + CheckExpectedRangesByTimestamp(audio_.get(), "{ [20,40) }"); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 2)); + } + + ProcessFrames("0K 10K", ""); + EXPECT_FALSE(new_media_segment_); + + if (using_sequence_mode) { + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,40) }"); + EXPECT_EQ(frame_duration_ * 2, timestamp_offset_); + CheckReadsThenReadStalls(audio_.get(), "0:20 10:30 20:0 30:10"); + } else { + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,40) }"); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + // TODO(wolenetz): Fix this need to seek to 0ms, possibly by having + // SourceBufferStream defer initial seek until next read. See + // http://crbug.com/371493. + audio_->AbortReads(); + audio_->Seek(base::TimeDelta()); + audio_->StartReturningData(); + CheckReadsThenReadStalls(audio_.get(), "0 10 20 30"); + } +} + +TEST_P(FrameProcessorTest, AudioVideo_SequentialProcessFrames) { + // Tests AV: P(A0,A10;V0k,V10,V20)+P(A20,A30,A40,V30) -> + // (a0,a10,a20,a30,a40);(v0,v10,v20,v30) + InSequence s; + AddTestTracks(HAS_AUDIO | HAS_VIDEO); + new_media_segment_ = true; + if (GetParam()) + frame_processor_->SetSequenceMode(true); + + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 3)); + ProcessFrames("0K 10K", "0K 10 20"); + EXPECT_FALSE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,20) }"); + CheckExpectedRangesByTimestamp(video_.get(), "{ [0,30) }"); + + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 5)); + ProcessFrames("20K 30K 40K", "30"); + EXPECT_FALSE(new_media_segment_); + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,50) }"); + CheckExpectedRangesByTimestamp(video_.get(), "{ [0,40) }"); + + CheckReadsThenReadStalls(audio_.get(), "0 10 20 30 40"); + CheckReadsThenReadStalls(video_.get(), "0 10 20 30"); +} + +TEST_P(FrameProcessorTest, AudioVideo_Discontinuity) { + // Tests AV: P(A0,A10,A30,A40,A50;V0k,V10,V40,V50key) -> + // if sequence mode: TSO==10,(a0,a10,a30,a40,a50@60);(v0,v10,v50@60) + // if segments mode: TSO==0,(a0,a10,a30,a40,a50);(v0,v10,v50) + // This assumes A40K is processed before V40, which depends currently on + // MergeBufferQueues() behavior. + InSequence s; + AddTestTracks(HAS_AUDIO | HAS_VIDEO); + new_media_segment_ = true; + bool using_sequence_mode = GetParam(); + if (using_sequence_mode) { + frame_processor_->SetSequenceMode(true); + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 7)); + } else { + EXPECT_CALL(callbacks_, PossibleDurationIncrease(frame_duration_ * 6)); + } + + ProcessFrames("0K 10K 30K 40K 50K", "0K 10 40 50K"); + EXPECT_FALSE(new_media_segment_); + + if (using_sequence_mode) { + EXPECT_EQ(frame_duration_, timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,70) }"); + CheckExpectedRangesByTimestamp(video_.get(), "{ [0,70) }"); + CheckReadsThenReadStalls(audio_.get(), "0 10 30 40 60:50"); + CheckReadsThenReadStalls(video_.get(), "0 10 60:50"); + } else { + EXPECT_EQ(base::TimeDelta(), timestamp_offset_); + CheckExpectedRangesByTimestamp(audio_.get(), "{ [0,60) }"); + CheckExpectedRangesByTimestamp(video_.get(), "{ [0,20) [50,60) }"); + CheckReadsThenReadStalls(audio_.get(), "0 10 30 40 50"); + CheckReadsThenReadStalls(video_.get(), "0 10"); + video_->AbortReads(); + video_->Seek(frame_duration_ * 5); + video_->StartReturningData(); + CheckReadsThenReadStalls(video_.get(), "50"); + } +} + +INSTANTIATE_TEST_CASE_P(SequenceMode, FrameProcessorTest, Values(true)); +INSTANTIATE_TEST_CASE_P(SegmentsMode, FrameProcessorTest, Values(false)); + +} // namespace media diff --git a/media/filters/legacy_frame_processor.cc b/media/filters/legacy_frame_processor.cc index 71c2801..ef8f89f 100644 --- a/media/filters/legacy_frame_processor.cc +++ b/media/filters/legacy_frame_processor.cc @@ -50,10 +50,6 @@ bool LegacyFrameProcessor::ProcessFrames( MseTrackBuffer* video_track = FindTrack(kVideoTrackId); DCHECK(video_buffers.empty() || video_track); - // TODO(wolenetz): DCHECK + return false if any of these buffers have UNKNOWN - // type() in upcoming coded frame processing compliant implementation. See - // http://crbug.com/249422. - StreamParser::BufferQueue filtered_audio; StreamParser::BufferQueue filtered_video; StreamParser::TextBufferQueueMap filtered_text; @@ -110,13 +106,8 @@ bool LegacyFrameProcessor::ProcessFrames( } *new_media_segment = false; - DCHECK(segment_timestamp != kInfiniteDuration()); - for (TrackBufferMap::iterator itr = track_buffers_.begin(); - itr != track_buffers_.end(); - ++itr) { - itr->second->stream()->OnNewMediaSegment(segment_timestamp); - } + NotifyNewMediaSegmentStarting(segment_timestamp); } if (!filtered_audio.empty() && diff --git a/media/filters/pipeline_integration_test.cc b/media/filters/pipeline_integration_test.cc index 2d4c4d9e..2920816 100644 --- a/media/filters/pipeline_integration_test.cc +++ b/media/filters/pipeline_integration_test.cc @@ -449,10 +449,7 @@ class MockMediaSource { // Test parameter determines which coded frame processor is used to process // appended data, and is only applicable in tests where the pipeline is using a // (Mock)MediaSource (which are TEST_P, not TEST_F). If true, -// LegacyFrameProcessor is used. Otherwise, (not yet supported), a more -// compliant frame processor is used. -// TODO(wolenetz): Enable usage of new frame processor based on this flag. -// See http://crbug.com/249422. +// LegacyFrameProcessor is used. Otherwise, the new FrameProcessor is used. class PipelineIntegrationTest : public testing::TestWithParam<bool>, public PipelineIntegrationTestBase { @@ -1319,8 +1316,10 @@ TEST_F(PipelineIntegrationTest, ASSERT_TRUE(WaitUntilOnEnded()); } -// TODO(wolenetz): Enable MSE testing of new frame processor based on this flag, -// once the new processor has landed. See http://crbug.com/249422. +// For MediaSource tests, generate two sets of tests: one using FrameProcessor, +// and one using LegacyFrameProcessor. +INSTANTIATE_TEST_CASE_P(NewFrameProcessor, PipelineIntegrationTest, + Values(false)); INSTANTIATE_TEST_CASE_P(LegacyFrameProcessor, PipelineIntegrationTest, Values(true)); diff --git a/media/media.gyp b/media/media.gyp index a799028..5a974aa 100644 --- a/media/media.gyp +++ b/media/media.gyp @@ -391,6 +391,8 @@ 'filters/ffmpeg_video_decoder.h', 'filters/file_data_source.cc', 'filters/file_data_source.h', + 'filters/frame_processor.cc', + 'filters/frame_processor.h', 'filters/frame_processor_base.cc', 'filters/frame_processor_base.h', 'filters/gpu_video_accelerator_factories.cc', @@ -1043,6 +1045,7 @@ 'filters/ffmpeg_glue_unittest.cc', 'filters/ffmpeg_video_decoder_unittest.cc', 'filters/file_data_source_unittest.cc', + 'filters/frame_processor_unittest.cc', 'filters/h264_bit_reader_unittest.cc', 'filters/h264_parser_unittest.cc', 'filters/in_memory_url_protocol_unittest.cc', |