Fill video frame gaps created after discarding invalid frames.

In MSE, a new segment must start with a key frame.
However, for HLS, this is just a recommendation and some HLS streams
do not comply with that recommendation. The leading non key frames
must be discarded.
However, simply discarding these frames would create a gap
in the video buffer ranges. This gap is now filled with the first
valid key frame.

BUG=None

Review URL: https://codereview.chromium.org/250703005

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@268586 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 55 insertions, 11 deletions

diff --git a/media/formats/mp2t/mp2t_stream_parser.cc b/media/formats/mp2t/mp2t_stream_parser.cc
index c00bba7..e832d2c 100644
--- a/media/formats/mp2t/mp2t_stream_parser.cc
+++ b/media/formats/mp2t/mp2t_stream_parser.cc
@@ -211,6 +211,7 @@ void Mp2tStreamParser::Flush() {
   // stream parser already involves the end of the current segment.
   segment_started_ = false;
   first_video_frame_in_segment_ = true;
+  discarded_frames_dts_.clear();
 
   // Remove any bytes left in the TS buffer.
   // (i.e. any partial TS packet => less than 188 bytes).
@@ -539,23 +540,27 @@ void Mp2tStreamParser::OnEmitVideoBuffer(
   stream_parser_buffer->SetDecodeTimestamp(
       stream_parser_buffer->GetDecodeTimestamp() - time_offset_);
 
-  // Ignore the incoming buffer if it is not associated with any config.
-  if (buffer_queue_chain_.empty()) {
-    DVLOG(1) << "Ignoring video buffer with no corresponding video config:"
+  // Discard the incoming buffer:
+  // - if it is not associated with any config,
+  // - or if only non-key frames have been added to a new segment.
+  if (buffer_queue_chain_.empty() ||
+      (first_video_frame_in_segment_ && !stream_parser_buffer->IsKeyframe())) {
+    DVLOG(1) << "Discard video buffer:"
              << " keyframe=" << stream_parser_buffer->IsKeyframe()
              << " dts="
              << stream_parser_buffer->GetDecodeTimestamp().InMilliseconds();
+    if (discarded_frames_dts_.empty() ||
+        discarded_frames_min_pts_ > stream_parser_buffer->timestamp()) {
+      discarded_frames_min_pts_ = stream_parser_buffer->timestamp();
+    }
+    discarded_frames_dts_.push_back(
+        stream_parser_buffer->GetDecodeTimestamp());
     return;
   }
 
-  // A segment cannot start with a non key frame.
-  // Ignore the frame if that's the case.
-  if (first_video_frame_in_segment_ && !stream_parser_buffer->IsKeyframe()) {
-    DVLOG(1) << "Ignoring non-key frame:"
-             << " dts="
-             << stream_parser_buffer->GetDecodeTimestamp().InMilliseconds();
-    return;
-  }
+  // Fill the gap created by frames that have been discarded.
+  if (!discarded_frames_dts_.empty())
+    FillVideoGap(stream_parser_buffer);
 
   first_video_frame_in_segment_ = false;
   buffer_queue_chain_.back().video_queue.push_back(stream_parser_buffer);
@@ -619,5 +624,33 @@ bool Mp2tStreamParser::EmitRemainingBuffers() {
   return true;
 }
 
+void Mp2tStreamParser::FillVideoGap(
+    const scoped_refptr<StreamParserBuffer>& stream_parser_buffer) {
+  DCHECK(!buffer_queue_chain_.empty());
+  DCHECK(!discarded_frames_dts_.empty());
+  DCHECK(stream_parser_buffer->IsKeyframe());
+
+  // PTS is interpolated between the min PTS of discarded frames
+  // and the PTS of the first valid buffer.
+  base::TimeDelta pts = discarded_frames_min_pts_;
+  base::TimeDelta pts_delta =
+      (stream_parser_buffer->timestamp() - pts) / discarded_frames_dts_.size();
+
+  while (!discarded_frames_dts_.empty()) {
+    scoped_refptr<StreamParserBuffer> frame =
+        StreamParserBuffer::CopyFrom(
+            stream_parser_buffer->data(),
+            stream_parser_buffer->data_size(),
+            stream_parser_buffer->IsKeyframe(),
+            stream_parser_buffer->type(),
+            stream_parser_buffer->track_id());
+    frame->SetDecodeTimestamp(discarded_frames_dts_.front());
+    frame->set_timestamp(pts);
+    buffer_queue_chain_.back().video_queue.push_back(frame);
+    pts += pts_delta;
+    discarded_frames_dts_.pop_front();
+  }
+}
+
 }  // namespace mp2t
 }  // namespace media
diff --git a/media/formats/mp2t/mp2t_stream_parser.h b/media/formats/mp2t/mp2t_stream_parser.h
index 85629dc..61f3440 100644
--- a/media/formats/mp2t/mp2t_stream_parser.h
+++ b/media/formats/mp2t/mp2t_stream_parser.h
@@ -92,6 +92,12 @@ class MEDIA_EXPORT Mp2tStreamParser : public StreamParser {
       scoped_refptr<StreamParserBuffer> stream_parser_buffer);
   bool EmitRemainingBuffers();
 
+  // At the beginning of a new segment, some video frames might be discarded.
+  // This function fills the hole by duplicating the first valid key frame
+  // given by |stream_parser_buffer|.
+  void FillVideoGap(
+      const scoped_refptr<StreamParserBuffer>& stream_parser_buffer);
+
   // List of callbacks.
   InitCB init_cb_;
   NewConfigCB config_cb_;
@@ -115,6 +121,11 @@ class MEDIA_EXPORT Mp2tStreamParser : public StreamParser {
   int selected_audio_pid_;
   int selected_video_pid_;
 
+  // DTS of discarded buffers.
+  // Min PTS of discarded buffers.
+  std::list<base::TimeDelta> discarded_frames_dts_;
+  base::TimeDelta discarded_frames_min_pts_;
+
   // Pending audio & video buffers.
   std::list<BufferQueueWithConfig> buffer_queue_chain_;