Implemented HEVC video demuxing and parsing

Chromecast team is interested in adding HEVC/H265 video codec support. Since we are using hardware decoders on our platforms, we don't need to add software decoder, but we still need to be able to demux and parse HEVC in mp4 containers. HEVC is very similar to H264 in many aspects, so we can reuse a lot of the existing H264 functionality.

BUG=454948

Review URL: https://codereview.chromium.org/816353010

Cr-Commit-Position: refs/heads/master@{#347814}

7 files changed, 423 insertions, 9 deletions

diff --git a/media/formats/mp4/box_definitions.cc b/media/formats/mp4/box_definitions.cc
index 9b46138..fc0250d 100644
--- a/media/formats/mp4/box_definitions.cc
+++ b/media/formats/mp4/box_definitions.cc
@@ -5,9 +5,16 @@
 #include "media/formats/mp4/box_definitions.h"
 
 #include "base/logging.h"
+#include "media/base/video_types.h"
+#include "media/base/video_util.h"
+#include "media/formats/mp4/avc.h"
 #include "media/formats/mp4/es_descriptor.h"
 #include "media/formats/mp4/rcheck.h"
 
+#if defined(ENABLE_HEVC_DEMUXING)
+#include "media/formats/mp4/hevc.h"
+#endif
+
 namespace media {
 namespace mp4 {
 
@@ -455,7 +462,9 @@ VideoSampleEntry::VideoSampleEntry()
     : format(FOURCC_NULL),
       data_reference_index(0),
       width(0),
-      height(0) {}
+      height(0),
+      video_codec(kUnknownVideoCodec),
+      video_codec_profile(VIDEO_CODEC_PROFILE_UNKNOWN) {}
 
 VideoSampleEntry::~VideoSampleEntry() {}
 FourCC VideoSampleEntry::BoxType() const {
@@ -464,6 +473,26 @@ FourCC VideoSampleEntry::BoxType() const {
   return FOURCC_NULL;
 }
 
+namespace {
+
+bool IsFormatValidH264(const FourCC& format,
+                       const ProtectionSchemeInfo& sinf) {
+  return format == FOURCC_AVC1 || format == FOURCC_AVC3 ||
+      (format == FOURCC_ENCV && (sinf.format.format == FOURCC_AVC1 ||
+                                 sinf.format.format == FOURCC_AVC3));
+}
+
+#if defined(ENABLE_HEVC_DEMUXING)
+bool IsFormatValidHEVC(const FourCC& format,
+                       const ProtectionSchemeInfo& sinf) {
+  return format == FOURCC_HEV1 || format == FOURCC_HVC1 ||
+      (format == FOURCC_ENCV && (sinf.format.format == FOURCC_HEV1 ||
+                                 sinf.format.format == FOURCC_HVC1));
+}
+#endif
+
+}
+
 bool VideoSampleEntry::Parse(BoxReader* reader) {
   format = reader->type();
   RCHECK(reader->SkipBytes(6) &&
@@ -485,21 +514,44 @@ bool VideoSampleEntry::Parse(BoxReader* reader) {
     }
   }
 
-  if (IsFormatValid()) {
+  if (IsFormatValidH264(format, sinf)) {
+    DVLOG(2) << __FUNCTION__
+             << " reading AVCDecoderConfigurationRecord (avcC)";
     scoped_ptr<AVCDecoderConfigurationRecord> avcConfig(
         new AVCDecoderConfigurationRecord());
     RCHECK(reader->ReadChild(avcConfig.get()));
     frame_bitstream_converter = make_scoped_refptr(
         new AVCBitstreamConverter(avcConfig.Pass()));
+    video_codec = kCodecH264;
+    video_codec_profile = H264PROFILE_MAIN;
+#if defined(ENABLE_HEVC_DEMUXING)
+  } else if (IsFormatValidHEVC(format, sinf)) {
+    DVLOG(2) << __FUNCTION__
+             << " parsing HEVCDecoderConfigurationRecord (hvcC)";
+    scoped_ptr<HEVCDecoderConfigurationRecord> hevcConfig(
+        new HEVCDecoderConfigurationRecord());
+    RCHECK(reader->ReadChild(hevcConfig.get()));
+    frame_bitstream_converter = make_scoped_refptr(
+        new HEVCBitstreamConverter(hevcConfig.Pass()));
+    video_codec = kCodecHEVC;
+#endif
+  } else {
+    // Unknown/unsupported format
+    MEDIA_LOG(ERROR, reader->media_log()) << __FUNCTION__
+                                          << " unsupported video format "
+                                          << FourCCToString(format);
+    return false;
   }
 
   return true;
 }
 
 bool VideoSampleEntry::IsFormatValid() const {
-  return format == FOURCC_AVC1 || format == FOURCC_AVC3 ||
-      (format == FOURCC_ENCV && (sinf.format.format == FOURCC_AVC1 ||
-                                 sinf.format.format == FOURCC_AVC3));
+#if defined(ENABLE_HEVC_DEMUXING)
+  if (IsFormatValidHEVC(format, sinf))
+    return true;
+#endif
+  return IsFormatValidH264(format, sinf);
 }
 
 ElementaryStreamDescriptor::ElementaryStreamDescriptor()
diff --git a/media/formats/mp4/box_definitions.h b/media/formats/mp4/box_definitions.h
index 39a19b7..cdcd775 100644
--- a/media/formats/mp4/box_definitions.h
+++ b/media/formats/mp4/box_definitions.h
@@ -12,6 +12,7 @@
 #include "base/compiler_specific.h"
 #include "media/base/media_export.h"
 #include "media/base/media_log.h"
+#include "media/base/video_decoder_config.h"
 #include "media/formats/mp4/aac.h"
 #include "media/formats/mp4/avc.h"
 #include "media/formats/mp4/box_reader.h"
@@ -207,6 +208,9 @@ struct MEDIA_EXPORT VideoSampleEntry : Box {
   PixelAspectRatioBox pixel_aspect;
   ProtectionSchemeInfo sinf;
 
+  VideoCodec video_codec;
+  VideoCodecProfile video_codec_profile;
+
   bool IsFormatValid() const;
 
   scoped_refptr<BitstreamConverter> frame_bitstream_converter;
diff --git a/media/formats/mp4/fourccs.h b/media/formats/mp4/fourccs.h
index d9086fa..fd97797 100644
--- a/media/formats/mp4/fourccs.h
+++ b/media/formats/mp4/fourccs.h
@@ -31,6 +31,11 @@ enum FourCC {
   FOURCC_FTYP = 0x66747970,
   FOURCC_HDLR = 0x68646c72,
   FOURCC_HINT = 0x68696e74,
+#if defined(ENABLE_HEVC_DEMUXING)
+  FOURCC_HEV1 = 0x68657631,
+  FOURCC_HVC1 = 0x68766331,
+  FOURCC_HVCC = 0x68766343,
+#endif
   FOURCC_IODS = 0x696f6473,
   FOURCC_MDAT = 0x6d646174,
   FOURCC_MDHD = 0x6d646864,
diff --git a/media/formats/mp4/hevc.cc b/media/formats/mp4/hevc.cc
new file mode 100644
index 0000000..4d6b53e
--- /dev/null
+++ b/media/formats/mp4/hevc.cc
@@ -0,0 +1,237 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/formats/mp4/hevc.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "base/logging.h"
+#include "media/base/decrypt_config.h"
+#include "media/filters/h265_parser.h"
+#include "media/formats/mp4/avc.h"
+#include "media/formats/mp4/box_definitions.h"
+#include "media/formats/mp4/box_reader.h"
+
+namespace media {
+namespace mp4 {
+
+HEVCDecoderConfigurationRecord::HEVCDecoderConfigurationRecord()
+    : configurationVersion(0),
+      general_profile_space(0),
+      general_tier_flag(0),
+      general_profile_idc(0),
+      general_profile_compatibility_flags(0),
+      general_constraint_indicator_flags(0),
+      general_level_idc(0),
+      min_spatial_segmentation_idc(0),
+      parallelismType(0),
+      chromaFormat(0),
+      bitDepthLumaMinus8(0),
+      bitDepthChromaMinus8(0),
+      avgFrameRate(0),
+      constantFrameRate(0),
+      numTemporalLayers(0),
+      temporalIdNested(0),
+      lengthSizeMinusOne(0),
+      numOfArrays(0) {}
+
+HEVCDecoderConfigurationRecord::~HEVCDecoderConfigurationRecord() {}
+FourCC HEVCDecoderConfigurationRecord::BoxType() const { return FOURCC_HVCC; }
+
+bool HEVCDecoderConfigurationRecord::Parse(BoxReader* reader) {
+  return ParseInternal(reader, reader->media_log());
+}
+
+bool HEVCDecoderConfigurationRecord::Parse(const uint8* data, int data_size) {
+  BufferReader reader(data, data_size);
+  return ParseInternal(&reader, new MediaLog());
+}
+
+HEVCDecoderConfigurationRecord::HVCCNALArray::HVCCNALArray()
+    : first_byte(0) {}
+
+HEVCDecoderConfigurationRecord::HVCCNALArray::~HVCCNALArray() {}
+
+bool HEVCDecoderConfigurationRecord::ParseInternal(
+    BufferReader* reader,
+    const scoped_refptr<MediaLog>& media_log) {
+  uint8 profile_indication = 0;
+  uint32 general_constraint_indicator_flags_hi = 0;
+  uint16 general_constraint_indicator_flags_lo = 0;
+  uint8 misc = 0;
+  RCHECK(reader->Read1(&configurationVersion) && configurationVersion == 1 &&
+         reader->Read1(&profile_indication) &&
+         reader->Read4(&general_profile_compatibility_flags) &&
+         reader->Read4(&general_constraint_indicator_flags_hi) &&
+         reader->Read2(&general_constraint_indicator_flags_lo) &&
+         reader->Read1(&general_level_idc) &&
+         reader->Read2(&min_spatial_segmentation_idc) &&
+         reader->Read1(&parallelismType) &&
+         reader->Read1(&chromaFormat) &&
+         reader->Read1(&bitDepthLumaMinus8) &&
+         reader->Read1(&bitDepthChromaMinus8) &&
+         reader->Read2(&avgFrameRate) &&
+         reader->Read1(&misc) &&
+         reader->Read1(&numOfArrays));
+
+  general_profile_space = profile_indication >> 6;
+  general_tier_flag = (profile_indication >> 5) & 1;
+  general_profile_idc = profile_indication & 0x1f;
+
+  general_constraint_indicator_flags = general_constraint_indicator_flags_hi;
+  general_constraint_indicator_flags <<= 16;
+  general_constraint_indicator_flags |= general_constraint_indicator_flags_lo;
+
+  min_spatial_segmentation_idc &= 0xfff;
+  parallelismType &= 3;
+  chromaFormat &= 3;
+  bitDepthLumaMinus8 &= 7;
+  bitDepthChromaMinus8 &= 7;
+
+  constantFrameRate = misc >> 6;
+  numTemporalLayers = (misc >> 3) & 7;
+  temporalIdNested = (misc >> 2) & 1;
+  lengthSizeMinusOne = misc & 3;
+
+  DVLOG(2) << __FUNCTION__ << " numOfArrays=" << (int)numOfArrays;
+  arrays.resize(numOfArrays);
+  for (uint32 j = 0; j < numOfArrays; j++) {
+    RCHECK(reader->Read1(&arrays[j].first_byte));
+    uint16 numNalus = 0;
+    RCHECK(reader->Read2(&numNalus));
+    arrays[j].units.resize(numNalus);
+    for (uint32 i = 0; i < numNalus; ++i) {
+      uint16 naluLength = 0;
+      RCHECK(reader->Read2(&naluLength) &&
+             reader->ReadVec(&arrays[j].units[i], naluLength));
+      DVLOG(4) << __FUNCTION__ << " naluType="
+               << (int)(arrays[j].first_byte & 0x3f)
+               << " size=" << arrays[j].units[i].size();
+    }
+  }
+
+  if (media_log.get()) {
+    MEDIA_LOG(INFO, media_log) << "Video codec: hevc";
+  }
+
+  return true;
+}
+
+static const uint8 kAnnexBStartCode[] = {0, 0, 0, 1};
+static const int kAnnexBStartCodeSize = 4;
+
+bool HEVC::InsertParamSetsAnnexB(
+    const HEVCDecoderConfigurationRecord& hevc_config,
+    std::vector<uint8>* buffer,
+    std::vector<SubsampleEntry>* subsamples) {
+  DCHECK(HEVC::IsValidAnnexB(*buffer, *subsamples));
+
+  scoped_ptr<H265Parser> parser(new H265Parser());
+  const uint8* start = &(*buffer)[0];
+  parser->SetEncryptedStream(start, buffer->size(), *subsamples);
+
+  H265NALU nalu;
+  if (parser->AdvanceToNextNALU(&nalu) != H265Parser::kOk)
+    return false;
+
+  std::vector<uint8>::iterator config_insert_point = buffer->begin();
+
+  if (nalu.nal_unit_type == H265NALU::AUD_NUT) {
+    // Move insert point to just after the AUD.
+    config_insert_point += (nalu.data + nalu.size) - start;
+  }
+
+  // Clear |parser| and |start| since they aren't needed anymore and
+  // will hold stale pointers once the insert happens.
+  parser.reset();
+  start = NULL;
+
+  std::vector<uint8> param_sets;
+  RCHECK(HEVC::ConvertConfigToAnnexB(hevc_config, &param_sets));
+  DVLOG(4) << __FUNCTION__ << " converted hvcC to AnnexB "
+           << " size=" << param_sets.size() << " inserted at "
+           << (int)(config_insert_point - buffer->begin());
+
+  if (subsamples && !subsamples->empty()) {
+    int subsample_index = AVC::FindSubsampleIndex(*buffer, subsamples,
+                                                  &(*config_insert_point));
+    // Update the size of the subsample where SPS/PPS is to be inserted.
+    (*subsamples)[subsample_index].clear_bytes += param_sets.size();
+  }
+
+  buffer->insert(config_insert_point,
+                 param_sets.begin(), param_sets.end());
+
+  DCHECK(HEVC::IsValidAnnexB(*buffer, *subsamples));
+  return true;
+}
+
+bool HEVC::ConvertConfigToAnnexB(
+    const HEVCDecoderConfigurationRecord& hevc_config,
+    std::vector<uint8>* buffer) {
+  DCHECK(buffer->empty());
+  buffer->clear();
+
+  for (size_t j = 0; j < hevc_config.arrays.size(); j++) {
+    uint8 naluType = hevc_config.arrays[j].first_byte & 0x3f;
+    for (size_t i = 0; i < hevc_config.arrays[j].units.size(); ++i) {
+      DVLOG(3) << __FUNCTION__ << " naluType=" << (int)naluType
+               << " size=" << hevc_config.arrays[j].units[i].size();
+      buffer->insert(buffer->end(), kAnnexBStartCode,
+                     kAnnexBStartCode + kAnnexBStartCodeSize);
+      buffer->insert(buffer->end(), hevc_config.arrays[j].units[i].begin(),
+                     hevc_config.arrays[j].units[i].end());
+    }
+  }
+
+  return true;
+}
+
+// Verifies AnnexB NALU order according to section 7.4.2.4.4 of ISO/IEC 23008-2.
+bool HEVC::IsValidAnnexB(const std::vector<uint8>& buffer,
+                         const std::vector<SubsampleEntry>& subsamples) {
+  return IsValidAnnexB(&buffer[0], buffer.size(), subsamples);
+}
+
+bool HEVC::IsValidAnnexB(const uint8* buffer, size_t size,
+                         const std::vector<SubsampleEntry>& subsamples) {
+  DCHECK(buffer);
+
+  if (size == 0)
+    return true;
+
+  // TODO(servolk): Implement this, see crbug.com/527595
+  return true;
+}
+
+HEVCBitstreamConverter::HEVCBitstreamConverter(
+    scoped_ptr<HEVCDecoderConfigurationRecord> hevc_config)
+  : hevc_config_(hevc_config.Pass()) {
+    DCHECK(hevc_config_);
+}
+
+HEVCBitstreamConverter::~HEVCBitstreamConverter() {
+}
+
+bool HEVCBitstreamConverter::ConvertFrame(
+    std::vector<uint8>* frame_buf,
+    bool is_keyframe,
+    std::vector<SubsampleEntry>* subsamples) const {
+  RCHECK(AVC::ConvertFrameToAnnexB(hevc_config_->lengthSizeMinusOne + 1,
+                                   frame_buf, subsamples));
+
+  if (is_keyframe) {
+    // If this is a keyframe, we (re-)inject HEVC params headers at the start of
+    // a frame. If subsample info is present, we also update the clear byte
+    // count for that first subsample.
+    RCHECK(HEVC::InsertParamSetsAnnexB(*hevc_config_, frame_buf, subsamples));
+  }
+
+  DCHECK(HEVC::IsValidAnnexB(*frame_buf, *subsamples));
+  return true;
+}
+
+}  // namespace mp4
+}  // namespace media
diff --git a/media/formats/mp4/hevc.h b/media/formats/mp4/hevc.h
new file mode 100644
index 0000000..06974c0d
--- /dev/null
+++ b/media/formats/mp4/hevc.h
@@ -0,0 +1,106 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_FORMATS_MP4_HEVC_H_
+#define MEDIA_FORMATS_MP4_HEVC_H_
+
+#include <vector>
+
+#include "base/basictypes.h"
+#include "base/memory/scoped_ptr.h"
+#include "media/base/media_export.h"
+#include "media/formats/mp4/bitstream_converter.h"
+#include "media/formats/mp4/box_definitions.h"
+
+namespace media {
+
+struct SubsampleEntry;
+
+namespace mp4 {
+
+struct MEDIA_EXPORT HEVCDecoderConfigurationRecord : Box {
+  DECLARE_BOX_METHODS(HEVCDecoderConfigurationRecord);
+
+  // Parses HEVCDecoderConfigurationRecord data encoded in |data|.
+  // Note: This method is intended to parse data outside the MP4StreamParser
+  //       context and therefore the box header is not expected to be present
+  //       in |data|.
+  // Returns true if |data| was successfully parsed.
+  bool Parse(const uint8* data, int data_size);
+
+  uint8 configurationVersion;
+  uint8 general_profile_space;
+  uint8 general_tier_flag;
+  uint8 general_profile_idc;
+  uint32 general_profile_compatibility_flags;
+  uint64 general_constraint_indicator_flags;
+  uint8 general_level_idc;
+  uint16 min_spatial_segmentation_idc;
+  uint8 parallelismType;
+  uint8 chromaFormat;
+  uint8 bitDepthLumaMinus8;
+  uint8 bitDepthChromaMinus8;
+  uint16 avgFrameRate;
+  uint8 constantFrameRate;
+  uint8 numTemporalLayers;
+  uint8 temporalIdNested;
+  uint8 lengthSizeMinusOne;
+  uint8 numOfArrays;
+
+  typedef std::vector<uint8> HVCCNALUnit;
+  struct HVCCNALArray {
+    HVCCNALArray();
+    ~HVCCNALArray();
+    uint8 first_byte;
+    std::vector<HVCCNALUnit> units;
+  };
+  std::vector<HVCCNALArray> arrays;
+
+ private:
+  bool ParseInternal(BufferReader* reader,
+                     const scoped_refptr<MediaLog>& media_log);
+};
+
+class MEDIA_EXPORT HEVC {
+ public:
+  static bool ConvertConfigToAnnexB(
+      const HEVCDecoderConfigurationRecord& hevc_config,
+      std::vector<uint8>* buffer);
+
+  static bool InsertParamSetsAnnexB(
+      const HEVCDecoderConfigurationRecord& hevc_config,
+      std::vector<uint8>* buffer,
+      std::vector<SubsampleEntry>* subsamples);
+
+  // Verifies that the contents of |buffer| conform to
+  // Section 7.4.2.4.4 of ISO/IEC 23008-2.
+  // |subsamples| contains the information about what parts of the buffer are
+  // encrypted and which parts are clear.
+  // Returns true if |buffer| contains conformant Annex B data
+  // TODO(servolk): Remove the std::vector version when we can use,
+  // C++11's std::vector<T>::data() method.
+  static bool IsValidAnnexB(const std::vector<uint8>& buffer,
+                            const std::vector<SubsampleEntry>& subsamples);
+  static bool IsValidAnnexB(const uint8* buffer, size_t size,
+                            const std::vector<SubsampleEntry>& subsamples);
+};
+
+class HEVCBitstreamConverter : public BitstreamConverter {
+ public:
+  explicit HEVCBitstreamConverter(
+      scoped_ptr<HEVCDecoderConfigurationRecord> hevc_config);
+
+  // BitstreamConverter interface
+  bool ConvertFrame(std::vector<uint8>* frame_buf,
+                    bool is_keyframe,
+                    std::vector<SubsampleEntry>* subsamples) const override;
+ private:
+  ~HEVCBitstreamConverter() override;
+  scoped_ptr<HEVCDecoderConfigurationRecord> hevc_config_;
+};
+
+}  // namespace mp4
+}  // namespace media
+
+#endif  // MEDIA_FORMATS_MP4_HEVC_H_
diff --git a/media/formats/mp4/mp4_stream_parser.cc b/media/formats/mp4/mp4_stream_parser.cc
index 59164d9..18698b3 100644
--- a/media/formats/mp4/mp4_stream_parser.cc
+++ b/media/formats/mp4/mp4_stream_parser.cc
@@ -305,9 +305,9 @@ bool MP4StreamParser::ParseMoov(BoxReader* reader) {
 
       is_video_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
       DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_;
-      video_config.Initialize(kCodecH264, H264PROFILE_MAIN, PIXEL_FORMAT_YV12,
-                              COLOR_SPACE_HD_REC709, coded_size, visible_rect,
-                              natural_size,
+      video_config.Initialize(entry.video_codec, entry.video_codec_profile,
+                              PIXEL_FORMAT_YV12, COLOR_SPACE_HD_REC709,
+                              coded_size, visible_rect, natural_size,
                               // No decoder-specific buffer needed for AVC;
                               // SPS/PPS are embedded in the video stream
                               NULL, 0, is_video_track_encrypted_);
@@ -478,7 +478,8 @@ bool MP4StreamParser::EnqueueSample(BufferQueue* audio_buffers,
     DCHECK(runs_->video_description().frame_bitstream_converter);
     if (!runs_->video_description().frame_bitstream_converter->ConvertFrame(
         &frame_buf, runs_->is_keyframe(), &subsamples)) {
-      MEDIA_LOG(ERROR, media_log_) << "Failed to prepare AVC sample for decode";
+      MEDIA_LOG(ERROR, media_log_)
+          << "Failed to prepare video sample for decode";
       *err = true;
       return false;
     }
diff --git a/media/formats/mp4/mp4_stream_parser_unittest.cc b/media/formats/mp4/mp4_stream_parser_unittest.cc
index 620b81f..60e4165 100644
--- a/media/formats/mp4/mp4_stream_parser_unittest.cc
+++ b/media/formats/mp4/mp4_stream_parser_unittest.cc
@@ -294,6 +294,15 @@ TEST_F(MP4StreamParserTest, VideoSamplesStartWithAUDs) {
   ParseMP4File("bear-1280x720-av_with-aud-nalus_frag.mp4", 512);
 }
 
+#if defined(ENABLE_HEVC_DEMUXING)
+TEST_F(MP4StreamParserTest, HEVC_in_MP4_container) {
+  InitializeParserAndExpectLiveness(DemuxerStream::LIVENESS_RECORDED);
+  scoped_refptr<DecoderBuffer> buffer = ReadTestDataFile("bear-hevc-frag.mp4");
+  EXPECT_MEDIA_LOG(VideoCodecLog("hevc"));
+  EXPECT_TRUE(AppendDataInPieces(buffer->data(), buffer->data_size(), 512));
+}
+#endif
+
 TEST_F(MP4StreamParserTest, CENC) {
   // Encrypted test mp4 files have non-zero duration and are treated as
   // recorded streams.