Add brotli content-encoding filter.

BUG=472009

Review URL: https://codereview.chromium.org/1431723002

Cr-Commit-Position: refs/heads/master@{#367153}

6 files changed, 419 insertions, 1 deletions

diff --git a/net/filter/brotli_filter.cc b/net/filter/brotli_filter.cc
new file mode 100644
index 0000000..66e778a
--- /dev/null
+++ b/net/filter/brotli_filter.cc
@@ -0,0 +1,113 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/filter/brotli_filter.h"
+
+#include "base/macros.h"
+#include "base/numerics/safe_conversions.h"
+#include "base/numerics/safe_math.h"
+#include "third_party/brotli/dec/decode.h"
+
+namespace net {
+
+// BrotliFilter applies Brotli content decoding to a data stream.
+// Brotli format specification: http://www.ietf.org/id/draft-alakuijala-brotli
+//
+// BrotliFilter is a subclass of Filter. See the latter's header file filter.h
+// for sample usage.
+class BrotliFilter : public Filter {
+ public:
+  BrotliFilter(FilterType type)
+      : Filter(type), decoding_status_(DECODING_IN_PROGRESS) {
+    BrotliStateInit(&brotli_state_);
+  }
+
+  ~BrotliFilter() override { BrotliStateCleanup(&brotli_state_); }
+
+  // Decodes the pre-filter data and writes the output into the |dest_buffer|
+  // passed in.
+  // The function returns FilterStatus. See filter.h for its description.
+  //
+  // Upon entry, |*dest_len| is the total size (in number of chars) of the
+  // destination buffer. Upon exit, |*dest_len| is the actual number of chars
+  // written into the destination buffer.
+  //
+  // This function will fail if there is no pre-filter data in the
+  // |stream_buffer_|. On the other hand, |*dest_len| can be 0 upon successful
+  // return. For example, decompressor may process some pre-filter data
+  // but not produce output yet.
+  FilterStatus ReadFilteredData(char* dest_buffer, int* dest_len) override {
+    if (!dest_buffer || !dest_len)
+      return Filter::FILTER_ERROR;
+
+    if (decoding_status_ == DECODING_DONE) {
+      *dest_len = 0;
+      return Filter::FILTER_DONE;
+    }
+
+    if (decoding_status_ != DECODING_IN_PROGRESS)
+      return Filter::FILTER_ERROR;
+
+    size_t output_buffer_size = base::checked_cast<size_t>(*dest_len);
+    size_t input_buffer_size = base::checked_cast<size_t>(stream_data_len_);
+
+    size_t available_in = input_buffer_size;
+    const uint8_t* next_in = bit_cast<uint8_t*>(next_stream_data_);
+    size_t available_out = output_buffer_size;
+    uint8_t* next_out = bit_cast<uint8_t*>(dest_buffer);
+    size_t total_out = 0;
+    BrotliResult result =
+        BrotliDecompressStream(&available_in, &next_in, &available_out,
+                               &next_out, &total_out, &brotli_state_);
+
+    CHECK(available_in <= input_buffer_size);
+    CHECK(available_out <= output_buffer_size);
+
+    base::CheckedNumeric<size_t> safe_bytes_written(output_buffer_size);
+    safe_bytes_written -= available_out;
+    int bytes_written =
+        base::checked_cast<int>(safe_bytes_written.ValueOrDie());
+
+    switch (result) {
+      case BROTLI_RESULT_NEEDS_MORE_OUTPUT:
+      // Fall through.
+      case BROTLI_RESULT_SUCCESS:
+        *dest_len = bytes_written;
+        stream_data_len_ = base::checked_cast<int>(available_in);
+        next_stream_data_ = bit_cast<char*>(next_in);
+        if (result == BROTLI_RESULT_SUCCESS) {
+          decoding_status_ = DECODING_DONE;
+          return Filter::FILTER_DONE;
+        }
+        return Filter::FILTER_OK;
+
+      case BROTLI_RESULT_NEEDS_MORE_INPUT:
+        *dest_len = bytes_written;
+        stream_data_len_ = 0;
+        next_stream_data_ = nullptr;
+        return Filter::FILTER_NEED_MORE_DATA;
+
+      default:
+        decoding_status_ = DECODING_ERROR;
+        return Filter::FILTER_ERROR;
+    }
+  }
+
+ private:
+  enum DecodingStatus { DECODING_IN_PROGRESS, DECODING_DONE, DECODING_ERROR };
+
+  // Tracks the status of decoding.
+  // This variable is updated only by ReadFilteredData.
+  DecodingStatus decoding_status_;
+
+  BrotliState brotli_state_;
+
+  DISALLOW_COPY_AND_ASSIGN(BrotliFilter);
+};
+
+Filter* CreateBrotliFilter(Filter::FilterType type_id) {
+  return new BrotliFilter(type_id);
+}
+
+}  // namespace net
diff --git a/net/filter/brotli_filter.h b/net/filter/brotli_filter.h
new file mode 100644
index 0000000..5570ae4
--- /dev/null
+++ b/net/filter/brotli_filter.h
@@ -0,0 +1,17 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_FILTER_BROTLI_FILTER_H_
+#define NET_FILTER_BROTLI_FILTER_H_
+
+#include "net/filter/filter.h"
+
+namespace net {
+
+// Creates instance of filter or returns nullptr if brotli is not supported.
+Filter* CreateBrotliFilter(Filter::FilterType type_id);
+
+}  // namespace net
+
+#endif  // NET_FILTER_BROTLI_FILTER_H__
diff --git a/net/filter/brotli_filter_disabled.cc b/net/filter/brotli_filter_disabled.cc
new file mode 100644
index 0000000..1081ff4
--- /dev/null
+++ b/net/filter/brotli_filter_disabled.cc
@@ -0,0 +1,13 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/filter/brotli_filter.h"
+
+namespace net {
+
+Filter* CreateBrotliFilter(Filter::FilterType type_id) {
+  return nullptr;
+}
+
+}  // namespace net
diff --git a/net/filter/brotli_filter_unittest.cc b/net/filter/brotli_filter_unittest.cc
new file mode 100644
index 0000000..9c9ce65
--- /dev/null
+++ b/net/filter/brotli_filter_unittest.cc
@@ -0,0 +1,253 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/files/file_util.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/path_service.h"
+#include "net/base/io_buffer.h"
+#include "net/filter/brotli_filter.h"
+#include "net/filter/mock_filter_context.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "testing/platform_test.h"
+
+namespace {
+const int kDefaultBufferSize = 4096;
+const int kSmallBufferSize = 128;
+}  // namespace
+
+namespace net {
+
+// These tests use the path service, which uses autoreleased objects on the
+// Mac, so this needs to be a PlatformTest.
+class BrotliUnitTest : public PlatformTest {
+ protected:
+  void SetUp() override {
+    PlatformTest::SetUp();
+
+    // Get the path of data directory.
+    base::FilePath data_dir;
+    PathService::Get(base::DIR_SOURCE_ROOT, &data_dir);
+    data_dir = data_dir.AppendASCII("net");
+    data_dir = data_dir.AppendASCII("data");
+    data_dir = data_dir.AppendASCII("filter_unittests");
+
+    // Read data from the original file into buffer.
+    base::FilePath file_path;
+    file_path = data_dir.AppendASCII("google.txt");
+    ASSERT_TRUE(base::ReadFileToString(file_path, &source_buffer_));
+
+    // Read data from the encoded file into buffer.
+    base::FilePath encoded_file_path;
+    encoded_file_path = data_dir.AppendASCII("google.br");
+    ASSERT_TRUE(base::ReadFileToString(encoded_file_path, &encoded_buffer_));
+    ASSERT_GE(kDefaultBufferSize, static_cast<int>(encoded_buffer_.size()));
+  }
+
+  // Use filter to decode compressed data, and compare the decoded result with
+  // the orginal data.
+  // Parameters: |source| and |source_len| are original data and its size.
+  // |encoded_source| and |encoded_source_len| are compressed data and its size.
+  // |output_buffer_size| specifies the size of buffer to read out data from
+  // filter.
+  void DecodeAndCompareWithFilter(Filter* filter,
+                                  const char* source,
+                                  int source_len,
+                                  const char* encoded_source,
+                                  int encoded_source_len,
+                                  int output_buffer_size) {
+    // Make sure we have enough space to hold the decoding output.
+    ASSERT_GE(kDefaultBufferSize, source_len);
+    ASSERT_GE(kDefaultBufferSize, output_buffer_size);
+
+    char decode_buffer[kDefaultBufferSize];
+    char* decode_next = decode_buffer;
+    int decode_avail_size = kDefaultBufferSize;
+
+    const char* encode_next = encoded_source;
+    int encode_avail_size = encoded_source_len;
+
+    int code = Filter::FILTER_OK;
+    while (code != Filter::FILTER_DONE) {
+      int encode_data_len =
+          std::min(encode_avail_size, filter->stream_buffer_size());
+      memcpy(filter->stream_buffer()->data(), encode_next, encode_data_len);
+      filter->FlushStreamBuffer(encode_data_len);
+      encode_next += encode_data_len;
+      encode_avail_size -= encode_data_len;
+
+      while (true) {
+        int decode_data_len = std::min(decode_avail_size, output_buffer_size);
+
+        code = filter->ReadData(decode_next, &decode_data_len);
+        decode_next += decode_data_len;
+        decode_avail_size -= decode_data_len;
+
+        ASSERT_NE(Filter::FILTER_ERROR, code);
+
+        if (code == Filter::FILTER_NEED_MORE_DATA ||
+            code == Filter::FILTER_DONE) {
+          break;
+        }
+      }
+    }
+
+    // Compare the decoding result with source data
+    int decode_total_data_len = kDefaultBufferSize - decode_avail_size;
+    EXPECT_EQ(source_len, decode_total_data_len);
+    EXPECT_EQ(memcmp(source, decode_buffer, source_len), 0);
+  }
+
+  // Unsafe function to use filter to decode compressed data.
+  // Parameters: |source| and |source_len| are compressed data and its size.
+  // |dest| is the buffer for decoding results. Upon entry, |*dest_len| is the
+  // size of the output buffer. Upon exit, |*dest_len| is the number of chars
+  // written into the buffer.
+  int DecodeAllWithFilter(Filter* filter,
+                          const char* source,
+                          int source_len,
+                          char* dest,
+                          int* dest_len) {
+    memcpy(filter->stream_buffer()->data(), source, source_len);
+    filter->FlushStreamBuffer(source_len);
+    return filter->ReadData(dest, dest_len);
+  }
+
+  void InitFilter() {
+    std::vector<Filter::FilterType> filter_types;
+    filter_types.push_back(Filter::FILTER_TYPE_BROTLI);
+    filter_.reset(Filter::Factory(filter_types, filter_context_));
+    ASSERT_TRUE(filter_.get());
+    ASSERT_LE(kDefaultBufferSize, filter_->stream_buffer_size());
+  }
+
+  void InitFilterWithBufferSize(int buffer_size) {
+    std::vector<Filter::FilterType> filter_types;
+    filter_types.push_back(Filter::FILTER_TYPE_BROTLI);
+    filter_.reset(
+        Filter::FactoryForTests(filter_types, filter_context_, buffer_size));
+    ASSERT_TRUE(filter_.get());
+  }
+
+  const char* source_buffer() const { return source_buffer_.data(); }
+  int source_len() const { return static_cast<int>(source_buffer_.size()); }
+
+  const char* encoded_buffer() const { return encoded_buffer_.data(); }
+  int encoded_len() const { return static_cast<int>(encoded_buffer_.size()); }
+
+  scoped_ptr<Filter> filter_;
+
+ private:
+  MockFilterContext filter_context_;
+  std::string source_buffer_;
+  std::string encoded_buffer_;
+};
+
+// Basic scenario: decoding brotli data with big enough buffer.
+TEST_F(BrotliUnitTest, DecodeBrotli) {
+  InitFilter();
+  memcpy(filter_->stream_buffer()->data(), encoded_buffer(), encoded_len());
+  filter_->FlushStreamBuffer(encoded_len());
+
+  ASSERT_GE(kDefaultBufferSize, source_len());
+  char decode_buffer[kDefaultBufferSize];
+  int decode_size = kDefaultBufferSize;
+  filter_->ReadData(decode_buffer, &decode_size);
+
+  // Compare the decoding result with source data
+  EXPECT_EQ(source_len(), decode_size);
+  EXPECT_EQ(memcmp(source_buffer(), decode_buffer, source_len()), 0);
+}
+
+// Tests we can call filter repeatedly to get all the data decoded.
+// To do that, we create a filter with a small buffer that can not hold all
+// the input data.
+TEST_F(BrotliUnitTest, DecodeWithSmallBuffer) {
+  InitFilterWithBufferSize(kSmallBufferSize);
+  EXPECT_EQ(kSmallBufferSize, filter_->stream_buffer_size());
+  DecodeAndCompareWithFilter(filter_.get(), source_buffer(), source_len(),
+                             encoded_buffer(), encoded_len(),
+                             kDefaultBufferSize);
+}
+
+// Tests we can still decode with just 1 byte buffer in the filter.
+// The purpose of this test: sometimes the filter will consume input without
+// generating output. Verify filter can handle it correctly.
+TEST_F(BrotliUnitTest, DecodeWithOneByteBuffer) {
+  InitFilterWithBufferSize(1);
+  EXPECT_EQ(1, filter_->stream_buffer_size());
+  DecodeAndCompareWithFilter(filter_.get(), source_buffer(), source_len(),
+                             encoded_buffer(), encoded_len(),
+                             kDefaultBufferSize);
+}
+
+// Tests we can decode when caller has small buffer to read out from filter.
+TEST_F(BrotliUnitTest, DecodeWithSmallOutputBuffer) {
+  InitFilter();
+  DecodeAndCompareWithFilter(filter_.get(), source_buffer(), source_len(),
+                             encoded_buffer(), encoded_len(), kSmallBufferSize);
+}
+
+// Tests we can decode when caller has small buffer and input is also broken
+// into small parts. This may uncover some corner cases that doesn't happen with
+// one-byte buffers.
+TEST_F(BrotliUnitTest, DecodeWithSmallInputAndOutputBuffer) {
+  InitFilterWithBufferSize(kSmallBufferSize);
+  DecodeAndCompareWithFilter(filter_.get(), source_buffer(), source_len(),
+                             encoded_buffer(), encoded_len(), kSmallBufferSize);
+}
+
+// Tests we can still decode with just 1 byte buffer in the filter and just 1
+// byte buffer in the caller.
+TEST_F(BrotliUnitTest, DecodeWithOneByteInputAndOutputBuffer) {
+  InitFilterWithBufferSize(1);
+  EXPECT_EQ(1, filter_->stream_buffer_size());
+  DecodeAndCompareWithFilter(filter_.get(), source_buffer(), source_len(),
+                             encoded_buffer(), encoded_len(), 1);
+}
+
+// Decoding deflate stream with corrupted data.
+TEST_F(BrotliUnitTest, DecodeCorruptedData) {
+  char corrupt_data[kDefaultBufferSize];
+  int corrupt_data_len = encoded_len();
+  memcpy(corrupt_data, encoded_buffer(), encoded_len());
+
+  int pos = corrupt_data_len / 2;
+  corrupt_data[pos] = !corrupt_data[pos];
+
+  // Decode the corrupted data with filter
+  InitFilter();
+  char corrupt_decode_buffer[kDefaultBufferSize];
+  int corrupt_decode_size = kDefaultBufferSize;
+
+  int code = DecodeAllWithFilter(filter_.get(), corrupt_data, corrupt_data_len,
+                                 corrupt_decode_buffer, &corrupt_decode_size);
+
+  // Expect failures
+  EXPECT_EQ(Filter::FILTER_ERROR, code);
+}
+
+// Decoding deflate stream with missing data.
+TEST_F(BrotliUnitTest, DecodeMissingData) {
+  char corrupt_data[kDefaultBufferSize];
+  int corrupt_data_len = encoded_len();
+  memcpy(corrupt_data, encoded_buffer(), encoded_len());
+
+  int pos = corrupt_data_len / 2;
+  int len = corrupt_data_len - pos - 1;
+  memmove(&corrupt_data[pos], &corrupt_data[pos + 1], len);
+  --corrupt_data_len;
+
+  // Decode the corrupted data with filter
+  InitFilter();
+  char corrupt_decode_buffer[kDefaultBufferSize];
+  int corrupt_decode_size = kDefaultBufferSize;
+
+  int code = DecodeAllWithFilter(filter_.get(), corrupt_data, corrupt_data_len,
+                                 corrupt_decode_buffer, &corrupt_decode_size);
+
+  // Expect failures
+  EXPECT_EQ(Filter::FILTER_ERROR, code);
+}
+
+}  // namespace net
diff --git a/net/filter/filter.cc b/net/filter/filter.cc
index f9c3712..466aa10 100644
--- a/net/filter/filter.cc
+++ b/net/filter/filter.cc
@@ -28,6 +28,7 @@
 #include "base/values.h"
 #include "net/base/io_buffer.h"
 #include "net/base/sdch_net_log_params.h"
+#include "net/filter/brotli_filter.h"
 #include "net/filter/gzip_filter.h"
 #include "net/filter/sdch_filter.h"
 #include "net/url_request/url_request_context.h"
@@ -38,6 +39,7 @@ namespace net {
 namespace {
 
 // Filter types (using canonical lower case only):
+const char kBrotli[]       = "br";
 const char kDeflate[]      = "deflate";
 const char kGZip[]         = "gzip";
 const char kXGZip[]        = "x-gzip";
@@ -62,6 +64,8 @@ void LogSdchProblem(const FilterContext& filter_context,
 
 std::string FilterTypeAsString(Filter::FilterType type_id) {
   switch (type_id) {
+    case Filter::FILTER_TYPE_BROTLI:
+      return "FILTER_TYPE_BROTLI";
     case Filter::FILTER_TYPE_DEFLATE:
       return "FILTER_TYPE_DEFLATE";
     case Filter::FILTER_TYPE_GZIP:
@@ -184,7 +188,9 @@ bool Filter::FlushStreamBuffer(int stream_data_len) {
 Filter::FilterType Filter::ConvertEncodingToType(
     const std::string& filter_type) {
   FilterType type_id;
-  if (base::LowerCaseEqualsASCII(filter_type, kDeflate)) {
+  if (base::LowerCaseEqualsASCII(filter_type, kBrotli)) {
+    type_id = FILTER_TYPE_BROTLI;
+  } else if (base::LowerCaseEqualsASCII(filter_type, kDeflate)) {
     type_id = FILTER_TYPE_DEFLATE;
   } else if (base::LowerCaseEqualsASCII(filter_type, kGZip) ||
              base::LowerCaseEqualsASCII(filter_type, kXGZip)) {
@@ -350,6 +356,16 @@ Filter::FilterStatus Filter::CopyOut(char* dest_buffer, int* dest_len) {
 }
 
 // static
+Filter* Filter::InitBrotliFilter(FilterType type_id, int buffer_size) {
+  scoped_ptr<Filter> brotli_filter(CreateBrotliFilter(type_id));
+  if (!brotli_filter.get())
+    return nullptr;
+
+  brotli_filter->InitBuffer(buffer_size);
+  return brotli_filter.release();
+}
+
+// static
 Filter* Filter::InitGZipFilter(FilterType type_id, int buffer_size) {
   scoped_ptr<GZipFilter> gz_filter(new GZipFilter(type_id));
   gz_filter->InitBuffer(buffer_size);
@@ -372,6 +388,9 @@ Filter* Filter::PrependNewFilter(FilterType type_id,
                                  Filter* filter_list) {
   scoped_ptr<Filter> first_filter;  // Soon to be start of chain.
   switch (type_id) {
+    case FILTER_TYPE_BROTLI:
+      first_filter.reset(InitBrotliFilter(type_id, buffer_size));
+      break;
     case FILTER_TYPE_GZIP_HELPING_SDCH:
     case FILTER_TYPE_DEFLATE:
     case FILTER_TYPE_GZIP:
diff --git a/net/filter/filter.h b/net/filter/filter.h
index 6e111e1..e9df3db 100644
--- a/net/filter/filter.h
+++ b/net/filter/filter.h
@@ -153,6 +153,7 @@ class NET_EXPORT_PRIVATE Filter {
 
   // Specifies type of filters that can be created.
   enum FilterType {
+    FILTER_TYPE_BROTLI,
     FILTER_TYPE_DEFLATE,
     FILTER_TYPE_GZIP,
     FILTER_TYPE_GZIP_HELPING_SDCH,  // Gzip possible, but pass through allowed.
@@ -233,6 +234,7 @@ class NET_EXPORT_PRIVATE Filter {
   std::string OrderedFilterList() const;
 
  protected:
+  friend class BrotliUnitTest;
   friend class GZipUnitTest;
   friend class SdchFilterChainingTest;
   FRIEND_TEST_ALL_PREFIXES(FilterTest, ThreeFilterChain);
@@ -285,6 +287,7 @@ class NET_EXPORT_PRIVATE Filter {
 
   // Helper methods for PrependNewFilter. If initialization is successful,
   // they return a fully initialized Filter. Otherwise, return NULL.
+  static Filter* InitBrotliFilter(FilterType type_id, int buffer_size);
   static Filter* InitGZipFilter(FilterType type_id, int buffer_size);
   static Filter* InitSdchFilter(FilterType type_id,
                                 const FilterContext& filter_context,