diff options
author | jar@google.com <jar@google.com@0039d316-1c4b-4281-b951-d872f2087c98> | 2008-10-27 17:39:28 +0000 |
---|---|---|
committer | jar@google.com <jar@google.com@0039d316-1c4b-4281-b951-d872f2087c98> | 2008-10-27 17:39:28 +0000 |
commit | 423041b0a7182411149472ae1e7fd87317173097 (patch) | |
tree | 494092141aa32f87e6bf02159579ae590d0a3fbf /net | |
parent | c05ef7e4fe5aaadb4193217209a11440bd4d2c27 (diff) | |
download | chromium_src-423041b0a7182411149472ae1e7fd87317173097.zip chromium_src-423041b0a7182411149472ae1e7fd87317173097.tar.gz chromium_src-423041b0a7182411149472ae1e7fd87317173097.tar.bz2 |
Clean up filter and content encoding handling
Centralize translation functions (text of "Content-Encoding" to enum)
in filter.cc
Centralize error recovery (for damaged content encoding headers) in filter.cc
Error recovery includes a loss of SDCH encoding headers, plus handling
of Apache server bug with gzip files are tagged as also being gzip encoded.
Centralize and add a pile of unit tests to this filter code.
r=openvcdiff,huanr
Review URL: http://codereview.chromium.org/8018
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@4004 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'net')
-rw-r--r-- | net/base/bzip2_filter_unittest.cc | 73 | ||||
-rw-r--r-- | net/base/filter.cc | 108 | ||||
-rw-r--r-- | net/base/filter.h | 45 | ||||
-rw-r--r-- | net/base/filter_unittest.cc | 133 | ||||
-rw-r--r-- | net/base/gzip_filter_unittest.cc | 91 | ||||
-rw-r--r-- | net/base/sdch_filter.cc | 10 | ||||
-rw-r--r-- | net/base/sdch_filter_unittest.cc | 121 | ||||
-rw-r--r-- | net/base/sdch_manager.cc | 10 | ||||
-rw-r--r-- | net/base/sdch_manager.h | 5 | ||||
-rw-r--r-- | net/build/net_unittests.vcproj | 10 | ||||
-rw-r--r-- | net/net_unittests.scons | 1 | ||||
-rw-r--r-- | net/url_request/url_request_http_job.cc | 121 | ||||
-rw-r--r-- | net/url_request/url_request_http_job.h | 3 | ||||
-rw-r--r-- | net/url_request/url_request_job.cc | 8 | ||||
-rw-r--r-- | net/url_request/url_request_job.h | 14 |
15 files changed, 408 insertions, 345 deletions
diff --git a/net/base/bzip2_filter_unittest.cc b/net/base/bzip2_filter_unittest.cc index 19a36af..19b5883 100644 --- a/net/base/bzip2_filter_unittest.cc +++ b/net/base/bzip2_filter_unittest.cc @@ -182,10 +182,9 @@ class BZip2FilterUnitTest : public PlatformTest { // Basic scenario: decoding bzip2 data with big enough buffer. TEST_F(BZip2FilterUnitTest, DecodeBZip2) { // Decode the compressed data with filter - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); memcpy(filter->stream_buffer(), bzip2_encode_buffer_, bzip2_encode_len_); filter->FlushStreamBuffer(bzip2_encode_len_); @@ -205,10 +204,9 @@ TEST_F(BZip2FilterUnitTest, DecodeBZip2) { // To do that, we create a filter with a small buffer that can not hold all // the input data. TEST_F(BZip2FilterUnitTest, DecodeWithSmallInputBuffer) { - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kSmallBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kSmallBufferSize)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), source_len(), bzip2_encode_buffer_, bzip2_encode_len_, @@ -217,10 +215,9 @@ TEST_F(BZip2FilterUnitTest, DecodeWithSmallInputBuffer) { // Tests we can decode when caller has small buffer to read out from filter. TEST_F(BZip2FilterUnitTest, DecodeWithSmallOutputBuffer) { - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), source_len(), bzip2_encode_buffer_, bzip2_encode_len_, @@ -232,10 +229,9 @@ TEST_F(BZip2FilterUnitTest, DecodeWithSmallOutputBuffer) { // header correctly. (2) Sometimes the filter will consume input without // generating output. Verify filter can handle it correctly. TEST_F(BZip2FilterUnitTest, DecodeWithOneByteInputBuffer) { - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, 1)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, 1)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), source_len(), bzip2_encode_buffer_, bzip2_encode_len_, @@ -245,10 +241,9 @@ TEST_F(BZip2FilterUnitTest, DecodeWithOneByteInputBuffer) { // Tests we can still decode with just 1 byte buffer in the filter and just 1 // byte buffer in the caller. TEST_F(BZip2FilterUnitTest, DecodeWithOneByteInputAndOutputBuffer) { - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, 1)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, 1)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), source_len(), bzip2_encode_buffer_, bzip2_encode_len_, 1, false); @@ -264,10 +259,9 @@ TEST_F(BZip2FilterUnitTest, DecodeCorruptedData) { int corrupt_decode_size = kDefaultBufferSize; // Decode the correct data with filter - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter1( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter1(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter1.get()); Filter::FilterStatus code = DecodeAllWithFilter(filter1.get(), @@ -280,8 +274,7 @@ TEST_F(BZip2FilterUnitTest, DecodeCorruptedData) { EXPECT_TRUE(code == Filter::FILTER_DONE); // Decode the corrupted data with filter - scoped_ptr<Filter> filter2( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + scoped_ptr<Filter> filter2(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter2.get()); int pos = corrupt_data_len / 2; @@ -309,10 +302,9 @@ TEST_F(BZip2FilterUnitTest, DecodeMissingData) { --corrupt_data_len; // Decode the corrupted data with filter - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); char corrupt_decode_buffer[kDefaultBufferSize]; int corrupt_decode_size = kDefaultBufferSize; @@ -335,10 +327,9 @@ TEST_F(BZip2FilterUnitTest, DecodeCorruptedHeader) { corrupt_data[2] = !corrupt_data[2]; // Decode the corrupted data with filter - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); char corrupt_decode_buffer[kDefaultBufferSize]; int corrupt_decode_size = kDefaultBufferSize; @@ -362,10 +353,9 @@ TEST_F(BZip2FilterUnitTest, DecodeWithExtraDataAndSmallOutputBuffer) { memcpy(more_data, bzip2_encode_buffer_, bzip2_encode_len_); memcpy(more_data + bzip2_encode_len_, kExtraData, kExtraDataBufferSize); - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), @@ -382,10 +372,9 @@ TEST_F(BZip2FilterUnitTest, DecodeWithExtraDataAndSmallInputBuffer) { memcpy(more_data, bzip2_encode_buffer_, bzip2_encode_len_); memcpy(more_data + bzip2_encode_len_, kExtraData, kExtraDataBufferSize); - std::vector<std::string> filters; - filters.push_back("bzip2"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kSmallBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_BZIP2); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kSmallBufferSize)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), diff --git a/net/base/filter.cc b/net/base/filter.cc index 1174c2f..d8717db 100644 --- a/net/base/filter.cc +++ b/net/base/filter.cc @@ -11,7 +11,7 @@ namespace { -// Filter types: +// Filter types (using canonical lower case only): const char kDeflate[] = "deflate"; const char kGZip[] = "gzip"; const char kXGZip[] = "x-gzip"; @@ -32,53 +32,34 @@ const char kApplicationGzip[] = "application/gzip"; const char kApplicationXGunzip[] = "application/x-gunzip"; const char kApplicationXCompress[] = "application/x-compress"; const char kApplicationCompress[] = "application/compress"; +const char kTextHtml[] = "text/html"; } // namespace -Filter* Filter::Factory(const std::vector<std::string>& filter_types, - const std::string& mime_type, +Filter* Filter::Factory(const std::vector<FilterType>& filter_types, int buffer_size) { if (filter_types.empty() || buffer_size < 0) return NULL; - std::string safe_mime_type = (filter_types.size() > 1) ? "" : mime_type; Filter* filter_list = NULL; // Linked list of filters. - FilterType type_id = FILTER_TYPE_UNSUPPORTED; for (size_t i = 0; i < filter_types.size(); i++) { - type_id = ConvertEncodingToType(filter_types[i], safe_mime_type); - filter_list = PrependNewFilter(type_id, buffer_size, filter_list); + filter_list = PrependNewFilter(filter_types[i], buffer_size, filter_list); if (!filter_list) return NULL; } - // Handle proxy that changes content encoding "sdch,gzip" into "sdch". - if (1 == filter_types.size() && FILTER_TYPE_SDCH == type_id) - filter_list = PrependNewFilter(FILTER_TYPE_GZIP_HELPING_SDCH, buffer_size, - filter_list); return filter_list; } // static -Filter::FilterType Filter::ConvertEncodingToType(const std::string& filter_type, - const std::string& mime_type) { +Filter::FilterType Filter::ConvertEncodingToType( + const std::string& filter_type) { FilterType type_id; if (LowerCaseEqualsASCII(filter_type, kDeflate)) { type_id = FILTER_TYPE_DEFLATE; } else if (LowerCaseEqualsASCII(filter_type, kGZip) || LowerCaseEqualsASCII(filter_type, kXGZip)) { - if (LowerCaseEqualsASCII(mime_type, kApplicationXGzip) || - LowerCaseEqualsASCII(mime_type, kApplicationGzip) || - LowerCaseEqualsASCII(mime_type, kApplicationXGunzip)) { - // The server has told us that it sent us gziped content with a gzip - // content encoding. Sadly, Apache mistakenly sets these headers for all - // .gz files. We match Firefox's nsHttpChannel::ProcessNormal and ignore - // the Content-Encoding here. - // TODO(jar): Move all this encoding type "fixup" into the - // GetContentEncoding() methods. Combine this defaulting with SDCH fixup. - type_id = FILTER_TYPE_UNSUPPORTED; - } else { - type_id = FILTER_TYPE_GZIP; - } + type_id = FILTER_TYPE_GZIP; } else if (LowerCaseEqualsASCII(filter_type, kBZip2) || LowerCaseEqualsASCII(filter_type, kXBZip2)) { type_id = FILTER_TYPE_BZIP2; @@ -93,6 +74,79 @@ Filter::FilterType Filter::ConvertEncodingToType(const std::string& filter_type, } // static +void Filter::FixupEncodingTypes( + bool is_sdch_response, + const std::string& mime_type, + std::vector<FilterType>* encoding_types) { + + if ((1 == encoding_types->size()) && + (FILTER_TYPE_GZIP == encoding_types->front())) { + if (LowerCaseEqualsASCII(mime_type, kApplicationXGzip) || + LowerCaseEqualsASCII(mime_type, kApplicationGzip) || + LowerCaseEqualsASCII(mime_type, kApplicationXGunzip)) + // The server has told us that it sent us gziped content with a gzip + // content encoding. Sadly, Apache mistakenly sets these headers for all + // .gz files. We match Firefox's nsHttpChannel::ProcessNormal and ignore + // the Content-Encoding here. + encoding_types->clear(); + return; + } + + if (!is_sdch_response) + return; + + // If content encoding included SDCH, then everything is fine. + if (!encoding_types->empty() && + (FILTER_TYPE_SDCH == encoding_types->front())) { + // Some proxies (found currently in Argentina) strip the Content-Encoding + // text from "sdch,gzip" to a mere "sdch" without modifying the compressed + // payload. To handle this gracefully, we simulate the "probably" deleted + // ",gzip" by appending a tentative gzip decode, which will default to a + // no-op pass through filter if it doesn't get gzip headers where expected. + if (1 == encoding_types->size()) + encoding_types->push_back(FILTER_TYPE_GZIP_HELPING_SDCH); + return; + } + + // SDCH "search results" protective hack: To make sure we don't break the only + // currently deployed SDCH enabled server! Be VERY cautious about proxies that + // strip all content-encoding to not include sdch. IF we don't see content + // encodings that seem to match what we'd expect from a server that asked us + // to use a dictionary (and we advertised said dictionary in the GET), then + // we set the encoding to (try to) use SDCH to decode. Note that SDCH will + // degrade into a pass-through filter if it doesn't have a viable dictionary + // hash in its header. Also note that a solo "sdch" will implicitly create + // a "sdch,gzip" decoding filter, where the gzip portion will degrade to a + // pass through if a gzip header is not encountered. Hence we can replace + // "gzip" with "sdch" and "everything will work." + // The one failure mode comes when we advertise a dictionary, and the server + // tries to *send* a gzipped file (not gzip encode content), and then we could + // do a gzip decode :-(. Since current server support does not ever see such + // a transfer, we are safe (for now). + if (LowerCaseEqualsASCII(mime_type, kTextHtml)) { + // Suspicious case: Advertised dictionary, but server didn't use sdch, even + // though it is text_html content. + if (encoding_types->empty()) + SdchManager::SdchErrorRecovery(SdchManager::ADDED_CONTENT_ENCODING); + else if (1 == encoding_types->size()) + SdchManager::SdchErrorRecovery(SdchManager::FIXED_CONTENT_ENCODING); + else + SdchManager::SdchErrorRecovery(SdchManager::FIXED_CONTENT_ENCODINGS); + encoding_types->clear(); + encoding_types->push_back(FILTER_TYPE_SDCH); + encoding_types->push_back(FILTER_TYPE_GZIP_HELPING_SDCH); + return; + } + + // It didn't have SDCH encoding... but it wasn't HTML... so maybe it really + // wasn't SDCH encoded. It would be nice if we knew this, and didn't bother + // to propose a dictionary etc., but current SDCH spec does not provide a nice + // way for us to conclude that. Perhaps in the future, this case will be much + // more rare. + return; +} + +// static Filter* Filter::PrependNewFilter(FilterType type_id, int buffer_size, Filter* filter_list) { Filter* first_filter = NULL; // Soon to be start of chain. @@ -247,7 +301,7 @@ void Filter::SetURL(const GURL& url) { next_filter_->SetURL(url); } -void Filter::SetMimeType(std::string& mime_type) { +void Filter::SetMimeType(const std::string& mime_type) { mime_type_ = mime_type; if (next_filter_.get()) next_filter_->SetMimeType(mime_type); diff --git a/net/base/filter.h b/net/base/filter.h index ab61732..98a531c 100644 --- a/net/base/filter.h +++ b/net/base/filter.h @@ -55,6 +55,17 @@ class Filter { FILTER_ERROR }; + // Specifies type of filters that can be created. + enum FilterType { + FILTER_TYPE_DEFLATE, + FILTER_TYPE_GZIP, + FILTER_TYPE_BZIP2, + FILTER_TYPE_GZIP_HELPING_SDCH, + FILTER_TYPE_SDCH, + FILTER_TYPE_UNSUPPORTED, + }; + + virtual ~Filter(); // Creates a Filter object. @@ -71,8 +82,7 @@ class Filter { // (decoding) order. For example, types[0] = "sdch", types[1] = "gzip" will // cause data to first be gunizip filtered, and the resulting output from that // filter will be sdch decoded. - static Filter* Factory(const std::vector<std::string>& filter_types, - const std::string& mime_type, + static Filter* Factory(const std::vector<FilterType>& filter_types, int buffer_size); // External call to obtain data from this filter chain. If ther is no @@ -107,20 +117,24 @@ class Filter { void SetURL(const GURL& url); const GURL& url() const { return url_; } - void SetMimeType(std::string& mime_type); + void SetMimeType(const std::string& mime_type); const std::string& mime_type() const { return mime_type_; } + // Translate the text of a filter name (from Content-Encoding header) into a + // FilterType. + static FilterType ConvertEncodingToType(const std::string& filter_type); + + // Given a array of encoding_types, try to do some error recovery adjustment + // to the list. This includes handling known bugs in the Apache server (where + // redundant gzip encoding is specified), as well as issues regarding SDCH + // encoding, where various proxies and anti-virus products modify or strip the + // encodings. These fixups require context, which includes whether this + // response was made to an SDCH request (i.e., an available dictionary was + // advertised in the GET), as well as the mime type of the content. + static void FixupEncodingTypes(bool is_sdch_response, + const std::string& mime_type, + std::vector<FilterType>* encoding_types); protected: - // Specifies type of filters that can be created. - enum FilterType { - FILTER_TYPE_DEFLATE, - FILTER_TYPE_GZIP, - FILTER_TYPE_BZIP2, - FILTER_TYPE_GZIP_HELPING_SDCH, - FILTER_TYPE_SDCH, // open-vcdiff compression relative to a dictionary. - FILTER_TYPE_UNSUPPORTED - }; - Filter(); FRIEND_TEST(SdchFilterTest, ContentTypeId); @@ -144,11 +158,6 @@ class Filter { // Buffer_size is the maximum size of stream_buffer_ in number of chars. bool InitBuffer(int buffer_size); - // Translate the text of a filter name (from Content-Encoding header) into a - // FilterType, in the context of a mime type. - static FilterType ConvertEncodingToType(const std::string& filter_type, - const std::string& mime_type); - // A factory helper for creating filters for within a chain of potentially // multiple encodings. If a chain of filters is created, then this may be // called multiple times during the filter creation process. In most simple diff --git a/net/base/filter_unittest.cc b/net/base/filter_unittest.cc new file mode 100644 index 0000000..ce04574 --- /dev/null +++ b/net/base/filter_unittest.cc @@ -0,0 +1,133 @@ +// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "net/base/filter.h" +#include "testing/gtest/include/gtest/gtest.h" + +class FilterTest : public testing::Test { +}; + +TEST(FilterTest, ContentTypeId) { + // Check for basic translation of Content-Encoding, including case variations. + EXPECT_EQ(Filter::FILTER_TYPE_DEFLATE, + Filter::ConvertEncodingToType("deflate")); + EXPECT_EQ(Filter::FILTER_TYPE_DEFLATE, + Filter::ConvertEncodingToType("deflAte")); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP, + Filter::ConvertEncodingToType("gzip")); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP, + Filter::ConvertEncodingToType("GzIp")); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP, + Filter::ConvertEncodingToType("x-gzip")); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP, + Filter::ConvertEncodingToType("X-GzIp")); + EXPECT_EQ(Filter::FILTER_TYPE_BZIP2, + Filter::ConvertEncodingToType("bzip2")); + EXPECT_EQ(Filter::FILTER_TYPE_BZIP2, + Filter::ConvertEncodingToType("BZiP2")); + EXPECT_EQ(Filter::FILTER_TYPE_BZIP2, + Filter::ConvertEncodingToType("x-bzip2")); + EXPECT_EQ(Filter::FILTER_TYPE_BZIP2, + Filter::ConvertEncodingToType("X-BZiP2")); + EXPECT_EQ(Filter::FILTER_TYPE_SDCH, + Filter::ConvertEncodingToType("sdch")); + EXPECT_EQ(Filter::FILTER_TYPE_SDCH, + Filter::ConvertEncodingToType("sDcH")); + EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, + Filter::ConvertEncodingToType("weird")); + EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, + Filter::ConvertEncodingToType("strange")); +} + +// Check various fixups that modify content encoding lists. +TEST(FilterTest, ApacheGzip) { + // Check that redundant gzip mime type removes only solo gzip encoding. + const bool is_sdch_response = false; + const std::string kGzipMime1("application/x-gzip"); + const std::string kGzipMime2("application/gzip"); + const std::string kGzipMime3("application/x-gunzip"); + std::vector<Filter::FilterType> encoding_types; + + // First show it removes the gzip, given any gzip style mime type. + encoding_types.clear(); + encoding_types.push_back(Filter::FILTER_TYPE_GZIP); + Filter::FixupEncodingTypes(is_sdch_response, kGzipMime1, &encoding_types); + EXPECT_TRUE(encoding_types.empty()); + + encoding_types.clear(); + encoding_types.push_back(Filter::FILTER_TYPE_GZIP); + Filter::FixupEncodingTypes(is_sdch_response, kGzipMime2, &encoding_types); + EXPECT_TRUE(encoding_types.empty()); + + encoding_types.clear(); + encoding_types.push_back(Filter::FILTER_TYPE_GZIP); + Filter::FixupEncodingTypes(is_sdch_response, kGzipMime3, &encoding_types); + EXPECT_TRUE(encoding_types.empty()); + + // Check to be sure it doesn't remove everything when it has such a type. + encoding_types.clear(); + encoding_types.push_back(Filter::FILTER_TYPE_SDCH); + Filter::FixupEncodingTypes(is_sdch_response, kGzipMime1, &encoding_types); + EXPECT_EQ(1, encoding_types.size()); + EXPECT_EQ(Filter::FILTER_TYPE_SDCH, encoding_types.front()); + + // Check to be sure that gzip can survive with other mime types. + encoding_types.clear(); + encoding_types.push_back(Filter::FILTER_TYPE_GZIP); + Filter::FixupEncodingTypes(is_sdch_response, "other/mime", &encoding_types); + EXPECT_EQ(1, encoding_types.size()); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP, encoding_types.front()); +} + +TEST(FilterTest, SdchEncoding) { + // Handle content encodings including SDCH. + const bool is_sdch_response = true; + const std::string kTextHtmlMime("text/html"); + std::vector<Filter::FilterType> encoding_types; + + // Check for most common encoding, and verify it survives unchanged. + encoding_types.clear(); + encoding_types.push_back(Filter::FILTER_TYPE_SDCH); + encoding_types.push_back(Filter::FILTER_TYPE_GZIP); + Filter::FixupEncodingTypes(is_sdch_response, kTextHtmlMime, &encoding_types); + EXPECT_EQ(2, encoding_types.size()); + EXPECT_EQ(Filter::FILTER_TYPE_SDCH, encoding_types[0]); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP, encoding_types[1]); + + // Unchanged even with other mime types. + encoding_types.clear(); + encoding_types.push_back(Filter::FILTER_TYPE_SDCH); + encoding_types.push_back(Filter::FILTER_TYPE_GZIP); + Filter::FixupEncodingTypes(is_sdch_response, "other/type", &encoding_types); + EXPECT_EQ(2, encoding_types.size()); + EXPECT_EQ(Filter::FILTER_TYPE_SDCH, encoding_types[0]); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP, encoding_types[1]); + + // Solo SDCH is extended to include optional gunzip. + encoding_types.clear(); + encoding_types.push_back(Filter::FILTER_TYPE_SDCH); + Filter::FixupEncodingTypes(is_sdch_response, "other/type", &encoding_types); + EXPECT_EQ(2, encoding_types.size()); + EXPECT_EQ(Filter::FILTER_TYPE_SDCH, encoding_types[0]); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP_HELPING_SDCH, encoding_types[1]); +} + +TEST(FilterTest, MissingSdchEncoding) { + // Handle interesting case where entire SDCH encoding assertion "got lost." + const bool is_sdch_response = true; + const std::string kTextHtmlMime("text/html"); + std::vector<Filter::FilterType> encoding_types; + + // Loss of encoding, but it was an SDCH response with html type. + encoding_types.clear(); + Filter::FixupEncodingTypes(is_sdch_response, kTextHtmlMime, &encoding_types); + EXPECT_EQ(2, encoding_types.size()); + EXPECT_EQ(Filter::FILTER_TYPE_SDCH, encoding_types[0]); + EXPECT_EQ(Filter::FILTER_TYPE_GZIP_HELPING_SDCH, encoding_types[1]); + + // No encoding, but it was an SDCH response with non-html type. + encoding_types.clear(); + Filter::FixupEncodingTypes(is_sdch_response, "other/mime", &encoding_types); + EXPECT_TRUE(encoding_types.empty()); +} diff --git a/net/base/gzip_filter_unittest.cc b/net/base/gzip_filter_unittest.cc index 22c1ae3..b58758f 100644 --- a/net/base/gzip_filter_unittest.cc +++ b/net/base/gzip_filter_unittest.cc @@ -74,7 +74,7 @@ class GZipUnitTest : public PlatformTest { int code = CompressAll(ENCODE_DEFLATE , source_buffer(), source_len(), deflate_encode_buffer_, &deflate_encode_len_); ASSERT_TRUE(code == Z_STREAM_END); - ASSERT_TRUE(deflate_encode_len_ > 0); + ASSERT_GT(deflate_encode_len_, 0); ASSERT_TRUE(deflate_encode_len_ <= kDefaultBufferSize); // Encode the data with gzip @@ -85,7 +85,7 @@ class GZipUnitTest : public PlatformTest { code = CompressAll(ENCODE_GZIP, source_buffer(), source_len(), gzip_encode_buffer_, &gzip_encode_len_); ASSERT_TRUE(code == Z_STREAM_END); - ASSERT_TRUE(gzip_encode_len_ > 0); + ASSERT_GT(gzip_encode_len_, 0); ASSERT_TRUE(gzip_encode_len_ <= kDefaultBufferSize); } @@ -228,10 +228,9 @@ class GZipUnitTest : public PlatformTest { // Basic scenario: decoding deflate data with big enough buffer. TEST_F(GZipUnitTest, DecodeDeflate) { // Decode the compressed data with filter - std::vector<std::string> filters; - filters.push_back("deflate"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_DEFLATE); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); memcpy(filter->stream_buffer(), deflate_encode_buffer_, deflate_encode_len_); filter->FlushStreamBuffer(deflate_encode_len_); @@ -248,10 +247,9 @@ TEST_F(GZipUnitTest, DecodeDeflate) { // Basic scenario: decoding gzip data with big enough buffer. TEST_F(GZipUnitTest, DecodeGZip) { // Decode the compressed data with filter - std::vector<std::string> filters; - filters.push_back("gzip"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_GZIP); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); memcpy(filter->stream_buffer(), gzip_encode_buffer_, gzip_encode_len_); filter->FlushStreamBuffer(gzip_encode_len_); @@ -269,10 +267,9 @@ TEST_F(GZipUnitTest, DecodeGZip) { // To do that, we create a filter with a small buffer that can not hold all // the input data. TEST_F(GZipUnitTest, DecodeWithSmallBuffer) { - std::vector<std::string> filters; - filters.push_back("deflate"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kSmallBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_DEFLATE); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kSmallBufferSize)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), source_len(), deflate_encode_buffer_, deflate_encode_len_, @@ -284,10 +281,9 @@ TEST_F(GZipUnitTest, DecodeWithSmallBuffer) { // header correctly. (2) Sometimes the filter will consume input without // generating output. Verify filter can handle it correctly. TEST_F(GZipUnitTest, DecodeWithOneByteBuffer) { - std::vector<std::string> filters; - filters.push_back("gzip"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, 1)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_GZIP); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, 1)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), source_len(), gzip_encode_buffer_, gzip_encode_len_, @@ -296,10 +292,9 @@ TEST_F(GZipUnitTest, DecodeWithOneByteBuffer) { // Tests we can decode when caller has small buffer to read out from filter. TEST_F(GZipUnitTest, DecodeWithSmallOutputBuffer) { - std::vector<std::string> filters; - filters.push_back("deflate"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_DEFLATE); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), source_len(), deflate_encode_buffer_, deflate_encode_len_, @@ -309,10 +304,9 @@ TEST_F(GZipUnitTest, DecodeWithSmallOutputBuffer) { // Tests we can still decode with just 1 byte buffer in the filter and just 1 // byte buffer in the caller. TEST_F(GZipUnitTest, DecodeWithOneByteInputAndOutputBuffer) { - std::vector<std::string> filters; - filters.push_back("gzip"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, 1)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_GZIP); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, 1)); ASSERT_TRUE(filter.get()); DecodeAndCompareWithFilter(filter.get(), source_buffer(), source_len(), gzip_encode_buffer_, gzip_encode_len_, 1); @@ -328,10 +322,9 @@ TEST_F(GZipUnitTest, DecodeCorruptedData) { corrupt_data[pos] = !corrupt_data[pos]; // Decode the corrupted data with filter - std::vector<std::string> filters; - filters.push_back("deflate"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_DEFLATE); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); char corrupt_decode_buffer[kDefaultBufferSize]; int corrupt_decode_size = kDefaultBufferSize; @@ -355,10 +348,9 @@ TEST_F(GZipUnitTest, DecodeMissingData) { --corrupt_data_len; // Decode the corrupted data with filter - std::vector<std::string> filters; - filters.push_back("deflate"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_DEFLATE); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); char corrupt_decode_buffer[kDefaultBufferSize]; int corrupt_decode_size = kDefaultBufferSize; @@ -379,10 +371,9 @@ TEST_F(GZipUnitTest, DecodeCorruptedHeader) { corrupt_data[2] = !corrupt_data[2]; // Decode the corrupted data with filter - std::vector<std::string> filters; - filters.push_back("gzip"); - scoped_ptr<Filter> filter( - Filter::Factory(filters, kApplicationOctetStream, kDefaultBufferSize)); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_GZIP); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kDefaultBufferSize)); ASSERT_TRUE(filter.get()); char corrupt_decode_buffer[kDefaultBufferSize]; int corrupt_decode_size = kDefaultBufferSize; @@ -394,28 +385,4 @@ TEST_F(GZipUnitTest, DecodeCorruptedHeader) { EXPECT_TRUE(code == Filter::FILTER_ERROR); } -TEST_F(GZipUnitTest, ApacheWorkaround) { - const int kBufferSize = kDefaultBufferSize; // To fit in 80 cols. - scoped_ptr<Filter> filter; - - std::vector<std::string> gzip_filters, x_gzip_filters; - gzip_filters.push_back("gzip"); - x_gzip_filters.push_back("x-gzip"); - - filter.reset(Filter::Factory(gzip_filters, kApplicationXGzip, kBufferSize)); - EXPECT_FALSE(filter.get()); - filter.reset(Filter::Factory(gzip_filters, kApplicationGzip, kBufferSize)); - EXPECT_FALSE(filter.get()); - filter.reset(Filter::Factory(gzip_filters, kApplicationXGunzip, kBufferSize)); - EXPECT_FALSE(filter.get()); - - filter.reset(Filter::Factory(x_gzip_filters, kApplicationXGzip, kBufferSize)); - EXPECT_FALSE(filter.get()); - filter.reset(Filter::Factory(x_gzip_filters, kApplicationGzip, kBufferSize)); - EXPECT_FALSE(filter.get()); - filter.reset(Filter::Factory(x_gzip_filters, kApplicationXGunzip, - kBufferSize)); - EXPECT_FALSE(filter.get()); -} - } // namespace diff --git a/net/base/sdch_filter.cc b/net/base/sdch_filter.cc index 9be9530..6de5c6a 100644 --- a/net/base/sdch_filter.cc +++ b/net/base/sdch_filter.cc @@ -29,7 +29,7 @@ SdchFilter::~SdchFilter() { static int filter_use_count = 0; ++filter_use_count; if (META_REFRESH_RECOVERY == decoding_status_) { - HISTOGRAM_COUNTS(L"Sdch.FilterUseBeforeDisabling", filter_use_count); + UMA_HISTOGRAM_COUNTS(L"Sdch.FilterUseBeforeDisabling", filter_use_count); } if (vcdiff_streaming_decoder_.get()) { @@ -37,9 +37,8 @@ SdchFilter::~SdchFilter() { decoding_status_ = DECODING_ERROR; } - // TODO(jar): Use UMA_HISTOGRAM when we turn sdch on by default. - HISTOGRAM_COUNTS(L"Sdch.Bytes read", source_bytes_); - HISTOGRAM_COUNTS(L"Sdch.Bytes output", output_bytes_); + UMA_HISTOGRAM_COUNTS(L"Sdch.Bytes read", source_bytes_); + UMA_HISTOGRAM_COUNTS(L"Sdch.Bytes output", output_bytes_); if (dictionary_) dictionary_->Release(); @@ -82,6 +81,8 @@ Filter::FilterStatus SdchFilter::ReadFilteredData(char* dest_buffer, DCHECK(0 == dest_buffer_excess_index_); DCHECK(dest_buffer_excess_.empty()); if (!dictionary_hash_is_plausible_) { + // One of the first 9 bytes precluded consideration as a hash. + // This can't be an SDCH payload. SdchManager::SdchErrorRecovery(SdchManager::PASSING_THROUGH_NON_SDCH); decoding_status_ = PASS_THROUGH; dest_buffer_excess_ = dictionary_hash_; // Send what we scanned. @@ -185,6 +186,7 @@ Filter::FilterStatus SdchFilter::InitializeDictionary() { if (!dictionary_) { DCHECK(dictionary_hash_.size() == kServerIdLength); + // Since dictionary was not found, check to see if hash was even plausible. for (size_t i = 0; i < kServerIdLength - 1; ++i) { char base64_char = dictionary_hash_[i]; if (!isalnum(base64_char) && '-' != base64_char && '_' != base64_char) { diff --git a/net/base/sdch_filter_unittest.cc b/net/base/sdch_filter_unittest.cc index 0b0ba8c..58a32e8 100644 --- a/net/base/sdch_filter_unittest.cc +++ b/net/base/sdch_filter_unittest.cc @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// TODO(jar): Change file name to *_unittest.cc (with double "t'). - #include <algorithm> #include <string> #include <vector> @@ -130,12 +128,11 @@ static std::string NewSdchDictionary(const std::string& domain) { //------------------------------------------------------------------------------ TEST_F(SdchFilterTest, BasicBadDictionary) { - std::vector<std::string> filters; - filters.push_back("sdch"); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_SDCH); const int kInputBufferSize(30); char output_buffer[20]; - scoped_ptr<Filter> filter(Filter::Factory(filters, "missing-mime", - kInputBufferSize)); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kInputBufferSize)); std::string url_string("http://ignore.com"); filter->SetURL(GURL(url_string)); @@ -225,13 +222,12 @@ TEST_F(SdchFilterTest, BasicDictionary) { std::string compressed(NewSdchCompressedData(dictionary)); - std::vector<std::string> filters; - filters.push_back("sdch"); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_SDCH); // Decode with a large buffer (larger than test input, or compressed data). const int kInputBufferSize(100); - scoped_ptr<Filter> filter(Filter::Factory(filters, "missing-mime", - kInputBufferSize)); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kInputBufferSize)); filter->SetURL(url); size_t feed_block_size = 100; @@ -242,7 +238,7 @@ TEST_F(SdchFilterTest, BasicDictionary) { EXPECT_EQ(output, expanded_); // Decode with really small buffers (size 1) to check for edge effects. - filter.reset((Filter::Factory(filters, "missing-mime", kInputBufferSize))); + filter.reset((Filter::Factory(filter_types, kInputBufferSize))); filter->SetURL(url); feed_block_size = 1; @@ -265,14 +261,13 @@ TEST_F(SdchFilterTest, CrossDomainDictionaryUse) { std::string compressed(NewSdchCompressedData(dictionary)); - std::vector<std::string> filters; - filters.push_back("sdch"); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_SDCH); const int kInputBufferSize(100); // Decode with content arriving from the "wrong" domain. // This tests CanSet() in the sdch_manager_-> - scoped_ptr<Filter> filter((Filter::Factory(filters, "missing-mime", - kInputBufferSize))); + scoped_ptr<Filter> filter((Filter::Factory(filter_types, kInputBufferSize))); GURL wrong_domain_url("http://www.wrongdomain.com"); filter->SetURL(wrong_domain_url); @@ -307,13 +302,12 @@ TEST_F(SdchFilterTest, DictionaryPathValidation) { std::string compressed_for_path(NewSdchCompressedData(dictionary_with_path)); - std::vector<std::string> filters; - filters.push_back("sdch"); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_SDCH); const int kInputBufferSize(100); // Test decode the path data, arriving from a valid path. - scoped_ptr<Filter> filter((Filter::Factory(filters, "missing-mime", - kInputBufferSize))); + scoped_ptr<Filter> filter((Filter::Factory(filter_types, kInputBufferSize))); filter->SetURL(GURL(url_string + path)); size_t feed_block_size = 100; @@ -325,7 +319,7 @@ TEST_F(SdchFilterTest, DictionaryPathValidation) { EXPECT_EQ(output, expanded_); // Test decode the path data, arriving from a invalid path. - filter.reset((Filter::Factory(filters, "missing-mime", kInputBufferSize))); + filter.reset((Filter::Factory(filter_types, kInputBufferSize))); filter->SetURL(GURL(url_string)); feed_block_size = 100; @@ -361,13 +355,12 @@ TEST_F(SdchFilterTest, DictionaryPortValidation) { std::string compressed_for_port(NewSdchCompressedData(dictionary_with_port)); - std::vector<std::string> filters; - filters.push_back("sdch"); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_SDCH); const int kInputBufferSize(100); // Test decode the port data, arriving from a valid port. - scoped_ptr<Filter> filter((Filter::Factory(filters, "missing-mime", - kInputBufferSize))); + scoped_ptr<Filter> filter((Filter::Factory(filter_types, kInputBufferSize))); filter->SetURL(GURL(url_string + ":" + port)); size_t feed_block_size = 100; @@ -378,7 +371,7 @@ TEST_F(SdchFilterTest, DictionaryPortValidation) { EXPECT_EQ(output, expanded_); // Test decode the port data, arriving from a valid (default) port. - filter.reset((Filter::Factory(filters, "missing-mime", kInputBufferSize))); + filter.reset((Filter::Factory(filter_types, kInputBufferSize))); filter->SetURL(GURL(url_string)); // Default port. feed_block_size = 100; @@ -389,7 +382,7 @@ TEST_F(SdchFilterTest, DictionaryPortValidation) { EXPECT_EQ(output, expanded_); // Test decode the port data, arriving from a invalid port. - filter.reset((Filter::Factory(filters, "missing-mime", kInputBufferSize))); + filter.reset((Filter::Factory(filter_types, kInputBufferSize))); filter->SetURL(GURL(url_string + ":" + port + "1")); feed_block_size = 100; @@ -480,14 +473,13 @@ TEST_F(SdchFilterTest, FilterChaining) { std::string gzip_compressed_sdch = gzip_compress(sdch_compressed); // Construct a chained filter. - std::vector<std::string> filters; - filters.push_back("sdch"); - filters.push_back("gzip"); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_SDCH); + filter_types.push_back(Filter::FILTER_TYPE_GZIP); // First try with a large buffer (larger than test input, or compressed data). const int kInputBufferSize(100); - scoped_ptr<Filter> filter(Filter::Factory(filters, "missing-mime", - kInputBufferSize)); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kInputBufferSize)); filter->SetURL(url); // Verify that chained filter is waiting for data. @@ -504,7 +496,7 @@ TEST_F(SdchFilterTest, FilterChaining) { EXPECT_EQ(output, expanded_); // Next try with a tiny buffer to cover edge effects. - filter.reset(Filter::Factory(filters, "missing-mime", kInputBufferSize)); + filter.reset(Filter::Factory(filter_types, kInputBufferSize)); filter->SetURL(url); feed_block_size = 1; @@ -530,15 +522,15 @@ TEST_F(SdchFilterTest, DefaultGzipIfSdch) { // Use Gzip to compress the sdch sdch_compressed data. std::string gzip_compressed_sdch = gzip_compress(sdch_compressed); - // Only claim to have sdch content, but really usethe gzipped sdch content. + // Only claim to have sdch content, but really use the gzipped sdch content. // System should automatically add the missing (optional) gzip. - std::vector<std::string> filters; - filters.push_back("sdch"); + std::vector<Filter::FilterType> filter_types; + filter_types.push_back(Filter::FILTER_TYPE_SDCH); + Filter::FixupEncodingTypes(true, "anything/mime", &filter_types); // First try with a large buffer (larger than test input, or compressed data). const int kInputBufferSize(100); - scoped_ptr<Filter> filter(Filter::Factory(filters, "missing-mime", - kInputBufferSize)); + scoped_ptr<Filter> filter(Filter::Factory(filter_types, kInputBufferSize)); filter->SetURL(url); // Verify that chained filter is waiting for data. @@ -555,7 +547,7 @@ TEST_F(SdchFilterTest, DefaultGzipIfSdch) { EXPECT_EQ(output, expanded_); // Next try with a tiny buffer to cover edge effects. - filter.reset(Filter::Factory(filters, "missing-mime", kInputBufferSize)); + filter.reset(Filter::Factory(filter_types, kInputBufferSize)); filter->SetURL(url); feed_block_size = 1; @@ -589,56 +581,3 @@ TEST_F(SdchFilterTest, DomainBlacklisting) { EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(test_url)); EXPECT_FALSE(SdchManager::Global()->IsInSupportedDomain(google_url)); } - - -// TODO(jar): move this sort of test into filter_unittest.cc, or -// url_request_http_job_unittest.cc if that is more applicable after refactoring -// to use array of enums rather than array of strings to express content -// encodings. -TEST_F(SdchFilterTest, ContentTypeId) { - // Check for basic translation of Content-Encoding, including case variations. - EXPECT_EQ(Filter::FILTER_TYPE_DEFLATE, - Filter::ConvertEncodingToType("deflate", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_DEFLATE, - Filter::ConvertEncodingToType("deflAte", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_GZIP, - Filter::ConvertEncodingToType("gzip", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_GZIP, - Filter::ConvertEncodingToType("GzIp", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_GZIP, - Filter::ConvertEncodingToType("x-gzip", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_GZIP, - Filter::ConvertEncodingToType("X-GzIp", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_BZIP2, - Filter::ConvertEncodingToType("bzip2", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_BZIP2, - Filter::ConvertEncodingToType("BZiP2", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_BZIP2, - Filter::ConvertEncodingToType("x-bzip2", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_BZIP2, - Filter::ConvertEncodingToType("X-BZiP2", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_SDCH, - Filter::ConvertEncodingToType("sdch", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_SDCH, - Filter::ConvertEncodingToType("sDcH", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, - Filter::ConvertEncodingToType("weird", "nothing")); - EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, - Filter::ConvertEncodingToType("strange", "nothing")); - - // Check to see that apache bug (marking things as gzipped because of their - // on-disk file type) is ignored. - EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, - Filter::ConvertEncodingToType("gzip", "application/x-gzip")); - EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, - Filter::ConvertEncodingToType("gzip", "application/gzip")); - EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, - Filter::ConvertEncodingToType("gzip", "application/x-gunzip")); - - EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, - Filter::ConvertEncodingToType("x-gzip", "application/x-gzip")); - EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, - Filter::ConvertEncodingToType("x-gzip", "application/gzip")); - EXPECT_EQ(Filter::FILTER_TYPE_UNSUPPORTED, - Filter::ConvertEncodingToType("x-gzip", "application/x-gunzip")); -} diff --git a/net/base/sdch_manager.cc b/net/base/sdch_manager.cc index 5b86a650..365c8de 100644 --- a/net/base/sdch_manager.cc +++ b/net/base/sdch_manager.cc @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "base/field_trial.h" #include "base/histogram.h" #include "base/logging.h" #include "base/sha2.h" @@ -25,7 +26,7 @@ SdchManager* SdchManager::Global() { void SdchManager::SdchErrorRecovery(ProblemCodes problem) { static LinearHistogram histogram(L"Sdch.ProblemCodes", MIN_PROBLEM_CODE, MAX_PROBLEM_CODE - 1, MAX_PROBLEM_CODE); - // TODO(jar): Set UMA flag for uploading. + histogram.SetFlags(kUmaTargetedHistogramFlag); histogram.Add(problem); } @@ -51,11 +52,13 @@ SdchManager::~SdchManager() { global_ = NULL; } -// static +// static bool SdchManager::BlacklistDomain(const GURL& url) { if (!global_ ) return false; std::string domain(url.host()); + UMA_HISTOGRAM_TIMES(L"Sdch.UptimeBeforeBlacklisting", + Time::Now() - FieldTrialList::application_start_time()); global_->blacklisted_domains_.insert(url.host()); return true; } @@ -171,7 +174,7 @@ bool SdchManager::AddSdchDictionary(const std::string& dictionary_text, if (!Dictionary::CanSet(domain, path, ports, dictionary_url)) return false; - HISTOGRAM_COUNTS(L"Sdch.Dictionary size loaded", dictionary_text.size()); + UMA_HISTOGRAM_COUNTS(L"Sdch.Dictionary size loaded", dictionary_text.size()); DLOG(INFO) << "Loaded dictionary with client hash " << client_hash << " and server hash " << server_hash; Dictionary* dictionary = @@ -187,7 +190,6 @@ void SdchManager::GetVcdiffDictionary(const std::string& server_hash, *dictionary = NULL; DictionaryMap::iterator it = dictionaries_.find(server_hash); if (it == dictionaries_.end()) { - SdchErrorRecovery(DICTIONARY_NOT_FOUND_FOR_HASH); return; } Dictionary* matching_dictionary = it->second; diff --git a/net/base/sdch_manager.h b/net/base/sdch_manager.h index 1d00163..3aa6884 100644 --- a/net/base/sdch_manager.h +++ b/net/base/sdch_manager.h @@ -66,8 +66,7 @@ class SdchManager { DECODE_BODY_ERROR, // Dictionary selection for use problems. - DICTIONARY_NOT_FOUND_FOR_HASH = 10, - DICTIONARY_FOUND_HAS_WRONG_DOMAIN, + DICTIONARY_FOUND_HAS_WRONG_DOMAIN = 10, DICTIONARY_FOUND_HAS_WRONG_PORT_LIST, DICTIONARY_FOUND_HAS_WRONG_PATH, DICTIONARY_FOUND_HAS_WRONG_SCHEME, @@ -92,7 +91,7 @@ class SdchManager { DICTIONARY_SELECTED_FOR_SSL, DICTIONARY_ALREADY_LOADED, - MAX_PROBLEM_CODE // Used to bound histogram + MAX_PROBLEM_CODE // Used to bound histogram. }; // There is one instance of |Dictionary| for each memory-cached SDCH diff --git a/net/build/net_unittests.vcproj b/net/build/net_unittests.vcproj index a52ce6e..f3d2936 100644 --- a/net/build/net_unittests.vcproj +++ b/net/build/net_unittests.vcproj @@ -227,15 +227,15 @@ Name="http" > <File - RelativePath="..\http\http_auth_unittest.cc" + RelativePath="..\http\http_auth_handler_basic_unittest.cc" > </File> <File - RelativePath="..\http\http_auth_handler_basic_unittest.cc" + RelativePath="..\http\http_auth_handler_digest_unittest.cc" > </File> <File - RelativePath="..\http\http_auth_handler_digest_unittest.cc" + RelativePath="..\http\http_auth_unittest.cc" > </File> <File @@ -327,6 +327,10 @@ > </File> <File + RelativePath="..\base\filter_unittest.cc" + > + </File> + <File RelativePath="..\base\gzip_filter_unittest.cc" > </File> diff --git a/net/net_unittests.scons b/net/net_unittests.scons index bea4aa7..82d0a40 100644 --- a/net/net_unittests.scons +++ b/net/net_unittests.scons @@ -103,6 +103,7 @@ if env['PLATFORM'] == 'win32': input_files.extend([ 'base/directory_lister_unittest.cc', 'base/sdch_filter_unittest.cc', + 'base/filter_unittest.cc', 'base/ssl_config_service_unittest.cc', 'base/ssl_client_socket_unittest.cc', 'base/wininet_util_unittest.cc', diff --git a/net/url_request/url_request_http_job.cc b/net/url_request/url_request_http_job.cc index d6286af..49a8925 100644 --- a/net/url_request/url_request_http_job.cc +++ b/net/url_request/url_request_http_job.cc @@ -10,6 +10,7 @@ #include "base/message_loop.h" #include "base/string_util.h" #include "net/base/cookie_monster.h" +#include "net/base/filter.h" #include "net/base/load_flags.h" #include "net/base/net_errors.h" #include "net/base/net_util.h" @@ -162,67 +163,24 @@ int URLRequestHttpJob::GetResponseCode() { } bool URLRequestHttpJob::GetContentEncodings( - std::vector<std::string>* encoding_types) { + std::vector<Filter::FilterType>* encoding_types) { DCHECK(transaction_.get()); - if (!response_info_) return false; + DCHECK(encoding_types->empty()); std::string encoding_type; void* iter = NULL; while (response_info_->headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) { - encoding_types->push_back(StringToLowerASCII(encoding_type)); + encoding_types->push_back(Filter::ConvertEncodingToType(encoding_type)); } - // TODO(jar): Transition to returning enums, rather than strings, and perform - // all content encoding fixups here, rather than doing some in the - // FilterFactor(). Note that enums generated can be more specific than mere - // restatement of strings. For example, rather than just having a GZIP - // encoding we can have a GZIP_OPTIONAL encoding to help with odd SDCH related - // fixups. - - // TODO(jar): Refactor code so that content-encoding error recovery is - // testable via unit tests. - - if (!IsSdchResponse()) - return !encoding_types->empty(); - - // If content encoding included SDCH, then everything is fine. - if (!encoding_types->empty() && ("sdch" == encoding_types->front())) - return !encoding_types->empty(); - - // SDCH "search results" protective hack: To make sure we don't break the only - // currently deployed SDCH enabled server, be VERY cautious about proxies that - // strip all content-encoding to not include sdch. IF we don't see content - // encodings that seem to match what we'd expect from a server that asked us - // to use a dictionary (and we advertised said dictionary in the GET), then - // we set the encoding to (try to) use SDCH to decode. Note that SDCH will - // degrade into a pass-through filter if it doesn't have a viable dictionary - // hash in its header. Also note that a solo "sdch" will implicitly create - // a "sdch,gzip" decoding filter, where the gzip portion will degrade to a - // pass through if a gzip header is not encountered. Hence we can replace - // "gzip" with "sdch" and "everything will work." - // The one failure mode comes when we advertise a dictionary, and the server - // tries to *send* a gzipped file (not gzip encode content), and then we could - // do a gzip decode :-(. Since current server support does not ever see such - // a transfer, we are safe (for now). - - std::string mime_type; - GetMimeType(&mime_type); - if (std::string::npos != mime_type.find_first_of("text/html")) { - // Suspicious case: Advertised dictionary, but server didn't use sdch, even - // though it is text_html content. - if (encoding_types->empty()) - SdchManager::SdchErrorRecovery(SdchManager::ADDED_CONTENT_ENCODING); - else if (encoding_types->size() == 1) - SdchManager::SdchErrorRecovery(SdchManager::FIXED_CONTENT_ENCODING); - else - SdchManager::SdchErrorRecovery(SdchManager::FIXED_CONTENT_ENCODINGS); - encoding_types->clear(); - encoding_types->push_back("sdch"); // Handle SDCH/GZIP-opt encoding. + if (!encoding_types->empty()) { + std::string mime_type; + GetMimeType(&mime_type); + Filter::FixupEncodingTypes(IsSdchResponse(), mime_type, encoding_types); } - return !encoding_types->empty(); } @@ -534,6 +492,39 @@ void URLRequestHttpJob::StartTransaction() { } void URLRequestHttpJob::AddExtraHeaders() { + // Supply Accept-Encoding headers first so that it is more likely that they + // will be in the first transmitted packet. This can sometimes make it easier + // to filter and analyze the streams to assure that a proxy has not damaged + // these headers. Some proxies deliberately corrupt Accept-Encoding headers. + if (!SdchManager::Global() || + !SdchManager::Global()->IsInSupportedDomain(request_->url())) { + // Tell the server what compression formats we support (other than SDCH). + request_info_.extra_headers += "Accept-Encoding: gzip,deflate,bzip2\r\n"; + } else { + // Supply SDCH related headers, as well as accepting that encoding. + // Tell the server what compression formats we support. + request_info_.extra_headers += "Accept-Encoding: " + "gzip,deflate,bzip2,sdch\r\n"; + + // TODO(jar): See if it is worth optimizing away these bytes when the URL is + // probably an img or such. (and SDCH encoding is not likely). + std::string avail_dictionaries; + SdchManager::Global()->GetAvailDictionaryList(request_->url(), + &avail_dictionaries); + if (!avail_dictionaries.empty()) { + request_info_.extra_headers += "Avail-Dictionary: " + + avail_dictionaries + "\r\n"; + request_info_.load_flags |= net::LOAD_SDCH_DICTIONARY_ADVERTISED; + } + + scoped_ptr<FileVersionInfo> file_version_info( + FileVersionInfo::CreateFileVersionInfoForCurrentModule()); + request_info_.extra_headers += "X-SDCH: Chrome "; + request_info_.extra_headers += + WideToASCII(file_version_info->product_version()); + request_info_.extra_headers += "\r\n"; + } + URLRequestContext* context = request_->context(); if (context) { // Add in the cookie header. TODO might we need more than one header? @@ -553,36 +544,6 @@ void URLRequestHttpJob::AddExtraHeaders() { request_info_.extra_headers += "Accept-Charset: " + context->accept_charset() + "\r\n"; } - - if (!SdchManager::Global() || - !SdchManager::Global()->IsInSupportedDomain(request_->url())) { - // Tell the server what compression formats we support (other than SDCH). - request_info_.extra_headers += "Accept-Encoding: gzip,deflate,bzip2\r\n"; - return; - } - - // Supply SDCH related headers, as well as accepting that encoding. - - // TODO(jar): See if it is worth optimizing away these bytes when the URL is - // probably an img or such. (and SDCH encoding is not likely). - std::string avail_dictionaries; - SdchManager::Global()->GetAvailDictionaryList(request_->url(), - &avail_dictionaries); - if (!avail_dictionaries.empty()) { - request_info_.extra_headers += "Avail-Dictionary: " - + avail_dictionaries + "\r\n"; - request_info_.load_flags |= net::LOAD_SDCH_DICTIONARY_ADVERTISED; - } - - scoped_ptr<FileVersionInfo> file_version_info( - FileVersionInfo::CreateFileVersionInfoForCurrentModule()); - request_info_.extra_headers += "X-SDCH: Chrome "; - request_info_.extra_headers += - WideToASCII(file_version_info->product_version()); - request_info_.extra_headers += "\r\n"; - - // Tell the server what compression formats we support. - request_info_.extra_headers += "Accept-Encoding: gzip,deflate,bzip2,sdch\r\n"; } void URLRequestHttpJob::FetchResponseCookies() { diff --git a/net/url_request/url_request_http_job.h b/net/url_request/url_request_http_job.h index c8e3ba2..f36a072 100644 --- a/net/url_request/url_request_http_job.h +++ b/net/url_request/url_request_http_job.h @@ -41,7 +41,8 @@ class URLRequestHttpJob : public URLRequestJob { virtual void GetResponseInfo(net::HttpResponseInfo* info); virtual bool GetResponseCookies(std::vector<std::string>* cookies); virtual int GetResponseCode(); - virtual bool GetContentEncodings(std::vector<std::string>* encoding_type); + virtual bool GetContentEncodings( + std::vector<Filter::FilterType>* encoding_type); virtual bool IsSdchResponse() const; virtual bool IsRedirectResponse(GURL* location, int* http_status_code); virtual bool IsSafeRedirect(const GURL& location); diff --git a/net/url_request/url_request_job.cc b/net/url_request/url_request_job.cc index f4924bb..10323f6 100644 --- a/net/url_request/url_request_job.cc +++ b/net/url_request/url_request_job.cc @@ -47,12 +47,12 @@ void URLRequestJob::DetachRequest() { } void URLRequestJob::SetupFilter() { - std::vector<std::string> encoding_types; + std::vector<Filter::FilterType> encoding_types; if (GetContentEncodings(&encoding_types)) { - std::string mime_type; - GetMimeType(&mime_type); - filter_.reset(Filter::Factory(encoding_types, mime_type, kFilterBufSize)); + filter_.reset(Filter::Factory(encoding_types, kFilterBufSize)); if (filter_.get()) { + std::string mime_type; + GetMimeType(&mime_type); filter_->SetURL(request_->url()); filter_->SetMimeType(mime_type); } diff --git a/net/url_request/url_request_job.h b/net/url_request/url_request_job.h index 2df62fd..6b309be 100644 --- a/net/url_request/url_request_job.h +++ b/net/url_request/url_request_job.h @@ -114,12 +114,14 @@ class URLRequestJob : public base::RefCountedThreadSafe<URLRequestJob> { // some types of requests. Returns true on success. Calling this on a request // that doesn't have or specify an encoding type will return false. // Returns a array of strings showing the sequential encodings used on the - // content. For example, types[0] = "sdch" and types[1] = gzip, means the - // content was first encoded by sdch, and then encoded by gzip. To decode, - // a series of filters must be applied in the reverse order (in the above - // example, ungzip first, and then sdch expand). - // TODO(jar): Cleaner API would return an array of enums. - virtual bool GetContentEncodings(std::vector<std::string>* encoding_types) { + // content. + // For example, encoding_types[0] = FILTER_TYPE_SDCH and encoding_types[1] = + // FILTER_TYPE_GZIP, means the content was first encoded by sdch, and then + // result was encoded by gzip. To decode, a series of filters must be applied + // in the reverse order (in the above example, ungzip first, and then sdch + // expand). + virtual bool GetContentEncodings( + std::vector<Filter::FilterType>* encoding_types) { return false; } |