diff options
author | gangwu <gangwu@chromium.org> | 2015-08-06 15:15:41 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-08-06 22:16:21 +0000 |
commit | fbf1a591aa9653333a748764c1465344b7782fc8 (patch) | |
tree | d915a8282c243c41794659d2ba5a49d1154f1521 | |
parent | 7c900e6e00f63108f144cf0435097c7d953c04ff (diff) | |
download | chromium_src-fbf1a591aa9653333a748764c1465344b7782fc8.zip chromium_src-fbf1a591aa9653333a748764c1465344b7782fc8.tar.gz chromium_src-fbf1a591aa9653333a748764c1465344b7782fc8.tar.bz2 |
Enable compress HTTP call's content between sync client and
sync server.
BUG=509728
Review URL: https://codereview.chromium.org/1246523003
Cr-Commit-Position: refs/heads/master@{#342213}
-rw-r--r-- | sync/BUILD.gn | 1 | ||||
-rw-r--r-- | sync/internal_api/http_bridge.cc | 138 | ||||
-rw-r--r-- | sync/internal_api/http_bridge_unittest.cc | 157 | ||||
-rwxr-xr-x | sync/tools/testserver/sync_testserver.py | 8 | ||||
-rw-r--r-- | testing/variations/fieldtrial_testing_config_android.json | 8 | ||||
-rw-r--r-- | testing/variations/fieldtrial_testing_config_chromeos.json | 8 | ||||
-rw-r--r-- | testing/variations/fieldtrial_testing_config_ios.json | 8 | ||||
-rw-r--r-- | testing/variations/fieldtrial_testing_config_linux.json | 8 | ||||
-rw-r--r-- | testing/variations/fieldtrial_testing_config_mac.json | 8 | ||||
-rw-r--r-- | testing/variations/fieldtrial_testing_config_win.json | 8 | ||||
-rw-r--r-- | tools/metrics/histograms/histograms.xml | 32 |
11 files changed, 382 insertions, 2 deletions
diff --git a/sync/BUILD.gn b/sync/BUILD.gn index 30e1fdd..584550f 100644 --- a/sync/BUILD.gn +++ b/sync/BUILD.gn @@ -695,6 +695,7 @@ test("sync_unit_tests") { "//testing/gtest", "//third_party/leveldatabase", "//third_party/protobuf:protobuf_lite", + "//third_party/zlib", ] if (is_android) { diff --git a/sync/internal_api/http_bridge.cc b/sync/internal_api/http_bridge.cc index c64822d..e5729f2 100644 --- a/sync/internal_api/http_bridge.cc +++ b/sync/internal_api/http_bridge.cc @@ -5,8 +5,10 @@ #include "sync/internal_api/public/http_bridge.h" #include "base/message_loop/message_loop.h" +#include "base/metrics/field_trial.h" #include "base/metrics/histogram_macros.h" #include "base/strings/string_number_conversions.h" +#include "base/strings/string_util.h" #include "base/strings/stringprintf.h" #include "net/base/load_flags.h" #include "net/base/net_errors.h" @@ -20,6 +22,7 @@ #include "net/url_request/url_request_job_factory_impl.h" #include "net/url_request/url_request_status.h" #include "sync/internal_api/public/base/cancelation_signal.h" +#include "third_party/zlib/zlib.h" namespace syncer { @@ -35,6 +38,109 @@ void LogTimeout(bool timed_out) { UMA_HISTOGRAM_BOOLEAN("Sync.URLFetchTimedOut", timed_out); } +bool IsSyncHttpContentCompressionEnabled() { + const std::string group_name = + base::FieldTrialList::FindFullName("SyncHttpContentCompression"); + return StartsWith(group_name, "Enabled", base::CompareCase::SENSITIVE); +} + +void RecordSyncRequestContentLengthHistograms(int64 compressed_content_length, + int64 original_content_length) { + UMA_HISTOGRAM_COUNTS("Sync.RequestContentLength.Compressed", + compressed_content_length); + UMA_HISTOGRAM_COUNTS("Sync.RequestContentLength.Original", + original_content_length); +} + +void RecordSyncResponseContentLengthHistograms(int64 compressed_content_length, + int64 original_content_length) { + UMA_HISTOGRAM_COUNTS("Sync.ResponseContentLength.Compressed", + compressed_content_length); + UMA_HISTOGRAM_COUNTS("Sync.ResponseContentLength.Original", + original_content_length); +} + +// ----------------------------------------------------------------------------- +// The rest of the code in the anon namespace is copied from +// components/metrics/compression_utils.cc +// TODO(gangwu): crbug.com/515695. The following code is copied from +// components/metrics/compression_utils.cc. We copied them because if we +// reference them, we will get cycle dependency warning. Once the functions +// have been moved from //component to //base, we can remove the following +// functions. +//------------------------------------------------------------------------------ +// The difference in bytes between a zlib header and a gzip header. +const size_t kGzipZlibHeaderDifferenceBytes = 16; + +// Pass an integer greater than the following get a gzip header instead of a +// zlib header when calling deflateInit2() and inflateInit2(). +const int kWindowBitsToGetGzipHeader = 16; + +// This describes the amount of memory zlib uses to compress data. It can go +// from 1 to 9, with 8 being the default. For details, see: +// http://www.zlib.net/manual.html (search for memLevel). +const int kZlibMemoryLevel = 8; + +// This code is taken almost verbatim from third_party/zlib/compress.c. The only +// difference is deflateInit2() is called which sets the window bits to be > 16. +// That causes a gzip header to be emitted rather than a zlib header. +int GzipCompressHelper(Bytef* dest, + uLongf* dest_length, + const Bytef* source, + uLong source_length) { + z_stream stream; + + stream.next_in = bit_cast<Bytef*>(source); + stream.avail_in = static_cast<uInt>(source_length); + stream.next_out = dest; + stream.avail_out = static_cast<uInt>(*dest_length); + if (static_cast<uLong>(stream.avail_out) != *dest_length) + return Z_BUF_ERROR; + + stream.zalloc = static_cast<alloc_func>(0); + stream.zfree = static_cast<free_func>(0); + stream.opaque = static_cast<voidpf>(0); + + gz_header gzip_header; + memset(&gzip_header, 0, sizeof(gzip_header)); + int err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, + MAX_WBITS + kWindowBitsToGetGzipHeader, + kZlibMemoryLevel, Z_DEFAULT_STRATEGY); + if (err != Z_OK) + return err; + + err = deflateSetHeader(&stream, &gzip_header); + if (err != Z_OK) + return err; + + err = deflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) { + deflateEnd(&stream); + return err == Z_OK ? Z_BUF_ERROR : err; + } + *dest_length = stream.total_out; + + err = deflateEnd(&stream); + return err; +} + +bool GzipCompress(const std::string& input, std::string* output) { + const uLongf input_size = static_cast<uLongf>(input.size()); + std::vector<Bytef> compressed_data(kGzipZlibHeaderDifferenceBytes + + compressBound(input_size)); + + uLongf compressed_size = static_cast<uLongf>(compressed_data.size()); + if (GzipCompressHelper(&compressed_data.front(), &compressed_size, + bit_cast<const Bytef*>(input.data()), + input_size) != Z_OK) { + return false; + } + + compressed_data.resize(compressed_size); + output->assign(compressed_data.begin(), compressed_data.end()); + return true; +} + } // namespace HttpBridgeFactory::HttpBridgeFactory( @@ -203,19 +309,36 @@ void HttpBridge::MakeAsynchronousPost() { base::Bind(&HttpBridge::OnURLFetchTimedOut, this)); DCHECK(request_context_getter_.get()); + fetch_state_.start_time = base::Time::Now(); fetch_state_.url_poster = net::URLFetcher::Create(url_for_request_, net::URLFetcher::POST, this) .release(); fetch_state_.url_poster->SetRequestContext(request_context_getter_.get()); - fetch_state_.url_poster->SetUploadData(content_type_, request_content_); fetch_state_.url_poster->SetExtraRequestHeaders(extra_headers_); + + int64 compressed_content_size = 0; + if (IsSyncHttpContentCompressionEnabled()) { + std::string compressed_request_content; + GzipCompress(request_content_, &compressed_request_content); + compressed_content_size = compressed_request_content.size(); + fetch_state_.url_poster->SetUploadData(content_type_, + compressed_request_content); + fetch_state_.url_poster->AddExtraRequestHeader("Content-Encoding: gzip"); + } else { + fetch_state_.url_poster->SetUploadData(content_type_, request_content_); + fetch_state_.url_poster->AddExtraRequestHeader(base::StringPrintf( + "%s: %s", net::HttpRequestHeaders::kAcceptEncoding, "deflate")); + } + + RecordSyncRequestContentLengthHistograms(compressed_content_size, + request_content_.size()); + fetch_state_.url_poster->AddExtraRequestHeader(base::StringPrintf( "%s: %s", net::HttpRequestHeaders::kUserAgent, user_agent_.c_str())); fetch_state_.url_poster->SetLoadFlags(net::LOAD_BYPASS_CACHE | net::LOAD_DISABLE_CACHE | net::LOAD_DO_NOT_SAVE_COOKIES | net::LOAD_DO_NOT_SEND_COOKIES); - fetch_state_.start_time = base::Time::Now(); fetch_state_.url_poster->Start(); } @@ -315,6 +438,17 @@ void HttpBridge::OnURLFetchComplete(const net::URLFetcher* source) { fetch_state_.response_headers = source->GetResponseHeaders(); UpdateNetworkTime(); + int64 compressed_content_length = fetch_state_.response_content.size(); + int64 original_content_length = compressed_content_length; + if (fetch_state_.response_headers && + fetch_state_.response_headers->HasHeaderValue("content-encoding", + "gzip")) { + compressed_content_length = + fetch_state_.response_headers->GetContentLength(); + } + RecordSyncResponseContentLengthHistograms(compressed_content_length, + original_content_length); + // End of the line for url_poster_. It lives only on the IO loop. // We defer deletion because we're inside a callback from a component of the // URLFetcher, so it seems most natural / "polite" to let the stack unwind. diff --git a/sync/internal_api/http_bridge_unittest.cc b/sync/internal_api/http_bridge_unittest.cc index 3c6db88..d1cc914 100644 --- a/sync/internal_api/http_bridge_unittest.cc +++ b/sync/internal_api/http_bridge_unittest.cc @@ -2,8 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "base/metrics/field_trial.h" #include "base/strings/stringprintf.h" #include "base/synchronization/waitable_event.h" +#include "base/test/mock_entropy_provider.h" #include "base/threading/thread.h" #include "net/http/http_response_headers.h" #include "net/test/spawned_test_server/spawned_test_server.h" @@ -13,15 +15,95 @@ #include "sync/internal_api/public/base/cancelation_signal.h" #include "sync/internal_api/public/http_bridge.h" #include "testing/gtest/include/gtest/gtest.h" +#include "third_party/zlib/zlib.h" namespace syncer { namespace { + // TODO(timsteele): Should use PathService here. See Chromium Issue 3113. const base::FilePath::CharType kDocRoot[] = FILE_PATH_LITERAL("chrome/test/data"); + +// ----------------------------------------------------------------------------- +// The rest of the code in the anon namespace is copied from +// components/metrics/compression_utils.cc +// TODO(gangwu): crbug.com/515695. The following codes are copied from +// components/metrics/compression_utils.cc, we copied them because if we +// reference them, we will get cycle dependency warning. Once the functions +// have been moved from //component to //base, we can remove the following +// functions. +//------------------------------------------------------------------------------ +// Pass an integer greater than the following get a gzip header instead of a +// zlib header when calling deflateInit2() and inflateInit2(). +const int kWindowBitsToGetGzipHeader = 16; + +// This code is taken almost verbatim from third_party/zlib/uncompr.c. The only +// difference is inflateInit2() is called which sets the window bits to be > 16. +// That causes a gzip header to be parsed rather than a zlib header. +int GzipUncompressHelper(Bytef* dest, + uLongf* dest_length, + const Bytef* source, + uLong source_length) { + z_stream stream; + + stream.next_in = bit_cast<Bytef*>(source); + stream.avail_in = static_cast<uInt>(source_length); + if (static_cast<uLong>(stream.avail_in) != source_length) + return Z_BUF_ERROR; + + stream.next_out = dest; + stream.avail_out = static_cast<uInt>(*dest_length); + if (static_cast<uLong>(stream.avail_out) != *dest_length) + return Z_BUF_ERROR; + + stream.zalloc = static_cast<alloc_func>(0); + stream.zfree = static_cast<free_func>(0); + + int err = inflateInit2(&stream, MAX_WBITS + kWindowBitsToGetGzipHeader); + if (err != Z_OK) + return err; + + err = inflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) { + inflateEnd(&stream); + if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) + return Z_DATA_ERROR; + return err; + } + *dest_length = stream.total_out; + + err = inflateEnd(&stream); + return err; +} + +// Returns the uncompressed size from GZIP-compressed |compressed_data|. +uint32 GetUncompressedSize(const std::string& compressed_data) { + // The uncompressed size is stored in the last 4 bytes of |input| in LE. + uint32 size; + if (compressed_data.length() < sizeof(size)) + return 0; + memcpy(&size, &compressed_data[compressed_data.length() - sizeof(size)], + sizeof(size)); + return base::ByteSwapToLE32(size); +} + +bool GzipUncompress(const std::string& input, std::string* output) { + std::string uncompressed_output; + uLongf uncompressed_size = static_cast<uLongf>(GetUncompressedSize(input)); + uncompressed_output.resize(uncompressed_size); + if (GzipUncompressHelper(bit_cast<Bytef*>(uncompressed_output.data()), + &uncompressed_size, + bit_cast<const Bytef*>(input.data()), + static_cast<uLongf>(input.length())) == Z_OK) { + output->swap(uncompressed_output); + return true; + } + return false; } +} // namespace + const char kUserAgent[] = "user-agent"; class SyncHttpBridgeTest : public testing::Test { @@ -146,9 +228,12 @@ class ShuntedHttpBridge : public HttpBridge { std::string response_content = "success!"; net::TestURLFetcher fetcher(0, GURL("http://www.google.com"), NULL); + scoped_refptr<net::HttpResponseHeaders> response_headers( + new net::HttpResponseHeaders("")); fetcher.set_response_code(200); fetcher.set_cookies(cookies); fetcher.SetResponseString(response_content); + fetcher.set_response_headers(response_headers); OnURLFetchComplete(&fetcher); } SyncHttpBridgeTest* test_; @@ -232,6 +317,74 @@ TEST_F(SyncHttpBridgeTest, TestMakeSynchronousPostLiveWithPayload) { EXPECT_EQ(payload, std::string(http_bridge->GetResponseContent())); } +// Full round-trip test of the HttpBridge with compressed data, check if the +// data is correctly compressed. +TEST_F(SyncHttpBridgeTest, CompressedRequestPayloadCheck) { + ASSERT_TRUE(test_server_.Start()); + + scoped_refptr<HttpBridge> http_bridge(BuildBridge()); + + std::string payload = "this should be echoed back"; + GURL echo = test_server_.GetURL("echo"); + http_bridge->SetURL(echo.spec().c_str(), echo.IntPort()); + http_bridge->SetPostPayload("application/x-www-form-urlencoded", + payload.length(), payload.c_str()); + int os_error = 0; + int response_code = 0; + base::FieldTrialList field_trial_list(new base::MockEntropyProvider()); + base::FieldTrialList::CreateFieldTrial("SyncHttpContentCompression", + "Enabled"); + bool success = http_bridge->MakeSynchronousPost(&os_error, &response_code); + EXPECT_TRUE(success); + EXPECT_EQ(200, response_code); + EXPECT_EQ(0, os_error); + + EXPECT_NE(payload.length() + 1, + static_cast<size_t>(http_bridge->GetResponseContentLength())); + std::string compressed_payload(http_bridge->GetResponseContent(), + http_bridge->GetResponseContentLength()); + std::string uncompressed_payload; + GzipUncompress(compressed_payload, &uncompressed_payload); + EXPECT_EQ(payload, uncompressed_payload); +} + +// Full round-trip test of the HttpBridge with compression, check if header +// fields("Content-Encoding" ,"Accept-Encoding" and user agent) are set +// correctly. +TEST_F(SyncHttpBridgeTest, CompressedRequestHeaderCheck) { + ASSERT_TRUE(test_server_.Start()); + + scoped_refptr<HttpBridge> http_bridge(BuildBridge()); + + GURL echo_header = test_server_.GetURL("echoall"); + http_bridge->SetURL(echo_header.spec().c_str(), echo_header.IntPort()); + + std::string test_payload = "###TEST PAYLOAD###"; + http_bridge->SetPostPayload("text/html", test_payload.length() + 1, + test_payload.c_str()); + + int os_error = 0; + int response_code = 0; + base::FieldTrialList field_trial_list(new base::MockEntropyProvider()); + base::FieldTrialList::CreateFieldTrial("SyncHttpContentCompression", + "Enabled"); + bool success = http_bridge->MakeSynchronousPost(&os_error, &response_code); + EXPECT_TRUE(success); + EXPECT_EQ(200, response_code); + EXPECT_EQ(0, os_error); + + std::string response(http_bridge->GetResponseContent(), + http_bridge->GetResponseContentLength()); + EXPECT_NE(std::string::npos, response.find("Content-Encoding: gzip")); + EXPECT_NE(std::string::npos, + response.find(base::StringPrintf( + "%s: %s", net::HttpRequestHeaders::kAcceptEncoding, + "gzip, deflate"))); + EXPECT_NE(std::string::npos, + response.find(base::StringPrintf( + "%s: %s", net::HttpRequestHeaders::kUserAgent, kUserAgent))); +} + // Full round-trip test of the HttpBridge. TEST_F(SyncHttpBridgeTest, TestMakeSynchronousPostLiveComprehensive) { ASSERT_TRUE(test_server_.Start()); @@ -256,6 +409,10 @@ TEST_F(SyncHttpBridgeTest, TestMakeSynchronousPostLiveComprehensive) { http_bridge->GetResponseContentLength()); EXPECT_EQ(std::string::npos, response.find("Cookie:")); EXPECT_NE(std::string::npos, + response.find(base::StringPrintf( + "%s: %s", net::HttpRequestHeaders::kAcceptEncoding, + "deflate"))); + EXPECT_NE(std::string::npos, response.find(base::StringPrintf("%s: %s", net::HttpRequestHeaders::kUserAgent, kUserAgent))); EXPECT_NE(std::string::npos, response.find(test_payload.c_str())); diff --git a/sync/tools/testserver/sync_testserver.py b/sync/tools/testserver/sync_testserver.py index d65be40..32c746e 100755 --- a/sync/tools/testserver/sync_testserver.py +++ b/sync/tools/testserver/sync_testserver.py @@ -13,8 +13,10 @@ specify an explicit port and xmpp_port if necessary. import asyncore import BaseHTTPServer import errno +import gzip import os import select +import StringIO import socket import sys import urlparse @@ -196,6 +198,12 @@ class SyncPageHandler(testserver_base.BasePageHandler): length = int(self.headers.getheader('content-length')) raw_request = self.rfile.read(length) + if self.headers.getheader('Content-Encoding'): + encode = self.headers.getheader('Content-Encoding') + if encode == "gzip": + raw_request = gzip.GzipFile( + fileobj=StringIO.StringIO(raw_request)).read() + http_response = 200 raw_reply = None if not self.server.GetAuthenticated(): diff --git a/testing/variations/fieldtrial_testing_config_android.json b/testing/variations/fieldtrial_testing_config_android.json index b44df26..e31f2d1 100644 --- a/testing/variations/fieldtrial_testing_config_android.json +++ b/testing/variations/fieldtrial_testing_config_android.json @@ -201,5 +201,13 @@ "state": "enabled" } } + ], + "SyncHttpContentCompression": [ + { + "group_name": "Enabled" + }, + { + "group_name": "Disabled" + } ] } diff --git a/testing/variations/fieldtrial_testing_config_chromeos.json b/testing/variations/fieldtrial_testing_config_chromeos.json index 10a9d12..985829d 100644 --- a/testing/variations/fieldtrial_testing_config_chromeos.json +++ b/testing/variations/fieldtrial_testing_config_chromeos.json @@ -32,5 +32,13 @@ { "group_name": "Enabled" } + ], + "SyncHttpContentCompression": [ + { + "group_name": "Enabled" + }, + { + "group_name": "Disabled" + } ] } diff --git a/testing/variations/fieldtrial_testing_config_ios.json b/testing/variations/fieldtrial_testing_config_ios.json index 0cff2d3..9ee0eb2 100644 --- a/testing/variations/fieldtrial_testing_config_ios.json +++ b/testing/variations/fieldtrial_testing_config_ios.json @@ -3,5 +3,13 @@ { "group_name": "Enabled" } + ], + "SyncHttpContentCompression": [ + { + "group_name": "Enabled" + }, + { + "group_name": "Disabled" + } ] } diff --git a/testing/variations/fieldtrial_testing_config_linux.json b/testing/variations/fieldtrial_testing_config_linux.json index 5bd6bf5..8abf8ef 100644 --- a/testing/variations/fieldtrial_testing_config_linux.json +++ b/testing/variations/fieldtrial_testing_config_linux.json @@ -50,5 +50,13 @@ { "group_name": "Enabled" } + ], + "SyncHttpContentCompression": [ + { + "group_name": "Enabled" + }, + { + "group_name": "Disabled" + } ] } diff --git a/testing/variations/fieldtrial_testing_config_mac.json b/testing/variations/fieldtrial_testing_config_mac.json index 5bd6bf5..8abf8ef 100644 --- a/testing/variations/fieldtrial_testing_config_mac.json +++ b/testing/variations/fieldtrial_testing_config_mac.json @@ -50,5 +50,13 @@ { "group_name": "Enabled" } + ], + "SyncHttpContentCompression": [ + { + "group_name": "Enabled" + }, + { + "group_name": "Disabled" + } ] } diff --git a/testing/variations/fieldtrial_testing_config_win.json b/testing/variations/fieldtrial_testing_config_win.json index 7d12d16..6a94e1e 100644 --- a/testing/variations/fieldtrial_testing_config_win.json +++ b/testing/variations/fieldtrial_testing_config_win.json @@ -110,5 +110,13 @@ { "group_name": "Enabled" } + ], + "SyncHttpContentCompression": [ + { + "group_name": "Enabled" + }, + { + "group_name": "Disabled" + } ] } diff --git a/tools/metrics/histograms/histograms.xml b/tools/metrics/histograms/histograms.xml index e251270..fb7613a 100644 --- a/tools/metrics/histograms/histograms.xml +++ b/tools/metrics/histograms/histograms.xml @@ -44831,12 +44831,44 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries. </summary> </histogram> +<histogram name="Sync.RequestContentLength.Compressed" units="bytes"> + <owner>gangwu@chromium.org</owner> + <summary> + The request content size for a single HTTP/HTTPS call from sync client to + server. The content is compressed by gzip. + </summary> +</histogram> + +<histogram name="Sync.RequestContentLength.Original" units="bytes"> + <owner>gangwu@chromium.org</owner> + <summary> + The original request content size for a single HTTP/HTTPS call from sync + client to server. It is the size before content got compressed. + </summary> +</histogram> + <histogram name="Sync.ResolveSimpleConflict" enum="SyncSimpleConflictResolutions"> <owner>zea@chromium.org</owner> <summary>Enumeration of types of simple conflict resolutions.</summary> </histogram> +<histogram name="Sync.ResponseContentLength.Compressed" units="bytes"> + <owner>gangwu@chromium.org</owner> + <summary> + The response content size for a single HTTP/HTTPS call from sync server to + client. The content is compressed by gzip. + </summary> +</histogram> + +<histogram name="Sync.ResponseContentLength.Original" units="bytes"> + <owner>gangwu@chromium.org</owner> + <summary> + The original response content size for a single HTTP/HTTPS call from sync + server and client. It is the size after content got uncompressed. + </summary> +</histogram> + <histogram name="Sync.RestoreBackendInitializeSucess" enum="BooleanSuccess"> <obsolete> Deprecated 11/2011. Was counted incorrectly. Replaced by |