Make MetricsService upload logs through an interface.

This allows targets that do not wish to depend on net
to use the MetricsService code.

Also, moves compression code to be in the net/ directory
of the metrics component.

BUG=375771,374295
R=isherman@chromium.org,blundell@chromium.org,davidben@chromium.org
TBR=agl@chromium.org

Review URL: https://codereview.chromium.org/291153013

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@274074 0039d316-1c4b-4281-b951-d872f2087c98

13 files changed, 409 insertions, 0 deletions

diff --git a/components/components_tests.gyp b/components/components_tests.gyp
index 0674d26..56dd758 100644
--- a/components/components_tests.gyp
+++ b/components/components_tests.gyp
@@ -111,6 +111,7 @@
             'metrics/metrics_log_base_unittest.cc',
             'metrics/metrics_log_manager_unittest.cc',
             'metrics/metrics_reporting_scheduler_unittest.cc',
+            'metrics/net/compression_utils_unittest.cc',
             'metrics/persisted_logs_unittest.cc',
             'navigation_interception/intercept_navigation_resource_throttle_unittest.cc',
             'os_crypt/ie7_password_win_unittest.cc',
@@ -263,6 +264,10 @@
             # Dependencies of language_usage_metrics
             'components.gyp:language_usage_metrics',
 
+            # Dependencies of metrics
+            'components.gyp:metrics',
+            'components.gyp:metrics_net',
+
             # Dependencies of os_crypt
             'components.gyp:os_crypt',
 
diff --git a/components/metrics.gypi b/components/metrics.gypi
index fd79d50..f9896fb 100644
--- a/components/metrics.gypi
+++ b/components/metrics.gypi
@@ -25,6 +25,8 @@
         'metrics/metrics_hashes.h',
         'metrics/metrics_log_base.cc',
         'metrics/metrics_log_base.h',
+        'metrics/metrics_log_uploader.cc',
+        'metrics/metrics_log_uploader.h',
         'metrics/metrics_log_manager.cc',
         'metrics/metrics_log_manager.h',
         'metrics/metrics_pref_names.cc',
@@ -51,6 +53,24 @@
       ],
     },
     {
+      'target_name': 'metrics_net',
+      'type': 'static_library',
+      'include_dirs': [
+        '..',
+      ],
+      'dependencies': [
+        '../net/net.gyp:net',
+        '../third_party/zlib/zlib.gyp:zlib',
+        'metrics',
+      ],
+      'sources': [
+        'metrics/net/compression_utils.cc',
+        'metrics/net/compression_utils.h',
+        'metrics/net/net_metrics_log_uploader.cc',
+        'metrics/net/net_metrics_log_uploader.h',
+      ],
+    },
+    {
       # Protobuf compiler / generator for UMA (User Metrics Analysis).
       'target_name': 'component_metrics_proto',
       'type': 'static_library',
diff --git a/components/metrics/metrics_log_uploader.cc b/components/metrics/metrics_log_uploader.cc
new file mode 100644
index 0000000..41b83ed
--- /dev/null
+++ b/components/metrics/metrics_log_uploader.cc
@@ -0,0 +1,21 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/metrics/metrics_log_uploader.h"
+
+namespace metrics {
+
+MetricsLogUploader::MetricsLogUploader(
+    const std::string& server_url,
+    const std::string& mime_type,
+    const base::Callback<void(int)>& on_upload_complete)
+    : server_url_(server_url),
+      mime_type_(mime_type),
+      on_upload_complete_(on_upload_complete) {
+}
+
+MetricsLogUploader::~MetricsLogUploader() {
+}
+
+}  // namespace metrics
diff --git a/components/metrics/metrics_log_uploader.h b/components/metrics/metrics_log_uploader.h
new file mode 100644
index 0000000..0deecc8
--- /dev/null
+++ b/components/metrics/metrics_log_uploader.h
@@ -0,0 +1,44 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_METRICS_METRICS_LOG_UPLOADER_H_
+#define COMPONENTS_METRICS_METRICS_LOG_UPLOADER_H_
+
+#include <string>
+
+#include "base/basictypes.h"
+#include "base/callback.h"
+#include "base/macros.h"
+
+namespace metrics {
+
+// MetricsLogUploader is an abstract base class for uploading UMA logs on behalf
+// of MetricsService.
+class MetricsLogUploader {
+ public:
+  // Constructs the uploader that will upload logs to the specified |server_url|
+  // with the given |mime_type|. The |on_upload_complete| callback will be
+  // called with the HTTP response code of the upload or with -1 on an error.
+  MetricsLogUploader(const std::string& server_url,
+                     const std::string& mime_type,
+                     const base::Callback<void(int)>& on_upload_complete);
+  virtual ~MetricsLogUploader();
+
+  // Uploads a log with the specified |log_data| and |log_hash|. |log_hash| is
+  // expected to be the hex-encoded SHA1 hash of |log_data|.
+  virtual bool UploadLog(const std::string& log_data,
+                         const std::string& log_hash) = 0;
+
+ protected:
+  const std::string server_url_;
+  const std::string mime_type_;
+  const base::Callback<void(int)> on_upload_complete_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(MetricsLogUploader);
+};
+
+}  // namespace metrics
+
+#endif  // COMPONENTS_METRICS_METRICS_LOG_UPLOADER_H_
diff --git a/components/metrics/metrics_service_client.h b/components/metrics/metrics_service_client.h
index dbad4a3..1fcfb6e 100644
--- a/components/metrics/metrics_service_client.h
+++ b/components/metrics/metrics_service_client.h
@@ -9,10 +9,13 @@
 
 #include "base/basictypes.h"
 #include "base/callback_forward.h"
+#include "base/memory/scoped_ptr.h"
 #include "components/metrics/proto/system_profile.pb.h"
 
 namespace metrics {
 
+class MetricsLogUploader;
+
 // An abstraction of operations that depend on the embedder's (e.g. Chrome)
 // environment.
 class MetricsServiceClient {
@@ -50,6 +53,13 @@ class MetricsServiceClient {
   // extra histograms that will go in that log. Asynchronous API - the client
   // implementation should call |done_callback| when complete.
   virtual void CollectFinalMetrics(const base::Closure& done_callback) = 0;
+
+  // Creates a MetricsLogUploader with the specified parameters (see comments on
+  // MetricsLogUploader for details).
+  virtual scoped_ptr<MetricsLogUploader> CreateUploader(
+      const std::string& server_url,
+      const std::string& mime_type,
+      const base::Callback<void(int)>& on_upload_complete) = 0;
 };
 
 }  // namespace metrics
diff --git a/components/metrics/net/DEPS b/components/metrics/net/DEPS
new file mode 100644
index 0000000..71ea427
--- /dev/null
+++ b/components/metrics/net/DEPS
@@ -0,0 +1,4 @@
+include_rules = [ 
+  "+net",
+  "+third_party/zlib",
+]
diff --git a/components/metrics/net/compression_utils.cc b/components/metrics/net/compression_utils.cc
new file mode 100644
index 0000000..e8efe18
--- /dev/null
+++ b/components/metrics/net/compression_utils.cc
@@ -0,0 +1,96 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/metrics/net/compression_utils.h"
+
+#include <vector>
+
+#include "base/basictypes.h"
+#include "third_party/zlib/zlib.h"
+
+namespace {
+
+// The difference in bytes between a zlib header and a gzip header.
+const size_t kGzipZlibHeaderDifferenceBytes = 16;
+
+// Pass an integer greater than the following get a gzip header instead of a
+// zlib header when calling deflateInit2_.
+const int kWindowBitsToGetGzipHeader = 16;
+
+// This describes the amount of memory zlib uses to compress data. It can go
+// from 1 to 9, with 8 being the default. For details, see:
+// http://www.zlib.net/manual.html (search for memLevel).
+const int kZlibMemoryLevel = 8;
+
+// This code is taken almost verbatim from third_party/zlib/compress.c. The only
+// difference is deflateInit2_ is called which sets the window bits to be > 16.
+// That causes a gzip header to be emitted rather than a zlib header.
+int GzipCompressHelper(Bytef* dest,
+                       uLongf* dest_length,
+                       const Bytef* source,
+                       uLong source_length) {
+    z_stream stream;
+
+    stream.next_in = bit_cast<Bytef*>(source);
+    stream.avail_in = static_cast<uInt>(source_length);
+    stream.next_out = dest;
+    stream.avail_out = static_cast<uInt>(*dest_length);
+    if (static_cast<uLong>(stream.avail_out) != *dest_length)
+      return Z_BUF_ERROR;
+
+    stream.zalloc = static_cast<alloc_func>(0);
+    stream.zfree = static_cast<free_func>(0);
+    stream.opaque = static_cast<voidpf>(0);
+
+    gz_header gzip_header;
+    memset(&gzip_header, 0, sizeof(gzip_header));
+    int err = deflateInit2_(&stream,
+                            Z_DEFAULT_COMPRESSION,
+                            Z_DEFLATED,
+                            MAX_WBITS + kWindowBitsToGetGzipHeader,
+                            kZlibMemoryLevel,
+                            Z_DEFAULT_STRATEGY,
+                            ZLIB_VERSION,
+                            sizeof(z_stream));
+    if (err != Z_OK)
+      return err;
+
+    err = deflateSetHeader(&stream, &gzip_header);
+    if (err != Z_OK)
+      return err;
+
+    err = deflate(&stream, Z_FINISH);
+    if (err != Z_STREAM_END) {
+      deflateEnd(&stream);
+      return err == Z_OK ? Z_BUF_ERROR : err;
+    }
+    *dest_length = stream.total_out;
+
+    err = deflateEnd(&stream);
+    return err;
+}
+
+}  // namespace
+
+namespace metrics {
+
+bool GzipCompress(const std::string& input, std::string* output) {
+  const uLongf input_size = static_cast<uLongf>(input.size());
+  std::vector<Bytef> compressed_data(kGzipZlibHeaderDifferenceBytes +
+                                     compressBound(input_size));
+
+  uLongf compressed_size = static_cast<uLongf>(compressed_data.size());
+  if (GzipCompressHelper(&compressed_data.front(),
+                         &compressed_size,
+                         bit_cast<const Bytef*>(input.data()),
+                         input_size) != Z_OK) {
+    return false;
+  }
+
+  compressed_data.resize(compressed_size);
+  output->assign(compressed_data.begin(), compressed_data.end());
+  return true;
+}
+
+}  // namespace metrics
diff --git a/components/metrics/net/compression_utils.h b/components/metrics/net/compression_utils.h
new file mode 100644
index 0000000..ee875dd
--- /dev/null
+++ b/components/metrics/net/compression_utils.h
@@ -0,0 +1,17 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_METRICS_NET_COMPRESSION_UTILS_H_
+#define COMPONENTS_METRICS_NET_COMPRESSION_UTILS_H_
+
+#include <string>
+
+namespace metrics {
+
+// Compresses the text in |input| using gzip, storing the result in |output|.
+bool GzipCompress(const std::string& input, std::string* output);
+
+}  // namespace metrics
+
+#endif  // COMPONENTS_METRICS_NET_COMPRESSION_UTILS_H_
diff --git a/components/metrics/net/compression_utils_unittest.cc b/components/metrics/net/compression_utils_unittest.cc
new file mode 100644
index 0000000..05305d4
--- /dev/null
+++ b/components/metrics/net/compression_utils_unittest.cc
@@ -0,0 +1,52 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/metrics/net/compression_utils.h"
+
+#include <string>
+
+#include "base/base_paths.h"
+#include "base/basictypes.h"
+#include "base/file_util.h"
+#include "base/path_service.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace metrics {
+
+namespace {
+
+// The data to be compressed by gzip. This is the hex representation of "hello
+// world".
+const uint8 kData[] =
+    {0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f,
+     0x72, 0x6c, 0x64};
+
+// This is the string representation of gzip compressed string above. It was
+// obtained by running echo -n "hello world" | gzip -c | hexdump -e '8 1 ",
+// 0x%x"' followed by 0'ing out the OS byte (10th byte) in the header. This is
+// so that the test passes on all platforms (that run various OS'es).
+const uint8 kCompressedData[] =
+    {0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+     0x00, 0xcb, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf,
+     0x2f, 0xca, 0x49, 0x01, 0x00, 0x85, 0x11, 0x4a, 0x0d,
+     0x0b, 0x00, 0x00, 0x00};
+
+// Re-enable C4309.
+#if defined(OS_WIN)
+#pragma warning( default: 4309 )
+#endif
+
+}  // namespace
+
+TEST(CompressionUtilsTest, GzipCompression) {
+  std::string data(reinterpret_cast<const char*>(kData), arraysize(kData));
+  std::string compressed_data;
+  EXPECT_TRUE(GzipCompress(data, &compressed_data));
+  std::string golden_compressed_data(
+      reinterpret_cast<const char*>(kCompressedData),
+      arraysize(kCompressedData));
+  EXPECT_EQ(golden_compressed_data, compressed_data);
+}
+
+}  // namespace metrics
diff --git a/components/metrics/net/net_metrics_log_uploader.cc b/components/metrics/net/net_metrics_log_uploader.cc
new file mode 100644
index 0000000..09042bd
--- /dev/null
+++ b/components/metrics/net/net_metrics_log_uploader.cc
@@ -0,0 +1,74 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/metrics/net/net_metrics_log_uploader.h"
+
+#include "base/metrics/histogram.h"
+#include "components/metrics/net/compression_utils.h"
+#include "net/base/load_flags.h"
+#include "net/url_request/url_fetcher.h"
+#include "url/gurl.h"
+
+namespace metrics {
+
+NetMetricsLogUploader::NetMetricsLogUploader(
+    net::URLRequestContextGetter* request_context_getter,
+    const std::string& server_url,
+    const std::string& mime_type,
+    const base::Callback<void(int)>& on_upload_complete)
+    : MetricsLogUploader(server_url, mime_type, on_upload_complete),
+      request_context_getter_(request_context_getter) {
+}
+
+NetMetricsLogUploader::~NetMetricsLogUploader() {
+}
+
+bool NetMetricsLogUploader::UploadLog(const std::string& log_data,
+                                      const std::string& log_hash) {
+  std::string compressed_log_data;
+  if (!GzipCompress(log_data, &compressed_log_data)) {
+    NOTREACHED();
+    return false;
+  }
+
+  UMA_HISTOGRAM_PERCENTAGE(
+      "UMA.ProtoCompressionRatio",
+      static_cast<int>(100 * compressed_log_data.size() / log_data.size()));
+  UMA_HISTOGRAM_CUSTOM_COUNTS(
+      "UMA.ProtoGzippedKBSaved",
+      static_cast<int>((log_data.size() - compressed_log_data.size()) / 1024),
+      1, 2000, 50);
+
+  current_fetch_.reset(
+      net::URLFetcher::Create(GURL(server_url_), net::URLFetcher::POST, this));
+  current_fetch_->SetRequestContext(request_context_getter_);
+  current_fetch_->SetUploadData(mime_type_, compressed_log_data);
+
+  // Tell the server that we're uploading gzipped protobufs.
+  current_fetch_->SetExtraRequestHeaders("content-encoding: gzip");
+
+  DCHECK(!log_hash.empty());
+  current_fetch_->AddExtraRequestHeader("X-Chrome-UMA-Log-SHA1: " + log_hash);
+
+  // We already drop cookies server-side, but we might as well strip them out
+  // client-side as well.
+  current_fetch_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES |
+                               net::LOAD_DO_NOT_SEND_COOKIES);
+  return true;
+}
+
+void NetMetricsLogUploader::OnURLFetchComplete(const net::URLFetcher* source) {
+  // We're not allowed to re-use the existing |URLFetcher|s, so free them here.
+  // Note however that |source| is aliased to the fetcher, so we should be
+  // careful not to delete it too early.
+  DCHECK_EQ(current_fetch_.get(), source);
+
+  int response_code = source->GetResponseCode();
+  if (response_code == net::URLFetcher::RESPONSE_CODE_INVALID)
+    response_code = -1;
+  on_upload_complete_.Run(response_code);
+  current_fetch_.reset();
+}
+
+}  // namespace metrics
diff --git a/components/metrics/net/net_metrics_log_uploader.h b/components/metrics/net/net_metrics_log_uploader.h
new file mode 100644
index 0000000..4753fd7
--- /dev/null
+++ b/components/metrics/net/net_metrics_log_uploader.h
@@ -0,0 +1,54 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_METRICS_NET_NET_METRICS_LOG_UPLOADER_H_
+#define COMPONENTS_METRICS_NET_NET_METRICS_LOG_UPLOADER_H_
+
+#include <string>
+
+#include "base/memory/scoped_ptr.h"
+#include "components/metrics/metrics_log_uploader.h"
+#include "net/url_request/url_fetcher_delegate.h"
+
+namespace net {
+class URLFetcher;
+class URLRequestContextGetter;
+}
+
+namespace metrics {
+
+// Implementation of MetricsLogUploader using the Chrome network stack.
+class NetMetricsLogUploader : public MetricsLogUploader,
+                              public net::URLFetcherDelegate {
+ public:
+  // Constructs a NetMetricsLogUploader with the specified request context and
+  // other params (see comments on MetricsLogUploader for details). The caller
+  // must ensure that |request_context_getter| remains valid for the lifetime
+  // of this class.
+  NetMetricsLogUploader(net::URLRequestContextGetter* request_context_getter,
+                        const std::string& server_url,
+                        const std::string& mime_type,
+                        const base::Callback<void(int)>& on_upload_complete);
+  virtual ~NetMetricsLogUploader();
+
+  // MetricsLogUploader:
+  virtual bool UploadLog(const std::string& log_data,
+                         const std::string& log_hash) OVERRIDE;
+
+ private:
+  // net::URLFetcherDelegate:
+  virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
+
+  // The request context for fetches done using the network stack.
+  net::URLRequestContextGetter* const request_context_getter_;
+
+  // The outstanding transmission appears as a URL Fetch operation.
+  scoped_ptr<net::URLFetcher> current_fetch_;
+
+  DISALLOW_COPY_AND_ASSIGN(NetMetricsLogUploader);
+};
+
+}  // namespace metrics
+
+#endif  // COMPONENTS_METRICS_NET_NET_METRICS_LOG_UPLOADER_H_
diff --git a/components/metrics/test_metrics_service_client.cc b/components/metrics/test_metrics_service_client.cc
index 891b90f..8d1b337 100644
--- a/components/metrics/test_metrics_service_client.cc
+++ b/components/metrics/test_metrics_service_client.cc
@@ -5,6 +5,7 @@
 #include "components/metrics/test_metrics_service_client.h"
 
 #include "base/callback.h"
+#include "components/metrics/metrics_log_uploader.h"
 
 namespace metrics {
 
@@ -55,4 +56,11 @@ void TestMetricsServiceClient::CollectFinalMetrics(
   done_callback.Run();
 }
 
+scoped_ptr<MetricsLogUploader> TestMetricsServiceClient::CreateUploader(
+    const std::string& server_url,
+    const std::string& mime_type,
+    const base::Callback<void(int)>& on_upload_complete) {
+  return scoped_ptr<MetricsLogUploader>();
+}
+
 }  // namespace metrics
diff --git a/components/metrics/test_metrics_service_client.h b/components/metrics/test_metrics_service_client.h
index b5b7ba9..407109d 100644
--- a/components/metrics/test_metrics_service_client.h
+++ b/components/metrics/test_metrics_service_client.h
@@ -32,6 +32,10 @@ class TestMetricsServiceClient : public MetricsServiceClient {
       const base::Closure& done_callback) OVERRIDE;
   virtual void CollectFinalMetrics(const base::Closure& done_callback)
       OVERRIDE;
+  virtual scoped_ptr<MetricsLogUploader> CreateUploader(
+      const std::string& server_url,
+      const std::string& mime_type,
+      const base::Callback<void(int)>& on_upload_complete) OVERRIDE;
 
   const std::string& get_client_id() const { return client_id_; }