Precache component

Layered component that precaches resources that we predict the user will want to fetch in the future, based on a list of page URLs that the user commonly visits. It interacts with a server component to determine the list of resources to precache.

This issue is a continuation of https://codereview.chromium.org/25481003, new issue created so that CL creator and CL author match.

BUG=306185

Review URL: https://codereview.chromium.org/27216002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@229341 0039d316-1c4b-4281-b951-d872f2087c98

11 files changed, 785 insertions, 0 deletions

diff --git a/components/precache/DEPS b/components/precache/DEPS
new file mode 100644
index 0000000..0caaca9
--- /dev/null
+++ b/components/precache/DEPS
@@ -0,0 +1,8 @@
+include_rules = [
+  # Precache is a layered component; subdirectories must introduce the ability
+  # to use the content layer explicitly as appropriate.
+  # http://www.chromium.org/developers/design-documents/layered-components-design
+  "-components/precache",
+  "+components/precache/core",
+  "-content/public",
+]
diff --git a/components/precache/OWNERS b/components/precache/OWNERS
new file mode 100644
index 0000000..38f02cc1
--- /dev/null
+++ b/components/precache/OWNERS
@@ -0,0 +1,2 @@
+bengr@chromium.org
+sclittle@chromium.org
diff --git a/components/precache/README b/components/precache/README
new file mode 100644
index 0000000..438ab7b
--- /dev/null
+++ b/components/precache/README
@@ -0,0 +1,11 @@
+The precache component contains code for an experimental prototype to
+proactively load Web resources into the network stack's disk cache.
+
+To enable this feature use the command line flag --enable-precache.
+
+Precache is a layered component. See:
+http://www.chromium.org/developers/design-documents/layered-components-design
+
+Folder structure:
+
+  core/   contains the core logic of the component.
diff --git a/components/precache/core/DEPS b/components/precache/core/DEPS
new file mode 100644
index 0000000..52b15f6
--- /dev/null
+++ b/components/precache/core/DEPS
@@ -0,0 +1,10 @@
+include_rules = [
+  "+net/base",
+  "+net/url_request",
+]
+
+specific_include_rules = {
+  '.*_[a-z]*test\.cc': [
+    "+net/http",
+  ],
+}
diff --git a/components/precache/core/precache_fetcher.cc b/components/precache/core/precache_fetcher.cc
new file mode 100644
index 0000000..42b281f
--- /dev/null
+++ b/components/precache/core/precache_fetcher.cc
@@ -0,0 +1,241 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/precache/core/precache_fetcher.h"
+
+#include <string>
+
+#include "base/bind.h"
+#include "base/callback.h"
+#include "base/command_line.h"
+#include "base/compiler_specific.h"
+#include "base/containers/hash_tables.h"
+#include "components/precache/core/precache_switches.h"
+#include "components/precache/core/proto/precache.pb.h"
+#include "net/base/escape.h"
+#include "net/base/load_flags.h"
+#include "net/url_request/url_fetcher.h"
+#include "net/url_request/url_fetcher_delegate.h"
+#include "net/url_request/url_request_context_getter.h"
+#include "net/url_request/url_request_status.h"
+
+using net::URLFetcher;
+
+namespace precache {
+
+namespace {
+
+GURL GetConfigURL() {
+  const CommandLine& command_line = *CommandLine::ForCurrentProcess();
+  if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
+    return GURL(
+        command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
+  }
+
+#if defined(PRECACHE_CONFIG_SETTINGS_URL)
+  return GURL(PRECACHE_CONFIG_SETTINGS_URL);
+#else
+  // The precache config settings URL could not be determined, so return an
+  // empty, invalid GURL.
+  return GURL();
+#endif
+}
+
+std::string GetManifestURLPrefix() {
+  const CommandLine& command_line = *CommandLine::ForCurrentProcess();
+  if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
+    return command_line.GetSwitchValueASCII(
+        switches::kPrecacheManifestURLPrefix);
+  }
+
+#if defined(PRECACHE_MANIFEST_URL_PREFIX)
+  return PRECACHE_MANIFEST_URL_PREFIX;
+#else
+  // The precache manifest URL prefix could not be determined, so return an
+  // empty string.
+  return std::string();
+#endif
+}
+
+// Construct the URL of the precache manifest for the given starting URL.
+// The server is expecting a request for a URL consisting of the manifest URL
+// prefix followed by the doubly escaped starting URL.
+GURL ConstructManifestURL(const GURL& starting_url) {
+  return GURL(
+      GetManifestURLPrefix() +
+      net::EscapeQueryParamValue(
+          net::EscapeQueryParamValue(starting_url.spec(), false), false));
+}
+
+// Attempts to parse a protobuf message from the response string of a
+// URLFetcher. If parsing is successful, the message parameter will contain the
+// parsed protobuf and this function will return true. Otherwise, returns false.
+bool ParseProtoFromFetchResponse(const URLFetcher& source,
+                                 ::google::protobuf::MessageLite* message) {
+  std::string response_string;
+
+  if (!source.GetStatus().is_success()) {
+    DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
+    return false;
+  }
+  if (!source.GetResponseAsString(&response_string)) {
+    DLOG(WARNING) << "No response string present: "
+                  << source.GetOriginalURL().spec();
+    return false;
+  }
+  if (!message->ParseFromString(response_string)) {
+    DLOG(WARNING) << "Unable to parse proto served from "
+                  << source.GetOriginalURL().spec();
+    return false;
+  }
+  return true;
+}
+
+}  // namespace
+
+// Class that fetches a URL, and runs the specified callback when the fetch is
+// complete. This class exists so that a different method can be run in
+// response to different kinds of fetches, e.g. OnConfigFetchComplete when
+// configuration settings are fetched, OnManifestFetchComplete when a manifest
+// is fetched, etc.
+class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
+ public:
+  // Construct a new Fetcher. This will create and start a new URLFetcher for
+  // the specified URL using the specified request context.
+  Fetcher(net::URLRequestContextGetter* request_context, const GURL& url,
+          const base::Callback<void(const URLFetcher&)>& callback);
+  virtual ~Fetcher() {}
+  virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
+
+ private:
+  const base::Callback<void(const URLFetcher&)> callback_;
+  scoped_ptr<URLFetcher> url_fetcher_;
+
+  DISALLOW_COPY_AND_ASSIGN(Fetcher);
+};
+
+PrecacheFetcher::Fetcher::Fetcher(
+    net::URLRequestContextGetter* request_context, const GURL& url,
+    const base::Callback<void(const URLFetcher&)>& callback)
+    : callback_(callback) {
+  url_fetcher_.reset(URLFetcher::Create(url, URLFetcher::GET, this));
+  url_fetcher_->SetRequestContext(request_context);
+  url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_PROMPT_FOR_LOGIN);
+  url_fetcher_->Start();
+}
+
+void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
+  callback_.Run(*source);
+}
+
+PrecacheFetcher::PrecacheFetcher(
+    const std::list<GURL>& starting_urls,
+    net::URLRequestContextGetter* request_context,
+    PrecacheFetcher::PrecacheDelegate* precache_delegate)
+    : starting_urls_(starting_urls),
+      request_context_(request_context),
+      precache_delegate_(precache_delegate) {
+  DCHECK(request_context_);    // Request context must be non-NULL.
+  DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
+
+  DCHECK_NE(GURL(), GetConfigURL())
+      << "Could not determine the precache config settings URL.";
+  DCHECK_NE(std::string(), GetManifestURLPrefix())
+      << "Could not determine the precache manifest URL prefix.";
+}
+
+PrecacheFetcher::~PrecacheFetcher() {
+}
+
+void PrecacheFetcher::Start() {
+  DCHECK(!fetcher_);  // Start shouldn't be called repeatedly.
+
+  GURL config_url = GetConfigURL();
+  DCHECK(config_url.is_valid());
+
+  // Fetch the precache configuration settings from the server.
+  fetcher_.reset(new Fetcher(request_context_, config_url,
+                             base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
+                                        base::Unretained(this))));
+}
+
+void PrecacheFetcher::StartNextFetch() {
+  if (!resource_urls_to_fetch_.empty()) {
+    // Fetch the next resource URL.
+    fetcher_.reset(
+        new Fetcher(request_context_, resource_urls_to_fetch_.front(),
+                    base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
+                               base::Unretained(this))));
+
+    resource_urls_to_fetch_.pop_front();
+    return;
+  }
+
+  if (!manifest_urls_to_fetch_.empty()) {
+    // Fetch the next manifest URL.
+    fetcher_.reset(
+        new Fetcher(request_context_, manifest_urls_to_fetch_.front(),
+                    base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
+                               base::Unretained(this))));
+
+    manifest_urls_to_fetch_.pop_front();
+    return;
+  }
+
+  // There are no more URLs to fetch, so end the precache cycle.
+  precache_delegate_->OnDone();
+  // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
+  // is called.
+}
+
+void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
+  PrecacheConfigurationSettings config;
+
+  if (ParseProtoFromFetchResponse(source, &config)) {
+    // A hash set of strings is used instead of GURLs because there is no
+    // standard hash function defined for GURLs.
+    base::hash_set<std::string> whitelisted_urls;
+    for (int i = 0; i < config.whitelisted_starting_url_size(); ++i) {
+      // Instead of using the raw URL string, construct a GURL and take the spec
+      // of it so that the URL string is canonicalized.
+      whitelisted_urls.insert(GURL(config.whitelisted_starting_url(i)).spec());
+    }
+
+    // Only fetch manifests for starting URLs up to the maximum rank that are in
+    // the whitelist.
+    int64 rank = 0;
+    for (std::list<GURL>::const_iterator it = starting_urls_.begin();
+         it != starting_urls_.end() &&
+             rank < config.maximum_rank_starting_url();
+         ++it, ++rank) {
+      if (whitelisted_urls.find(it->spec()) != whitelisted_urls.end()) {
+        manifest_urls_to_fetch_.push_back(ConstructManifestURL(*it));
+      }
+    }
+  }
+
+  StartNextFetch();
+}
+
+void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
+  PrecacheManifest manifest;
+
+  if (ParseProtoFromFetchResponse(source, &manifest)) {
+    for (int i = 0; i < manifest.resource_size(); ++i) {
+      if (manifest.resource(i).has_url()) {
+        resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
+      }
+    }
+  }
+
+  StartNextFetch();
+}
+
+void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
+  // The resource has already been put in the cache during the fetch process, so
+  // nothing more needs to be done for the resource.
+  StartNextFetch();
+}
+
+}  // namespace precache
diff --git a/components/precache/core/precache_fetcher.h b/components/precache/core/precache_fetcher.h
new file mode 100644
index 0000000..a3dde62
--- /dev/null
+++ b/components/precache/core/precache_fetcher.h
@@ -0,0 +1,128 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
+#define COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
+
+#include <list>
+
+#include "base/basictypes.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
+#include "url/gurl.h"
+
+namespace net {
+class URLFetcher;
+class URLRequestContextGetter;
+}
+
+namespace precache {
+
+// Public interface to code that fetches resources that the user is likely to
+// want to fetch in the future, putting them in the network stack disk cache.
+// Precaching is intended to be done when Chrome is not actively in use, likely
+// hours ahead of the time when the resources are actually needed.
+//
+// This class takes as input a prioritized list of page URLs that the user
+// commonly visits, referred to as starting URLs. This class interacts with a
+// server, sending it the list of starting URLs sequentially. For each starting
+// URL, the server returns a manifest of resource URLs that are good candidates
+// for precaching. Every resource returned is fetched, and responses are cached
+// as they are received. Destroying the PrecacheFetcher while it is precaching
+// will cancel any fetch in progress and cancel precaching.
+//
+// The URLs of the server-side component must be specified in order for the
+// PrecacheFetcher to work. This includes the URL that the precache
+// configuration settings are fetched from and the prefix of URLs where precache
+// manifests are fetched from. These can be set by using command line switches
+// or by providing default values.
+//
+// Sample interaction:
+//
+// class MyPrecacheFetcherDelegate : public PrecacheFetcher::PrecacheDelegate {
+//  public:
+//   void PrecacheResourcesForTopURLs(
+//       net::URLRequestContextGetter* request_context,
+//       const std::list<GURL>& top_urls) {
+//     fetcher_.reset(new PrecacheFetcher(request_context, top_urls, this));
+//     fetcher_->Start();
+//   }
+//
+//   virtual void OnDone() {
+//     // Do something when precaching is done.
+//   }
+//
+//  private:
+//   scoped_ptr<PrecacheFetcher> fetcher_;
+// };
+class PrecacheFetcher {
+ public:
+  class PrecacheDelegate {
+   public:
+    // Called when the fetching of resources has finished, whether the resources
+    // were fetched or not. If the PrecacheFetcher is destroyed before OnDone is
+    // called, then precaching will be canceled and OnDone will not be called.
+    virtual void OnDone() = 0;
+  };
+
+  // Constructs a new PrecacheFetcher. The |starting_urls| parameter is a
+  // prioritized list of page URLs that the user commonly visits. These URLs are
+  // used by a server side component to construct a list of resource URLs that
+  // the user is likely to fetch.
+  PrecacheFetcher(const std::list<GURL>& starting_urls,
+                  net::URLRequestContextGetter* request_context,
+                  PrecacheDelegate* precache_delegate);
+
+  virtual ~PrecacheFetcher();
+
+  // Starts fetching resources to precache. URLs are fetched sequentially. Can
+  // be called from any thread. Start should only be called once on a
+  // PrecacheFetcher instance.
+  void Start();
+
+ private:
+  class Fetcher;
+
+  // Fetches the next resource or manifest URL, if any remain. Fetching is done
+  // sequentially and depth-first: all resources are fetched for a manifest
+  // before the next manifest is fetched. This is done to limit the length of
+  // the |resource_urls_to_fetch_| list, reducing the memory usage.
+  void StartNextFetch();
+
+  // Called when the precache configuration settings have been fetched.
+  // Determines the list of manifest URLs to fetch according to the URLs that
+  // are present in both the list of |starting_urls_| and the whitelist
+  // contained in the precache configuration settings. If the fetch of the
+  // configuration settings fails, then precaching ends.
+  void OnConfigFetchComplete(const net::URLFetcher& source);
+
+  // Called when a precache manifest has been fetched. Builds the list of
+  // resource URLs to fetch according to the URLs in the manifest. If the fetch
+  // of a manifest fails, then it skips to the next manifest.
+  void OnManifestFetchComplete(const net::URLFetcher& source);
+
+  // Called when a resource has been fetched.
+  void OnResourceFetchComplete(const net::URLFetcher& source);
+
+  // The prioritized list of starting URLs that the server will pick resource
+  // URLs to be precached for.
+  const std::list<GURL> starting_urls_;
+
+  // The request context used when fetching URLs.
+  scoped_refptr<net::URLRequestContextGetter> request_context_;
+
+  // Non-owning pointer. Should not be NULL.
+  PrecacheDelegate* precache_delegate_;
+
+  scoped_ptr<Fetcher> fetcher_;
+
+  std::list<GURL> manifest_urls_to_fetch_;
+  std::list<GURL> resource_urls_to_fetch_;
+
+  DISALLOW_COPY_AND_ASSIGN(PrecacheFetcher);
+};
+
+}  // namespace precache
+
+#endif  // COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
diff --git a/components/precache/core/precache_fetcher_unittest.cc b/components/precache/core/precache_fetcher_unittest.cc
new file mode 100644
index 0000000..da2bdca
--- /dev/null
+++ b/components/precache/core/precache_fetcher_unittest.cc
@@ -0,0 +1,289 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/precache/core/precache_fetcher.h"
+
+#include <list>
+#include <set>
+#include <string>
+
+#include "base/basictypes.h"
+#include "base/bind.h"
+#include "base/callback.h"
+#include "base/command_line.h"
+#include "base/compiler_specific.h"
+#include "base/message_loop/message_loop.h"
+#include "components/precache/core/precache_switches.h"
+#include "components/precache/core/proto/precache.pb.h"
+#include "net/http/http_response_headers.h"
+#include "net/url_request/test_url_fetcher_factory.h"
+#include "net/url_request/url_request_test_util.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace precache {
+
+namespace {
+
+class TestURLFetcherCallback {
+ public:
+  scoped_ptr<net::FakeURLFetcher> CreateURLFetcher(
+      const GURL& url, net::URLFetcherDelegate* delegate,
+      const std::string& response_data, bool success) {
+    scoped_ptr<net::FakeURLFetcher> fetcher(
+        new net::FakeURLFetcher(url, delegate, response_data, success));
+
+    if (success) {
+      scoped_refptr<net::HttpResponseHeaders> download_headers =
+          new net::HttpResponseHeaders("");
+      download_headers->AddHeader("Content-Type: text/html");
+      fetcher->set_response_headers(download_headers);
+    }
+
+    requested_urls_.insert(url);
+    return fetcher.Pass();
+  }
+
+  const std::multiset<GURL>& requested_urls() const {
+    return requested_urls_;
+  }
+
+ private:
+  // Multiset with one entry for each URL requested.
+  std::multiset<GURL> requested_urls_;
+};
+
+class TestPrecacheDelegate : public PrecacheFetcher::PrecacheDelegate {
+ public:
+  TestPrecacheDelegate() : was_on_done_called_(false) {}
+
+  virtual void OnDone() OVERRIDE {
+    was_on_done_called_ = true;
+  }
+
+  bool was_on_done_called() const {
+    return was_on_done_called_;
+  }
+
+ private:
+  bool was_on_done_called_;
+};
+
+class PrecacheFetcherTest : public testing::Test {
+ public:
+  PrecacheFetcherTest()
+      : request_context_(new net::TestURLRequestContextGetter(
+            base::MessageLoopProxy::current())),
+        factory_(NULL, base::Bind(&TestURLFetcherCallback::CreateURLFetcher,
+                                  base::Unretained(&url_callback_))) {}
+
+ protected:
+  base::MessageLoopForUI loop_;
+  scoped_refptr<net::TestURLRequestContextGetter> request_context_;
+  TestURLFetcherCallback url_callback_;
+  net::FakeURLFetcherFactory factory_;
+  TestPrecacheDelegate precache_delegate_;
+};
+
+const char kConfigURL[] = "http://config-url.com";
+const char kManfiestURLPrefix[] = "http://manifest-url-prefix.com/";
+const char kManifestFetchFailureURL[] =
+    "http://manifest-url-prefix.com/"
+    "http%253A%252F%252Fmanifest-fetch-failure.com%252F";
+const char kBadManifestURL[] =
+    "http://manifest-url-prefix.com/http%253A%252F%252Fbad-manifest.com%252F";
+const char kGoodManifestURL[] =
+    "http://manifest-url-prefix.com/http%253A%252F%252Fgood-manifest.com%252F";
+const char kResourceFetchFailureURL[] = "http://resource-fetch-failure.com";
+const char kGoodResourceURL[] = "http://good-resource.com";
+
+TEST_F(PrecacheFetcherTest, FullPrecache) {
+  CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+      switches::kPrecacheConfigSettingsURL, kConfigURL);
+  CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+      switches::kPrecacheManifestURLPrefix, kManfiestURLPrefix);
+
+  std::list<GURL> starting_urls;
+  starting_urls.push_back(GURL("http://not-whitelisted.com"));
+  starting_urls.push_back(GURL("http://manifest-fetch-failure.com"));
+  starting_urls.push_back(GURL("http://bad-manifest.com"));
+  starting_urls.push_back(GURL("http://good-manifest.com"));
+  starting_urls.push_back(GURL("http://not-in-top-4.com"));
+
+  PrecacheConfigurationSettings config;
+  config.add_whitelisted_starting_url("http://whitelisted-unused.com");
+  config.add_whitelisted_starting_url("http://manifest-fetch-failure.com");
+  config.add_whitelisted_starting_url("http://bad-manifest.com");
+  config.add_whitelisted_starting_url("http://good-manifest.com");
+  config.add_whitelisted_starting_url("http://not-in-top-4.com");
+  config.set_maximum_rank_starting_url(4);
+
+  PrecacheManifest good_manifest;
+  good_manifest.add_resource()->set_url(kResourceFetchFailureURL);
+  good_manifest.add_resource();  // Resource with no URL, should not be fetched.
+  good_manifest.add_resource()->set_url(kGoodResourceURL);
+
+  factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(), true);
+  factory_.SetFakeResponse(GURL(kManifestFetchFailureURL), "", false);
+  factory_.SetFakeResponse(GURL(kBadManifestURL), "bad protobuf", true);
+  factory_.SetFakeResponse(GURL(kGoodManifestURL),
+                           good_manifest.SerializeAsString(), true);
+  factory_.SetFakeResponse(GURL(kResourceFetchFailureURL), "", false);
+  factory_.SetFakeResponse(GURL(kGoodResourceURL), "good", true);
+
+  PrecacheFetcher precache_fetcher(starting_urls, request_context_.get(),
+                                   &precache_delegate_);
+  precache_fetcher.Start();
+
+  base::MessageLoop::current()->RunUntilIdle();
+
+  std::multiset<GURL> expected_requested_urls;
+  expected_requested_urls.insert(GURL(kConfigURL));
+  expected_requested_urls.insert(GURL(kManifestFetchFailureURL));
+  expected_requested_urls.insert(GURL(kBadManifestURL));
+  expected_requested_urls.insert(GURL(kGoodManifestURL));
+  expected_requested_urls.insert(GURL(kResourceFetchFailureURL));
+  expected_requested_urls.insert(GURL(kGoodResourceURL));
+
+  EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
+
+  EXPECT_TRUE(precache_delegate_.was_on_done_called());
+}
+
+TEST_F(PrecacheFetcherTest, ConfigFetchFailure) {
+  CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+      switches::kPrecacheConfigSettingsURL, kConfigURL);
+
+  std::list<GURL> starting_urls(1, GURL("http://starting-url.com"));
+
+  factory_.SetFakeResponse(GURL(kConfigURL), "", false);
+
+  PrecacheFetcher precache_fetcher(starting_urls, request_context_.get(),
+                                   &precache_delegate_);
+  precache_fetcher.Start();
+
+  base::MessageLoop::current()->RunUntilIdle();
+
+  std::multiset<GURL> expected_requested_urls;
+  expected_requested_urls.insert(GURL(kConfigURL));
+  EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
+
+  EXPECT_TRUE(precache_delegate_.was_on_done_called());
+}
+
+TEST_F(PrecacheFetcherTest, BadConfig) {
+  CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+      switches::kPrecacheConfigSettingsURL, kConfigURL);
+
+  std::list<GURL> starting_urls(1, GURL("http://starting-url.com"));
+
+  factory_.SetFakeResponse(GURL(kConfigURL), "bad protobuf", true);
+
+  PrecacheFetcher precache_fetcher(starting_urls, request_context_.get(),
+                                   &precache_delegate_);
+  precache_fetcher.Start();
+
+  base::MessageLoop::current()->RunUntilIdle();
+
+  std::multiset<GURL> expected_requested_urls;
+  expected_requested_urls.insert(GURL(kConfigURL));
+  EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
+
+  EXPECT_TRUE(precache_delegate_.was_on_done_called());
+}
+
+TEST_F(PrecacheFetcherTest, Cancel) {
+  CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+      switches::kPrecacheConfigSettingsURL, kConfigURL);
+
+  std::list<GURL> starting_urls(1, GURL("http://starting-url.com"));
+
+  PrecacheConfigurationSettings config;
+  config.add_whitelisted_starting_url("http://starting-url.com");
+  config.set_maximum_rank_starting_url(1);
+
+  factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(), true);
+
+  scoped_ptr<PrecacheFetcher> precache_fetcher(new PrecacheFetcher(
+      starting_urls, request_context_.get(), &precache_delegate_));
+  precache_fetcher->Start();
+
+  // Destroy the PrecacheFetcher to cancel precaching. This should not cause
+  // OnDone to be called on the precache delegate.
+  precache_fetcher.reset();
+
+  base::MessageLoop::current()->RunUntilIdle();
+
+  std::multiset<GURL> expected_requested_urls;
+  expected_requested_urls.insert(GURL(kConfigURL));
+  EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
+
+  EXPECT_FALSE(precache_delegate_.was_on_done_called());
+}
+
+#if defined(PRECACHE_CONFIG_SETTINGS_URL)
+
+// If the default precache configuration settings URL is defined, then test that
+// it works with the PrecacheFetcher.
+TEST_F(PrecacheFetcherTest, PrecacheUsingDefaultConfigSettingsURL) {
+  std::list<GURL> starting_urls(1, GURL("http://starting-url.com"));
+
+  factory_.SetFakeResponse(GURL(PRECACHE_CONFIG_SETTINGS_URL),
+                           PrecacheConfigurationSettings().SerializeAsString(),
+                           true);
+
+  PrecacheFetcher precache_fetcher(starting_urls, request_context_.get(),
+                                   &precache_delegate_);
+  precache_fetcher.Start();
+
+  base::MessageLoop::current()->RunUntilIdle();
+
+  std::multiset<GURL> expected_requested_urls;
+  expected_requested_urls.insert(GURL(PRECACHE_CONFIG_SETTINGS_URL));
+  EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
+
+  EXPECT_TRUE(precache_delegate_.was_on_done_called());
+}
+
+#endif  // PRECACHE_CONFIG_SETTINGS_URL
+
+#if defined(PRECACHE_MANIFEST_URL_PREFIX)
+
+// If the default precache manifest URL prefix is defined, then test that it
+// works with the PrecacheFetcher.
+TEST_F(PrecacheFetcherTest, PrecacheUsingDefaultManifestURLPrefix) {
+  CommandLine::ForCurrentProcess()->AppendSwitchASCII(
+      switches::kPrecacheConfigSettingsURL, kConfigURL);
+
+  std::list<GURL> starting_urls(1, GURL("http://starting-url.com"));
+
+  PrecacheConfigurationSettings config;
+  config.add_whitelisted_starting_url("http://starting-url.com");
+  config.set_maximum_rank_starting_url(1);
+
+  GURL manifest_url(PRECACHE_MANIFEST_URL_PREFIX
+                    "http%253A%252F%252Fstarting-url.com%252F");
+
+  factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(), true);
+  factory_.SetFakeResponse(manifest_url, PrecacheManifest().SerializeAsString(),
+                           true);
+
+  PrecacheFetcher precache_fetcher(starting_urls, request_context_.get(),
+                                   &precache_delegate_);
+  precache_fetcher.Start();
+
+  base::MessageLoop::current()->RunUntilIdle();
+
+  std::multiset<GURL> expected_requested_urls;
+  expected_requested_urls.insert(GURL(kConfigURL));
+  expected_requested_urls.insert(manifest_url);
+  EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
+
+  EXPECT_TRUE(precache_delegate_.was_on_done_called());
+}
+
+#endif  // PRECACHE_MANIFEST_URL_PREFIX
+
+}  // namespace
+
+}  // namespace precache
diff --git a/components/precache/core/precache_switches.cc b/components/precache/core/precache_switches.cc
new file mode 100644
index 0000000..156546e
--- /dev/null
+++ b/components/precache/core/precache_switches.cc
@@ -0,0 +1,17 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/precache/core/precache_switches.h"
+
+namespace precache {
+namespace switches {
+
+// The URL that provides the PrecacheConfigurationSettings proto.
+const char kPrecacheConfigSettingsURL[] = "precache-config-settings-url";
+
+// Precache manifests will be served from URLs with this prefix.
+const char kPrecacheManifestURLPrefix[] = "precache-manifest-url-prefix";
+
+}  // namespace switches
+}  // namespace precache
diff --git a/components/precache/core/precache_switches.h b/components/precache/core/precache_switches.h
new file mode 100644
index 0000000..c0d2aba
--- /dev/null
+++ b/components/precache/core/precache_switches.h
@@ -0,0 +1,19 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_PRECACHE_CORE_PRECACHE_SWITCHES_H_
+#define COMPONENTS_PRECACHE_CORE_PRECACHE_SWITCHES_H_
+
+namespace precache {
+namespace switches {
+
+// All switches in alphabetical order. The switches should be documented
+// alongside the definition of their values in the .cc file.
+extern const char kPrecacheConfigSettingsURL[];
+extern const char kPrecacheManifestURLPrefix[];
+
+}  // namespace switches
+}  // namespace precache
+
+#endif  // COMPONENTS_PRECACHE_CORE_PRECACHE_SWITCHES_H_
diff --git a/components/precache/core/proto/precache.proto b/components/precache/core/proto/precache.proto
new file mode 100644
index 0000000..0429d58
--- /dev/null
+++ b/components/precache/core/proto/precache.proto
@@ -0,0 +1,38 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto2";
+
+package precache;
+
+// Chrome requires this.
+option optimize_for = LITE_RUNTIME;
+
+// Information about a cacheable resource to be precached.
+message PrecacheResource {
+  // The URL of the resource. This field must always be present.
+  optional string url = 1;
+};
+
+// A manifest of cacheable resources to be precached for a specific starting
+// URL.
+message PrecacheManifest {
+  // List of resources that we predict that the user will need if they are
+  // likely to fetch the starting URL.
+  repeated PrecacheResource resource = 1;
+};
+
+message PrecacheConfigurationSettings {
+  // The whitelist of starting URLs that are currently supported. Precaching
+  // should only be attempted for starting URLs that are in this list.
+  repeated string whitelisted_starting_url = 1;
+
+  // The maximum rank of the user's most visited URLs to consider precaching
+  // resources for, starting from 1. For example, a value of 10 means that only
+  // URLs that are both in the user's top 10 most visited URLs and in the
+  // whitelist will be considered as starting URLs for resource precaching.
+  // This is specified by the server for testing purposes, so that we can
+  // easily adjust how aggressively we precache resources.
+  optional int64 maximum_rank_starting_url = 2 [default = 10];
+};
diff --git a/components/precache/precache_defines.gypi b/components/precache/precache_defines.gypi
new file mode 100644
index 0000000..eb8c25a
--- /dev/null
+++ b/components/precache/precache_defines.gypi
@@ -0,0 +1,22 @@
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+{
+  'variables': {
+    'precache_config_settings_url%': 'http://www.gstatic.com/chrome/wifiprefetch/precache_config',
+    'precache_manifest_url_prefix%': 'http://www.gstatic.com/chrome/wifiprefetch/precache_manifest_',
+  },
+  'conditions': [
+    ['precache_config_settings_url != ""', {
+      'defines': [
+        'PRECACHE_CONFIG_SETTINGS_URL="<(precache_config_settings_url)"',
+      ],
+    }],
+    ['precache_manifest_url_prefix != ""', {
+      'defines': [
+        'PRECACHE_MANIFEST_URL_PREFIX="<(precache_manifest_url_prefix)"',
+      ],
+    }],
+  ],
+}