summaryrefslogtreecommitdiffstats
path: root/chrome
diff options
context:
space:
mode:
authorzhenw <zhenw@chromium.org>2014-09-15 15:06:25 -0700
committerCommit bot <commit-bot@chromium.org>2014-09-15 22:09:14 +0000
commit825722d5939dc0832d124b70958501611e6a4628 (patch)
tree4bdd2610512fd377868bf894be22875e0947c6ee /chrome
parent0761425c09c0d1f364095aa745f57b59b8843236 (diff)
downloadchromium_src-825722d5939dc0832d124b70958501611e6a4628.zip
chromium_src-825722d5939dc0832d124b70958501611e6a4628.tar.gz
chromium_src-825722d5939dc0832d124b70958501611e6a4628.tar.bz2
Revert CL 117933003. Re-add speculative resource prefetching code.
https://codereview.chromium.org/117933003/ The speculative resource prefetching code was experimental code developed by shishir@. He found that it has little improvement on desktop Chrome (win). We though this should be beneficial to mobile browsers. After discussing with tburkard@ and kenjibaheux@, we decided to bring the code back and do more analysis on mobile devices. Reverting the patchset to re-add the code is the first step. The following design doc has discussed all related approaches and action items. https://docs.google.com/a/google.com/document/d/1ie3hu-zNNXvmTXm3aJAtKUGOh6nZfbNjA0aZE1bzzIg/edit?usp=sharing BUG=408399, 405690 Review URL: https://codereview.chromium.org/462423004 Cr-Commit-Position: refs/heads/master@{#294899}
Diffstat (limited to 'chrome')
-rw-r--r--chrome/browser/net/resource_prefetch_predictor_observer.cc202
-rw-r--r--chrome/browser/net/resource_prefetch_predictor_observer.h50
-rw-r--r--chrome/browser/predictors/predictor_database.cc19
-rw-r--r--chrome/browser/predictors/predictor_database.h2
-rw-r--r--chrome/browser/predictors/resource_prefetch_common.cc245
-rw-r--r--chrome/browser/predictors/resource_prefetch_common.h116
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor.cc1222
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor.h314
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor_factory.cc52
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor_factory.h38
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc62
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h34
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor_tables.cc513
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor_tables.h160
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor_tables_unittest.cc472
-rw-r--r--chrome/browser/predictors/resource_prefetch_predictor_unittest.cc925
-rw-r--r--chrome/browser/predictors/resource_prefetcher.cc240
-rw-r--r--chrome/browser/predictors/resource_prefetcher.h165
-rw-r--r--chrome/browser/predictors/resource_prefetcher_manager.cc135
-rw-r--r--chrome/browser/predictors/resource_prefetcher_manager.h87
-rw-r--r--chrome/browser/predictors/resource_prefetcher_unittest.cc343
-rw-r--r--chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc7
-rw-r--r--chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc2
-rw-r--r--chrome/browser/profiles/profile_io_data.cc15
-rw-r--r--chrome/browser/profiles/profile_io_data.h14
-rw-r--r--chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc14
-rw-r--r--chrome/browser/resources/predictors/predictors.html4
-rw-r--r--chrome/browser/resources/predictors/predictors.js1
-rw-r--r--chrome/browser/resources/predictors/resource_prefetch_predictor.html49
-rw-r--r--chrome/browser/resources/predictors/resource_prefetch_predictor.js102
-rw-r--r--chrome/browser/ui/tab_helpers.cc8
-rw-r--r--chrome/browser/ui/webui/predictors/predictors_handler.cc77
-rw-r--r--chrome/browser/ui/webui/predictors/predictors_handler.h12
-rw-r--r--chrome/chrome_browser.gypi16
-rw-r--r--chrome/chrome_tests_unit.gypi3
-rw-r--r--chrome/common/chrome_switches.cc14
-rw-r--r--chrome/common/chrome_switches.h4
37 files changed, 5737 insertions, 1 deletions
diff --git a/chrome/browser/net/resource_prefetch_predictor_observer.cc b/chrome/browser/net/resource_prefetch_predictor_observer.cc
new file mode 100644
index 0000000..5a3f4a1
--- /dev/null
+++ b/chrome/browser/net/resource_prefetch_predictor_observer.cc
@@ -0,0 +1,202 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/net/resource_prefetch_predictor_observer.h"
+
+#include <string>
+
+#include "base/metrics/histogram.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/browser/resource_request_info.h"
+#include "net/url_request/url_request.h"
+#include "url/gurl.h"
+
+using content::BrowserThread;
+using predictors::ResourcePrefetchPredictor;
+
+namespace {
+
+// Enum for measuring statistics pertaining to observed request, responses and
+// redirects.
+enum RequestStats {
+ REQUEST_STATS_TOTAL_RESPONSES = 0,
+ REQUEST_STATS_TOTAL_PROCESSED_RESPONSES = 1,
+ REQUEST_STATS_NO_RESOURCE_REQUEST_INFO = 2,
+ REQUEST_STATS_NO_RENDER_FRAME_ID_FROM_REQUEST_INFO = 3,
+ REQUEST_STATS_MAX = 4,
+};
+
+// Specific to main frame requests.
+enum MainFrameRequestStats {
+ MAIN_FRAME_REQUEST_STATS_TOTAL_REQUESTS = 0,
+ MAIN_FRAME_REQUEST_STATS_PROCESSED_REQUESTS = 1,
+ MAIN_FRAME_REQUEST_STATS_TOTAL_REDIRECTS = 2,
+ MAIN_FRAME_REQUEST_STATS_PROCESSED_REDIRECTS = 3,
+ MAIN_FRAME_REQUEST_STATS_TOTAL_RESPONSES = 4,
+ MAIN_FRAME_REQUEST_STATS_PROCESSED_RESPONSES = 5,
+ MAIN_FRAME_REQUEST_STATS_MAX = 6,
+};
+
+void ReportRequestStats(RequestStats stat) {
+ UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.RequestStats",
+ stat,
+ REQUEST_STATS_MAX);
+}
+
+void ReportMainFrameRequestStats(MainFrameRequestStats stat) {
+ UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.MainFrameRequestStats",
+ stat,
+ MAIN_FRAME_REQUEST_STATS_MAX);
+}
+
+bool SummarizeResponse(net::URLRequest* request,
+ ResourcePrefetchPredictor::URLRequestSummary* summary) {
+ const content::ResourceRequestInfo* info =
+ content::ResourceRequestInfo::ForRequest(request);
+ if (!info) {
+ ReportRequestStats(REQUEST_STATS_NO_RESOURCE_REQUEST_INFO);
+ return false;
+ }
+
+ int render_process_id, render_frame_id;
+ if (!info->GetAssociatedRenderFrame(&render_process_id, &render_frame_id)) {
+ ReportRequestStats(REQUEST_STATS_NO_RENDER_FRAME_ID_FROM_REQUEST_INFO);
+ return false;
+ }
+
+ summary->navigation_id.render_process_id = render_process_id;
+ summary->navigation_id.render_frame_id = render_frame_id;
+ summary->navigation_id.main_frame_url = request->first_party_for_cookies();
+ summary->navigation_id.creation_time = request->creation_time();
+ summary->resource_url = request->original_url();
+ summary->resource_type = info->GetResourceType();
+ request->GetMimeType(&summary->mime_type);
+ summary->was_cached = request->was_cached();
+
+ // Use the mime_type to determine the resource type for subresources since
+ // types such as PREFETCH, SUB_RESOURCE, etc are not useful.
+ if (summary->resource_type != content::RESOURCE_TYPE_MAIN_FRAME) {
+ summary->resource_type =
+ ResourcePrefetchPredictor::GetResourceTypeFromMimeType(
+ summary->mime_type,
+ summary->resource_type);
+ }
+ return true;
+}
+
+} // namespace
+
+namespace chrome_browser_net {
+
+ResourcePrefetchPredictorObserver::ResourcePrefetchPredictorObserver(
+ ResourcePrefetchPredictor* predictor)
+ : predictor_(predictor->AsWeakPtr()) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+}
+
+ResourcePrefetchPredictorObserver::~ResourcePrefetchPredictorObserver() {
+ CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI) ||
+ BrowserThread::CurrentlyOn(BrowserThread::IO));
+}
+
+void ResourcePrefetchPredictorObserver::OnRequestStarted(
+ net::URLRequest* request,
+ content::ResourceType resource_type,
+ int child_id,
+ int frame_id) {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+ if (resource_type == content::RESOURCE_TYPE_MAIN_FRAME)
+ ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_REQUESTS);
+
+ if (!ResourcePrefetchPredictor::ShouldRecordRequest(request, resource_type))
+ return;
+
+ ResourcePrefetchPredictor::URLRequestSummary summary;
+ summary.navigation_id.render_process_id = child_id;
+ summary.navigation_id.render_frame_id = frame_id;
+ summary.navigation_id.main_frame_url = request->first_party_for_cookies();
+ summary.resource_url = request->original_url();
+ summary.resource_type = resource_type;
+
+ BrowserThread::PostTask(
+ BrowserThread::UI,
+ FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictor::RecordURLRequest,
+ predictor_,
+ summary));
+
+ if (resource_type == content::RESOURCE_TYPE_MAIN_FRAME)
+ ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_REQUESTS);
+}
+
+void ResourcePrefetchPredictorObserver::OnRequestRedirected(
+ const GURL& redirect_url,
+ net::URLRequest* request) {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+ const content::ResourceRequestInfo* request_info =
+ content::ResourceRequestInfo::ForRequest(request);
+ if (request_info &&
+ request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) {
+ ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_REDIRECTS);
+ }
+
+ if (!ResourcePrefetchPredictor::ShouldRecordRedirect(request))
+ return;
+
+ ResourcePrefetchPredictor::URLRequestSummary summary;
+ if (!SummarizeResponse(request, &summary))
+ return;
+
+ summary.redirect_url = redirect_url;
+
+ BrowserThread::PostTask(
+ BrowserThread::UI,
+ FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictor::RecordURLRedirect,
+ predictor_,
+ summary));
+
+ if (request_info &&
+ request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) {
+ ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_REDIRECTS);
+ }
+}
+
+void ResourcePrefetchPredictorObserver::OnResponseStarted(
+ net::URLRequest* request) {
+ DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+ ReportRequestStats(REQUEST_STATS_TOTAL_RESPONSES);
+
+ const content::ResourceRequestInfo* request_info =
+ content::ResourceRequestInfo::ForRequest(request);
+ if (request_info &&
+ request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) {
+ ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_RESPONSES);
+ }
+
+ if (!ResourcePrefetchPredictor::ShouldRecordResponse(request))
+ return;
+ ResourcePrefetchPredictor::URLRequestSummary summary;
+ if (!SummarizeResponse(request, &summary))
+ return;
+
+ BrowserThread::PostTask(
+ BrowserThread::UI,
+ FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictor::RecordURLResponse,
+ predictor_,
+ summary));
+
+ ReportRequestStats(REQUEST_STATS_TOTAL_PROCESSED_RESPONSES);
+ if (request_info &&
+ request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) {
+ ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_RESPONSES);
+ }
+}
+
+} // namespace chrome_browser_net
diff --git a/chrome/browser/net/resource_prefetch_predictor_observer.h b/chrome/browser/net/resource_prefetch_predictor_observer.h
new file mode 100644
index 0000000..e2fd480
--- /dev/null
+++ b/chrome/browser/net/resource_prefetch_predictor_observer.h
@@ -0,0 +1,50 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_
+#define CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_
+
+#include "base/basictypes.h"
+#include "base/memory/weak_ptr.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "content/public/common/resource_type.h"
+
+namespace net {
+class URLRequest;
+}
+
+class GURL;
+
+namespace chrome_browser_net {
+
+// Observes resource requests in the ResourceDispatcherHostDelegate and notifies
+// the ResourcePrefetchPredictor about the ones it is interested in.
+// - Has an instance per profile, and is owned by the corresponding
+// ProfileIOData.
+// - Needs to be constructed on UI thread. Rest of the functions can only be
+// called on the IO thread. Can be destroyed on UI or IO thread.
+class ResourcePrefetchPredictorObserver {
+ public:
+ explicit ResourcePrefetchPredictorObserver(
+ predictors::ResourcePrefetchPredictor* predictor);
+ ~ResourcePrefetchPredictorObserver();
+
+ // Parts of the ResourceDispatcherHostDelegate that we want to observe.
+ void OnRequestStarted(net::URLRequest* request,
+ content::ResourceType resource_type,
+ int child_id,
+ int frame_id);
+ void OnRequestRedirected(const GURL& redirect_url, net::URLRequest* request);
+ void OnResponseStarted(net::URLRequest* request);
+
+ private:
+ // Owned by profile.
+ base::WeakPtr<predictors::ResourcePrefetchPredictor> predictor_;
+
+ DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorObserver);
+};
+
+} // namespace chrome_browser_net
+
+#endif // CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_
diff --git a/chrome/browser/predictors/predictor_database.cc b/chrome/browser/predictors/predictor_database.cc
index eb2bf5c..9f8460a 100644
--- a/chrome/browser/predictors/predictor_database.cc
+++ b/chrome/browser/predictors/predictor_database.cc
@@ -12,6 +12,8 @@
#include "base/strings/stringprintf.h"
#include "chrome/browser/predictors/autocomplete_action_predictor_table.h"
#include "chrome/browser/predictors/logged_in_predictor_table.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
#include "chrome/browser/prerender/prerender_field_trial.h"
#include "chrome/browser/profiles/profile.h"
#include "content/public/browser/browser_thread.h"
@@ -50,6 +52,7 @@ class PredictorDatabaseInternal
// Cancels pending DB transactions. Should only be called on the UI thread.
void SetCancelled();
+ bool is_resource_prefetch_predictor_enabled_;
base::FilePath db_path_;
scoped_ptr<sql::Connection> db_;
@@ -57,6 +60,7 @@ class PredictorDatabaseInternal
// to using a WeakPtr instead.
scoped_refptr<AutocompleteActionPredictorTable> autocomplete_table_;
scoped_refptr<LoggedInPredictorTable> logged_in_table_;
+ scoped_refptr<ResourcePrefetchPredictorTables> resource_prefetch_tables_;
DISALLOW_COPY_AND_ASSIGN(PredictorDatabaseInternal);
};
@@ -66,8 +70,12 @@ PredictorDatabaseInternal::PredictorDatabaseInternal(Profile* profile)
: db_path_(profile->GetPath().Append(kPredictorDatabaseName)),
db_(new sql::Connection()),
autocomplete_table_(new AutocompleteActionPredictorTable()),
- logged_in_table_(new LoggedInPredictorTable()) {
+ logged_in_table_(new LoggedInPredictorTable()),
+ resource_prefetch_tables_(new ResourcePrefetchPredictorTables()) {
db_->set_histogram_tag("Predictor");
+ ResourcePrefetchPredictorConfig config;
+ is_resource_prefetch_predictor_enabled_ =
+ IsSpeculativeResourcePrefetchingEnabled(profile, &config);
}
PredictorDatabaseInternal::~PredictorDatabaseInternal() {
@@ -89,6 +97,7 @@ void PredictorDatabaseInternal::Initialize() {
autocomplete_table_->Initialize(db_.get());
logged_in_table_->Initialize(db_.get());
+ resource_prefetch_tables_->Initialize(db_.get());
LogDatabaseStats();
}
@@ -99,6 +108,7 @@ void PredictorDatabaseInternal::SetCancelled() {
autocomplete_table_->SetCancelled();
logged_in_table_->SetCancelled();
+ resource_prefetch_tables_->SetCancelled();
}
void PredictorDatabaseInternal::LogDatabaseStats() {
@@ -113,6 +123,8 @@ void PredictorDatabaseInternal::LogDatabaseStats() {
autocomplete_table_->LogDatabaseStats();
logged_in_table_->LogDatabaseStats();
+ if (is_resource_prefetch_predictor_enabled_)
+ resource_prefetch_tables_->LogDatabaseStats();
}
PredictorDatabase::PredictorDatabase(Profile* profile)
@@ -138,6 +150,11 @@ scoped_refptr<LoggedInPredictorTable>
return db_->logged_in_table_;
}
+scoped_refptr<ResourcePrefetchPredictorTables>
+ PredictorDatabase::resource_prefetch_tables() {
+ return db_->resource_prefetch_tables_;
+}
+
sql::Connection* PredictorDatabase::GetDatabase() {
return db_->db_.get();
}
diff --git a/chrome/browser/predictors/predictor_database.h b/chrome/browser/predictors/predictor_database.h
index 624b00b..bc53a9a 100644
--- a/chrome/browser/predictors/predictor_database.h
+++ b/chrome/browser/predictors/predictor_database.h
@@ -19,6 +19,7 @@ namespace predictors {
class AutocompleteActionPredictorTable;
class LoggedInPredictorTable;
class PredictorDatabaseInternal;
+class ResourcePrefetchPredictorTables;
class PredictorDatabase : public KeyedService {
public:
@@ -26,6 +27,7 @@ class PredictorDatabase : public KeyedService {
virtual ~PredictorDatabase();
scoped_refptr<AutocompleteActionPredictorTable> autocomplete_table();
+ scoped_refptr<ResourcePrefetchPredictorTables> resource_prefetch_tables();
scoped_refptr<LoggedInPredictorTable> logged_in_table();
// Used for testing.
diff --git a/chrome/browser/predictors/resource_prefetch_common.cc b/chrome/browser/predictors/resource_prefetch_common.cc
new file mode 100644
index 0000000..731bda6
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_common.cc
@@ -0,0 +1,245 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+
+#include "base/command_line.h"
+#include "base/metrics/field_trial.h"
+#include "base/prefs/pref_service.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/common/chrome_switches.h"
+#include "chrome/common/pref_names.h"
+#include "content/public/browser/render_process_host.h"
+#include "content/public/browser/render_view_host.h"
+#include "content/public/browser/web_contents.h"
+
+namespace predictors {
+
+const char kSpeculativePrefetchingTrialName[] =
+ "SpeculativeResourcePrefetching";
+
+bool IsSpeculativeResourcePrefetchingEnabled(
+ Profile* profile,
+ ResourcePrefetchPredictorConfig* config) {
+ DCHECK(config);
+
+ // Off the record - disabled.
+ if (!profile || profile->IsOffTheRecord())
+ return false;
+
+ // If the user has explicitly disabled "predictive actions" - disabled.
+ if (!profile->GetPrefs() ||
+ !profile->GetPrefs()->GetBoolean(prefs::kNetworkPredictionEnabled)) {
+ return false;
+ }
+
+ // The config has the default params already set. The command line with just
+ // enable them with the default params.
+ if (CommandLine::ForCurrentProcess()->HasSwitch(
+ switches::kSpeculativeResourcePrefetching)) {
+ const std::string value =
+ CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
+ switches::kSpeculativeResourcePrefetching);
+
+ if (value == switches::kSpeculativeResourcePrefetchingDisabled) {
+ return false;
+ } else if (value == switches::kSpeculativeResourcePrefetchingLearning) {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ return true;
+ } else if (value == switches::kSpeculativeResourcePrefetchingEnabled) {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+ return true;
+ }
+ }
+
+ std::string trial = base::FieldTrialList::FindFullName(
+ kSpeculativePrefetchingTrialName);
+
+ if (trial == "LearningHost") {
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ return true;
+ } else if (trial == "LearningURL") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ return true;
+ } else if (trial == "Learning") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ return true;
+ } else if (trial == "PrefetchingHost") {
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+ return true;
+ } else if (trial == "PrefetchingURL") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ return true;
+ } else if (trial == "Prefetching") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+ return true;
+ } else if (trial == "PrefetchingLowConfidence") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+ config->min_url_visit_count = 1;
+ config->min_resource_confidence_to_trigger_prefetch = 0.5f;
+ config->min_resource_hits_to_trigger_prefetch = 1;
+ return true;
+ } else if (trial == "PrefetchingHighConfidence") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+ config->min_url_visit_count = 3;
+ config->min_resource_confidence_to_trigger_prefetch = 0.9f;
+ config->min_resource_hits_to_trigger_prefetch = 3;
+ return true;
+ } else if (trial == "PrefetchingMoreResources") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+ config->max_resources_per_entry = 100;
+ return true;
+ } else if (trial == "LearningSmallDB") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+
+ config->max_urls_to_track = 200;
+ config->max_hosts_to_track = 100;
+ return true;
+ } else if (trial == "PrefetchingSmallDB") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+ config->max_urls_to_track = 200;
+ config->max_hosts_to_track = 100;
+ return true;
+ } else if (trial == "PrefetchingSmallDBLowConfidence") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+ config->max_urls_to_track = 200;
+ config->max_hosts_to_track = 100;
+ config->min_url_visit_count = 1;
+ config->min_resource_confidence_to_trigger_prefetch = 0.5f;
+ config->min_resource_hits_to_trigger_prefetch = 1;
+ return true;
+ } else if (trial == "PrefetchingSmallDBHighConfidence") {
+ config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING;
+ config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING;
+
+ config->max_urls_to_track = 200;
+ config->max_hosts_to_track = 100;
+ config->min_url_visit_count = 3;
+ config->min_resource_confidence_to_trigger_prefetch = 0.9f;
+ config->min_resource_hits_to_trigger_prefetch = 3;
+ return true;
+ }
+
+ return false;
+}
+
+NavigationID::NavigationID()
+ : render_process_id(-1),
+ render_frame_id(-1) {
+}
+
+NavigationID::NavigationID(const NavigationID& other)
+ : render_process_id(other.render_process_id),
+ render_frame_id(other.render_frame_id),
+ main_frame_url(other.main_frame_url),
+ creation_time(other.creation_time) {
+}
+
+NavigationID::NavigationID(content::WebContents* web_contents)
+ : render_process_id(web_contents->GetRenderProcessHost()->GetID()),
+ render_frame_id(web_contents->GetRenderViewHost()->GetRoutingID()),
+ main_frame_url(web_contents->GetURL()) {
+}
+
+bool NavigationID::is_valid() const {
+ return render_process_id != -1 && render_frame_id != -1 &&
+ !main_frame_url.is_empty();
+}
+
+bool NavigationID::operator<(const NavigationID& rhs) const {
+ DCHECK(is_valid() && rhs.is_valid());
+ if (render_process_id != rhs.render_process_id)
+ return render_process_id < rhs.render_process_id;
+ else if (render_frame_id != rhs.render_frame_id)
+ return render_frame_id < rhs.render_frame_id;
+ else
+ return main_frame_url < rhs.main_frame_url;
+}
+
+bool NavigationID::operator==(const NavigationID& rhs) const {
+ DCHECK(is_valid() && rhs.is_valid());
+ return IsSameRenderer(rhs) && main_frame_url == rhs.main_frame_url;
+}
+
+bool NavigationID::IsSameRenderer(const NavigationID& other) const {
+ DCHECK(is_valid() && other.is_valid());
+ return render_process_id == other.render_process_id &&
+ render_frame_id == other.render_frame_id;
+}
+
+ResourcePrefetchPredictorConfig::ResourcePrefetchPredictorConfig()
+ : mode(0),
+ max_navigation_lifetime_seconds(60),
+ max_urls_to_track(500),
+ max_hosts_to_track(200),
+ min_url_visit_count(2),
+ max_resources_per_entry(50),
+ max_consecutive_misses(3),
+ min_resource_confidence_to_trigger_prefetch(0.8f),
+ min_resource_hits_to_trigger_prefetch(3),
+ max_prefetches_inflight_per_navigation(24),
+ max_prefetches_inflight_per_host_per_navigation(3) {
+}
+
+ResourcePrefetchPredictorConfig::~ResourcePrefetchPredictorConfig() {
+}
+
+bool ResourcePrefetchPredictorConfig::IsLearningEnabled() const {
+ return IsURLLearningEnabled() || IsHostLearningEnabled();
+}
+
+bool ResourcePrefetchPredictorConfig::IsPrefetchingEnabled() const {
+ return IsURLPrefetchingEnabled() || IsHostPrefetchingEnabled();
+}
+
+bool ResourcePrefetchPredictorConfig::IsURLLearningEnabled() const {
+ return (mode & URL_LEARNING) > 0;
+}
+
+bool ResourcePrefetchPredictorConfig::IsHostLearningEnabled() const {
+ return (mode & HOST_LEARNING) > 0;
+}
+
+bool ResourcePrefetchPredictorConfig::IsURLPrefetchingEnabled() const {
+ return (mode & URL_PREFETCHING) > 0;
+}
+
+bool ResourcePrefetchPredictorConfig::IsHostPrefetchingEnabled() const {
+ return (mode & HOST_PRFETCHING) > 0;
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetch_common.h b/chrome/browser/predictors/resource_prefetch_common.h
new file mode 100644
index 0000000..f94a50d
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_common.h
@@ -0,0 +1,116 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_
+
+#include "base/time/time.h"
+#include "url/gurl.h"
+
+class Profile;
+
+namespace content {
+class WebContents;
+}
+
+namespace predictors {
+
+struct ResourcePrefetchPredictorConfig;
+
+// Returns true if prefetching is enabled. And will initilize the |config|
+// fields to the appropritate values.
+bool IsSpeculativeResourcePrefetchingEnabled(
+ Profile* profile,
+ ResourcePrefetchPredictorConfig* config);
+
+// Represents the type of key based on which prefetch data is stored.
+enum PrefetchKeyType {
+ PREFETCH_KEY_TYPE_HOST,
+ PREFETCH_KEY_TYPE_URL
+};
+
+// Represents a single navigation for a render frame.
+struct NavigationID {
+ // TODO(shishir): Maybe take process_id, frame_id and url as input in
+ // constructor.
+ NavigationID();
+ NavigationID(const NavigationID& other);
+ explicit NavigationID(content::WebContents* web_contents);
+ bool operator<(const NavigationID& rhs) const;
+ bool operator==(const NavigationID& rhs) const;
+
+ bool IsSameRenderer(const NavigationID& other) const;
+
+ // Returns true iff the render_process_id_, render_frame_id_ and
+ // frame_url_ has been set correctly.
+ bool is_valid() const;
+
+ int render_process_id;
+ int render_frame_id;
+ GURL main_frame_url;
+
+ // NOTE: Even though we store the creation time here, it is not used during
+ // comparison of two NavigationIDs because it cannot always be determined
+ // correctly.
+ base::TimeTicks creation_time;
+};
+
+// Represents the config for the resource prefetch prediction algorithm. It is
+// useful for running experiments.
+struct ResourcePrefetchPredictorConfig {
+ // Initializes the config with default values.
+ ResourcePrefetchPredictorConfig();
+ ~ResourcePrefetchPredictorConfig();
+
+ // The mode the prefetcher is running in. Forms a bit map.
+ enum Mode {
+ URL_LEARNING = 1 << 0,
+ HOST_LEARNING = 1 << 1,
+ URL_PREFETCHING = 1 << 2, // Should also turn on URL_LEARNING.
+ HOST_PRFETCHING = 1 << 3 // Should also turn on HOST_LEARNING.
+ };
+ int mode;
+
+ // Helpers to deal with mode.
+ bool IsLearningEnabled() const;
+ bool IsPrefetchingEnabled() const;
+ bool IsURLLearningEnabled() const;
+ bool IsHostLearningEnabled() const;
+ bool IsURLPrefetchingEnabled() const;
+ bool IsHostPrefetchingEnabled() const;
+
+ // If a navigation hasn't seen a load complete event in this much time, it
+ // is considered abandoned.
+ size_t max_navigation_lifetime_seconds;
+
+ // Size of LRU caches for the URL and host data.
+ size_t max_urls_to_track;
+ size_t max_hosts_to_track;
+
+ // The number of times we should have seen a visit to this URL in history
+ // to start tracking it. This is to ensure we don't bother with oneoff
+ // entries. For hosts we track each one.
+ size_t min_url_visit_count;
+
+ // The maximum number of resources to store per entry.
+ size_t max_resources_per_entry;
+ // The number of consecutive misses after we stop tracking a resource URL.
+ size_t max_consecutive_misses;
+
+ // The minimum confidence (accuracy of hits) required for a resource to be
+ // prefetched.
+ float min_resource_confidence_to_trigger_prefetch;
+ // The minimum number of times we must have a URL on record to prefetch it.
+ size_t min_resource_hits_to_trigger_prefetch;
+
+ // Maximum number of prefetches that can be inflight for a single navigation.
+ size_t max_prefetches_inflight_per_navigation;
+ // Maximum number of prefetches that can be inflight for a host for a single
+ // navigation.
+ size_t max_prefetches_inflight_per_host_per_navigation;
+};
+
+} // namespace predictors
+
+#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_
diff --git a/chrome/browser/predictors/resource_prefetch_predictor.cc b/chrome/browser/predictors/resource_prefetch_predictor.cc
new file mode 100644
index 0000000..d0b02a2
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor.cc
@@ -0,0 +1,1222 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+
+#include <map>
+#include <set>
+#include <utility>
+
+#include "base/command_line.h"
+#include "base/metrics/histogram.h"
+#include "base/stl_util.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/stringprintf.h"
+#include "base/time/time.h"
+#include "chrome/browser/chrome_notification_types.h"
+#include "chrome/browser/history/history_database.h"
+#include "chrome/browser/history/history_db_task.h"
+#include "chrome/browser/history/history_notifications.h"
+#include "chrome/browser/history/history_service.h"
+#include "chrome/browser/history/history_service_factory.h"
+#include "chrome/browser/predictors/predictor_database.h"
+#include "chrome/browser/predictors/predictor_database_factory.h"
+#include "chrome/browser/predictors/resource_prefetcher_manager.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/common/chrome_switches.h"
+#include "chrome/common/url_constants.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/navigation_controller.h"
+#include "content/public/browser/notification_service.h"
+#include "content/public/browser/notification_source.h"
+#include "content/public/browser/notification_types.h"
+#include "content/public/browser/resource_request_info.h"
+#include "content/public/browser/web_contents.h"
+#include "net/base/mime_util.h"
+#include "net/http/http_response_headers.h"
+#include "net/url_request/url_request.h"
+#include "net/url_request/url_request_context_getter.h"
+
+using content::BrowserThread;
+
+namespace {
+
+// For reporting whether a subresource is handled or not, and for what reasons.
+enum ResourceStatus {
+ RESOURCE_STATUS_HANDLED = 0,
+ RESOURCE_STATUS_NOT_HTTP_PAGE = 1,
+ RESOURCE_STATUS_NOT_HTTP_RESOURCE = 2,
+ RESOURCE_STATUS_UNSUPPORTED_MIME_TYPE = 4,
+ RESOURCE_STATUS_NOT_GET = 8,
+ RESOURCE_STATUS_URL_TOO_LONG = 16,
+ RESOURCE_STATUS_NOT_CACHEABLE = 32,
+ RESOURCE_STATUS_HEADERS_MISSING = 64,
+ RESOURCE_STATUS_MAX = 128,
+};
+
+// For reporting various interesting events that occur during the loading of a
+// single main frame.
+enum NavigationEvent {
+ NAVIGATION_EVENT_REQUEST_STARTED = 0,
+ NAVIGATION_EVENT_REQUEST_REDIRECTED = 1,
+ NAVIGATION_EVENT_REQUEST_REDIRECTED_EMPTY_URL = 2,
+ NAVIGATION_EVENT_REQUEST_EXPIRED = 3,
+ NAVIGATION_EVENT_RESPONSE_STARTED = 4,
+ NAVIGATION_EVENT_ONLOAD = 5,
+ NAVIGATION_EVENT_ONLOAD_EMPTY_URL = 6,
+ NAVIGATION_EVENT_ONLOAD_UNTRACKED_URL = 7,
+ NAVIGATION_EVENT_ONLOAD_TRACKED_URL = 8,
+ NAVIGATION_EVENT_SHOULD_TRACK_URL = 9,
+ NAVIGATION_EVENT_SHOULD_NOT_TRACK_URL = 10,
+ NAVIGATION_EVENT_URL_TABLE_FULL = 11,
+ NAVIGATION_EVENT_HAVE_PREDICTIONS_FOR_URL = 12,
+ NAVIGATION_EVENT_NO_PREDICTIONS_FOR_URL = 13,
+ NAVIGATION_EVENT_MAIN_FRAME_URL_TOO_LONG = 14,
+ NAVIGATION_EVENT_HOST_TOO_LONG = 15,
+ NAVIGATION_EVENT_COUNT = 16,
+};
+
+// For reporting events of interest that are not tied to any navigation.
+enum ReportingEvent {
+ REPORTING_EVENT_ALL_HISTORY_CLEARED = 0,
+ REPORTING_EVENT_PARTIAL_HISTORY_CLEARED = 1,
+ REPORTING_EVENT_COUNT = 2
+};
+
+void RecordNavigationEvent(NavigationEvent event) {
+ UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.NavigationEvent",
+ event,
+ NAVIGATION_EVENT_COUNT);
+}
+
+} // namespace
+
+namespace predictors {
+
+////////////////////////////////////////////////////////////////////////////////
+// History lookup task.
+
+// Used to fetch the visit count for a URL from the History database.
+class GetUrlVisitCountTask : public history::HistoryDBTask {
+ public:
+ typedef ResourcePrefetchPredictor::URLRequestSummary URLRequestSummary;
+ typedef base::Callback<void(
+ size_t, // Visit count.
+ const NavigationID&,
+ const std::vector<URLRequestSummary>&)> VisitInfoCallback;
+
+ GetUrlVisitCountTask(
+ const NavigationID& navigation_id,
+ std::vector<URLRequestSummary>* requests,
+ VisitInfoCallback callback)
+ : visit_count_(0),
+ navigation_id_(navigation_id),
+ requests_(requests),
+ callback_(callback) {
+ DCHECK(requests_.get());
+ }
+
+ virtual bool RunOnDBThread(history::HistoryBackend* backend,
+ history::HistoryDatabase* db) OVERRIDE {
+ history::URLRow url_row;
+ if (db->GetRowForURL(navigation_id_.main_frame_url, &url_row))
+ visit_count_ = url_row.visit_count();
+ return true;
+ }
+
+ virtual void DoneRunOnMainThread() OVERRIDE {
+ callback_.Run(visit_count_, navigation_id_, *requests_);
+ }
+
+ private:
+ virtual ~GetUrlVisitCountTask() { }
+
+ int visit_count_;
+ NavigationID navigation_id_;
+ scoped_ptr<std::vector<URLRequestSummary> > requests_;
+ VisitInfoCallback callback_;
+
+ DISALLOW_COPY_AND_ASSIGN(GetUrlVisitCountTask);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// ResourcePrefetchPredictor static functions.
+
+// static
+bool ResourcePrefetchPredictor::ShouldRecordRequest(
+ net::URLRequest* request,
+ content::ResourceType resource_type) {
+ const content::ResourceRequestInfo* request_info =
+ content::ResourceRequestInfo::ForRequest(request);
+ if (!request_info)
+ return false;
+
+ if (!request_info->IsMainFrame())
+ return false;
+
+ return resource_type == content::RESOURCE_TYPE_MAIN_FRAME &&
+ IsHandledMainPage(request);
+}
+
+// static
+bool ResourcePrefetchPredictor::ShouldRecordResponse(
+ net::URLRequest* response) {
+ const content::ResourceRequestInfo* request_info =
+ content::ResourceRequestInfo::ForRequest(response);
+ if (!request_info)
+ return false;
+
+ if (!request_info->IsMainFrame())
+ return false;
+
+ return request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME ?
+ IsHandledMainPage(response) : IsHandledSubresource(response);
+}
+
+// static
+bool ResourcePrefetchPredictor::ShouldRecordRedirect(
+ net::URLRequest* response) {
+ const content::ResourceRequestInfo* request_info =
+ content::ResourceRequestInfo::ForRequest(response);
+ if (!request_info)
+ return false;
+
+ if (!request_info->IsMainFrame())
+ return false;
+
+ return request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME &&
+ IsHandledMainPage(response);
+}
+
+// static
+bool ResourcePrefetchPredictor::IsHandledMainPage(net::URLRequest* request) {
+ return request->original_url().scheme() == url::kHttpScheme;
+}
+
+// static
+bool ResourcePrefetchPredictor::IsHandledSubresource(
+ net::URLRequest* response) {
+ int resource_status = 0;
+ if (response->first_party_for_cookies().scheme() != url::kHttpScheme)
+ resource_status |= RESOURCE_STATUS_NOT_HTTP_PAGE;
+
+ if (response->original_url().scheme() != url::kHttpScheme)
+ resource_status |= RESOURCE_STATUS_NOT_HTTP_RESOURCE;
+
+ std::string mime_type;
+ response->GetMimeType(&mime_type);
+ if (!mime_type.empty() &&
+ !net::IsSupportedImageMimeType(mime_type.c_str()) &&
+ !net::IsSupportedJavascriptMimeType(mime_type.c_str()) &&
+ !net::MatchesMimeType("text/css", mime_type)) {
+ resource_status |= RESOURCE_STATUS_UNSUPPORTED_MIME_TYPE;
+ }
+
+ if (response->method() != "GET")
+ resource_status |= RESOURCE_STATUS_NOT_GET;
+
+ if (response->original_url().spec().length() >
+ ResourcePrefetchPredictorTables::kMaxStringLength) {
+ resource_status |= RESOURCE_STATUS_URL_TOO_LONG;
+ }
+
+ if (!response->response_info().headers.get())
+ resource_status |= RESOURCE_STATUS_HEADERS_MISSING;
+
+ if (!IsCacheable(response))
+ resource_status |= RESOURCE_STATUS_NOT_CACHEABLE;
+
+ UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.ResourceStatus",
+ resource_status,
+ RESOURCE_STATUS_MAX);
+
+ return resource_status == 0;
+}
+
+// static
+bool ResourcePrefetchPredictor::IsCacheable(const net::URLRequest* response) {
+ if (response->was_cached())
+ return true;
+
+ // For non cached responses, we will ensure that the freshness lifetime is
+ // some sane value.
+ const net::HttpResponseInfo& response_info = response->response_info();
+ if (!response_info.headers.get())
+ return false;
+ base::Time response_time(response_info.response_time);
+ response_time += base::TimeDelta::FromSeconds(1);
+ base::TimeDelta freshness = response_info.headers->GetFreshnessLifetime(
+ response_time);
+ return freshness > base::TimeDelta();
+}
+
+// static
+content::ResourceType ResourcePrefetchPredictor::GetResourceTypeFromMimeType(
+ const std::string& mime_type,
+ content::ResourceType fallback) {
+ if (net::IsSupportedImageMimeType(mime_type.c_str()))
+ return content::RESOURCE_TYPE_IMAGE;
+ else if (net::IsSupportedJavascriptMimeType(mime_type.c_str()))
+ return content::RESOURCE_TYPE_SCRIPT;
+ else if (net::MatchesMimeType("text/css", mime_type))
+ return content::RESOURCE_TYPE_STYLESHEET;
+ else
+ return fallback;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// ResourcePrefetchPredictor structs.
+
+ResourcePrefetchPredictor::URLRequestSummary::URLRequestSummary()
+ : resource_type(content::RESOURCE_TYPE_LAST_TYPE),
+ was_cached(false) {
+}
+
+ResourcePrefetchPredictor::URLRequestSummary::URLRequestSummary(
+ const URLRequestSummary& other)
+ : navigation_id(other.navigation_id),
+ resource_url(other.resource_url),
+ resource_type(other.resource_type),
+ mime_type(other.mime_type),
+ was_cached(other.was_cached),
+ redirect_url(other.redirect_url) {
+}
+
+ResourcePrefetchPredictor::URLRequestSummary::~URLRequestSummary() {
+}
+
+ResourcePrefetchPredictor::Result::Result(
+ PrefetchKeyType i_key_type,
+ ResourcePrefetcher::RequestVector* i_requests)
+ : key_type(i_key_type),
+ requests(i_requests) {
+}
+
+ResourcePrefetchPredictor::Result::~Result() {
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// ResourcePrefetchPredictor.
+
+ResourcePrefetchPredictor::ResourcePrefetchPredictor(
+ const ResourcePrefetchPredictorConfig& config,
+ Profile* profile)
+ : profile_(profile),
+ config_(config),
+ initialization_state_(NOT_INITIALIZED),
+ tables_(PredictorDatabaseFactory::GetForProfile(
+ profile)->resource_prefetch_tables()),
+ results_map_deleter_(&results_map_) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ // Some form of learning has to be enabled.
+ DCHECK(config_.IsLearningEnabled());
+ if (config_.IsURLPrefetchingEnabled())
+ DCHECK(config_.IsURLLearningEnabled());
+ if (config_.IsHostPrefetchingEnabled())
+ DCHECK(config_.IsHostLearningEnabled());
+}
+
+ResourcePrefetchPredictor::~ResourcePrefetchPredictor() {
+}
+
+void ResourcePrefetchPredictor::RecordURLRequest(
+ const URLRequestSummary& request) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+ if (initialization_state_ != INITIALIZED)
+ return;
+
+ CHECK_EQ(request.resource_type, content::RESOURCE_TYPE_MAIN_FRAME);
+ OnMainFrameRequest(request);
+}
+
+void ResourcePrefetchPredictor::RecordURLResponse(
+ const URLRequestSummary& response) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+ if (initialization_state_ != INITIALIZED)
+ return;
+
+ if (response.resource_type == content::RESOURCE_TYPE_MAIN_FRAME)
+ OnMainFrameResponse(response);
+ else
+ OnSubresourceResponse(response);
+}
+
+void ResourcePrefetchPredictor::RecordURLRedirect(
+ const URLRequestSummary& response) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+ if (initialization_state_ != INITIALIZED)
+ return;
+
+ CHECK_EQ(response.resource_type, content::RESOURCE_TYPE_MAIN_FRAME);
+ OnMainFrameRedirect(response);
+}
+
+void ResourcePrefetchPredictor::RecordMainFrameLoadComplete(
+ const NavigationID& navigation_id) {
+ switch (initialization_state_) {
+ case NOT_INITIALIZED:
+ StartInitialization();
+ break;
+ case INITIALIZING:
+ break;
+ case INITIALIZED: {
+ RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD);
+ // WebContents can return an empty URL if the navigation entry
+ // corresponding to the navigation has not been created yet.
+ if (navigation_id.main_frame_url.is_empty())
+ RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD_EMPTY_URL);
+ else
+ OnNavigationComplete(navigation_id);
+ break;
+ }
+ default:
+ NOTREACHED() << "Unexpected initialization_state_: "
+ << initialization_state_;
+ }
+}
+
+void ResourcePrefetchPredictor::FinishedPrefetchForNavigation(
+ const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ ResourcePrefetcher::RequestVector* requests) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ Result* result = new Result(key_type, requests);
+ // Add the results to the results map.
+ if (!results_map_.insert(std::make_pair(navigation_id, result)).second) {
+ DLOG(FATAL) << "Returning results for existing navigation.";
+ delete result;
+ }
+}
+
+void ResourcePrefetchPredictor::Observe(
+ int type,
+ const content::NotificationSource& source,
+ const content::NotificationDetails& details) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ switch (type) {
+ case chrome::NOTIFICATION_HISTORY_LOADED: {
+ DCHECK_EQ(initialization_state_, INITIALIZING);
+ notification_registrar_.Remove(this,
+ chrome::NOTIFICATION_HISTORY_LOADED,
+ content::Source<Profile>(profile_));
+ OnHistoryAndCacheLoaded();
+ break;
+ }
+
+ case chrome::NOTIFICATION_HISTORY_URLS_DELETED: {
+ DCHECK_EQ(initialization_state_, INITIALIZED);
+ const content::Details<const history::URLsDeletedDetails>
+ urls_deleted_details =
+ content::Details<const history::URLsDeletedDetails>(details);
+ if (urls_deleted_details->all_history) {
+ DeleteAllUrls();
+ UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.ReportingEvent",
+ REPORTING_EVENT_ALL_HISTORY_CLEARED,
+ REPORTING_EVENT_COUNT);
+ } else {
+ DeleteUrls(urls_deleted_details->rows);
+ UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.ReportingEvent",
+ REPORTING_EVENT_PARTIAL_HISTORY_CLEARED,
+ REPORTING_EVENT_COUNT);
+ }
+ break;
+ }
+
+ default:
+ NOTREACHED() << "Unexpected notification observed.";
+ break;
+ }
+}
+
+void ResourcePrefetchPredictor::Shutdown() {
+ if (prefetch_manager_.get()) {
+ prefetch_manager_->ShutdownOnUIThread();
+ prefetch_manager_ = NULL;
+ }
+}
+
+void ResourcePrefetchPredictor::OnMainFrameRequest(
+ const URLRequestSummary& request) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+ DCHECK_EQ(INITIALIZED, initialization_state_);
+
+ RecordNavigationEvent(NAVIGATION_EVENT_REQUEST_STARTED);
+
+ StartPrefetching(request.navigation_id);
+
+ // Cleanup older navigations.
+ CleanupAbandonedNavigations(request.navigation_id);
+
+ // New empty navigation entry.
+ inflight_navigations_.insert(std::make_pair(
+ request.navigation_id,
+ make_linked_ptr(new std::vector<URLRequestSummary>())));
+}
+
+void ResourcePrefetchPredictor::OnMainFrameResponse(
+ const URLRequestSummary& response) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+ if (initialization_state_ != INITIALIZED)
+ return;
+
+ RecordNavigationEvent(NAVIGATION_EVENT_RESPONSE_STARTED);
+
+ StopPrefetching(response.navigation_id);
+}
+
+void ResourcePrefetchPredictor::OnMainFrameRedirect(
+ const URLRequestSummary& response) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ RecordNavigationEvent(NAVIGATION_EVENT_REQUEST_REDIRECTED);
+
+ // TODO(shishir): There are significant gains to be had here if we can use the
+ // start URL in a redirect chain as the key to start prefetching. We can save
+ // of redirect times considerably assuming that the redirect chains do not
+ // change.
+
+ // Stop any inflight prefetching. Remove the older navigation.
+ StopPrefetching(response.navigation_id);
+ inflight_navigations_.erase(response.navigation_id);
+
+ // A redirect will not lead to another OnMainFrameRequest call, so record the
+ // redirect url as a new navigation.
+
+ // The redirect url may be empty if the url was invalid.
+ if (response.redirect_url.is_empty()) {
+ RecordNavigationEvent(NAVIGATION_EVENT_REQUEST_REDIRECTED_EMPTY_URL);
+ return;
+ }
+
+ NavigationID navigation_id(response.navigation_id);
+ navigation_id.main_frame_url = response.redirect_url;
+ inflight_navigations_.insert(std::make_pair(
+ navigation_id,
+ make_linked_ptr(new std::vector<URLRequestSummary>())));
+}
+
+void ResourcePrefetchPredictor::OnSubresourceResponse(
+ const URLRequestSummary& response) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ NavigationMap::const_iterator nav_it =
+ inflight_navigations_.find(response.navigation_id);
+ if (nav_it == inflight_navigations_.end()) {
+ return;
+ }
+
+ nav_it->second->push_back(response);
+}
+
+void ResourcePrefetchPredictor::OnNavigationComplete(
+ const NavigationID& navigation_id) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ NavigationMap::iterator nav_it =
+ inflight_navigations_.find(navigation_id);
+ if (nav_it == inflight_navigations_.end()) {
+ RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD_UNTRACKED_URL);
+ return;
+ }
+ RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD_TRACKED_URL);
+
+ // Report any stats.
+ if (prefetch_manager_.get()) {
+ ResultsMap::iterator results_it = results_map_.find(navigation_id);
+ bool have_prefetch_results = results_it != results_map_.end();
+ UMA_HISTOGRAM_BOOLEAN("ResourcePrefetchPredictor.HavePrefetchResults",
+ have_prefetch_results);
+ if (have_prefetch_results) {
+ ReportAccuracyStats(results_it->second->key_type,
+ *(nav_it->second),
+ results_it->second->requests.get());
+ }
+ } else {
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests(
+ new ResourcePrefetcher::RequestVector);
+ PrefetchKeyType key_type;
+ if (GetPrefetchData(navigation_id, requests.get(), &key_type)) {
+ RecordNavigationEvent(NAVIGATION_EVENT_HAVE_PREDICTIONS_FOR_URL);
+ ReportPredictedAccuracyStats(key_type,
+ *(nav_it->second),
+ *requests);
+ } else {
+ RecordNavigationEvent(NAVIGATION_EVENT_NO_PREDICTIONS_FOR_URL);
+ }
+ }
+
+ // Remove the navigation from the inflight navigations.
+ std::vector<URLRequestSummary>* requests = (nav_it->second).release();
+ inflight_navigations_.erase(nav_it);
+
+ // Kick off history lookup to determine if we should record the URL.
+ HistoryService* history_service = HistoryServiceFactory::GetForProfile(
+ profile_, Profile::EXPLICIT_ACCESS);
+ DCHECK(history_service);
+ history_service->ScheduleDBTask(
+ scoped_ptr<history::HistoryDBTask>(
+ new GetUrlVisitCountTask(
+ navigation_id,
+ requests,
+ base::Bind(&ResourcePrefetchPredictor::OnVisitCountLookup,
+ AsWeakPtr()))),
+ &history_lookup_consumer_);
+}
+
+bool ResourcePrefetchPredictor::GetPrefetchData(
+ const NavigationID& navigation_id,
+ ResourcePrefetcher::RequestVector* prefetch_requests,
+ PrefetchKeyType* key_type) {
+ DCHECK(prefetch_requests);
+ DCHECK(key_type);
+
+ *key_type = PREFETCH_KEY_TYPE_URL;
+ const GURL& main_frame_url = navigation_id.main_frame_url;
+
+ bool use_url_data = config_.IsPrefetchingEnabled() ?
+ config_.IsURLPrefetchingEnabled() : config_.IsURLLearningEnabled();
+ if (use_url_data) {
+ PrefetchDataMap::const_iterator iterator =
+ url_table_cache_->find(main_frame_url.spec());
+ if (iterator != url_table_cache_->end())
+ PopulatePrefetcherRequest(iterator->second, prefetch_requests);
+ }
+ if (!prefetch_requests->empty())
+ return true;
+
+ bool use_host_data = config_.IsPrefetchingEnabled() ?
+ config_.IsHostPrefetchingEnabled() : config_.IsHostLearningEnabled();
+ if (use_host_data) {
+ PrefetchDataMap::const_iterator iterator =
+ host_table_cache_->find(main_frame_url.host());
+ if (iterator != host_table_cache_->end()) {
+ *key_type = PREFETCH_KEY_TYPE_HOST;
+ PopulatePrefetcherRequest(iterator->second, prefetch_requests);
+ }
+ }
+
+ return !prefetch_requests->empty();
+}
+
+void ResourcePrefetchPredictor::PopulatePrefetcherRequest(
+ const PrefetchData& data,
+ ResourcePrefetcher::RequestVector* requests) {
+ for (ResourceRows::const_iterator it = data.resources.begin();
+ it != data.resources.end(); ++it) {
+ float confidence = static_cast<float>(it->number_of_hits) /
+ (it->number_of_hits + it->number_of_misses);
+ if (confidence < config_.min_resource_confidence_to_trigger_prefetch ||
+ it->number_of_hits < config_.min_resource_hits_to_trigger_prefetch) {
+ continue;
+ }
+
+ ResourcePrefetcher::Request* req = new ResourcePrefetcher::Request(
+ it->resource_url);
+ requests->push_back(req);
+ }
+}
+
+void ResourcePrefetchPredictor::StartPrefetching(
+ const NavigationID& navigation_id) {
+ if (!prefetch_manager_.get()) // Prefetching not enabled.
+ return;
+
+ // Prefer URL based data first.
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests(
+ new ResourcePrefetcher::RequestVector);
+ PrefetchKeyType key_type;
+ if (!GetPrefetchData(navigation_id, requests.get(), &key_type)) {
+ // No prefetching data at host or URL level.
+ return;
+ }
+
+ BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
+ base::Bind(&ResourcePrefetcherManager::MaybeAddPrefetch,
+ prefetch_manager_,
+ navigation_id,
+ key_type,
+ base::Passed(&requests)));
+}
+
+void ResourcePrefetchPredictor::StopPrefetching(
+ const NavigationID& navigation_id) {
+ if (!prefetch_manager_.get()) // Not enabled.
+ return;
+
+ BrowserThread::PostTask(
+ BrowserThread::IO, FROM_HERE,
+ base::Bind(&ResourcePrefetcherManager::MaybeRemovePrefetch,
+ prefetch_manager_,
+ navigation_id));
+}
+
+void ResourcePrefetchPredictor::StartInitialization() {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ DCHECK_EQ(initialization_state_, NOT_INITIALIZED);
+ initialization_state_ = INITIALIZING;
+
+ // Create local caches using the database as loaded.
+ scoped_ptr<PrefetchDataMap> url_data_map(new PrefetchDataMap());
+ scoped_ptr<PrefetchDataMap> host_data_map(new PrefetchDataMap());
+ PrefetchDataMap* url_data_ptr = url_data_map.get();
+ PrefetchDataMap* host_data_ptr = host_data_map.get();
+
+ BrowserThread::PostTaskAndReply(
+ BrowserThread::DB, FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictorTables::GetAllData,
+ tables_, url_data_ptr, host_data_ptr),
+ base::Bind(&ResourcePrefetchPredictor::CreateCaches, AsWeakPtr(),
+ base::Passed(&url_data_map), base::Passed(&host_data_map)));
+}
+
+void ResourcePrefetchPredictor::CreateCaches(
+ scoped_ptr<PrefetchDataMap> url_data_map,
+ scoped_ptr<PrefetchDataMap> host_data_map) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ DCHECK_EQ(initialization_state_, INITIALIZING);
+ DCHECK(!url_table_cache_);
+ DCHECK(!host_table_cache_);
+ DCHECK(inflight_navigations_.empty());
+
+ url_table_cache_.reset(url_data_map.release());
+ host_table_cache_.reset(host_data_map.release());
+
+ UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.UrlTableMainFrameUrlCount",
+ url_table_cache_->size());
+ UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HostTableHostCount",
+ host_table_cache_->size());
+
+ // Add notifications for history loading if it is not ready.
+ HistoryService* history_service = HistoryServiceFactory::GetForProfile(
+ profile_, Profile::EXPLICIT_ACCESS);
+ if (!history_service) {
+ notification_registrar_.Add(this, chrome::NOTIFICATION_HISTORY_LOADED,
+ content::Source<Profile>(profile_));
+ } else {
+ OnHistoryAndCacheLoaded();
+ }
+}
+
+void ResourcePrefetchPredictor::OnHistoryAndCacheLoaded() {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+ DCHECK_EQ(initialization_state_, INITIALIZING);
+
+ notification_registrar_.Add(this,
+ chrome::NOTIFICATION_HISTORY_URLS_DELETED,
+ content::Source<Profile>(profile_));
+
+ // Initialize the prefetch manager only if prefetching is enabled.
+ if (config_.IsPrefetchingEnabled()) {
+ prefetch_manager_ = new ResourcePrefetcherManager(
+ this, config_, profile_->GetRequestContext());
+ }
+
+ initialization_state_ = INITIALIZED;
+}
+
+void ResourcePrefetchPredictor::CleanupAbandonedNavigations(
+ const NavigationID& navigation_id) {
+ static const base::TimeDelta max_navigation_age =
+ base::TimeDelta::FromSeconds(config_.max_navigation_lifetime_seconds);
+
+ base::TimeTicks time_now = base::TimeTicks::Now();
+ for (NavigationMap::iterator it = inflight_navigations_.begin();
+ it != inflight_navigations_.end();) {
+ if (it->first.IsSameRenderer(navigation_id) ||
+ (time_now - it->first.creation_time > max_navigation_age)) {
+ inflight_navigations_.erase(it++);
+ RecordNavigationEvent(NAVIGATION_EVENT_REQUEST_EXPIRED);
+ } else {
+ ++it;
+ }
+ }
+ for (ResultsMap::iterator it = results_map_.begin();
+ it != results_map_.end();) {
+ if (it->first.IsSameRenderer(navigation_id) ||
+ (time_now - it->first.creation_time > max_navigation_age)) {
+ delete it->second;
+ results_map_.erase(it++);
+ } else {
+ ++it;
+ }
+ }
+}
+
+void ResourcePrefetchPredictor::DeleteAllUrls() {
+ inflight_navigations_.clear();
+ url_table_cache_->clear();
+ host_table_cache_->clear();
+
+ BrowserThread::PostTask(BrowserThread::DB, FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictorTables::DeleteAllData, tables_));
+}
+
+void ResourcePrefetchPredictor::DeleteUrls(const history::URLRows& urls) {
+ // Check all the urls in the database and pick out the ones that are present
+ // in the cache.
+ std::vector<std::string> urls_to_delete, hosts_to_delete;
+
+ for (history::URLRows::const_iterator it = urls.begin(); it != urls.end();
+ ++it) {
+ const std::string url_spec = it->url().spec();
+ if (url_table_cache_->find(url_spec) != url_table_cache_->end()) {
+ urls_to_delete.push_back(url_spec);
+ url_table_cache_->erase(url_spec);
+ }
+
+ const std::string host = it->url().host();
+ if (host_table_cache_->find(host) != host_table_cache_->end()) {
+ hosts_to_delete.push_back(host);
+ host_table_cache_->erase(host);
+ }
+ }
+
+ if (!urls_to_delete.empty() || !hosts_to_delete.empty()) {
+ BrowserThread::PostTask(BrowserThread::DB, FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictorTables::DeleteData,
+ tables_,
+ urls_to_delete,
+ hosts_to_delete));
+ }
+}
+
+void ResourcePrefetchPredictor::RemoveOldestEntryInPrefetchDataMap(
+ PrefetchKeyType key_type,
+ PrefetchDataMap* data_map) {
+ if (data_map->empty())
+ return;
+
+ base::Time oldest_time;
+ std::string key_to_delete;
+ for (PrefetchDataMap::iterator it = data_map->begin();
+ it != data_map->end(); ++it) {
+ if (key_to_delete.empty() || it->second.last_visit < oldest_time) {
+ key_to_delete = it->first;
+ oldest_time = it->second.last_visit;
+ }
+ }
+
+ data_map->erase(key_to_delete);
+ BrowserThread::PostTask(BrowserThread::DB, FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictorTables::DeleteSingleDataPoint,
+ tables_,
+ key_to_delete,
+ key_type));
+}
+
+void ResourcePrefetchPredictor::OnVisitCountLookup(
+ size_t visit_count,
+ const NavigationID& navigation_id,
+ const std::vector<URLRequestSummary>& requests) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HistoryVisitCountForUrl",
+ visit_count);
+
+ // URL level data - merge only if we are already saving the data, or we it
+ // meets the cutoff requirement.
+ const std::string url_spec = navigation_id.main_frame_url.spec();
+ bool already_tracking = url_table_cache_->find(url_spec) !=
+ url_table_cache_->end();
+ bool should_track_url = already_tracking ||
+ (visit_count >= config_.min_url_visit_count);
+
+ if (should_track_url) {
+ RecordNavigationEvent(NAVIGATION_EVENT_SHOULD_TRACK_URL);
+
+ if (config_.IsURLLearningEnabled()) {
+ LearnNavigation(url_spec, PREFETCH_KEY_TYPE_URL, requests,
+ config_.max_urls_to_track, url_table_cache_.get());
+ }
+ } else {
+ RecordNavigationEvent(NAVIGATION_EVENT_SHOULD_NOT_TRACK_URL);
+ }
+
+ // Host level data - no cutoff, always learn the navigation if enabled.
+ if (config_.IsHostLearningEnabled()) {
+ LearnNavigation(navigation_id.main_frame_url.host(),
+ PREFETCH_KEY_TYPE_HOST,
+ requests,
+ config_.max_hosts_to_track,
+ host_table_cache_.get());
+ }
+
+ // Remove the navigation from the results map.
+ ResultsMap::iterator results_it = results_map_.find(navigation_id);
+ if (results_it != results_map_.end()) {
+ delete results_it->second;
+ results_map_.erase(results_it);
+ }
+}
+
+void ResourcePrefetchPredictor::LearnNavigation(
+ const std::string& key,
+ PrefetchKeyType key_type,
+ const std::vector<URLRequestSummary>& new_resources,
+ size_t max_data_map_size,
+ PrefetchDataMap* data_map) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ // If the primary key is too long reject it.
+ if (key.length() > ResourcePrefetchPredictorTables::kMaxStringLength) {
+ if (key_type == PREFETCH_KEY_TYPE_HOST)
+ RecordNavigationEvent(NAVIGATION_EVENT_HOST_TOO_LONG);
+ else
+ RecordNavigationEvent(NAVIGATION_EVENT_MAIN_FRAME_URL_TOO_LONG);
+ return;
+ }
+
+ PrefetchDataMap::iterator cache_entry = data_map->find(key);
+ if (cache_entry == data_map->end()) {
+ if (data_map->size() >= max_data_map_size) {
+ // The table is full, delete an entry.
+ RemoveOldestEntryInPrefetchDataMap(key_type, data_map);
+ }
+
+ cache_entry = data_map->insert(std::make_pair(
+ key, PrefetchData(key_type, key))).first;
+ cache_entry->second.last_visit = base::Time::Now();
+ size_t new_resources_size = new_resources.size();
+ std::set<GURL> resources_seen;
+ for (size_t i = 0; i < new_resources_size; ++i) {
+ if (resources_seen.find(new_resources[i].resource_url) !=
+ resources_seen.end()) {
+ continue;
+ }
+ ResourceRow row_to_add;
+ row_to_add.resource_url = new_resources[i].resource_url;
+ row_to_add.resource_type = new_resources[i].resource_type;
+ row_to_add.number_of_hits = 1;
+ row_to_add.average_position = i + 1;
+ cache_entry->second.resources.push_back(row_to_add);
+ resources_seen.insert(new_resources[i].resource_url);
+ }
+ } else {
+ ResourceRows& old_resources = cache_entry->second.resources;
+ cache_entry->second.last_visit = base::Time::Now();
+
+ // Build indices over the data.
+ std::map<GURL, int> new_index, old_index;
+ int new_resources_size = static_cast<int>(new_resources.size());
+ for (int i = 0; i < new_resources_size; ++i) {
+ const URLRequestSummary& summary = new_resources[i];
+ // Take the first occurence of every url.
+ if (new_index.find(summary.resource_url) == new_index.end())
+ new_index[summary.resource_url] = i;
+ }
+ int old_resources_size = static_cast<int>(old_resources.size());
+ for (int i = 0; i < old_resources_size; ++i) {
+ const ResourceRow& row = old_resources[i];
+ DCHECK(old_index.find(row.resource_url) == old_index.end());
+ old_index[row.resource_url] = i;
+ }
+
+ // Go through the old urls and update their hit/miss counts.
+ for (int i = 0; i < old_resources_size; ++i) {
+ ResourceRow& old_row = old_resources[i];
+ if (new_index.find(old_row.resource_url) == new_index.end()) {
+ ++old_row.number_of_misses;
+ ++old_row.consecutive_misses;
+ } else {
+ const URLRequestSummary& new_row =
+ new_resources[new_index[old_row.resource_url]];
+
+ // Update the resource type since it could have changed.
+ if (new_row.resource_type != content::RESOURCE_TYPE_LAST_TYPE)
+ old_row.resource_type = new_row.resource_type;
+
+ int position = new_index[old_row.resource_url] + 1;
+ int total = old_row.number_of_hits + old_row.number_of_misses;
+ old_row.average_position =
+ ((old_row.average_position * total) + position) / (total + 1);
+ ++old_row.number_of_hits;
+ old_row.consecutive_misses = 0;
+ }
+ }
+
+ // Add the new ones that we have not seen before.
+ for (int i = 0; i < new_resources_size; ++i) {
+ const URLRequestSummary& summary = new_resources[i];
+ if (old_index.find(summary.resource_url) != old_index.end())
+ continue;
+
+ // Only need to add new stuff.
+ ResourceRow row_to_add;
+ row_to_add.resource_url = summary.resource_url;
+ row_to_add.resource_type = summary.resource_type;
+ row_to_add.number_of_hits = 1;
+ row_to_add.average_position = i + 1;
+ old_resources.push_back(row_to_add);
+
+ // To ensure we dont add the same url twice.
+ old_index[summary.resource_url] = 0;
+ }
+ }
+
+ // Trim and sort the resources after the update.
+ ResourceRows& resources = cache_entry->second.resources;
+ for (ResourceRows::iterator it = resources.begin();
+ it != resources.end();) {
+ it->UpdateScore();
+ if (it->consecutive_misses >= config_.max_consecutive_misses)
+ it = resources.erase(it);
+ else
+ ++it;
+ }
+ std::sort(resources.begin(), resources.end(),
+ ResourcePrefetchPredictorTables::ResourceRowSorter());
+ if (resources.size() > config_.max_resources_per_entry)
+ resources.resize(config_.max_resources_per_entry);
+
+ // If the row has no resources, remove it from the cache and delete the
+ // entry in the database. Else update the database.
+ if (resources.empty()) {
+ data_map->erase(key);
+ BrowserThread::PostTask(
+ BrowserThread::DB, FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictorTables::DeleteSingleDataPoint,
+ tables_,
+ key,
+ key_type));
+ } else {
+ bool is_host = key_type == PREFETCH_KEY_TYPE_HOST;
+ PrefetchData empty_data(
+ !is_host ? PREFETCH_KEY_TYPE_HOST : PREFETCH_KEY_TYPE_URL,
+ std::string());
+ const PrefetchData& host_data = is_host ? cache_entry->second : empty_data;
+ const PrefetchData& url_data = is_host ? empty_data : cache_entry->second;
+ BrowserThread::PostTask(
+ BrowserThread::DB, FROM_HERE,
+ base::Bind(&ResourcePrefetchPredictorTables::UpdateData,
+ tables_,
+ url_data,
+ host_data));
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Accuracy measurement.
+
+void ResourcePrefetchPredictor::ReportAccuracyStats(
+ PrefetchKeyType key_type,
+ const std::vector<URLRequestSummary>& actual,
+ ResourcePrefetcher::RequestVector* prefetched) const {
+ // Annotate the results.
+ std::map<GURL, bool> actual_resources;
+ for (std::vector<URLRequestSummary>::const_iterator it = actual.begin();
+ it != actual.end(); ++it) {
+ actual_resources[it->resource_url] = it->was_cached;
+ }
+
+ int prefetch_cancelled = 0, prefetch_failed = 0, prefetch_not_started = 0;
+ // 'a_' -> actual, 'p_' -> predicted.
+ int p_cache_a_cache = 0, p_cache_a_network = 0, p_cache_a_notused = 0,
+ p_network_a_cache = 0, p_network_a_network = 0, p_network_a_notused = 0;
+
+ for (ResourcePrefetcher::RequestVector::iterator it = prefetched->begin();
+ it != prefetched->end(); ++it) {
+ ResourcePrefetcher::Request* req = *it;
+
+ // Set the usage states if the resource was actually used.
+ std::map<GURL, bool>::iterator actual_it = actual_resources.find(
+ req->resource_url);
+ if (actual_it != actual_resources.end()) {
+ if (actual_it->second) {
+ req->usage_status =
+ ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE;
+ } else {
+ req->usage_status =
+ ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK;
+ }
+ }
+
+ switch (req->prefetch_status) {
+ // TODO(shishir): Add histogram for each cancellation reason.
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_REDIRECTED:
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_AUTH_REQUIRED:
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_CERT_REQUIRED:
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_CERT_ERROR:
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_CANCELLED:
+ ++prefetch_cancelled;
+ break;
+
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_FAILED:
+ ++prefetch_failed;
+ break;
+
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_FROM_CACHE:
+ if (req->usage_status ==
+ ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE)
+ ++p_cache_a_cache;
+ else if (req->usage_status ==
+ ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK)
+ ++p_cache_a_network;
+ else
+ ++p_cache_a_notused;
+ break;
+
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_FROM_NETWORK:
+ if (req->usage_status ==
+ ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE)
+ ++p_network_a_cache;
+ else if (req->usage_status ==
+ ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK)
+ ++p_network_a_network;
+ else
+ ++p_network_a_notused;
+ break;
+
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_NOT_STARTED:
+ ++prefetch_not_started;
+ break;
+
+ case ResourcePrefetcher::Request::PREFETCH_STATUS_STARTED:
+ DLOG(FATAL) << "Invalid prefetch status";
+ break;
+ }
+ }
+
+ int total_prefetched = p_cache_a_cache + p_cache_a_network + p_cache_a_notused
+ + p_network_a_cache + p_network_a_network + p_network_a_notused;
+
+ std::string histogram_type = key_type == PREFETCH_KEY_TYPE_HOST ? "Host." :
+ "Url.";
+
+ // Macros to avoid using the STATIC_HISTOGRAM_POINTER_BLOCK in UMA_HISTOGRAM
+ // definitions.
+#define RPP_HISTOGRAM_PERCENTAGE(suffix, value) \
+ { \
+ std::string name = "ResourcePrefetchPredictor." + histogram_type + suffix; \
+ std::string g_name = "ResourcePrefetchPredictor." + std::string(suffix); \
+ base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( \
+ name, 1, 101, 102, base::Histogram::kUmaTargetedHistogramFlag); \
+ histogram->Add(value); \
+ UMA_HISTOGRAM_PERCENTAGE(g_name, value); \
+ }
+
+ RPP_HISTOGRAM_PERCENTAGE("PrefetchCancelled",
+ prefetch_cancelled * 100.0 / total_prefetched);
+ RPP_HISTOGRAM_PERCENTAGE("PrefetchFailed",
+ prefetch_failed * 100.0 / total_prefetched);
+ RPP_HISTOGRAM_PERCENTAGE("PrefetchFromCacheUsedFromCache",
+ p_cache_a_cache * 100.0 / total_prefetched);
+ RPP_HISTOGRAM_PERCENTAGE("PrefetchFromCacheUsedFromNetwork",
+ p_cache_a_network * 100.0 / total_prefetched);
+ RPP_HISTOGRAM_PERCENTAGE("PrefetchFromCacheNotUsed",
+ p_cache_a_notused * 100.0 / total_prefetched);
+ RPP_HISTOGRAM_PERCENTAGE("PrefetchFromNetworkUsedFromCache",
+ p_network_a_cache * 100.0 / total_prefetched);
+ RPP_HISTOGRAM_PERCENTAGE("PrefetchFromNetworkUsedFromNetwork",
+ p_network_a_network * 100.0 / total_prefetched);
+ RPP_HISTOGRAM_PERCENTAGE("PrefetchFromNetworkNotUsed",
+ p_network_a_notused * 100.0 / total_prefetched);
+
+ RPP_HISTOGRAM_PERCENTAGE(
+ "PrefetchNotStarted",
+ prefetch_not_started * 100.0 / (prefetch_not_started + total_prefetched));
+
+#undef RPP_HISTOGRAM_PERCENTAGE
+}
+
+void ResourcePrefetchPredictor::ReportPredictedAccuracyStats(
+ PrefetchKeyType key_type,
+ const std::vector<URLRequestSummary>& actual,
+ const ResourcePrefetcher::RequestVector& predicted) const {
+ std::map<GURL, bool> actual_resources;
+ int from_network = 0;
+ for (std::vector<URLRequestSummary>::const_iterator it = actual.begin();
+ it != actual.end(); ++it) {
+ actual_resources[it->resource_url] = it->was_cached;
+ if (!it->was_cached)
+ ++from_network;
+ }
+
+ // Measure the accuracy at 25, 50 predicted resources.
+ ReportPredictedAccuracyStatsHelper(key_type, predicted, actual_resources,
+ from_network, 25);
+ ReportPredictedAccuracyStatsHelper(key_type, predicted, actual_resources,
+ from_network, 50);
+}
+
+void ResourcePrefetchPredictor::ReportPredictedAccuracyStatsHelper(
+ PrefetchKeyType key_type,
+ const ResourcePrefetcher::RequestVector& predicted,
+ const std::map<GURL, bool>& actual,
+ size_t total_resources_fetched_from_network,
+ size_t max_assumed_prefetched) const {
+ int prefetch_cached = 0, prefetch_network = 0, prefetch_missed = 0;
+ int num_assumed_prefetched = std::min(predicted.size(),
+ max_assumed_prefetched);
+ if (num_assumed_prefetched == 0)
+ return;
+
+ for (int i = 0; i < num_assumed_prefetched; ++i) {
+ const ResourcePrefetcher::Request& row = *(predicted[i]);
+ std::map<GURL, bool>::const_iterator it = actual.find(row.resource_url);
+ if (it == actual.end()) {
+ ++prefetch_missed;
+ } else if (it->second) {
+ ++prefetch_cached;
+ } else {
+ ++prefetch_network;
+ }
+ }
+
+ std::string prefix = key_type == PREFETCH_KEY_TYPE_HOST ?
+ "ResourcePrefetchPredictor.Host.Predicted" :
+ "ResourcePrefetchPredictor.Url.Predicted";
+ std::string suffix = "_" + base::IntToString(max_assumed_prefetched);
+
+ // Macros to avoid using the STATIC_HISTOGRAM_POINTER_BLOCK in UMA_HISTOGRAM
+ // definitions.
+#define RPP_PREDICTED_HISTOGRAM_COUNTS(name, value) \
+ { \
+ std::string full_name = prefix + name + suffix; \
+ base::HistogramBase* histogram = base::Histogram::FactoryGet( \
+ full_name, 1, 1000000, 50, \
+ base::Histogram::kUmaTargetedHistogramFlag); \
+ histogram->Add(value); \
+ }
+
+#define RPP_PREDICTED_HISTOGRAM_PERCENTAGE(name, value) \
+ { \
+ std::string full_name = prefix + name + suffix; \
+ base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( \
+ full_name, 1, 101, 102, base::Histogram::kUmaTargetedHistogramFlag); \
+ histogram->Add(value); \
+ }
+
+ RPP_PREDICTED_HISTOGRAM_COUNTS("PrefetchCount", num_assumed_prefetched);
+ RPP_PREDICTED_HISTOGRAM_COUNTS("PrefetchMisses_Count", prefetch_missed);
+ RPP_PREDICTED_HISTOGRAM_COUNTS("PrefetchFromCache_Count", prefetch_cached);
+ RPP_PREDICTED_HISTOGRAM_COUNTS("PrefetchFromNetwork_Count", prefetch_network);
+
+ RPP_PREDICTED_HISTOGRAM_PERCENTAGE(
+ "PrefetchMisses_PercentOfTotalPrefetched",
+ prefetch_missed * 100.0 / num_assumed_prefetched);
+ RPP_PREDICTED_HISTOGRAM_PERCENTAGE(
+ "PrefetchFromCache_PercentOfTotalPrefetched",
+ prefetch_cached * 100.0 / num_assumed_prefetched);
+ RPP_PREDICTED_HISTOGRAM_PERCENTAGE(
+ "PrefetchFromNetwork_PercentOfTotalPrefetched",
+ prefetch_network * 100.0 / num_assumed_prefetched);
+
+ // Measure the ratio of total number of resources prefetched from network vs
+ // the total number of resources fetched by the page from the network.
+ if (total_resources_fetched_from_network > 0) {
+ RPP_PREDICTED_HISTOGRAM_PERCENTAGE(
+ "PrefetchFromNetworkPercentOfTotalFromNetwork",
+ prefetch_network * 100.0 / total_resources_fetched_from_network);
+ }
+
+#undef RPP_PREDICTED_HISTOGRAM_PERCENTAGE
+#undef RPP_PREDICTED_HISTOGRAM_COUNTS
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetch_predictor.h b/chrome/browser/predictors/resource_prefetch_predictor.h
new file mode 100644
index 0000000..5e6157b
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor.h
@@ -0,0 +1,314 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "base/gtest_prod_util.h"
+#include "base/memory/linked_ptr.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/task/cancelable_task_tracker.h"
+#include "base/time/time.h"
+#include "chrome/browser/history/history_types.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
+#include "chrome/browser/predictors/resource_prefetcher.h"
+#include "components/keyed_service/core/keyed_service.h"
+#include "content/public/browser/notification_observer.h"
+#include "content/public/browser/notification_registrar.h"
+#include "content/public/common/resource_type.h"
+#include "url/gurl.h"
+
+class PredictorsHandler;
+class Profile;
+
+namespace content {
+class WebContents;
+}
+
+namespace net {
+class URLRequest;
+}
+
+namespace predictors {
+
+class ResourcePrefetcherManager;
+
+// Contains logic for learning what can be prefetched and for kicking off
+// speculative prefetching.
+// - The class is a profile keyed service owned by the profile.
+// - All the non-static methods of this class need to be called on the UI
+// thread.
+//
+// The overall flow of the resource prefetching algorithm is as follows:
+//
+// * ResourcePrefetchPredictorObserver - Listens for URL requests, responses and
+// redirects on the IO thread (via ResourceDispatcherHostDelegate) and posts
+// tasks to the ResourcePrefetchPredictor on the UI thread. This is owned by
+// the ProfileIOData for the profile.
+// * ResourcePrefetchPredictorTables - Persists ResourcePrefetchPredictor data
+// to a sql database. Runs entirely on the DB thread. Owned by the
+// PredictorDatabase.
+// * ResourcePrefetchPredictor - Learns about resource requirements per URL in
+// the UI thread through the ResourcePrefetchPredictorObserver and persists
+// it to disk in the DB thread through the ResourcePrefetchPredictorTables. It
+// initiates resource prefetching using the ResourcePrefetcherManager. Owned
+// by profile.
+// * ResourcePrefetcherManager - Manages the ResourcePrefetchers that do the
+// prefetching on the IO thread. The manager is owned by the
+// ResourcePrefetchPredictor and interfaces between the predictor on the UI
+// thread and the prefetchers on the IO thread.
+// * ResourcePrefetcher - Lives entirely on the IO thread, owned by the
+// ResourcePrefetcherManager, and issues net::URLRequest to fetch resources.
+//
+// TODO(shishir): Do speculative prefetching for https resources and/or https
+// main frame urls.
+// TODO(zhenw): Currently only main frame requests/redirects/responses are
+// recorded. Consider recording sub-frame responses independently or together
+// with main frame.
+class ResourcePrefetchPredictor
+ : public KeyedService,
+ public content::NotificationObserver,
+ public base::SupportsWeakPtr<ResourcePrefetchPredictor> {
+ public:
+ // Stores the data that we need to get from the URLRequest.
+ struct URLRequestSummary {
+ URLRequestSummary();
+ URLRequestSummary(const URLRequestSummary& other);
+ ~URLRequestSummary();
+
+ NavigationID navigation_id;
+ GURL resource_url;
+ content::ResourceType resource_type;
+
+ // Only for responses.
+ std::string mime_type;
+ bool was_cached;
+ GURL redirect_url; // Empty unless request was redirected to a valid url.
+ };
+
+ ResourcePrefetchPredictor(const ResourcePrefetchPredictorConfig& config,
+ Profile* profile);
+ virtual ~ResourcePrefetchPredictor();
+
+ // Thread safe.
+ static bool ShouldRecordRequest(net::URLRequest* request,
+ content::ResourceType resource_type);
+ static bool ShouldRecordResponse(net::URLRequest* response);
+ static bool ShouldRecordRedirect(net::URLRequest* response);
+
+ // Determines the ResourceType from the mime type, defaulting to the
+ // |fallback| if the ResourceType could not be determined.
+ static content::ResourceType GetResourceTypeFromMimeType(
+ const std::string& mime_type,
+ content::ResourceType fallback);
+
+ // 'ResourcePrefetchPredictorObserver' calls the below functions to inform the
+ // predictor of main frame and resource requests. Should only be called if the
+ // corresponding Should* functions return true.
+ void RecordURLRequest(const URLRequestSummary& request);
+ void RecordURLResponse(const URLRequestSummary& response);
+ void RecordURLRedirect(const URLRequestSummary& response);
+
+ // Called when the main frame of a page completes loading.
+ void RecordMainFrameLoadComplete(const NavigationID& navigation_id);
+
+ // Called by ResourcePrefetcherManager to notify that prefetching has finished
+ // for a navigation. Should take ownership of |requests|.
+ virtual void FinishedPrefetchForNavigation(
+ const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ ResourcePrefetcher::RequestVector* requests);
+
+ private:
+ friend class ::PredictorsHandler;
+ friend class ResourcePrefetchPredictorTest;
+
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, DeleteUrls);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
+ LazilyInitializeEmpty);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
+ LazilyInitializeWithData);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
+ NavigationNotRecorded);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, NavigationUrlInDB);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, NavigationUrlNotInDB);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
+ NavigationUrlNotInDBAndDBFull);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, OnMainFrameRequest);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, OnMainFrameRedirect);
+ FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
+ OnSubresourceResponse);
+
+ enum InitializationState {
+ NOT_INITIALIZED = 0,
+ INITIALIZING = 1,
+ INITIALIZED = 2
+ };
+
+ // Stores prefetching results.
+ struct Result {
+ // Takes ownership of requests.
+ Result(PrefetchKeyType key_type,
+ ResourcePrefetcher::RequestVector* requests);
+ ~Result();
+
+ PrefetchKeyType key_type;
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(Result);
+ };
+
+ typedef ResourcePrefetchPredictorTables::ResourceRow ResourceRow;
+ typedef ResourcePrefetchPredictorTables::ResourceRows ResourceRows;
+ typedef ResourcePrefetchPredictorTables::PrefetchData PrefetchData;
+ typedef ResourcePrefetchPredictorTables::PrefetchDataMap PrefetchDataMap;
+ typedef std::map<NavigationID, linked_ptr<std::vector<URLRequestSummary> > >
+ NavigationMap;
+ typedef std::map<NavigationID, Result*> ResultsMap;
+
+ // Returns true if the main page request is supported for prediction.
+ static bool IsHandledMainPage(net::URLRequest* request);
+
+ // Returns true if the subresource request is supported for prediction.
+ static bool IsHandledSubresource(net::URLRequest* request);
+
+ // Returns true if the request (should have a response in it) is cacheable.
+ static bool IsCacheable(const net::URLRequest* request);
+
+ // content::NotificationObserver methods OVERRIDE.
+ virtual void Observe(int type,
+ const content::NotificationSource& source,
+ const content::NotificationDetails& details) OVERRIDE;
+
+ // KeyedService methods OVERRIDE.
+ virtual void Shutdown() OVERRIDE;
+
+ // Functions called on different network events pertaining to the loading of
+ // main frame resource or sub resources.
+ void OnMainFrameRequest(const URLRequestSummary& request);
+ void OnMainFrameResponse(const URLRequestSummary& response);
+ void OnMainFrameRedirect(const URLRequestSummary& response);
+ void OnSubresourceResponse(const URLRequestSummary& response);
+
+ // Called when onload completes for a navigation. We treat this point as the
+ // "completion" of the navigation. The resources requested by the page upto
+ // this point are the only ones considered for prefetching.
+ void OnNavigationComplete(const NavigationID& navigation_id);
+
+ // Returns true if there is PrefetchData that can be used for the
+ // navigation and fills in the |prefetch_data| to resources that need to be
+ // prefetched.
+ bool GetPrefetchData(const NavigationID& navigation_id,
+ ResourcePrefetcher::RequestVector* prefetch_requests,
+ PrefetchKeyType* key_type);
+
+ // Converts a PrefetchData into a ResourcePrefetcher::RequestVector.
+ void PopulatePrefetcherRequest(const PrefetchData& data,
+ ResourcePrefetcher::RequestVector* requests);
+
+ // Starts prefetching if it is enabled and prefetching data exists for the
+ // NavigationID either at the URL or at the host level.
+ void StartPrefetching(const NavigationID& navigation_id);
+
+ // Stops prefetching that may be in progress corresponding to |navigation_id|.
+ void StopPrefetching(const NavigationID& navigation_id);
+
+ // Starts initialization by posting a task to the DB thread to read the
+ // predictor database.
+ void StartInitialization();
+
+ // Callback for task to read predictor database. Takes ownership of
+ // |url_data_map| and |host_data_map|.
+ void CreateCaches(scoped_ptr<PrefetchDataMap> url_data_map,
+ scoped_ptr<PrefetchDataMap> host_data_map);
+
+ // Called during initialization when history is read and the predictor
+ // database has been read.
+ void OnHistoryAndCacheLoaded();
+
+ // Removes data for navigations where the onload never fired. Will cleanup
+ // inflight_navigations_ and results_map_.
+ void CleanupAbandonedNavigations(const NavigationID& navigation_id);
+
+ // Deletes all URLs from the predictor database, the caches and removes all
+ // inflight navigations.
+ void DeleteAllUrls();
+
+ // Deletes data for the input |urls| and their corresponding hosts from the
+ // predictor database and caches.
+ void DeleteUrls(const history::URLRows& urls);
+
+ // Callback for GetUrlVisitCountTask.
+ void OnVisitCountLookup(size_t visit_count,
+ const NavigationID& navigation_id,
+ const std::vector<URLRequestSummary>& requests);
+
+ // Removes the oldest entry in the input |data_map|, also deleting it from the
+ // predictor database.
+ void RemoveOldestEntryInPrefetchDataMap(PrefetchKeyType key_type,
+ PrefetchDataMap* data_map);
+
+ // Merges resources in |new_resources| into the |data_map| and correspondingly
+ // updates the predictor database.
+ void LearnNavigation(const std::string& key,
+ PrefetchKeyType key_type,
+ const std::vector<URLRequestSummary>& new_resources,
+ size_t max_data_map_size,
+ PrefetchDataMap* data_map);
+
+ // Reports accuracy by comparing prefetched resources with resources that are
+ // actually used by the page.
+ void ReportAccuracyStats(PrefetchKeyType key_type,
+ const std::vector<URLRequestSummary>& actual,
+ ResourcePrefetcher::RequestVector* prefetched) const;
+
+ // Reports predicted accuracy i.e. by comparing resources that are actually
+ // used by the page with those that may have been prefetched.
+ void ReportPredictedAccuracyStats(
+ PrefetchKeyType key_type,
+ const std::vector<URLRequestSummary>& actual,
+ const ResourcePrefetcher::RequestVector& predicted) const;
+ void ReportPredictedAccuracyStatsHelper(
+ PrefetchKeyType key_type,
+ const ResourcePrefetcher::RequestVector& predicted,
+ const std::map<GURL, bool>& actual,
+ size_t total_resources_fetched_from_network,
+ size_t max_assumed_prefetched) const;
+
+ // Used for testing to inject mock tables.
+ void set_mock_tables(scoped_refptr<ResourcePrefetchPredictorTables> tables) {
+ tables_ = tables;
+ }
+
+ Profile* const profile_;
+ ResourcePrefetchPredictorConfig const config_;
+ InitializationState initialization_state_;
+ scoped_refptr<ResourcePrefetchPredictorTables> tables_;
+ scoped_refptr<ResourcePrefetcherManager> prefetch_manager_;
+ content::NotificationRegistrar notification_registrar_;
+ base::CancelableTaskTracker history_lookup_consumer_;
+
+ // Map of all the navigations in flight to their resource requests.
+ NavigationMap inflight_navigations_;
+
+ // Copy of the data in the predictor tables.
+ scoped_ptr<PrefetchDataMap> url_table_cache_;
+ scoped_ptr<PrefetchDataMap> host_table_cache_;
+
+ ResultsMap results_map_;
+ STLValueDeleter<ResultsMap> results_map_deleter_;
+
+ DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictor);
+};
+
+} // namespace predictors
+
+#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_factory.cc b/chrome/browser/predictors/resource_prefetch_predictor_factory.cc
new file mode 100644
index 0000000..6bb4287
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor_factory.cc
@@ -0,0 +1,52 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
+
+#include "chrome/browser/history/history_service_factory.h"
+#include "chrome/browser/predictors/predictor_database_factory.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/profiles/profile.h"
+#include "components/keyed_service/content/browser_context_dependency_manager.h"
+#include "components/keyed_service/content/browser_context_keyed_service_factory.h"
+
+namespace predictors {
+
+// static
+ResourcePrefetchPredictor* ResourcePrefetchPredictorFactory::GetForProfile(
+ content::BrowserContext* context) {
+ return static_cast<ResourcePrefetchPredictor*>(
+ GetInstance()->GetServiceForBrowserContext(context, true));
+}
+
+// static
+ResourcePrefetchPredictorFactory*
+ResourcePrefetchPredictorFactory::GetInstance() {
+ return Singleton<ResourcePrefetchPredictorFactory>::get();
+}
+
+ResourcePrefetchPredictorFactory::ResourcePrefetchPredictorFactory()
+ : BrowserContextKeyedServiceFactory(
+ "ResourcePrefetchPredictor",
+ BrowserContextDependencyManager::GetInstance()) {
+ DependsOn(HistoryServiceFactory::GetInstance());
+ DependsOn(PredictorDatabaseFactory::GetInstance());
+}
+
+ResourcePrefetchPredictorFactory::~ResourcePrefetchPredictorFactory() {}
+
+KeyedService*
+ ResourcePrefetchPredictorFactory::BuildServiceInstanceFor(
+ content::BrowserContext* context) const {
+ Profile* profile = Profile::FromBrowserContext(context);
+
+ ResourcePrefetchPredictorConfig config;
+ if (!IsSpeculativeResourcePrefetchingEnabled(profile, &config))
+ return NULL;
+
+ return new ResourcePrefetchPredictor(config, profile);
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_factory.h b/chrome/browser/predictors/resource_prefetch_predictor_factory.h
new file mode 100644
index 0000000..7c6f5e8
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor_factory.h
@@ -0,0 +1,38 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_
+
+#include "base/basictypes.h"
+#include "base/memory/singleton.h"
+#include "components/keyed_service/content/browser_context_keyed_service_factory.h"
+
+namespace predictors {
+
+class ResourcePrefetchPredictor;
+
+class ResourcePrefetchPredictorFactory
+ : public BrowserContextKeyedServiceFactory {
+ public:
+ static ResourcePrefetchPredictor* GetForProfile(
+ content::BrowserContext* context);
+ static ResourcePrefetchPredictorFactory* GetInstance();
+
+ private:
+ friend struct DefaultSingletonTraits<ResourcePrefetchPredictorFactory>;
+
+ ResourcePrefetchPredictorFactory();
+ virtual ~ResourcePrefetchPredictorFactory();
+
+ // RefcountedBrowserContextKeyedServiceFactory:
+ virtual KeyedService* BuildServiceInstanceFor(
+ content::BrowserContext* context) const OVERRIDE;
+
+ DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorFactory);
+};
+
+} // namespace predictors
+
+#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc b/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc
new file mode 100644
index 0000000..ec827f1
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc
@@ -0,0 +1,62 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h"
+
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
+#include "chrome/browser/profiles/profile.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/load_from_memory_cache_details.h"
+
+DEFINE_WEB_CONTENTS_USER_DATA_KEY(
+ predictors::ResourcePrefetchPredictorTabHelper);
+
+using content::BrowserThread;
+
+namespace predictors {
+
+ResourcePrefetchPredictorTabHelper::ResourcePrefetchPredictorTabHelper(
+ content::WebContents* web_contents)
+ : content::WebContentsObserver(web_contents) {
+}
+
+ResourcePrefetchPredictorTabHelper::~ResourcePrefetchPredictorTabHelper() {
+}
+
+void ResourcePrefetchPredictorTabHelper::DocumentOnLoadCompletedInMainFrame() {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ ResourcePrefetchPredictor* predictor =
+ ResourcePrefetchPredictorFactory::GetForProfile(
+ web_contents()->GetBrowserContext());
+ if (!predictor)
+ return;
+
+ NavigationID navigation_id(web_contents());
+ predictor->RecordMainFrameLoadComplete(navigation_id);
+}
+
+void ResourcePrefetchPredictorTabHelper::DidLoadResourceFromMemoryCache(
+ const content::LoadFromMemoryCacheDetails& details) {
+ DCHECK_CURRENTLY_ON(BrowserThread::UI);
+
+ ResourcePrefetchPredictor* predictor =
+ ResourcePrefetchPredictorFactory::GetForProfile(
+ web_contents()->GetBrowserContext());
+ if (!predictor)
+ return;
+
+ ResourcePrefetchPredictor::URLRequestSummary summary;
+ summary.navigation_id = NavigationID(web_contents());
+ summary.resource_url = details.url;
+ summary.mime_type = details.mime_type;
+ summary.resource_type =
+ ResourcePrefetchPredictor::GetResourceTypeFromMimeType(
+ details.mime_type, details.resource_type);
+ summary.was_cached = true;
+ predictor->RecordURLResponse(summary);
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h b/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h
new file mode 100644
index 0000000..4580bc8
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h
@@ -0,0 +1,34 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_
+
+#include "content/public/browser/web_contents_observer.h"
+#include "content/public/browser/web_contents_user_data.h"
+
+namespace predictors {
+
+class ResourcePrefetchPredictorTabHelper
+ : public content::WebContentsObserver,
+ public content::WebContentsUserData<ResourcePrefetchPredictorTabHelper> {
+ public:
+ virtual ~ResourcePrefetchPredictorTabHelper();
+
+ // content::WebContentsObserver implementation
+ virtual void DocumentOnLoadCompletedInMainFrame() OVERRIDE;
+ virtual void DidLoadResourceFromMemoryCache(
+ const content::LoadFromMemoryCacheDetails& details) OVERRIDE;
+
+ private:
+ explicit ResourcePrefetchPredictorTabHelper(
+ content::WebContents* web_contents);
+ friend class content::WebContentsUserData<ResourcePrefetchPredictorTabHelper>;
+
+ DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorTabHelper);
+};
+
+} // namespace predictors
+
+#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tables.cc b/chrome/browser/predictors/resource_prefetch_predictor_tables.cc
new file mode 100644
index 0000000..89ebd7d
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor_tables.cc
@@ -0,0 +1,513 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "base/logging.h"
+#include "base/metrics/histogram.h"
+#include "base/strings/stringprintf.h"
+#include "content/public/browser/browser_thread.h"
+#include "sql/statement.h"
+
+using content::BrowserThread;
+using sql::Statement;
+
+namespace {
+
+const char kUrlResourceTableName[] = "resource_prefetch_predictor_url";
+const char kUrlMetadataTableName[] = "resource_prefetch_predictor_url_metadata";
+const char kHostResourceTableName[] = "resource_prefetch_predictor_host";
+const char kHostMetadataTableName[] =
+ "resource_prefetch_predictor_host_metadata";
+
+void BindResourceRowToStatement(
+ const predictors::ResourcePrefetchPredictorTables::ResourceRow& row,
+ const std::string& primary_key,
+ Statement* statement) {
+ statement->BindString(0, primary_key);
+ statement->BindString(1, row.resource_url.spec());
+ statement->BindInt(2, static_cast<int>(row.resource_type));
+ statement->BindInt(3, row.number_of_hits);
+ statement->BindInt(4, row.number_of_misses);
+ statement->BindInt(5, row.consecutive_misses);
+ statement->BindDouble(6, row.average_position);
+}
+
+bool StepAndInitializeResourceRow(
+ Statement* statement,
+ predictors::ResourcePrefetchPredictorTables::ResourceRow* row) {
+ if (!statement->Step())
+ return false;
+
+ row->primary_key = statement->ColumnString(0);
+ row->resource_url = GURL(statement->ColumnString(1));
+ row->resource_type = static_cast<content::ResourceType>(
+ statement->ColumnInt(2));
+ row->number_of_hits = statement->ColumnInt(3);
+ row->number_of_misses = statement->ColumnInt(4);
+ row->consecutive_misses = statement->ColumnInt(5);
+ row->average_position = statement->ColumnDouble(6);
+ return true;
+}
+
+} // namespace
+
+namespace predictors {
+
+// static
+const size_t ResourcePrefetchPredictorTables::kMaxStringLength = 1024;
+
+ResourcePrefetchPredictorTables::ResourceRow::ResourceRow()
+ : resource_type(content::RESOURCE_TYPE_LAST_TYPE),
+ number_of_hits(0),
+ number_of_misses(0),
+ consecutive_misses(0),
+ average_position(0.0),
+ score(0.0) {
+}
+
+ResourcePrefetchPredictorTables::ResourceRow::ResourceRow(
+ const ResourceRow& other)
+ : primary_key(other.primary_key),
+ resource_url(other.resource_url),
+ resource_type(other.resource_type),
+ number_of_hits(other.number_of_hits),
+ number_of_misses(other.number_of_misses),
+ consecutive_misses(other.consecutive_misses),
+ average_position(other.average_position),
+ score(other.score) {
+}
+
+ResourcePrefetchPredictorTables::ResourceRow::ResourceRow(
+ const std::string& i_primary_key,
+ const std::string& i_resource_url,
+ content::ResourceType i_resource_type,
+ int i_number_of_hits,
+ int i_number_of_misses,
+ int i_consecutive_misses,
+ double i_average_position)
+ : primary_key(i_primary_key),
+ resource_url(i_resource_url),
+ resource_type(i_resource_type),
+ number_of_hits(i_number_of_hits),
+ number_of_misses(i_number_of_misses),
+ consecutive_misses(i_consecutive_misses),
+ average_position(i_average_position) {
+ UpdateScore();
+}
+
+void ResourcePrefetchPredictorTables::ResourceRow::UpdateScore() {
+ // The score is calculated so that when the rows are sorted, the stylesheets
+ // and scripts appear first, sorted by position(ascending) and then the rest
+ // of the resources sorted by position(ascending).
+ static const int kMaxResourcesPerType = 100;
+ switch (resource_type) {
+ case content::RESOURCE_TYPE_STYLESHEET:
+ case content::RESOURCE_TYPE_SCRIPT:
+ score = (2 * kMaxResourcesPerType) - average_position;
+ break;
+
+ case content::RESOURCE_TYPE_IMAGE:
+ default:
+ score = kMaxResourcesPerType - average_position;
+ break;
+ }
+}
+
+bool ResourcePrefetchPredictorTables::ResourceRow::operator==(
+ const ResourceRow& rhs) const {
+ return primary_key == rhs.primary_key &&
+ resource_url == rhs.resource_url &&
+ resource_type == rhs.resource_type &&
+ number_of_hits == rhs.number_of_hits &&
+ number_of_misses == rhs.number_of_misses &&
+ consecutive_misses == rhs.consecutive_misses &&
+ average_position == rhs.average_position &&
+ score == rhs.score;
+}
+
+bool ResourcePrefetchPredictorTables::ResourceRowSorter::operator()(
+ const ResourceRow& x, const ResourceRow& y) const {
+ return x.score > y.score;
+}
+
+ResourcePrefetchPredictorTables::PrefetchData::PrefetchData(
+ PrefetchKeyType i_key_type,
+ const std::string& i_primary_key)
+ : key_type(i_key_type),
+ primary_key(i_primary_key) {
+}
+
+ResourcePrefetchPredictorTables::PrefetchData::PrefetchData(
+ const PrefetchData& other)
+ : key_type(other.key_type),
+ primary_key(other.primary_key),
+ last_visit(other.last_visit),
+ resources(other.resources) {
+}
+
+ResourcePrefetchPredictorTables::PrefetchData::~PrefetchData() {
+}
+
+bool ResourcePrefetchPredictorTables::PrefetchData::operator==(
+ const PrefetchData& rhs) const {
+ return key_type == rhs.key_type && primary_key == rhs.primary_key &&
+ resources == rhs.resources;
+}
+
+void ResourcePrefetchPredictorTables::GetAllData(
+ PrefetchDataMap* url_data_map,
+ PrefetchDataMap* host_data_map) {
+ DCHECK_CURRENTLY_ON(BrowserThread::DB);
+ if (CantAccessDatabase())
+ return;
+
+ DCHECK(url_data_map);
+ DCHECK(host_data_map);
+ url_data_map->clear();
+ host_data_map->clear();
+
+ std::vector<std::string> urls_to_delete, hosts_to_delete;
+ GetAllDataHelper(PREFETCH_KEY_TYPE_URL, url_data_map, &urls_to_delete);
+ GetAllDataHelper(PREFETCH_KEY_TYPE_HOST, host_data_map, &hosts_to_delete);
+
+ if (!urls_to_delete.empty() || !hosts_to_delete.empty())
+ DeleteData(urls_to_delete, hosts_to_delete);
+}
+
+void ResourcePrefetchPredictorTables::UpdateData(
+ const PrefetchData& url_data,
+ const PrefetchData& host_data) {
+ DCHECK_CURRENTLY_ON(BrowserThread::DB);
+ if (CantAccessDatabase())
+ return;
+
+ DCHECK(!url_data.is_host() && host_data.is_host());
+ DCHECK(!url_data.primary_key.empty() || !host_data.primary_key.empty());
+
+ DB()->BeginTransaction();
+
+ bool success = (url_data.primary_key.empty() || UpdateDataHelper(url_data)) &&
+ (host_data.primary_key.empty() || UpdateDataHelper(host_data));
+ if (!success)
+ DB()->RollbackTransaction();
+
+ DB()->CommitTransaction();
+}
+
+void ResourcePrefetchPredictorTables::DeleteData(
+ const std::vector<std::string>& urls,
+ const std::vector<std::string>& hosts) {
+ DCHECK_CURRENTLY_ON(BrowserThread::DB);
+ if (CantAccessDatabase())
+ return;
+
+ DCHECK(!urls.empty() || !hosts.empty());
+
+ if (!urls.empty())
+ DeleteDataHelper(PREFETCH_KEY_TYPE_URL, urls);
+ if (!hosts.empty())
+ DeleteDataHelper(PREFETCH_KEY_TYPE_HOST, hosts);
+}
+
+void ResourcePrefetchPredictorTables::DeleteSingleDataPoint(
+ const std::string& key,
+ PrefetchKeyType key_type) {
+ DCHECK_CURRENTLY_ON(BrowserThread::DB);
+ if (CantAccessDatabase())
+ return;
+
+ DeleteDataHelper(key_type, std::vector<std::string>(1, key));
+}
+
+void ResourcePrefetchPredictorTables::DeleteAllData() {
+ if (CantAccessDatabase())
+ return;
+
+ Statement deleter(DB()->GetUniqueStatement(
+ base::StringPrintf("DELETE FROM %s", kUrlResourceTableName).c_str()));
+ deleter.Run();
+ deleter.Assign(DB()->GetUniqueStatement(
+ base::StringPrintf("DELETE FROM %s", kUrlMetadataTableName).c_str()));
+ deleter.Run();
+ deleter.Assign(DB()->GetUniqueStatement(
+ base::StringPrintf("DELETE FROM %s", kHostResourceTableName).c_str()));
+ deleter.Run();
+ deleter.Assign(DB()->GetUniqueStatement(
+ base::StringPrintf("DELETE FROM %s", kHostMetadataTableName).c_str()));
+ deleter.Run();
+}
+
+ResourcePrefetchPredictorTables::ResourcePrefetchPredictorTables()
+ : PredictorTableBase() {
+}
+
+ResourcePrefetchPredictorTables::~ResourcePrefetchPredictorTables() {
+}
+
+void ResourcePrefetchPredictorTables::GetAllDataHelper(
+ PrefetchKeyType key_type,
+ PrefetchDataMap* data_map,
+ std::vector<std::string>* to_delete) {
+ bool is_host = key_type == PREFETCH_KEY_TYPE_HOST;
+
+ // Read the resources table and organize it per primary key.
+ const char* resource_table_name = is_host ? kHostResourceTableName :
+ kUrlResourceTableName;
+ Statement resource_reader(DB()->GetUniqueStatement(
+ base::StringPrintf("SELECT * FROM %s", resource_table_name).c_str()));
+
+ ResourceRow row;
+ while (StepAndInitializeResourceRow(&resource_reader, &row)) {
+ row.UpdateScore();
+ std::string primary_key = row.primary_key;
+ // Don't need to store primary key since the data is grouped by primary key.
+ row.primary_key.clear();
+
+ PrefetchDataMap::iterator it = data_map->find(primary_key);
+ if (it == data_map->end()) {
+ it = data_map->insert(std::make_pair(
+ primary_key, PrefetchData(key_type, primary_key))).first;
+ }
+ it->second.resources.push_back(row);
+ }
+
+ // Sort each of the resource row vectors by score.
+ for (PrefetchDataMap::iterator it = data_map->begin(); it != data_map->end();
+ ++it) {
+ std::sort(it->second.resources.begin(),
+ it->second.resources.end(),
+ ResourceRowSorter());
+ }
+
+ // Read the metadata and keep track of entries that have metadata, but no
+ // resource entries, so they can be deleted.
+ const char* metadata_table_name = is_host ? kHostMetadataTableName :
+ kUrlMetadataTableName;
+ Statement metadata_reader(DB()->GetUniqueStatement(
+ base::StringPrintf("SELECT * FROM %s", metadata_table_name).c_str()));
+
+ while (metadata_reader.Step()) {
+ std::string primary_key = metadata_reader.ColumnString(0);
+
+ PrefetchDataMap::iterator it = data_map->find(primary_key);
+ if (it != data_map->end()) {
+ int64 last_visit = metadata_reader.ColumnInt64(1);
+ it->second.last_visit = base::Time::FromInternalValue(last_visit);
+ } else {
+ to_delete->push_back(primary_key);
+ }
+ }
+}
+
+bool ResourcePrefetchPredictorTables::UpdateDataHelper(
+ const PrefetchData& data) {
+ DCHECK(!data.primary_key.empty());
+
+ if (!StringsAreSmallerThanDBLimit(data)) {
+ UMA_HISTOGRAM_BOOLEAN("ResourcePrefetchPredictor.DbStringTooLong", true);
+ return false;
+ }
+
+ // Delete the older data from both the tables.
+ scoped_ptr<Statement> deleter(data.is_host() ?
+ GetHostResourceDeleteStatement() : GetUrlResourceDeleteStatement());
+ deleter->BindString(0, data.primary_key);
+ if (!deleter->Run())
+ return false;
+
+ deleter.reset(data.is_host() ? GetHostMetadataDeleteStatement() :
+ GetUrlMetadataDeleteStatement());
+ deleter->BindString(0, data.primary_key);
+ if (!deleter->Run())
+ return false;
+
+ // Add the new data to the tables.
+ const ResourceRows& resources = data.resources;
+ for (ResourceRows::const_iterator it = resources.begin();
+ it != resources.end(); ++it) {
+ scoped_ptr<Statement> resource_inserter(data.is_host() ?
+ GetHostResourceUpdateStatement() : GetUrlResourceUpdateStatement());
+ BindResourceRowToStatement(*it, data.primary_key, resource_inserter.get());
+ if (!resource_inserter->Run())
+ return false;
+ }
+
+ scoped_ptr<Statement> metadata_inserter(data.is_host() ?
+ GetHostMetadataUpdateStatement() : GetUrlMetadataUpdateStatement());
+ metadata_inserter->BindString(0, data.primary_key);
+ metadata_inserter->BindInt64(1, data.last_visit.ToInternalValue());
+ if (!metadata_inserter->Run())
+ return false;
+
+ return true;
+}
+
+void ResourcePrefetchPredictorTables::DeleteDataHelper(
+ PrefetchKeyType key_type,
+ const std::vector<std::string>& keys) {
+ bool is_host = key_type == PREFETCH_KEY_TYPE_HOST;
+
+ for (std::vector<std::string>::const_iterator it = keys.begin();
+ it != keys.end(); ++it) {
+ scoped_ptr<Statement> deleter(is_host ? GetHostResourceDeleteStatement() :
+ GetUrlResourceDeleteStatement());
+ deleter->BindString(0, *it);
+ deleter->Run();
+
+ deleter.reset(is_host ? GetHostMetadataDeleteStatement() :
+ GetUrlMetadataDeleteStatement());
+ deleter->BindString(0, *it);
+ deleter->Run();
+ }
+}
+
+bool ResourcePrefetchPredictorTables::StringsAreSmallerThanDBLimit(
+ const PrefetchData& data) const {
+ if (data.primary_key.length() > kMaxStringLength)
+ return false;
+
+ for (ResourceRows::const_iterator it = data.resources.begin();
+ it != data.resources.end(); ++it) {
+ if (it->resource_url.spec().length() > kMaxStringLength)
+ return false;
+ }
+ return true;
+}
+
+void ResourcePrefetchPredictorTables::CreateTableIfNonExistent() {
+ DCHECK_CURRENTLY_ON(BrowserThread::DB);
+ if (CantAccessDatabase())
+ return;
+
+ const char resource_table_creator[] =
+ "CREATE TABLE %s ( "
+ "main_page_url TEXT, "
+ "resource_url TEXT, "
+ "resource_type INTEGER, "
+ "number_of_hits INTEGER, "
+ "number_of_misses INTEGER, "
+ "consecutive_misses INTEGER, "
+ "average_position DOUBLE, "
+ "PRIMARY KEY(main_page_url, resource_url))";
+ const char* metadata_table_creator =
+ "CREATE TABLE %s ( "
+ "main_page_url TEXT, "
+ "last_visit_time INTEGER, "
+ "PRIMARY KEY(main_page_url))";
+
+ sql::Connection* db = DB();
+ bool success =
+ (db->DoesTableExist(kUrlResourceTableName) ||
+ db->Execute(base::StringPrintf(resource_table_creator,
+ kUrlResourceTableName).c_str())) &&
+ (db->DoesTableExist(kUrlMetadataTableName) ||
+ db->Execute(base::StringPrintf(metadata_table_creator,
+ kUrlMetadataTableName).c_str())) &&
+ (db->DoesTableExist(kHostResourceTableName) ||
+ db->Execute(base::StringPrintf(resource_table_creator,
+ kHostResourceTableName).c_str())) &&
+ (db->DoesTableExist(kHostMetadataTableName) ||
+ db->Execute(base::StringPrintf(metadata_table_creator,
+ kHostMetadataTableName).c_str()));
+
+ if (!success)
+ ResetDB();
+}
+
+void ResourcePrefetchPredictorTables::LogDatabaseStats() {
+ DCHECK_CURRENTLY_ON(BrowserThread::DB);
+ if (CantAccessDatabase())
+ return;
+
+ Statement statement(DB()->GetUniqueStatement(
+ base::StringPrintf("SELECT count(*) FROM %s",
+ kUrlResourceTableName).c_str()));
+ if (statement.Step())
+ UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.UrlTableRowCount",
+ statement.ColumnInt(0));
+
+ statement.Assign(DB()->GetUniqueStatement(
+ base::StringPrintf("SELECT count(*) FROM %s",
+ kHostResourceTableName).c_str()));
+ if (statement.Step())
+ UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HostTableRowCount",
+ statement.ColumnInt(0));
+}
+
+Statement*
+ ResourcePrefetchPredictorTables::GetUrlResourceDeleteStatement() {
+ return new Statement(DB()->GetCachedStatement(
+ SQL_FROM_HERE,
+ base::StringPrintf("DELETE FROM %s WHERE main_page_url=?",
+ kUrlResourceTableName).c_str()));
+}
+
+Statement*
+ ResourcePrefetchPredictorTables::GetUrlResourceUpdateStatement() {
+ return new Statement(DB()->GetCachedStatement(
+ SQL_FROM_HERE,
+ base::StringPrintf(
+ "INSERT INTO %s "
+ "(main_page_url, resource_url, resource_type, number_of_hits, "
+ "number_of_misses, consecutive_misses, average_position) "
+ "VALUES (?,?,?,?,?,?,?)", kUrlResourceTableName).c_str()));
+}
+
+Statement*
+ ResourcePrefetchPredictorTables::GetUrlMetadataDeleteStatement() {
+ return new Statement(DB()->GetCachedStatement(
+ SQL_FROM_HERE,
+ base::StringPrintf("DELETE FROM %s WHERE main_page_url=?",
+ kUrlMetadataTableName).c_str()));
+}
+
+Statement*
+ ResourcePrefetchPredictorTables::GetUrlMetadataUpdateStatement() {
+ return new Statement(DB()->GetCachedStatement(
+ SQL_FROM_HERE,
+ base::StringPrintf(
+ "INSERT INTO %s (main_page_url, last_visit_time) VALUES (?,?)",
+ kUrlMetadataTableName).c_str()));
+}
+
+Statement*
+ ResourcePrefetchPredictorTables::GetHostResourceDeleteStatement() {
+ return new Statement(DB()->GetCachedStatement(
+ SQL_FROM_HERE,
+ base::StringPrintf("DELETE FROM %s WHERE main_page_url=?",
+ kHostResourceTableName).c_str()));
+}
+
+Statement*
+ ResourcePrefetchPredictorTables::GetHostResourceUpdateStatement() {
+ return new Statement(DB()->GetCachedStatement(
+ SQL_FROM_HERE,
+ base::StringPrintf(
+ "INSERT INTO %s "
+ "(main_page_url, resource_url, resource_type, number_of_hits, "
+ "number_of_misses, consecutive_misses, average_position) "
+ "VALUES (?,?,?,?,?,?,?)", kHostResourceTableName).c_str()));
+}
+
+Statement*
+ ResourcePrefetchPredictorTables::GetHostMetadataDeleteStatement() {
+ return new Statement(DB()->GetCachedStatement(
+ SQL_FROM_HERE,
+ base::StringPrintf("DELETE FROM %s WHERE main_page_url=?",
+ kHostMetadataTableName).c_str()));
+}
+
+Statement* ResourcePrefetchPredictorTables::GetHostMetadataUpdateStatement() {
+ return new Statement(DB()->GetCachedStatement(
+ SQL_FROM_HERE,
+ base::StringPrintf(
+ "INSERT INTO %s (main_page_url, last_visit_time) VALUES (?,?)",
+ kHostMetadataTableName).c_str()));
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tables.h b/chrome/browser/predictors/resource_prefetch_predictor_tables.h
new file mode 100644
index 0000000..bd266a0
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor_tables.h
@@ -0,0 +1,160 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_
+
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "base/time/time.h"
+#include "chrome/browser/predictors/predictor_table_base.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "content/public/common/resource_type.h"
+#include "url/gurl.h"
+
+namespace sql {
+class Statement;
+}
+
+namespace predictors {
+
+// Interface for database tables used by the ResourcePrefetchPredictor.
+// All methods except the constructor and destructor need to be called on the DB
+// thread.
+//
+// Currently manages:
+// - UrlResourceTable - resources per Urls.
+// - UrlMetadataTable - misc data for Urls (like last visit time).
+// - HostResourceTable - resources per host.
+// - HostMetadataTable - misc data for hosts.
+class ResourcePrefetchPredictorTables : public PredictorTableBase {
+ public:
+ // Used in the UrlResourceTable and HostResourceTable to store resources
+ // required for the page or host.
+ struct ResourceRow {
+ ResourceRow();
+ ResourceRow(const ResourceRow& other);
+ ResourceRow(const std::string& main_frame_url,
+ const std::string& resource_url,
+ content::ResourceType resource_type,
+ int number_of_hits,
+ int number_of_misses,
+ int consecutive_misses,
+ double average_position);
+ void UpdateScore();
+ bool operator==(const ResourceRow& rhs) const;
+
+ // Stores the host for host based data, main frame Url for the Url based
+ // data. This field is cleared for efficiency reasons and the code outside
+ // this class should not assume it is set.
+ std::string primary_key;
+
+ GURL resource_url;
+ content::ResourceType resource_type;
+ size_t number_of_hits;
+ size_t number_of_misses;
+ size_t consecutive_misses;
+ double average_position;
+
+ // Not stored.
+ float score;
+ };
+ typedef std::vector<ResourceRow> ResourceRows;
+
+ // Sorts the ResourceRows by score, descending.
+ struct ResourceRowSorter {
+ bool operator()(const ResourceRow& x, const ResourceRow& y) const;
+ };
+
+ // Aggregated data for a Url or Host. Although the data differs slightly, we
+ // store them in the same structure, because most of the fields are common and
+ // it allows us to use the same functions.
+ struct PrefetchData {
+ PrefetchData(PrefetchKeyType key_type, const std::string& primary_key);
+ PrefetchData(const PrefetchData& other);
+ ~PrefetchData();
+ bool operator==(const PrefetchData& rhs) const;
+
+ bool is_host() const { return key_type == PREFETCH_KEY_TYPE_HOST; }
+
+ // Is the data a host as opposed to a Url?
+ PrefetchKeyType key_type; // Not const to be able to assign.
+ std::string primary_key; // is_host() ? main frame url : host.
+
+ base::Time last_visit;
+ ResourceRows resources;
+ };
+ // Map from primary key to PrefetchData for the key.
+ typedef std::map<std::string, PrefetchData> PrefetchDataMap;
+
+ // Returns data for all Urls and Hosts.
+ virtual void GetAllData(PrefetchDataMap* url_data_map,
+ PrefetchDataMap* host_data_map);
+
+ // Updates data for a Url and a host. If either of the |url_data| or
+ // |host_data| has an empty primary key, it will be ignored.
+ // Note that the Urls and primary key in |url_data| and |host_data| should be
+ // less than |kMaxStringLength| in length.
+ virtual void UpdateData(const PrefetchData& url_data,
+ const PrefetchData& host_data);
+
+ // Delete data for the input |urls| and |hosts|.
+ virtual void DeleteData(const std::vector<std::string>& urls,
+ const std::vector<std::string>& hosts);
+
+ // Wrapper over DeleteData for convenience.
+ virtual void DeleteSingleDataPoint(const std::string& key,
+ PrefetchKeyType key_type);
+
+ // Deletes all data in all the tables.
+ virtual void DeleteAllData();
+
+ // The maximum length of the string that can be stored in the DB.
+ static const size_t kMaxStringLength;
+
+ private:
+ friend class PredictorDatabaseInternal;
+ friend class MockResourcePrefetchPredictorTables;
+
+ ResourcePrefetchPredictorTables();
+ virtual ~ResourcePrefetchPredictorTables();
+
+ // Helper functions below help perform functions on the Url and host table
+ // using the same code.
+ void GetAllDataHelper(PrefetchKeyType key_type,
+ PrefetchDataMap* data_map,
+ std::vector<std::string>* to_delete);
+ bool UpdateDataHelper(const PrefetchData& data);
+ void DeleteDataHelper(PrefetchKeyType key_type,
+ const std::vector<std::string>& keys);
+
+ // Returns true if the strings in the |data| are less than |kMaxStringLength|
+ // in length.
+ bool StringsAreSmallerThanDBLimit(const PrefetchData& data) const;
+
+ // PredictorTableBase methods.
+ virtual void CreateTableIfNonExistent() OVERRIDE;
+ virtual void LogDatabaseStats() OVERRIDE;
+
+ // Helpers to return Statements for cached Statements. The caller must take
+ // ownership of the return Statements.
+ sql::Statement* GetUrlResourceDeleteStatement();
+ sql::Statement* GetUrlResourceUpdateStatement();
+ sql::Statement* GetUrlMetadataDeleteStatement();
+ sql::Statement* GetUrlMetadataUpdateStatement();
+
+ sql::Statement* GetHostResourceDeleteStatement();
+ sql::Statement* GetHostResourceUpdateStatement();
+ sql::Statement* GetHostMetadataDeleteStatement();
+ sql::Statement* GetHostMetadataUpdateStatement();
+
+ DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorTables);
+};
+
+} // namespace predictors
+
+#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tables_unittest.cc b/chrome/browser/predictors/resource_prefetch_predictor_tables_unittest.cc
new file mode 100644
index 0000000..63ed7c4
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor_tables_unittest.cc
@@ -0,0 +1,472 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <set>
+#include <utility>
+#include <vector>
+
+#include "base/message_loop/message_loop.h"
+#include "base/strings/utf_string_conversions.h"
+#include "chrome/browser/predictors/predictor_database.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
+#include "chrome/test/base/testing_profile.h"
+#include "content/public/test/test_browser_thread.h"
+#include "sql/statement.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace predictors {
+
+class ResourcePrefetchPredictorTablesTest : public testing::Test {
+ public:
+ ResourcePrefetchPredictorTablesTest();
+ virtual ~ResourcePrefetchPredictorTablesTest();
+ virtual void SetUp() OVERRIDE;
+ virtual void TearDown() OVERRIDE;
+
+ protected:
+ void TestGetAllData();
+ void TestUpdateData();
+ void TestDeleteData();
+ void TestDeleteSingleDataPoint();
+ void TestDeleteAllData();
+
+ base::MessageLoop loop_;
+ content::TestBrowserThread db_thread_;
+ TestingProfile profile_;
+ scoped_ptr<PredictorDatabase> db_;
+ scoped_refptr<ResourcePrefetchPredictorTables> tables_;
+
+ private:
+ typedef ResourcePrefetchPredictorTables::ResourceRow ResourceRow;
+ typedef std::vector<ResourceRow> ResourceRows;
+ typedef ResourcePrefetchPredictorTables::PrefetchData PrefetchData;
+ typedef ResourcePrefetchPredictorTables::PrefetchDataMap PrefetchDataMap;
+
+ // Initializes the tables, |test_url_data_| and |test_host_data_|.
+ void InitializeSampleData();
+
+ // Checks that the input PrefetchData are the same, although the resources
+ // can be in different order.
+ void TestPrefetchDataAreEqual(const PrefetchDataMap& lhs,
+ const PrefetchDataMap& rhs) const;
+ void TestResourceRowsAreEqual(const ResourceRows& lhs,
+ const ResourceRows& rhs) const;
+
+ void AddKey(PrefetchDataMap* m, const std::string& key) const;
+
+ // Useful for debugging test.
+ void PrintPrefetchData(const PrefetchData& data) const {
+ LOG(ERROR) << "[" << data.key_type << "," << data.primary_key
+ << "," << data.last_visit.ToInternalValue() << "]";
+ for (ResourceRows::const_iterator it = data.resources.begin();
+ it != data.resources.end(); ++it) {
+ LOG(ERROR) << "\t\t" << it->resource_url << "\t" << it->resource_type
+ << "\t" << it->number_of_hits << "\t" << it->number_of_misses
+ << "\t" << it->consecutive_misses
+ << "\t" << it->average_position
+ << "\t" << it->score;
+ }
+ }
+
+ PrefetchDataMap test_url_data_;
+ PrefetchDataMap test_host_data_;
+};
+
+class ResourcePrefetchPredictorTablesReopenTest
+ : public ResourcePrefetchPredictorTablesTest {
+ public:
+ virtual void SetUp() OVERRIDE {
+ // Write data to the table, and then reopen the db.
+ ResourcePrefetchPredictorTablesTest::SetUp();
+ ResourcePrefetchPredictorTablesTest::TearDown();
+
+ db_.reset(new PredictorDatabase(&profile_));
+ loop_.RunUntilIdle();
+ tables_ = db_->resource_prefetch_tables();
+ }
+};
+
+ResourcePrefetchPredictorTablesTest::ResourcePrefetchPredictorTablesTest()
+ : loop_(base::MessageLoop::TYPE_DEFAULT),
+ db_thread_(content::BrowserThread::DB, &loop_),
+ db_(new PredictorDatabase(&profile_)),
+ tables_(db_->resource_prefetch_tables()) {
+ loop_.RunUntilIdle();
+}
+
+ResourcePrefetchPredictorTablesTest::~ResourcePrefetchPredictorTablesTest() {
+}
+
+void ResourcePrefetchPredictorTablesTest::SetUp() {
+ tables_->DeleteAllData();
+ InitializeSampleData();
+}
+
+void ResourcePrefetchPredictorTablesTest::TearDown() {
+ tables_ = NULL;
+ db_.reset();
+ loop_.RunUntilIdle();
+}
+
+void ResourcePrefetchPredictorTablesTest::TestGetAllData() {
+ PrefetchDataMap actual_url_data, actual_host_data;
+ tables_->GetAllData(&actual_url_data, &actual_host_data);
+
+ TestPrefetchDataAreEqual(test_url_data_, actual_url_data);
+ TestPrefetchDataAreEqual(test_host_data_, actual_host_data);
+}
+
+void ResourcePrefetchPredictorTablesTest::TestDeleteData() {
+ std::vector<std::string> urls_to_delete, hosts_to_delete;
+ urls_to_delete.push_back("http://www.google.com");
+ urls_to_delete.push_back("http://www.yahoo.com");
+ hosts_to_delete.push_back("www.yahoo.com");
+
+ tables_->DeleteData(urls_to_delete, hosts_to_delete);
+
+ PrefetchDataMap actual_url_data, actual_host_data;
+ tables_->GetAllData(&actual_url_data, &actual_host_data);
+
+ PrefetchDataMap expected_url_data, expected_host_data;
+ AddKey(&expected_url_data, "http://www.reddit.com");
+ AddKey(&expected_host_data, "www.facebook.com");
+
+ TestPrefetchDataAreEqual(expected_url_data, actual_url_data);
+ TestPrefetchDataAreEqual(expected_host_data, actual_host_data);
+}
+
+void ResourcePrefetchPredictorTablesTest::TestDeleteSingleDataPoint() {
+ // Delete a URL.
+ tables_->DeleteSingleDataPoint("http://www.reddit.com",
+ PREFETCH_KEY_TYPE_URL);
+
+ PrefetchDataMap actual_url_data, actual_host_data;
+ tables_->GetAllData(&actual_url_data, &actual_host_data);
+
+ PrefetchDataMap expected_url_data;
+ AddKey(&expected_url_data, "http://www.google.com");
+ AddKey(&expected_url_data, "http://www.yahoo.com");
+
+ TestPrefetchDataAreEqual(expected_url_data, actual_url_data);
+ TestPrefetchDataAreEqual(test_host_data_, actual_host_data);
+
+ // Delete a host.
+ tables_->DeleteSingleDataPoint("www.facebook.com", PREFETCH_KEY_TYPE_HOST);
+ actual_url_data.clear();
+ actual_host_data.clear();
+ tables_->GetAllData(&actual_url_data, &actual_host_data);
+
+ PrefetchDataMap expected_host_data;
+ AddKey(&expected_host_data, "www.yahoo.com");
+
+ TestPrefetchDataAreEqual(expected_url_data, actual_url_data);
+ TestPrefetchDataAreEqual(expected_host_data, actual_host_data);
+}
+
+void ResourcePrefetchPredictorTablesTest::TestUpdateData() {
+ PrefetchData google(PREFETCH_KEY_TYPE_URL, "http://www.google.com");
+ google.last_visit = base::Time::FromInternalValue(10);
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/style.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 6,
+ 2,
+ 0,
+ 1.0));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 6,
+ 4,
+ 1,
+ 4.2));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/a.xml",
+ content::RESOURCE_TYPE_LAST_TYPE,
+ 1,
+ 0,
+ 0,
+ 6.1));
+ google.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.resources.google.com/script.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 12,
+ 0,
+ 0,
+ 8.5));
+
+ PrefetchData yahoo(PREFETCH_KEY_TYPE_HOST, "www.yahoo.com");
+ yahoo.last_visit = base::Time::FromInternalValue(7);
+ yahoo.resources.push_back(ResourceRow(std::string(),
+ "http://www.yahoo.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 120,
+ 1,
+ 1,
+ 10.0));
+
+ tables_->UpdateData(google, yahoo);
+
+ PrefetchDataMap actual_url_data, actual_host_data;
+ tables_->GetAllData(&actual_url_data, &actual_host_data);
+
+ PrefetchDataMap expected_url_data, expected_host_data;
+ AddKey(&expected_url_data, "http://www.reddit.com");
+ AddKey(&expected_url_data, "http://www.yahoo.com");
+ expected_url_data.insert(std::make_pair("http://www.google.com", google));
+
+ AddKey(&expected_host_data, "www.facebook.com");
+ expected_host_data.insert(std::make_pair("www.yahoo.com", yahoo));
+
+ TestPrefetchDataAreEqual(expected_url_data, actual_url_data);
+ TestPrefetchDataAreEqual(expected_host_data, actual_host_data);
+}
+
+void ResourcePrefetchPredictorTablesTest::TestDeleteAllData() {
+ tables_->DeleteAllData();
+
+ PrefetchDataMap actual_url_data, actual_host_data;
+ tables_->GetAllData(&actual_url_data, &actual_host_data);
+ EXPECT_TRUE(actual_url_data.empty());
+ EXPECT_TRUE(actual_host_data.empty());
+}
+
+void ResourcePrefetchPredictorTablesTest::TestPrefetchDataAreEqual(
+ const PrefetchDataMap& lhs,
+ const PrefetchDataMap& rhs) const {
+ EXPECT_EQ(lhs.size(), rhs.size());
+
+ for (PrefetchDataMap::const_iterator rhs_it = rhs.begin();
+ rhs_it != rhs.end(); ++rhs_it) {
+ PrefetchDataMap::const_iterator lhs_it = lhs.find(rhs_it->first);
+ ASSERT_TRUE(lhs_it != lhs.end()) << rhs_it->first;
+
+ TestResourceRowsAreEqual(lhs_it->second.resources,
+ rhs_it->second.resources);
+ }
+}
+
+void ResourcePrefetchPredictorTablesTest::TestResourceRowsAreEqual(
+ const ResourceRows& lhs,
+ const ResourceRows& rhs) const {
+ EXPECT_EQ(lhs.size(), rhs.size());
+
+ std::set<GURL> resources_seen;
+ for (ResourceRows::const_iterator rhs_it = rhs.begin();
+ rhs_it != rhs.end(); ++rhs_it) {
+ const GURL& resource = rhs_it->resource_url;
+ EXPECT_FALSE(ContainsKey(resources_seen, resource));
+
+ for (ResourceRows::const_iterator lhs_it = lhs.begin();
+ lhs_it != lhs.end(); ++lhs_it) {
+ if (*rhs_it == *lhs_it) {
+ resources_seen.insert(resource);
+ break;
+ }
+ }
+ EXPECT_TRUE(ContainsKey(resources_seen, resource));
+ }
+ EXPECT_EQ(lhs.size(), resources_seen.size());
+}
+
+void ResourcePrefetchPredictorTablesTest::AddKey(PrefetchDataMap* m,
+ const std::string& key) const {
+ PrefetchDataMap::const_iterator it = test_url_data_.find(key);
+ if (it != test_url_data_.end()) {
+ m->insert(std::make_pair(it->first, it->second));
+ return;
+ }
+ it = test_host_data_.find(key);
+ ASSERT_TRUE(it != test_host_data_.end());
+ m->insert(std::make_pair(it->first, it->second));
+}
+
+void ResourcePrefetchPredictorTablesTest::InitializeSampleData() {
+ { // Url data.
+ PrefetchData google(PREFETCH_KEY_TYPE_URL, "http://www.google.com");
+ google.last_visit = base::Time::FromInternalValue(1);
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/style.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 5,
+ 2,
+ 1,
+ 1.1));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/script.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 4,
+ 0,
+ 1,
+ 2.1));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 6,
+ 3,
+ 0,
+ 2.2));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/a.font",
+ content::RESOURCE_TYPE_LAST_TYPE,
+ 2,
+ 0,
+ 0,
+ 5.1));
+ google.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.resources.google.com/script.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 11,
+ 0,
+ 0,
+ 8.5));
+
+ PrefetchData reddit(PREFETCH_KEY_TYPE_URL, "http://www.reddit.com");
+ reddit.last_visit = base::Time::FromInternalValue(2);
+ reddit.resources
+ .push_back(ResourceRow(std::string(),
+ "http://reddit-resource.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 4,
+ 0,
+ 1,
+ 1.0));
+ reddit.resources
+ .push_back(ResourceRow(std::string(),
+ "http://reddit-resource.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 2,
+ 0,
+ 0,
+ 2.1));
+
+ PrefetchData yahoo(PREFETCH_KEY_TYPE_URL, "http://www.yahoo.com");
+ yahoo.last_visit = base::Time::FromInternalValue(3);
+ yahoo.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 20,
+ 1,
+ 0,
+ 10.0));
+
+ test_url_data_.clear();
+ test_url_data_.insert(std::make_pair("http://www.google.com", google));
+ test_url_data_.insert(std::make_pair("http://www.reddit.com", reddit));
+ test_url_data_.insert(std::make_pair("http://www.yahoo.com", yahoo));
+
+ PrefetchData empty_host_data(PREFETCH_KEY_TYPE_HOST, std::string());
+ tables_->UpdateData(google, empty_host_data);
+ tables_->UpdateData(reddit, empty_host_data);
+ tables_->UpdateData(yahoo, empty_host_data);
+ }
+
+ { // Host data.
+ PrefetchData facebook(PREFETCH_KEY_TYPE_HOST, "www.facebook.com");
+ facebook.last_visit = base::Time::FromInternalValue(4);
+ facebook.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.facebook.com/style.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 5,
+ 2,
+ 1,
+ 1.1));
+ facebook.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.facebook.com/script.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 4,
+ 0,
+ 1,
+ 2.1));
+ facebook.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.facebook.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 6,
+ 3,
+ 0,
+ 2.2));
+ facebook.resources.push_back(ResourceRow(std::string(),
+ "http://www.facebook.com/a.font",
+ content::RESOURCE_TYPE_LAST_TYPE,
+ 2,
+ 0,
+ 0,
+ 5.1));
+ facebook.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.resources.facebook.com/script.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 11,
+ 0,
+ 0,
+ 8.5));
+
+ PrefetchData yahoo(PREFETCH_KEY_TYPE_HOST, "www.yahoo.com");
+ yahoo.last_visit = base::Time::FromInternalValue(5);
+ yahoo.resources.push_back(ResourceRow(std::string(),
+ "http://www.google.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 20,
+ 1,
+ 0,
+ 10.0));
+
+ test_host_data_.clear();
+ test_host_data_.insert(std::make_pair("www.facebook.com", facebook));
+ test_host_data_.insert(std::make_pair("www.yahoo.com", yahoo));
+
+ PrefetchData empty_url_data(PREFETCH_KEY_TYPE_URL, std::string());
+ tables_->UpdateData(empty_url_data, facebook);
+ tables_->UpdateData(empty_url_data, yahoo);
+ }
+}
+
+// Test cases.
+
+TEST_F(ResourcePrefetchPredictorTablesTest, GetAllData) {
+ TestGetAllData();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesTest, UpdateData) {
+ TestUpdateData();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesTest, DeleteData) {
+ TestDeleteData();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesTest, DeleteSingleDataPoint) {
+ TestDeleteSingleDataPoint();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesTest, DeleteAllData) {
+ TestDeleteAllData();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesReopenTest, GetAllData) {
+ TestGetAllData();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesReopenTest, UpdateData) {
+ TestUpdateData();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesReopenTest, DeleteData) {
+ TestDeleteData();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesReopenTest, DeleteSingleDataPoint) {
+ TestDeleteSingleDataPoint();
+}
+
+TEST_F(ResourcePrefetchPredictorTablesReopenTest, DeleteAllData) {
+ TestDeleteAllData();
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc b/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc
new file mode 100644
index 0000000..6e79971
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc
@@ -0,0 +1,925 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <iostream>
+#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/message_loop/message_loop.h"
+#include "base/run_loop.h"
+#include "base/time/time.h"
+#include "chrome/browser/history/history_service.h"
+#include "chrome/browser/history/history_service_factory.h"
+#include "chrome/browser/history/history_types.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
+#include "chrome/test/base/testing_profile.h"
+#include "content/public/test/test_browser_thread.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using testing::ContainerEq;
+using testing::Pointee;
+using testing::SetArgPointee;
+using testing::StrictMock;
+
+namespace predictors {
+
+typedef ResourcePrefetchPredictor::URLRequestSummary URLRequestSummary;
+typedef ResourcePrefetchPredictorTables::ResourceRow ResourceRow;
+typedef std::vector<ResourceRow> ResourceRows;
+typedef ResourcePrefetchPredictorTables::PrefetchData PrefetchData;
+typedef ResourcePrefetchPredictorTables::PrefetchDataMap PrefetchDataMap;
+
+// For printing failures nicely.
+void PrintTo(const ResourceRow& row, ::std::ostream* os) {
+ *os << "[" << row.primary_key << "," << row.resource_url
+ << "," << row.resource_type << "," << row.number_of_hits
+ << "," << row.number_of_misses << "," << row.consecutive_misses
+ << "," << row.average_position << "," << row.score << "]";
+}
+
+void PrintTo(const PrefetchData& data, ::std::ostream* os) {
+ *os << "[" << data.key_type << "," << data.primary_key
+ << "," << data.last_visit.ToInternalValue() << "]\n";
+ for (ResourceRows::const_iterator it = data.resources.begin();
+ it != data.resources.end(); ++it) {
+ *os << "\t\t";
+ PrintTo(*it, os);
+ *os << "\n";
+ }
+}
+
+class MockResourcePrefetchPredictorTables
+ : public ResourcePrefetchPredictorTables {
+ public:
+ MockResourcePrefetchPredictorTables() { }
+
+ MOCK_METHOD2(GetAllData, void(PrefetchDataMap* url_data_map,
+ PrefetchDataMap* host_data_map));
+ MOCK_METHOD2(UpdateData, void(const PrefetchData& url_data,
+ const PrefetchData& host_data));
+ MOCK_METHOD2(DeleteData, void(const std::vector<std::string>& urls,
+ const std::vector<std::string>& hosts));
+ MOCK_METHOD2(DeleteSingleDataPoint, void(const std::string& key,
+ PrefetchKeyType key_type));
+ MOCK_METHOD0(DeleteAllData, void());
+
+ protected:
+ ~MockResourcePrefetchPredictorTables() { }
+};
+
+class ResourcePrefetchPredictorTest : public testing::Test {
+ public:
+ ResourcePrefetchPredictorTest();
+ virtual ~ResourcePrefetchPredictorTest();
+ virtual void SetUp() OVERRIDE;
+ virtual void TearDown() OVERRIDE;
+
+ protected:
+ void AddUrlToHistory(const std::string& url, int visit_count) {
+ HistoryServiceFactory::GetForProfile(profile_.get(),
+ Profile::EXPLICIT_ACCESS)->
+ AddPageWithDetails(
+ GURL(url),
+ base::string16(),
+ visit_count,
+ 0,
+ base::Time::Now(),
+ false,
+ history::SOURCE_BROWSED);
+ profile_->BlockUntilHistoryProcessesPendingRequests();
+ }
+
+ NavigationID CreateNavigationID(int process_id,
+ int render_frame_id,
+ const std::string& main_frame_url) {
+ NavigationID navigation_id;
+ navigation_id.render_process_id = process_id;
+ navigation_id.render_frame_id = render_frame_id;
+ navigation_id.main_frame_url = GURL(main_frame_url);
+ navigation_id.creation_time = base::TimeTicks::Now();
+ return navigation_id;
+ }
+
+ ResourcePrefetchPredictor::URLRequestSummary CreateURLRequestSummary(
+ int process_id,
+ int render_frame_id,
+ const std::string& main_frame_url,
+ const std::string& resource_url,
+ content::ResourceType resource_type,
+ const std::string& mime_type,
+ bool was_cached) {
+ ResourcePrefetchPredictor::URLRequestSummary summary;
+ summary.navigation_id = CreateNavigationID(process_id, render_frame_id,
+ main_frame_url);
+ summary.resource_url = GURL(resource_url);
+ summary.resource_type = resource_type;
+ summary.mime_type = mime_type;
+ summary.was_cached = was_cached;
+ return summary;
+ }
+
+ void InitializePredictor() {
+ predictor_->StartInitialization();
+ base::RunLoop loop;
+ loop.RunUntilIdle(); // Runs the DB lookup.
+ profile_->BlockUntilHistoryProcessesPendingRequests();
+ }
+
+ bool URLRequestSummaryAreEqual(const URLRequestSummary& lhs,
+ const URLRequestSummary& rhs) {
+ return lhs.navigation_id == rhs.navigation_id &&
+ lhs.resource_url == rhs.resource_url &&
+ lhs.resource_type == rhs.resource_type &&
+ lhs.mime_type == rhs.mime_type &&
+ lhs.was_cached == rhs.was_cached;
+ }
+
+ void ResetPredictor() {
+ ResourcePrefetchPredictorConfig config;
+ config.max_urls_to_track = 3;
+ config.max_hosts_to_track = 2;
+ config.min_url_visit_count = 2;
+ config.max_resources_per_entry = 4;
+ config.max_consecutive_misses = 2;
+
+ // TODO(shishir): Enable the prefetching mode in the tests.
+ config.mode |= ResourcePrefetchPredictorConfig::URL_LEARNING;
+ config.mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING;
+ predictor_.reset(new ResourcePrefetchPredictor(config, profile_.get()));
+ predictor_->set_mock_tables(mock_tables_);
+ }
+
+ void InitializeSampleData();
+
+ base::MessageLoop loop_;
+ content::TestBrowserThread ui_thread_;
+ content::TestBrowserThread db_thread_;
+ scoped_ptr<TestingProfile> profile_;
+
+ scoped_ptr<ResourcePrefetchPredictor> predictor_;
+ scoped_refptr<StrictMock<MockResourcePrefetchPredictorTables> > mock_tables_;
+
+ PrefetchDataMap test_url_data_;
+ PrefetchDataMap test_host_data_;
+ PrefetchData empty_url_data_;
+ PrefetchData empty_host_data_;
+};
+
+ResourcePrefetchPredictorTest::ResourcePrefetchPredictorTest()
+ : loop_(base::MessageLoop::TYPE_DEFAULT),
+ ui_thread_(content::BrowserThread::UI, &loop_),
+ db_thread_(content::BrowserThread::DB, &loop_),
+ profile_(new TestingProfile()),
+ mock_tables_(new StrictMock<MockResourcePrefetchPredictorTables>()),
+ empty_url_data_(PREFETCH_KEY_TYPE_URL, std::string()),
+ empty_host_data_(PREFETCH_KEY_TYPE_HOST, std::string()) {}
+
+ResourcePrefetchPredictorTest::~ResourcePrefetchPredictorTest() {
+ profile_.reset(NULL);
+ loop_.RunUntilIdle();
+}
+
+void ResourcePrefetchPredictorTest::SetUp() {
+ InitializeSampleData();
+
+ ASSERT_TRUE(profile_->CreateHistoryService(true, false));
+ profile_->BlockUntilHistoryProcessesPendingRequests();
+ EXPECT_TRUE(HistoryServiceFactory::GetForProfile(profile_.get(),
+ Profile::EXPLICIT_ACCESS));
+ // Initialize the predictor with empty data.
+ ResetPredictor();
+ EXPECT_EQ(predictor_->initialization_state_,
+ ResourcePrefetchPredictor::NOT_INITIALIZED);
+ EXPECT_CALL(*mock_tables_.get(),
+ GetAllData(Pointee(ContainerEq(PrefetchDataMap())),
+ Pointee(ContainerEq(PrefetchDataMap()))));
+ InitializePredictor();
+ EXPECT_TRUE(predictor_->inflight_navigations_.empty());
+ EXPECT_EQ(predictor_->initialization_state_,
+ ResourcePrefetchPredictor::INITIALIZED);
+}
+
+void ResourcePrefetchPredictorTest::TearDown() {
+ predictor_.reset(NULL);
+ profile_->DestroyHistoryService();
+}
+
+void ResourcePrefetchPredictorTest::InitializeSampleData() {
+ { // Url data.
+ PrefetchData google(PREFETCH_KEY_TYPE_URL, "http://www.google.com/");
+ google.last_visit = base::Time::FromInternalValue(1);
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 3,
+ 2,
+ 1,
+ 1.0));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script3.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 4,
+ 0,
+ 1,
+ 2.1));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script4.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 11,
+ 0,
+ 0,
+ 2.1));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/image1.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 6,
+ 3,
+ 0,
+ 2.2));
+ google.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/a.font",
+ content::RESOURCE_TYPE_LAST_TYPE,
+ 2,
+ 0,
+ 0,
+ 5.1));
+
+ PrefetchData reddit(PREFETCH_KEY_TYPE_URL, "http://www.reddit.com/");
+ reddit.last_visit = base::Time::FromInternalValue(2);
+ reddit.resources
+ .push_back(ResourceRow(std::string(),
+ "http://reddit-resource.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 4,
+ 0,
+ 1,
+ 1.0));
+ reddit.resources
+ .push_back(ResourceRow(std::string(),
+ "http://reddit-resource.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 2,
+ 0,
+ 0,
+ 2.1));
+
+ PrefetchData yahoo(PREFETCH_KEY_TYPE_URL, "http://www.yahoo.com/");
+ yahoo.last_visit = base::Time::FromInternalValue(3);
+ yahoo.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 20,
+ 1,
+ 0,
+ 10.0));
+
+ test_url_data_.clear();
+ test_url_data_.insert(std::make_pair("http://www.google.com/", google));
+ test_url_data_.insert(std::make_pair("http://www.reddit.com/", reddit));
+ test_url_data_.insert(std::make_pair("http://www.yahoo.com/", yahoo));
+ }
+
+ { // Host data.
+ PrefetchData facebook(PREFETCH_KEY_TYPE_HOST, "www.facebook.com");
+ facebook.last_visit = base::Time::FromInternalValue(4);
+ facebook.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.facebook.com/style.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 5,
+ 2,
+ 1,
+ 1.1));
+ facebook.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.facebook.com/script.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 4,
+ 0,
+ 1,
+ 2.1));
+ facebook.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.facebook.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 6,
+ 3,
+ 0,
+ 2.2));
+ facebook.resources.push_back(ResourceRow(std::string(),
+ "http://www.facebook.com/a.font",
+ content::RESOURCE_TYPE_LAST_TYPE,
+ 2,
+ 0,
+ 0,
+ 5.1));
+ facebook.resources
+ .push_back(ResourceRow(std::string(),
+ "http://www.resources.facebook.com/script.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 11,
+ 0,
+ 0,
+ 8.5));
+
+ PrefetchData yahoo(PREFETCH_KEY_TYPE_HOST, "www.yahoo.com");
+ yahoo.last_visit = base::Time::FromInternalValue(5);
+ yahoo.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/image.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 20,
+ 1,
+ 0,
+ 10.0));
+
+ test_host_data_.clear();
+ test_host_data_.insert(std::make_pair("www.facebook.com", facebook));
+ test_host_data_.insert(std::make_pair("www.yahoo.com", yahoo));
+ }
+}
+
+TEST_F(ResourcePrefetchPredictorTest, LazilyInitializeEmpty) {
+ // Tests that the predictor initializes correctly without any data.
+ EXPECT_TRUE(predictor_->url_table_cache_->empty());
+ EXPECT_TRUE(predictor_->host_table_cache_->empty());
+}
+
+TEST_F(ResourcePrefetchPredictorTest, LazilyInitializeWithData) {
+ // Tests that the history and the db tables data are loaded correctly.
+ AddUrlToHistory("http://www.google.com/", 4);
+ AddUrlToHistory("http://www.yahoo.com/", 2);
+
+ EXPECT_CALL(*mock_tables_.get(),
+ GetAllData(Pointee(ContainerEq(PrefetchDataMap())),
+ Pointee(ContainerEq(PrefetchDataMap()))))
+ .WillOnce(DoAll(SetArgPointee<0>(test_url_data_),
+ SetArgPointee<1>(test_host_data_)));
+
+ ResetPredictor();
+ InitializePredictor();
+
+ // Test that the internal variables correctly initialized.
+ EXPECT_EQ(predictor_->initialization_state_,
+ ResourcePrefetchPredictor::INITIALIZED);
+ EXPECT_TRUE(predictor_->inflight_navigations_.empty());
+
+ EXPECT_EQ(test_url_data_, *predictor_->url_table_cache_);
+ EXPECT_EQ(test_host_data_, *predictor_->host_table_cache_);
+}
+
+TEST_F(ResourcePrefetchPredictorTest, NavigationNotRecorded) {
+ // Single navigation but history count is low, so should not record.
+ AddUrlToHistory("http://www.google.com", 1);
+
+ URLRequestSummary main_frame =
+ CreateURLRequestSummary(1,
+ 1,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ predictor_->RecordURLRequest(main_frame);
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ // Now add a few subresources.
+ URLRequestSummary resource1 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET, "text/css", false);
+ predictor_->RecordURLResponse(resource1);
+ URLRequestSummary resource2 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", false);
+ predictor_->RecordURLResponse(resource2);
+ URLRequestSummary resource3 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", false);
+ predictor_->RecordURLResponse(resource3);
+
+ PrefetchData host_data(PREFETCH_KEY_TYPE_HOST, "www.google.com");
+ host_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 1,
+ 0,
+ 0,
+ 1.0));
+ host_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 1,
+ 0,
+ 0,
+ 2.0));
+ host_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 1,
+ 0,
+ 0,
+ 3.0));
+ EXPECT_CALL(*mock_tables_.get(), UpdateData(empty_url_data_, host_data));
+
+ predictor_->OnNavigationComplete(main_frame.navigation_id);
+ profile_->BlockUntilHistoryProcessesPendingRequests();
+}
+
+TEST_F(ResourcePrefetchPredictorTest, NavigationUrlNotInDB) {
+ // Single navigation that will be recorded. Will check for duplicate
+ // resources and also for number of resources saved.
+ AddUrlToHistory("http://www.google.com", 4);
+
+ URLRequestSummary main_frame =
+ CreateURLRequestSummary(1,
+ 1,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ predictor_->RecordURLRequest(main_frame);
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ URLRequestSummary resource1 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET, "text/css", false);
+ predictor_->RecordURLResponse(resource1);
+ URLRequestSummary resource2 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", false);
+ predictor_->RecordURLResponse(resource2);
+ URLRequestSummary resource3 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", false);
+ predictor_->RecordURLResponse(resource3);
+ URLRequestSummary resource4 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", true);
+ predictor_->RecordURLResponse(resource4);
+ URLRequestSummary resource5 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/image1.png",
+ content::RESOURCE_TYPE_IMAGE, "image/png", false);
+ predictor_->RecordURLResponse(resource5);
+ URLRequestSummary resource6 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/image2.png",
+ content::RESOURCE_TYPE_IMAGE, "image/png", false);
+ predictor_->RecordURLResponse(resource6);
+ URLRequestSummary resource7 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/style2.css",
+ content::RESOURCE_TYPE_STYLESHEET, "text/css", true);
+ predictor_->RecordURLResponse(resource7);
+
+ PrefetchData url_data(PREFETCH_KEY_TYPE_URL, "http://www.google.com/");
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 1,
+ 0,
+ 0,
+ 1.0));
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 1,
+ 0,
+ 0,
+ 2.0));
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 1,
+ 0,
+ 0,
+ 3.0));
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/style2.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 1,
+ 0,
+ 0,
+ 7.0));
+ EXPECT_CALL(*mock_tables_.get(), UpdateData(url_data, empty_host_data_));
+
+ PrefetchData host_data(PREFETCH_KEY_TYPE_HOST, "www.google.com");
+ host_data.resources = url_data.resources;
+ EXPECT_CALL(*mock_tables_.get(), UpdateData(empty_url_data_, host_data));
+
+ predictor_->OnNavigationComplete(main_frame.navigation_id);
+ profile_->BlockUntilHistoryProcessesPendingRequests();
+}
+
+TEST_F(ResourcePrefetchPredictorTest, NavigationUrlInDB) {
+ // Tests that navigation is recorded correctly for URL already present in
+ // the database cache.
+ AddUrlToHistory("http://www.google.com", 4);
+
+ EXPECT_CALL(*mock_tables_.get(),
+ GetAllData(Pointee(ContainerEq(PrefetchDataMap())),
+ Pointee(ContainerEq(PrefetchDataMap()))))
+ .WillOnce(DoAll(SetArgPointee<0>(test_url_data_),
+ SetArgPointee<1>(test_host_data_)));
+ ResetPredictor();
+ InitializePredictor();
+ EXPECT_EQ(3, static_cast<int>(predictor_->url_table_cache_->size()));
+ EXPECT_EQ(2, static_cast<int>(predictor_->host_table_cache_->size()));
+
+ URLRequestSummary main_frame =
+ CreateURLRequestSummary(1,
+ 1,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ predictor_->RecordURLRequest(main_frame);
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ URLRequestSummary resource1 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET, "text/css", false);
+ predictor_->RecordURLResponse(resource1);
+ URLRequestSummary resource2 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", false);
+ predictor_->RecordURLResponse(resource2);
+ URLRequestSummary resource3 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", false);
+ predictor_->RecordURLResponse(resource3);
+ URLRequestSummary resource4 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", true);
+ predictor_->RecordURLResponse(resource4);
+ URLRequestSummary resource5 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/image1.png",
+ content::RESOURCE_TYPE_IMAGE, "image/png", false);
+ predictor_->RecordURLResponse(resource5);
+ URLRequestSummary resource6 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/image2.png",
+ content::RESOURCE_TYPE_IMAGE, "image/png", false);
+ predictor_->RecordURLResponse(resource6);
+ URLRequestSummary resource7 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/style2.css",
+ content::RESOURCE_TYPE_STYLESHEET, "text/css", true);
+ predictor_->RecordURLResponse(resource7);
+
+ PrefetchData url_data(PREFETCH_KEY_TYPE_URL, "http://www.google.com/");
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 4,
+ 2,
+ 0,
+ 1.0));
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 1,
+ 0,
+ 0,
+ 2.0));
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script4.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 11,
+ 1,
+ 1,
+ 2.1));
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 1,
+ 0,
+ 0,
+ 3.0));
+ EXPECT_CALL(*mock_tables_.get(), UpdateData(url_data, empty_host_data_));
+
+ EXPECT_CALL(
+ *mock_tables_.get(),
+ DeleteSingleDataPoint("www.facebook.com", PREFETCH_KEY_TYPE_HOST));
+
+ PrefetchData host_data(PREFETCH_KEY_TYPE_HOST, "www.google.com");
+ host_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 1,
+ 0,
+ 0,
+ 1.0));
+ host_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 1,
+ 0,
+ 0,
+ 2.0));
+ host_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT,
+ 1,
+ 0,
+ 0,
+ 3.0));
+ host_data.resources.push_back(ResourceRow(std::string(),
+ "http://google.com/style2.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 1,
+ 0,
+ 0,
+ 7.0));
+ EXPECT_CALL(*mock_tables_.get(), UpdateData(empty_url_data_, host_data));
+
+ predictor_->OnNavigationComplete(main_frame.navigation_id);
+ profile_->BlockUntilHistoryProcessesPendingRequests();
+}
+
+TEST_F(ResourcePrefetchPredictorTest, NavigationUrlNotInDBAndDBFull) {
+ // Tests that a URL is deleted before another is added if the cache is full.
+ AddUrlToHistory("http://www.nike.com/", 4);
+
+ EXPECT_CALL(*mock_tables_.get(),
+ GetAllData(Pointee(ContainerEq(PrefetchDataMap())),
+ Pointee(ContainerEq(PrefetchDataMap()))))
+ .WillOnce(DoAll(SetArgPointee<0>(test_url_data_),
+ SetArgPointee<1>(test_host_data_)));
+ ResetPredictor();
+ InitializePredictor();
+ EXPECT_EQ(3, static_cast<int>(predictor_->url_table_cache_->size()));
+ EXPECT_EQ(2, static_cast<int>(predictor_->host_table_cache_->size()));
+
+ URLRequestSummary main_frame =
+ CreateURLRequestSummary(1,
+ 1,
+ "http://www.nike.com",
+ "http://www.nike.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ predictor_->RecordURLRequest(main_frame);
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ URLRequestSummary resource1 = CreateURLRequestSummary(
+ 1, 1, "http://www.nike.com", "http://nike.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET, "text/css", false);
+ predictor_->RecordURLResponse(resource1);
+ URLRequestSummary resource2 = CreateURLRequestSummary(
+ 1, 1, "http://www.nike.com", "http://nike.com/image2.png",
+ content::RESOURCE_TYPE_IMAGE, "image/png", false);
+ predictor_->RecordURLResponse(resource2);
+
+ EXPECT_CALL(
+ *mock_tables_.get(),
+ DeleteSingleDataPoint("http://www.google.com/", PREFETCH_KEY_TYPE_URL));
+ EXPECT_CALL(
+ *mock_tables_.get(),
+ DeleteSingleDataPoint("www.facebook.com", PREFETCH_KEY_TYPE_HOST));
+
+ PrefetchData url_data(PREFETCH_KEY_TYPE_URL, "http://www.nike.com/");
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://nike.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET,
+ 1,
+ 0,
+ 0,
+ 1.0));
+ url_data.resources.push_back(ResourceRow(std::string(),
+ "http://nike.com/image2.png",
+ content::RESOURCE_TYPE_IMAGE,
+ 1,
+ 0,
+ 0,
+ 2.0));
+ EXPECT_CALL(*mock_tables_.get(), UpdateData(url_data, empty_host_data_));
+
+ PrefetchData host_data(PREFETCH_KEY_TYPE_HOST, "www.nike.com");
+ host_data.resources = url_data.resources;
+ EXPECT_CALL(*mock_tables_.get(), UpdateData(empty_url_data_, host_data));
+
+ predictor_->OnNavigationComplete(main_frame.navigation_id);
+ profile_->BlockUntilHistoryProcessesPendingRequests();
+}
+
+TEST_F(ResourcePrefetchPredictorTest, DeleteUrls) {
+ // Add some dummy entries to cache.
+ predictor_->url_table_cache_->insert(std::make_pair(
+ "http://www.google.com/page1.html",
+ PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.google.com/page1.html")));
+ predictor_->url_table_cache_->insert(std::make_pair(
+ "http://www.google.com/page2.html",
+ PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.google.com/page2.html")));
+ predictor_->url_table_cache_->insert(std::make_pair(
+ "http://www.yahoo.com/",
+ PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.yahoo.com/")));
+ predictor_->url_table_cache_->insert(std::make_pair(
+ "http://www.apple.com/",
+ PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.apple.com/")));
+ predictor_->url_table_cache_->insert(std::make_pair(
+ "http://www.nike.com/",
+ PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.nike.com/")));
+
+ predictor_->host_table_cache_->insert(std::make_pair(
+ "www.google.com",
+ PrefetchData(PREFETCH_KEY_TYPE_HOST, "www.google.com")));
+ predictor_->host_table_cache_->insert(std::make_pair(
+ "www.yahoo.com",
+ PrefetchData(PREFETCH_KEY_TYPE_HOST, "www.yahoo.com")));
+ predictor_->host_table_cache_->insert(std::make_pair(
+ "www.apple.com",
+ PrefetchData(PREFETCH_KEY_TYPE_HOST, "www.apple.com")));
+
+ history::URLRows rows;
+ rows.push_back(history::URLRow(GURL("http://www.google.com/page2.html")));
+ rows.push_back(history::URLRow(GURL("http://www.apple.com")));
+ rows.push_back(history::URLRow(GURL("http://www.nike.com")));
+
+ std::vector<std::string> urls_to_delete, hosts_to_delete;
+ urls_to_delete.push_back("http://www.google.com/page2.html");
+ urls_to_delete.push_back("http://www.apple.com/");
+ urls_to_delete.push_back("http://www.nike.com/");
+ hosts_to_delete.push_back("www.google.com");
+ hosts_to_delete.push_back("www.apple.com");
+
+ EXPECT_CALL(
+ *mock_tables_.get(),
+ DeleteData(ContainerEq(urls_to_delete), ContainerEq(hosts_to_delete)));
+
+ predictor_->DeleteUrls(rows);
+ EXPECT_EQ(2, static_cast<int>(predictor_->url_table_cache_->size()));
+ EXPECT_EQ(1, static_cast<int>(predictor_->host_table_cache_->size()));
+
+ EXPECT_CALL(*mock_tables_.get(), DeleteAllData());
+
+ predictor_->DeleteAllUrls();
+ EXPECT_TRUE(predictor_->url_table_cache_->empty());
+ EXPECT_TRUE(predictor_->host_table_cache_->empty());
+}
+
+TEST_F(ResourcePrefetchPredictorTest, OnMainFrameRequest) {
+ URLRequestSummary summary1 =
+ CreateURLRequestSummary(1,
+ 1,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ URLRequestSummary summary2 =
+ CreateURLRequestSummary(1,
+ 2,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ URLRequestSummary summary3 =
+ CreateURLRequestSummary(2,
+ 1,
+ "http://www.yahoo.com",
+ "http://www.yahoo.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+
+ predictor_->OnMainFrameRequest(summary1);
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+ predictor_->OnMainFrameRequest(summary2);
+ EXPECT_EQ(2, static_cast<int>(predictor_->inflight_navigations_.size()));
+ predictor_->OnMainFrameRequest(summary3);
+ EXPECT_EQ(3, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ // Insert anther with same navigation id. It should replace.
+ URLRequestSummary summary4 =
+ CreateURLRequestSummary(1,
+ 1,
+ "http://www.nike.com",
+ "http://www.nike.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ URLRequestSummary summary5 =
+ CreateURLRequestSummary(1,
+ 2,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+
+ predictor_->OnMainFrameRequest(summary4);
+ EXPECT_EQ(3, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ // Change this creation time so that it will go away on the next insert.
+ summary5.navigation_id.creation_time = base::TimeTicks::Now() -
+ base::TimeDelta::FromDays(1);
+ predictor_->OnMainFrameRequest(summary5);
+ EXPECT_EQ(3, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ URLRequestSummary summary6 =
+ CreateURLRequestSummary(3,
+ 1,
+ "http://www.shoes.com",
+ "http://www.shoes.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ predictor_->OnMainFrameRequest(summary6);
+ EXPECT_EQ(3, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ EXPECT_TRUE(predictor_->inflight_navigations_.find(summary3.navigation_id) !=
+ predictor_->inflight_navigations_.end());
+ EXPECT_TRUE(predictor_->inflight_navigations_.find(summary4.navigation_id) !=
+ predictor_->inflight_navigations_.end());
+ EXPECT_TRUE(predictor_->inflight_navigations_.find(summary6.navigation_id) !=
+ predictor_->inflight_navigations_.end());
+}
+
+TEST_F(ResourcePrefetchPredictorTest, OnMainFrameRedirect) {
+ URLRequestSummary summary1 =
+ CreateURLRequestSummary(1,
+ 1,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ URLRequestSummary summary2 =
+ CreateURLRequestSummary(1,
+ 2,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ URLRequestSummary summary3 =
+ CreateURLRequestSummary(2,
+ 1,
+ "http://www.yahoo.com",
+ "http://www.yahoo.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+
+ predictor_->OnMainFrameRedirect(summary1);
+ EXPECT_TRUE(predictor_->inflight_navigations_.empty());
+
+ predictor_->OnMainFrameRequest(summary1);
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+ predictor_->OnMainFrameRequest(summary2);
+ EXPECT_EQ(2, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ predictor_->OnMainFrameRedirect(summary3);
+ EXPECT_EQ(2, static_cast<int>(predictor_->inflight_navigations_.size()));
+ predictor_->OnMainFrameRedirect(summary1);
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+ predictor_->OnMainFrameRedirect(summary2);
+ EXPECT_TRUE(predictor_->inflight_navigations_.empty());
+}
+
+TEST_F(ResourcePrefetchPredictorTest, OnSubresourceResponse) {
+ // If there is no inflight navigation, nothing happens.
+ URLRequestSummary resource1 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/style1.css",
+ content::RESOURCE_TYPE_STYLESHEET, "text/css", false);
+ predictor_->OnSubresourceResponse(resource1);
+ EXPECT_TRUE(predictor_->inflight_navigations_.empty());
+
+ // Add an inflight navigation.
+ URLRequestSummary main_frame1 =
+ CreateURLRequestSummary(1,
+ 1,
+ "http://www.google.com",
+ "http://www.google.com",
+ content::RESOURCE_TYPE_MAIN_FRAME,
+ std::string(),
+ false);
+ predictor_->OnMainFrameRequest(main_frame1);
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+
+ // Now add a few subresources.
+ URLRequestSummary resource2 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script1.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", false);
+ URLRequestSummary resource3 = CreateURLRequestSummary(
+ 1, 1, "http://www.google.com", "http://google.com/script2.js",
+ content::RESOURCE_TYPE_SCRIPT, "text/javascript", false);
+ predictor_->OnSubresourceResponse(resource1);
+ predictor_->OnSubresourceResponse(resource2);
+ predictor_->OnSubresourceResponse(resource3);
+
+ EXPECT_EQ(1, static_cast<int>(predictor_->inflight_navigations_.size()));
+ EXPECT_EQ(3, static_cast<int>(
+ predictor_->inflight_navigations_[main_frame1.navigation_id]->size()));
+ EXPECT_TRUE(URLRequestSummaryAreEqual(
+ resource1,
+ predictor_->inflight_navigations_[main_frame1.navigation_id]->at(0)));
+ EXPECT_TRUE(URLRequestSummaryAreEqual(
+ resource2,
+ predictor_->inflight_navigations_[main_frame1.navigation_id]->at(1)));
+ EXPECT_TRUE(URLRequestSummaryAreEqual(
+ resource3,
+ predictor_->inflight_navigations_[main_frame1.navigation_id]->at(2)));
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetcher.cc b/chrome/browser/predictors/resource_prefetcher.cc
new file mode 100644
index 0000000..b602a2b
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetcher.cc
@@ -0,0 +1,240 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetcher.h"
+
+#include <iterator>
+
+#include "base/stl_util.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/base/io_buffer.h"
+#include "net/base/request_priority.h"
+#include "net/url_request/url_request_context.h"
+
+namespace {
+
+// The size of the buffer used to read the resource.
+static const size_t kResourceBufferSizeBytes = 50000;
+
+} // namespace
+
+namespace predictors {
+
+ResourcePrefetcher::Request::Request(const GURL& i_resource_url)
+ : resource_url(i_resource_url),
+ prefetch_status(PREFETCH_STATUS_NOT_STARTED),
+ usage_status(USAGE_STATUS_NOT_REQUESTED) {
+}
+
+ResourcePrefetcher::Request::Request(const Request& other)
+ : resource_url(other.resource_url),
+ prefetch_status(other.prefetch_status),
+ usage_status(other.usage_status) {
+}
+
+ResourcePrefetcher::ResourcePrefetcher(
+ Delegate* delegate,
+ const ResourcePrefetchPredictorConfig& config,
+ const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ scoped_ptr<RequestVector> requests)
+ : state_(INITIALIZED),
+ delegate_(delegate),
+ config_(config),
+ navigation_id_(navigation_id),
+ key_type_(key_type),
+ request_vector_(requests.Pass()) {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+ DCHECK(request_vector_.get());
+
+ std::copy(request_vector_->begin(), request_vector_->end(),
+ std::back_inserter(request_queue_));
+}
+
+ResourcePrefetcher::~ResourcePrefetcher() {
+ // Delete any pending net::URLRequests.
+ STLDeleteContainerPairFirstPointers(inflight_requests_.begin(),
+ inflight_requests_.end());
+}
+
+void ResourcePrefetcher::Start() {
+ DCHECK(thread_checker_.CalledOnValidThread());
+
+ CHECK_EQ(state_, INITIALIZED);
+ state_ = RUNNING;
+
+ TryToLaunchPrefetchRequests();
+}
+
+void ResourcePrefetcher::Stop() {
+ DCHECK(thread_checker_.CalledOnValidThread());
+
+ if (state_ == FINISHED)
+ return;
+
+ state_ = STOPPED;
+}
+
+void ResourcePrefetcher::TryToLaunchPrefetchRequests() {
+ CHECK(state_ == RUNNING || state_ == STOPPED);
+
+ // Try to launch new requests if the state is RUNNING.
+ if (state_ == RUNNING) {
+ bool request_available = true;
+
+ // Loop through the requests while we are under the
+ // max_prefetches_inflight_per_host_per_navigation limit, looking for a URL
+ // for which the max_prefetches_inflight_per_host_per_navigation limit has
+ // not been reached. Try to launch as many requests as possible.
+ while ((inflight_requests_.size() <
+ config_.max_prefetches_inflight_per_navigation) &&
+ request_available) {
+ std::list<Request*>::iterator request_it = request_queue_.begin();
+ for (; request_it != request_queue_.end(); ++request_it) {
+ const std::string& host = (*request_it)->resource_url.host();
+
+ std::map<std::string, size_t>::iterator host_it =
+ host_inflight_counts_.find(host);
+ if (host_it == host_inflight_counts_.end() ||
+ host_it->second <
+ config_.max_prefetches_inflight_per_host_per_navigation)
+ break;
+ }
+ request_available = request_it != request_queue_.end();
+
+ if (request_available) {
+ SendRequest(*request_it);
+ request_queue_.erase(request_it);
+ }
+ }
+ }
+
+ // If the inflight_requests_ is empty, we cant launch any more. Finish.
+ if (inflight_requests_.empty()) {
+ CHECK(host_inflight_counts_.empty());
+ CHECK(request_queue_.empty() || state_ == STOPPED);
+
+ state_ = FINISHED;
+ delegate_->ResourcePrefetcherFinished(this, request_vector_.release());
+ }
+}
+
+void ResourcePrefetcher::SendRequest(Request* request) {
+ request->prefetch_status = Request::PREFETCH_STATUS_STARTED;
+
+ net::URLRequest* url_request =
+ delegate_->GetURLRequestContext()->CreateRequest(
+ request->resource_url, net::LOW, this, NULL).release();
+
+ inflight_requests_[url_request] = request;
+ host_inflight_counts_[url_request->original_url().host()] += 1;
+
+ url_request->set_method("GET");
+ url_request->set_first_party_for_cookies(navigation_id_.main_frame_url);
+ url_request->SetReferrer(navigation_id_.main_frame_url.spec());
+ StartURLRequest(url_request);
+}
+
+void ResourcePrefetcher::StartURLRequest(net::URLRequest* request) {
+ request->Start();
+}
+
+void ResourcePrefetcher::FinishRequest(net::URLRequest* request,
+ Request::PrefetchStatus status) {
+ std::map<net::URLRequest*, Request*>::iterator request_it =
+ inflight_requests_.find(request);
+ CHECK(request_it != inflight_requests_.end());
+
+ const std::string host = request->original_url().host();
+ std::map<std::string, size_t>::iterator host_it = host_inflight_counts_.find(
+ host);
+ CHECK_GT(host_it->second, 0U);
+ host_it->second -= 1;
+ if (host_it->second == 0)
+ host_inflight_counts_.erase(host);
+
+ request_it->second->prefetch_status = status;
+ inflight_requests_.erase(request_it);
+
+ delete request;
+
+ TryToLaunchPrefetchRequests();
+}
+
+void ResourcePrefetcher::ReadFullResponse(net::URLRequest* request) {
+ bool status = true;
+ while (status) {
+ int bytes_read = 0;
+ scoped_refptr<net::IOBuffer> buffer(new net::IOBuffer(
+ kResourceBufferSizeBytes));
+ status = request->Read(buffer.get(), kResourceBufferSizeBytes, &bytes_read);
+
+ if (status) {
+ status = ShouldContinueReadingRequest(request, bytes_read);
+ } else if (request->status().error()) {
+ FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+ return;
+ }
+ }
+}
+
+bool ResourcePrefetcher::ShouldContinueReadingRequest(net::URLRequest* request,
+ int bytes_read) {
+ if (bytes_read == 0) { // When bytes_read == 0, no more data.
+ if (request->was_cached())
+ FinishRequest(request, Request::PREFETCH_STATUS_FROM_CACHE);
+ else
+ FinishRequest(request, Request::PREFETCH_STATUS_FROM_NETWORK);
+ return false;
+ }
+
+ return true;
+}
+
+void ResourcePrefetcher::OnReceivedRedirect(
+ net::URLRequest* request,
+ const net::RedirectInfo& redirect_info,
+ bool* defer_redirect) {
+ FinishRequest(request, Request::PREFETCH_STATUS_REDIRECTED);
+}
+
+void ResourcePrefetcher::OnAuthRequired(net::URLRequest* request,
+ net::AuthChallengeInfo* auth_info) {
+ FinishRequest(request, Request::PREFETCH_STATUS_AUTH_REQUIRED);
+}
+
+void ResourcePrefetcher::OnCertificateRequested(
+ net::URLRequest* request,
+ net::SSLCertRequestInfo* cert_request_info) {
+ FinishRequest(request, Request::PREFETCH_STATUS_CERT_REQUIRED);
+}
+
+void ResourcePrefetcher::OnSSLCertificateError(net::URLRequest* request,
+ const net::SSLInfo& ssl_info,
+ bool fatal) {
+ FinishRequest(request, Request::PREFETCH_STATUS_CERT_ERROR);
+}
+
+void ResourcePrefetcher::OnResponseStarted(net::URLRequest* request) {
+ if (request->status().error()) {
+ FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+ return;
+ }
+
+ // TODO(shishir): Do not read cached entries, or ones that are not cacheable.
+ ReadFullResponse(request);
+}
+
+void ResourcePrefetcher::OnReadCompleted(net::URLRequest* request,
+ int bytes_read) {
+ if (request->status().error()) {
+ FinishRequest(request, Request::PREFETCH_STATUS_FAILED);
+ return;
+ }
+
+ if (ShouldContinueReadingRequest(request, bytes_read))
+ ReadFullResponse(request);
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetcher.h b/chrome/browser/predictors/resource_prefetcher.h
new file mode 100644
index 0000000..18b1bff
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetcher.h
@@ -0,0 +1,165 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_
+
+#include <list>
+#include <map>
+#include <vector>
+
+#include "base/gtest_prod_util.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/memory/scoped_vector.h"
+#include "base/threading/thread_checker.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "net/url_request/redirect_info.h"
+#include "net/url_request/url_request.h"
+#include "url/gurl.h"
+
+namespace net {
+class URLRequestContext;
+}
+
+namespace predictors {
+
+// Responsible for prefetching resources for a single navigation based on the
+// input list of resources.
+// - Limits the max number of resources in flight for any host and also across
+// hosts.
+// - When stopped, will wait for the pending requests to finish.
+// - Lives entirely on the IO thread.
+class ResourcePrefetcher : public net::URLRequest::Delegate {
+ public:
+ // Denotes the prefetch request for a single subresource.
+ struct Request {
+ explicit Request(const GURL& i_resource_url);
+ Request(const Request& other);
+
+ enum PrefetchStatus {
+ PREFETCH_STATUS_NOT_STARTED,
+ PREFETCH_STATUS_STARTED,
+
+ // Cancellation reasons.
+ PREFETCH_STATUS_REDIRECTED,
+ PREFETCH_STATUS_AUTH_REQUIRED,
+ PREFETCH_STATUS_CERT_REQUIRED,
+ PREFETCH_STATUS_CERT_ERROR,
+ PREFETCH_STATUS_CANCELLED,
+ PREFETCH_STATUS_FAILED,
+
+ // Successful prefetch states.
+ PREFETCH_STATUS_FROM_CACHE,
+ PREFETCH_STATUS_FROM_NETWORK
+ };
+
+ enum UsageStatus {
+ USAGE_STATUS_NOT_REQUESTED,
+ USAGE_STATUS_FROM_CACHE,
+ USAGE_STATUS_FROM_NETWORK,
+ USAGE_STATUS_NAVIGATION_ABANDONED
+ };
+
+ GURL resource_url;
+ PrefetchStatus prefetch_status;
+ UsageStatus usage_status;
+ };
+ typedef ScopedVector<Request> RequestVector;
+
+ // Used to communicate when the prefetching is done. All methods are invoked
+ // on the IO thread.
+ class Delegate {
+ public:
+ virtual ~Delegate() { }
+
+ // Called when the ResourcePrefetcher is finished, i.e. there is nothing
+ // pending in flight. Should take ownership of |requests|.
+ virtual void ResourcePrefetcherFinished(
+ ResourcePrefetcher* prefetcher,
+ RequestVector* requests) = 0;
+
+ virtual net::URLRequestContext* GetURLRequestContext() = 0;
+ };
+
+ // |delegate| has to outlive the ResourcePrefetcher. The ResourcePrefetcher
+ // takes ownership of |requests|.
+ ResourcePrefetcher(Delegate* delegate,
+ const ResourcePrefetchPredictorConfig& config,
+ const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ scoped_ptr<RequestVector> requests);
+ virtual ~ResourcePrefetcher();
+
+ void Start(); // Kicks off the prefetching. Can only be called once.
+ void Stop(); // No additional prefetches will be queued after this.
+
+ const NavigationID& navigation_id() const { return navigation_id_; }
+ PrefetchKeyType key_type() const { return key_type_; }
+
+ private:
+ friend class ResourcePrefetcherTest;
+ friend class TestResourcePrefetcher;
+
+ // Launches new prefetch requests if possible.
+ void TryToLaunchPrefetchRequests();
+
+ // Starts a net::URLRequest for the input |request|.
+ void SendRequest(Request* request);
+
+ // Called by |SendRequest| to start the |request|. This is necessary to stub
+ // out the Start() call to net::URLRequest for unittesting.
+ virtual void StartURLRequest(net::URLRequest* request);
+
+ // Marks the request as finished, with the given status.
+ void FinishRequest(net::URLRequest* request, Request::PrefetchStatus status);
+
+ // Reads the response data from the response - required for the resource to
+ // be cached correctly. Stubbed out during testing.
+ virtual void ReadFullResponse(net::URLRequest* request);
+
+ // Returns true if the request has more data that needs to be read. If it
+ // returns false, the request should not be referenced again.
+ bool ShouldContinueReadingRequest(net::URLRequest* request, int bytes_read);
+
+ // net::URLRequest::Delegate methods.
+ virtual void OnReceivedRedirect(net::URLRequest* request,
+ const net::RedirectInfo& redirect_info,
+ bool* defer_redirect) OVERRIDE;
+ virtual void OnAuthRequired(net::URLRequest* request,
+ net::AuthChallengeInfo* auth_info) OVERRIDE;
+ virtual void OnCertificateRequested(
+ net::URLRequest* request,
+ net::SSLCertRequestInfo* cert_request_info) OVERRIDE;
+ virtual void OnSSLCertificateError(net::URLRequest* request,
+ const net::SSLInfo& ssl_info,
+ bool fatal) OVERRIDE;
+ virtual void OnResponseStarted(net::URLRequest* request) OVERRIDE;
+ virtual void OnReadCompleted(net::URLRequest* request,
+ int bytes_read) OVERRIDE;
+
+ enum PrefetcherState {
+ INITIALIZED = 0, // Prefetching hasn't started.
+ RUNNING = 1, // Prefetching started, allowed to add more requests.
+ STOPPED = 2, // Prefetching started, not allowed to add more requests.
+ FINISHED = 3 // No more inflight request, new requests not possible.
+ };
+
+ base::ThreadChecker thread_checker_;
+ PrefetcherState state_;
+ Delegate* const delegate_;
+ ResourcePrefetchPredictorConfig const config_;
+ NavigationID navigation_id_;
+ PrefetchKeyType key_type_;
+ scoped_ptr<RequestVector> request_vector_;
+
+ std::map<net::URLRequest*, Request*> inflight_requests_;
+ std::list<Request*> request_queue_;
+ std::map<std::string, size_t> host_inflight_counts_;
+
+ DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcher);
+};
+
+} // namespace predictors
+
+#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_
diff --git a/chrome/browser/predictors/resource_prefetcher_manager.cc b/chrome/browser/predictors/resource_prefetcher_manager.cc
new file mode 100644
index 0000000..15d1b76
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetcher_manager.cc
@@ -0,0 +1,135 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/predictors/resource_prefetcher_manager.h"
+
+#include "base/bind.h"
+#include "base/stl_util.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/url_request/url_request.h"
+#include "net/url_request/url_request_context_getter.h"
+
+using content::BrowserThread;
+
+namespace predictors {
+
+ResourcePrefetcherManager::ResourcePrefetcherManager(
+ ResourcePrefetchPredictor* predictor,
+ const ResourcePrefetchPredictorConfig& config,
+ net::URLRequestContextGetter* context_getter)
+ : predictor_(predictor),
+ config_(config),
+ context_getter_(context_getter) {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+ CHECK(predictor_);
+ CHECK(context_getter_);
+}
+
+ResourcePrefetcherManager::~ResourcePrefetcherManager() {
+ DCHECK(prefetcher_map_.empty())
+ << "Did not call ShutdownOnUIThread or ShutdownOnIOThread. "
+ " Will leak Prefetcher pointers.";
+}
+
+void ResourcePrefetcherManager::ShutdownOnUIThread() {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+ predictor_ = NULL;
+ BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
+ base::Bind(&ResourcePrefetcherManager::ShutdownOnIOThread,
+ this));
+}
+
+void ResourcePrefetcherManager::ShutdownOnIOThread() {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+ STLDeleteContainerPairSecondPointers(prefetcher_map_.begin(),
+ prefetcher_map_.end());
+}
+
+void ResourcePrefetcherManager::MaybeAddPrefetch(
+ const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests) {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+
+ // Don't add a duplicate prefetch for the same host or URL.
+ std::string key = key_type == PREFETCH_KEY_TYPE_HOST ?
+ navigation_id.main_frame_url.host() : navigation_id.main_frame_url.spec();
+ if (ContainsKey(prefetcher_map_, key))
+ return;
+
+ ResourcePrefetcher* prefetcher = new ResourcePrefetcher(
+ this, config_, navigation_id, key_type, requests.Pass());
+ prefetcher_map_.insert(std::make_pair(key, prefetcher));
+ prefetcher->Start();
+}
+
+void ResourcePrefetcherManager::MaybeRemovePrefetch(
+ const NavigationID& navigation_id) {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+
+ // Look for a URL based prefetch first.
+ PrefetcherMap::iterator it = prefetcher_map_.find(
+ navigation_id.main_frame_url.spec());
+ if (it != prefetcher_map_.end() &&
+ it->second->navigation_id() == navigation_id) {
+ it->second->Stop();
+ return;
+ }
+
+ // No URL based prefetching, look for host based.
+ it = prefetcher_map_.find(navigation_id.main_frame_url.host());
+ if (it != prefetcher_map_.end() &&
+ it->second->navigation_id() == navigation_id) {
+ it->second->Stop();
+ }
+}
+
+void ResourcePrefetcherManager::ResourcePrefetcherFinished(
+ ResourcePrefetcher* resource_prefetcher,
+ ResourcePrefetcher::RequestVector* requests) {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+
+ // |predictor_| can only be accessed from the UI thread.
+ scoped_ptr<ResourcePrefetcher::RequestVector> scoped_requests(requests);
+ BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
+ base::Bind(&ResourcePrefetcherManager::ResourcePrefetcherFinishedOnUI,
+ this,
+ resource_prefetcher->navigation_id(),
+ resource_prefetcher->key_type(),
+ base::Passed(&scoped_requests)));
+
+ const GURL& main_frame_url =
+ resource_prefetcher->navigation_id().main_frame_url;
+ const std::string key =
+ resource_prefetcher->key_type() == PREFETCH_KEY_TYPE_HOST ?
+ main_frame_url.host() : main_frame_url.spec();
+ PrefetcherMap::iterator it = prefetcher_map_.find(key);
+ DCHECK(it != prefetcher_map_.end());
+ delete it->second;
+ prefetcher_map_.erase(it);
+}
+
+void ResourcePrefetcherManager::ResourcePrefetcherFinishedOnUI(
+ const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests) {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+ // |predictor_| may have been set to NULL if the predictor is shutting down.
+ if (predictor_) {
+ predictor_->FinishedPrefetchForNavigation(navigation_id,
+ key_type,
+ requests.release());
+ }
+}
+
+net::URLRequestContext* ResourcePrefetcherManager::GetURLRequestContext() {
+ DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+
+ return context_getter_->GetURLRequestContext();
+}
+
+} // namespace predictors
diff --git a/chrome/browser/predictors/resource_prefetcher_manager.h b/chrome/browser/predictors/resource_prefetcher_manager.h
new file mode 100644
index 0000000..91b23ed
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetcher_manager.h
@@ -0,0 +1,87 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_
+#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_
+
+#include <map>
+
+#include "base/memory/ref_counted.h"
+#include "chrome/browser/predictors/resource_prefetch_common.h"
+#include "chrome/browser/predictors/resource_prefetcher.h"
+
+namespace net {
+class URLRequestContextGetter;
+}
+
+namespace predictors {
+
+struct NavigationID;
+class ResourcePrefetchPredictor;
+
+// Manages prefetches for multple navigations.
+// - Created and owned by the resource prefetch predictor.
+// - Needs to be refcounted as it is de-referenced on two different threads.
+// - Created on the UI thread, but most functions are called in the IO thread.
+// - Will only allow one inflight prefresh per main frame URL.
+class ResourcePrefetcherManager
+ : public ResourcePrefetcher::Delegate,
+ public base::RefCountedThreadSafe<ResourcePrefetcherManager> {
+ public:
+ // The |predictor| should be alive till ShutdownOnIOThread is called.
+ ResourcePrefetcherManager(ResourcePrefetchPredictor* predictor,
+ const ResourcePrefetchPredictorConfig& config,
+ net::URLRequestContextGetter* getter);
+
+ // UI thread.
+ void ShutdownOnUIThread();
+
+ // --- IO Thread methods.
+
+ // The prefetchers need to be deleted on the IO thread.
+ void ShutdownOnIOThread();
+
+ // Will create a new ResourcePrefetcher for the main frame url of the input
+ // navigation if there isn't one already for the same URL or host (for host
+ // based).
+ void MaybeAddPrefetch(const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests);
+
+ // Stops the ResourcePrefetcher for the input navigation, if one was in
+ // progress.
+ void MaybeRemovePrefetch(const NavigationID& navigation_id);
+
+ // ResourcePrefetcher::Delegate methods.
+ virtual void ResourcePrefetcherFinished(
+ ResourcePrefetcher* prefetcher,
+ ResourcePrefetcher::RequestVector* requests) OVERRIDE;
+ virtual net::URLRequestContext* GetURLRequestContext() OVERRIDE;
+
+ private:
+ friend class base::RefCountedThreadSafe<ResourcePrefetcherManager>;
+ friend class MockResourcePrefetcherManager;
+
+ typedef std::map<std::string, ResourcePrefetcher*> PrefetcherMap;
+
+ virtual ~ResourcePrefetcherManager();
+
+ // UI Thread. |predictor_| needs to be called on the UI thread.
+ void ResourcePrefetcherFinishedOnUI(
+ const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests);
+
+ ResourcePrefetchPredictor* predictor_;
+ const ResourcePrefetchPredictorConfig config_;
+ net::URLRequestContextGetter* const context_getter_;
+
+ PrefetcherMap prefetcher_map_; // Owns the ResourcePrefetcher pointers.
+
+ DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcherManager);
+};
+
+} // namespace predictors
+
+#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_
diff --git a/chrome/browser/predictors/resource_prefetcher_unittest.cc b/chrome/browser/predictors/resource_prefetcher_unittest.cc
new file mode 100644
index 0000000..bc21f89
--- /dev/null
+++ b/chrome/browser/predictors/resource_prefetcher_unittest.cc
@@ -0,0 +1,343 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/message_loop/message_loop.h"
+#include "chrome/browser/predictors/resource_prefetcher.h"
+#include "chrome/browser/predictors/resource_prefetcher_manager.h"
+#include "chrome/test/base/testing_profile.h"
+#include "content/public/test/test_browser_thread.h"
+#include "net/url_request/redirect_info.h"
+#include "net/url_request/url_request.h"
+#include "net/url_request/url_request_test_util.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using testing::Eq;
+using testing::Property;
+
+namespace predictors {
+
+// Wrapper over the ResourcePrefetcher that stubs out the StartURLRequest call
+// since we do not want to do network fetches in this unittest.
+class TestResourcePrefetcher : public ResourcePrefetcher {
+ public:
+ TestResourcePrefetcher(ResourcePrefetcher::Delegate* delegate,
+ const ResourcePrefetchPredictorConfig& config,
+ const NavigationID& navigation_id,
+ PrefetchKeyType key_type,
+ scoped_ptr<RequestVector> requests)
+ : ResourcePrefetcher(delegate, config, navigation_id,
+ key_type, requests.Pass()) { }
+
+ virtual ~TestResourcePrefetcher() { }
+
+ MOCK_METHOD1(StartURLRequest, void(net::URLRequest* request));
+
+ void ReadFullResponse(net::URLRequest* request) OVERRIDE {
+ FinishRequest(request, Request::PREFETCH_STATUS_FROM_CACHE);
+ }
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(TestResourcePrefetcher);
+};
+
+
+// Delegate for ResourcePrefetcher.
+class TestResourcePrefetcherDelegate : public ResourcePrefetcher::Delegate {
+ public:
+ explicit TestResourcePrefetcherDelegate(base::MessageLoop* loop)
+ : request_context_getter_(new net::TestURLRequestContextGetter(
+ loop->message_loop_proxy())) { }
+ ~TestResourcePrefetcherDelegate() { }
+
+ virtual net::URLRequestContext* GetURLRequestContext() OVERRIDE {
+ return request_context_getter_->GetURLRequestContext();
+ }
+
+ MOCK_METHOD2(ResourcePrefetcherFinished,
+ void(ResourcePrefetcher* prefetcher,
+ ResourcePrefetcher::RequestVector* requests));
+
+ private:
+ scoped_refptr<net::TestURLRequestContextGetter> request_context_getter_;
+
+ DISALLOW_COPY_AND_ASSIGN(TestResourcePrefetcherDelegate);
+};
+
+
+// The following unittest tests most of the ResourcePrefetcher except for:
+// 1. Call to ReadFullResponse. There does not seem to be a good way to test the
+// function in a unittest, and probably requires a browser_test.
+// 2. Setting of the Prefetch status for cache vs non cache.
+class ResourcePrefetcherTest : public testing::Test {
+ public:
+ ResourcePrefetcherTest();
+ virtual ~ResourcePrefetcherTest();
+
+ protected:
+ typedef ResourcePrefetcher::Request Request;
+
+ void AddStartUrlRequestExpectation(const std::string& url) {
+ EXPECT_CALL(*prefetcher_,
+ StartURLRequest(Property(&net::URLRequest::original_url,
+ Eq(GURL(url)))));
+ }
+
+ void CheckPrefetcherState(size_t inflight, size_t queue, size_t host) {
+ EXPECT_EQ(prefetcher_->inflight_requests_.size(), inflight);
+ EXPECT_EQ(prefetcher_->request_queue_.size(), queue);
+ EXPECT_EQ(prefetcher_->host_inflight_counts_.size(), host);
+ }
+
+ net::URLRequest* GetInFlightRequest(const std::string& url_str) {
+ GURL url(url_str);
+
+ for (std::list<Request*>::const_iterator it =
+ prefetcher_->request_queue_.begin();
+ it != prefetcher_->request_queue_.end(); ++it) {
+ EXPECT_NE((*it)->resource_url, url);
+ }
+ for (std::map<net::URLRequest*, Request*>::const_iterator it =
+ prefetcher_->inflight_requests_.begin();
+ it != prefetcher_->inflight_requests_.end(); ++it) {
+ if (it->first->original_url() == url)
+ return it->first;
+ }
+ EXPECT_TRUE(false) << "Infligh request not found: " << url_str;
+ return NULL;
+ }
+
+
+ void OnReceivedRedirect(const std::string& url) {
+ prefetcher_->OnReceivedRedirect(
+ GetInFlightRequest(url), net::RedirectInfo(), NULL);
+ }
+ void OnAuthRequired(const std::string& url) {
+ prefetcher_->OnAuthRequired(GetInFlightRequest(url), NULL);
+ }
+ void OnCertificateRequested(const std::string& url) {
+ prefetcher_->OnCertificateRequested(GetInFlightRequest(url), NULL);
+ }
+ void OnSSLCertificateError(const std::string& url) {
+ prefetcher_->OnSSLCertificateError(GetInFlightRequest(url),
+ net::SSLInfo(), false);
+ }
+ void OnResponse(const std::string& url) {
+ prefetcher_->OnResponseStarted(GetInFlightRequest(url));
+ }
+
+ base::MessageLoop loop_;
+ content::TestBrowserThread io_thread_;
+ ResourcePrefetchPredictorConfig config_;
+ TestResourcePrefetcherDelegate prefetcher_delegate_;
+ scoped_ptr<TestResourcePrefetcher> prefetcher_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcherTest);
+};
+
+ResourcePrefetcherTest::ResourcePrefetcherTest()
+ : loop_(base::MessageLoop::TYPE_IO),
+ io_thread_(content::BrowserThread::IO, &loop_),
+ prefetcher_delegate_(&loop_) {
+ config_.max_prefetches_inflight_per_navigation = 5;
+ config_.max_prefetches_inflight_per_host_per_navigation = 2;
+}
+
+ResourcePrefetcherTest::~ResourcePrefetcherTest() {
+}
+
+TEST_F(ResourcePrefetcherTest, TestPrefetcherFinishes) {
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests(
+ new ResourcePrefetcher::RequestVector);
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://www.google.com/resource1.html")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://www.google.com/resource2.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://yahoo.com/resource1.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://yahoo.com/resource2.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://yahoo.com/resource3.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://m.google.com/resource1.jpg")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://www.google.com/resource3.html")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://m.google.com/resource2.html")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://m.google.com/resource3.css")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://m.google.com/resource4.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://yahoo.com/resource4.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://yahoo.com/resource5.png")));
+
+ NavigationID navigation_id;
+ navigation_id.render_process_id = 1;
+ navigation_id.render_frame_id = 2;
+ navigation_id.main_frame_url = GURL("http://www.google.com");
+
+ // Needed later for comparison.
+ ResourcePrefetcher::RequestVector* requests_ptr = requests.get();
+
+ prefetcher_.reset(new TestResourcePrefetcher(&prefetcher_delegate_,
+ config_,
+ navigation_id,
+ PREFETCH_KEY_TYPE_URL,
+ requests.Pass()));
+
+ // Starting the prefetcher maxes out the number of possible requests.
+ AddStartUrlRequestExpectation("http://www.google.com/resource1.html");
+ AddStartUrlRequestExpectation("http://www.google.com/resource2.png");
+ AddStartUrlRequestExpectation("http://yahoo.com/resource1.png");
+ AddStartUrlRequestExpectation("http://yahoo.com/resource2.png");
+ AddStartUrlRequestExpectation("http://m.google.com/resource1.jpg");
+
+ prefetcher_->Start();
+ CheckPrefetcherState(5, 7, 3);
+
+ AddStartUrlRequestExpectation("http://m.google.com/resource2.html");
+ OnResponse("http://m.google.com/resource1.jpg");
+ CheckPrefetcherState(5, 6, 3);
+
+ AddStartUrlRequestExpectation("http://www.google.com/resource3.html");
+ OnSSLCertificateError("http://www.google.com/resource1.html");
+ CheckPrefetcherState(5, 5, 3);
+
+ AddStartUrlRequestExpectation("http://m.google.com/resource3.css");
+ OnResponse("http://m.google.com/resource2.html");
+ CheckPrefetcherState(5, 4, 3);
+
+ AddStartUrlRequestExpectation("http://m.google.com/resource4.png");
+ OnReceivedRedirect("http://www.google.com/resource3.html");
+ CheckPrefetcherState(5, 3, 3);
+
+ OnResponse("http://www.google.com/resource2.png");
+ CheckPrefetcherState(4, 3, 2);
+
+ AddStartUrlRequestExpectation("http://yahoo.com/resource3.png");
+ OnReceivedRedirect("http://yahoo.com/resource2.png");
+ CheckPrefetcherState(4, 2, 2);
+
+ AddStartUrlRequestExpectation("http://yahoo.com/resource4.png");
+ OnResponse("http://yahoo.com/resource1.png");
+ CheckPrefetcherState(4, 1, 2);
+
+ AddStartUrlRequestExpectation("http://yahoo.com/resource5.png");
+ OnResponse("http://yahoo.com/resource4.png");
+ CheckPrefetcherState(4, 0, 2);
+
+ OnResponse("http://yahoo.com/resource5.png");
+ CheckPrefetcherState(3, 0, 2);
+
+ OnCertificateRequested("http://m.google.com/resource4.png");
+ CheckPrefetcherState(2, 0, 2);
+
+ OnAuthRequired("http://m.google.com/resource3.css");
+ CheckPrefetcherState(1, 0, 1);
+
+ // Expect the final call.
+ EXPECT_CALL(prefetcher_delegate_,
+ ResourcePrefetcherFinished(Eq(prefetcher_.get()),
+ Eq(requests_ptr)));
+
+ OnResponse("http://yahoo.com/resource3.png");
+ CheckPrefetcherState(0, 0, 0);
+
+ // Check the prefetch status.
+ EXPECT_EQ(Request::PREFETCH_STATUS_CERT_ERROR,
+ (*requests_ptr)[0]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE,
+ (*requests_ptr)[1]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE,
+ (*requests_ptr)[2]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_REDIRECTED,
+ (*requests_ptr)[3]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE,
+ (*requests_ptr)[4]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE,
+ (*requests_ptr)[5]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_REDIRECTED,
+ (*requests_ptr)[6]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE,
+ (*requests_ptr)[7]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_AUTH_REQUIRED,
+ (*requests_ptr)[8]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_CERT_REQUIRED,
+ (*requests_ptr)[9]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE,
+ (*requests_ptr)[10]->prefetch_status);
+ EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE,
+ (*requests_ptr)[11]->prefetch_status);
+}
+
+TEST_F(ResourcePrefetcherTest, TestPrefetcherStopped) {
+ scoped_ptr<ResourcePrefetcher::RequestVector> requests(
+ new ResourcePrefetcher::RequestVector);
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://www.google.com/resource1.html")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://www.google.com/resource2.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://yahoo.com/resource1.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://yahoo.com/resource2.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://yahoo.com/resource3.png")));
+ requests->push_back(new ResourcePrefetcher::Request(GURL(
+ "http://m.google.com/resource1.jpg")));
+
+ NavigationID navigation_id;
+ navigation_id.render_process_id = 1;
+ navigation_id.render_frame_id = 2;
+ navigation_id.main_frame_url = GURL("http://www.google.com");
+
+ // Needed later for comparison.
+ ResourcePrefetcher::RequestVector* requests_ptr = requests.get();
+
+ prefetcher_.reset(new TestResourcePrefetcher(&prefetcher_delegate_,
+ config_,
+ navigation_id,
+ PREFETCH_KEY_TYPE_HOST,
+ requests.Pass()));
+
+ // Starting the prefetcher maxes out the number of possible requests.
+ AddStartUrlRequestExpectation("http://www.google.com/resource1.html");
+ AddStartUrlRequestExpectation("http://www.google.com/resource2.png");
+ AddStartUrlRequestExpectation("http://yahoo.com/resource1.png");
+ AddStartUrlRequestExpectation("http://yahoo.com/resource2.png");
+ AddStartUrlRequestExpectation("http://m.google.com/resource1.jpg");
+
+ prefetcher_->Start();
+ CheckPrefetcherState(5, 1, 3);
+
+ OnResponse("http://www.google.com/resource1.html");
+ CheckPrefetcherState(4, 1, 3);
+
+ prefetcher_->Stop(); // No more queueing.
+
+ OnResponse("http://www.google.com/resource2.png");
+ CheckPrefetcherState(3, 1, 2);
+
+ OnResponse("http://yahoo.com/resource1.png");
+ CheckPrefetcherState(2, 1, 2);
+
+ OnResponse("http://yahoo.com/resource2.png");
+ CheckPrefetcherState(1, 1, 1);
+
+ // Expect the final call.
+ EXPECT_CALL(prefetcher_delegate_,
+ ResourcePrefetcherFinished(Eq(prefetcher_.get()),
+ Eq(requests_ptr)));
+
+ OnResponse("http://m.google.com/resource1.jpg");
+ CheckPrefetcherState(0, 1, 0);
+}
+
+} // namespace predictors
diff --git a/chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc b/chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc
index 40fc4b6..211502c6 100644
--- a/chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc
+++ b/chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc
@@ -67,6 +67,13 @@ IN_PROC_BROWSER_TEST_F(CloudPrintPolicyTest, NormalPassedFlag) {
IN_PROC_BROWSER_TEST_F(CloudPrintPolicyTest, DISABLED_CloudPrintPolicyFlag) {
CommandLine new_command_line(GetCommandLineForRelaunch());
new_command_line.AppendSwitch(switches::kCheckCloudPrintConnectorPolicy);
+ // This is important for the test as the way the browser process is launched
+ // here causes the predictor databases to be initialized multiple times. This
+ // is not an issue for production where the process is launched as a service
+ // and a Profile is not created. See http://crbug.com/140466 for more details.
+ new_command_line.AppendSwitchASCII(
+ switches::kSpeculativeResourcePrefetching,
+ switches::kSpeculativeResourcePrefetchingDisabled);
base::ProcessHandle handle;
bool launched =
diff --git a/chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc b/chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc
index a31257b..3a36d23 100644
--- a/chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc
+++ b/chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc
@@ -27,6 +27,7 @@
#include "chrome/browser/policy/profile_policy_connector_factory.h"
#include "chrome/browser/predictors/autocomplete_action_predictor_factory.h"
#include "chrome/browser/predictors/predictor_database_factory.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
#include "chrome/browser/prerender/prerender_link_manager_factory.h"
#include "chrome/browser/prerender/prerender_manager_factory.h"
#include "chrome/browser/printing/cloud_print/cloud_print_proxy_service_factory.h"
@@ -239,6 +240,7 @@ EnsureBrowserContextKeyedServiceFactoriesBuilt() {
#endif
predictors::AutocompleteActionPredictorFactory::GetInstance();
predictors::PredictorDatabaseFactory::GetInstance();
+ predictors::ResourcePrefetchPredictorFactory::GetInstance();
prerender::PrerenderManagerFactory::GetInstance();
prerender::PrerenderLinkManagerFactory::GetInstance();
ProfileSyncServiceFactory::GetInstance();
diff --git a/chrome/browser/profiles/profile_io_data.cc b/chrome/browser/profiles/profile_io_data.cc
index c61fda3..5608a36 100644
--- a/chrome/browser/profiles/profile_io_data.cc
+++ b/chrome/browser/profiles/profile_io_data.cc
@@ -42,6 +42,9 @@
#include "chrome/browser/net/chrome_url_request_context_getter.h"
#include "chrome/browser/net/cookie_store_util.h"
#include "chrome/browser/net/proxy_service_factory.h"
+#include "chrome/browser/net/resource_prefetch_predictor_observer.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/profiles/profile_manager.h"
#include "chrome/browser/signin/signin_names_io_thread.h"
@@ -357,6 +360,13 @@ void ProfileIOData::InitializeOnUIThread(Profile* profile) {
extensions::ExtensionSystem::Get(profile)->info_map();
#endif
+ if (predictors::ResourcePrefetchPredictor* predictor =
+ predictors::ResourcePrefetchPredictorFactory::GetForProfile(
+ profile)) {
+ resource_prefetch_predictor_observer_.reset(
+ new chrome_browser_net::ResourcePrefetchPredictorObserver(predictor));
+ }
+
ProtocolHandlerRegistry* protocol_handler_registry =
ProtocolHandlerRegistryFactory::GetForBrowserContext(profile);
DCHECK(protocol_handler_registry);
@@ -1054,6 +1064,11 @@ void ProfileIOData::Init(
resource_context_->host_resolver_ = io_thread_globals->host_resolver.get();
resource_context_->request_context_ = main_request_context_.get();
+ if (profile_params_->resource_prefetch_predictor_observer_) {
+ resource_prefetch_predictor_observer_.reset(
+ profile_params_->resource_prefetch_predictor_observer_.release());
+ }
+
#if defined(ENABLE_MANAGED_USERS)
supervised_user_url_filter_ = profile_params_->supervised_user_url_filter;
#endif
diff --git a/chrome/browser/profiles/profile_io_data.h b/chrome/browser/profiles/profile_io_data.h
index 34fcb64..ad9aaee 100644
--- a/chrome/browser/profiles/profile_io_data.h
+++ b/chrome/browser/profiles/profile_io_data.h
@@ -42,6 +42,10 @@ class ProtocolHandlerRegistry;
class SigninNamesOnIOThread;
class SupervisedUserURLFilter;
+namespace chrome_browser_net {
+class ResourcePrefetchPredictorObserver;
+}
+
namespace extensions {
class InfoMap;
}
@@ -228,6 +232,11 @@ class ProfileIOData {
return &incognito_availibility_pref_;
}
+ chrome_browser_net::ResourcePrefetchPredictorObserver*
+ resource_prefetch_predictor_observer() const {
+ return resource_prefetch_predictor_observer_.get();
+ }
+
#if defined(ENABLE_CONFIGURATION_POLICY)
policy::PolicyHeaderIOHelper* policy_header_helper() const {
return policy_header_helper_.get();
@@ -310,6 +319,8 @@ class ProfileIOData {
#if defined(ENABLE_EXTENSIONS)
scoped_refptr<extensions::InfoMap> extension_info_map;
#endif
+ scoped_ptr<chrome_browser_net::ResourcePrefetchPredictorObserver>
+ resource_prefetch_predictor_observer_;
// This pointer exists only as a means of conveying a url job factory
// pointer from the protocol handler registry on the UI thread to the
@@ -601,6 +612,9 @@ class ProfileIOData {
mutable scoped_refptr<HostContentSettingsMap> host_content_settings_map_;
+ mutable scoped_ptr<chrome_browser_net::ResourcePrefetchPredictorObserver>
+ resource_prefetch_predictor_observer_;
+
mutable scoped_ptr<ChromeHttpUserAgentSettings>
chrome_http_user_agent_settings_;
diff --git a/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc b/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc
index c3c6480..5a724f2 100644
--- a/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc
+++ b/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc
@@ -16,6 +16,7 @@
#include "chrome/browser/content_settings/host_content_settings_map.h"
#include "chrome/browser/download/download_request_limiter.h"
#include "chrome/browser/download/download_resource_throttle.h"
+#include "chrome/browser/net/resource_prefetch_predictor_observer.h"
#include "chrome/browser/prefetch/prefetch.h"
#include "chrome/browser/prerender/prerender_manager.h"
#include "chrome/browser/prerender/prerender_manager_factory.h"
@@ -410,6 +411,11 @@ void ChromeResourceDispatcherHostDelegate::RequestBeginning(
throttles);
}
#endif
+
+ if (io_data->resource_prefetch_predictor_observer()) {
+ io_data->resource_prefetch_predictor_observer()->OnRequestStarted(
+ request, resource_type, info->GetChildID(), info->GetRenderFrameID());
+ }
}
void ChromeResourceDispatcherHostDelegate::DownloadStarting(
@@ -671,6 +677,9 @@ void ChromeResourceDispatcherHostDelegate::OnResponseStarted(
}
}
+ if (io_data->resource_prefetch_predictor_observer())
+ io_data->resource_prefetch_predictor_observer()->OnResponseStarted(request);
+
// Ignores x-frame-options for the chrome signin UI.
const std::string request_spec(
request->first_party_for_cookies().GetOrigin().spec());
@@ -714,6 +723,11 @@ void ChromeResourceDispatcherHostDelegate::OnRequestRedirected(
// management UI is built on top of it.
signin::AppendMirrorRequestHeaderIfPossible(request, redirect_url, io_data);
+ if (io_data->resource_prefetch_predictor_observer()) {
+ io_data->resource_prefetch_predictor_observer()->OnRequestRedirected(
+ redirect_url, request);
+ }
+
#if defined(ENABLE_CONFIGURATION_POLICY)
if (io_data->policy_header_helper())
io_data->policy_header_helper()->AddPolicyHeaders(redirect_url, request);
diff --git a/chrome/browser/resources/predictors/predictors.html b/chrome/browser/resources/predictors/predictors.html
index 83a64ed..01ba3b0 100644
--- a/chrome/browser/resources/predictors/predictors.html
+++ b/chrome/browser/resources/predictors/predictors.html
@@ -23,11 +23,15 @@
<tabbox id="predictor-page">
<tabs>
<tab>Autocomplete Action Predictor</tab>
+ <tab>Resource Prefetch Predictor</tab>
</tabs>
<tabpanels>
<tabpanel>
<include src="autocomplete_action_predictor.html" />
</tabpanel>
+ <tabpanel>
+ <include src="resource_prefetch_predictor.html" />
+ </tabpanel>
</tabpanels>
</tabbox>
<script src="chrome://resources/js/jstemplate_compiled.js"></script>
diff --git a/chrome/browser/resources/predictors/predictors.js b/chrome/browser/resources/predictors/predictors.js
index b9ea1f6..ef5033c 100644
--- a/chrome/browser/resources/predictors/predictors.js
+++ b/chrome/browser/resources/predictors/predictors.js
@@ -3,6 +3,7 @@
// found in the LICENSE file.
<include src="autocomplete_action_predictor.js"/>
+<include src="resource_prefetch_predictor.js"/>
if (cr.isWindows)
document.documentElement.setAttribute('os', 'win');
diff --git a/chrome/browser/resources/predictors/resource_prefetch_predictor.html b/chrome/browser/resources/predictors/resource_prefetch_predictor.html
new file mode 100644
index 0000000..36903fa
--- /dev/null
+++ b/chrome/browser/resources/predictors/resource_prefetch_predictor.html
@@ -0,0 +1,49 @@
+<div id='rpp_enabled'>
+ <tabbox id="rpp_data">
+ <tabs>
+ <tab>URL Table Cache</tab>
+ <tab>Host Table Cache</tab>
+ </tabs>
+ <tabpanels>
+ <tabpanel>
+ <table id="">
+ <thead>
+ <tr>
+ <th>Main Frame Url</th>
+ <th>Resource Url</th>
+ <th>Resource Type</th>
+ <th>Num Hits</th>
+ <th>Num Misses</th>
+ <th>Consec Misses</th>
+ <th>Average Position</th>
+ <th>Score</th>
+ </tr>
+ </thead>
+ <tbody id="rpp_url_body">
+ </tbody>
+ </table>
+ </tabpanel>
+ <tabpanel>
+ <table id="">
+ <thead>
+ <tr>
+ <th>Host</th>
+ <th>Resource Url</th>
+ <th>Resource Type</th>
+ <th>Num Hits</th>
+ <th>Num Misses</th>
+ <th>Consec Misses</th>
+ <th>Average Position</th>
+ <th>Score</th>
+ </tr>
+ </thead>
+ <tbody id="rpp_host_body">
+ </tbody>
+ </table>
+ </tabpanel>
+ </tabpanels>
+ </tabbox>
+</div>
+<div id='rpp_disabled'>
+ Resource prefetch prediction is disabled.
+</div> \ No newline at end of file
diff --git a/chrome/browser/resources/predictors/resource_prefetch_predictor.js b/chrome/browser/resources/predictors/resource_prefetch_predictor.js
new file mode 100644
index 0000000..f725105
--- /dev/null
+++ b/chrome/browser/resources/predictors/resource_prefetch_predictor.js
@@ -0,0 +1,102 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/**
+ * Requests the database from the backend.
+ */
+function requestResourcePrefetchPredictorDb() {
+ chrome.send('requestResourcePrefetchPredictorDb');
+}
+
+/**
+ * Callback from backend with the database contents. Sets up some globals and
+ * calls to create the UI.
+ * @param {Dictionary} database Information about ResourcePrefetchPredictor
+ * including the database as a flattened list, a boolean indicating if the
+ * system is enabled.
+ */
+function updateResourcePrefetchPredictorDb(database) {
+ updateResourcePrefetchPredictorDbView(database);
+}
+
+/**
+ * Truncates the string to keep the database readable.
+ * @param {string} str The string to truncate.
+ * @return {string} The truncated string.
+ */
+function truncateString(str) {
+ return str.length < 100 ? str : str.substring(0, 99);
+}
+
+/**
+ * Updates the table from the database.
+ * @param {Dictionary} database Information about ResourcePrefetchPredictor
+ * including the database as a flattened list, a boolean indicating if the
+ * system is enabled and the current hit weight.
+ */
+function updateResourcePrefetchPredictorDbView(database) {
+ if (!database.enabled) {
+ $('rpp_enabled').style.display = 'none';
+ $('rpp_disabled').style.display = 'block';
+ return;
+ } else {
+ $('rpp_enabled').style.display = 'block';
+ $('rpp_disabled').style.display = 'none';
+ }
+
+ var hasUrlData = database.url_db && database.url_db.length > 0;
+ var hasHostData = database.host_db && database.host_db.length > 0;
+
+ if (hasUrlData)
+ renderCacheData($('rpp_url_body'), database.url_db);
+ if (hasHostData)
+ renderCacheData($('rpp_host_body'), database.host_db);
+}
+
+/**
+ * Renders cache data for URL or host based data.
+ * @param {HTMLElement} body element of table to render into.
+ * @param {Dictionary} database to render.
+ */
+function renderCacheData(body, database) {
+ body.textContent = '';
+ for (var i = 0; i < database.length; ++i) {
+ var main = database[i];
+
+ for (var j = 0; j < main.resources.length; ++j) {
+ var resource = main.resources[j];
+ var row = document.createElement('tr');
+
+ if (j == 0) {
+ var t = document.createElement('td');
+ t.rowSpan = main.resources.length;
+ t.textContent = truncateString(main.main_frame_url);
+ t.className = 'last';
+ row.appendChild(t);
+ }
+
+ if (j == main.resources.length - 1)
+ row.className = 'last';
+
+ row.appendChild(document.createElement('td')).textContent =
+ truncateString(resource.resource_url);
+ row.appendChild(document.createElement('td')).textContent =
+ resource.resource_type;
+ row.appendChild(document.createElement('td')).textContent =
+ resource.number_of_hits;
+ row.appendChild(document.createElement('td')).textContent =
+ resource.number_of_misses;
+ row.appendChild(document.createElement('td')).textContent =
+ resource.consecutive_misses;
+ row.appendChild(document.createElement('td')).textContent =
+ resource.position;
+ row.appendChild(document.createElement('td')).textContent =
+ resource.score;
+ body.appendChild(row);
+ }
+ }
+}
+
+document.addEventListener('DOMContentLoaded',
+ requestResourcePrefetchPredictorDb);
diff --git a/chrome/browser/ui/tab_helpers.cc b/chrome/browser/ui/tab_helpers.cc
index 797bb3a..036271b 100644
--- a/chrome/browser/ui/tab_helpers.cc
+++ b/chrome/browser/ui/tab_helpers.cc
@@ -12,6 +12,8 @@
#include "chrome/browser/infobars/infobar_service.h"
#include "chrome/browser/net/net_error_tab_helper.h"
#include "chrome/browser/password_manager/chrome_password_manager_client.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h"
#include "chrome/browser/prerender/prerender_tab_helper.h"
#include "chrome/browser/sessions/session_tab_helper.h"
#include "chrome/browser/tab_contents/navigation_metrics_recorder.h"
@@ -227,4 +229,10 @@ void TabHelpers::AttachTabHelpers(WebContents* web_contents) {
ChromePasswordManagerClient::GetManagerFromWebContents(web_contents));
}
#endif
+
+ if (predictors::ResourcePrefetchPredictorFactory::GetForProfile(
+ web_contents->GetBrowserContext())) {
+ predictors::ResourcePrefetchPredictorTabHelper::CreateForWebContents(
+ web_contents);
+ }
}
diff --git a/chrome/browser/ui/webui/predictors/predictors_handler.cc b/chrome/browser/ui/webui/predictors/predictors_handler.cc
index bd7f39b..5845797 100644
--- a/chrome/browser/ui/webui/predictors/predictors_handler.cc
+++ b/chrome/browser/ui/webui/predictors/predictors_handler.cc
@@ -8,14 +8,39 @@
#include "base/values.h"
#include "chrome/browser/predictors/autocomplete_action_predictor.h"
#include "chrome/browser/predictors/autocomplete_action_predictor_factory.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
#include "chrome/browser/profiles/profile.h"
#include "content/public/browser/web_ui.h"
+#include "content/public/common/resource_type.h"
using predictors::AutocompleteActionPredictor;
+using predictors::ResourcePrefetchPredictor;
+using predictors::ResourcePrefetchPredictorTables;
+
+namespace {
+
+std::string ConvertResourceType(content::ResourceType type) {
+ switch (type) {
+ case content::RESOURCE_TYPE_IMAGE:
+ return "Image";
+ case content::RESOURCE_TYPE_STYLESHEET:
+ return "Stylesheet";
+ case content::RESOURCE_TYPE_SCRIPT:
+ return "Script";
+ default:
+ return "Unknown";
+ }
+}
+
+} // namespace
PredictorsHandler::PredictorsHandler(Profile* profile) {
autocomplete_action_predictor_ =
predictors::AutocompleteActionPredictorFactory::GetForProfile(profile);
+ resource_prefetch_predictor_ =
+ predictors::ResourcePrefetchPredictorFactory::GetForProfile(profile);
}
PredictorsHandler::~PredictorsHandler() { }
@@ -24,6 +49,9 @@ void PredictorsHandler::RegisterMessages() {
web_ui()->RegisterMessageCallback("requestAutocompleteActionPredictorDb",
base::Bind(&PredictorsHandler::RequestAutocompleteActionPredictorDb,
base::Unretained(this)));
+ web_ui()->RegisterMessageCallback("requestResourcePrefetchPredictorDb",
+ base::Bind(&PredictorsHandler::RequestResourcePrefetchPredictorDb,
+ base::Unretained(this)));
}
void PredictorsHandler::RequestAutocompleteActionPredictorDb(
@@ -51,3 +79,52 @@ void PredictorsHandler::RequestAutocompleteActionPredictorDb(
web_ui()->CallJavascriptFunction("updateAutocompleteActionPredictorDb", dict);
}
+
+void PredictorsHandler::RequestResourcePrefetchPredictorDb(
+ const base::ListValue* args) {
+ const bool enabled = (resource_prefetch_predictor_ != NULL);
+ base::DictionaryValue dict;
+ dict.SetBoolean("enabled", enabled);
+
+ if (enabled) {
+ // Url Database cache.
+ base::ListValue* db = new base::ListValue();
+ AddPrefetchDataMapToListValue(
+ *resource_prefetch_predictor_->url_table_cache_, db);
+ dict.Set("url_db", db);
+
+ db = new base::ListValue();
+ AddPrefetchDataMapToListValue(
+ *resource_prefetch_predictor_->host_table_cache_, db);
+ dict.Set("host_db", db);
+ }
+
+ web_ui()->CallJavascriptFunction("updateResourcePrefetchPredictorDb", dict);
+}
+
+void PredictorsHandler::AddPrefetchDataMapToListValue(
+ const ResourcePrefetchPredictor::PrefetchDataMap& data_map,
+ base::ListValue* db) const {
+ for (ResourcePrefetchPredictor::PrefetchDataMap::const_iterator it =
+ data_map.begin(); it != data_map.end(); ++it) {
+ base::DictionaryValue* main = new base::DictionaryValue();
+ main->SetString("main_frame_url", it->first);
+ base::ListValue* resources = new base::ListValue();
+ for (ResourcePrefetchPredictor::ResourceRows::const_iterator
+ row = it->second.resources.begin();
+ row != it->second.resources.end(); ++row) {
+ base::DictionaryValue* resource = new base::DictionaryValue();
+ resource->SetString("resource_url", row->resource_url.spec());
+ resource->SetString("resource_type",
+ ConvertResourceType(row->resource_type));
+ resource->SetInteger("number_of_hits", row->number_of_hits);
+ resource->SetInteger("number_of_misses", row->number_of_misses);
+ resource->SetInteger("consecutive_misses", row->consecutive_misses);
+ resource->SetDouble("position", row->average_position);
+ resource->SetDouble("score", row->score);
+ resources->Append(resource);
+ }
+ main->Set("resources", resources);
+ db->Append(main);
+ }
+}
diff --git a/chrome/browser/ui/webui/predictors/predictors_handler.h b/chrome/browser/ui/webui/predictors/predictors_handler.h
index 1c2c422..f3355e4 100644
--- a/chrome/browser/ui/webui/predictors/predictors_handler.h
+++ b/chrome/browser/ui/webui/predictors/predictors_handler.h
@@ -6,6 +6,7 @@
#define CHROME_BROWSER_UI_WEBUI_PREDICTORS_PREDICTORS_HANDLER_H_
#include "base/compiler_specific.h"
+#include "chrome/browser/predictors/resource_prefetch_predictor.h"
#include "content/public/browser/web_ui_message_handler.h"
namespace base {
@@ -14,6 +15,7 @@ class ListValue;
namespace predictors {
class AutocompleteActionPredictor;
+class ResourcePrefetchPredictor;
}
class Profile;
@@ -32,7 +34,17 @@ class PredictorsHandler : public content::WebUIMessageHandler {
// calls into JS with the resulting DictionaryValue.
void RequestAutocompleteActionPredictorDb(const base::ListValue* args);
+ // Fetches stats for the ResourcePrefetchPredictor and returns it as a
+ // DictionaryValue to the JS.
+ void RequestResourcePrefetchPredictorDb(const base::ListValue* args);
+
+ // Helper for RequestResourcePrefetchPredictorDb.
+ void AddPrefetchDataMapToListValue(
+ const predictors::ResourcePrefetchPredictor::PrefetchDataMap& data_map,
+ base::ListValue* db) const;
+
predictors::AutocompleteActionPredictor* autocomplete_action_predictor_;
+ predictors::ResourcePrefetchPredictor* resource_prefetch_predictor_;
DISALLOW_COPY_AND_ASSIGN(PredictorsHandler);
};
diff --git a/chrome/chrome_browser.gypi b/chrome/chrome_browser.gypi
index 6805eab..92d0dc6 100644
--- a/chrome/chrome_browser.gypi
+++ b/chrome/chrome_browser.gypi
@@ -780,6 +780,8 @@
'browser/net/quota_policy_channel_id_store.h',
'browser/net/referrer.cc',
'browser/net/referrer.h',
+ 'browser/net/resource_prefetch_predictor_observer.cc',
+ 'browser/net/resource_prefetch_predictor_observer.h',
'browser/net/safe_search_util.cc',
'browser/net/safe_search_util.h',
'browser/net/service_providers_win.cc',
@@ -851,6 +853,20 @@
'browser/predictors/predictor_database_factory.h',
'browser/predictors/predictor_table_base.cc',
'browser/predictors/predictor_table_base.h',
+ 'browser/predictors/resource_prefetch_common.cc',
+ 'browser/predictors/resource_prefetch_common.h',
+ 'browser/predictors/resource_prefetch_predictor.cc',
+ 'browser/predictors/resource_prefetch_predictor.h',
+ 'browser/predictors/resource_prefetch_predictor_factory.cc',
+ 'browser/predictors/resource_prefetch_predictor_factory.h',
+ 'browser/predictors/resource_prefetch_predictor_tab_helper.cc',
+ 'browser/predictors/resource_prefetch_predictor_tab_helper.h',
+ 'browser/predictors/resource_prefetch_predictor_tables.cc',
+ 'browser/predictors/resource_prefetch_predictor_tables.h',
+ 'browser/predictors/resource_prefetcher.cc',
+ 'browser/predictors/resource_prefetcher.h',
+ 'browser/predictors/resource_prefetcher_manager.cc',
+ 'browser/predictors/resource_prefetcher_manager.h',
'browser/pref_service_flags_storage.cc',
'browser/pref_service_flags_storage.h',
'browser/prefetch/prefetch.cc',
diff --git a/chrome/chrome_tests_unit.gypi b/chrome/chrome_tests_unit.gypi
index 45dade8..9922379 100644
--- a/chrome/chrome_tests_unit.gypi
+++ b/chrome/chrome_tests_unit.gypi
@@ -594,6 +594,9 @@
'browser/power/process_power_collector_unittest.cc',
'browser/predictors/autocomplete_action_predictor_table_unittest.cc',
'browser/predictors/autocomplete_action_predictor_unittest.cc',
+ 'browser/predictors/resource_prefetch_predictor_unittest.cc',
+ 'browser/predictors/resource_prefetch_predictor_tables_unittest.cc',
+ 'browser/predictors/resource_prefetcher_unittest.cc',
'browser/prefs/chrome_pref_service_unittest.cc',
'browser/prefs/command_line_pref_store_unittest.cc',
'browser/prefs/incognito_mode_prefs_unittest.cc',
diff --git a/chrome/common/chrome_switches.cc b/chrome/common/chrome_switches.cc
index d6dc7ca..a04c81a 100644
--- a/chrome/common/chrome_switches.cc
+++ b/chrome/common/chrome_switches.cc
@@ -1080,6 +1080,20 @@ const char kSimulateOutdated[] = "simulate-outdated";
// Simulates that current version is outdated and auto-update is off.
const char kSimulateOutdatedNoAU[] = "simulate-outdated-no-au";
+// Speculative resource prefetching.
+const char kSpeculativeResourcePrefetching[] =
+ "speculative-resource-prefetching";
+
+// Speculative resource prefetching is disabled.
+const char kSpeculativeResourcePrefetchingDisabled[] = "disabled";
+
+// Speculative resource prefetching will only learn about resources that need to
+// be prefetched but will not prefetch them.
+const char kSpeculativeResourcePrefetchingLearning[] = "learning";
+
+// Speculative resource prefetching is enabled.
+const char kSpeculativeResourcePrefetchingEnabled[] = "enabled";
+
// Specifies the URL where spelling service feedback data will be sent instead
// of the default URL. This switch is for temporary testing only.
// TODO(rouslan): Remove this flag when feedback testing is complete. Revisit by
diff --git a/chrome/common/chrome_switches.h b/chrome/common/chrome_switches.h
index a91277a..65cda6b 100644
--- a/chrome/common/chrome_switches.h
+++ b/chrome/common/chrome_switches.h
@@ -298,6 +298,10 @@ extern const char kSimulateUpgrade[];
extern const char kSimulateCriticalUpdate[];
extern const char kSimulateOutdated[];
extern const char kSimulateOutdatedNoAU[];
+extern const char kSpeculativeResourcePrefetching[];
+extern const char kSpeculativeResourcePrefetchingDisabled[];
+extern const char kSpeculativeResourcePrefetchingEnabled[];
+extern const char kSpeculativeResourcePrefetchingLearning[];
extern const char kSpellingServiceFeedbackUrl[];
extern const char kSpellingServiceFeedbackIntervalSeconds[];
extern const char kSSLVersionMax[];