From 825722d5939dc0832d124b70958501611e6a4628 Mon Sep 17 00:00:00 2001 From: zhenw Date: Mon, 15 Sep 2014 15:06:25 -0700 Subject: Revert CL 117933003. Re-add speculative resource prefetching code. https://codereview.chromium.org/117933003/ The speculative resource prefetching code was experimental code developed by shishir@. He found that it has little improvement on desktop Chrome (win). We though this should be beneficial to mobile browsers. After discussing with tburkard@ and kenjibaheux@, we decided to bring the code back and do more analysis on mobile devices. Reverting the patchset to re-add the code is the first step. The following design doc has discussed all related approaches and action items. https://docs.google.com/a/google.com/document/d/1ie3hu-zNNXvmTXm3aJAtKUGOh6nZfbNjA0aZE1bzzIg/edit?usp=sharing BUG=408399, 405690 Review URL: https://codereview.chromium.org/462423004 Cr-Commit-Position: refs/heads/master@{#294899} --- .../net/resource_prefetch_predictor_observer.cc | 202 ++++ .../net/resource_prefetch_predictor_observer.h | 50 + chrome/browser/predictors/predictor_database.cc | 19 +- chrome/browser/predictors/predictor_database.h | 2 + .../browser/predictors/resource_prefetch_common.cc | 245 ++++ .../browser/predictors/resource_prefetch_common.h | 116 ++ .../predictors/resource_prefetch_predictor.cc | 1222 ++++++++++++++++++++ .../predictors/resource_prefetch_predictor.h | 314 +++++ .../resource_prefetch_predictor_factory.cc | 52 + .../resource_prefetch_predictor_factory.h | 38 + .../resource_prefetch_predictor_tab_helper.cc | 62 + .../resource_prefetch_predictor_tab_helper.h | 34 + .../resource_prefetch_predictor_tables.cc | 513 ++++++++ .../resource_prefetch_predictor_tables.h | 160 +++ .../resource_prefetch_predictor_tables_unittest.cc | 472 ++++++++ .../resource_prefetch_predictor_unittest.cc | 925 +++++++++++++++ chrome/browser/predictors/resource_prefetcher.cc | 240 ++++ chrome/browser/predictors/resource_prefetcher.h | 165 +++ .../predictors/resource_prefetcher_manager.cc | 135 +++ .../predictors/resource_prefetcher_manager.h | 87 ++ .../predictors/resource_prefetcher_unittest.cc | 343 ++++++ .../test/cloud_print_policy_browsertest.cc | 7 + .../chrome_browser_main_extra_parts_profiles.cc | 2 + chrome/browser/profiles/profile_io_data.cc | 15 + chrome/browser/profiles/profile_io_data.h | 14 + .../chrome_resource_dispatcher_host_delegate.cc | 14 + .../browser/resources/predictors/predictors.html | 4 + chrome/browser/resources/predictors/predictors.js | 1 + .../predictors/resource_prefetch_predictor.html | 49 + .../predictors/resource_prefetch_predictor.js | 102 ++ chrome/browser/ui/tab_helpers.cc | 8 + .../ui/webui/predictors/predictors_handler.cc | 77 ++ .../ui/webui/predictors/predictors_handler.h | 12 + chrome/chrome_browser.gypi | 16 + chrome/chrome_tests_unit.gypi | 3 + chrome/common/chrome_switches.cc | 14 + chrome/common/chrome_switches.h | 4 + 37 files changed, 5737 insertions(+), 1 deletion(-) create mode 100644 chrome/browser/net/resource_prefetch_predictor_observer.cc create mode 100644 chrome/browser/net/resource_prefetch_predictor_observer.h create mode 100644 chrome/browser/predictors/resource_prefetch_common.cc create mode 100644 chrome/browser/predictors/resource_prefetch_common.h create mode 100644 chrome/browser/predictors/resource_prefetch_predictor.cc create mode 100644 chrome/browser/predictors/resource_prefetch_predictor.h create mode 100644 chrome/browser/predictors/resource_prefetch_predictor_factory.cc create mode 100644 chrome/browser/predictors/resource_prefetch_predictor_factory.h create mode 100644 chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc create mode 100644 chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h create mode 100644 chrome/browser/predictors/resource_prefetch_predictor_tables.cc create mode 100644 chrome/browser/predictors/resource_prefetch_predictor_tables.h create mode 100644 chrome/browser/predictors/resource_prefetch_predictor_tables_unittest.cc create mode 100644 chrome/browser/predictors/resource_prefetch_predictor_unittest.cc create mode 100644 chrome/browser/predictors/resource_prefetcher.cc create mode 100644 chrome/browser/predictors/resource_prefetcher.h create mode 100644 chrome/browser/predictors/resource_prefetcher_manager.cc create mode 100644 chrome/browser/predictors/resource_prefetcher_manager.h create mode 100644 chrome/browser/predictors/resource_prefetcher_unittest.cc create mode 100644 chrome/browser/resources/predictors/resource_prefetch_predictor.html create mode 100644 chrome/browser/resources/predictors/resource_prefetch_predictor.js (limited to 'chrome') diff --git a/chrome/browser/net/resource_prefetch_predictor_observer.cc b/chrome/browser/net/resource_prefetch_predictor_observer.cc new file mode 100644 index 0000000..5a3f4a1 --- /dev/null +++ b/chrome/browser/net/resource_prefetch_predictor_observer.cc @@ -0,0 +1,202 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/net/resource_prefetch_predictor_observer.h" + +#include + +#include "base/metrics/histogram.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/render_frame_host.h" +#include "content/public/browser/resource_request_info.h" +#include "net/url_request/url_request.h" +#include "url/gurl.h" + +using content::BrowserThread; +using predictors::ResourcePrefetchPredictor; + +namespace { + +// Enum for measuring statistics pertaining to observed request, responses and +// redirects. +enum RequestStats { + REQUEST_STATS_TOTAL_RESPONSES = 0, + REQUEST_STATS_TOTAL_PROCESSED_RESPONSES = 1, + REQUEST_STATS_NO_RESOURCE_REQUEST_INFO = 2, + REQUEST_STATS_NO_RENDER_FRAME_ID_FROM_REQUEST_INFO = 3, + REQUEST_STATS_MAX = 4, +}; + +// Specific to main frame requests. +enum MainFrameRequestStats { + MAIN_FRAME_REQUEST_STATS_TOTAL_REQUESTS = 0, + MAIN_FRAME_REQUEST_STATS_PROCESSED_REQUESTS = 1, + MAIN_FRAME_REQUEST_STATS_TOTAL_REDIRECTS = 2, + MAIN_FRAME_REQUEST_STATS_PROCESSED_REDIRECTS = 3, + MAIN_FRAME_REQUEST_STATS_TOTAL_RESPONSES = 4, + MAIN_FRAME_REQUEST_STATS_PROCESSED_RESPONSES = 5, + MAIN_FRAME_REQUEST_STATS_MAX = 6, +}; + +void ReportRequestStats(RequestStats stat) { + UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.RequestStats", + stat, + REQUEST_STATS_MAX); +} + +void ReportMainFrameRequestStats(MainFrameRequestStats stat) { + UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.MainFrameRequestStats", + stat, + MAIN_FRAME_REQUEST_STATS_MAX); +} + +bool SummarizeResponse(net::URLRequest* request, + ResourcePrefetchPredictor::URLRequestSummary* summary) { + const content::ResourceRequestInfo* info = + content::ResourceRequestInfo::ForRequest(request); + if (!info) { + ReportRequestStats(REQUEST_STATS_NO_RESOURCE_REQUEST_INFO); + return false; + } + + int render_process_id, render_frame_id; + if (!info->GetAssociatedRenderFrame(&render_process_id, &render_frame_id)) { + ReportRequestStats(REQUEST_STATS_NO_RENDER_FRAME_ID_FROM_REQUEST_INFO); + return false; + } + + summary->navigation_id.render_process_id = render_process_id; + summary->navigation_id.render_frame_id = render_frame_id; + summary->navigation_id.main_frame_url = request->first_party_for_cookies(); + summary->navigation_id.creation_time = request->creation_time(); + summary->resource_url = request->original_url(); + summary->resource_type = info->GetResourceType(); + request->GetMimeType(&summary->mime_type); + summary->was_cached = request->was_cached(); + + // Use the mime_type to determine the resource type for subresources since + // types such as PREFETCH, SUB_RESOURCE, etc are not useful. + if (summary->resource_type != content::RESOURCE_TYPE_MAIN_FRAME) { + summary->resource_type = + ResourcePrefetchPredictor::GetResourceTypeFromMimeType( + summary->mime_type, + summary->resource_type); + } + return true; +} + +} // namespace + +namespace chrome_browser_net { + +ResourcePrefetchPredictorObserver::ResourcePrefetchPredictorObserver( + ResourcePrefetchPredictor* predictor) + : predictor_(predictor->AsWeakPtr()) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); +} + +ResourcePrefetchPredictorObserver::~ResourcePrefetchPredictorObserver() { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI) || + BrowserThread::CurrentlyOn(BrowserThread::IO)); +} + +void ResourcePrefetchPredictorObserver::OnRequestStarted( + net::URLRequest* request, + content::ResourceType resource_type, + int child_id, + int frame_id) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + + if (resource_type == content::RESOURCE_TYPE_MAIN_FRAME) + ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_REQUESTS); + + if (!ResourcePrefetchPredictor::ShouldRecordRequest(request, resource_type)) + return; + + ResourcePrefetchPredictor::URLRequestSummary summary; + summary.navigation_id.render_process_id = child_id; + summary.navigation_id.render_frame_id = frame_id; + summary.navigation_id.main_frame_url = request->first_party_for_cookies(); + summary.resource_url = request->original_url(); + summary.resource_type = resource_type; + + BrowserThread::PostTask( + BrowserThread::UI, + FROM_HERE, + base::Bind(&ResourcePrefetchPredictor::RecordURLRequest, + predictor_, + summary)); + + if (resource_type == content::RESOURCE_TYPE_MAIN_FRAME) + ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_REQUESTS); +} + +void ResourcePrefetchPredictorObserver::OnRequestRedirected( + const GURL& redirect_url, + net::URLRequest* request) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + + const content::ResourceRequestInfo* request_info = + content::ResourceRequestInfo::ForRequest(request); + if (request_info && + request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) { + ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_REDIRECTS); + } + + if (!ResourcePrefetchPredictor::ShouldRecordRedirect(request)) + return; + + ResourcePrefetchPredictor::URLRequestSummary summary; + if (!SummarizeResponse(request, &summary)) + return; + + summary.redirect_url = redirect_url; + + BrowserThread::PostTask( + BrowserThread::UI, + FROM_HERE, + base::Bind(&ResourcePrefetchPredictor::RecordURLRedirect, + predictor_, + summary)); + + if (request_info && + request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) { + ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_REDIRECTS); + } +} + +void ResourcePrefetchPredictorObserver::OnResponseStarted( + net::URLRequest* request) { + DCHECK_CURRENTLY_ON(BrowserThread::IO); + + ReportRequestStats(REQUEST_STATS_TOTAL_RESPONSES); + + const content::ResourceRequestInfo* request_info = + content::ResourceRequestInfo::ForRequest(request); + if (request_info && + request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) { + ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_TOTAL_RESPONSES); + } + + if (!ResourcePrefetchPredictor::ShouldRecordResponse(request)) + return; + ResourcePrefetchPredictor::URLRequestSummary summary; + if (!SummarizeResponse(request, &summary)) + return; + + BrowserThread::PostTask( + BrowserThread::UI, + FROM_HERE, + base::Bind(&ResourcePrefetchPredictor::RecordURLResponse, + predictor_, + summary)); + + ReportRequestStats(REQUEST_STATS_TOTAL_PROCESSED_RESPONSES); + if (request_info && + request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME) { + ReportMainFrameRequestStats(MAIN_FRAME_REQUEST_STATS_PROCESSED_RESPONSES); + } +} + +} // namespace chrome_browser_net diff --git a/chrome/browser/net/resource_prefetch_predictor_observer.h b/chrome/browser/net/resource_prefetch_predictor_observer.h new file mode 100644 index 0000000..e2fd480 --- /dev/null +++ b/chrome/browser/net/resource_prefetch_predictor_observer.h @@ -0,0 +1,50 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_ +#define CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_ + +#include "base/basictypes.h" +#include "base/memory/weak_ptr.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "content/public/common/resource_type.h" + +namespace net { +class URLRequest; +} + +class GURL; + +namespace chrome_browser_net { + +// Observes resource requests in the ResourceDispatcherHostDelegate and notifies +// the ResourcePrefetchPredictor about the ones it is interested in. +// - Has an instance per profile, and is owned by the corresponding +// ProfileIOData. +// - Needs to be constructed on UI thread. Rest of the functions can only be +// called on the IO thread. Can be destroyed on UI or IO thread. +class ResourcePrefetchPredictorObserver { + public: + explicit ResourcePrefetchPredictorObserver( + predictors::ResourcePrefetchPredictor* predictor); + ~ResourcePrefetchPredictorObserver(); + + // Parts of the ResourceDispatcherHostDelegate that we want to observe. + void OnRequestStarted(net::URLRequest* request, + content::ResourceType resource_type, + int child_id, + int frame_id); + void OnRequestRedirected(const GURL& redirect_url, net::URLRequest* request); + void OnResponseStarted(net::URLRequest* request); + + private: + // Owned by profile. + base::WeakPtr predictor_; + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorObserver); +}; + +} // namespace chrome_browser_net + +#endif // CHROME_BROWSER_NET_RESOURCE_PREFETCH_PREDICTOR_OBSERVER_H_ diff --git a/chrome/browser/predictors/predictor_database.cc b/chrome/browser/predictors/predictor_database.cc index eb2bf5c..9f8460a 100644 --- a/chrome/browser/predictors/predictor_database.cc +++ b/chrome/browser/predictors/predictor_database.cc @@ -12,6 +12,8 @@ #include "base/strings/stringprintf.h" #include "chrome/browser/predictors/autocomplete_action_predictor_table.h" #include "chrome/browser/predictors/logged_in_predictor_table.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h" #include "chrome/browser/prerender/prerender_field_trial.h" #include "chrome/browser/profiles/profile.h" #include "content/public/browser/browser_thread.h" @@ -50,6 +52,7 @@ class PredictorDatabaseInternal // Cancels pending DB transactions. Should only be called on the UI thread. void SetCancelled(); + bool is_resource_prefetch_predictor_enabled_; base::FilePath db_path_; scoped_ptr db_; @@ -57,6 +60,7 @@ class PredictorDatabaseInternal // to using a WeakPtr instead. scoped_refptr autocomplete_table_; scoped_refptr logged_in_table_; + scoped_refptr resource_prefetch_tables_; DISALLOW_COPY_AND_ASSIGN(PredictorDatabaseInternal); }; @@ -66,8 +70,12 @@ PredictorDatabaseInternal::PredictorDatabaseInternal(Profile* profile) : db_path_(profile->GetPath().Append(kPredictorDatabaseName)), db_(new sql::Connection()), autocomplete_table_(new AutocompleteActionPredictorTable()), - logged_in_table_(new LoggedInPredictorTable()) { + logged_in_table_(new LoggedInPredictorTable()), + resource_prefetch_tables_(new ResourcePrefetchPredictorTables()) { db_->set_histogram_tag("Predictor"); + ResourcePrefetchPredictorConfig config; + is_resource_prefetch_predictor_enabled_ = + IsSpeculativeResourcePrefetchingEnabled(profile, &config); } PredictorDatabaseInternal::~PredictorDatabaseInternal() { @@ -89,6 +97,7 @@ void PredictorDatabaseInternal::Initialize() { autocomplete_table_->Initialize(db_.get()); logged_in_table_->Initialize(db_.get()); + resource_prefetch_tables_->Initialize(db_.get()); LogDatabaseStats(); } @@ -99,6 +108,7 @@ void PredictorDatabaseInternal::SetCancelled() { autocomplete_table_->SetCancelled(); logged_in_table_->SetCancelled(); + resource_prefetch_tables_->SetCancelled(); } void PredictorDatabaseInternal::LogDatabaseStats() { @@ -113,6 +123,8 @@ void PredictorDatabaseInternal::LogDatabaseStats() { autocomplete_table_->LogDatabaseStats(); logged_in_table_->LogDatabaseStats(); + if (is_resource_prefetch_predictor_enabled_) + resource_prefetch_tables_->LogDatabaseStats(); } PredictorDatabase::PredictorDatabase(Profile* profile) @@ -138,6 +150,11 @@ scoped_refptr return db_->logged_in_table_; } +scoped_refptr + PredictorDatabase::resource_prefetch_tables() { + return db_->resource_prefetch_tables_; +} + sql::Connection* PredictorDatabase::GetDatabase() { return db_->db_.get(); } diff --git a/chrome/browser/predictors/predictor_database.h b/chrome/browser/predictors/predictor_database.h index 624b00b..bc53a9a 100644 --- a/chrome/browser/predictors/predictor_database.h +++ b/chrome/browser/predictors/predictor_database.h @@ -19,6 +19,7 @@ namespace predictors { class AutocompleteActionPredictorTable; class LoggedInPredictorTable; class PredictorDatabaseInternal; +class ResourcePrefetchPredictorTables; class PredictorDatabase : public KeyedService { public: @@ -26,6 +27,7 @@ class PredictorDatabase : public KeyedService { virtual ~PredictorDatabase(); scoped_refptr autocomplete_table(); + scoped_refptr resource_prefetch_tables(); scoped_refptr logged_in_table(); // Used for testing. diff --git a/chrome/browser/predictors/resource_prefetch_common.cc b/chrome/browser/predictors/resource_prefetch_common.cc new file mode 100644 index 0000000..731bda6 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_common.cc @@ -0,0 +1,245 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetch_common.h" + +#include "base/command_line.h" +#include "base/metrics/field_trial.h" +#include "base/prefs/pref_service.h" +#include "chrome/browser/profiles/profile.h" +#include "chrome/common/chrome_switches.h" +#include "chrome/common/pref_names.h" +#include "content/public/browser/render_process_host.h" +#include "content/public/browser/render_view_host.h" +#include "content/public/browser/web_contents.h" + +namespace predictors { + +const char kSpeculativePrefetchingTrialName[] = + "SpeculativeResourcePrefetching"; + +bool IsSpeculativeResourcePrefetchingEnabled( + Profile* profile, + ResourcePrefetchPredictorConfig* config) { + DCHECK(config); + + // Off the record - disabled. + if (!profile || profile->IsOffTheRecord()) + return false; + + // If the user has explicitly disabled "predictive actions" - disabled. + if (!profile->GetPrefs() || + !profile->GetPrefs()->GetBoolean(prefs::kNetworkPredictionEnabled)) { + return false; + } + + // The config has the default params already set. The command line with just + // enable them with the default params. + if (CommandLine::ForCurrentProcess()->HasSwitch( + switches::kSpeculativeResourcePrefetching)) { + const std::string value = + CommandLine::ForCurrentProcess()->GetSwitchValueASCII( + switches::kSpeculativeResourcePrefetching); + + if (value == switches::kSpeculativeResourcePrefetchingDisabled) { + return false; + } else if (value == switches::kSpeculativeResourcePrefetchingLearning) { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + return true; + } else if (value == switches::kSpeculativeResourcePrefetchingEnabled) { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + return true; + } + } + + std::string trial = base::FieldTrialList::FindFullName( + kSpeculativePrefetchingTrialName); + + if (trial == "LearningHost") { + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + return true; + } else if (trial == "LearningURL") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + return true; + } else if (trial == "Learning") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + return true; + } else if (trial == "PrefetchingHost") { + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + return true; + } else if (trial == "PrefetchingURL") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + return true; + } else if (trial == "Prefetching") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + return true; + } else if (trial == "PrefetchingLowConfidence") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + + config->min_url_visit_count = 1; + config->min_resource_confidence_to_trigger_prefetch = 0.5f; + config->min_resource_hits_to_trigger_prefetch = 1; + return true; + } else if (trial == "PrefetchingHighConfidence") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + + config->min_url_visit_count = 3; + config->min_resource_confidence_to_trigger_prefetch = 0.9f; + config->min_resource_hits_to_trigger_prefetch = 3; + return true; + } else if (trial == "PrefetchingMoreResources") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + + config->max_resources_per_entry = 100; + return true; + } else if (trial == "LearningSmallDB") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + + config->max_urls_to_track = 200; + config->max_hosts_to_track = 100; + return true; + } else if (trial == "PrefetchingSmallDB") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + + config->max_urls_to_track = 200; + config->max_hosts_to_track = 100; + return true; + } else if (trial == "PrefetchingSmallDBLowConfidence") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + + config->max_urls_to_track = 200; + config->max_hosts_to_track = 100; + config->min_url_visit_count = 1; + config->min_resource_confidence_to_trigger_prefetch = 0.5f; + config->min_resource_hits_to_trigger_prefetch = 1; + return true; + } else if (trial == "PrefetchingSmallDBHighConfidence") { + config->mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + config->mode |= ResourcePrefetchPredictorConfig::URL_PREFETCHING; + config->mode |= ResourcePrefetchPredictorConfig::HOST_PRFETCHING; + + config->max_urls_to_track = 200; + config->max_hosts_to_track = 100; + config->min_url_visit_count = 3; + config->min_resource_confidence_to_trigger_prefetch = 0.9f; + config->min_resource_hits_to_trigger_prefetch = 3; + return true; + } + + return false; +} + +NavigationID::NavigationID() + : render_process_id(-1), + render_frame_id(-1) { +} + +NavigationID::NavigationID(const NavigationID& other) + : render_process_id(other.render_process_id), + render_frame_id(other.render_frame_id), + main_frame_url(other.main_frame_url), + creation_time(other.creation_time) { +} + +NavigationID::NavigationID(content::WebContents* web_contents) + : render_process_id(web_contents->GetRenderProcessHost()->GetID()), + render_frame_id(web_contents->GetRenderViewHost()->GetRoutingID()), + main_frame_url(web_contents->GetURL()) { +} + +bool NavigationID::is_valid() const { + return render_process_id != -1 && render_frame_id != -1 && + !main_frame_url.is_empty(); +} + +bool NavigationID::operator<(const NavigationID& rhs) const { + DCHECK(is_valid() && rhs.is_valid()); + if (render_process_id != rhs.render_process_id) + return render_process_id < rhs.render_process_id; + else if (render_frame_id != rhs.render_frame_id) + return render_frame_id < rhs.render_frame_id; + else + return main_frame_url < rhs.main_frame_url; +} + +bool NavigationID::operator==(const NavigationID& rhs) const { + DCHECK(is_valid() && rhs.is_valid()); + return IsSameRenderer(rhs) && main_frame_url == rhs.main_frame_url; +} + +bool NavigationID::IsSameRenderer(const NavigationID& other) const { + DCHECK(is_valid() && other.is_valid()); + return render_process_id == other.render_process_id && + render_frame_id == other.render_frame_id; +} + +ResourcePrefetchPredictorConfig::ResourcePrefetchPredictorConfig() + : mode(0), + max_navigation_lifetime_seconds(60), + max_urls_to_track(500), + max_hosts_to_track(200), + min_url_visit_count(2), + max_resources_per_entry(50), + max_consecutive_misses(3), + min_resource_confidence_to_trigger_prefetch(0.8f), + min_resource_hits_to_trigger_prefetch(3), + max_prefetches_inflight_per_navigation(24), + max_prefetches_inflight_per_host_per_navigation(3) { +} + +ResourcePrefetchPredictorConfig::~ResourcePrefetchPredictorConfig() { +} + +bool ResourcePrefetchPredictorConfig::IsLearningEnabled() const { + return IsURLLearningEnabled() || IsHostLearningEnabled(); +} + +bool ResourcePrefetchPredictorConfig::IsPrefetchingEnabled() const { + return IsURLPrefetchingEnabled() || IsHostPrefetchingEnabled(); +} + +bool ResourcePrefetchPredictorConfig::IsURLLearningEnabled() const { + return (mode & URL_LEARNING) > 0; +} + +bool ResourcePrefetchPredictorConfig::IsHostLearningEnabled() const { + return (mode & HOST_LEARNING) > 0; +} + +bool ResourcePrefetchPredictorConfig::IsURLPrefetchingEnabled() const { + return (mode & URL_PREFETCHING) > 0; +} + +bool ResourcePrefetchPredictorConfig::IsHostPrefetchingEnabled() const { + return (mode & HOST_PRFETCHING) > 0; +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetch_common.h b/chrome/browser/predictors/resource_prefetch_common.h new file mode 100644 index 0000000..f94a50d --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_common.h @@ -0,0 +1,116 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_ + +#include "base/time/time.h" +#include "url/gurl.h" + +class Profile; + +namespace content { +class WebContents; +} + +namespace predictors { + +struct ResourcePrefetchPredictorConfig; + +// Returns true if prefetching is enabled. And will initilize the |config| +// fields to the appropritate values. +bool IsSpeculativeResourcePrefetchingEnabled( + Profile* profile, + ResourcePrefetchPredictorConfig* config); + +// Represents the type of key based on which prefetch data is stored. +enum PrefetchKeyType { + PREFETCH_KEY_TYPE_HOST, + PREFETCH_KEY_TYPE_URL +}; + +// Represents a single navigation for a render frame. +struct NavigationID { + // TODO(shishir): Maybe take process_id, frame_id and url as input in + // constructor. + NavigationID(); + NavigationID(const NavigationID& other); + explicit NavigationID(content::WebContents* web_contents); + bool operator<(const NavigationID& rhs) const; + bool operator==(const NavigationID& rhs) const; + + bool IsSameRenderer(const NavigationID& other) const; + + // Returns true iff the render_process_id_, render_frame_id_ and + // frame_url_ has been set correctly. + bool is_valid() const; + + int render_process_id; + int render_frame_id; + GURL main_frame_url; + + // NOTE: Even though we store the creation time here, it is not used during + // comparison of two NavigationIDs because it cannot always be determined + // correctly. + base::TimeTicks creation_time; +}; + +// Represents the config for the resource prefetch prediction algorithm. It is +// useful for running experiments. +struct ResourcePrefetchPredictorConfig { + // Initializes the config with default values. + ResourcePrefetchPredictorConfig(); + ~ResourcePrefetchPredictorConfig(); + + // The mode the prefetcher is running in. Forms a bit map. + enum Mode { + URL_LEARNING = 1 << 0, + HOST_LEARNING = 1 << 1, + URL_PREFETCHING = 1 << 2, // Should also turn on URL_LEARNING. + HOST_PRFETCHING = 1 << 3 // Should also turn on HOST_LEARNING. + }; + int mode; + + // Helpers to deal with mode. + bool IsLearningEnabled() const; + bool IsPrefetchingEnabled() const; + bool IsURLLearningEnabled() const; + bool IsHostLearningEnabled() const; + bool IsURLPrefetchingEnabled() const; + bool IsHostPrefetchingEnabled() const; + + // If a navigation hasn't seen a load complete event in this much time, it + // is considered abandoned. + size_t max_navigation_lifetime_seconds; + + // Size of LRU caches for the URL and host data. + size_t max_urls_to_track; + size_t max_hosts_to_track; + + // The number of times we should have seen a visit to this URL in history + // to start tracking it. This is to ensure we don't bother with oneoff + // entries. For hosts we track each one. + size_t min_url_visit_count; + + // The maximum number of resources to store per entry. + size_t max_resources_per_entry; + // The number of consecutive misses after we stop tracking a resource URL. + size_t max_consecutive_misses; + + // The minimum confidence (accuracy of hits) required for a resource to be + // prefetched. + float min_resource_confidence_to_trigger_prefetch; + // The minimum number of times we must have a URL on record to prefetch it. + size_t min_resource_hits_to_trigger_prefetch; + + // Maximum number of prefetches that can be inflight for a single navigation. + size_t max_prefetches_inflight_per_navigation; + // Maximum number of prefetches that can be inflight for a host for a single + // navigation. + size_t max_prefetches_inflight_per_host_per_navigation; +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_ diff --git a/chrome/browser/predictors/resource_prefetch_predictor.cc b/chrome/browser/predictors/resource_prefetch_predictor.cc new file mode 100644 index 0000000..d0b02a2 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor.cc @@ -0,0 +1,1222 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetch_predictor.h" + +#include +#include +#include + +#include "base/command_line.h" +#include "base/metrics/histogram.h" +#include "base/stl_util.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/stringprintf.h" +#include "base/time/time.h" +#include "chrome/browser/chrome_notification_types.h" +#include "chrome/browser/history/history_database.h" +#include "chrome/browser/history/history_db_task.h" +#include "chrome/browser/history/history_notifications.h" +#include "chrome/browser/history/history_service.h" +#include "chrome/browser/history/history_service_factory.h" +#include "chrome/browser/predictors/predictor_database.h" +#include "chrome/browser/predictors/predictor_database_factory.h" +#include "chrome/browser/predictors/resource_prefetcher_manager.h" +#include "chrome/browser/profiles/profile.h" +#include "chrome/common/chrome_switches.h" +#include "chrome/common/url_constants.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/navigation_controller.h" +#include "content/public/browser/notification_service.h" +#include "content/public/browser/notification_source.h" +#include "content/public/browser/notification_types.h" +#include "content/public/browser/resource_request_info.h" +#include "content/public/browser/web_contents.h" +#include "net/base/mime_util.h" +#include "net/http/http_response_headers.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context_getter.h" + +using content::BrowserThread; + +namespace { + +// For reporting whether a subresource is handled or not, and for what reasons. +enum ResourceStatus { + RESOURCE_STATUS_HANDLED = 0, + RESOURCE_STATUS_NOT_HTTP_PAGE = 1, + RESOURCE_STATUS_NOT_HTTP_RESOURCE = 2, + RESOURCE_STATUS_UNSUPPORTED_MIME_TYPE = 4, + RESOURCE_STATUS_NOT_GET = 8, + RESOURCE_STATUS_URL_TOO_LONG = 16, + RESOURCE_STATUS_NOT_CACHEABLE = 32, + RESOURCE_STATUS_HEADERS_MISSING = 64, + RESOURCE_STATUS_MAX = 128, +}; + +// For reporting various interesting events that occur during the loading of a +// single main frame. +enum NavigationEvent { + NAVIGATION_EVENT_REQUEST_STARTED = 0, + NAVIGATION_EVENT_REQUEST_REDIRECTED = 1, + NAVIGATION_EVENT_REQUEST_REDIRECTED_EMPTY_URL = 2, + NAVIGATION_EVENT_REQUEST_EXPIRED = 3, + NAVIGATION_EVENT_RESPONSE_STARTED = 4, + NAVIGATION_EVENT_ONLOAD = 5, + NAVIGATION_EVENT_ONLOAD_EMPTY_URL = 6, + NAVIGATION_EVENT_ONLOAD_UNTRACKED_URL = 7, + NAVIGATION_EVENT_ONLOAD_TRACKED_URL = 8, + NAVIGATION_EVENT_SHOULD_TRACK_URL = 9, + NAVIGATION_EVENT_SHOULD_NOT_TRACK_URL = 10, + NAVIGATION_EVENT_URL_TABLE_FULL = 11, + NAVIGATION_EVENT_HAVE_PREDICTIONS_FOR_URL = 12, + NAVIGATION_EVENT_NO_PREDICTIONS_FOR_URL = 13, + NAVIGATION_EVENT_MAIN_FRAME_URL_TOO_LONG = 14, + NAVIGATION_EVENT_HOST_TOO_LONG = 15, + NAVIGATION_EVENT_COUNT = 16, +}; + +// For reporting events of interest that are not tied to any navigation. +enum ReportingEvent { + REPORTING_EVENT_ALL_HISTORY_CLEARED = 0, + REPORTING_EVENT_PARTIAL_HISTORY_CLEARED = 1, + REPORTING_EVENT_COUNT = 2 +}; + +void RecordNavigationEvent(NavigationEvent event) { + UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.NavigationEvent", + event, + NAVIGATION_EVENT_COUNT); +} + +} // namespace + +namespace predictors { + +//////////////////////////////////////////////////////////////////////////////// +// History lookup task. + +// Used to fetch the visit count for a URL from the History database. +class GetUrlVisitCountTask : public history::HistoryDBTask { + public: + typedef ResourcePrefetchPredictor::URLRequestSummary URLRequestSummary; + typedef base::Callback&)> VisitInfoCallback; + + GetUrlVisitCountTask( + const NavigationID& navigation_id, + std::vector* requests, + VisitInfoCallback callback) + : visit_count_(0), + navigation_id_(navigation_id), + requests_(requests), + callback_(callback) { + DCHECK(requests_.get()); + } + + virtual bool RunOnDBThread(history::HistoryBackend* backend, + history::HistoryDatabase* db) OVERRIDE { + history::URLRow url_row; + if (db->GetRowForURL(navigation_id_.main_frame_url, &url_row)) + visit_count_ = url_row.visit_count(); + return true; + } + + virtual void DoneRunOnMainThread() OVERRIDE { + callback_.Run(visit_count_, navigation_id_, *requests_); + } + + private: + virtual ~GetUrlVisitCountTask() { } + + int visit_count_; + NavigationID navigation_id_; + scoped_ptr > requests_; + VisitInfoCallback callback_; + + DISALLOW_COPY_AND_ASSIGN(GetUrlVisitCountTask); +}; + +//////////////////////////////////////////////////////////////////////////////// +// ResourcePrefetchPredictor static functions. + +// static +bool ResourcePrefetchPredictor::ShouldRecordRequest( + net::URLRequest* request, + content::ResourceType resource_type) { + const content::ResourceRequestInfo* request_info = + content::ResourceRequestInfo::ForRequest(request); + if (!request_info) + return false; + + if (!request_info->IsMainFrame()) + return false; + + return resource_type == content::RESOURCE_TYPE_MAIN_FRAME && + IsHandledMainPage(request); +} + +// static +bool ResourcePrefetchPredictor::ShouldRecordResponse( + net::URLRequest* response) { + const content::ResourceRequestInfo* request_info = + content::ResourceRequestInfo::ForRequest(response); + if (!request_info) + return false; + + if (!request_info->IsMainFrame()) + return false; + + return request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME ? + IsHandledMainPage(response) : IsHandledSubresource(response); +} + +// static +bool ResourcePrefetchPredictor::ShouldRecordRedirect( + net::URLRequest* response) { + const content::ResourceRequestInfo* request_info = + content::ResourceRequestInfo::ForRequest(response); + if (!request_info) + return false; + + if (!request_info->IsMainFrame()) + return false; + + return request_info->GetResourceType() == content::RESOURCE_TYPE_MAIN_FRAME && + IsHandledMainPage(response); +} + +// static +bool ResourcePrefetchPredictor::IsHandledMainPage(net::URLRequest* request) { + return request->original_url().scheme() == url::kHttpScheme; +} + +// static +bool ResourcePrefetchPredictor::IsHandledSubresource( + net::URLRequest* response) { + int resource_status = 0; + if (response->first_party_for_cookies().scheme() != url::kHttpScheme) + resource_status |= RESOURCE_STATUS_NOT_HTTP_PAGE; + + if (response->original_url().scheme() != url::kHttpScheme) + resource_status |= RESOURCE_STATUS_NOT_HTTP_RESOURCE; + + std::string mime_type; + response->GetMimeType(&mime_type); + if (!mime_type.empty() && + !net::IsSupportedImageMimeType(mime_type.c_str()) && + !net::IsSupportedJavascriptMimeType(mime_type.c_str()) && + !net::MatchesMimeType("text/css", mime_type)) { + resource_status |= RESOURCE_STATUS_UNSUPPORTED_MIME_TYPE; + } + + if (response->method() != "GET") + resource_status |= RESOURCE_STATUS_NOT_GET; + + if (response->original_url().spec().length() > + ResourcePrefetchPredictorTables::kMaxStringLength) { + resource_status |= RESOURCE_STATUS_URL_TOO_LONG; + } + + if (!response->response_info().headers.get()) + resource_status |= RESOURCE_STATUS_HEADERS_MISSING; + + if (!IsCacheable(response)) + resource_status |= RESOURCE_STATUS_NOT_CACHEABLE; + + UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.ResourceStatus", + resource_status, + RESOURCE_STATUS_MAX); + + return resource_status == 0; +} + +// static +bool ResourcePrefetchPredictor::IsCacheable(const net::URLRequest* response) { + if (response->was_cached()) + return true; + + // For non cached responses, we will ensure that the freshness lifetime is + // some sane value. + const net::HttpResponseInfo& response_info = response->response_info(); + if (!response_info.headers.get()) + return false; + base::Time response_time(response_info.response_time); + response_time += base::TimeDelta::FromSeconds(1); + base::TimeDelta freshness = response_info.headers->GetFreshnessLifetime( + response_time); + return freshness > base::TimeDelta(); +} + +// static +content::ResourceType ResourcePrefetchPredictor::GetResourceTypeFromMimeType( + const std::string& mime_type, + content::ResourceType fallback) { + if (net::IsSupportedImageMimeType(mime_type.c_str())) + return content::RESOURCE_TYPE_IMAGE; + else if (net::IsSupportedJavascriptMimeType(mime_type.c_str())) + return content::RESOURCE_TYPE_SCRIPT; + else if (net::MatchesMimeType("text/css", mime_type)) + return content::RESOURCE_TYPE_STYLESHEET; + else + return fallback; +} + +//////////////////////////////////////////////////////////////////////////////// +// ResourcePrefetchPredictor structs. + +ResourcePrefetchPredictor::URLRequestSummary::URLRequestSummary() + : resource_type(content::RESOURCE_TYPE_LAST_TYPE), + was_cached(false) { +} + +ResourcePrefetchPredictor::URLRequestSummary::URLRequestSummary( + const URLRequestSummary& other) + : navigation_id(other.navigation_id), + resource_url(other.resource_url), + resource_type(other.resource_type), + mime_type(other.mime_type), + was_cached(other.was_cached), + redirect_url(other.redirect_url) { +} + +ResourcePrefetchPredictor::URLRequestSummary::~URLRequestSummary() { +} + +ResourcePrefetchPredictor::Result::Result( + PrefetchKeyType i_key_type, + ResourcePrefetcher::RequestVector* i_requests) + : key_type(i_key_type), + requests(i_requests) { +} + +ResourcePrefetchPredictor::Result::~Result() { +} + +//////////////////////////////////////////////////////////////////////////////// +// ResourcePrefetchPredictor. + +ResourcePrefetchPredictor::ResourcePrefetchPredictor( + const ResourcePrefetchPredictorConfig& config, + Profile* profile) + : profile_(profile), + config_(config), + initialization_state_(NOT_INITIALIZED), + tables_(PredictorDatabaseFactory::GetForProfile( + profile)->resource_prefetch_tables()), + results_map_deleter_(&results_map_) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + // Some form of learning has to be enabled. + DCHECK(config_.IsLearningEnabled()); + if (config_.IsURLPrefetchingEnabled()) + DCHECK(config_.IsURLLearningEnabled()); + if (config_.IsHostPrefetchingEnabled()) + DCHECK(config_.IsHostLearningEnabled()); +} + +ResourcePrefetchPredictor::~ResourcePrefetchPredictor() { +} + +void ResourcePrefetchPredictor::RecordURLRequest( + const URLRequestSummary& request) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (initialization_state_ != INITIALIZED) + return; + + CHECK_EQ(request.resource_type, content::RESOURCE_TYPE_MAIN_FRAME); + OnMainFrameRequest(request); +} + +void ResourcePrefetchPredictor::RecordURLResponse( + const URLRequestSummary& response) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (initialization_state_ != INITIALIZED) + return; + + if (response.resource_type == content::RESOURCE_TYPE_MAIN_FRAME) + OnMainFrameResponse(response); + else + OnSubresourceResponse(response); +} + +void ResourcePrefetchPredictor::RecordURLRedirect( + const URLRequestSummary& response) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (initialization_state_ != INITIALIZED) + return; + + CHECK_EQ(response.resource_type, content::RESOURCE_TYPE_MAIN_FRAME); + OnMainFrameRedirect(response); +} + +void ResourcePrefetchPredictor::RecordMainFrameLoadComplete( + const NavigationID& navigation_id) { + switch (initialization_state_) { + case NOT_INITIALIZED: + StartInitialization(); + break; + case INITIALIZING: + break; + case INITIALIZED: { + RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD); + // WebContents can return an empty URL if the navigation entry + // corresponding to the navigation has not been created yet. + if (navigation_id.main_frame_url.is_empty()) + RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD_EMPTY_URL); + else + OnNavigationComplete(navigation_id); + break; + } + default: + NOTREACHED() << "Unexpected initialization_state_: " + << initialization_state_; + } +} + +void ResourcePrefetchPredictor::FinishedPrefetchForNavigation( + const NavigationID& navigation_id, + PrefetchKeyType key_type, + ResourcePrefetcher::RequestVector* requests) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + Result* result = new Result(key_type, requests); + // Add the results to the results map. + if (!results_map_.insert(std::make_pair(navigation_id, result)).second) { + DLOG(FATAL) << "Returning results for existing navigation."; + delete result; + } +} + +void ResourcePrefetchPredictor::Observe( + int type, + const content::NotificationSource& source, + const content::NotificationDetails& details) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + switch (type) { + case chrome::NOTIFICATION_HISTORY_LOADED: { + DCHECK_EQ(initialization_state_, INITIALIZING); + notification_registrar_.Remove(this, + chrome::NOTIFICATION_HISTORY_LOADED, + content::Source(profile_)); + OnHistoryAndCacheLoaded(); + break; + } + + case chrome::NOTIFICATION_HISTORY_URLS_DELETED: { + DCHECK_EQ(initialization_state_, INITIALIZED); + const content::Details + urls_deleted_details = + content::Details(details); + if (urls_deleted_details->all_history) { + DeleteAllUrls(); + UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.ReportingEvent", + REPORTING_EVENT_ALL_HISTORY_CLEARED, + REPORTING_EVENT_COUNT); + } else { + DeleteUrls(urls_deleted_details->rows); + UMA_HISTOGRAM_ENUMERATION("ResourcePrefetchPredictor.ReportingEvent", + REPORTING_EVENT_PARTIAL_HISTORY_CLEARED, + REPORTING_EVENT_COUNT); + } + break; + } + + default: + NOTREACHED() << "Unexpected notification observed."; + break; + } +} + +void ResourcePrefetchPredictor::Shutdown() { + if (prefetch_manager_.get()) { + prefetch_manager_->ShutdownOnUIThread(); + prefetch_manager_ = NULL; + } +} + +void ResourcePrefetchPredictor::OnMainFrameRequest( + const URLRequestSummary& request) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK_EQ(INITIALIZED, initialization_state_); + + RecordNavigationEvent(NAVIGATION_EVENT_REQUEST_STARTED); + + StartPrefetching(request.navigation_id); + + // Cleanup older navigations. + CleanupAbandonedNavigations(request.navigation_id); + + // New empty navigation entry. + inflight_navigations_.insert(std::make_pair( + request.navigation_id, + make_linked_ptr(new std::vector()))); +} + +void ResourcePrefetchPredictor::OnMainFrameResponse( + const URLRequestSummary& response) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (initialization_state_ != INITIALIZED) + return; + + RecordNavigationEvent(NAVIGATION_EVENT_RESPONSE_STARTED); + + StopPrefetching(response.navigation_id); +} + +void ResourcePrefetchPredictor::OnMainFrameRedirect( + const URLRequestSummary& response) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + RecordNavigationEvent(NAVIGATION_EVENT_REQUEST_REDIRECTED); + + // TODO(shishir): There are significant gains to be had here if we can use the + // start URL in a redirect chain as the key to start prefetching. We can save + // of redirect times considerably assuming that the redirect chains do not + // change. + + // Stop any inflight prefetching. Remove the older navigation. + StopPrefetching(response.navigation_id); + inflight_navigations_.erase(response.navigation_id); + + // A redirect will not lead to another OnMainFrameRequest call, so record the + // redirect url as a new navigation. + + // The redirect url may be empty if the url was invalid. + if (response.redirect_url.is_empty()) { + RecordNavigationEvent(NAVIGATION_EVENT_REQUEST_REDIRECTED_EMPTY_URL); + return; + } + + NavigationID navigation_id(response.navigation_id); + navigation_id.main_frame_url = response.redirect_url; + inflight_navigations_.insert(std::make_pair( + navigation_id, + make_linked_ptr(new std::vector()))); +} + +void ResourcePrefetchPredictor::OnSubresourceResponse( + const URLRequestSummary& response) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + NavigationMap::const_iterator nav_it = + inflight_navigations_.find(response.navigation_id); + if (nav_it == inflight_navigations_.end()) { + return; + } + + nav_it->second->push_back(response); +} + +void ResourcePrefetchPredictor::OnNavigationComplete( + const NavigationID& navigation_id) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + NavigationMap::iterator nav_it = + inflight_navigations_.find(navigation_id); + if (nav_it == inflight_navigations_.end()) { + RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD_UNTRACKED_URL); + return; + } + RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD_TRACKED_URL); + + // Report any stats. + if (prefetch_manager_.get()) { + ResultsMap::iterator results_it = results_map_.find(navigation_id); + bool have_prefetch_results = results_it != results_map_.end(); + UMA_HISTOGRAM_BOOLEAN("ResourcePrefetchPredictor.HavePrefetchResults", + have_prefetch_results); + if (have_prefetch_results) { + ReportAccuracyStats(results_it->second->key_type, + *(nav_it->second), + results_it->second->requests.get()); + } + } else { + scoped_ptr requests( + new ResourcePrefetcher::RequestVector); + PrefetchKeyType key_type; + if (GetPrefetchData(navigation_id, requests.get(), &key_type)) { + RecordNavigationEvent(NAVIGATION_EVENT_HAVE_PREDICTIONS_FOR_URL); + ReportPredictedAccuracyStats(key_type, + *(nav_it->second), + *requests); + } else { + RecordNavigationEvent(NAVIGATION_EVENT_NO_PREDICTIONS_FOR_URL); + } + } + + // Remove the navigation from the inflight navigations. + std::vector* requests = (nav_it->second).release(); + inflight_navigations_.erase(nav_it); + + // Kick off history lookup to determine if we should record the URL. + HistoryService* history_service = HistoryServiceFactory::GetForProfile( + profile_, Profile::EXPLICIT_ACCESS); + DCHECK(history_service); + history_service->ScheduleDBTask( + scoped_ptr( + new GetUrlVisitCountTask( + navigation_id, + requests, + base::Bind(&ResourcePrefetchPredictor::OnVisitCountLookup, + AsWeakPtr()))), + &history_lookup_consumer_); +} + +bool ResourcePrefetchPredictor::GetPrefetchData( + const NavigationID& navigation_id, + ResourcePrefetcher::RequestVector* prefetch_requests, + PrefetchKeyType* key_type) { + DCHECK(prefetch_requests); + DCHECK(key_type); + + *key_type = PREFETCH_KEY_TYPE_URL; + const GURL& main_frame_url = navigation_id.main_frame_url; + + bool use_url_data = config_.IsPrefetchingEnabled() ? + config_.IsURLPrefetchingEnabled() : config_.IsURLLearningEnabled(); + if (use_url_data) { + PrefetchDataMap::const_iterator iterator = + url_table_cache_->find(main_frame_url.spec()); + if (iterator != url_table_cache_->end()) + PopulatePrefetcherRequest(iterator->second, prefetch_requests); + } + if (!prefetch_requests->empty()) + return true; + + bool use_host_data = config_.IsPrefetchingEnabled() ? + config_.IsHostPrefetchingEnabled() : config_.IsHostLearningEnabled(); + if (use_host_data) { + PrefetchDataMap::const_iterator iterator = + host_table_cache_->find(main_frame_url.host()); + if (iterator != host_table_cache_->end()) { + *key_type = PREFETCH_KEY_TYPE_HOST; + PopulatePrefetcherRequest(iterator->second, prefetch_requests); + } + } + + return !prefetch_requests->empty(); +} + +void ResourcePrefetchPredictor::PopulatePrefetcherRequest( + const PrefetchData& data, + ResourcePrefetcher::RequestVector* requests) { + for (ResourceRows::const_iterator it = data.resources.begin(); + it != data.resources.end(); ++it) { + float confidence = static_cast(it->number_of_hits) / + (it->number_of_hits + it->number_of_misses); + if (confidence < config_.min_resource_confidence_to_trigger_prefetch || + it->number_of_hits < config_.min_resource_hits_to_trigger_prefetch) { + continue; + } + + ResourcePrefetcher::Request* req = new ResourcePrefetcher::Request( + it->resource_url); + requests->push_back(req); + } +} + +void ResourcePrefetchPredictor::StartPrefetching( + const NavigationID& navigation_id) { + if (!prefetch_manager_.get()) // Prefetching not enabled. + return; + + // Prefer URL based data first. + scoped_ptr requests( + new ResourcePrefetcher::RequestVector); + PrefetchKeyType key_type; + if (!GetPrefetchData(navigation_id, requests.get(), &key_type)) { + // No prefetching data at host or URL level. + return; + } + + BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, + base::Bind(&ResourcePrefetcherManager::MaybeAddPrefetch, + prefetch_manager_, + navigation_id, + key_type, + base::Passed(&requests))); +} + +void ResourcePrefetchPredictor::StopPrefetching( + const NavigationID& navigation_id) { + if (!prefetch_manager_.get()) // Not enabled. + return; + + BrowserThread::PostTask( + BrowserThread::IO, FROM_HERE, + base::Bind(&ResourcePrefetcherManager::MaybeRemovePrefetch, + prefetch_manager_, + navigation_id)); +} + +void ResourcePrefetchPredictor::StartInitialization() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + DCHECK_EQ(initialization_state_, NOT_INITIALIZED); + initialization_state_ = INITIALIZING; + + // Create local caches using the database as loaded. + scoped_ptr url_data_map(new PrefetchDataMap()); + scoped_ptr host_data_map(new PrefetchDataMap()); + PrefetchDataMap* url_data_ptr = url_data_map.get(); + PrefetchDataMap* host_data_ptr = host_data_map.get(); + + BrowserThread::PostTaskAndReply( + BrowserThread::DB, FROM_HERE, + base::Bind(&ResourcePrefetchPredictorTables::GetAllData, + tables_, url_data_ptr, host_data_ptr), + base::Bind(&ResourcePrefetchPredictor::CreateCaches, AsWeakPtr(), + base::Passed(&url_data_map), base::Passed(&host_data_map))); +} + +void ResourcePrefetchPredictor::CreateCaches( + scoped_ptr url_data_map, + scoped_ptr host_data_map) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + DCHECK_EQ(initialization_state_, INITIALIZING); + DCHECK(!url_table_cache_); + DCHECK(!host_table_cache_); + DCHECK(inflight_navigations_.empty()); + + url_table_cache_.reset(url_data_map.release()); + host_table_cache_.reset(host_data_map.release()); + + UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.UrlTableMainFrameUrlCount", + url_table_cache_->size()); + UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HostTableHostCount", + host_table_cache_->size()); + + // Add notifications for history loading if it is not ready. + HistoryService* history_service = HistoryServiceFactory::GetForProfile( + profile_, Profile::EXPLICIT_ACCESS); + if (!history_service) { + notification_registrar_.Add(this, chrome::NOTIFICATION_HISTORY_LOADED, + content::Source(profile_)); + } else { + OnHistoryAndCacheLoaded(); + } +} + +void ResourcePrefetchPredictor::OnHistoryAndCacheLoaded() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK_EQ(initialization_state_, INITIALIZING); + + notification_registrar_.Add(this, + chrome::NOTIFICATION_HISTORY_URLS_DELETED, + content::Source(profile_)); + + // Initialize the prefetch manager only if prefetching is enabled. + if (config_.IsPrefetchingEnabled()) { + prefetch_manager_ = new ResourcePrefetcherManager( + this, config_, profile_->GetRequestContext()); + } + + initialization_state_ = INITIALIZED; +} + +void ResourcePrefetchPredictor::CleanupAbandonedNavigations( + const NavigationID& navigation_id) { + static const base::TimeDelta max_navigation_age = + base::TimeDelta::FromSeconds(config_.max_navigation_lifetime_seconds); + + base::TimeTicks time_now = base::TimeTicks::Now(); + for (NavigationMap::iterator it = inflight_navigations_.begin(); + it != inflight_navigations_.end();) { + if (it->first.IsSameRenderer(navigation_id) || + (time_now - it->first.creation_time > max_navigation_age)) { + inflight_navigations_.erase(it++); + RecordNavigationEvent(NAVIGATION_EVENT_REQUEST_EXPIRED); + } else { + ++it; + } + } + for (ResultsMap::iterator it = results_map_.begin(); + it != results_map_.end();) { + if (it->first.IsSameRenderer(navigation_id) || + (time_now - it->first.creation_time > max_navigation_age)) { + delete it->second; + results_map_.erase(it++); + } else { + ++it; + } + } +} + +void ResourcePrefetchPredictor::DeleteAllUrls() { + inflight_navigations_.clear(); + url_table_cache_->clear(); + host_table_cache_->clear(); + + BrowserThread::PostTask(BrowserThread::DB, FROM_HERE, + base::Bind(&ResourcePrefetchPredictorTables::DeleteAllData, tables_)); +} + +void ResourcePrefetchPredictor::DeleteUrls(const history::URLRows& urls) { + // Check all the urls in the database and pick out the ones that are present + // in the cache. + std::vector urls_to_delete, hosts_to_delete; + + for (history::URLRows::const_iterator it = urls.begin(); it != urls.end(); + ++it) { + const std::string url_spec = it->url().spec(); + if (url_table_cache_->find(url_spec) != url_table_cache_->end()) { + urls_to_delete.push_back(url_spec); + url_table_cache_->erase(url_spec); + } + + const std::string host = it->url().host(); + if (host_table_cache_->find(host) != host_table_cache_->end()) { + hosts_to_delete.push_back(host); + host_table_cache_->erase(host); + } + } + + if (!urls_to_delete.empty() || !hosts_to_delete.empty()) { + BrowserThread::PostTask(BrowserThread::DB, FROM_HERE, + base::Bind(&ResourcePrefetchPredictorTables::DeleteData, + tables_, + urls_to_delete, + hosts_to_delete)); + } +} + +void ResourcePrefetchPredictor::RemoveOldestEntryInPrefetchDataMap( + PrefetchKeyType key_type, + PrefetchDataMap* data_map) { + if (data_map->empty()) + return; + + base::Time oldest_time; + std::string key_to_delete; + for (PrefetchDataMap::iterator it = data_map->begin(); + it != data_map->end(); ++it) { + if (key_to_delete.empty() || it->second.last_visit < oldest_time) { + key_to_delete = it->first; + oldest_time = it->second.last_visit; + } + } + + data_map->erase(key_to_delete); + BrowserThread::PostTask(BrowserThread::DB, FROM_HERE, + base::Bind(&ResourcePrefetchPredictorTables::DeleteSingleDataPoint, + tables_, + key_to_delete, + key_type)); +} + +void ResourcePrefetchPredictor::OnVisitCountLookup( + size_t visit_count, + const NavigationID& navigation_id, + const std::vector& requests) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HistoryVisitCountForUrl", + visit_count); + + // URL level data - merge only if we are already saving the data, or we it + // meets the cutoff requirement. + const std::string url_spec = navigation_id.main_frame_url.spec(); + bool already_tracking = url_table_cache_->find(url_spec) != + url_table_cache_->end(); + bool should_track_url = already_tracking || + (visit_count >= config_.min_url_visit_count); + + if (should_track_url) { + RecordNavigationEvent(NAVIGATION_EVENT_SHOULD_TRACK_URL); + + if (config_.IsURLLearningEnabled()) { + LearnNavigation(url_spec, PREFETCH_KEY_TYPE_URL, requests, + config_.max_urls_to_track, url_table_cache_.get()); + } + } else { + RecordNavigationEvent(NAVIGATION_EVENT_SHOULD_NOT_TRACK_URL); + } + + // Host level data - no cutoff, always learn the navigation if enabled. + if (config_.IsHostLearningEnabled()) { + LearnNavigation(navigation_id.main_frame_url.host(), + PREFETCH_KEY_TYPE_HOST, + requests, + config_.max_hosts_to_track, + host_table_cache_.get()); + } + + // Remove the navigation from the results map. + ResultsMap::iterator results_it = results_map_.find(navigation_id); + if (results_it != results_map_.end()) { + delete results_it->second; + results_map_.erase(results_it); + } +} + +void ResourcePrefetchPredictor::LearnNavigation( + const std::string& key, + PrefetchKeyType key_type, + const std::vector& new_resources, + size_t max_data_map_size, + PrefetchDataMap* data_map) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + // If the primary key is too long reject it. + if (key.length() > ResourcePrefetchPredictorTables::kMaxStringLength) { + if (key_type == PREFETCH_KEY_TYPE_HOST) + RecordNavigationEvent(NAVIGATION_EVENT_HOST_TOO_LONG); + else + RecordNavigationEvent(NAVIGATION_EVENT_MAIN_FRAME_URL_TOO_LONG); + return; + } + + PrefetchDataMap::iterator cache_entry = data_map->find(key); + if (cache_entry == data_map->end()) { + if (data_map->size() >= max_data_map_size) { + // The table is full, delete an entry. + RemoveOldestEntryInPrefetchDataMap(key_type, data_map); + } + + cache_entry = data_map->insert(std::make_pair( + key, PrefetchData(key_type, key))).first; + cache_entry->second.last_visit = base::Time::Now(); + size_t new_resources_size = new_resources.size(); + std::set resources_seen; + for (size_t i = 0; i < new_resources_size; ++i) { + if (resources_seen.find(new_resources[i].resource_url) != + resources_seen.end()) { + continue; + } + ResourceRow row_to_add; + row_to_add.resource_url = new_resources[i].resource_url; + row_to_add.resource_type = new_resources[i].resource_type; + row_to_add.number_of_hits = 1; + row_to_add.average_position = i + 1; + cache_entry->second.resources.push_back(row_to_add); + resources_seen.insert(new_resources[i].resource_url); + } + } else { + ResourceRows& old_resources = cache_entry->second.resources; + cache_entry->second.last_visit = base::Time::Now(); + + // Build indices over the data. + std::map new_index, old_index; + int new_resources_size = static_cast(new_resources.size()); + for (int i = 0; i < new_resources_size; ++i) { + const URLRequestSummary& summary = new_resources[i]; + // Take the first occurence of every url. + if (new_index.find(summary.resource_url) == new_index.end()) + new_index[summary.resource_url] = i; + } + int old_resources_size = static_cast(old_resources.size()); + for (int i = 0; i < old_resources_size; ++i) { + const ResourceRow& row = old_resources[i]; + DCHECK(old_index.find(row.resource_url) == old_index.end()); + old_index[row.resource_url] = i; + } + + // Go through the old urls and update their hit/miss counts. + for (int i = 0; i < old_resources_size; ++i) { + ResourceRow& old_row = old_resources[i]; + if (new_index.find(old_row.resource_url) == new_index.end()) { + ++old_row.number_of_misses; + ++old_row.consecutive_misses; + } else { + const URLRequestSummary& new_row = + new_resources[new_index[old_row.resource_url]]; + + // Update the resource type since it could have changed. + if (new_row.resource_type != content::RESOURCE_TYPE_LAST_TYPE) + old_row.resource_type = new_row.resource_type; + + int position = new_index[old_row.resource_url] + 1; + int total = old_row.number_of_hits + old_row.number_of_misses; + old_row.average_position = + ((old_row.average_position * total) + position) / (total + 1); + ++old_row.number_of_hits; + old_row.consecutive_misses = 0; + } + } + + // Add the new ones that we have not seen before. + for (int i = 0; i < new_resources_size; ++i) { + const URLRequestSummary& summary = new_resources[i]; + if (old_index.find(summary.resource_url) != old_index.end()) + continue; + + // Only need to add new stuff. + ResourceRow row_to_add; + row_to_add.resource_url = summary.resource_url; + row_to_add.resource_type = summary.resource_type; + row_to_add.number_of_hits = 1; + row_to_add.average_position = i + 1; + old_resources.push_back(row_to_add); + + // To ensure we dont add the same url twice. + old_index[summary.resource_url] = 0; + } + } + + // Trim and sort the resources after the update. + ResourceRows& resources = cache_entry->second.resources; + for (ResourceRows::iterator it = resources.begin(); + it != resources.end();) { + it->UpdateScore(); + if (it->consecutive_misses >= config_.max_consecutive_misses) + it = resources.erase(it); + else + ++it; + } + std::sort(resources.begin(), resources.end(), + ResourcePrefetchPredictorTables::ResourceRowSorter()); + if (resources.size() > config_.max_resources_per_entry) + resources.resize(config_.max_resources_per_entry); + + // If the row has no resources, remove it from the cache and delete the + // entry in the database. Else update the database. + if (resources.empty()) { + data_map->erase(key); + BrowserThread::PostTask( + BrowserThread::DB, FROM_HERE, + base::Bind(&ResourcePrefetchPredictorTables::DeleteSingleDataPoint, + tables_, + key, + key_type)); + } else { + bool is_host = key_type == PREFETCH_KEY_TYPE_HOST; + PrefetchData empty_data( + !is_host ? PREFETCH_KEY_TYPE_HOST : PREFETCH_KEY_TYPE_URL, + std::string()); + const PrefetchData& host_data = is_host ? cache_entry->second : empty_data; + const PrefetchData& url_data = is_host ? empty_data : cache_entry->second; + BrowserThread::PostTask( + BrowserThread::DB, FROM_HERE, + base::Bind(&ResourcePrefetchPredictorTables::UpdateData, + tables_, + url_data, + host_data)); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Accuracy measurement. + +void ResourcePrefetchPredictor::ReportAccuracyStats( + PrefetchKeyType key_type, + const std::vector& actual, + ResourcePrefetcher::RequestVector* prefetched) const { + // Annotate the results. + std::map actual_resources; + for (std::vector::const_iterator it = actual.begin(); + it != actual.end(); ++it) { + actual_resources[it->resource_url] = it->was_cached; + } + + int prefetch_cancelled = 0, prefetch_failed = 0, prefetch_not_started = 0; + // 'a_' -> actual, 'p_' -> predicted. + int p_cache_a_cache = 0, p_cache_a_network = 0, p_cache_a_notused = 0, + p_network_a_cache = 0, p_network_a_network = 0, p_network_a_notused = 0; + + for (ResourcePrefetcher::RequestVector::iterator it = prefetched->begin(); + it != prefetched->end(); ++it) { + ResourcePrefetcher::Request* req = *it; + + // Set the usage states if the resource was actually used. + std::map::iterator actual_it = actual_resources.find( + req->resource_url); + if (actual_it != actual_resources.end()) { + if (actual_it->second) { + req->usage_status = + ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE; + } else { + req->usage_status = + ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK; + } + } + + switch (req->prefetch_status) { + // TODO(shishir): Add histogram for each cancellation reason. + case ResourcePrefetcher::Request::PREFETCH_STATUS_REDIRECTED: + case ResourcePrefetcher::Request::PREFETCH_STATUS_AUTH_REQUIRED: + case ResourcePrefetcher::Request::PREFETCH_STATUS_CERT_REQUIRED: + case ResourcePrefetcher::Request::PREFETCH_STATUS_CERT_ERROR: + case ResourcePrefetcher::Request::PREFETCH_STATUS_CANCELLED: + ++prefetch_cancelled; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_FAILED: + ++prefetch_failed; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_FROM_CACHE: + if (req->usage_status == + ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE) + ++p_cache_a_cache; + else if (req->usage_status == + ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK) + ++p_cache_a_network; + else + ++p_cache_a_notused; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_FROM_NETWORK: + if (req->usage_status == + ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE) + ++p_network_a_cache; + else if (req->usage_status == + ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK) + ++p_network_a_network; + else + ++p_network_a_notused; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_NOT_STARTED: + ++prefetch_not_started; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_STARTED: + DLOG(FATAL) << "Invalid prefetch status"; + break; + } + } + + int total_prefetched = p_cache_a_cache + p_cache_a_network + p_cache_a_notused + + p_network_a_cache + p_network_a_network + p_network_a_notused; + + std::string histogram_type = key_type == PREFETCH_KEY_TYPE_HOST ? "Host." : + "Url."; + + // Macros to avoid using the STATIC_HISTOGRAM_POINTER_BLOCK in UMA_HISTOGRAM + // definitions. +#define RPP_HISTOGRAM_PERCENTAGE(suffix, value) \ + { \ + std::string name = "ResourcePrefetchPredictor." + histogram_type + suffix; \ + std::string g_name = "ResourcePrefetchPredictor." + std::string(suffix); \ + base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( \ + name, 1, 101, 102, base::Histogram::kUmaTargetedHistogramFlag); \ + histogram->Add(value); \ + UMA_HISTOGRAM_PERCENTAGE(g_name, value); \ + } + + RPP_HISTOGRAM_PERCENTAGE("PrefetchCancelled", + prefetch_cancelled * 100.0 / total_prefetched); + RPP_HISTOGRAM_PERCENTAGE("PrefetchFailed", + prefetch_failed * 100.0 / total_prefetched); + RPP_HISTOGRAM_PERCENTAGE("PrefetchFromCacheUsedFromCache", + p_cache_a_cache * 100.0 / total_prefetched); + RPP_HISTOGRAM_PERCENTAGE("PrefetchFromCacheUsedFromNetwork", + p_cache_a_network * 100.0 / total_prefetched); + RPP_HISTOGRAM_PERCENTAGE("PrefetchFromCacheNotUsed", + p_cache_a_notused * 100.0 / total_prefetched); + RPP_HISTOGRAM_PERCENTAGE("PrefetchFromNetworkUsedFromCache", + p_network_a_cache * 100.0 / total_prefetched); + RPP_HISTOGRAM_PERCENTAGE("PrefetchFromNetworkUsedFromNetwork", + p_network_a_network * 100.0 / total_prefetched); + RPP_HISTOGRAM_PERCENTAGE("PrefetchFromNetworkNotUsed", + p_network_a_notused * 100.0 / total_prefetched); + + RPP_HISTOGRAM_PERCENTAGE( + "PrefetchNotStarted", + prefetch_not_started * 100.0 / (prefetch_not_started + total_prefetched)); + +#undef RPP_HISTOGRAM_PERCENTAGE +} + +void ResourcePrefetchPredictor::ReportPredictedAccuracyStats( + PrefetchKeyType key_type, + const std::vector& actual, + const ResourcePrefetcher::RequestVector& predicted) const { + std::map actual_resources; + int from_network = 0; + for (std::vector::const_iterator it = actual.begin(); + it != actual.end(); ++it) { + actual_resources[it->resource_url] = it->was_cached; + if (!it->was_cached) + ++from_network; + } + + // Measure the accuracy at 25, 50 predicted resources. + ReportPredictedAccuracyStatsHelper(key_type, predicted, actual_resources, + from_network, 25); + ReportPredictedAccuracyStatsHelper(key_type, predicted, actual_resources, + from_network, 50); +} + +void ResourcePrefetchPredictor::ReportPredictedAccuracyStatsHelper( + PrefetchKeyType key_type, + const ResourcePrefetcher::RequestVector& predicted, + const std::map& actual, + size_t total_resources_fetched_from_network, + size_t max_assumed_prefetched) const { + int prefetch_cached = 0, prefetch_network = 0, prefetch_missed = 0; + int num_assumed_prefetched = std::min(predicted.size(), + max_assumed_prefetched); + if (num_assumed_prefetched == 0) + return; + + for (int i = 0; i < num_assumed_prefetched; ++i) { + const ResourcePrefetcher::Request& row = *(predicted[i]); + std::map::const_iterator it = actual.find(row.resource_url); + if (it == actual.end()) { + ++prefetch_missed; + } else if (it->second) { + ++prefetch_cached; + } else { + ++prefetch_network; + } + } + + std::string prefix = key_type == PREFETCH_KEY_TYPE_HOST ? + "ResourcePrefetchPredictor.Host.Predicted" : + "ResourcePrefetchPredictor.Url.Predicted"; + std::string suffix = "_" + base::IntToString(max_assumed_prefetched); + + // Macros to avoid using the STATIC_HISTOGRAM_POINTER_BLOCK in UMA_HISTOGRAM + // definitions. +#define RPP_PREDICTED_HISTOGRAM_COUNTS(name, value) \ + { \ + std::string full_name = prefix + name + suffix; \ + base::HistogramBase* histogram = base::Histogram::FactoryGet( \ + full_name, 1, 1000000, 50, \ + base::Histogram::kUmaTargetedHistogramFlag); \ + histogram->Add(value); \ + } + +#define RPP_PREDICTED_HISTOGRAM_PERCENTAGE(name, value) \ + { \ + std::string full_name = prefix + name + suffix; \ + base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( \ + full_name, 1, 101, 102, base::Histogram::kUmaTargetedHistogramFlag); \ + histogram->Add(value); \ + } + + RPP_PREDICTED_HISTOGRAM_COUNTS("PrefetchCount", num_assumed_prefetched); + RPP_PREDICTED_HISTOGRAM_COUNTS("PrefetchMisses_Count", prefetch_missed); + RPP_PREDICTED_HISTOGRAM_COUNTS("PrefetchFromCache_Count", prefetch_cached); + RPP_PREDICTED_HISTOGRAM_COUNTS("PrefetchFromNetwork_Count", prefetch_network); + + RPP_PREDICTED_HISTOGRAM_PERCENTAGE( + "PrefetchMisses_PercentOfTotalPrefetched", + prefetch_missed * 100.0 / num_assumed_prefetched); + RPP_PREDICTED_HISTOGRAM_PERCENTAGE( + "PrefetchFromCache_PercentOfTotalPrefetched", + prefetch_cached * 100.0 / num_assumed_prefetched); + RPP_PREDICTED_HISTOGRAM_PERCENTAGE( + "PrefetchFromNetwork_PercentOfTotalPrefetched", + prefetch_network * 100.0 / num_assumed_prefetched); + + // Measure the ratio of total number of resources prefetched from network vs + // the total number of resources fetched by the page from the network. + if (total_resources_fetched_from_network > 0) { + RPP_PREDICTED_HISTOGRAM_PERCENTAGE( + "PrefetchFromNetworkPercentOfTotalFromNetwork", + prefetch_network * 100.0 / total_resources_fetched_from_network); + } + +#undef RPP_PREDICTED_HISTOGRAM_PERCENTAGE +#undef RPP_PREDICTED_HISTOGRAM_COUNTS +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetch_predictor.h b/chrome/browser/predictors/resource_prefetch_predictor.h new file mode 100644 index 0000000..5e6157b --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor.h @@ -0,0 +1,314 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_ + +#include +#include +#include + +#include "base/gtest_prod_util.h" +#include "base/memory/linked_ptr.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/weak_ptr.h" +#include "base/task/cancelable_task_tracker.h" +#include "base/time/time.h" +#include "chrome/browser/history/history_types.h" +#include "chrome/browser/predictors/resource_prefetch_common.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h" +#include "chrome/browser/predictors/resource_prefetcher.h" +#include "components/keyed_service/core/keyed_service.h" +#include "content/public/browser/notification_observer.h" +#include "content/public/browser/notification_registrar.h" +#include "content/public/common/resource_type.h" +#include "url/gurl.h" + +class PredictorsHandler; +class Profile; + +namespace content { +class WebContents; +} + +namespace net { +class URLRequest; +} + +namespace predictors { + +class ResourcePrefetcherManager; + +// Contains logic for learning what can be prefetched and for kicking off +// speculative prefetching. +// - The class is a profile keyed service owned by the profile. +// - All the non-static methods of this class need to be called on the UI +// thread. +// +// The overall flow of the resource prefetching algorithm is as follows: +// +// * ResourcePrefetchPredictorObserver - Listens for URL requests, responses and +// redirects on the IO thread (via ResourceDispatcherHostDelegate) and posts +// tasks to the ResourcePrefetchPredictor on the UI thread. This is owned by +// the ProfileIOData for the profile. +// * ResourcePrefetchPredictorTables - Persists ResourcePrefetchPredictor data +// to a sql database. Runs entirely on the DB thread. Owned by the +// PredictorDatabase. +// * ResourcePrefetchPredictor - Learns about resource requirements per URL in +// the UI thread through the ResourcePrefetchPredictorObserver and persists +// it to disk in the DB thread through the ResourcePrefetchPredictorTables. It +// initiates resource prefetching using the ResourcePrefetcherManager. Owned +// by profile. +// * ResourcePrefetcherManager - Manages the ResourcePrefetchers that do the +// prefetching on the IO thread. The manager is owned by the +// ResourcePrefetchPredictor and interfaces between the predictor on the UI +// thread and the prefetchers on the IO thread. +// * ResourcePrefetcher - Lives entirely on the IO thread, owned by the +// ResourcePrefetcherManager, and issues net::URLRequest to fetch resources. +// +// TODO(shishir): Do speculative prefetching for https resources and/or https +// main frame urls. +// TODO(zhenw): Currently only main frame requests/redirects/responses are +// recorded. Consider recording sub-frame responses independently or together +// with main frame. +class ResourcePrefetchPredictor + : public KeyedService, + public content::NotificationObserver, + public base::SupportsWeakPtr { + public: + // Stores the data that we need to get from the URLRequest. + struct URLRequestSummary { + URLRequestSummary(); + URLRequestSummary(const URLRequestSummary& other); + ~URLRequestSummary(); + + NavigationID navigation_id; + GURL resource_url; + content::ResourceType resource_type; + + // Only for responses. + std::string mime_type; + bool was_cached; + GURL redirect_url; // Empty unless request was redirected to a valid url. + }; + + ResourcePrefetchPredictor(const ResourcePrefetchPredictorConfig& config, + Profile* profile); + virtual ~ResourcePrefetchPredictor(); + + // Thread safe. + static bool ShouldRecordRequest(net::URLRequest* request, + content::ResourceType resource_type); + static bool ShouldRecordResponse(net::URLRequest* response); + static bool ShouldRecordRedirect(net::URLRequest* response); + + // Determines the ResourceType from the mime type, defaulting to the + // |fallback| if the ResourceType could not be determined. + static content::ResourceType GetResourceTypeFromMimeType( + const std::string& mime_type, + content::ResourceType fallback); + + // 'ResourcePrefetchPredictorObserver' calls the below functions to inform the + // predictor of main frame and resource requests. Should only be called if the + // corresponding Should* functions return true. + void RecordURLRequest(const URLRequestSummary& request); + void RecordURLResponse(const URLRequestSummary& response); + void RecordURLRedirect(const URLRequestSummary& response); + + // Called when the main frame of a page completes loading. + void RecordMainFrameLoadComplete(const NavigationID& navigation_id); + + // Called by ResourcePrefetcherManager to notify that prefetching has finished + // for a navigation. Should take ownership of |requests|. + virtual void FinishedPrefetchForNavigation( + const NavigationID& navigation_id, + PrefetchKeyType key_type, + ResourcePrefetcher::RequestVector* requests); + + private: + friend class ::PredictorsHandler; + friend class ResourcePrefetchPredictorTest; + + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, DeleteUrls); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, + LazilyInitializeEmpty); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, + LazilyInitializeWithData); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, + NavigationNotRecorded); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, NavigationUrlInDB); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, NavigationUrlNotInDB); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, + NavigationUrlNotInDBAndDBFull); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, OnMainFrameRequest); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, OnMainFrameRedirect); + FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, + OnSubresourceResponse); + + enum InitializationState { + NOT_INITIALIZED = 0, + INITIALIZING = 1, + INITIALIZED = 2 + }; + + // Stores prefetching results. + struct Result { + // Takes ownership of requests. + Result(PrefetchKeyType key_type, + ResourcePrefetcher::RequestVector* requests); + ~Result(); + + PrefetchKeyType key_type; + scoped_ptr requests; + + private: + DISALLOW_COPY_AND_ASSIGN(Result); + }; + + typedef ResourcePrefetchPredictorTables::ResourceRow ResourceRow; + typedef ResourcePrefetchPredictorTables::ResourceRows ResourceRows; + typedef ResourcePrefetchPredictorTables::PrefetchData PrefetchData; + typedef ResourcePrefetchPredictorTables::PrefetchDataMap PrefetchDataMap; + typedef std::map > > + NavigationMap; + typedef std::map ResultsMap; + + // Returns true if the main page request is supported for prediction. + static bool IsHandledMainPage(net::URLRequest* request); + + // Returns true if the subresource request is supported for prediction. + static bool IsHandledSubresource(net::URLRequest* request); + + // Returns true if the request (should have a response in it) is cacheable. + static bool IsCacheable(const net::URLRequest* request); + + // content::NotificationObserver methods OVERRIDE. + virtual void Observe(int type, + const content::NotificationSource& source, + const content::NotificationDetails& details) OVERRIDE; + + // KeyedService methods OVERRIDE. + virtual void Shutdown() OVERRIDE; + + // Functions called on different network events pertaining to the loading of + // main frame resource or sub resources. + void OnMainFrameRequest(const URLRequestSummary& request); + void OnMainFrameResponse(const URLRequestSummary& response); + void OnMainFrameRedirect(const URLRequestSummary& response); + void OnSubresourceResponse(const URLRequestSummary& response); + + // Called when onload completes for a navigation. We treat this point as the + // "completion" of the navigation. The resources requested by the page upto + // this point are the only ones considered for prefetching. + void OnNavigationComplete(const NavigationID& navigation_id); + + // Returns true if there is PrefetchData that can be used for the + // navigation and fills in the |prefetch_data| to resources that need to be + // prefetched. + bool GetPrefetchData(const NavigationID& navigation_id, + ResourcePrefetcher::RequestVector* prefetch_requests, + PrefetchKeyType* key_type); + + // Converts a PrefetchData into a ResourcePrefetcher::RequestVector. + void PopulatePrefetcherRequest(const PrefetchData& data, + ResourcePrefetcher::RequestVector* requests); + + // Starts prefetching if it is enabled and prefetching data exists for the + // NavigationID either at the URL or at the host level. + void StartPrefetching(const NavigationID& navigation_id); + + // Stops prefetching that may be in progress corresponding to |navigation_id|. + void StopPrefetching(const NavigationID& navigation_id); + + // Starts initialization by posting a task to the DB thread to read the + // predictor database. + void StartInitialization(); + + // Callback for task to read predictor database. Takes ownership of + // |url_data_map| and |host_data_map|. + void CreateCaches(scoped_ptr url_data_map, + scoped_ptr host_data_map); + + // Called during initialization when history is read and the predictor + // database has been read. + void OnHistoryAndCacheLoaded(); + + // Removes data for navigations where the onload never fired. Will cleanup + // inflight_navigations_ and results_map_. + void CleanupAbandonedNavigations(const NavigationID& navigation_id); + + // Deletes all URLs from the predictor database, the caches and removes all + // inflight navigations. + void DeleteAllUrls(); + + // Deletes data for the input |urls| and their corresponding hosts from the + // predictor database and caches. + void DeleteUrls(const history::URLRows& urls); + + // Callback for GetUrlVisitCountTask. + void OnVisitCountLookup(size_t visit_count, + const NavigationID& navigation_id, + const std::vector& requests); + + // Removes the oldest entry in the input |data_map|, also deleting it from the + // predictor database. + void RemoveOldestEntryInPrefetchDataMap(PrefetchKeyType key_type, + PrefetchDataMap* data_map); + + // Merges resources in |new_resources| into the |data_map| and correspondingly + // updates the predictor database. + void LearnNavigation(const std::string& key, + PrefetchKeyType key_type, + const std::vector& new_resources, + size_t max_data_map_size, + PrefetchDataMap* data_map); + + // Reports accuracy by comparing prefetched resources with resources that are + // actually used by the page. + void ReportAccuracyStats(PrefetchKeyType key_type, + const std::vector& actual, + ResourcePrefetcher::RequestVector* prefetched) const; + + // Reports predicted accuracy i.e. by comparing resources that are actually + // used by the page with those that may have been prefetched. + void ReportPredictedAccuracyStats( + PrefetchKeyType key_type, + const std::vector& actual, + const ResourcePrefetcher::RequestVector& predicted) const; + void ReportPredictedAccuracyStatsHelper( + PrefetchKeyType key_type, + const ResourcePrefetcher::RequestVector& predicted, + const std::map& actual, + size_t total_resources_fetched_from_network, + size_t max_assumed_prefetched) const; + + // Used for testing to inject mock tables. + void set_mock_tables(scoped_refptr tables) { + tables_ = tables; + } + + Profile* const profile_; + ResourcePrefetchPredictorConfig const config_; + InitializationState initialization_state_; + scoped_refptr tables_; + scoped_refptr prefetch_manager_; + content::NotificationRegistrar notification_registrar_; + base::CancelableTaskTracker history_lookup_consumer_; + + // Map of all the navigations in flight to their resource requests. + NavigationMap inflight_navigations_; + + // Copy of the data in the predictor tables. + scoped_ptr url_table_cache_; + scoped_ptr host_table_cache_; + + ResultsMap results_map_; + STLValueDeleter results_map_deleter_; + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictor); +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_ diff --git a/chrome/browser/predictors/resource_prefetch_predictor_factory.cc b/chrome/browser/predictors/resource_prefetch_predictor_factory.cc new file mode 100644 index 0000000..6bb4287 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor_factory.cc @@ -0,0 +1,52 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h" + +#include "chrome/browser/history/history_service_factory.h" +#include "chrome/browser/predictors/predictor_database_factory.h" +#include "chrome/browser/predictors/resource_prefetch_common.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "chrome/browser/profiles/profile.h" +#include "components/keyed_service/content/browser_context_dependency_manager.h" +#include "components/keyed_service/content/browser_context_keyed_service_factory.h" + +namespace predictors { + +// static +ResourcePrefetchPredictor* ResourcePrefetchPredictorFactory::GetForProfile( + content::BrowserContext* context) { + return static_cast( + GetInstance()->GetServiceForBrowserContext(context, true)); +} + +// static +ResourcePrefetchPredictorFactory* +ResourcePrefetchPredictorFactory::GetInstance() { + return Singleton::get(); +} + +ResourcePrefetchPredictorFactory::ResourcePrefetchPredictorFactory() + : BrowserContextKeyedServiceFactory( + "ResourcePrefetchPredictor", + BrowserContextDependencyManager::GetInstance()) { + DependsOn(HistoryServiceFactory::GetInstance()); + DependsOn(PredictorDatabaseFactory::GetInstance()); +} + +ResourcePrefetchPredictorFactory::~ResourcePrefetchPredictorFactory() {} + +KeyedService* + ResourcePrefetchPredictorFactory::BuildServiceInstanceFor( + content::BrowserContext* context) const { + Profile* profile = Profile::FromBrowserContext(context); + + ResourcePrefetchPredictorConfig config; + if (!IsSpeculativeResourcePrefetchingEnabled(profile, &config)) + return NULL; + + return new ResourcePrefetchPredictor(config, profile); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetch_predictor_factory.h b/chrome/browser/predictors/resource_prefetch_predictor_factory.h new file mode 100644 index 0000000..7c6f5e8 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor_factory.h @@ -0,0 +1,38 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_ + +#include "base/basictypes.h" +#include "base/memory/singleton.h" +#include "components/keyed_service/content/browser_context_keyed_service_factory.h" + +namespace predictors { + +class ResourcePrefetchPredictor; + +class ResourcePrefetchPredictorFactory + : public BrowserContextKeyedServiceFactory { + public: + static ResourcePrefetchPredictor* GetForProfile( + content::BrowserContext* context); + static ResourcePrefetchPredictorFactory* GetInstance(); + + private: + friend struct DefaultSingletonTraits; + + ResourcePrefetchPredictorFactory(); + virtual ~ResourcePrefetchPredictorFactory(); + + // RefcountedBrowserContextKeyedServiceFactory: + virtual KeyedService* BuildServiceInstanceFor( + content::BrowserContext* context) const OVERRIDE; + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorFactory); +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_FACTORY_H_ diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc b/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc new file mode 100644 index 0000000..ec827f1 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.cc @@ -0,0 +1,62 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h" + +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h" +#include "chrome/browser/profiles/profile.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/load_from_memory_cache_details.h" + +DEFINE_WEB_CONTENTS_USER_DATA_KEY( + predictors::ResourcePrefetchPredictorTabHelper); + +using content::BrowserThread; + +namespace predictors { + +ResourcePrefetchPredictorTabHelper::ResourcePrefetchPredictorTabHelper( + content::WebContents* web_contents) + : content::WebContentsObserver(web_contents) { +} + +ResourcePrefetchPredictorTabHelper::~ResourcePrefetchPredictorTabHelper() { +} + +void ResourcePrefetchPredictorTabHelper::DocumentOnLoadCompletedInMainFrame() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + ResourcePrefetchPredictor* predictor = + ResourcePrefetchPredictorFactory::GetForProfile( + web_contents()->GetBrowserContext()); + if (!predictor) + return; + + NavigationID navigation_id(web_contents()); + predictor->RecordMainFrameLoadComplete(navigation_id); +} + +void ResourcePrefetchPredictorTabHelper::DidLoadResourceFromMemoryCache( + const content::LoadFromMemoryCacheDetails& details) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + ResourcePrefetchPredictor* predictor = + ResourcePrefetchPredictorFactory::GetForProfile( + web_contents()->GetBrowserContext()); + if (!predictor) + return; + + ResourcePrefetchPredictor::URLRequestSummary summary; + summary.navigation_id = NavigationID(web_contents()); + summary.resource_url = details.url; + summary.mime_type = details.mime_type; + summary.resource_type = + ResourcePrefetchPredictor::GetResourceTypeFromMimeType( + details.mime_type, details.resource_type); + summary.was_cached = true; + predictor->RecordURLResponse(summary); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h b/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h new file mode 100644 index 0000000..4580bc8 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h @@ -0,0 +1,34 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_ + +#include "content/public/browser/web_contents_observer.h" +#include "content/public/browser/web_contents_user_data.h" + +namespace predictors { + +class ResourcePrefetchPredictorTabHelper + : public content::WebContentsObserver, + public content::WebContentsUserData { + public: + virtual ~ResourcePrefetchPredictorTabHelper(); + + // content::WebContentsObserver implementation + virtual void DocumentOnLoadCompletedInMainFrame() OVERRIDE; + virtual void DidLoadResourceFromMemoryCache( + const content::LoadFromMemoryCacheDetails& details) OVERRIDE; + + private: + explicit ResourcePrefetchPredictorTabHelper( + content::WebContents* web_contents); + friend class content::WebContentsUserData; + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorTabHelper); +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TAB_HELPER_H_ diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tables.cc b/chrome/browser/predictors/resource_prefetch_predictor_tables.cc new file mode 100644 index 0000000..89ebd7d --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor_tables.cc @@ -0,0 +1,513 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h" + +#include +#include + +#include "base/logging.h" +#include "base/metrics/histogram.h" +#include "base/strings/stringprintf.h" +#include "content/public/browser/browser_thread.h" +#include "sql/statement.h" + +using content::BrowserThread; +using sql::Statement; + +namespace { + +const char kUrlResourceTableName[] = "resource_prefetch_predictor_url"; +const char kUrlMetadataTableName[] = "resource_prefetch_predictor_url_metadata"; +const char kHostResourceTableName[] = "resource_prefetch_predictor_host"; +const char kHostMetadataTableName[] = + "resource_prefetch_predictor_host_metadata"; + +void BindResourceRowToStatement( + const predictors::ResourcePrefetchPredictorTables::ResourceRow& row, + const std::string& primary_key, + Statement* statement) { + statement->BindString(0, primary_key); + statement->BindString(1, row.resource_url.spec()); + statement->BindInt(2, static_cast(row.resource_type)); + statement->BindInt(3, row.number_of_hits); + statement->BindInt(4, row.number_of_misses); + statement->BindInt(5, row.consecutive_misses); + statement->BindDouble(6, row.average_position); +} + +bool StepAndInitializeResourceRow( + Statement* statement, + predictors::ResourcePrefetchPredictorTables::ResourceRow* row) { + if (!statement->Step()) + return false; + + row->primary_key = statement->ColumnString(0); + row->resource_url = GURL(statement->ColumnString(1)); + row->resource_type = static_cast( + statement->ColumnInt(2)); + row->number_of_hits = statement->ColumnInt(3); + row->number_of_misses = statement->ColumnInt(4); + row->consecutive_misses = statement->ColumnInt(5); + row->average_position = statement->ColumnDouble(6); + return true; +} + +} // namespace + +namespace predictors { + +// static +const size_t ResourcePrefetchPredictorTables::kMaxStringLength = 1024; + +ResourcePrefetchPredictorTables::ResourceRow::ResourceRow() + : resource_type(content::RESOURCE_TYPE_LAST_TYPE), + number_of_hits(0), + number_of_misses(0), + consecutive_misses(0), + average_position(0.0), + score(0.0) { +} + +ResourcePrefetchPredictorTables::ResourceRow::ResourceRow( + const ResourceRow& other) + : primary_key(other.primary_key), + resource_url(other.resource_url), + resource_type(other.resource_type), + number_of_hits(other.number_of_hits), + number_of_misses(other.number_of_misses), + consecutive_misses(other.consecutive_misses), + average_position(other.average_position), + score(other.score) { +} + +ResourcePrefetchPredictorTables::ResourceRow::ResourceRow( + const std::string& i_primary_key, + const std::string& i_resource_url, + content::ResourceType i_resource_type, + int i_number_of_hits, + int i_number_of_misses, + int i_consecutive_misses, + double i_average_position) + : primary_key(i_primary_key), + resource_url(i_resource_url), + resource_type(i_resource_type), + number_of_hits(i_number_of_hits), + number_of_misses(i_number_of_misses), + consecutive_misses(i_consecutive_misses), + average_position(i_average_position) { + UpdateScore(); +} + +void ResourcePrefetchPredictorTables::ResourceRow::UpdateScore() { + // The score is calculated so that when the rows are sorted, the stylesheets + // and scripts appear first, sorted by position(ascending) and then the rest + // of the resources sorted by position(ascending). + static const int kMaxResourcesPerType = 100; + switch (resource_type) { + case content::RESOURCE_TYPE_STYLESHEET: + case content::RESOURCE_TYPE_SCRIPT: + score = (2 * kMaxResourcesPerType) - average_position; + break; + + case content::RESOURCE_TYPE_IMAGE: + default: + score = kMaxResourcesPerType - average_position; + break; + } +} + +bool ResourcePrefetchPredictorTables::ResourceRow::operator==( + const ResourceRow& rhs) const { + return primary_key == rhs.primary_key && + resource_url == rhs.resource_url && + resource_type == rhs.resource_type && + number_of_hits == rhs.number_of_hits && + number_of_misses == rhs.number_of_misses && + consecutive_misses == rhs.consecutive_misses && + average_position == rhs.average_position && + score == rhs.score; +} + +bool ResourcePrefetchPredictorTables::ResourceRowSorter::operator()( + const ResourceRow& x, const ResourceRow& y) const { + return x.score > y.score; +} + +ResourcePrefetchPredictorTables::PrefetchData::PrefetchData( + PrefetchKeyType i_key_type, + const std::string& i_primary_key) + : key_type(i_key_type), + primary_key(i_primary_key) { +} + +ResourcePrefetchPredictorTables::PrefetchData::PrefetchData( + const PrefetchData& other) + : key_type(other.key_type), + primary_key(other.primary_key), + last_visit(other.last_visit), + resources(other.resources) { +} + +ResourcePrefetchPredictorTables::PrefetchData::~PrefetchData() { +} + +bool ResourcePrefetchPredictorTables::PrefetchData::operator==( + const PrefetchData& rhs) const { + return key_type == rhs.key_type && primary_key == rhs.primary_key && + resources == rhs.resources; +} + +void ResourcePrefetchPredictorTables::GetAllData( + PrefetchDataMap* url_data_map, + PrefetchDataMap* host_data_map) { + DCHECK_CURRENTLY_ON(BrowserThread::DB); + if (CantAccessDatabase()) + return; + + DCHECK(url_data_map); + DCHECK(host_data_map); + url_data_map->clear(); + host_data_map->clear(); + + std::vector urls_to_delete, hosts_to_delete; + GetAllDataHelper(PREFETCH_KEY_TYPE_URL, url_data_map, &urls_to_delete); + GetAllDataHelper(PREFETCH_KEY_TYPE_HOST, host_data_map, &hosts_to_delete); + + if (!urls_to_delete.empty() || !hosts_to_delete.empty()) + DeleteData(urls_to_delete, hosts_to_delete); +} + +void ResourcePrefetchPredictorTables::UpdateData( + const PrefetchData& url_data, + const PrefetchData& host_data) { + DCHECK_CURRENTLY_ON(BrowserThread::DB); + if (CantAccessDatabase()) + return; + + DCHECK(!url_data.is_host() && host_data.is_host()); + DCHECK(!url_data.primary_key.empty() || !host_data.primary_key.empty()); + + DB()->BeginTransaction(); + + bool success = (url_data.primary_key.empty() || UpdateDataHelper(url_data)) && + (host_data.primary_key.empty() || UpdateDataHelper(host_data)); + if (!success) + DB()->RollbackTransaction(); + + DB()->CommitTransaction(); +} + +void ResourcePrefetchPredictorTables::DeleteData( + const std::vector& urls, + const std::vector& hosts) { + DCHECK_CURRENTLY_ON(BrowserThread::DB); + if (CantAccessDatabase()) + return; + + DCHECK(!urls.empty() || !hosts.empty()); + + if (!urls.empty()) + DeleteDataHelper(PREFETCH_KEY_TYPE_URL, urls); + if (!hosts.empty()) + DeleteDataHelper(PREFETCH_KEY_TYPE_HOST, hosts); +} + +void ResourcePrefetchPredictorTables::DeleteSingleDataPoint( + const std::string& key, + PrefetchKeyType key_type) { + DCHECK_CURRENTLY_ON(BrowserThread::DB); + if (CantAccessDatabase()) + return; + + DeleteDataHelper(key_type, std::vector(1, key)); +} + +void ResourcePrefetchPredictorTables::DeleteAllData() { + if (CantAccessDatabase()) + return; + + Statement deleter(DB()->GetUniqueStatement( + base::StringPrintf("DELETE FROM %s", kUrlResourceTableName).c_str())); + deleter.Run(); + deleter.Assign(DB()->GetUniqueStatement( + base::StringPrintf("DELETE FROM %s", kUrlMetadataTableName).c_str())); + deleter.Run(); + deleter.Assign(DB()->GetUniqueStatement( + base::StringPrintf("DELETE FROM %s", kHostResourceTableName).c_str())); + deleter.Run(); + deleter.Assign(DB()->GetUniqueStatement( + base::StringPrintf("DELETE FROM %s", kHostMetadataTableName).c_str())); + deleter.Run(); +} + +ResourcePrefetchPredictorTables::ResourcePrefetchPredictorTables() + : PredictorTableBase() { +} + +ResourcePrefetchPredictorTables::~ResourcePrefetchPredictorTables() { +} + +void ResourcePrefetchPredictorTables::GetAllDataHelper( + PrefetchKeyType key_type, + PrefetchDataMap* data_map, + std::vector* to_delete) { + bool is_host = key_type == PREFETCH_KEY_TYPE_HOST; + + // Read the resources table and organize it per primary key. + const char* resource_table_name = is_host ? kHostResourceTableName : + kUrlResourceTableName; + Statement resource_reader(DB()->GetUniqueStatement( + base::StringPrintf("SELECT * FROM %s", resource_table_name).c_str())); + + ResourceRow row; + while (StepAndInitializeResourceRow(&resource_reader, &row)) { + row.UpdateScore(); + std::string primary_key = row.primary_key; + // Don't need to store primary key since the data is grouped by primary key. + row.primary_key.clear(); + + PrefetchDataMap::iterator it = data_map->find(primary_key); + if (it == data_map->end()) { + it = data_map->insert(std::make_pair( + primary_key, PrefetchData(key_type, primary_key))).first; + } + it->second.resources.push_back(row); + } + + // Sort each of the resource row vectors by score. + for (PrefetchDataMap::iterator it = data_map->begin(); it != data_map->end(); + ++it) { + std::sort(it->second.resources.begin(), + it->second.resources.end(), + ResourceRowSorter()); + } + + // Read the metadata and keep track of entries that have metadata, but no + // resource entries, so they can be deleted. + const char* metadata_table_name = is_host ? kHostMetadataTableName : + kUrlMetadataTableName; + Statement metadata_reader(DB()->GetUniqueStatement( + base::StringPrintf("SELECT * FROM %s", metadata_table_name).c_str())); + + while (metadata_reader.Step()) { + std::string primary_key = metadata_reader.ColumnString(0); + + PrefetchDataMap::iterator it = data_map->find(primary_key); + if (it != data_map->end()) { + int64 last_visit = metadata_reader.ColumnInt64(1); + it->second.last_visit = base::Time::FromInternalValue(last_visit); + } else { + to_delete->push_back(primary_key); + } + } +} + +bool ResourcePrefetchPredictorTables::UpdateDataHelper( + const PrefetchData& data) { + DCHECK(!data.primary_key.empty()); + + if (!StringsAreSmallerThanDBLimit(data)) { + UMA_HISTOGRAM_BOOLEAN("ResourcePrefetchPredictor.DbStringTooLong", true); + return false; + } + + // Delete the older data from both the tables. + scoped_ptr deleter(data.is_host() ? + GetHostResourceDeleteStatement() : GetUrlResourceDeleteStatement()); + deleter->BindString(0, data.primary_key); + if (!deleter->Run()) + return false; + + deleter.reset(data.is_host() ? GetHostMetadataDeleteStatement() : + GetUrlMetadataDeleteStatement()); + deleter->BindString(0, data.primary_key); + if (!deleter->Run()) + return false; + + // Add the new data to the tables. + const ResourceRows& resources = data.resources; + for (ResourceRows::const_iterator it = resources.begin(); + it != resources.end(); ++it) { + scoped_ptr resource_inserter(data.is_host() ? + GetHostResourceUpdateStatement() : GetUrlResourceUpdateStatement()); + BindResourceRowToStatement(*it, data.primary_key, resource_inserter.get()); + if (!resource_inserter->Run()) + return false; + } + + scoped_ptr metadata_inserter(data.is_host() ? + GetHostMetadataUpdateStatement() : GetUrlMetadataUpdateStatement()); + metadata_inserter->BindString(0, data.primary_key); + metadata_inserter->BindInt64(1, data.last_visit.ToInternalValue()); + if (!metadata_inserter->Run()) + return false; + + return true; +} + +void ResourcePrefetchPredictorTables::DeleteDataHelper( + PrefetchKeyType key_type, + const std::vector& keys) { + bool is_host = key_type == PREFETCH_KEY_TYPE_HOST; + + for (std::vector::const_iterator it = keys.begin(); + it != keys.end(); ++it) { + scoped_ptr deleter(is_host ? GetHostResourceDeleteStatement() : + GetUrlResourceDeleteStatement()); + deleter->BindString(0, *it); + deleter->Run(); + + deleter.reset(is_host ? GetHostMetadataDeleteStatement() : + GetUrlMetadataDeleteStatement()); + deleter->BindString(0, *it); + deleter->Run(); + } +} + +bool ResourcePrefetchPredictorTables::StringsAreSmallerThanDBLimit( + const PrefetchData& data) const { + if (data.primary_key.length() > kMaxStringLength) + return false; + + for (ResourceRows::const_iterator it = data.resources.begin(); + it != data.resources.end(); ++it) { + if (it->resource_url.spec().length() > kMaxStringLength) + return false; + } + return true; +} + +void ResourcePrefetchPredictorTables::CreateTableIfNonExistent() { + DCHECK_CURRENTLY_ON(BrowserThread::DB); + if (CantAccessDatabase()) + return; + + const char resource_table_creator[] = + "CREATE TABLE %s ( " + "main_page_url TEXT, " + "resource_url TEXT, " + "resource_type INTEGER, " + "number_of_hits INTEGER, " + "number_of_misses INTEGER, " + "consecutive_misses INTEGER, " + "average_position DOUBLE, " + "PRIMARY KEY(main_page_url, resource_url))"; + const char* metadata_table_creator = + "CREATE TABLE %s ( " + "main_page_url TEXT, " + "last_visit_time INTEGER, " + "PRIMARY KEY(main_page_url))"; + + sql::Connection* db = DB(); + bool success = + (db->DoesTableExist(kUrlResourceTableName) || + db->Execute(base::StringPrintf(resource_table_creator, + kUrlResourceTableName).c_str())) && + (db->DoesTableExist(kUrlMetadataTableName) || + db->Execute(base::StringPrintf(metadata_table_creator, + kUrlMetadataTableName).c_str())) && + (db->DoesTableExist(kHostResourceTableName) || + db->Execute(base::StringPrintf(resource_table_creator, + kHostResourceTableName).c_str())) && + (db->DoesTableExist(kHostMetadataTableName) || + db->Execute(base::StringPrintf(metadata_table_creator, + kHostMetadataTableName).c_str())); + + if (!success) + ResetDB(); +} + +void ResourcePrefetchPredictorTables::LogDatabaseStats() { + DCHECK_CURRENTLY_ON(BrowserThread::DB); + if (CantAccessDatabase()) + return; + + Statement statement(DB()->GetUniqueStatement( + base::StringPrintf("SELECT count(*) FROM %s", + kUrlResourceTableName).c_str())); + if (statement.Step()) + UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.UrlTableRowCount", + statement.ColumnInt(0)); + + statement.Assign(DB()->GetUniqueStatement( + base::StringPrintf("SELECT count(*) FROM %s", + kHostResourceTableName).c_str())); + if (statement.Step()) + UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HostTableRowCount", + statement.ColumnInt(0)); +} + +Statement* + ResourcePrefetchPredictorTables::GetUrlResourceDeleteStatement() { + return new Statement(DB()->GetCachedStatement( + SQL_FROM_HERE, + base::StringPrintf("DELETE FROM %s WHERE main_page_url=?", + kUrlResourceTableName).c_str())); +} + +Statement* + ResourcePrefetchPredictorTables::GetUrlResourceUpdateStatement() { + return new Statement(DB()->GetCachedStatement( + SQL_FROM_HERE, + base::StringPrintf( + "INSERT INTO %s " + "(main_page_url, resource_url, resource_type, number_of_hits, " + "number_of_misses, consecutive_misses, average_position) " + "VALUES (?,?,?,?,?,?,?)", kUrlResourceTableName).c_str())); +} + +Statement* + ResourcePrefetchPredictorTables::GetUrlMetadataDeleteStatement() { + return new Statement(DB()->GetCachedStatement( + SQL_FROM_HERE, + base::StringPrintf("DELETE FROM %s WHERE main_page_url=?", + kUrlMetadataTableName).c_str())); +} + +Statement* + ResourcePrefetchPredictorTables::GetUrlMetadataUpdateStatement() { + return new Statement(DB()->GetCachedStatement( + SQL_FROM_HERE, + base::StringPrintf( + "INSERT INTO %s (main_page_url, last_visit_time) VALUES (?,?)", + kUrlMetadataTableName).c_str())); +} + +Statement* + ResourcePrefetchPredictorTables::GetHostResourceDeleteStatement() { + return new Statement(DB()->GetCachedStatement( + SQL_FROM_HERE, + base::StringPrintf("DELETE FROM %s WHERE main_page_url=?", + kHostResourceTableName).c_str())); +} + +Statement* + ResourcePrefetchPredictorTables::GetHostResourceUpdateStatement() { + return new Statement(DB()->GetCachedStatement( + SQL_FROM_HERE, + base::StringPrintf( + "INSERT INTO %s " + "(main_page_url, resource_url, resource_type, number_of_hits, " + "number_of_misses, consecutive_misses, average_position) " + "VALUES (?,?,?,?,?,?,?)", kHostResourceTableName).c_str())); +} + +Statement* + ResourcePrefetchPredictorTables::GetHostMetadataDeleteStatement() { + return new Statement(DB()->GetCachedStatement( + SQL_FROM_HERE, + base::StringPrintf("DELETE FROM %s WHERE main_page_url=?", + kHostMetadataTableName).c_str())); +} + +Statement* ResourcePrefetchPredictorTables::GetHostMetadataUpdateStatement() { + return new Statement(DB()->GetCachedStatement( + SQL_FROM_HERE, + base::StringPrintf( + "INSERT INTO %s (main_page_url, last_visit_time) VALUES (?,?)", + kHostMetadataTableName).c_str())); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tables.h b/chrome/browser/predictors/resource_prefetch_predictor_tables.h new file mode 100644 index 0000000..bd266a0 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor_tables.h @@ -0,0 +1,160 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_ + + +#include +#include +#include + +#include "base/time/time.h" +#include "chrome/browser/predictors/predictor_table_base.h" +#include "chrome/browser/predictors/resource_prefetch_common.h" +#include "content/public/common/resource_type.h" +#include "url/gurl.h" + +namespace sql { +class Statement; +} + +namespace predictors { + +// Interface for database tables used by the ResourcePrefetchPredictor. +// All methods except the constructor and destructor need to be called on the DB +// thread. +// +// Currently manages: +// - UrlResourceTable - resources per Urls. +// - UrlMetadataTable - misc data for Urls (like last visit time). +// - HostResourceTable - resources per host. +// - HostMetadataTable - misc data for hosts. +class ResourcePrefetchPredictorTables : public PredictorTableBase { + public: + // Used in the UrlResourceTable and HostResourceTable to store resources + // required for the page or host. + struct ResourceRow { + ResourceRow(); + ResourceRow(const ResourceRow& other); + ResourceRow(const std::string& main_frame_url, + const std::string& resource_url, + content::ResourceType resource_type, + int number_of_hits, + int number_of_misses, + int consecutive_misses, + double average_position); + void UpdateScore(); + bool operator==(const ResourceRow& rhs) const; + + // Stores the host for host based data, main frame Url for the Url based + // data. This field is cleared for efficiency reasons and the code outside + // this class should not assume it is set. + std::string primary_key; + + GURL resource_url; + content::ResourceType resource_type; + size_t number_of_hits; + size_t number_of_misses; + size_t consecutive_misses; + double average_position; + + // Not stored. + float score; + }; + typedef std::vector ResourceRows; + + // Sorts the ResourceRows by score, descending. + struct ResourceRowSorter { + bool operator()(const ResourceRow& x, const ResourceRow& y) const; + }; + + // Aggregated data for a Url or Host. Although the data differs slightly, we + // store them in the same structure, because most of the fields are common and + // it allows us to use the same functions. + struct PrefetchData { + PrefetchData(PrefetchKeyType key_type, const std::string& primary_key); + PrefetchData(const PrefetchData& other); + ~PrefetchData(); + bool operator==(const PrefetchData& rhs) const; + + bool is_host() const { return key_type == PREFETCH_KEY_TYPE_HOST; } + + // Is the data a host as opposed to a Url? + PrefetchKeyType key_type; // Not const to be able to assign. + std::string primary_key; // is_host() ? main frame url : host. + + base::Time last_visit; + ResourceRows resources; + }; + // Map from primary key to PrefetchData for the key. + typedef std::map PrefetchDataMap; + + // Returns data for all Urls and Hosts. + virtual void GetAllData(PrefetchDataMap* url_data_map, + PrefetchDataMap* host_data_map); + + // Updates data for a Url and a host. If either of the |url_data| or + // |host_data| has an empty primary key, it will be ignored. + // Note that the Urls and primary key in |url_data| and |host_data| should be + // less than |kMaxStringLength| in length. + virtual void UpdateData(const PrefetchData& url_data, + const PrefetchData& host_data); + + // Delete data for the input |urls| and |hosts|. + virtual void DeleteData(const std::vector& urls, + const std::vector& hosts); + + // Wrapper over DeleteData for convenience. + virtual void DeleteSingleDataPoint(const std::string& key, + PrefetchKeyType key_type); + + // Deletes all data in all the tables. + virtual void DeleteAllData(); + + // The maximum length of the string that can be stored in the DB. + static const size_t kMaxStringLength; + + private: + friend class PredictorDatabaseInternal; + friend class MockResourcePrefetchPredictorTables; + + ResourcePrefetchPredictorTables(); + virtual ~ResourcePrefetchPredictorTables(); + + // Helper functions below help perform functions on the Url and host table + // using the same code. + void GetAllDataHelper(PrefetchKeyType key_type, + PrefetchDataMap* data_map, + std::vector* to_delete); + bool UpdateDataHelper(const PrefetchData& data); + void DeleteDataHelper(PrefetchKeyType key_type, + const std::vector& keys); + + // Returns true if the strings in the |data| are less than |kMaxStringLength| + // in length. + bool StringsAreSmallerThanDBLimit(const PrefetchData& data) const; + + // PredictorTableBase methods. + virtual void CreateTableIfNonExistent() OVERRIDE; + virtual void LogDatabaseStats() OVERRIDE; + + // Helpers to return Statements for cached Statements. The caller must take + // ownership of the return Statements. + sql::Statement* GetUrlResourceDeleteStatement(); + sql::Statement* GetUrlResourceUpdateStatement(); + sql::Statement* GetUrlMetadataDeleteStatement(); + sql::Statement* GetUrlMetadataUpdateStatement(); + + sql::Statement* GetHostResourceDeleteStatement(); + sql::Statement* GetHostResourceUpdateStatement(); + sql::Statement* GetHostMetadataDeleteStatement(); + sql::Statement* GetHostMetadataUpdateStatement(); + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorTables); +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_TABLES_H_ diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tables_unittest.cc b/chrome/browser/predictors/resource_prefetch_predictor_tables_unittest.cc new file mode 100644 index 0000000..63ed7c4 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor_tables_unittest.cc @@ -0,0 +1,472 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include +#include + +#include "base/message_loop/message_loop.h" +#include "base/strings/utf_string_conversions.h" +#include "chrome/browser/predictors/predictor_database.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h" +#include "chrome/test/base/testing_profile.h" +#include "content/public/test/test_browser_thread.h" +#include "sql/statement.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace predictors { + +class ResourcePrefetchPredictorTablesTest : public testing::Test { + public: + ResourcePrefetchPredictorTablesTest(); + virtual ~ResourcePrefetchPredictorTablesTest(); + virtual void SetUp() OVERRIDE; + virtual void TearDown() OVERRIDE; + + protected: + void TestGetAllData(); + void TestUpdateData(); + void TestDeleteData(); + void TestDeleteSingleDataPoint(); + void TestDeleteAllData(); + + base::MessageLoop loop_; + content::TestBrowserThread db_thread_; + TestingProfile profile_; + scoped_ptr db_; + scoped_refptr tables_; + + private: + typedef ResourcePrefetchPredictorTables::ResourceRow ResourceRow; + typedef std::vector ResourceRows; + typedef ResourcePrefetchPredictorTables::PrefetchData PrefetchData; + typedef ResourcePrefetchPredictorTables::PrefetchDataMap PrefetchDataMap; + + // Initializes the tables, |test_url_data_| and |test_host_data_|. + void InitializeSampleData(); + + // Checks that the input PrefetchData are the same, although the resources + // can be in different order. + void TestPrefetchDataAreEqual(const PrefetchDataMap& lhs, + const PrefetchDataMap& rhs) const; + void TestResourceRowsAreEqual(const ResourceRows& lhs, + const ResourceRows& rhs) const; + + void AddKey(PrefetchDataMap* m, const std::string& key) const; + + // Useful for debugging test. + void PrintPrefetchData(const PrefetchData& data) const { + LOG(ERROR) << "[" << data.key_type << "," << data.primary_key + << "," << data.last_visit.ToInternalValue() << "]"; + for (ResourceRows::const_iterator it = data.resources.begin(); + it != data.resources.end(); ++it) { + LOG(ERROR) << "\t\t" << it->resource_url << "\t" << it->resource_type + << "\t" << it->number_of_hits << "\t" << it->number_of_misses + << "\t" << it->consecutive_misses + << "\t" << it->average_position + << "\t" << it->score; + } + } + + PrefetchDataMap test_url_data_; + PrefetchDataMap test_host_data_; +}; + +class ResourcePrefetchPredictorTablesReopenTest + : public ResourcePrefetchPredictorTablesTest { + public: + virtual void SetUp() OVERRIDE { + // Write data to the table, and then reopen the db. + ResourcePrefetchPredictorTablesTest::SetUp(); + ResourcePrefetchPredictorTablesTest::TearDown(); + + db_.reset(new PredictorDatabase(&profile_)); + loop_.RunUntilIdle(); + tables_ = db_->resource_prefetch_tables(); + } +}; + +ResourcePrefetchPredictorTablesTest::ResourcePrefetchPredictorTablesTest() + : loop_(base::MessageLoop::TYPE_DEFAULT), + db_thread_(content::BrowserThread::DB, &loop_), + db_(new PredictorDatabase(&profile_)), + tables_(db_->resource_prefetch_tables()) { + loop_.RunUntilIdle(); +} + +ResourcePrefetchPredictorTablesTest::~ResourcePrefetchPredictorTablesTest() { +} + +void ResourcePrefetchPredictorTablesTest::SetUp() { + tables_->DeleteAllData(); + InitializeSampleData(); +} + +void ResourcePrefetchPredictorTablesTest::TearDown() { + tables_ = NULL; + db_.reset(); + loop_.RunUntilIdle(); +} + +void ResourcePrefetchPredictorTablesTest::TestGetAllData() { + PrefetchDataMap actual_url_data, actual_host_data; + tables_->GetAllData(&actual_url_data, &actual_host_data); + + TestPrefetchDataAreEqual(test_url_data_, actual_url_data); + TestPrefetchDataAreEqual(test_host_data_, actual_host_data); +} + +void ResourcePrefetchPredictorTablesTest::TestDeleteData() { + std::vector urls_to_delete, hosts_to_delete; + urls_to_delete.push_back("http://www.google.com"); + urls_to_delete.push_back("http://www.yahoo.com"); + hosts_to_delete.push_back("www.yahoo.com"); + + tables_->DeleteData(urls_to_delete, hosts_to_delete); + + PrefetchDataMap actual_url_data, actual_host_data; + tables_->GetAllData(&actual_url_data, &actual_host_data); + + PrefetchDataMap expected_url_data, expected_host_data; + AddKey(&expected_url_data, "http://www.reddit.com"); + AddKey(&expected_host_data, "www.facebook.com"); + + TestPrefetchDataAreEqual(expected_url_data, actual_url_data); + TestPrefetchDataAreEqual(expected_host_data, actual_host_data); +} + +void ResourcePrefetchPredictorTablesTest::TestDeleteSingleDataPoint() { + // Delete a URL. + tables_->DeleteSingleDataPoint("http://www.reddit.com", + PREFETCH_KEY_TYPE_URL); + + PrefetchDataMap actual_url_data, actual_host_data; + tables_->GetAllData(&actual_url_data, &actual_host_data); + + PrefetchDataMap expected_url_data; + AddKey(&expected_url_data, "http://www.google.com"); + AddKey(&expected_url_data, "http://www.yahoo.com"); + + TestPrefetchDataAreEqual(expected_url_data, actual_url_data); + TestPrefetchDataAreEqual(test_host_data_, actual_host_data); + + // Delete a host. + tables_->DeleteSingleDataPoint("www.facebook.com", PREFETCH_KEY_TYPE_HOST); + actual_url_data.clear(); + actual_host_data.clear(); + tables_->GetAllData(&actual_url_data, &actual_host_data); + + PrefetchDataMap expected_host_data; + AddKey(&expected_host_data, "www.yahoo.com"); + + TestPrefetchDataAreEqual(expected_url_data, actual_url_data); + TestPrefetchDataAreEqual(expected_host_data, actual_host_data); +} + +void ResourcePrefetchPredictorTablesTest::TestUpdateData() { + PrefetchData google(PREFETCH_KEY_TYPE_URL, "http://www.google.com"); + google.last_visit = base::Time::FromInternalValue(10); + google.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/style.css", + content::RESOURCE_TYPE_STYLESHEET, + 6, + 2, + 0, + 1.0)); + google.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 6, + 4, + 1, + 4.2)); + google.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/a.xml", + content::RESOURCE_TYPE_LAST_TYPE, + 1, + 0, + 0, + 6.1)); + google.resources + .push_back(ResourceRow(std::string(), + "http://www.resources.google.com/script.js", + content::RESOURCE_TYPE_SCRIPT, + 12, + 0, + 0, + 8.5)); + + PrefetchData yahoo(PREFETCH_KEY_TYPE_HOST, "www.yahoo.com"); + yahoo.last_visit = base::Time::FromInternalValue(7); + yahoo.resources.push_back(ResourceRow(std::string(), + "http://www.yahoo.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 120, + 1, + 1, + 10.0)); + + tables_->UpdateData(google, yahoo); + + PrefetchDataMap actual_url_data, actual_host_data; + tables_->GetAllData(&actual_url_data, &actual_host_data); + + PrefetchDataMap expected_url_data, expected_host_data; + AddKey(&expected_url_data, "http://www.reddit.com"); + AddKey(&expected_url_data, "http://www.yahoo.com"); + expected_url_data.insert(std::make_pair("http://www.google.com", google)); + + AddKey(&expected_host_data, "www.facebook.com"); + expected_host_data.insert(std::make_pair("www.yahoo.com", yahoo)); + + TestPrefetchDataAreEqual(expected_url_data, actual_url_data); + TestPrefetchDataAreEqual(expected_host_data, actual_host_data); +} + +void ResourcePrefetchPredictorTablesTest::TestDeleteAllData() { + tables_->DeleteAllData(); + + PrefetchDataMap actual_url_data, actual_host_data; + tables_->GetAllData(&actual_url_data, &actual_host_data); + EXPECT_TRUE(actual_url_data.empty()); + EXPECT_TRUE(actual_host_data.empty()); +} + +void ResourcePrefetchPredictorTablesTest::TestPrefetchDataAreEqual( + const PrefetchDataMap& lhs, + const PrefetchDataMap& rhs) const { + EXPECT_EQ(lhs.size(), rhs.size()); + + for (PrefetchDataMap::const_iterator rhs_it = rhs.begin(); + rhs_it != rhs.end(); ++rhs_it) { + PrefetchDataMap::const_iterator lhs_it = lhs.find(rhs_it->first); + ASSERT_TRUE(lhs_it != lhs.end()) << rhs_it->first; + + TestResourceRowsAreEqual(lhs_it->second.resources, + rhs_it->second.resources); + } +} + +void ResourcePrefetchPredictorTablesTest::TestResourceRowsAreEqual( + const ResourceRows& lhs, + const ResourceRows& rhs) const { + EXPECT_EQ(lhs.size(), rhs.size()); + + std::set resources_seen; + for (ResourceRows::const_iterator rhs_it = rhs.begin(); + rhs_it != rhs.end(); ++rhs_it) { + const GURL& resource = rhs_it->resource_url; + EXPECT_FALSE(ContainsKey(resources_seen, resource)); + + for (ResourceRows::const_iterator lhs_it = lhs.begin(); + lhs_it != lhs.end(); ++lhs_it) { + if (*rhs_it == *lhs_it) { + resources_seen.insert(resource); + break; + } + } + EXPECT_TRUE(ContainsKey(resources_seen, resource)); + } + EXPECT_EQ(lhs.size(), resources_seen.size()); +} + +void ResourcePrefetchPredictorTablesTest::AddKey(PrefetchDataMap* m, + const std::string& key) const { + PrefetchDataMap::const_iterator it = test_url_data_.find(key); + if (it != test_url_data_.end()) { + m->insert(std::make_pair(it->first, it->second)); + return; + } + it = test_host_data_.find(key); + ASSERT_TRUE(it != test_host_data_.end()); + m->insert(std::make_pair(it->first, it->second)); +} + +void ResourcePrefetchPredictorTablesTest::InitializeSampleData() { + { // Url data. + PrefetchData google(PREFETCH_KEY_TYPE_URL, "http://www.google.com"); + google.last_visit = base::Time::FromInternalValue(1); + google.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/style.css", + content::RESOURCE_TYPE_STYLESHEET, + 5, + 2, + 1, + 1.1)); + google.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/script.js", + content::RESOURCE_TYPE_SCRIPT, + 4, + 0, + 1, + 2.1)); + google.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 6, + 3, + 0, + 2.2)); + google.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/a.font", + content::RESOURCE_TYPE_LAST_TYPE, + 2, + 0, + 0, + 5.1)); + google.resources + .push_back(ResourceRow(std::string(), + "http://www.resources.google.com/script.js", + content::RESOURCE_TYPE_SCRIPT, + 11, + 0, + 0, + 8.5)); + + PrefetchData reddit(PREFETCH_KEY_TYPE_URL, "http://www.reddit.com"); + reddit.last_visit = base::Time::FromInternalValue(2); + reddit.resources + .push_back(ResourceRow(std::string(), + "http://reddit-resource.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, + 4, + 0, + 1, + 1.0)); + reddit.resources + .push_back(ResourceRow(std::string(), + "http://reddit-resource.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, + 2, + 0, + 0, + 2.1)); + + PrefetchData yahoo(PREFETCH_KEY_TYPE_URL, "http://www.yahoo.com"); + yahoo.last_visit = base::Time::FromInternalValue(3); + yahoo.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 20, + 1, + 0, + 10.0)); + + test_url_data_.clear(); + test_url_data_.insert(std::make_pair("http://www.google.com", google)); + test_url_data_.insert(std::make_pair("http://www.reddit.com", reddit)); + test_url_data_.insert(std::make_pair("http://www.yahoo.com", yahoo)); + + PrefetchData empty_host_data(PREFETCH_KEY_TYPE_HOST, std::string()); + tables_->UpdateData(google, empty_host_data); + tables_->UpdateData(reddit, empty_host_data); + tables_->UpdateData(yahoo, empty_host_data); + } + + { // Host data. + PrefetchData facebook(PREFETCH_KEY_TYPE_HOST, "www.facebook.com"); + facebook.last_visit = base::Time::FromInternalValue(4); + facebook.resources + .push_back(ResourceRow(std::string(), + "http://www.facebook.com/style.css", + content::RESOURCE_TYPE_STYLESHEET, + 5, + 2, + 1, + 1.1)); + facebook.resources + .push_back(ResourceRow(std::string(), + "http://www.facebook.com/script.js", + content::RESOURCE_TYPE_SCRIPT, + 4, + 0, + 1, + 2.1)); + facebook.resources + .push_back(ResourceRow(std::string(), + "http://www.facebook.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 6, + 3, + 0, + 2.2)); + facebook.resources.push_back(ResourceRow(std::string(), + "http://www.facebook.com/a.font", + content::RESOURCE_TYPE_LAST_TYPE, + 2, + 0, + 0, + 5.1)); + facebook.resources + .push_back(ResourceRow(std::string(), + "http://www.resources.facebook.com/script.js", + content::RESOURCE_TYPE_SCRIPT, + 11, + 0, + 0, + 8.5)); + + PrefetchData yahoo(PREFETCH_KEY_TYPE_HOST, "www.yahoo.com"); + yahoo.last_visit = base::Time::FromInternalValue(5); + yahoo.resources.push_back(ResourceRow(std::string(), + "http://www.google.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 20, + 1, + 0, + 10.0)); + + test_host_data_.clear(); + test_host_data_.insert(std::make_pair("www.facebook.com", facebook)); + test_host_data_.insert(std::make_pair("www.yahoo.com", yahoo)); + + PrefetchData empty_url_data(PREFETCH_KEY_TYPE_URL, std::string()); + tables_->UpdateData(empty_url_data, facebook); + tables_->UpdateData(empty_url_data, yahoo); + } +} + +// Test cases. + +TEST_F(ResourcePrefetchPredictorTablesTest, GetAllData) { + TestGetAllData(); +} + +TEST_F(ResourcePrefetchPredictorTablesTest, UpdateData) { + TestUpdateData(); +} + +TEST_F(ResourcePrefetchPredictorTablesTest, DeleteData) { + TestDeleteData(); +} + +TEST_F(ResourcePrefetchPredictorTablesTest, DeleteSingleDataPoint) { + TestDeleteSingleDataPoint(); +} + +TEST_F(ResourcePrefetchPredictorTablesTest, DeleteAllData) { + TestDeleteAllData(); +} + +TEST_F(ResourcePrefetchPredictorTablesReopenTest, GetAllData) { + TestGetAllData(); +} + +TEST_F(ResourcePrefetchPredictorTablesReopenTest, UpdateData) { + TestUpdateData(); +} + +TEST_F(ResourcePrefetchPredictorTablesReopenTest, DeleteData) { + TestDeleteData(); +} + +TEST_F(ResourcePrefetchPredictorTablesReopenTest, DeleteSingleDataPoint) { + TestDeleteSingleDataPoint(); +} + +TEST_F(ResourcePrefetchPredictorTablesReopenTest, DeleteAllData) { + TestDeleteAllData(); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc b/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc new file mode 100644 index 0000000..6e79971 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc @@ -0,0 +1,925 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "base/message_loop/message_loop.h" +#include "base/run_loop.h" +#include "base/time/time.h" +#include "chrome/browser/history/history_service.h" +#include "chrome/browser/history/history_service_factory.h" +#include "chrome/browser/history/history_types.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h" +#include "chrome/test/base/testing_profile.h" +#include "content/public/test/test_browser_thread.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using testing::ContainerEq; +using testing::Pointee; +using testing::SetArgPointee; +using testing::StrictMock; + +namespace predictors { + +typedef ResourcePrefetchPredictor::URLRequestSummary URLRequestSummary; +typedef ResourcePrefetchPredictorTables::ResourceRow ResourceRow; +typedef std::vector ResourceRows; +typedef ResourcePrefetchPredictorTables::PrefetchData PrefetchData; +typedef ResourcePrefetchPredictorTables::PrefetchDataMap PrefetchDataMap; + +// For printing failures nicely. +void PrintTo(const ResourceRow& row, ::std::ostream* os) { + *os << "[" << row.primary_key << "," << row.resource_url + << "," << row.resource_type << "," << row.number_of_hits + << "," << row.number_of_misses << "," << row.consecutive_misses + << "," << row.average_position << "," << row.score << "]"; +} + +void PrintTo(const PrefetchData& data, ::std::ostream* os) { + *os << "[" << data.key_type << "," << data.primary_key + << "," << data.last_visit.ToInternalValue() << "]\n"; + for (ResourceRows::const_iterator it = data.resources.begin(); + it != data.resources.end(); ++it) { + *os << "\t\t"; + PrintTo(*it, os); + *os << "\n"; + } +} + +class MockResourcePrefetchPredictorTables + : public ResourcePrefetchPredictorTables { + public: + MockResourcePrefetchPredictorTables() { } + + MOCK_METHOD2(GetAllData, void(PrefetchDataMap* url_data_map, + PrefetchDataMap* host_data_map)); + MOCK_METHOD2(UpdateData, void(const PrefetchData& url_data, + const PrefetchData& host_data)); + MOCK_METHOD2(DeleteData, void(const std::vector& urls, + const std::vector& hosts)); + MOCK_METHOD2(DeleteSingleDataPoint, void(const std::string& key, + PrefetchKeyType key_type)); + MOCK_METHOD0(DeleteAllData, void()); + + protected: + ~MockResourcePrefetchPredictorTables() { } +}; + +class ResourcePrefetchPredictorTest : public testing::Test { + public: + ResourcePrefetchPredictorTest(); + virtual ~ResourcePrefetchPredictorTest(); + virtual void SetUp() OVERRIDE; + virtual void TearDown() OVERRIDE; + + protected: + void AddUrlToHistory(const std::string& url, int visit_count) { + HistoryServiceFactory::GetForProfile(profile_.get(), + Profile::EXPLICIT_ACCESS)-> + AddPageWithDetails( + GURL(url), + base::string16(), + visit_count, + 0, + base::Time::Now(), + false, + history::SOURCE_BROWSED); + profile_->BlockUntilHistoryProcessesPendingRequests(); + } + + NavigationID CreateNavigationID(int process_id, + int render_frame_id, + const std::string& main_frame_url) { + NavigationID navigation_id; + navigation_id.render_process_id = process_id; + navigation_id.render_frame_id = render_frame_id; + navigation_id.main_frame_url = GURL(main_frame_url); + navigation_id.creation_time = base::TimeTicks::Now(); + return navigation_id; + } + + ResourcePrefetchPredictor::URLRequestSummary CreateURLRequestSummary( + int process_id, + int render_frame_id, + const std::string& main_frame_url, + const std::string& resource_url, + content::ResourceType resource_type, + const std::string& mime_type, + bool was_cached) { + ResourcePrefetchPredictor::URLRequestSummary summary; + summary.navigation_id = CreateNavigationID(process_id, render_frame_id, + main_frame_url); + summary.resource_url = GURL(resource_url); + summary.resource_type = resource_type; + summary.mime_type = mime_type; + summary.was_cached = was_cached; + return summary; + } + + void InitializePredictor() { + predictor_->StartInitialization(); + base::RunLoop loop; + loop.RunUntilIdle(); // Runs the DB lookup. + profile_->BlockUntilHistoryProcessesPendingRequests(); + } + + bool URLRequestSummaryAreEqual(const URLRequestSummary& lhs, + const URLRequestSummary& rhs) { + return lhs.navigation_id == rhs.navigation_id && + lhs.resource_url == rhs.resource_url && + lhs.resource_type == rhs.resource_type && + lhs.mime_type == rhs.mime_type && + lhs.was_cached == rhs.was_cached; + } + + void ResetPredictor() { + ResourcePrefetchPredictorConfig config; + config.max_urls_to_track = 3; + config.max_hosts_to_track = 2; + config.min_url_visit_count = 2; + config.max_resources_per_entry = 4; + config.max_consecutive_misses = 2; + + // TODO(shishir): Enable the prefetching mode in the tests. + config.mode |= ResourcePrefetchPredictorConfig::URL_LEARNING; + config.mode |= ResourcePrefetchPredictorConfig::HOST_LEARNING; + predictor_.reset(new ResourcePrefetchPredictor(config, profile_.get())); + predictor_->set_mock_tables(mock_tables_); + } + + void InitializeSampleData(); + + base::MessageLoop loop_; + content::TestBrowserThread ui_thread_; + content::TestBrowserThread db_thread_; + scoped_ptr profile_; + + scoped_ptr predictor_; + scoped_refptr > mock_tables_; + + PrefetchDataMap test_url_data_; + PrefetchDataMap test_host_data_; + PrefetchData empty_url_data_; + PrefetchData empty_host_data_; +}; + +ResourcePrefetchPredictorTest::ResourcePrefetchPredictorTest() + : loop_(base::MessageLoop::TYPE_DEFAULT), + ui_thread_(content::BrowserThread::UI, &loop_), + db_thread_(content::BrowserThread::DB, &loop_), + profile_(new TestingProfile()), + mock_tables_(new StrictMock()), + empty_url_data_(PREFETCH_KEY_TYPE_URL, std::string()), + empty_host_data_(PREFETCH_KEY_TYPE_HOST, std::string()) {} + +ResourcePrefetchPredictorTest::~ResourcePrefetchPredictorTest() { + profile_.reset(NULL); + loop_.RunUntilIdle(); +} + +void ResourcePrefetchPredictorTest::SetUp() { + InitializeSampleData(); + + ASSERT_TRUE(profile_->CreateHistoryService(true, false)); + profile_->BlockUntilHistoryProcessesPendingRequests(); + EXPECT_TRUE(HistoryServiceFactory::GetForProfile(profile_.get(), + Profile::EXPLICIT_ACCESS)); + // Initialize the predictor with empty data. + ResetPredictor(); + EXPECT_EQ(predictor_->initialization_state_, + ResourcePrefetchPredictor::NOT_INITIALIZED); + EXPECT_CALL(*mock_tables_.get(), + GetAllData(Pointee(ContainerEq(PrefetchDataMap())), + Pointee(ContainerEq(PrefetchDataMap())))); + InitializePredictor(); + EXPECT_TRUE(predictor_->inflight_navigations_.empty()); + EXPECT_EQ(predictor_->initialization_state_, + ResourcePrefetchPredictor::INITIALIZED); +} + +void ResourcePrefetchPredictorTest::TearDown() { + predictor_.reset(NULL); + profile_->DestroyHistoryService(); +} + +void ResourcePrefetchPredictorTest::InitializeSampleData() { + { // Url data. + PrefetchData google(PREFETCH_KEY_TYPE_URL, "http://www.google.com/"); + google.last_visit = base::Time::FromInternalValue(1); + google.resources.push_back(ResourceRow(std::string(), + "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, + 3, + 2, + 1, + 1.0)); + google.resources.push_back(ResourceRow(std::string(), + "http://google.com/script3.js", + content::RESOURCE_TYPE_SCRIPT, + 4, + 0, + 1, + 2.1)); + google.resources.push_back(ResourceRow(std::string(), + "http://google.com/script4.js", + content::RESOURCE_TYPE_SCRIPT, + 11, + 0, + 0, + 2.1)); + google.resources.push_back(ResourceRow(std::string(), + "http://google.com/image1.png", + content::RESOURCE_TYPE_IMAGE, + 6, + 3, + 0, + 2.2)); + google.resources.push_back(ResourceRow(std::string(), + "http://google.com/a.font", + content::RESOURCE_TYPE_LAST_TYPE, + 2, + 0, + 0, + 5.1)); + + PrefetchData reddit(PREFETCH_KEY_TYPE_URL, "http://www.reddit.com/"); + reddit.last_visit = base::Time::FromInternalValue(2); + reddit.resources + .push_back(ResourceRow(std::string(), + "http://reddit-resource.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, + 4, + 0, + 1, + 1.0)); + reddit.resources + .push_back(ResourceRow(std::string(), + "http://reddit-resource.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, + 2, + 0, + 0, + 2.1)); + + PrefetchData yahoo(PREFETCH_KEY_TYPE_URL, "http://www.yahoo.com/"); + yahoo.last_visit = base::Time::FromInternalValue(3); + yahoo.resources.push_back(ResourceRow(std::string(), + "http://google.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 20, + 1, + 0, + 10.0)); + + test_url_data_.clear(); + test_url_data_.insert(std::make_pair("http://www.google.com/", google)); + test_url_data_.insert(std::make_pair("http://www.reddit.com/", reddit)); + test_url_data_.insert(std::make_pair("http://www.yahoo.com/", yahoo)); + } + + { // Host data. + PrefetchData facebook(PREFETCH_KEY_TYPE_HOST, "www.facebook.com"); + facebook.last_visit = base::Time::FromInternalValue(4); + facebook.resources + .push_back(ResourceRow(std::string(), + "http://www.facebook.com/style.css", + content::RESOURCE_TYPE_STYLESHEET, + 5, + 2, + 1, + 1.1)); + facebook.resources + .push_back(ResourceRow(std::string(), + "http://www.facebook.com/script.js", + content::RESOURCE_TYPE_SCRIPT, + 4, + 0, + 1, + 2.1)); + facebook.resources + .push_back(ResourceRow(std::string(), + "http://www.facebook.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 6, + 3, + 0, + 2.2)); + facebook.resources.push_back(ResourceRow(std::string(), + "http://www.facebook.com/a.font", + content::RESOURCE_TYPE_LAST_TYPE, + 2, + 0, + 0, + 5.1)); + facebook.resources + .push_back(ResourceRow(std::string(), + "http://www.resources.facebook.com/script.js", + content::RESOURCE_TYPE_SCRIPT, + 11, + 0, + 0, + 8.5)); + + PrefetchData yahoo(PREFETCH_KEY_TYPE_HOST, "www.yahoo.com"); + yahoo.last_visit = base::Time::FromInternalValue(5); + yahoo.resources.push_back(ResourceRow(std::string(), + "http://google.com/image.png", + content::RESOURCE_TYPE_IMAGE, + 20, + 1, + 0, + 10.0)); + + test_host_data_.clear(); + test_host_data_.insert(std::make_pair("www.facebook.com", facebook)); + test_host_data_.insert(std::make_pair("www.yahoo.com", yahoo)); + } +} + +TEST_F(ResourcePrefetchPredictorTest, LazilyInitializeEmpty) { + // Tests that the predictor initializes correctly without any data. + EXPECT_TRUE(predictor_->url_table_cache_->empty()); + EXPECT_TRUE(predictor_->host_table_cache_->empty()); +} + +TEST_F(ResourcePrefetchPredictorTest, LazilyInitializeWithData) { + // Tests that the history and the db tables data are loaded correctly. + AddUrlToHistory("http://www.google.com/", 4); + AddUrlToHistory("http://www.yahoo.com/", 2); + + EXPECT_CALL(*mock_tables_.get(), + GetAllData(Pointee(ContainerEq(PrefetchDataMap())), + Pointee(ContainerEq(PrefetchDataMap())))) + .WillOnce(DoAll(SetArgPointee<0>(test_url_data_), + SetArgPointee<1>(test_host_data_))); + + ResetPredictor(); + InitializePredictor(); + + // Test that the internal variables correctly initialized. + EXPECT_EQ(predictor_->initialization_state_, + ResourcePrefetchPredictor::INITIALIZED); + EXPECT_TRUE(predictor_->inflight_navigations_.empty()); + + EXPECT_EQ(test_url_data_, *predictor_->url_table_cache_); + EXPECT_EQ(test_host_data_, *predictor_->host_table_cache_); +} + +TEST_F(ResourcePrefetchPredictorTest, NavigationNotRecorded) { + // Single navigation but history count is low, so should not record. + AddUrlToHistory("http://www.google.com", 1); + + URLRequestSummary main_frame = + CreateURLRequestSummary(1, + 1, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + predictor_->RecordURLRequest(main_frame); + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + + // Now add a few subresources. + URLRequestSummary resource1 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, "text/css", false); + predictor_->RecordURLResponse(resource1); + URLRequestSummary resource2 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", false); + predictor_->RecordURLResponse(resource2); + URLRequestSummary resource3 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", false); + predictor_->RecordURLResponse(resource3); + + PrefetchData host_data(PREFETCH_KEY_TYPE_HOST, "www.google.com"); + host_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, + 1, + 0, + 0, + 1.0)); + host_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, + 1, + 0, + 0, + 2.0)); + host_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, + 1, + 0, + 0, + 3.0)); + EXPECT_CALL(*mock_tables_.get(), UpdateData(empty_url_data_, host_data)); + + predictor_->OnNavigationComplete(main_frame.navigation_id); + profile_->BlockUntilHistoryProcessesPendingRequests(); +} + +TEST_F(ResourcePrefetchPredictorTest, NavigationUrlNotInDB) { + // Single navigation that will be recorded. Will check for duplicate + // resources and also for number of resources saved. + AddUrlToHistory("http://www.google.com", 4); + + URLRequestSummary main_frame = + CreateURLRequestSummary(1, + 1, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + predictor_->RecordURLRequest(main_frame); + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + + URLRequestSummary resource1 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, "text/css", false); + predictor_->RecordURLResponse(resource1); + URLRequestSummary resource2 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", false); + predictor_->RecordURLResponse(resource2); + URLRequestSummary resource3 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", false); + predictor_->RecordURLResponse(resource3); + URLRequestSummary resource4 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", true); + predictor_->RecordURLResponse(resource4); + URLRequestSummary resource5 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/image1.png", + content::RESOURCE_TYPE_IMAGE, "image/png", false); + predictor_->RecordURLResponse(resource5); + URLRequestSummary resource6 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/image2.png", + content::RESOURCE_TYPE_IMAGE, "image/png", false); + predictor_->RecordURLResponse(resource6); + URLRequestSummary resource7 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/style2.css", + content::RESOURCE_TYPE_STYLESHEET, "text/css", true); + predictor_->RecordURLResponse(resource7); + + PrefetchData url_data(PREFETCH_KEY_TYPE_URL, "http://www.google.com/"); + url_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, + 1, + 0, + 0, + 1.0)); + url_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, + 1, + 0, + 0, + 2.0)); + url_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, + 1, + 0, + 0, + 3.0)); + url_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/style2.css", + content::RESOURCE_TYPE_STYLESHEET, + 1, + 0, + 0, + 7.0)); + EXPECT_CALL(*mock_tables_.get(), UpdateData(url_data, empty_host_data_)); + + PrefetchData host_data(PREFETCH_KEY_TYPE_HOST, "www.google.com"); + host_data.resources = url_data.resources; + EXPECT_CALL(*mock_tables_.get(), UpdateData(empty_url_data_, host_data)); + + predictor_->OnNavigationComplete(main_frame.navigation_id); + profile_->BlockUntilHistoryProcessesPendingRequests(); +} + +TEST_F(ResourcePrefetchPredictorTest, NavigationUrlInDB) { + // Tests that navigation is recorded correctly for URL already present in + // the database cache. + AddUrlToHistory("http://www.google.com", 4); + + EXPECT_CALL(*mock_tables_.get(), + GetAllData(Pointee(ContainerEq(PrefetchDataMap())), + Pointee(ContainerEq(PrefetchDataMap())))) + .WillOnce(DoAll(SetArgPointee<0>(test_url_data_), + SetArgPointee<1>(test_host_data_))); + ResetPredictor(); + InitializePredictor(); + EXPECT_EQ(3, static_cast(predictor_->url_table_cache_->size())); + EXPECT_EQ(2, static_cast(predictor_->host_table_cache_->size())); + + URLRequestSummary main_frame = + CreateURLRequestSummary(1, + 1, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + predictor_->RecordURLRequest(main_frame); + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + + URLRequestSummary resource1 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, "text/css", false); + predictor_->RecordURLResponse(resource1); + URLRequestSummary resource2 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", false); + predictor_->RecordURLResponse(resource2); + URLRequestSummary resource3 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", false); + predictor_->RecordURLResponse(resource3); + URLRequestSummary resource4 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", true); + predictor_->RecordURLResponse(resource4); + URLRequestSummary resource5 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/image1.png", + content::RESOURCE_TYPE_IMAGE, "image/png", false); + predictor_->RecordURLResponse(resource5); + URLRequestSummary resource6 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/image2.png", + content::RESOURCE_TYPE_IMAGE, "image/png", false); + predictor_->RecordURLResponse(resource6); + URLRequestSummary resource7 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/style2.css", + content::RESOURCE_TYPE_STYLESHEET, "text/css", true); + predictor_->RecordURLResponse(resource7); + + PrefetchData url_data(PREFETCH_KEY_TYPE_URL, "http://www.google.com/"); + url_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, + 4, + 2, + 0, + 1.0)); + url_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, + 1, + 0, + 0, + 2.0)); + url_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script4.js", + content::RESOURCE_TYPE_SCRIPT, + 11, + 1, + 1, + 2.1)); + url_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, + 1, + 0, + 0, + 3.0)); + EXPECT_CALL(*mock_tables_.get(), UpdateData(url_data, empty_host_data_)); + + EXPECT_CALL( + *mock_tables_.get(), + DeleteSingleDataPoint("www.facebook.com", PREFETCH_KEY_TYPE_HOST)); + + PrefetchData host_data(PREFETCH_KEY_TYPE_HOST, "www.google.com"); + host_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, + 1, + 0, + 0, + 1.0)); + host_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, + 1, + 0, + 0, + 2.0)); + host_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, + 1, + 0, + 0, + 3.0)); + host_data.resources.push_back(ResourceRow(std::string(), + "http://google.com/style2.css", + content::RESOURCE_TYPE_STYLESHEET, + 1, + 0, + 0, + 7.0)); + EXPECT_CALL(*mock_tables_.get(), UpdateData(empty_url_data_, host_data)); + + predictor_->OnNavigationComplete(main_frame.navigation_id); + profile_->BlockUntilHistoryProcessesPendingRequests(); +} + +TEST_F(ResourcePrefetchPredictorTest, NavigationUrlNotInDBAndDBFull) { + // Tests that a URL is deleted before another is added if the cache is full. + AddUrlToHistory("http://www.nike.com/", 4); + + EXPECT_CALL(*mock_tables_.get(), + GetAllData(Pointee(ContainerEq(PrefetchDataMap())), + Pointee(ContainerEq(PrefetchDataMap())))) + .WillOnce(DoAll(SetArgPointee<0>(test_url_data_), + SetArgPointee<1>(test_host_data_))); + ResetPredictor(); + InitializePredictor(); + EXPECT_EQ(3, static_cast(predictor_->url_table_cache_->size())); + EXPECT_EQ(2, static_cast(predictor_->host_table_cache_->size())); + + URLRequestSummary main_frame = + CreateURLRequestSummary(1, + 1, + "http://www.nike.com", + "http://www.nike.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + predictor_->RecordURLRequest(main_frame); + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + + URLRequestSummary resource1 = CreateURLRequestSummary( + 1, 1, "http://www.nike.com", "http://nike.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, "text/css", false); + predictor_->RecordURLResponse(resource1); + URLRequestSummary resource2 = CreateURLRequestSummary( + 1, 1, "http://www.nike.com", "http://nike.com/image2.png", + content::RESOURCE_TYPE_IMAGE, "image/png", false); + predictor_->RecordURLResponse(resource2); + + EXPECT_CALL( + *mock_tables_.get(), + DeleteSingleDataPoint("http://www.google.com/", PREFETCH_KEY_TYPE_URL)); + EXPECT_CALL( + *mock_tables_.get(), + DeleteSingleDataPoint("www.facebook.com", PREFETCH_KEY_TYPE_HOST)); + + PrefetchData url_data(PREFETCH_KEY_TYPE_URL, "http://www.nike.com/"); + url_data.resources.push_back(ResourceRow(std::string(), + "http://nike.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, + 1, + 0, + 0, + 1.0)); + url_data.resources.push_back(ResourceRow(std::string(), + "http://nike.com/image2.png", + content::RESOURCE_TYPE_IMAGE, + 1, + 0, + 0, + 2.0)); + EXPECT_CALL(*mock_tables_.get(), UpdateData(url_data, empty_host_data_)); + + PrefetchData host_data(PREFETCH_KEY_TYPE_HOST, "www.nike.com"); + host_data.resources = url_data.resources; + EXPECT_CALL(*mock_tables_.get(), UpdateData(empty_url_data_, host_data)); + + predictor_->OnNavigationComplete(main_frame.navigation_id); + profile_->BlockUntilHistoryProcessesPendingRequests(); +} + +TEST_F(ResourcePrefetchPredictorTest, DeleteUrls) { + // Add some dummy entries to cache. + predictor_->url_table_cache_->insert(std::make_pair( + "http://www.google.com/page1.html", + PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.google.com/page1.html"))); + predictor_->url_table_cache_->insert(std::make_pair( + "http://www.google.com/page2.html", + PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.google.com/page2.html"))); + predictor_->url_table_cache_->insert(std::make_pair( + "http://www.yahoo.com/", + PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.yahoo.com/"))); + predictor_->url_table_cache_->insert(std::make_pair( + "http://www.apple.com/", + PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.apple.com/"))); + predictor_->url_table_cache_->insert(std::make_pair( + "http://www.nike.com/", + PrefetchData(PREFETCH_KEY_TYPE_URL, "http://www.nike.com/"))); + + predictor_->host_table_cache_->insert(std::make_pair( + "www.google.com", + PrefetchData(PREFETCH_KEY_TYPE_HOST, "www.google.com"))); + predictor_->host_table_cache_->insert(std::make_pair( + "www.yahoo.com", + PrefetchData(PREFETCH_KEY_TYPE_HOST, "www.yahoo.com"))); + predictor_->host_table_cache_->insert(std::make_pair( + "www.apple.com", + PrefetchData(PREFETCH_KEY_TYPE_HOST, "www.apple.com"))); + + history::URLRows rows; + rows.push_back(history::URLRow(GURL("http://www.google.com/page2.html"))); + rows.push_back(history::URLRow(GURL("http://www.apple.com"))); + rows.push_back(history::URLRow(GURL("http://www.nike.com"))); + + std::vector urls_to_delete, hosts_to_delete; + urls_to_delete.push_back("http://www.google.com/page2.html"); + urls_to_delete.push_back("http://www.apple.com/"); + urls_to_delete.push_back("http://www.nike.com/"); + hosts_to_delete.push_back("www.google.com"); + hosts_to_delete.push_back("www.apple.com"); + + EXPECT_CALL( + *mock_tables_.get(), + DeleteData(ContainerEq(urls_to_delete), ContainerEq(hosts_to_delete))); + + predictor_->DeleteUrls(rows); + EXPECT_EQ(2, static_cast(predictor_->url_table_cache_->size())); + EXPECT_EQ(1, static_cast(predictor_->host_table_cache_->size())); + + EXPECT_CALL(*mock_tables_.get(), DeleteAllData()); + + predictor_->DeleteAllUrls(); + EXPECT_TRUE(predictor_->url_table_cache_->empty()); + EXPECT_TRUE(predictor_->host_table_cache_->empty()); +} + +TEST_F(ResourcePrefetchPredictorTest, OnMainFrameRequest) { + URLRequestSummary summary1 = + CreateURLRequestSummary(1, + 1, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + URLRequestSummary summary2 = + CreateURLRequestSummary(1, + 2, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + URLRequestSummary summary3 = + CreateURLRequestSummary(2, + 1, + "http://www.yahoo.com", + "http://www.yahoo.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + + predictor_->OnMainFrameRequest(summary1); + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + predictor_->OnMainFrameRequest(summary2); + EXPECT_EQ(2, static_cast(predictor_->inflight_navigations_.size())); + predictor_->OnMainFrameRequest(summary3); + EXPECT_EQ(3, static_cast(predictor_->inflight_navigations_.size())); + + // Insert anther with same navigation id. It should replace. + URLRequestSummary summary4 = + CreateURLRequestSummary(1, + 1, + "http://www.nike.com", + "http://www.nike.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + URLRequestSummary summary5 = + CreateURLRequestSummary(1, + 2, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + + predictor_->OnMainFrameRequest(summary4); + EXPECT_EQ(3, static_cast(predictor_->inflight_navigations_.size())); + + // Change this creation time so that it will go away on the next insert. + summary5.navigation_id.creation_time = base::TimeTicks::Now() - + base::TimeDelta::FromDays(1); + predictor_->OnMainFrameRequest(summary5); + EXPECT_EQ(3, static_cast(predictor_->inflight_navigations_.size())); + + URLRequestSummary summary6 = + CreateURLRequestSummary(3, + 1, + "http://www.shoes.com", + "http://www.shoes.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + predictor_->OnMainFrameRequest(summary6); + EXPECT_EQ(3, static_cast(predictor_->inflight_navigations_.size())); + + EXPECT_TRUE(predictor_->inflight_navigations_.find(summary3.navigation_id) != + predictor_->inflight_navigations_.end()); + EXPECT_TRUE(predictor_->inflight_navigations_.find(summary4.navigation_id) != + predictor_->inflight_navigations_.end()); + EXPECT_TRUE(predictor_->inflight_navigations_.find(summary6.navigation_id) != + predictor_->inflight_navigations_.end()); +} + +TEST_F(ResourcePrefetchPredictorTest, OnMainFrameRedirect) { + URLRequestSummary summary1 = + CreateURLRequestSummary(1, + 1, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + URLRequestSummary summary2 = + CreateURLRequestSummary(1, + 2, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + URLRequestSummary summary3 = + CreateURLRequestSummary(2, + 1, + "http://www.yahoo.com", + "http://www.yahoo.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + + predictor_->OnMainFrameRedirect(summary1); + EXPECT_TRUE(predictor_->inflight_navigations_.empty()); + + predictor_->OnMainFrameRequest(summary1); + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + predictor_->OnMainFrameRequest(summary2); + EXPECT_EQ(2, static_cast(predictor_->inflight_navigations_.size())); + + predictor_->OnMainFrameRedirect(summary3); + EXPECT_EQ(2, static_cast(predictor_->inflight_navigations_.size())); + predictor_->OnMainFrameRedirect(summary1); + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + predictor_->OnMainFrameRedirect(summary2); + EXPECT_TRUE(predictor_->inflight_navigations_.empty()); +} + +TEST_F(ResourcePrefetchPredictorTest, OnSubresourceResponse) { + // If there is no inflight navigation, nothing happens. + URLRequestSummary resource1 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/style1.css", + content::RESOURCE_TYPE_STYLESHEET, "text/css", false); + predictor_->OnSubresourceResponse(resource1); + EXPECT_TRUE(predictor_->inflight_navigations_.empty()); + + // Add an inflight navigation. + URLRequestSummary main_frame1 = + CreateURLRequestSummary(1, + 1, + "http://www.google.com", + "http://www.google.com", + content::RESOURCE_TYPE_MAIN_FRAME, + std::string(), + false); + predictor_->OnMainFrameRequest(main_frame1); + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + + // Now add a few subresources. + URLRequestSummary resource2 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script1.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", false); + URLRequestSummary resource3 = CreateURLRequestSummary( + 1, 1, "http://www.google.com", "http://google.com/script2.js", + content::RESOURCE_TYPE_SCRIPT, "text/javascript", false); + predictor_->OnSubresourceResponse(resource1); + predictor_->OnSubresourceResponse(resource2); + predictor_->OnSubresourceResponse(resource3); + + EXPECT_EQ(1, static_cast(predictor_->inflight_navigations_.size())); + EXPECT_EQ(3, static_cast( + predictor_->inflight_navigations_[main_frame1.navigation_id]->size())); + EXPECT_TRUE(URLRequestSummaryAreEqual( + resource1, + predictor_->inflight_navigations_[main_frame1.navigation_id]->at(0))); + EXPECT_TRUE(URLRequestSummaryAreEqual( + resource2, + predictor_->inflight_navigations_[main_frame1.navigation_id]->at(1))); + EXPECT_TRUE(URLRequestSummaryAreEqual( + resource3, + predictor_->inflight_navigations_[main_frame1.navigation_id]->at(2))); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetcher.cc b/chrome/browser/predictors/resource_prefetcher.cc new file mode 100644 index 0000000..b602a2b --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher.cc @@ -0,0 +1,240 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetcher.h" + +#include + +#include "base/stl_util.h" +#include "content/public/browser/browser_thread.h" +#include "net/base/io_buffer.h" +#include "net/base/request_priority.h" +#include "net/url_request/url_request_context.h" + +namespace { + +// The size of the buffer used to read the resource. +static const size_t kResourceBufferSizeBytes = 50000; + +} // namespace + +namespace predictors { + +ResourcePrefetcher::Request::Request(const GURL& i_resource_url) + : resource_url(i_resource_url), + prefetch_status(PREFETCH_STATUS_NOT_STARTED), + usage_status(USAGE_STATUS_NOT_REQUESTED) { +} + +ResourcePrefetcher::Request::Request(const Request& other) + : resource_url(other.resource_url), + prefetch_status(other.prefetch_status), + usage_status(other.usage_status) { +} + +ResourcePrefetcher::ResourcePrefetcher( + Delegate* delegate, + const ResourcePrefetchPredictorConfig& config, + const NavigationID& navigation_id, + PrefetchKeyType key_type, + scoped_ptr requests) + : state_(INITIALIZED), + delegate_(delegate), + config_(config), + navigation_id_(navigation_id), + key_type_(key_type), + request_vector_(requests.Pass()) { + DCHECK_CURRENTLY_ON(content::BrowserThread::IO); + DCHECK(request_vector_.get()); + + std::copy(request_vector_->begin(), request_vector_->end(), + std::back_inserter(request_queue_)); +} + +ResourcePrefetcher::~ResourcePrefetcher() { + // Delete any pending net::URLRequests. + STLDeleteContainerPairFirstPointers(inflight_requests_.begin(), + inflight_requests_.end()); +} + +void ResourcePrefetcher::Start() { + DCHECK(thread_checker_.CalledOnValidThread()); + + CHECK_EQ(state_, INITIALIZED); + state_ = RUNNING; + + TryToLaunchPrefetchRequests(); +} + +void ResourcePrefetcher::Stop() { + DCHECK(thread_checker_.CalledOnValidThread()); + + if (state_ == FINISHED) + return; + + state_ = STOPPED; +} + +void ResourcePrefetcher::TryToLaunchPrefetchRequests() { + CHECK(state_ == RUNNING || state_ == STOPPED); + + // Try to launch new requests if the state is RUNNING. + if (state_ == RUNNING) { + bool request_available = true; + + // Loop through the requests while we are under the + // max_prefetches_inflight_per_host_per_navigation limit, looking for a URL + // for which the max_prefetches_inflight_per_host_per_navigation limit has + // not been reached. Try to launch as many requests as possible. + while ((inflight_requests_.size() < + config_.max_prefetches_inflight_per_navigation) && + request_available) { + std::list::iterator request_it = request_queue_.begin(); + for (; request_it != request_queue_.end(); ++request_it) { + const std::string& host = (*request_it)->resource_url.host(); + + std::map::iterator host_it = + host_inflight_counts_.find(host); + if (host_it == host_inflight_counts_.end() || + host_it->second < + config_.max_prefetches_inflight_per_host_per_navigation) + break; + } + request_available = request_it != request_queue_.end(); + + if (request_available) { + SendRequest(*request_it); + request_queue_.erase(request_it); + } + } + } + + // If the inflight_requests_ is empty, we cant launch any more. Finish. + if (inflight_requests_.empty()) { + CHECK(host_inflight_counts_.empty()); + CHECK(request_queue_.empty() || state_ == STOPPED); + + state_ = FINISHED; + delegate_->ResourcePrefetcherFinished(this, request_vector_.release()); + } +} + +void ResourcePrefetcher::SendRequest(Request* request) { + request->prefetch_status = Request::PREFETCH_STATUS_STARTED; + + net::URLRequest* url_request = + delegate_->GetURLRequestContext()->CreateRequest( + request->resource_url, net::LOW, this, NULL).release(); + + inflight_requests_[url_request] = request; + host_inflight_counts_[url_request->original_url().host()] += 1; + + url_request->set_method("GET"); + url_request->set_first_party_for_cookies(navigation_id_.main_frame_url); + url_request->SetReferrer(navigation_id_.main_frame_url.spec()); + StartURLRequest(url_request); +} + +void ResourcePrefetcher::StartURLRequest(net::URLRequest* request) { + request->Start(); +} + +void ResourcePrefetcher::FinishRequest(net::URLRequest* request, + Request::PrefetchStatus status) { + std::map::iterator request_it = + inflight_requests_.find(request); + CHECK(request_it != inflight_requests_.end()); + + const std::string host = request->original_url().host(); + std::map::iterator host_it = host_inflight_counts_.find( + host); + CHECK_GT(host_it->second, 0U); + host_it->second -= 1; + if (host_it->second == 0) + host_inflight_counts_.erase(host); + + request_it->second->prefetch_status = status; + inflight_requests_.erase(request_it); + + delete request; + + TryToLaunchPrefetchRequests(); +} + +void ResourcePrefetcher::ReadFullResponse(net::URLRequest* request) { + bool status = true; + while (status) { + int bytes_read = 0; + scoped_refptr buffer(new net::IOBuffer( + kResourceBufferSizeBytes)); + status = request->Read(buffer.get(), kResourceBufferSizeBytes, &bytes_read); + + if (status) { + status = ShouldContinueReadingRequest(request, bytes_read); + } else if (request->status().error()) { + FinishRequest(request, Request::PREFETCH_STATUS_FAILED); + return; + } + } +} + +bool ResourcePrefetcher::ShouldContinueReadingRequest(net::URLRequest* request, + int bytes_read) { + if (bytes_read == 0) { // When bytes_read == 0, no more data. + if (request->was_cached()) + FinishRequest(request, Request::PREFETCH_STATUS_FROM_CACHE); + else + FinishRequest(request, Request::PREFETCH_STATUS_FROM_NETWORK); + return false; + } + + return true; +} + +void ResourcePrefetcher::OnReceivedRedirect( + net::URLRequest* request, + const net::RedirectInfo& redirect_info, + bool* defer_redirect) { + FinishRequest(request, Request::PREFETCH_STATUS_REDIRECTED); +} + +void ResourcePrefetcher::OnAuthRequired(net::URLRequest* request, + net::AuthChallengeInfo* auth_info) { + FinishRequest(request, Request::PREFETCH_STATUS_AUTH_REQUIRED); +} + +void ResourcePrefetcher::OnCertificateRequested( + net::URLRequest* request, + net::SSLCertRequestInfo* cert_request_info) { + FinishRequest(request, Request::PREFETCH_STATUS_CERT_REQUIRED); +} + +void ResourcePrefetcher::OnSSLCertificateError(net::URLRequest* request, + const net::SSLInfo& ssl_info, + bool fatal) { + FinishRequest(request, Request::PREFETCH_STATUS_CERT_ERROR); +} + +void ResourcePrefetcher::OnResponseStarted(net::URLRequest* request) { + if (request->status().error()) { + FinishRequest(request, Request::PREFETCH_STATUS_FAILED); + return; + } + + // TODO(shishir): Do not read cached entries, or ones that are not cacheable. + ReadFullResponse(request); +} + +void ResourcePrefetcher::OnReadCompleted(net::URLRequest* request, + int bytes_read) { + if (request->status().error()) { + FinishRequest(request, Request::PREFETCH_STATUS_FAILED); + return; + } + + if (ShouldContinueReadingRequest(request, bytes_read)) + ReadFullResponse(request); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetcher.h b/chrome/browser/predictors/resource_prefetcher.h new file mode 100644 index 0000000..18b1bff --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher.h @@ -0,0 +1,165 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_ + +#include +#include +#include + +#include "base/gtest_prod_util.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/scoped_vector.h" +#include "base/threading/thread_checker.h" +#include "chrome/browser/predictors/resource_prefetch_common.h" +#include "net/url_request/redirect_info.h" +#include "net/url_request/url_request.h" +#include "url/gurl.h" + +namespace net { +class URLRequestContext; +} + +namespace predictors { + +// Responsible for prefetching resources for a single navigation based on the +// input list of resources. +// - Limits the max number of resources in flight for any host and also across +// hosts. +// - When stopped, will wait for the pending requests to finish. +// - Lives entirely on the IO thread. +class ResourcePrefetcher : public net::URLRequest::Delegate { + public: + // Denotes the prefetch request for a single subresource. + struct Request { + explicit Request(const GURL& i_resource_url); + Request(const Request& other); + + enum PrefetchStatus { + PREFETCH_STATUS_NOT_STARTED, + PREFETCH_STATUS_STARTED, + + // Cancellation reasons. + PREFETCH_STATUS_REDIRECTED, + PREFETCH_STATUS_AUTH_REQUIRED, + PREFETCH_STATUS_CERT_REQUIRED, + PREFETCH_STATUS_CERT_ERROR, + PREFETCH_STATUS_CANCELLED, + PREFETCH_STATUS_FAILED, + + // Successful prefetch states. + PREFETCH_STATUS_FROM_CACHE, + PREFETCH_STATUS_FROM_NETWORK + }; + + enum UsageStatus { + USAGE_STATUS_NOT_REQUESTED, + USAGE_STATUS_FROM_CACHE, + USAGE_STATUS_FROM_NETWORK, + USAGE_STATUS_NAVIGATION_ABANDONED + }; + + GURL resource_url; + PrefetchStatus prefetch_status; + UsageStatus usage_status; + }; + typedef ScopedVector RequestVector; + + // Used to communicate when the prefetching is done. All methods are invoked + // on the IO thread. + class Delegate { + public: + virtual ~Delegate() { } + + // Called when the ResourcePrefetcher is finished, i.e. there is nothing + // pending in flight. Should take ownership of |requests|. + virtual void ResourcePrefetcherFinished( + ResourcePrefetcher* prefetcher, + RequestVector* requests) = 0; + + virtual net::URLRequestContext* GetURLRequestContext() = 0; + }; + + // |delegate| has to outlive the ResourcePrefetcher. The ResourcePrefetcher + // takes ownership of |requests|. + ResourcePrefetcher(Delegate* delegate, + const ResourcePrefetchPredictorConfig& config, + const NavigationID& navigation_id, + PrefetchKeyType key_type, + scoped_ptr requests); + virtual ~ResourcePrefetcher(); + + void Start(); // Kicks off the prefetching. Can only be called once. + void Stop(); // No additional prefetches will be queued after this. + + const NavigationID& navigation_id() const { return navigation_id_; } + PrefetchKeyType key_type() const { return key_type_; } + + private: + friend class ResourcePrefetcherTest; + friend class TestResourcePrefetcher; + + // Launches new prefetch requests if possible. + void TryToLaunchPrefetchRequests(); + + // Starts a net::URLRequest for the input |request|. + void SendRequest(Request* request); + + // Called by |SendRequest| to start the |request|. This is necessary to stub + // out the Start() call to net::URLRequest for unittesting. + virtual void StartURLRequest(net::URLRequest* request); + + // Marks the request as finished, with the given status. + void FinishRequest(net::URLRequest* request, Request::PrefetchStatus status); + + // Reads the response data from the response - required for the resource to + // be cached correctly. Stubbed out during testing. + virtual void ReadFullResponse(net::URLRequest* request); + + // Returns true if the request has more data that needs to be read. If it + // returns false, the request should not be referenced again. + bool ShouldContinueReadingRequest(net::URLRequest* request, int bytes_read); + + // net::URLRequest::Delegate methods. + virtual void OnReceivedRedirect(net::URLRequest* request, + const net::RedirectInfo& redirect_info, + bool* defer_redirect) OVERRIDE; + virtual void OnAuthRequired(net::URLRequest* request, + net::AuthChallengeInfo* auth_info) OVERRIDE; + virtual void OnCertificateRequested( + net::URLRequest* request, + net::SSLCertRequestInfo* cert_request_info) OVERRIDE; + virtual void OnSSLCertificateError(net::URLRequest* request, + const net::SSLInfo& ssl_info, + bool fatal) OVERRIDE; + virtual void OnResponseStarted(net::URLRequest* request) OVERRIDE; + virtual void OnReadCompleted(net::URLRequest* request, + int bytes_read) OVERRIDE; + + enum PrefetcherState { + INITIALIZED = 0, // Prefetching hasn't started. + RUNNING = 1, // Prefetching started, allowed to add more requests. + STOPPED = 2, // Prefetching started, not allowed to add more requests. + FINISHED = 3 // No more inflight request, new requests not possible. + }; + + base::ThreadChecker thread_checker_; + PrefetcherState state_; + Delegate* const delegate_; + ResourcePrefetchPredictorConfig const config_; + NavigationID navigation_id_; + PrefetchKeyType key_type_; + scoped_ptr request_vector_; + + std::map inflight_requests_; + std::list request_queue_; + std::map host_inflight_counts_; + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcher); +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_ diff --git a/chrome/browser/predictors/resource_prefetcher_manager.cc b/chrome/browser/predictors/resource_prefetcher_manager.cc new file mode 100644 index 0000000..15d1b76 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher_manager.cc @@ -0,0 +1,135 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetcher_manager.h" + +#include "base/bind.h" +#include "base/stl_util.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "content/public/browser/browser_thread.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context_getter.h" + +using content::BrowserThread; + +namespace predictors { + +ResourcePrefetcherManager::ResourcePrefetcherManager( + ResourcePrefetchPredictor* predictor, + const ResourcePrefetchPredictorConfig& config, + net::URLRequestContextGetter* context_getter) + : predictor_(predictor), + config_(config), + context_getter_(context_getter) { + DCHECK_CURRENTLY_ON(content::BrowserThread::UI); + CHECK(predictor_); + CHECK(context_getter_); +} + +ResourcePrefetcherManager::~ResourcePrefetcherManager() { + DCHECK(prefetcher_map_.empty()) + << "Did not call ShutdownOnUIThread or ShutdownOnIOThread. " + " Will leak Prefetcher pointers."; +} + +void ResourcePrefetcherManager::ShutdownOnUIThread() { + DCHECK_CURRENTLY_ON(content::BrowserThread::UI); + + predictor_ = NULL; + BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, + base::Bind(&ResourcePrefetcherManager::ShutdownOnIOThread, + this)); +} + +void ResourcePrefetcherManager::ShutdownOnIOThread() { + DCHECK_CURRENTLY_ON(content::BrowserThread::IO); + STLDeleteContainerPairSecondPointers(prefetcher_map_.begin(), + prefetcher_map_.end()); +} + +void ResourcePrefetcherManager::MaybeAddPrefetch( + const NavigationID& navigation_id, + PrefetchKeyType key_type, + scoped_ptr requests) { + DCHECK_CURRENTLY_ON(content::BrowserThread::IO); + + // Don't add a duplicate prefetch for the same host or URL. + std::string key = key_type == PREFETCH_KEY_TYPE_HOST ? + navigation_id.main_frame_url.host() : navigation_id.main_frame_url.spec(); + if (ContainsKey(prefetcher_map_, key)) + return; + + ResourcePrefetcher* prefetcher = new ResourcePrefetcher( + this, config_, navigation_id, key_type, requests.Pass()); + prefetcher_map_.insert(std::make_pair(key, prefetcher)); + prefetcher->Start(); +} + +void ResourcePrefetcherManager::MaybeRemovePrefetch( + const NavigationID& navigation_id) { + DCHECK_CURRENTLY_ON(content::BrowserThread::IO); + + // Look for a URL based prefetch first. + PrefetcherMap::iterator it = prefetcher_map_.find( + navigation_id.main_frame_url.spec()); + if (it != prefetcher_map_.end() && + it->second->navigation_id() == navigation_id) { + it->second->Stop(); + return; + } + + // No URL based prefetching, look for host based. + it = prefetcher_map_.find(navigation_id.main_frame_url.host()); + if (it != prefetcher_map_.end() && + it->second->navigation_id() == navigation_id) { + it->second->Stop(); + } +} + +void ResourcePrefetcherManager::ResourcePrefetcherFinished( + ResourcePrefetcher* resource_prefetcher, + ResourcePrefetcher::RequestVector* requests) { + DCHECK_CURRENTLY_ON(content::BrowserThread::IO); + + // |predictor_| can only be accessed from the UI thread. + scoped_ptr scoped_requests(requests); + BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, + base::Bind(&ResourcePrefetcherManager::ResourcePrefetcherFinishedOnUI, + this, + resource_prefetcher->navigation_id(), + resource_prefetcher->key_type(), + base::Passed(&scoped_requests))); + + const GURL& main_frame_url = + resource_prefetcher->navigation_id().main_frame_url; + const std::string key = + resource_prefetcher->key_type() == PREFETCH_KEY_TYPE_HOST ? + main_frame_url.host() : main_frame_url.spec(); + PrefetcherMap::iterator it = prefetcher_map_.find(key); + DCHECK(it != prefetcher_map_.end()); + delete it->second; + prefetcher_map_.erase(it); +} + +void ResourcePrefetcherManager::ResourcePrefetcherFinishedOnUI( + const NavigationID& navigation_id, + PrefetchKeyType key_type, + scoped_ptr requests) { + DCHECK_CURRENTLY_ON(content::BrowserThread::UI); + + // |predictor_| may have been set to NULL if the predictor is shutting down. + if (predictor_) { + predictor_->FinishedPrefetchForNavigation(navigation_id, + key_type, + requests.release()); + } +} + +net::URLRequestContext* ResourcePrefetcherManager::GetURLRequestContext() { + DCHECK_CURRENTLY_ON(content::BrowserThread::IO); + + return context_getter_->GetURLRequestContext(); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetcher_manager.h b/chrome/browser/predictors/resource_prefetcher_manager.h new file mode 100644 index 0000000..91b23ed --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher_manager.h @@ -0,0 +1,87 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_ + +#include + +#include "base/memory/ref_counted.h" +#include "chrome/browser/predictors/resource_prefetch_common.h" +#include "chrome/browser/predictors/resource_prefetcher.h" + +namespace net { +class URLRequestContextGetter; +} + +namespace predictors { + +struct NavigationID; +class ResourcePrefetchPredictor; + +// Manages prefetches for multple navigations. +// - Created and owned by the resource prefetch predictor. +// - Needs to be refcounted as it is de-referenced on two different threads. +// - Created on the UI thread, but most functions are called in the IO thread. +// - Will only allow one inflight prefresh per main frame URL. +class ResourcePrefetcherManager + : public ResourcePrefetcher::Delegate, + public base::RefCountedThreadSafe { + public: + // The |predictor| should be alive till ShutdownOnIOThread is called. + ResourcePrefetcherManager(ResourcePrefetchPredictor* predictor, + const ResourcePrefetchPredictorConfig& config, + net::URLRequestContextGetter* getter); + + // UI thread. + void ShutdownOnUIThread(); + + // --- IO Thread methods. + + // The prefetchers need to be deleted on the IO thread. + void ShutdownOnIOThread(); + + // Will create a new ResourcePrefetcher for the main frame url of the input + // navigation if there isn't one already for the same URL or host (for host + // based). + void MaybeAddPrefetch(const NavigationID& navigation_id, + PrefetchKeyType key_type, + scoped_ptr requests); + + // Stops the ResourcePrefetcher for the input navigation, if one was in + // progress. + void MaybeRemovePrefetch(const NavigationID& navigation_id); + + // ResourcePrefetcher::Delegate methods. + virtual void ResourcePrefetcherFinished( + ResourcePrefetcher* prefetcher, + ResourcePrefetcher::RequestVector* requests) OVERRIDE; + virtual net::URLRequestContext* GetURLRequestContext() OVERRIDE; + + private: + friend class base::RefCountedThreadSafe; + friend class MockResourcePrefetcherManager; + + typedef std::map PrefetcherMap; + + virtual ~ResourcePrefetcherManager(); + + // UI Thread. |predictor_| needs to be called on the UI thread. + void ResourcePrefetcherFinishedOnUI( + const NavigationID& navigation_id, + PrefetchKeyType key_type, + scoped_ptr requests); + + ResourcePrefetchPredictor* predictor_; + const ResourcePrefetchPredictorConfig config_; + net::URLRequestContextGetter* const context_getter_; + + PrefetcherMap prefetcher_map_; // Owns the ResourcePrefetcher pointers. + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcherManager); +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_ diff --git a/chrome/browser/predictors/resource_prefetcher_unittest.cc b/chrome/browser/predictors/resource_prefetcher_unittest.cc new file mode 100644 index 0000000..bc21f89 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher_unittest.cc @@ -0,0 +1,343 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "base/message_loop/message_loop.h" +#include "chrome/browser/predictors/resource_prefetcher.h" +#include "chrome/browser/predictors/resource_prefetcher_manager.h" +#include "chrome/test/base/testing_profile.h" +#include "content/public/test/test_browser_thread.h" +#include "net/url_request/redirect_info.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using testing::Eq; +using testing::Property; + +namespace predictors { + +// Wrapper over the ResourcePrefetcher that stubs out the StartURLRequest call +// since we do not want to do network fetches in this unittest. +class TestResourcePrefetcher : public ResourcePrefetcher { + public: + TestResourcePrefetcher(ResourcePrefetcher::Delegate* delegate, + const ResourcePrefetchPredictorConfig& config, + const NavigationID& navigation_id, + PrefetchKeyType key_type, + scoped_ptr requests) + : ResourcePrefetcher(delegate, config, navigation_id, + key_type, requests.Pass()) { } + + virtual ~TestResourcePrefetcher() { } + + MOCK_METHOD1(StartURLRequest, void(net::URLRequest* request)); + + void ReadFullResponse(net::URLRequest* request) OVERRIDE { + FinishRequest(request, Request::PREFETCH_STATUS_FROM_CACHE); + } + + private: + DISALLOW_COPY_AND_ASSIGN(TestResourcePrefetcher); +}; + + +// Delegate for ResourcePrefetcher. +class TestResourcePrefetcherDelegate : public ResourcePrefetcher::Delegate { + public: + explicit TestResourcePrefetcherDelegate(base::MessageLoop* loop) + : request_context_getter_(new net::TestURLRequestContextGetter( + loop->message_loop_proxy())) { } + ~TestResourcePrefetcherDelegate() { } + + virtual net::URLRequestContext* GetURLRequestContext() OVERRIDE { + return request_context_getter_->GetURLRequestContext(); + } + + MOCK_METHOD2(ResourcePrefetcherFinished, + void(ResourcePrefetcher* prefetcher, + ResourcePrefetcher::RequestVector* requests)); + + private: + scoped_refptr request_context_getter_; + + DISALLOW_COPY_AND_ASSIGN(TestResourcePrefetcherDelegate); +}; + + +// The following unittest tests most of the ResourcePrefetcher except for: +// 1. Call to ReadFullResponse. There does not seem to be a good way to test the +// function in a unittest, and probably requires a browser_test. +// 2. Setting of the Prefetch status for cache vs non cache. +class ResourcePrefetcherTest : public testing::Test { + public: + ResourcePrefetcherTest(); + virtual ~ResourcePrefetcherTest(); + + protected: + typedef ResourcePrefetcher::Request Request; + + void AddStartUrlRequestExpectation(const std::string& url) { + EXPECT_CALL(*prefetcher_, + StartURLRequest(Property(&net::URLRequest::original_url, + Eq(GURL(url))))); + } + + void CheckPrefetcherState(size_t inflight, size_t queue, size_t host) { + EXPECT_EQ(prefetcher_->inflight_requests_.size(), inflight); + EXPECT_EQ(prefetcher_->request_queue_.size(), queue); + EXPECT_EQ(prefetcher_->host_inflight_counts_.size(), host); + } + + net::URLRequest* GetInFlightRequest(const std::string& url_str) { + GURL url(url_str); + + for (std::list::const_iterator it = + prefetcher_->request_queue_.begin(); + it != prefetcher_->request_queue_.end(); ++it) { + EXPECT_NE((*it)->resource_url, url); + } + for (std::map::const_iterator it = + prefetcher_->inflight_requests_.begin(); + it != prefetcher_->inflight_requests_.end(); ++it) { + if (it->first->original_url() == url) + return it->first; + } + EXPECT_TRUE(false) << "Infligh request not found: " << url_str; + return NULL; + } + + + void OnReceivedRedirect(const std::string& url) { + prefetcher_->OnReceivedRedirect( + GetInFlightRequest(url), net::RedirectInfo(), NULL); + } + void OnAuthRequired(const std::string& url) { + prefetcher_->OnAuthRequired(GetInFlightRequest(url), NULL); + } + void OnCertificateRequested(const std::string& url) { + prefetcher_->OnCertificateRequested(GetInFlightRequest(url), NULL); + } + void OnSSLCertificateError(const std::string& url) { + prefetcher_->OnSSLCertificateError(GetInFlightRequest(url), + net::SSLInfo(), false); + } + void OnResponse(const std::string& url) { + prefetcher_->OnResponseStarted(GetInFlightRequest(url)); + } + + base::MessageLoop loop_; + content::TestBrowserThread io_thread_; + ResourcePrefetchPredictorConfig config_; + TestResourcePrefetcherDelegate prefetcher_delegate_; + scoped_ptr prefetcher_; + + private: + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcherTest); +}; + +ResourcePrefetcherTest::ResourcePrefetcherTest() + : loop_(base::MessageLoop::TYPE_IO), + io_thread_(content::BrowserThread::IO, &loop_), + prefetcher_delegate_(&loop_) { + config_.max_prefetches_inflight_per_navigation = 5; + config_.max_prefetches_inflight_per_host_per_navigation = 2; +} + +ResourcePrefetcherTest::~ResourcePrefetcherTest() { +} + +TEST_F(ResourcePrefetcherTest, TestPrefetcherFinishes) { + scoped_ptr requests( + new ResourcePrefetcher::RequestVector); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource1.html"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource2.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource1.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource2.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource3.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource1.jpg"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource3.html"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource2.html"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource3.css"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource4.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource4.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource5.png"))); + + NavigationID navigation_id; + navigation_id.render_process_id = 1; + navigation_id.render_frame_id = 2; + navigation_id.main_frame_url = GURL("http://www.google.com"); + + // Needed later for comparison. + ResourcePrefetcher::RequestVector* requests_ptr = requests.get(); + + prefetcher_.reset(new TestResourcePrefetcher(&prefetcher_delegate_, + config_, + navigation_id, + PREFETCH_KEY_TYPE_URL, + requests.Pass())); + + // Starting the prefetcher maxes out the number of possible requests. + AddStartUrlRequestExpectation("http://www.google.com/resource1.html"); + AddStartUrlRequestExpectation("http://www.google.com/resource2.png"); + AddStartUrlRequestExpectation("http://yahoo.com/resource1.png"); + AddStartUrlRequestExpectation("http://yahoo.com/resource2.png"); + AddStartUrlRequestExpectation("http://m.google.com/resource1.jpg"); + + prefetcher_->Start(); + CheckPrefetcherState(5, 7, 3); + + AddStartUrlRequestExpectation("http://m.google.com/resource2.html"); + OnResponse("http://m.google.com/resource1.jpg"); + CheckPrefetcherState(5, 6, 3); + + AddStartUrlRequestExpectation("http://www.google.com/resource3.html"); + OnSSLCertificateError("http://www.google.com/resource1.html"); + CheckPrefetcherState(5, 5, 3); + + AddStartUrlRequestExpectation("http://m.google.com/resource3.css"); + OnResponse("http://m.google.com/resource2.html"); + CheckPrefetcherState(5, 4, 3); + + AddStartUrlRequestExpectation("http://m.google.com/resource4.png"); + OnReceivedRedirect("http://www.google.com/resource3.html"); + CheckPrefetcherState(5, 3, 3); + + OnResponse("http://www.google.com/resource2.png"); + CheckPrefetcherState(4, 3, 2); + + AddStartUrlRequestExpectation("http://yahoo.com/resource3.png"); + OnReceivedRedirect("http://yahoo.com/resource2.png"); + CheckPrefetcherState(4, 2, 2); + + AddStartUrlRequestExpectation("http://yahoo.com/resource4.png"); + OnResponse("http://yahoo.com/resource1.png"); + CheckPrefetcherState(4, 1, 2); + + AddStartUrlRequestExpectation("http://yahoo.com/resource5.png"); + OnResponse("http://yahoo.com/resource4.png"); + CheckPrefetcherState(4, 0, 2); + + OnResponse("http://yahoo.com/resource5.png"); + CheckPrefetcherState(3, 0, 2); + + OnCertificateRequested("http://m.google.com/resource4.png"); + CheckPrefetcherState(2, 0, 2); + + OnAuthRequired("http://m.google.com/resource3.css"); + CheckPrefetcherState(1, 0, 1); + + // Expect the final call. + EXPECT_CALL(prefetcher_delegate_, + ResourcePrefetcherFinished(Eq(prefetcher_.get()), + Eq(requests_ptr))); + + OnResponse("http://yahoo.com/resource3.png"); + CheckPrefetcherState(0, 0, 0); + + // Check the prefetch status. + EXPECT_EQ(Request::PREFETCH_STATUS_CERT_ERROR, + (*requests_ptr)[0]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE, + (*requests_ptr)[1]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE, + (*requests_ptr)[2]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_REDIRECTED, + (*requests_ptr)[3]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE, + (*requests_ptr)[4]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE, + (*requests_ptr)[5]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_REDIRECTED, + (*requests_ptr)[6]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE, + (*requests_ptr)[7]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_AUTH_REQUIRED, + (*requests_ptr)[8]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_CERT_REQUIRED, + (*requests_ptr)[9]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE, + (*requests_ptr)[10]->prefetch_status); + EXPECT_EQ(Request::PREFETCH_STATUS_FROM_CACHE, + (*requests_ptr)[11]->prefetch_status); +} + +TEST_F(ResourcePrefetcherTest, TestPrefetcherStopped) { + scoped_ptr requests( + new ResourcePrefetcher::RequestVector); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource1.html"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource2.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource1.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource2.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource3.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource1.jpg"))); + + NavigationID navigation_id; + navigation_id.render_process_id = 1; + navigation_id.render_frame_id = 2; + navigation_id.main_frame_url = GURL("http://www.google.com"); + + // Needed later for comparison. + ResourcePrefetcher::RequestVector* requests_ptr = requests.get(); + + prefetcher_.reset(new TestResourcePrefetcher(&prefetcher_delegate_, + config_, + navigation_id, + PREFETCH_KEY_TYPE_HOST, + requests.Pass())); + + // Starting the prefetcher maxes out the number of possible requests. + AddStartUrlRequestExpectation("http://www.google.com/resource1.html"); + AddStartUrlRequestExpectation("http://www.google.com/resource2.png"); + AddStartUrlRequestExpectation("http://yahoo.com/resource1.png"); + AddStartUrlRequestExpectation("http://yahoo.com/resource2.png"); + AddStartUrlRequestExpectation("http://m.google.com/resource1.jpg"); + + prefetcher_->Start(); + CheckPrefetcherState(5, 1, 3); + + OnResponse("http://www.google.com/resource1.html"); + CheckPrefetcherState(4, 1, 3); + + prefetcher_->Stop(); // No more queueing. + + OnResponse("http://www.google.com/resource2.png"); + CheckPrefetcherState(3, 1, 2); + + OnResponse("http://yahoo.com/resource1.png"); + CheckPrefetcherState(2, 1, 2); + + OnResponse("http://yahoo.com/resource2.png"); + CheckPrefetcherState(1, 1, 1); + + // Expect the final call. + EXPECT_CALL(prefetcher_delegate_, + ResourcePrefetcherFinished(Eq(prefetcher_.get()), + Eq(requests_ptr))); + + OnResponse("http://m.google.com/resource1.jpg"); + CheckPrefetcherState(0, 1, 0); +} + +} // namespace predictors diff --git a/chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc b/chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc index 40fc4b6..211502c6 100644 --- a/chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc +++ b/chrome/browser/printing/cloud_print/test/cloud_print_policy_browsertest.cc @@ -67,6 +67,13 @@ IN_PROC_BROWSER_TEST_F(CloudPrintPolicyTest, NormalPassedFlag) { IN_PROC_BROWSER_TEST_F(CloudPrintPolicyTest, DISABLED_CloudPrintPolicyFlag) { CommandLine new_command_line(GetCommandLineForRelaunch()); new_command_line.AppendSwitch(switches::kCheckCloudPrintConnectorPolicy); + // This is important for the test as the way the browser process is launched + // here causes the predictor databases to be initialized multiple times. This + // is not an issue for production where the process is launched as a service + // and a Profile is not created. See http://crbug.com/140466 for more details. + new_command_line.AppendSwitchASCII( + switches::kSpeculativeResourcePrefetching, + switches::kSpeculativeResourcePrefetchingDisabled); base::ProcessHandle handle; bool launched = diff --git a/chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc b/chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc index a31257b..3a36d23 100644 --- a/chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc +++ b/chrome/browser/profiles/chrome_browser_main_extra_parts_profiles.cc @@ -27,6 +27,7 @@ #include "chrome/browser/policy/profile_policy_connector_factory.h" #include "chrome/browser/predictors/autocomplete_action_predictor_factory.h" #include "chrome/browser/predictors/predictor_database_factory.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h" #include "chrome/browser/prerender/prerender_link_manager_factory.h" #include "chrome/browser/prerender/prerender_manager_factory.h" #include "chrome/browser/printing/cloud_print/cloud_print_proxy_service_factory.h" @@ -239,6 +240,7 @@ EnsureBrowserContextKeyedServiceFactoriesBuilt() { #endif predictors::AutocompleteActionPredictorFactory::GetInstance(); predictors::PredictorDatabaseFactory::GetInstance(); + predictors::ResourcePrefetchPredictorFactory::GetInstance(); prerender::PrerenderManagerFactory::GetInstance(); prerender::PrerenderLinkManagerFactory::GetInstance(); ProfileSyncServiceFactory::GetInstance(); diff --git a/chrome/browser/profiles/profile_io_data.cc b/chrome/browser/profiles/profile_io_data.cc index c61fda3..5608a36 100644 --- a/chrome/browser/profiles/profile_io_data.cc +++ b/chrome/browser/profiles/profile_io_data.cc @@ -42,6 +42,9 @@ #include "chrome/browser/net/chrome_url_request_context_getter.h" #include "chrome/browser/net/cookie_store_util.h" #include "chrome/browser/net/proxy_service_factory.h" +#include "chrome/browser/net/resource_prefetch_predictor_observer.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h" #include "chrome/browser/profiles/profile.h" #include "chrome/browser/profiles/profile_manager.h" #include "chrome/browser/signin/signin_names_io_thread.h" @@ -357,6 +360,13 @@ void ProfileIOData::InitializeOnUIThread(Profile* profile) { extensions::ExtensionSystem::Get(profile)->info_map(); #endif + if (predictors::ResourcePrefetchPredictor* predictor = + predictors::ResourcePrefetchPredictorFactory::GetForProfile( + profile)) { + resource_prefetch_predictor_observer_.reset( + new chrome_browser_net::ResourcePrefetchPredictorObserver(predictor)); + } + ProtocolHandlerRegistry* protocol_handler_registry = ProtocolHandlerRegistryFactory::GetForBrowserContext(profile); DCHECK(protocol_handler_registry); @@ -1054,6 +1064,11 @@ void ProfileIOData::Init( resource_context_->host_resolver_ = io_thread_globals->host_resolver.get(); resource_context_->request_context_ = main_request_context_.get(); + if (profile_params_->resource_prefetch_predictor_observer_) { + resource_prefetch_predictor_observer_.reset( + profile_params_->resource_prefetch_predictor_observer_.release()); + } + #if defined(ENABLE_MANAGED_USERS) supervised_user_url_filter_ = profile_params_->supervised_user_url_filter; #endif diff --git a/chrome/browser/profiles/profile_io_data.h b/chrome/browser/profiles/profile_io_data.h index 34fcb64..ad9aaee 100644 --- a/chrome/browser/profiles/profile_io_data.h +++ b/chrome/browser/profiles/profile_io_data.h @@ -42,6 +42,10 @@ class ProtocolHandlerRegistry; class SigninNamesOnIOThread; class SupervisedUserURLFilter; +namespace chrome_browser_net { +class ResourcePrefetchPredictorObserver; +} + namespace extensions { class InfoMap; } @@ -228,6 +232,11 @@ class ProfileIOData { return &incognito_availibility_pref_; } + chrome_browser_net::ResourcePrefetchPredictorObserver* + resource_prefetch_predictor_observer() const { + return resource_prefetch_predictor_observer_.get(); + } + #if defined(ENABLE_CONFIGURATION_POLICY) policy::PolicyHeaderIOHelper* policy_header_helper() const { return policy_header_helper_.get(); @@ -310,6 +319,8 @@ class ProfileIOData { #if defined(ENABLE_EXTENSIONS) scoped_refptr extension_info_map; #endif + scoped_ptr + resource_prefetch_predictor_observer_; // This pointer exists only as a means of conveying a url job factory // pointer from the protocol handler registry on the UI thread to the @@ -601,6 +612,9 @@ class ProfileIOData { mutable scoped_refptr host_content_settings_map_; + mutable scoped_ptr + resource_prefetch_predictor_observer_; + mutable scoped_ptr chrome_http_user_agent_settings_; diff --git a/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc b/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc index c3c6480..5a724f2 100644 --- a/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc +++ b/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc @@ -16,6 +16,7 @@ #include "chrome/browser/content_settings/host_content_settings_map.h" #include "chrome/browser/download/download_request_limiter.h" #include "chrome/browser/download/download_resource_throttle.h" +#include "chrome/browser/net/resource_prefetch_predictor_observer.h" #include "chrome/browser/prefetch/prefetch.h" #include "chrome/browser/prerender/prerender_manager.h" #include "chrome/browser/prerender/prerender_manager_factory.h" @@ -410,6 +411,11 @@ void ChromeResourceDispatcherHostDelegate::RequestBeginning( throttles); } #endif + + if (io_data->resource_prefetch_predictor_observer()) { + io_data->resource_prefetch_predictor_observer()->OnRequestStarted( + request, resource_type, info->GetChildID(), info->GetRenderFrameID()); + } } void ChromeResourceDispatcherHostDelegate::DownloadStarting( @@ -671,6 +677,9 @@ void ChromeResourceDispatcherHostDelegate::OnResponseStarted( } } + if (io_data->resource_prefetch_predictor_observer()) + io_data->resource_prefetch_predictor_observer()->OnResponseStarted(request); + // Ignores x-frame-options for the chrome signin UI. const std::string request_spec( request->first_party_for_cookies().GetOrigin().spec()); @@ -714,6 +723,11 @@ void ChromeResourceDispatcherHostDelegate::OnRequestRedirected( // management UI is built on top of it. signin::AppendMirrorRequestHeaderIfPossible(request, redirect_url, io_data); + if (io_data->resource_prefetch_predictor_observer()) { + io_data->resource_prefetch_predictor_observer()->OnRequestRedirected( + redirect_url, request); + } + #if defined(ENABLE_CONFIGURATION_POLICY) if (io_data->policy_header_helper()) io_data->policy_header_helper()->AddPolicyHeaders(redirect_url, request); diff --git a/chrome/browser/resources/predictors/predictors.html b/chrome/browser/resources/predictors/predictors.html index 83a64ed..01ba3b0 100644 --- a/chrome/browser/resources/predictors/predictors.html +++ b/chrome/browser/resources/predictors/predictors.html @@ -23,11 +23,15 @@ Autocomplete Action Predictor + Resource Prefetch Predictor + + + diff --git a/chrome/browser/resources/predictors/predictors.js b/chrome/browser/resources/predictors/predictors.js index b9ea1f6..ef5033c 100644 --- a/chrome/browser/resources/predictors/predictors.js +++ b/chrome/browser/resources/predictors/predictors.js @@ -3,6 +3,7 @@ // found in the LICENSE file. + if (cr.isWindows) document.documentElement.setAttribute('os', 'win'); diff --git a/chrome/browser/resources/predictors/resource_prefetch_predictor.html b/chrome/browser/resources/predictors/resource_prefetch_predictor.html new file mode 100644 index 0000000..36903fa --- /dev/null +++ b/chrome/browser/resources/predictors/resource_prefetch_predictor.html @@ -0,0 +1,49 @@ +
+ + + URL Table Cache + Host Table Cache + + + + + + + + + + + + + + + + + + +
Main Frame UrlResource UrlResource TypeNum HitsNum MissesConsec MissesAverage PositionScore
+ + + + + + + + + + + + + + + + + +
HostResource UrlResource TypeNum HitsNum MissesConsec MissesAverage PositionScore
+ + + +
+
+ Resource prefetch prediction is disabled. +
\ No newline at end of file diff --git a/chrome/browser/resources/predictors/resource_prefetch_predictor.js b/chrome/browser/resources/predictors/resource_prefetch_predictor.js new file mode 100644 index 0000000..f725105 --- /dev/null +++ b/chrome/browser/resources/predictors/resource_prefetch_predictor.js @@ -0,0 +1,102 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/** + * Requests the database from the backend. + */ +function requestResourcePrefetchPredictorDb() { + chrome.send('requestResourcePrefetchPredictorDb'); +} + +/** + * Callback from backend with the database contents. Sets up some globals and + * calls to create the UI. + * @param {Dictionary} database Information about ResourcePrefetchPredictor + * including the database as a flattened list, a boolean indicating if the + * system is enabled. + */ +function updateResourcePrefetchPredictorDb(database) { + updateResourcePrefetchPredictorDbView(database); +} + +/** + * Truncates the string to keep the database readable. + * @param {string} str The string to truncate. + * @return {string} The truncated string. + */ +function truncateString(str) { + return str.length < 100 ? str : str.substring(0, 99); +} + +/** + * Updates the table from the database. + * @param {Dictionary} database Information about ResourcePrefetchPredictor + * including the database as a flattened list, a boolean indicating if the + * system is enabled and the current hit weight. + */ +function updateResourcePrefetchPredictorDbView(database) { + if (!database.enabled) { + $('rpp_enabled').style.display = 'none'; + $('rpp_disabled').style.display = 'block'; + return; + } else { + $('rpp_enabled').style.display = 'block'; + $('rpp_disabled').style.display = 'none'; + } + + var hasUrlData = database.url_db && database.url_db.length > 0; + var hasHostData = database.host_db && database.host_db.length > 0; + + if (hasUrlData) + renderCacheData($('rpp_url_body'), database.url_db); + if (hasHostData) + renderCacheData($('rpp_host_body'), database.host_db); +} + +/** + * Renders cache data for URL or host based data. + * @param {HTMLElement} body element of table to render into. + * @param {Dictionary} database to render. + */ +function renderCacheData(body, database) { + body.textContent = ''; + for (var i = 0; i < database.length; ++i) { + var main = database[i]; + + for (var j = 0; j < main.resources.length; ++j) { + var resource = main.resources[j]; + var row = document.createElement('tr'); + + if (j == 0) { + var t = document.createElement('td'); + t.rowSpan = main.resources.length; + t.textContent = truncateString(main.main_frame_url); + t.className = 'last'; + row.appendChild(t); + } + + if (j == main.resources.length - 1) + row.className = 'last'; + + row.appendChild(document.createElement('td')).textContent = + truncateString(resource.resource_url); + row.appendChild(document.createElement('td')).textContent = + resource.resource_type; + row.appendChild(document.createElement('td')).textContent = + resource.number_of_hits; + row.appendChild(document.createElement('td')).textContent = + resource.number_of_misses; + row.appendChild(document.createElement('td')).textContent = + resource.consecutive_misses; + row.appendChild(document.createElement('td')).textContent = + resource.position; + row.appendChild(document.createElement('td')).textContent = + resource.score; + body.appendChild(row); + } + } +} + +document.addEventListener('DOMContentLoaded', + requestResourcePrefetchPredictorDb); diff --git a/chrome/browser/ui/tab_helpers.cc b/chrome/browser/ui/tab_helpers.cc index 797bb3a..036271b 100644 --- a/chrome/browser/ui/tab_helpers.cc +++ b/chrome/browser/ui/tab_helpers.cc @@ -12,6 +12,8 @@ #include "chrome/browser/infobars/infobar_service.h" #include "chrome/browser/net/net_error_tab_helper.h" #include "chrome/browser/password_manager/chrome_password_manager_client.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_tab_helper.h" #include "chrome/browser/prerender/prerender_tab_helper.h" #include "chrome/browser/sessions/session_tab_helper.h" #include "chrome/browser/tab_contents/navigation_metrics_recorder.h" @@ -227,4 +229,10 @@ void TabHelpers::AttachTabHelpers(WebContents* web_contents) { ChromePasswordManagerClient::GetManagerFromWebContents(web_contents)); } #endif + + if (predictors::ResourcePrefetchPredictorFactory::GetForProfile( + web_contents->GetBrowserContext())) { + predictors::ResourcePrefetchPredictorTabHelper::CreateForWebContents( + web_contents); + } } diff --git a/chrome/browser/ui/webui/predictors/predictors_handler.cc b/chrome/browser/ui/webui/predictors/predictors_handler.cc index bd7f39b..5845797 100644 --- a/chrome/browser/ui/webui/predictors/predictors_handler.cc +++ b/chrome/browser/ui/webui/predictors/predictors_handler.cc @@ -8,14 +8,39 @@ #include "base/values.h" #include "chrome/browser/predictors/autocomplete_action_predictor.h" #include "chrome/browser/predictors/autocomplete_action_predictor_factory.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_factory.h" +#include "chrome/browser/predictors/resource_prefetch_predictor_tables.h" #include "chrome/browser/profiles/profile.h" #include "content/public/browser/web_ui.h" +#include "content/public/common/resource_type.h" using predictors::AutocompleteActionPredictor; +using predictors::ResourcePrefetchPredictor; +using predictors::ResourcePrefetchPredictorTables; + +namespace { + +std::string ConvertResourceType(content::ResourceType type) { + switch (type) { + case content::RESOURCE_TYPE_IMAGE: + return "Image"; + case content::RESOURCE_TYPE_STYLESHEET: + return "Stylesheet"; + case content::RESOURCE_TYPE_SCRIPT: + return "Script"; + default: + return "Unknown"; + } +} + +} // namespace PredictorsHandler::PredictorsHandler(Profile* profile) { autocomplete_action_predictor_ = predictors::AutocompleteActionPredictorFactory::GetForProfile(profile); + resource_prefetch_predictor_ = + predictors::ResourcePrefetchPredictorFactory::GetForProfile(profile); } PredictorsHandler::~PredictorsHandler() { } @@ -24,6 +49,9 @@ void PredictorsHandler::RegisterMessages() { web_ui()->RegisterMessageCallback("requestAutocompleteActionPredictorDb", base::Bind(&PredictorsHandler::RequestAutocompleteActionPredictorDb, base::Unretained(this))); + web_ui()->RegisterMessageCallback("requestResourcePrefetchPredictorDb", + base::Bind(&PredictorsHandler::RequestResourcePrefetchPredictorDb, + base::Unretained(this))); } void PredictorsHandler::RequestAutocompleteActionPredictorDb( @@ -51,3 +79,52 @@ void PredictorsHandler::RequestAutocompleteActionPredictorDb( web_ui()->CallJavascriptFunction("updateAutocompleteActionPredictorDb", dict); } + +void PredictorsHandler::RequestResourcePrefetchPredictorDb( + const base::ListValue* args) { + const bool enabled = (resource_prefetch_predictor_ != NULL); + base::DictionaryValue dict; + dict.SetBoolean("enabled", enabled); + + if (enabled) { + // Url Database cache. + base::ListValue* db = new base::ListValue(); + AddPrefetchDataMapToListValue( + *resource_prefetch_predictor_->url_table_cache_, db); + dict.Set("url_db", db); + + db = new base::ListValue(); + AddPrefetchDataMapToListValue( + *resource_prefetch_predictor_->host_table_cache_, db); + dict.Set("host_db", db); + } + + web_ui()->CallJavascriptFunction("updateResourcePrefetchPredictorDb", dict); +} + +void PredictorsHandler::AddPrefetchDataMapToListValue( + const ResourcePrefetchPredictor::PrefetchDataMap& data_map, + base::ListValue* db) const { + for (ResourcePrefetchPredictor::PrefetchDataMap::const_iterator it = + data_map.begin(); it != data_map.end(); ++it) { + base::DictionaryValue* main = new base::DictionaryValue(); + main->SetString("main_frame_url", it->first); + base::ListValue* resources = new base::ListValue(); + for (ResourcePrefetchPredictor::ResourceRows::const_iterator + row = it->second.resources.begin(); + row != it->second.resources.end(); ++row) { + base::DictionaryValue* resource = new base::DictionaryValue(); + resource->SetString("resource_url", row->resource_url.spec()); + resource->SetString("resource_type", + ConvertResourceType(row->resource_type)); + resource->SetInteger("number_of_hits", row->number_of_hits); + resource->SetInteger("number_of_misses", row->number_of_misses); + resource->SetInteger("consecutive_misses", row->consecutive_misses); + resource->SetDouble("position", row->average_position); + resource->SetDouble("score", row->score); + resources->Append(resource); + } + main->Set("resources", resources); + db->Append(main); + } +} diff --git a/chrome/browser/ui/webui/predictors/predictors_handler.h b/chrome/browser/ui/webui/predictors/predictors_handler.h index 1c2c422..f3355e4 100644 --- a/chrome/browser/ui/webui/predictors/predictors_handler.h +++ b/chrome/browser/ui/webui/predictors/predictors_handler.h @@ -6,6 +6,7 @@ #define CHROME_BROWSER_UI_WEBUI_PREDICTORS_PREDICTORS_HANDLER_H_ #include "base/compiler_specific.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" #include "content/public/browser/web_ui_message_handler.h" namespace base { @@ -14,6 +15,7 @@ class ListValue; namespace predictors { class AutocompleteActionPredictor; +class ResourcePrefetchPredictor; } class Profile; @@ -32,7 +34,17 @@ class PredictorsHandler : public content::WebUIMessageHandler { // calls into JS with the resulting DictionaryValue. void RequestAutocompleteActionPredictorDb(const base::ListValue* args); + // Fetches stats for the ResourcePrefetchPredictor and returns it as a + // DictionaryValue to the JS. + void RequestResourcePrefetchPredictorDb(const base::ListValue* args); + + // Helper for RequestResourcePrefetchPredictorDb. + void AddPrefetchDataMapToListValue( + const predictors::ResourcePrefetchPredictor::PrefetchDataMap& data_map, + base::ListValue* db) const; + predictors::AutocompleteActionPredictor* autocomplete_action_predictor_; + predictors::ResourcePrefetchPredictor* resource_prefetch_predictor_; DISALLOW_COPY_AND_ASSIGN(PredictorsHandler); }; diff --git a/chrome/chrome_browser.gypi b/chrome/chrome_browser.gypi index 6805eab..92d0dc6 100644 --- a/chrome/chrome_browser.gypi +++ b/chrome/chrome_browser.gypi @@ -780,6 +780,8 @@ 'browser/net/quota_policy_channel_id_store.h', 'browser/net/referrer.cc', 'browser/net/referrer.h', + 'browser/net/resource_prefetch_predictor_observer.cc', + 'browser/net/resource_prefetch_predictor_observer.h', 'browser/net/safe_search_util.cc', 'browser/net/safe_search_util.h', 'browser/net/service_providers_win.cc', @@ -851,6 +853,20 @@ 'browser/predictors/predictor_database_factory.h', 'browser/predictors/predictor_table_base.cc', 'browser/predictors/predictor_table_base.h', + 'browser/predictors/resource_prefetch_common.cc', + 'browser/predictors/resource_prefetch_common.h', + 'browser/predictors/resource_prefetch_predictor.cc', + 'browser/predictors/resource_prefetch_predictor.h', + 'browser/predictors/resource_prefetch_predictor_factory.cc', + 'browser/predictors/resource_prefetch_predictor_factory.h', + 'browser/predictors/resource_prefetch_predictor_tab_helper.cc', + 'browser/predictors/resource_prefetch_predictor_tab_helper.h', + 'browser/predictors/resource_prefetch_predictor_tables.cc', + 'browser/predictors/resource_prefetch_predictor_tables.h', + 'browser/predictors/resource_prefetcher.cc', + 'browser/predictors/resource_prefetcher.h', + 'browser/predictors/resource_prefetcher_manager.cc', + 'browser/predictors/resource_prefetcher_manager.h', 'browser/pref_service_flags_storage.cc', 'browser/pref_service_flags_storage.h', 'browser/prefetch/prefetch.cc', diff --git a/chrome/chrome_tests_unit.gypi b/chrome/chrome_tests_unit.gypi index 45dade8..9922379 100644 --- a/chrome/chrome_tests_unit.gypi +++ b/chrome/chrome_tests_unit.gypi @@ -594,6 +594,9 @@ 'browser/power/process_power_collector_unittest.cc', 'browser/predictors/autocomplete_action_predictor_table_unittest.cc', 'browser/predictors/autocomplete_action_predictor_unittest.cc', + 'browser/predictors/resource_prefetch_predictor_unittest.cc', + 'browser/predictors/resource_prefetch_predictor_tables_unittest.cc', + 'browser/predictors/resource_prefetcher_unittest.cc', 'browser/prefs/chrome_pref_service_unittest.cc', 'browser/prefs/command_line_pref_store_unittest.cc', 'browser/prefs/incognito_mode_prefs_unittest.cc', diff --git a/chrome/common/chrome_switches.cc b/chrome/common/chrome_switches.cc index d6dc7ca..a04c81a 100644 --- a/chrome/common/chrome_switches.cc +++ b/chrome/common/chrome_switches.cc @@ -1080,6 +1080,20 @@ const char kSimulateOutdated[] = "simulate-outdated"; // Simulates that current version is outdated and auto-update is off. const char kSimulateOutdatedNoAU[] = "simulate-outdated-no-au"; +// Speculative resource prefetching. +const char kSpeculativeResourcePrefetching[] = + "speculative-resource-prefetching"; + +// Speculative resource prefetching is disabled. +const char kSpeculativeResourcePrefetchingDisabled[] = "disabled"; + +// Speculative resource prefetching will only learn about resources that need to +// be prefetched but will not prefetch them. +const char kSpeculativeResourcePrefetchingLearning[] = "learning"; + +// Speculative resource prefetching is enabled. +const char kSpeculativeResourcePrefetchingEnabled[] = "enabled"; + // Specifies the URL where spelling service feedback data will be sent instead // of the default URL. This switch is for temporary testing only. // TODO(rouslan): Remove this flag when feedback testing is complete. Revisit by diff --git a/chrome/common/chrome_switches.h b/chrome/common/chrome_switches.h index a91277a..65cda6b 100644 --- a/chrome/common/chrome_switches.h +++ b/chrome/common/chrome_switches.h @@ -298,6 +298,10 @@ extern const char kSimulateUpgrade[]; extern const char kSimulateCriticalUpdate[]; extern const char kSimulateOutdated[]; extern const char kSimulateOutdatedNoAU[]; +extern const char kSpeculativeResourcePrefetching[]; +extern const char kSpeculativeResourcePrefetchingDisabled[]; +extern const char kSpeculativeResourcePrefetchingEnabled[]; +extern const char kSpeculativeResourcePrefetchingLearning[]; extern const char kSpellingServiceFeedbackUrl[]; extern const char kSpellingServiceFeedbackIntervalSeconds[]; extern const char kSSLVersionMax[]; -- cgit v1.1