diff options
author | shishir@chromium.org <shishir@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-09-14 23:29:04 +0000 |
---|---|---|
committer | shishir@chromium.org <shishir@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-09-14 23:29:04 +0000 |
commit | 1bd55ae056e7772922da318c0282ad8d59c9d496 (patch) | |
tree | 5db42ca41693a4a5a0b0f6946fb1f2dc2e5776dd | |
parent | 95d4e5f8ca8b1456c5927df7376e2bd1b7cb8e3a (diff) | |
download | chromium_src-1bd55ae056e7772922da318c0282ad8d59c9d496.zip chromium_src-1bd55ae056e7772922da318c0282ad8d59c9d496.tar.gz chromium_src-1bd55ae056e7772922da318c0282ad8d59c9d496.tar.bz2 |
Adds speculative prefetching of resources.
The learning component was already checked in.
BUG=None
TEST=To be done.
Review URL: https://chromiumcodereview.appspot.com/10817004
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@156926 0039d316-1c4b-4281-b951-d872f2087c98
18 files changed, 1253 insertions, 51 deletions
diff --git a/chrome/browser/predictors/predictor_database.cc b/chrome/browser/predictors/predictor_database.cc index cb904fc..1949a3f 100644 --- a/chrome/browser/predictors/predictor_database.cc +++ b/chrome/browser/predictors/predictor_database.cc @@ -13,6 +13,7 @@ #include "chrome/browser/predictors/autocomplete_action_predictor_table.h" #include "chrome/browser/predictors/resource_prefetch_predictor.h" #include "chrome/browser/predictors/resource_prefetch_predictor_tables.h" +#include "chrome/browser/prerender/prerender_field_trial.h" #include "chrome/browser/profiles/profile.h" #include "content/public/browser/browser_thread.h" #include "sql/connection.h" @@ -66,7 +67,8 @@ PredictorDatabaseInternal::PredictorDatabaseInternal(Profile* profile) autocomplete_table_(new AutocompleteActionPredictorTable()), resource_prefetch_tables_(new ResourcePrefetchPredictorTables()) { is_resource_prefetch_predictor_enabled_ = - ResourcePrefetchPredictor::IsEnabled(profile); + prerender::IsSpeculativeResourcePrefetchingLearningEnabled(profile) || + prerender::IsSpeculativeResourcePrefetchingEnabled(profile); } PredictorDatabaseInternal::~PredictorDatabaseInternal() { diff --git a/chrome/browser/predictors/resource_prefetch_common.cc b/chrome/browser/predictors/resource_prefetch_common.cc index 5e87d43..ad4c0f9 100644 --- a/chrome/browser/predictors/resource_prefetch_common.cc +++ b/chrome/browser/predictors/resource_prefetch_common.cc @@ -52,4 +52,16 @@ bool NavigationID::IsSameRenderer(const NavigationID& other) const { render_view_id == other.render_view_id; } +ResourcePrefetchPredictorConfig::ResourcePrefetchPredictorConfig() + : max_navigation_lifetime_seconds(60), + max_urls_to_track(500), + min_url_visit_count(3), + max_resources_per_entry(50), + max_consecutive_misses(3), + min_resource_confidence_to_trigger_prefetch(0.8f), + min_resource_hits_to_trigger_prefetch(4), + max_prefetches_inflight_per_navigation(24), + max_prefetches_inflight_per_host_per_navigation(3) { +} + } // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetch_common.h b/chrome/browser/predictors/resource_prefetch_common.h index 9f63d18..5680cec 100644 --- a/chrome/browser/predictors/resource_prefetch_common.h +++ b/chrome/browser/predictors/resource_prefetch_common.h @@ -40,6 +40,39 @@ struct NavigationID { base::TimeTicks creation_time; }; +// Represents the config for the resource prefetch prediction algorithm. It is +// useful for running experiments. +struct ResourcePrefetchPredictorConfig { + // Initializes the config with default values. + ResourcePrefetchPredictorConfig(); + + // If a navigation hasn't seen a load complete event in this much time, it + // is considered abandoned. + int max_navigation_lifetime_seconds; + // Size of LRU caches for the URL data. + int max_urls_to_track; + // The number of times, we should have seen a visit to this URL in history + // to start tracking it. This is to ensure we dont bother with oneoff + // entries. + int min_url_visit_count; + // The maximum number of resources to store per entry. + int max_resources_per_entry; + // The number of consecutive misses after we stop tracking a resource URL. + int max_consecutive_misses; + + // The minimum confidence (accuracy of hits) required for a resource to be + // prefetched. + float min_resource_confidence_to_trigger_prefetch; + // The minimum number of times we must have a URL on record to prefetch it. + int min_resource_hits_to_trigger_prefetch; + + // Maximum number of prefetches that can be inflight for a single navigation. + int max_prefetches_inflight_per_navigation; + // Maximum number of prefetches that can be inflight for a host for a single + // navigation. + int max_prefetches_inflight_per_host_per_navigation; +}; + } // namespace predictors #endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_COMMON_H_ diff --git a/chrome/browser/predictors/resource_prefetch_predictor.cc b/chrome/browser/predictors/resource_prefetch_predictor.cc index 892e088..8cd0978 100644 --- a/chrome/browser/predictors/resource_prefetch_predictor.cc +++ b/chrome/browser/predictors/resource_prefetch_predictor.cc @@ -21,6 +21,7 @@ #include "chrome/browser/history/url_database.h" #include "chrome/browser/predictors/predictor_database.h" #include "chrome/browser/predictors/predictor_database_factory.h" +#include "chrome/browser/predictors/resource_prefetcher_manager.h" #include "chrome/browser/prerender/prerender_field_trial.h" #include "chrome/browser/profiles/profile.h" #include "chrome/common/chrome_notification_types.h" @@ -96,14 +97,6 @@ void RecordNavigationEvent(NavigationEvent event) { namespace predictors { -ResourcePrefetchPredictor::Config::Config() - : max_navigation_lifetime_seconds(60), - max_urls_to_track(500), - min_url_visit_count(3), - max_resources_per_entry(50), - max_consecutive_misses(3) { -} - ResourcePrefetchPredictor::URLRequestSummary::URLRequestSummary() : resource_type(ResourceType::LAST_TYPE), was_cached(false) { @@ -129,13 +122,14 @@ ResourcePrefetchPredictor::UrlTableCacheValue::~UrlTableCacheValue() { } ResourcePrefetchPredictor::ResourcePrefetchPredictor( - const Config& config, + const ResourcePrefetchPredictorConfig& config, Profile* profile) : profile_(profile), config_(config), initialization_state_(NOT_INITIALIZED), tables_(PredictorDatabaseFactory::GetForProfile( - profile)->resource_prefetch_tables()) { + profile)->resource_prefetch_tables()), + results_map_deleter_(&results_map_) { CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); notification_registrar_.Add(this, @@ -146,9 +140,11 @@ ResourcePrefetchPredictor::ResourcePrefetchPredictor( ResourcePrefetchPredictor::~ResourcePrefetchPredictor() { } -// static -bool ResourcePrefetchPredictor::IsEnabled(Profile* profile) { - return prerender::IsSpeculativeResourcePrefetchingLearningEnabled(profile); +void ResourcePrefetchPredictor::Shutdown() { + if (prefetch_manager_) { + prefetch_manager_->ShutdownOnUIThread(); + prefetch_manager_ = NULL; + } } void ResourcePrefetchPredictor::LazilyInitialize() { @@ -365,15 +361,60 @@ void ResourcePrefetchPredictor::OnMainFrameRequest( // New empty navigation entry. inflight_navigations_.insert(std::make_pair( request.navigation_id, std::vector<URLRequestSummary>())); + + // If prefetching is enabled, and we can prefetch something, start + // prefetching. + if (!prefetch_manager_.get()) + return; + + const GURL& main_frame_url = request.navigation_id.main_frame_url; + const UrlTableCacheMap::const_iterator value_iter = url_table_cache_.find( + main_frame_url); + if (value_iter == url_table_cache_.end()) + return; + + const UrlTableCacheValue& value = value_iter->second; + + scoped_ptr<ResourcePrefetcher::RequestVector> requests( + new ResourcePrefetcher::RequestVector); + for (UrlTableRowVector::const_iterator it = value.rows.begin(); + it != value.rows.end(); ++it) { + float confidence = static_cast<float>(it->number_of_hits) / + (it->number_of_hits + it->number_of_misses); + if (confidence < config_.min_resource_confidence_to_trigger_prefetch || + it->number_of_hits < config_.min_resource_hits_to_trigger_prefetch) { + continue; + } + + ResourcePrefetcher::Request* req = new ResourcePrefetcher::Request( + it->resource_url); + requests->push_back(req); + } + + if (requests->empty()) + return; + + BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, + base::Bind(&ResourcePrefetcherManager::MaybeAddPrefetch, + prefetch_manager_, + request.navigation_id, + base::Passed(&requests))); } void ResourcePrefetchPredictor::OnMainFrameResponse( const URLRequestSummary& response) { CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); + if (initialization_state_ != INITIALIZED) + return; RecordNavigationEvent(NAVIGATION_EVENT_RESPONSE_STARTED); - // TODO(shishir): The prefreshing will be stopped here. + if (prefetch_manager_.get()) + BrowserThread::PostTask( + BrowserThread::IO, FROM_HERE, + base::Bind(&ResourcePrefetcherManager::MaybeRemovePrefetch, + prefetch_manager_, + response.navigation_id)); } void ResourcePrefetchPredictor::OnMainFrameRedirect( @@ -445,6 +486,16 @@ void ResourcePrefetchPredictor::CleanupAbandonedNavigations( ++it; } } + for (ResultsMap::iterator it = results_map_.begin(); + it != results_map_.end();) { + if (it->first.IsSameRenderer(navigation_id) || + (time_now - it->first.creation_time > max_navigation_age)) { + delete it->second; + results_map_.erase(it++); + } else { + ++it; + } + } } void ResourcePrefetchPredictor::Observe( @@ -530,6 +581,18 @@ void ResourcePrefetchPredictor::Observe( } } +void ResourcePrefetchPredictor::FinishedPrefetchForNavigation( + const NavigationID& navigation_id, + ResourcePrefetcher::RequestVector* requests) { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); + + // Add the results to the results map. + if (!results_map_.insert(std::make_pair(navigation_id, requests)).second) { + DLOG(FATAL) << "Returning results for existing navigation."; + delete requests; + } +} + void ResourcePrefetchPredictor::OnHistoryAndCacheLoaded() { CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); DCHECK_EQ(initialization_state_, INITIALIZING); @@ -579,6 +642,12 @@ void ResourcePrefetchPredictor::OnHistoryAndCacheLoaded() { // TODO(shishir): Maybe listen for notifications for navigation being // abandoned and cleanup the inflight_navigations_. + // Initialize the prefetch manager only if prefetching is enabled. + if (prerender::IsSpeculativeResourcePrefetchingEnabled(profile_)) { + prefetch_manager_ = new ResourcePrefetcherManager( + this, config_, profile_->GetRequestContext()); + } + initialization_state_ = INITIALIZED; } @@ -616,7 +685,11 @@ void ResourcePrefetchPredictor::OnNavigationComplete( RecordNavigationEvent(NAVIGATION_EVENT_ONLOAD_TRACKED_URL); // Report any stats. - MaybeReportAccuracyStats(navigation_id); + if (prefetch_manager_.get()) { + MaybeReportAccuracyStats(navigation_id); + } else { + MaybeReportSimulatedAccuracyStats(navigation_id); + } // Update the URL table. const GURL& main_frame_url = navigation_id.main_frame_url; @@ -629,6 +702,8 @@ void ResourcePrefetchPredictor::OnNavigationComplete( // Remove the navigation. inflight_navigations_.erase(navigation_id); + delete results_map_[navigation_id]; + results_map_.erase(navigation_id); } void ResourcePrefetchPredictor::LearnUrlNavigation( @@ -637,7 +712,7 @@ void ResourcePrefetchPredictor::LearnUrlNavigation( CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); if (url_table_cache_.find(main_frame_url) == url_table_cache_.end()) { - if (url_table_cache_.size() >= config_.max_urls_to_track) + if (static_cast<int>(url_table_cache_.size()) >= config_.max_urls_to_track) RemoveAnEntryFromUrlDB(); url_table_cache_[main_frame_url].last_visit = base::Time::Now(); @@ -780,7 +855,7 @@ void ResourcePrefetchPredictor::RemoveAnEntryFromUrlDB() { urls_to_delete)); } -void ResourcePrefetchPredictor::MaybeReportAccuracyStats( +void ResourcePrefetchPredictor::MaybeReportSimulatedAccuracyStats( const NavigationID& navigation_id) const { const GURL& main_frame_url = navigation_id.main_frame_url; DCHECK(inflight_navigations_.find(navigation_id) != @@ -887,6 +962,130 @@ void ResourcePrefetchPredictor::ReportAccuracyHistograms( #undef RPP_PREDICTED_HISTOGRAM_COUNTS } +void ResourcePrefetchPredictor::MaybeReportAccuracyStats( + const NavigationID& navigation_id) { + NavigationMap::iterator nav_it = inflight_navigations_.find(navigation_id); + DCHECK(nav_it != inflight_navigations_.end()); + + ResultsMap::iterator results_it = results_map_.find(navigation_id); + bool have_prefetch_results = results_it != results_map_.end(); + UMA_HISTOGRAM_BOOLEAN("ResourcePrefetchPredictor.HavePrefetchResults", + have_prefetch_results); + if (!have_prefetch_results) + return; + + // Annotate the results. + const std::vector<URLRequestSummary>& actual = nav_it->second; + ResourcePrefetcher::RequestVector* prefetched = results_it->second; + + std::map<GURL, bool> actual_resources; + for (std::vector<URLRequestSummary>::const_iterator it = actual.begin(); + it != actual.end(); ++it) { + actual_resources[it->resource_url] = it->was_cached; + } + + int prefetch_cancelled = 0, prefetch_failed = 0, prefetch_not_started = 0; + // 'a_' -> actual, 'p_' -> predicted. + int p_cache_a_cache = 0, p_cache_a_network = 0, p_cache_a_notused = 0, + p_network_a_cache = 0, p_network_a_network = 0, p_network_a_notused = 0; + + for (ResourcePrefetcher::RequestVector::iterator it = prefetched->begin(); + it != prefetched->end(); ++it) { + ResourcePrefetcher::Request* req = *it; + + // Set the usage states if the resource was actually used. + std::map<GURL, bool>::iterator actual_it = actual_resources.find( + req->resource_url); + if (actual_it != actual_resources.end()) { + if (actual_it->second) { + req->usage_status = + ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE; + } else { + req->usage_status = + ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK; + } + } + + switch (req->prefetch_status) { + + // TODO(shishir): Add histogram for each cancellation reason. + case ResourcePrefetcher::Request::PREFETCH_STATUS_REDIRECTED: + case ResourcePrefetcher::Request::PREFETCH_STATUS_AUTH_REQUIRED: + case ResourcePrefetcher::Request::PREFETCH_STATUS_CERT_REQUIRED: + case ResourcePrefetcher::Request::PREFETCH_STATUS_CERT_ERROR: + case ResourcePrefetcher::Request::PREFETCH_STATUS_CANCELLED: + ++prefetch_cancelled; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_FAILED: + ++prefetch_failed; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_FROM_CACHE: + if (req->usage_status == + ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE) + ++p_cache_a_cache; + else if (req->usage_status == + ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK) + ++p_cache_a_network; + else + ++p_cache_a_notused; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_FROM_NETWORK: + if (req->usage_status == + ResourcePrefetcher::Request::USAGE_STATUS_FROM_CACHE) + ++p_network_a_cache; + else if (req->usage_status == + ResourcePrefetcher::Request::USAGE_STATUS_FROM_NETWORK) + ++p_network_a_network; + else + ++p_network_a_notused; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_NOT_STARTED: + ++prefetch_not_started; + break; + + case ResourcePrefetcher::Request::PREFETCH_STATUS_STARTED: + DLOG(FATAL) << "Invalid prefetch status"; + break; + } + } + + int total_prefetched = p_cache_a_cache + p_cache_a_network + p_cache_a_notused + + p_network_a_cache + p_network_a_network + p_network_a_notused; + + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchCancelled", + prefetch_cancelled * 100.0 / total_prefetched); + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchFailed", + prefetch_failed * 100.0 / total_prefetched); + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchFromCacheUsedFromCache", + p_cache_a_cache * 100.0 / total_prefetched); + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchFromCacheUsedFromNetwork", + p_cache_a_network * 100.0 / total_prefetched); + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchFromCacheNotUsed", + p_cache_a_notused * 100.0 / total_prefetched); + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchFromNetworkUsedFromCache", + p_network_a_cache * 100.0 / total_prefetched); + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchFromNetworkUsedFromNetwork", + p_network_a_network * 100.0 / total_prefetched); + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchFromNetworkNotUsed", + p_network_a_notused * 100.0 / total_prefetched); + + UMA_HISTOGRAM_PERCENTAGE( + "ResourcePrefetchPredictor.PrefetchNotStarted", + prefetch_not_started * 100.0 / (prefetch_not_started + total_prefetched)); +} + void ResourcePrefetchPredictor::DeleteAllUrls() { inflight_navigations_.clear(); url_table_cache_.clear(); diff --git a/chrome/browser/predictors/resource_prefetch_predictor.h b/chrome/browser/predictors/resource_prefetch_predictor.h index 09e0402..155625d 100644 --- a/chrome/browser/predictors/resource_prefetch_predictor.h +++ b/chrome/browser/predictors/resource_prefetch_predictor.h @@ -14,6 +14,7 @@ #include "base/memory/weak_ptr.h" #include "base/time.h" #include "chrome/browser/history/history_types.h" +#include "chrome/browser/predictors/resource_prefetcher.h" #include "chrome/browser/predictors/resource_prefetch_common.h" #include "chrome/browser/predictors/resource_prefetch_predictor_tables.h" #include "chrome/browser/profiles/profile_keyed_service.h" @@ -35,6 +36,8 @@ class URLRequest; namespace predictors { +class ResourcePrefetcherManager; + // Contains logic for learning what can be prefetched and for kicking off // speculative prefetching. // - The class is a profile keyed service owned by the profile. @@ -52,10 +55,16 @@ namespace predictors { // PredictorDatabase. // * ResourcePrefetchPredictor - Learns about resource requirements per URL in // the UI thread through the ResourcePrefetchPredictorObserver and perisists -// it to disk in the DB thread through the ResourcePrefetchPredictorTables. -// Owned by profile. +// it to disk in the DB thread through the ResourcePrefetchPredictorTables. It +// initiates resource prefetching using the ResourcePrefetcherManager. Owned +// by profile. +// * ResourcePrefetcherManager - Manages the ResourcePrefetchers that do the +// prefetching on the IO thread. The manager is owned by the +// ResourcePrefetchPredictor and interfaces between the predictor on the UI +// thread and the prefetchers on the IO thread. +// * ResourcePrefetcher - Lives entirely on the IO thread, owned by the +// ResourcePrefetcherManager, and issues net::URLRequest to fetch resources. // -// TODO(shishir): Implement the prefetching of resources. // TODO(shishir): Do speculative prefetching for https resources and/or https // main_frame urls. class ResourcePrefetchPredictor @@ -63,27 +72,6 @@ class ResourcePrefetchPredictor public content::NotificationObserver, public base::SupportsWeakPtr<ResourcePrefetchPredictor> { public: - // The following config allows us to change the predictor constants and run - // field trials with different constants. - struct Config { - // Initializes the config with default values. - Config(); - - // If a navigation hasn't seen a load complete event in this much time, it - // is considered abandoned. - int max_navigation_lifetime_seconds; // Default 60 - // Size of LRU caches for the Url data. - size_t max_urls_to_track; // Default 500 - // The number of times, we should have seen a visit to this Url in history - // to start tracking it. This is to ensure we dont bother with oneoff - // entries. - int min_url_visit_count; // Default 3 - // The maximum number of resources to store per entry. - int max_resources_per_entry; // Default 50 - // The number of consecutive misses after we stop tracking a resource Url. - int max_consecutive_misses; // Default 3 - }; - // Stores the data that we need to get from the URLRequest. struct URLRequestSummary { URLRequestSummary(); @@ -100,11 +88,11 @@ class ResourcePrefetchPredictor GURL redirect_url; // Empty unless request was redirected to a valid url. }; - ResourcePrefetchPredictor(const Config& config, Profile* profile); + ResourcePrefetchPredictor(const ResourcePrefetchPredictorConfig& config, + Profile* profile); virtual ~ResourcePrefetchPredictor(); // Thread safe. - static bool IsEnabled(Profile* profile); static bool ShouldRecordRequest(net::URLRequest* request, ResourceType::Type resource_type); static bool ShouldRecordResponse(net::URLRequest* response); @@ -118,6 +106,12 @@ class ResourcePrefetchPredictor void RecordUrlResponse(const URLRequestSummary& response); void RecordUrlRedirect(const URLRequestSummary& response); + // Called by ResourcePrefetcherManager to notify that prefetching has finished + // for a navigation. Should take ownership of |requests|. + virtual void FinishedPrefetchForNavigation( + const NavigationID& navigation_id, + ResourcePrefetcher::RequestVector* requests); + private: friend class ::PredictorsHandler; friend class ResourcePrefetchPredictorTest; @@ -158,12 +152,16 @@ class ResourcePrefetchPredictor typedef std::map<NavigationID, std::vector<URLRequestSummary> > NavigationMap; typedef std::map<GURL, UrlTableCacheValue> UrlTableCacheMap; + typedef std::map<NavigationID, ResourcePrefetcher::RequestVector*> ResultsMap; // content::NotificationObserver methods OVERRIDE. virtual void Observe(int type, const content::NotificationSource& source, const content::NotificationDetails& details) OVERRIDE; + // ProfileKeyedService methods OVERRIDE. + virtual void Shutdown() OVERRIDE; + static bool IsHandledMainPage(net::URLRequest* request); static bool IsHandledSubresource(net::URLRequest* response); static bool IsCacheable(const net::URLRequest* response); @@ -193,7 +191,9 @@ class ResourcePrefetchPredictor void OnNavigationComplete(const NavigationID& navigation_id); void LearnUrlNavigation(const GURL& main_frame_url, const std::vector<URLRequestSummary>& new_value); - void MaybeReportAccuracyStats(const NavigationID& navigation_id) const; + void MaybeReportAccuracyStats(const NavigationID& navigation_id); + void MaybeReportSimulatedAccuracyStats( + const NavigationID& navigation_id) const; void ReportAccuracyHistograms(const UrlTableRowVector& predicted, const std::map<GURL, bool>& actual_resources, int total_resources_fetched_from_network, @@ -203,13 +203,16 @@ class ResourcePrefetchPredictor scoped_refptr<ResourcePrefetchPredictorTables> tables); Profile* const profile_; - Config config_; + ResourcePrefetchPredictorConfig const config_; InitializationState initialization_state_; scoped_refptr<ResourcePrefetchPredictorTables> tables_; + scoped_refptr<ResourcePrefetcherManager> prefetch_manager_; content::NotificationRegistrar notification_registrar_; NavigationMap inflight_navigations_; UrlTableCacheMap url_table_cache_; + ResultsMap results_map_; + STLValueDeleter<ResultsMap> results_map_deleter_; DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictor); }; diff --git a/chrome/browser/predictors/resource_prefetch_predictor_factory.cc b/chrome/browser/predictors/resource_prefetch_predictor_factory.cc index b58c8ed..04de4e7 100644 --- a/chrome/browser/predictors/resource_prefetch_predictor_factory.cc +++ b/chrome/browser/predictors/resource_prefetch_predictor_factory.cc @@ -7,6 +7,7 @@ #include "chrome/browser/history/history_service_factory.h" #include "chrome/browser/predictors/predictor_database_factory.h" #include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "chrome/browser/prerender/prerender_field_trial.h" #include "chrome/browser/profiles/profile.h" #include "chrome/browser/profiles/profile_dependency_manager.h" @@ -37,10 +38,11 @@ ResourcePrefetchPredictorFactory::~ResourcePrefetchPredictorFactory() {} ProfileKeyedService* ResourcePrefetchPredictorFactory::BuildServiceInstanceFor( Profile* profile) const { - return ResourcePrefetchPredictor::IsEnabled(profile) ? - new ResourcePrefetchPredictor(ResourcePrefetchPredictor::Config(), - profile) : - NULL; + return (prerender::IsSpeculativeResourcePrefetchingEnabled(profile) || + prerender::IsSpeculativeResourcePrefetchingLearningEnabled(profile)) ? + new ResourcePrefetchPredictor(ResourcePrefetchPredictorConfig(), + profile) : + NULL; } } // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc b/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc index de0b00a..6ebec41 100644 --- a/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc +++ b/chrome/browser/predictors/resource_prefetch_predictor_unittest.cc @@ -100,7 +100,7 @@ class ResourcePrefetchPredictorTest : public testing::Test { } void ResetPredictor() { - ResourcePrefetchPredictor::Config config; + ResourcePrefetchPredictorConfig config; config.max_urls_to_track = 3; config.min_url_visit_count = 2; config.max_resources_per_entry = 4; diff --git a/chrome/browser/predictors/resource_prefetcher.cc b/chrome/browser/predictors/resource_prefetcher.cc new file mode 100644 index 0000000..5b1040c --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher.cc @@ -0,0 +1,225 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetcher.h" + +#include "base/stl_util.h" +#include "content/public/browser/browser_thread.h" +#include "net/base/io_buffer.h" + +namespace { + +// The size of the buffer used to read the resource. +static const size_t kResourceBufferSizeBytes = 50000; + +} // namespace + +namespace predictors { + +ResourcePrefetcher::Request::Request(const GURL& i_resource_url) + : resource_url(i_resource_url), + prefetch_status(PREFETCH_STATUS_NOT_STARTED), + usage_status(USAGE_STATUS_NOT_REQUESTED) { +} + +ResourcePrefetcher::Request::Request(const Request& other) + : resource_url(other.resource_url), + prefetch_status(other.prefetch_status), + usage_status(other.usage_status) { +} + +ResourcePrefetcher::ResourcePrefetcher( + Delegate* delegate, + const ResourcePrefetchPredictorConfig& config, + const NavigationID& navigation_id, + scoped_ptr<RequestVector> requests) + : state_(INITIALIZED), + delegate_(delegate), + config_(config), + navigation_id_(navigation_id), + request_vector_(requests.Pass()) { + CHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::IO)); + DCHECK(request_vector_.get()); + + std::copy(request_vector_->begin(), request_vector_->end(), + std::back_inserter(request_queue_)); +} + +ResourcePrefetcher::~ResourcePrefetcher() { + // Delete any pending net::URLRequests. + STLDeleteContainerPairFirstPointers(inflight_requests_.begin(), + inflight_requests_.end()); +} + +void ResourcePrefetcher::Start() { + CHECK(CalledOnValidThread()); + + CHECK_EQ(state_, INITIALIZED); + state_ = RUNNING; + + TryToLaunchPrefetchRequests(); +} + +void ResourcePrefetcher::Stop() { + CHECK(CalledOnValidThread()); + + if (state_ == FINISHED) + return; + + state_ = STOPPED; +} + +void ResourcePrefetcher::TryToLaunchPrefetchRequests() { + CHECK(state_ == RUNNING || state_ == STOPPED); + + // Try to launch new requests if the state is RUNNING. + if (state_ == RUNNING) { + bool request_available = true; + + // Loop through the requests while we are under the + // max_prefetches_inflight_per_host_per_navigation limit, looking for a URL + // for which the max_prefetches_inflight_per_host_per_navigation limit has + // not been reached. Try to launch as many requests as possible. + while ((static_cast<int>(inflight_requests_.size()) < + config_.max_prefetches_inflight_per_navigation) && + request_available) { + std::list<Request*>::iterator request_it = request_queue_.begin(); + for (; request_it != request_queue_.end(); ++request_it) { + const std::string& host = (*request_it)->resource_url.host(); + + std::map<std::string, int>::iterator host_it = + host_inflight_counts_.find(host); + if (host_it == host_inflight_counts_.end() || + host_it->second < + config_.max_prefetches_inflight_per_host_per_navigation) + break; + } + request_available = request_it != request_queue_.end(); + + if (request_available) { + SendRequest(*request_it); + request_queue_.erase(request_it); + } + } + } + + // If the inflight_requests_ is empty, we cant launch any more. Finish. + if (inflight_requests_.empty()) { + CHECK(host_inflight_counts_.empty()); + CHECK(request_queue_.empty() || state_ == STOPPED); + + state_ = FINISHED; + delegate_->ResourcePrefetcherFinished(this, request_vector_.release()); + } +} + +void ResourcePrefetcher::SendRequest(Request* request) { + request->prefetch_status = Request::PREFETCH_STATUS_STARTED; + + net::URLRequest* url_request = + new net::URLRequest(request->resource_url, + this, + delegate_->GetURLRequestContext()); + inflight_requests_[url_request] = request; + host_inflight_counts_[url_request->original_url().host()] += 1; + + url_request->set_method("GET"); + url_request->set_first_party_for_cookies(navigation_id_.main_frame_url); + url_request->set_referrer(navigation_id_.main_frame_url.spec()); + url_request->set_priority(net::LOW); + StartURLRequest(url_request); +} + +void ResourcePrefetcher::StartURLRequest(net::URLRequest* request) { + request->Start(); +} + +void ResourcePrefetcher::FinishRequest(net::URLRequest* request, + Request::PrefetchStatus status) { + std::map<net::URLRequest*, Request*>::iterator request_it = + inflight_requests_.find(request); + CHECK(request_it != inflight_requests_.end()); + + const std::string host = request->original_url().host(); + std::map<std::string, int>::iterator host_it = host_inflight_counts_.find( + host); + CHECK_GT(host_it->second, 0); + host_it->second -= 1; + if (host_it->second == 0) + host_inflight_counts_.erase(host); + + request_it->second->prefetch_status = status; + inflight_requests_.erase(request_it); + + delete request; + + TryToLaunchPrefetchRequests(); +} + +void ResourcePrefetcher::ReadFullResponse(net::URLRequest* request) { + bool status = true; + while (status) { + int bytes_read = 0; + scoped_refptr<net::IOBuffer> buffer(new net::IOBuffer( + kResourceBufferSizeBytes)); + status = request->Read(buffer, kResourceBufferSizeBytes, &bytes_read); + + if (status) { + if (request->status().error()) { + FinishRequest(request, Request::PREFETCH_STATUS_FAILED); + return; + } else if (bytes_read == 0) { + if (request->was_cached()) + FinishRequest(request, Request::PREFETCH_STATUS_FROM_CACHE); + else + FinishRequest(request, Request::PREFETCH_STATUS_FROM_NETWORK); + return; + } + } + } +} + +void ResourcePrefetcher::OnReceivedRedirect(net::URLRequest* request, + const GURL& new_url, + bool* defer_redirect) { + FinishRequest(request, Request::PREFETCH_STATUS_REDIRECTED); +} + +void ResourcePrefetcher::OnAuthRequired(net::URLRequest* request, + net::AuthChallengeInfo* auth_info) { + FinishRequest(request, Request::PREFETCH_STATUS_AUTH_REQUIRED); +} + +void ResourcePrefetcher::OnCertificateRequested( + net::URLRequest* request, + net::SSLCertRequestInfo* cert_request_info) { + FinishRequest(request, Request::PREFETCH_STATUS_CERT_REQUIRED); +} + +void ResourcePrefetcher::OnSSLCertificateError(net::URLRequest* request, + const net::SSLInfo& ssl_info, + bool fatal) { + FinishRequest(request, Request::PREFETCH_STATUS_CERT_ERROR); +} + +void ResourcePrefetcher::OnResponseStarted(net::URLRequest* request) { + if (request->status().error()) { + FinishRequest(request, Request::PREFETCH_STATUS_FAILED); + return; + } + + ReadFullResponse(request); +} + +void ResourcePrefetcher::OnReadCompleted(net::URLRequest* request, + int bytes_read) { + if (request->status().error()) { + FinishRequest(request, Request::PREFETCH_STATUS_FAILED); + return; + } + + ReadFullResponse(request); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetcher.h b/chrome/browser/predictors/resource_prefetcher.h new file mode 100644 index 0000000..117c6bd --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher.h @@ -0,0 +1,159 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_ + +#include <map> +#include <list> +#include <vector> + +#include "base/gtest_prod_util.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/scoped_vector.h" +#include "base/threading/non_thread_safe.h" +#include "chrome/browser/predictors/resource_prefetch_common.h" +#include "googleurl/src/gurl.h" +#include "net/url_request/url_request.h" + +namespace net { +class URLRequestContext; +} + +namespace predictors { + +// Responsible for prefetching resources for a single navigation based on the +// input list of resources. +// - Limits the max number of resources in flight for any host and also across +// hosts. +// - When stopped, will wait for the pending requests to finish. +// - Lives entirely on the IO thread. +class ResourcePrefetcher : public base::NonThreadSafe, + public net::URLRequest::Delegate { + public: + // Denotes the prefetch request for a single subresource. + struct Request { + explicit Request(const GURL& i_resource_url); + Request(const Request& other); + + enum PrefetchStatus { + PREFETCH_STATUS_NOT_STARTED, + PREFETCH_STATUS_STARTED, + + // Cancellation reasons. + PREFETCH_STATUS_REDIRECTED, + PREFETCH_STATUS_AUTH_REQUIRED, + PREFETCH_STATUS_CERT_REQUIRED, + PREFETCH_STATUS_CERT_ERROR, + PREFETCH_STATUS_CANCELLED, + PREFETCH_STATUS_FAILED, + + // Successful prefetch states. + PREFETCH_STATUS_FROM_CACHE, + PREFETCH_STATUS_FROM_NETWORK + }; + + enum UsageStatus { + USAGE_STATUS_NOT_REQUESTED, + USAGE_STATUS_FROM_CACHE, + USAGE_STATUS_FROM_NETWORK, + USAGE_STATUS_NAVIGATION_ABANDONED + }; + + GURL resource_url; + PrefetchStatus prefetch_status; + UsageStatus usage_status; + }; + typedef ScopedVector<Request> RequestVector; + + // Used to communicate when the prefetching is done. All methods are invoked + // on the IO thread. + class Delegate { + public: + virtual ~Delegate() { } + + // Called when the ResourcePrefetcher is finished, i.e. there is nothing + // pending in flight. Should take ownership of |requests|. + virtual void ResourcePrefetcherFinished( + ResourcePrefetcher* prefetcher, + RequestVector* requests) = 0; + + virtual net::URLRequestContext* GetURLRequestContext() = 0; + }; + + // |delegate| has to outlive the ResourcePrefetcher. The ResourcePrefetcher + // takes ownership of |requests|. + ResourcePrefetcher(Delegate* delegate, + const ResourcePrefetchPredictorConfig& config, + const NavigationID& navigation_id, + scoped_ptr<RequestVector> requests); + virtual ~ResourcePrefetcher(); + + void Start(); // Kicks off the prefetching. Can only be called once. + void Stop(); // No additional prefetches will be queued after this. + + const NavigationID& navigation_id() const { + return navigation_id_; + } + + private: + friend class ResourcePrefetcherTest; + friend class TestResourcePrefetcher; + + // Launches new prefetch requests if possible. + void TryToLaunchPrefetchRequests(); + + // Starts a net::URLRequest for the input |request|. + void SendRequest(Request* request); + + // Called by |SendRequest| to start the |request|. This is necessary to stub + // out the Start() call to net::URLRequest for unittesting. + virtual void StartURLRequest(net::URLRequest* request); + + // Marks the request as finished, with the given status. + void FinishRequest(net::URLRequest* request, Request::PrefetchStatus status); + + // Reads the response data from the response - required for the resource to + // be cached correctly. Stubbed out during testing. + virtual void ReadFullResponse(net::URLRequest* request); + + // net::URLRequest::Delegate methods. + virtual void OnReceivedRedirect(net::URLRequest* request, + const GURL& new_url, + bool* defer_redirect) OVERRIDE; + virtual void OnAuthRequired(net::URLRequest* request, + net::AuthChallengeInfo* auth_info) OVERRIDE; + virtual void OnCertificateRequested( + net::URLRequest* request, + net::SSLCertRequestInfo* cert_request_info) OVERRIDE; + virtual void OnSSLCertificateError(net::URLRequest* request, + const net::SSLInfo& ssl_info, + bool fatal) OVERRIDE; + virtual void OnResponseStarted(net::URLRequest* request) OVERRIDE; + virtual void OnReadCompleted(net::URLRequest* request, + int bytes_read) OVERRIDE; + + enum PrefetcherState { + INITIALIZED = 0, // Prefetching hasn't started. + RUNNING = 1, // Prefetching started, allowed to add more requests. + STOPPED = 2, // Prefetching started, not allowed to add more requests. + FINISHED = 3 // No more inflight request, new requests not possible. + }; + + PrefetcherState state_; + Delegate* const delegate_; + ResourcePrefetchPredictorConfig const config_; + NavigationID navigation_id_; + scoped_ptr<RequestVector> request_vector_; + + std::map<net::URLRequest*, Request*> inflight_requests_; + std::list<Request*> request_queue_; + std::map<std::string, int> host_inflight_counts_; + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcher); +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_H_ diff --git a/chrome/browser/predictors/resource_prefetcher_manager.cc b/chrome/browser/predictors/resource_prefetcher_manager.cc new file mode 100644 index 0000000..f5ed30e --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher_manager.cc @@ -0,0 +1,117 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/predictors/resource_prefetcher_manager.h" + +#include "base/bind.h" +#include "base/stl_util.h" +#include "chrome/browser/predictors/resource_prefetch_predictor.h" +#include "content/public/browser/browser_thread.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context_getter.h" + +using content::BrowserThread; + +namespace predictors { + +ResourcePrefetcherManager::ResourcePrefetcherManager( + ResourcePrefetchPredictor* predictor, + const ResourcePrefetchPredictorConfig& config, + net::URLRequestContextGetter* context_getter) + : predictor_(predictor), + config_(config), + context_getter_(context_getter) { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); + CHECK(predictor_); + CHECK(context_getter_); +} + +ResourcePrefetcherManager::~ResourcePrefetcherManager() { + DCHECK(prefetcher_map_.empty()) + << "Did not call ShutdownOnUIThread or ShutdownOnIOThread. " + " Will leak Prefetcher pointers."; +} + +void ResourcePrefetcherManager::ShutdownOnUIThread() { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); + + predictor_ = NULL; + BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, + base::Bind(&ResourcePrefetcherManager::ShutdownOnIOThread, + this)); +} + +void ResourcePrefetcherManager::ShutdownOnIOThread() { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); + STLDeleteContainerPairSecondPointers(prefetcher_map_.begin(), + prefetcher_map_.end()); +} + +void ResourcePrefetcherManager::MaybeAddPrefetch( + const NavigationID& navigation_id, + scoped_ptr<ResourcePrefetcher::RequestVector> requests) { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); + + const GURL& main_frame_url = navigation_id.main_frame_url; + PrefetcherMap::iterator prefetcher_it = prefetcher_map_.find(main_frame_url); + if (prefetcher_it != prefetcher_map_.end()) + return; + + ResourcePrefetcher* prefetcher = new ResourcePrefetcher( + this, config_, navigation_id, requests.Pass()); + prefetcher_map_.insert(std::make_pair(main_frame_url, prefetcher)); + prefetcher->Start(); +} + +void ResourcePrefetcherManager::MaybeRemovePrefetch( + const NavigationID& navigation_id) { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); + + PrefetcherMap::iterator it = prefetcher_map_.find( + navigation_id.main_frame_url); + if (it != prefetcher_map_.end() && + it->second->navigation_id() == navigation_id) { + it->second->Stop(); + } +} + +void ResourcePrefetcherManager::ResourcePrefetcherFinished( + ResourcePrefetcher* resource_prefetcher, + ResourcePrefetcher::RequestVector* requests) { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); + + // |predictor_| can only be accessed from the UI thread. + scoped_ptr<ResourcePrefetcher::RequestVector> requests_ptr(requests); + BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, + base::Bind(&ResourcePrefetcherManager::ResourcePrefetcherFinishedOnUI, + this, + resource_prefetcher->navigation_id(), + base::Passed(&requests_ptr))); + + const GURL& main_frame_url = + resource_prefetcher->navigation_id().main_frame_url; + PrefetcherMap::iterator it = prefetcher_map_.find(main_frame_url); + DCHECK(it != prefetcher_map_.end()); + delete it->second; + prefetcher_map_.erase(main_frame_url); +} + +void ResourcePrefetcherManager::ResourcePrefetcherFinishedOnUI( + const NavigationID& navigation_id, + scoped_ptr<ResourcePrefetcher::RequestVector> requests) { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); + + // |predictor_| may have been set to NULL if the predictor is shutting down. + if (predictor_) + predictor_->FinishedPrefetchForNavigation(navigation_id, + requests.release()); +} + +net::URLRequestContext* ResourcePrefetcherManager::GetURLRequestContext() { + CHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); + + return context_getter_->GetURLRequestContext(); +} + +} // namespace predictors diff --git a/chrome/browser/predictors/resource_prefetcher_manager.h b/chrome/browser/predictors/resource_prefetcher_manager.h new file mode 100644 index 0000000..3ac9c35 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher_manager.h @@ -0,0 +1,84 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_ +#define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_ + +#include <map> + +#include "base/memory/ref_counted.h" +#include "chrome/browser/predictors/resource_prefetcher.h" +#include "chrome/browser/predictors/resource_prefetch_common.h" + +namespace net { +class URLRequestContextGetter; +} + +namespace predictors { + +struct NavigationID; +class ResourcePrefetchPredictor; + +// Manages prefetches for multple navigations. +// - Created and owned by the resource prefetch predictor. +// - Needs to be refcounted as it is de-referenced on two different threads. +// - Created on the UI thread, but most functions are called in the IO thread. +// - Will only allow one inflight prefresh per main frame URL. +class ResourcePrefetcherManager + : public ResourcePrefetcher::Delegate, + public base::RefCountedThreadSafe<ResourcePrefetcherManager> { + public: + // The |predictor| should be alive till ShutdownOnIOThread is called. + ResourcePrefetcherManager(ResourcePrefetchPredictor* predictor, + const ResourcePrefetchPredictorConfig& config, + net::URLRequestContextGetter* getter); + + // UI thread. + void ShutdownOnUIThread(); + + // --- IO Thread methods. + + // The prefetchers need to be deleted on the IO thread. + void ShutdownOnIOThread(); + + // Will create a new ResourcePrefetcher for the main frame url of the input + // navigation if there isn't one already for the same URL. + void MaybeAddPrefetch(const NavigationID& navigation_id, + scoped_ptr<ResourcePrefetcher::RequestVector> requests); + + // Stops the ResourcePrefetcher for the input navigation, if one was in + // progress. + void MaybeRemovePrefetch(const NavigationID& navigation_id); + + // ResourcePrefetcher::Delegate methods. + virtual void ResourcePrefetcherFinished( + ResourcePrefetcher* prefetcher, + ResourcePrefetcher::RequestVector* requests) OVERRIDE; + virtual net::URLRequestContext* GetURLRequestContext() OVERRIDE; + + private: + friend class base::RefCountedThreadSafe<ResourcePrefetcherManager>; + friend class MockResourcePrefetcherManager; + + typedef std::map<GURL, ResourcePrefetcher*> PrefetcherMap; + + virtual ~ResourcePrefetcherManager(); + + // UI Thread. |predictor_| needs to be called on the UI thread. + void ResourcePrefetcherFinishedOnUI( + const NavigationID& navigation_id, + scoped_ptr<ResourcePrefetcher::RequestVector> requests); + + ResourcePrefetchPredictor* predictor_; + const ResourcePrefetchPredictorConfig config_; + net::URLRequestContextGetter* const context_getter_; + + PrefetcherMap prefetcher_map_; // Owns the ResourcePrefetcher pointers. + + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcherManager); +}; + +} // namespace predictors + +#endif // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCHER_MANAGER_H_ diff --git a/chrome/browser/predictors/resource_prefetcher_unittest.cc b/chrome/browser/predictors/resource_prefetcher_unittest.cc new file mode 100644 index 0000000..a0fae39 --- /dev/null +++ b/chrome/browser/predictors/resource_prefetcher_unittest.cc @@ -0,0 +1,337 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/memory/ref_counted.h" +#include "base/message_loop.h" +#include "base/memory/scoped_ptr.h" +#include "chrome/browser/predictors/resource_prefetcher.h" +#include "chrome/browser/predictors/resource_prefetcher_manager.h" +#include "chrome/test/base/testing_profile.h" +#include "content/public/test/test_browser_thread.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "testing/gmock/include/gmock/gmock.h" + +using testing::Eq; +using testing::Property; + +namespace predictors { + +// Wrapper over the ResourcePrefetcher that stubs out the StartURLRequest call +// since we do not want to do network fetches in this unittest. +class TestResourcePrefetcher : public ResourcePrefetcher { + public: + TestResourcePrefetcher(ResourcePrefetcher::Delegate* delegate, + const ResourcePrefetchPredictorConfig& config, + const NavigationID& navigation_id, + scoped_ptr<RequestVector> requests) + : ResourcePrefetcher(delegate, config, navigation_id, requests.Pass()) { } + + virtual ~TestResourcePrefetcher() { } + + MOCK_METHOD1(StartURLRequest, void(net::URLRequest* request)); + + void ReadFullResponse(net::URLRequest* request) OVERRIDE { + FinishRequest(request, Request::PREFETCH_STATUS_FROM_CACHE); + } + + private: + DISALLOW_COPY_AND_ASSIGN(TestResourcePrefetcher); +}; + + +// Delegate for ResourcePrefetcher. +class TestResourcePrefetcherDelegate : public ResourcePrefetcher::Delegate { + public: + explicit TestResourcePrefetcherDelegate(MessageLoop* loop) + : request_context_getter_(new TestURLRequestContextGetter( + loop->message_loop_proxy())) { } + ~TestResourcePrefetcherDelegate() { } + + virtual net::URLRequestContext* GetURLRequestContext() OVERRIDE { + return request_context_getter_->GetURLRequestContext(); + } + + MOCK_METHOD2(ResourcePrefetcherFinished, + void(ResourcePrefetcher* prefetcher, + ResourcePrefetcher::RequestVector* requests)); + + private: + TestURLRequestContextGetter* request_context_getter_; + + DISALLOW_COPY_AND_ASSIGN(TestResourcePrefetcherDelegate); +}; + + +// The following unittest tests most of the ResourcePrefetcher except for: +// 1. Call to ReadFullResponse. There does not seem to be a good way to test the +// function in a unittest, and probably requires a browser_test. +// 2. Setting of the Prefetch status for cache vs non cache. +class ResourcePrefetcherTest : public testing::Test { + public: + ResourcePrefetcherTest(); + ~ResourcePrefetcherTest(); + + protected: + typedef ResourcePrefetcher::Request Request; + + void AddStartUrlRequestExpectation(const std::string& url) { + EXPECT_CALL(*prefetcher_, + StartURLRequest(Property(&net::URLRequest::original_url, + Eq(GURL(url))))); + } + + void CheckPrefetcherState(size_t inflight, size_t queue, size_t host) { + EXPECT_EQ(prefetcher_->inflight_requests_.size(), inflight); + EXPECT_EQ(prefetcher_->request_queue_.size(), queue); + EXPECT_EQ(prefetcher_->host_inflight_counts_.size(), host); + } + + net::URLRequest* GetInFlightRequest(const std::string& url_str) { + GURL url(url_str); + + for (std::list<Request*>::const_iterator it = + prefetcher_->request_queue_.begin(); + it != prefetcher_->request_queue_.end(); ++it) { + EXPECT_NE((*it)->resource_url, url); + } + for (std::map<net::URLRequest*, Request*>::const_iterator it = + prefetcher_->inflight_requests_.begin(); + it != prefetcher_->inflight_requests_.end(); ++it) { + if (it->first->original_url() == url) + return it->first; + } + EXPECT_TRUE(false) << "Infligh request not found: " << url_str; + return NULL; + } + + + void OnReceivedRedirect(const std::string& url) { + prefetcher_->OnReceivedRedirect(GetInFlightRequest(url), GURL(""), NULL); + } + void OnAuthRequired(const std::string& url) { + prefetcher_->OnAuthRequired(GetInFlightRequest(url), NULL); + } + void OnCertificateRequested(const std::string& url) { + prefetcher_->OnCertificateRequested(GetInFlightRequest(url), NULL); + } + void OnSSLCertificateError(const std::string& url) { + prefetcher_->OnSSLCertificateError(GetInFlightRequest(url), + net::SSLInfo(), false); + } + void OnResponse(const std::string& url) { + prefetcher_->OnResponseStarted(GetInFlightRequest(url)); + } + + MessageLoop loop_; + content::TestBrowserThread io_thread_; + ResourcePrefetchPredictorConfig config_; + TestResourcePrefetcherDelegate prefetcher_delegate_; + scoped_ptr<TestResourcePrefetcher> prefetcher_; + + private: + DISALLOW_COPY_AND_ASSIGN(ResourcePrefetcherTest); +}; + +ResourcePrefetcherTest::ResourcePrefetcherTest() + : loop_(MessageLoop::TYPE_IO), + io_thread_(content::BrowserThread::IO, &loop_), + prefetcher_delegate_(&loop_) { + config_.max_prefetches_inflight_per_navigation = 5; + config_.max_prefetches_inflight_per_host_per_navigation = 2; +} + +ResourcePrefetcherTest::~ResourcePrefetcherTest() { +} + +TEST_F(ResourcePrefetcherTest, TestPrefetcherFinishes) { + scoped_ptr<ResourcePrefetcher::RequestVector> requests( + new ResourcePrefetcher::RequestVector); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource1.html"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource2.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource1.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource2.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource3.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource1.jpg"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource3.html"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource2.html"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource3.css"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource4.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource4.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource5.png"))); + + NavigationID navigation_id; + navigation_id.render_process_id = 1; + navigation_id.render_view_id = 2; + navigation_id.main_frame_url = GURL("http://www.google.com"); + + // Needed later for comparison. + ResourcePrefetcher::RequestVector* requests_ptr = requests.get(); + + prefetcher_.reset(new TestResourcePrefetcher(&prefetcher_delegate_, + config_, + navigation_id, + requests.Pass())); + + // Starting the prefetcher maxes out the number of possible requests. + AddStartUrlRequestExpectation("http://www.google.com/resource1.html"); + AddStartUrlRequestExpectation("http://www.google.com/resource2.png"); + AddStartUrlRequestExpectation("http://yahoo.com/resource1.png"); + AddStartUrlRequestExpectation("http://yahoo.com/resource2.png"); + AddStartUrlRequestExpectation("http://m.google.com/resource1.jpg"); + + prefetcher_->Start(); + CheckPrefetcherState(5, 7, 3); + + AddStartUrlRequestExpectation("http://m.google.com/resource2.html"); + OnResponse("http://m.google.com/resource1.jpg"); + CheckPrefetcherState(5, 6, 3); + + AddStartUrlRequestExpectation("http://www.google.com/resource3.html"); + OnSSLCertificateError("http://www.google.com/resource1.html"); + CheckPrefetcherState(5, 5, 3); + + AddStartUrlRequestExpectation("http://m.google.com/resource3.css"); + OnResponse("http://m.google.com/resource2.html"); + CheckPrefetcherState(5, 4, 3); + + AddStartUrlRequestExpectation("http://m.google.com/resource4.png"); + OnReceivedRedirect("http://www.google.com/resource3.html"); + CheckPrefetcherState(5, 3, 3); + + OnResponse("http://www.google.com/resource2.png"); + CheckPrefetcherState(4, 3, 2); + + AddStartUrlRequestExpectation("http://yahoo.com/resource3.png"); + OnReceivedRedirect("http://yahoo.com/resource2.png"); + CheckPrefetcherState(4, 2, 2); + + AddStartUrlRequestExpectation("http://yahoo.com/resource4.png"); + OnResponse("http://yahoo.com/resource1.png"); + CheckPrefetcherState(4, 1, 2); + + AddStartUrlRequestExpectation("http://yahoo.com/resource5.png"); + OnResponse("http://yahoo.com/resource4.png"); + CheckPrefetcherState(4, 0, 2); + + OnResponse("http://yahoo.com/resource5.png"); + CheckPrefetcherState(3, 0, 2); + + OnCertificateRequested("http://m.google.com/resource4.png"); + CheckPrefetcherState(2, 0, 2); + + OnAuthRequired("http://m.google.com/resource3.css"); + CheckPrefetcherState(1, 0, 1); + + // Expect the final call. + EXPECT_CALL(prefetcher_delegate_, + ResourcePrefetcherFinished(Eq(prefetcher_.get()), + Eq(requests_ptr))); + + OnResponse("http://yahoo.com/resource3.png"); + CheckPrefetcherState(0, 0, 0); + + // Check the prefetch status. + EXPECT_EQ((*requests_ptr)[0]->prefetch_status, + Request::PREFETCH_STATUS_CERT_ERROR); + EXPECT_EQ((*requests_ptr)[1]->prefetch_status, + Request::PREFETCH_STATUS_FROM_CACHE); + EXPECT_EQ((*requests_ptr)[2]->prefetch_status, + Request::PREFETCH_STATUS_FROM_CACHE); + EXPECT_EQ((*requests_ptr)[3]->prefetch_status, + Request::PREFETCH_STATUS_REDIRECTED); + EXPECT_EQ((*requests_ptr)[4]->prefetch_status, + Request::PREFETCH_STATUS_FROM_CACHE); + EXPECT_EQ((*requests_ptr)[5]->prefetch_status, + Request::PREFETCH_STATUS_FROM_CACHE); + EXPECT_EQ((*requests_ptr)[6]->prefetch_status, + Request::PREFETCH_STATUS_REDIRECTED); + EXPECT_EQ((*requests_ptr)[7]->prefetch_status, + Request::PREFETCH_STATUS_FROM_CACHE); + EXPECT_EQ((*requests_ptr)[8]->prefetch_status, + Request::PREFETCH_STATUS_AUTH_REQUIRED); + EXPECT_EQ((*requests_ptr)[9]->prefetch_status, + Request::PREFETCH_STATUS_CERT_REQUIRED); + EXPECT_EQ((*requests_ptr)[10]->prefetch_status, + Request::PREFETCH_STATUS_FROM_CACHE); + EXPECT_EQ((*requests_ptr)[11]->prefetch_status, + Request::PREFETCH_STATUS_FROM_CACHE); +} + +TEST_F(ResourcePrefetcherTest, TestPrefetcherStopped) { + scoped_ptr<ResourcePrefetcher::RequestVector> requests( + new ResourcePrefetcher::RequestVector); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource1.html"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://www.google.com/resource2.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource1.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource2.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://yahoo.com/resource3.png"))); + requests->push_back(new ResourcePrefetcher::Request(GURL( + "http://m.google.com/resource1.jpg"))); + + NavigationID navigation_id; + navigation_id.render_process_id = 1; + navigation_id.render_view_id = 2; + navigation_id.main_frame_url = GURL("http://www.google.com"); + + // Needed later for comparison. + ResourcePrefetcher::RequestVector* requests_ptr = requests.get(); + + prefetcher_.reset(new TestResourcePrefetcher(&prefetcher_delegate_, + config_, + navigation_id, + requests.Pass())); + + // Starting the prefetcher maxes out the number of possible requests. + AddStartUrlRequestExpectation("http://www.google.com/resource1.html"); + AddStartUrlRequestExpectation("http://www.google.com/resource2.png"); + AddStartUrlRequestExpectation("http://yahoo.com/resource1.png"); + AddStartUrlRequestExpectation("http://yahoo.com/resource2.png"); + AddStartUrlRequestExpectation("http://m.google.com/resource1.jpg"); + + prefetcher_->Start(); + CheckPrefetcherState(5, 1, 3); + + OnResponse("http://www.google.com/resource1.html"); + CheckPrefetcherState(4, 1, 3); + + prefetcher_->Stop(); // No more queueing. + + OnResponse("http://www.google.com/resource2.png"); + CheckPrefetcherState(3, 1, 2); + + OnResponse("http://yahoo.com/resource1.png"); + CheckPrefetcherState(2, 1, 2); + + OnResponse("http://yahoo.com/resource2.png"); + CheckPrefetcherState(1, 1, 1); + + // Expect the final call. + EXPECT_CALL(prefetcher_delegate_, + ResourcePrefetcherFinished(Eq(prefetcher_.get()), + Eq(requests_ptr))); + + OnResponse("http://m.google.com/resource1.jpg"); + CheckPrefetcherState(0, 1, 0); +} + +} // namespace predictors diff --git a/chrome/browser/prerender/prerender_field_trial.cc b/chrome/browser/prerender/prerender_field_trial.cc index 69edf7b..f50036f 100644 --- a/chrome/browser/prerender/prerender_field_trial.cc +++ b/chrome/browser/prerender/prerender_field_trial.cc @@ -293,4 +293,20 @@ bool IsSpeculativeResourcePrefetchingLearningEnabled(Profile* profile) { return group == g_speculative_prefetching_learning_default_group_number; } +bool IsSpeculativeResourcePrefetchingEnabled(Profile* profile) { + if (!profile) + return false; + + // Check if the user has set a command line flag. + if (CommandLine::ForCurrentProcess()->HasSwitch( + switches::kSpeculativeResourcePrefetching)) { + const std::string switch_value = + CommandLine::ForCurrentProcess()->GetSwitchValueASCII( + switches::kSpeculativeResourcePrefetching); + return switch_value == switches::kSpeculativeResourcePrefetchingEnabled; + } + + return false; +} + } // namespace prerender diff --git a/chrome/browser/prerender/prerender_field_trial.h b/chrome/browser/prerender/prerender_field_trial.h index cccecbf..3cd4fc3 100644 --- a/chrome/browser/prerender/prerender_field_trial.h +++ b/chrome/browser/prerender/prerender_field_trial.h @@ -25,6 +25,10 @@ bool IsOmniboxEnabled(Profile* profile); // resource prefetching learning experiment. bool IsSpeculativeResourcePrefetchingLearningEnabled(Profile* profile); +// Returns true iff the user has opted in or has been opted into the speculative +// resource prefetching experiment. +bool IsSpeculativeResourcePrefetchingEnabled(Profile* profile); + } // namespace prerender #endif // CHROME_BROWSER_PRERENDER_PRERENDER_FIELD_TRIAL_H_ diff --git a/chrome/chrome_browser.gypi b/chrome/chrome_browser.gypi index 0debbe4..d2ce42d 100644 --- a/chrome/chrome_browser.gypi +++ b/chrome/chrome_browser.gypi @@ -1357,6 +1357,10 @@ 'browser/predictors/resource_prefetch_predictor_factory.h', 'browser/predictors/resource_prefetch_predictor_tables.cc', 'browser/predictors/resource_prefetch_predictor_tables.h', + 'browser/predictors/resource_prefetcher.cc', + 'browser/predictors/resource_prefetcher.h', + 'browser/predictors/resource_prefetcher_manager.cc', + 'browser/predictors/resource_prefetcher_manager.h', 'browser/preferences_mac.cc', 'browser/preferences_mac.h', 'browser/prefs/browser_prefs.cc', diff --git a/chrome/chrome_tests.gypi b/chrome/chrome_tests.gypi index 0d85d39..e0dc618 100644 --- a/chrome/chrome_tests.gypi +++ b/chrome/chrome_tests.gypi @@ -1493,6 +1493,7 @@ 'browser/predictors/autocomplete_action_predictor_unittest.cc', 'browser/predictors/resource_prefetch_predictor_unittest.cc', 'browser/predictors/resource_prefetch_predictor_tables_unittest.cc', + 'browser/predictors/resource_prefetcher_unittest.cc', 'browser/preferences_mock_mac.cc', 'browser/preferences_mock_mac.h', 'browser/prefs/command_line_pref_store_unittest.cc', diff --git a/chrome/common/chrome_switches.cc b/chrome/common/chrome_switches.cc index 3752e6d..fa03557 100644 --- a/chrome/common/chrome_switches.cc +++ b/chrome/common/chrome_switches.cc @@ -1225,6 +1225,9 @@ const char kSpeculativeResourcePrefetchingDisabled[] = "disabled"; // be prefetched but will not prefetch them. const char kSpeculativeResourcePrefetchingLearning[] = "learning"; +// Speculative resource prefetching is enabled. +const char kSpeculativeResourcePrefetchingEnabled[] = "enabled"; + // Specifies the maximum SSL/TLS version ("ssl3", "tls1", "tls1.1", or // "tls1.2"). const char kSSLVersionMax[] = "ssl-version-max"; diff --git a/chrome/common/chrome_switches.h b/chrome/common/chrome_switches.h index 9f86745..6ba7f78 100644 --- a/chrome/common/chrome_switches.h +++ b/chrome/common/chrome_switches.h @@ -322,6 +322,7 @@ extern const char kSocketReusePolicy[]; extern const char kSpeculativeResourcePrefetching[]; extern const char kSpeculativeResourcePrefetchingDisabled[]; extern const char kSpeculativeResourcePrefetchingLearning[]; +extern const char kSpeculativeResourcePrefetchingEnabled[]; extern const char kSSLVersionMax[]; extern const char kSSLVersionMin[]; extern const char kStartMaximized[]; |