summaryrefslogtreecommitdiffstats
path: root/net/url_request
diff options
context:
space:
mode:
authorjoi@chromium.org <joi@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2010-11-25 02:29:06 +0000
committerjoi@chromium.org <joi@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2010-11-25 02:29:06 +0000
commit6b3f964f5de20d1d5d567bf67d16f5b246ac0299 (patch)
treedc0061b10ce3b9a3e52a2eb2e9784d1bad141da8 /net/url_request
parent9b41006d15b05b373724cce02c8b458cf173c9b9 (diff)
downloadchromium_src-6b3f964f5de20d1d5d567bf67d16f5b246ac0299.zip
chromium_src-6b3f964f5de20d1d5d567bf67d16f5b246ac0299.tar.gz
chromium_src-6b3f964f5de20d1d5d567bf67d16f5b246ac0299.tar.bz2
Implement exponential back-off mechanism.
Contributed by yzshen@google.com, original review http://codereview.chromium.org/4194001/ Implement exponential back-off mechanism. Enforce it at the URLRequestHttpJob level for all outgoing HTTP requests. The reason why to make this change is that we need back-off logic at a lower enough level to manage all outgoing HTTP traffic, so that the browser won't cause any DDoS attack. This change: 1) patches http://codereview.chromium.org/2487001/show, which is the exponential back-off implementation. 2) resolves conflicts with URLFetcher, by removing its own back-off logic: -- removes url_fetcher_protect.{h,cc}; -- integrates the sliding window mechanism of URLFetcherProtectEntry into RequestThrottlerEntry. 3) resolves conflicts with CloudPrintURLFetcher. 4) makes unit tests of CloudPrintURLFetcher, URLFetcher and URLRequest work. BUG=none TEST=pass all existing tests and also the newly-added request_throttler_unittest.cc Review URL: http://codereview.chromium.org/5276007 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@67375 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'net/url_request')
-rw-r--r--net/url_request/url_request_http_job.cc20
-rw-r--r--net/url_request/url_request_http_job.h4
-rw-r--r--net/url_request/url_request_throttler_entry.cc242
-rw-r--r--net/url_request/url_request_throttler_entry.h157
-rw-r--r--net/url_request/url_request_throttler_entry_interface.h63
-rw-r--r--net/url_request/url_request_throttler_header_adapter.cc27
-rw-r--r--net/url_request/url_request_throttler_header_adapter.h34
-rw-r--r--net/url_request/url_request_throttler_header_interface.h28
-rw-r--r--net/url_request/url_request_throttler_manager.cc107
-rw-r--r--net/url_request/url_request_throttler_manager.h101
-rw-r--r--net/url_request/url_request_throttler_unittest.cc346
11 files changed, 1127 insertions, 2 deletions
diff --git a/net/url_request/url_request_http_job.cc b/net/url_request/url_request_http_job.cc
index f00f490..826f635 100644
--- a/net/url_request/url_request_http_job.cc
+++ b/net/url_request/url_request_http_job.cc
@@ -33,6 +33,8 @@
#include "net/url_request/url_request_context.h"
#include "net/url_request/url_request_error_job.h"
#include "net/url_request/url_request_redirect_job.h"
+#include "net/url_request/url_request_throttler_header_adapter.h"
+#include "net/url_request/url_request_throttler_manager.h"
static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
@@ -91,6 +93,8 @@ URLRequestHttpJob::URLRequestHttpJob(URLRequest* request)
this, &URLRequestHttpJob::OnReadCompleted)),
read_in_progress_(false),
transaction_(NULL),
+ throttling_entry_(net::URLRequestThrottlerManager::GetInstance()->
+ RegisterRequestUrl(request->url())),
sdch_dictionary_advertised_(false),
sdch_test_activated_(false),
sdch_test_control_(false),
@@ -569,6 +573,12 @@ void URLRequestHttpJob::NotifyHeadersComplete() {
// also need this info.
is_cached_content_ = response_info_->was_cached;
+ if (!is_cached_content_) {
+ net::URLRequestThrottlerHeaderAdapter response_adapter(
+ response_info_->headers);
+ throttling_entry_->UpdateWithResponse(&response_adapter);
+ }
+
ProcessStrictTransportSecurityHeader();
if (SdchManager::Global() &&
@@ -618,6 +628,7 @@ void URLRequestHttpJob::StartTransaction() {
// with auth provided by username_ and password_.
int rv;
+
if (transaction_.get()) {
rv = transaction_->RestartWithAuth(username_, password_, &start_callback_);
username_.clear();
@@ -629,8 +640,13 @@ void URLRequestHttpJob::StartTransaction() {
rv = request_->context()->http_transaction_factory()->CreateTransaction(
&transaction_);
if (rv == net::OK) {
- rv = transaction_->Start(
- &request_info_, &start_callback_, request_->net_log());
+ if (!throttling_entry_->IsDuringExponentialBackoff()) {
+ rv = transaction_->Start(
+ &request_info_, &start_callback_, request_->net_log());
+ } else {
+ // Special error code for the exponential back-off module.
+ rv = net::ERR_TEMPORARILY_THROTTLED;
+ }
// Make sure the context is alive for the duration of the
// transaction.
context_ = request_->context();
diff --git a/net/url_request/url_request_http_job.h b/net/url_request/url_request_http_job.h
index 431756a..c981047 100644
--- a/net/url_request/url_request_http_job.h
+++ b/net/url_request/url_request_http_job.h
@@ -15,6 +15,7 @@
#include "net/base/completion_callback.h"
#include "net/http/http_request_info.h"
#include "net/url_request/url_request_job.h"
+#include "net/url_request/url_request_throttler_entry_interface.h"
namespace net {
class HttpResponseInfo;
@@ -112,6 +113,9 @@ class URLRequestHttpJob : public URLRequestJob {
scoped_ptr<net::HttpTransaction> transaction_;
+ // This is used to supervise traffic and enforce exponential back-off.
+ scoped_refptr<net::URLRequestThrottlerEntryInterface> throttling_entry_;
+
// Indicated if an SDCH dictionary was advertised, and hence an SDCH
// compressed response is expected. We use this to help detect (accidental?)
// proxy corruption of a response, which sometimes marks SDCH content as
diff --git a/net/url_request/url_request_throttler_entry.cc b/net/url_request/url_request_throttler_entry.cc
new file mode 100644
index 0000000..4abb438
--- /dev/null
+++ b/net/url_request/url_request_throttler_entry.cc
@@ -0,0 +1,242 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/url_request/url_request_throttler_entry.h"
+
+#include <cmath>
+
+#include "base/logging.h"
+#include "base/rand_util.h"
+#include "base/string_number_conversions.h"
+#include "net/url_request/url_request_throttler_header_interface.h"
+
+namespace net {
+
+const int URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs = 2000;
+const int URLRequestThrottlerEntry::kDefaultMaxSendThreshold = 20;
+const int URLRequestThrottlerEntry::kDefaultInitialBackoffMs = 700;
+const int URLRequestThrottlerEntry::kDefaultAdditionalConstantMs = 100;
+const double URLRequestThrottlerEntry::kDefaultMultiplyFactor = 1.4;
+const double URLRequestThrottlerEntry::kDefaultJitterFactor = 0.4;
+const int URLRequestThrottlerEntry::kDefaultMaximumBackoffMs = 60 * 60 * 1000;
+const int URLRequestThrottlerEntry::kDefaultEntryLifetimeMs = 120000;
+const char URLRequestThrottlerEntry::kRetryHeaderName[] = "X-Retry-After";
+
+URLRequestThrottlerEntry::URLRequestThrottlerEntry()
+ : sliding_window_period_(
+ base::TimeDelta::FromMilliseconds(kDefaultSlidingWindowPeriodMs)),
+ max_send_threshold_(kDefaultMaxSendThreshold),
+ initial_backoff_ms_(kDefaultInitialBackoffMs),
+ additional_constant_ms_(kDefaultAdditionalConstantMs),
+ multiply_factor_(kDefaultMultiplyFactor),
+ jitter_factor_(kDefaultJitterFactor),
+ maximum_backoff_ms_(kDefaultMaximumBackoffMs),
+ entry_lifetime_ms_(kDefaultEntryLifetimeMs) {
+ Initialize();
+}
+
+URLRequestThrottlerEntry::URLRequestThrottlerEntry(
+ int sliding_window_period_ms,
+ int max_send_threshold,
+ int initial_backoff_ms,
+ int additional_constant_ms,
+ double multiply_factor,
+ double jitter_factor,
+ int maximum_backoff_ms)
+ : sliding_window_period_(
+ base::TimeDelta::FromMilliseconds(sliding_window_period_ms)),
+ max_send_threshold_(max_send_threshold),
+ initial_backoff_ms_(initial_backoff_ms),
+ additional_constant_ms_(additional_constant_ms),
+ multiply_factor_(multiply_factor),
+ jitter_factor_(jitter_factor),
+ maximum_backoff_ms_(maximum_backoff_ms),
+ entry_lifetime_ms_(-1) {
+ DCHECK_GT(sliding_window_period_ms, 0);
+ DCHECK_GT(max_send_threshold_, 0);
+ DCHECK_GE(initial_backoff_ms_, 0);
+ DCHECK_GE(additional_constant_ms_, 0);
+ DCHECK_GT(multiply_factor_, 0);
+ DCHECK_GE(jitter_factor_, 0);
+ DCHECK_LT(jitter_factor_, 1);
+ DCHECK_GE(maximum_backoff_ms_, 0);
+
+ Initialize();
+}
+
+URLRequestThrottlerEntry::~URLRequestThrottlerEntry() {
+}
+
+void URLRequestThrottlerEntry::Initialize() {
+ // Since this method is called by the constructors, GetTimeNow() (a virtual
+ // method) is not used.
+ exponential_backoff_release_time_ = base::TimeTicks::Now();
+ failure_count_ = 0;
+ latest_response_was_failure_ = false;
+
+ sliding_window_release_time_ = base::TimeTicks::Now();
+}
+
+bool URLRequestThrottlerEntry::IsDuringExponentialBackoff() const {
+ return exponential_backoff_release_time_ > GetTimeNow();
+}
+
+int64 URLRequestThrottlerEntry::ReserveSendingTimeForNextRequest(
+ const base::TimeTicks& earliest_time) {
+ base::TimeTicks now = GetTimeNow();
+ // If a lot of requests were successfully made recently,
+ // sliding_window_release_time_ may be greater than
+ // exponential_backoff_release_time_.
+ base::TimeTicks recommended_sending_time =
+ std::max(std::max(now, earliest_time),
+ std::max(exponential_backoff_release_time_,
+ sliding_window_release_time_));
+
+ DCHECK(send_log_.empty() ||
+ recommended_sending_time >= send_log_.back());
+ // Log the new send event.
+ send_log_.push(recommended_sending_time);
+
+ sliding_window_release_time_ = recommended_sending_time;
+
+ // Drop the out-of-date events in the event list.
+ // We don't need to worry that the queue may become empty during this
+ // operation, since the last element is sliding_window_release_time_.
+ while ((send_log_.front() + sliding_window_period_ <=
+ sliding_window_release_time_) ||
+ send_log_.size() > static_cast<unsigned>(max_send_threshold_)) {
+ send_log_.pop();
+ }
+
+ // Check if there are too many send events in recent time.
+ if (send_log_.size() == static_cast<unsigned>(max_send_threshold_))
+ sliding_window_release_time_ = send_log_.front() + sliding_window_period_;
+
+ return (recommended_sending_time - now).InMillisecondsRoundedUp();
+}
+
+base::TimeTicks
+ URLRequestThrottlerEntry::GetExponentialBackoffReleaseTime() const {
+ return exponential_backoff_release_time_;
+}
+
+void URLRequestThrottlerEntry::UpdateWithResponse(
+ const URLRequestThrottlerHeaderInterface* response) {
+ if (response->GetResponseCode() >= 500) {
+ failure_count_++;
+ latest_response_was_failure_ = true;
+ exponential_backoff_release_time_ =
+ CalculateExponentialBackoffReleaseTime();
+ } else {
+ // We slowly decay the number of times delayed instead of resetting it to 0
+ // in order to stay stable if we received lots of requests with
+ // malformed bodies at the same time.
+ if (failure_count_ > 0)
+ failure_count_--;
+
+ latest_response_was_failure_ = false;
+
+ // The reason why we are not just cutting the release time to GetTimeNow()
+ // is on the one hand, it would unset delay put by our custom retry-after
+ // header and on the other we would like to push every request up to our
+ // "horizon" when dealing with multiple in-flight requests. Ex: If we send
+ // three requests and we receive 2 failures and 1 success. The success that
+ // follows those failures will not reset the release time, further requests
+ // will then need to wait the delay caused by the 2 failures.
+ exponential_backoff_release_time_ = std::max(
+ GetTimeNow(), exponential_backoff_release_time_);
+
+ std::string retry_header = response->GetNormalizedValue(kRetryHeaderName);
+ if (!retry_header.empty())
+ HandleCustomRetryAfter(retry_header);
+ }
+}
+
+bool URLRequestThrottlerEntry::IsEntryOutdated() const {
+ if (entry_lifetime_ms_ == -1)
+ return false;
+
+ base::TimeTicks now = GetTimeNow();
+
+ // If there are send events in the sliding window period, we still need this
+ // entry.
+ if (send_log_.size() > 0 &&
+ send_log_.back() + sliding_window_period_ > now) {
+ return false;
+ }
+
+ int64 unused_since_ms =
+ (now - exponential_backoff_release_time_).InMilliseconds();
+
+ // Release time is further than now, we are managing it.
+ if (unused_since_ms < 0)
+ return false;
+
+ // latest_response_was_failure_ is true indicates that the latest one or
+ // more requests encountered server errors or had malformed response bodies.
+ // In that case, we don't want to collect the entry unless it hasn't been used
+ // for longer than the maximum allowed back-off.
+ if (latest_response_was_failure_)
+ return unused_since_ms > std::max(maximum_backoff_ms_, entry_lifetime_ms_);
+
+ // Otherwise, consider the entry is outdated if it hasn't been used for the
+ // specified lifetime period.
+ return unused_since_ms > entry_lifetime_ms_;
+}
+
+void URLRequestThrottlerEntry::ReceivedContentWasMalformed() {
+ // For any response that is marked as malformed now, we have probably
+ // considered it as a success when receiving it and decreased the failure
+ // count by 1. As a result, we increase the failure count by 2 here to undo
+ // the effect and record a failure.
+ //
+ // Please note that this may lead to a larger failure count than expected,
+ // because we don't decrease the failure count for successful responses when
+ // it has already reached 0.
+ failure_count_ += 2;
+ latest_response_was_failure_ = true;
+ exponential_backoff_release_time_ = CalculateExponentialBackoffReleaseTime();
+}
+
+base::TimeTicks
+ URLRequestThrottlerEntry::CalculateExponentialBackoffReleaseTime() {
+ double delay = initial_backoff_ms_;
+ delay *= pow(multiply_factor_, failure_count_);
+ delay += additional_constant_ms_;
+ delay -= base::RandDouble() * jitter_factor_ * delay;
+
+ // Ensure that we do not exceed maximum delay.
+ int64 delay_int = static_cast<int64>(delay + 0.5);
+ delay_int = std::min(delay_int, static_cast<int64>(maximum_backoff_ms_));
+
+ return std::max(GetTimeNow() + base::TimeDelta::FromMilliseconds(delay_int),
+ exponential_backoff_release_time_);
+}
+
+base::TimeTicks URLRequestThrottlerEntry::GetTimeNow() const {
+ return base::TimeTicks::Now();
+}
+
+void URLRequestThrottlerEntry::HandleCustomRetryAfter(
+ const std::string& header_value) {
+ // Input parameter is the number of seconds to wait in a floating point value.
+ double time_in_sec = 0;
+ bool conversion_is_ok = base::StringToDouble(header_value, &time_in_sec);
+
+ // Conversion of custom retry-after header value failed.
+ if (!conversion_is_ok)
+ return;
+
+ // We must use an int value later so we transform this in milliseconds.
+ int64 value_ms = static_cast<int64>(0.5 + time_in_sec * 1000);
+
+ if (maximum_backoff_ms_ < value_ms || value_ms < 0)
+ return;
+
+ exponential_backoff_release_time_ = std::max(
+ (GetTimeNow() + base::TimeDelta::FromMilliseconds(value_ms)),
+ exponential_backoff_release_time_);
+}
+
+} // namespace net
diff --git a/net/url_request/url_request_throttler_entry.h b/net/url_request/url_request_throttler_entry.h
new file mode 100644
index 0000000..9b8955d
--- /dev/null
+++ b/net/url_request/url_request_throttler_entry.h
@@ -0,0 +1,157 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_URL_REQUEST_URL_REQUEST_THROTTLER_ENTRY_H_
+#define NET_URL_REQUEST_URL_REQUEST_THROTTLER_ENTRY_H_
+
+#include <queue>
+#include <string>
+
+#include "base/basictypes.h"
+#include "net/url_request/url_request_throttler_entry_interface.h"
+
+namespace net {
+
+// URLRequestThrottlerEntry represents an entry of URLRequestThrottlerManager.
+// It analyzes requests of a specific URL over some period of time, in order to
+// deduce the back-off time for every request.
+// The back-off algorithm consists of two parts. Firstly, exponential back-off
+// is used when receiving 5XX server errors or malformed response bodies.
+// The exponential back-off rule is enforced by URLRequestHttpJob. Any request
+// sent during the back-off period will be cancelled.
+// Secondly, a sliding window is used to count recent requests to a given
+// destination and provide guidance (to the application level only) on whether
+// too many requests have been sent and when a good time to send the next one
+// would be. This is never used to deny requests at the network level.
+class URLRequestThrottlerEntry : public URLRequestThrottlerEntryInterface {
+ public:
+ // Sliding window period.
+ static const int kDefaultSlidingWindowPeriodMs;
+
+ // Maximum number of requests allowed in sliding window period.
+ static const int kDefaultMaxSendThreshold;
+
+ // Initial delay for exponential back-off.
+ static const int kDefaultInitialBackoffMs;
+
+ // Additional constant to adjust back-off.
+ static const int kDefaultAdditionalConstantMs;
+
+ // Factor by which the waiting time will be multiplied.
+ static const double kDefaultMultiplyFactor;
+
+ // Fuzzing percentage. ex: 10% will spread requests randomly
+ // between 90%-100% of the calculated time.
+ static const double kDefaultJitterFactor;
+
+ // Maximum amount of time we are willing to delay our request.
+ static const int kDefaultMaximumBackoffMs;
+
+ // Time after which the entry is considered outdated.
+ static const int kDefaultEntryLifetimeMs;
+
+ // Name of the header that servers can use to ask clients to delay their next
+ // request.
+ static const char kRetryHeaderName[];
+
+ URLRequestThrottlerEntry();
+
+ // The life span of instances created with this constructor is set to
+ // infinite.
+ // It is only used by unit tests.
+ URLRequestThrottlerEntry(int sliding_window_period_ms,
+ int max_send_threshold,
+ int initial_backoff_ms,
+ int additional_constant_ms,
+ double multiply_factor,
+ double jitter_factor,
+ int maximum_backoff_ms);
+
+ // Implementation of URLRequestThrottlerEntryInterface.
+ virtual bool IsDuringExponentialBackoff() const;
+ virtual int64 ReserveSendingTimeForNextRequest(
+ const base::TimeTicks& earliest_time);
+ virtual base::TimeTicks GetExponentialBackoffReleaseTime() const;
+ virtual void UpdateWithResponse(
+ const URLRequestThrottlerHeaderInterface* response);
+ virtual void ReceivedContentWasMalformed();
+
+ // Used by the manager, returns true if the entry needs to be garbage
+ // collected.
+ bool IsEntryOutdated() const;
+
+ protected:
+ virtual ~URLRequestThrottlerEntry();
+
+ void Initialize();
+
+ // Calculates the release time for exponential back-off.
+ base::TimeTicks CalculateExponentialBackoffReleaseTime();
+
+ // Equivalent to TimeTicks::Now(), virtual to be mockable for testing purpose.
+ virtual base::TimeTicks GetTimeNow() const;
+
+ // Used internally to increase release time following a retry-after header.
+ void HandleCustomRetryAfter(const std::string& header_value);
+
+ // Used by tests.
+ void set_exponential_backoff_release_time(
+ const base::TimeTicks& release_time) {
+ exponential_backoff_release_time_ = release_time;
+ }
+
+ // Used by tests.
+ base::TimeTicks sliding_window_release_time() const {
+ return sliding_window_release_time_;
+ }
+
+ // Used by tests.
+ void set_sliding_window_release_time(const base::TimeTicks& release_time) {
+ sliding_window_release_time_ = release_time;
+ }
+
+ // Used by tests.
+ void set_failure_count(int failure_count) {
+ failure_count_ = failure_count;
+ }
+
+ private:
+ // Timestamp calculated by the exponential back-off algorithm at which we are
+ // allowed to start sending requests again.
+ base::TimeTicks exponential_backoff_release_time_;
+
+ // Number of times we encounter server errors or malformed response bodies.
+ int failure_count_;
+
+ // If true, the last request response was a failure.
+ // Note that this member can be false at the same time as failure_count_ can
+ // be greater than 0, since we gradually decrease failure_count_, instead of
+ // resetting it to 0 directly, when we receive successful responses.
+ bool latest_response_was_failure_;
+
+ // Timestamp calculated by the sliding window algorithm for when we advise
+ // clients the next request should be made, at the earliest. Advisory only,
+ // not used to deny requests.
+ base::TimeTicks sliding_window_release_time_;
+
+ // A list of the recent send events. We use them to decide whether there are
+ // too many requests sent in sliding window.
+ std::queue<base::TimeTicks> send_log_;
+
+ const base::TimeDelta sliding_window_period_;
+ const int max_send_threshold_;
+ const int initial_backoff_ms_;
+ const int additional_constant_ms_;
+ const double multiply_factor_;
+ const double jitter_factor_;
+ const int maximum_backoff_ms_;
+ // Set to -1 if the entry never expires.
+ const int entry_lifetime_ms_;
+
+ DISALLOW_COPY_AND_ASSIGN(URLRequestThrottlerEntry);
+};
+
+} // namespace net
+
+#endif // NET_URL_REQUEST_URL_REQUEST_THROTTLER_ENTRY_H_
diff --git a/net/url_request/url_request_throttler_entry_interface.h b/net/url_request/url_request_throttler_entry_interface.h
new file mode 100644
index 0000000..616e1d0
--- /dev/null
+++ b/net/url_request/url_request_throttler_entry_interface.h
@@ -0,0 +1,63 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_URL_REQUEST_URL_REQUEST_THROTTLER_ENTRY_INTERFACE_H_
+#define NET_URL_REQUEST_URL_REQUEST_THROTTLER_ENTRY_INTERFACE_H_
+
+#include "base/basictypes.h"
+#include "base/ref_counted.h"
+#include "base/time.h"
+
+namespace net {
+
+class URLRequestThrottlerHeaderInterface;
+
+// Interface provided on entries of the URL request throttler manager.
+class URLRequestThrottlerEntryInterface
+ : public base::RefCounted<URLRequestThrottlerEntryInterface> {
+ public:
+ URLRequestThrottlerEntryInterface() {}
+
+ // Returns true when we have encountered server errors and are doing
+ // exponential back-off.
+ // URLRequestHttpJob checks this method prior to every request; it cancels
+ // requests if this method returns true.
+ virtual bool IsDuringExponentialBackoff() const = 0;
+
+ // Calculates a recommended sending time for the next request and reserves it.
+ // The sending time is not earlier than the current exponential back-off
+ // release time or |earliest_time|. Moreover, the previous results of
+ // the method are taken into account, in order to make sure they are spread
+ // properly over time.
+ // Returns the recommended delay before sending the next request, in
+ // milliseconds. The return value is always positive or 0.
+ // Although it is not mandatory, respecting the value returned by this method
+ // is helpful to avoid traffic overload.
+ virtual int64 ReserveSendingTimeForNextRequest(
+ const base::TimeTicks& earliest_time) = 0;
+
+ // Returns the time after which requests are allowed.
+ virtual base::TimeTicks GetExponentialBackoffReleaseTime() const = 0;
+
+ // This method needs to be called each time a response is received.
+ virtual void UpdateWithResponse(
+ const URLRequestThrottlerHeaderInterface* response) = 0;
+
+ // Lets higher-level modules, that know how to parse particular response
+ // bodies, notify of receiving malformed content for the given URL. This will
+ // be handled by the throttler as if an HTTP 5xx response had been received to
+ // the request, i.e. it will count as a failure.
+ virtual void ReceivedContentWasMalformed() = 0;
+
+ protected:
+ virtual ~URLRequestThrottlerEntryInterface() {}
+
+ private:
+ friend class base::RefCounted<URLRequestThrottlerEntryInterface>;
+ DISALLOW_COPY_AND_ASSIGN(URLRequestThrottlerEntryInterface);
+};
+
+} // namespace net
+
+#endif // NET_URL_REQUEST_URL_REQUEST_THROTTLER_ENTRY_INTERFACE_H_
diff --git a/net/url_request/url_request_throttler_header_adapter.cc b/net/url_request/url_request_throttler_header_adapter.cc
new file mode 100644
index 0000000..e453071c
--- /dev/null
+++ b/net/url_request/url_request_throttler_header_adapter.cc
@@ -0,0 +1,27 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/url_request/url_request_throttler_header_adapter.h"
+
+#include "net/http/http_response_headers.h"
+
+namespace net {
+
+URLRequestThrottlerHeaderAdapter::URLRequestThrottlerHeaderAdapter(
+ net::HttpResponseHeaders* headers)
+ : response_header_(headers) {
+}
+
+std::string URLRequestThrottlerHeaderAdapter::GetNormalizedValue(
+ const std::string& key) const {
+ std::string return_value;
+ response_header_->GetNormalizedHeader(key, &return_value);
+ return return_value;
+}
+
+int URLRequestThrottlerHeaderAdapter::GetResponseCode() const {
+ return response_header_->response_code();
+}
+
+} // namespace net
diff --git a/net/url_request/url_request_throttler_header_adapter.h b/net/url_request/url_request_throttler_header_adapter.h
new file mode 100644
index 0000000..599a9f6
--- /dev/null
+++ b/net/url_request/url_request_throttler_header_adapter.h
@@ -0,0 +1,34 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_URL_REQUEST_URL_REQUEST_THROTTLER_HEADER_ADAPTER_H_
+#define NET_URL_REQUEST_URL_REQUEST_THROTTLER_HEADER_ADAPTER_H_
+
+#include <string>
+
+#include "base/ref_counted.h"
+#include "net/url_request/url_request_throttler_header_interface.h"
+
+namespace net {
+
+class HttpResponseHeaders;
+
+// Adapter for the HTTP header interface of the URL request throttler component.
+class URLRequestThrottlerHeaderAdapter
+ : public URLRequestThrottlerHeaderInterface {
+ public:
+ explicit URLRequestThrottlerHeaderAdapter(net::HttpResponseHeaders* headers);
+ virtual ~URLRequestThrottlerHeaderAdapter() {}
+
+ // Implementation of URLRequestThrottlerHeaderInterface
+ virtual std::string GetNormalizedValue(const std::string& key) const;
+ virtual int GetResponseCode() const;
+
+ private:
+ const scoped_refptr<net::HttpResponseHeaders> response_header_;
+};
+
+} // namespace net
+
+#endif // NET_URL_REQUEST_URL_REQUEST_THROTTLER_HEADER_ADAPTER_H_
diff --git a/net/url_request/url_request_throttler_header_interface.h b/net/url_request/url_request_throttler_header_interface.h
new file mode 100644
index 0000000..c69d185
--- /dev/null
+++ b/net/url_request/url_request_throttler_header_interface.h
@@ -0,0 +1,28 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_URL_REQUEST_URL_REQUEST_THROTTLER_HEADER_INTERFACE_H_
+#define NET_URL_REQUEST_URL_REQUEST_THROTTLER_HEADER_INTERFACE_H_
+
+#include <string>
+
+namespace net {
+
+// Interface to an HTTP header to enforce we have the methods we need.
+class URLRequestThrottlerHeaderInterface {
+ public:
+ virtual ~URLRequestThrottlerHeaderInterface() {}
+
+ // Method that enables us to fetch the header value by its key.
+ // ex: location: www.example.com -> key = "location" value = "www.example.com"
+ // If the key does not exist, it returns an empty string.
+ virtual std::string GetNormalizedValue(const std::string& key) const = 0;
+
+ // Returns the HTTP response code associated with the request.
+ virtual int GetResponseCode() const = 0;
+};
+
+} // namespace net
+
+#endif // NET_URL_REQUEST_URL_REQUEST_THROTTLER_HEADER_INTERFACE_H_
diff --git a/net/url_request/url_request_throttler_manager.cc b/net/url_request/url_request_throttler_manager.cc
new file mode 100644
index 0000000..5428d9a
--- /dev/null
+++ b/net/url_request/url_request_throttler_manager.cc
@@ -0,0 +1,107 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/url_request/url_request_throttler_manager.h"
+
+#include "base/string_util.h"
+
+namespace net {
+
+const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500;
+const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200;
+
+URLRequestThrottlerManager* URLRequestThrottlerManager::GetInstance() {
+ return Singleton<URLRequestThrottlerManager>::get();
+}
+
+scoped_refptr<URLRequestThrottlerEntryInterface>
+ URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
+ // Normalize the url.
+ std::string url_id = GetIdFromUrl(url);
+
+ // Periodically garbage collect old entries.
+ GarbageCollectEntriesIfNecessary();
+
+ // Find the entry in the map or create it.
+ scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id];
+ if (entry == NULL)
+ entry = new URLRequestThrottlerEntry();
+
+ return entry;
+}
+
+URLRequestThrottlerManager::URLRequestThrottlerManager()
+ : requests_since_last_gc_(0) {
+}
+
+URLRequestThrottlerManager::~URLRequestThrottlerManager() {
+ // Delete all entries.
+ url_entries_.clear();
+}
+
+std::string URLRequestThrottlerManager::GetIdFromUrl(const GURL& url) const {
+ if (!url.is_valid())
+ return url.possibly_invalid_spec();
+
+ if (url_id_replacements_ == NULL) {
+ url_id_replacements_.reset(new GURL::Replacements());
+
+ url_id_replacements_->ClearPassword();
+ url_id_replacements_->ClearUsername();
+ url_id_replacements_->ClearQuery();
+ url_id_replacements_->ClearRef();
+ }
+
+ GURL id = url.ReplaceComponents(*url_id_replacements_);
+ return StringToLowerASCII(id.spec());
+}
+
+void URLRequestThrottlerManager::GarbageCollectEntries() {
+ UrlEntryMap::iterator i = url_entries_.begin();
+
+ while (i != url_entries_.end()) {
+ if ((i->second)->IsEntryOutdated()) {
+ url_entries_.erase(i++);
+ } else {
+ ++i;
+ }
+ }
+
+ // In case something broke we want to make sure not to grow indefinitely.
+ while (url_entries_.size() > kMaximumNumberOfEntries) {
+ url_entries_.erase(url_entries_.begin());
+ }
+}
+
+void URLRequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
+ requests_since_last_gc_++;
+ if (requests_since_last_gc_ < kRequestsBetweenCollecting)
+ return;
+
+ requests_since_last_gc_ = 0;
+ GarbageCollectEntries();
+}
+
+void URLRequestThrottlerManager::OverrideEntryForTests(
+ const GURL& url,
+ URLRequestThrottlerEntry* entry) {
+ if (entry == NULL)
+ return;
+
+ // Normalize the url.
+ std::string url_id = GetIdFromUrl(url);
+
+ // Periodically garbage collect old entries.
+ GarbageCollectEntriesIfNecessary();
+
+ url_entries_[url_id] = entry;
+}
+
+void URLRequestThrottlerManager::EraseEntryForTests(const GURL& url) {
+ // Normalize the url.
+ std::string url_id = GetIdFromUrl(url);
+ url_entries_.erase(url_id);
+}
+
+} // namespace net
diff --git a/net/url_request/url_request_throttler_manager.h b/net/url_request/url_request_throttler_manager.h
new file mode 100644
index 0000000..6c8cd2f
--- /dev/null
+++ b/net/url_request/url_request_throttler_manager.h
@@ -0,0 +1,101 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_URL_REQUEST_URL_REQUEST_THROTTLER_MANAGER_H_
+#define NET_URL_REQUEST_URL_REQUEST_THROTTLER_MANAGER_H_
+
+#include <map>
+#include <string>
+
+#include "base/basictypes.h"
+#include "base/scoped_ptr.h"
+#include "base/singleton.h"
+#include "googleurl/src/gurl.h"
+#include "net/url_request/url_request_throttler_entry.h"
+
+namespace net {
+
+// Class that registers URL request throttler entries for URLs being accessed in
+// order to supervise traffic. URL requests for HTTP contents should register
+// their URLs in this manager on each request.
+// URLRequestThrottlerManager maintains a map of URL IDs to URL request
+// throttler entries. It creates URL request throttler entries when new URLs are
+// registered, and does garbage collection from time to time in order to clean
+// out outdated entries. URL ID consists of lowercased scheme, host, port and
+// path. All URLs converted to the same ID will share the same entry.
+//
+// NOTE: All usage of the singleton object of this class should be on the same
+// thread.
+class URLRequestThrottlerManager {
+ public:
+ static URLRequestThrottlerManager* GetInstance();
+
+ // Must be called for every request, returns the URL request throttler entry
+ // associated with the URL. The caller must inform this entry of some events.
+ // Please refer to url_request_throttler_entry_interface.h for further
+ // informations.
+ scoped_refptr<URLRequestThrottlerEntryInterface> RegisterRequestUrl(
+ const GURL& url);
+
+ // Registers a new entry in this service and overrides the existing entry (if
+ // any) for the URL. The service will hold a reference to the entry.
+ // It is only used by unit tests.
+ void OverrideEntryForTests(const GURL& url, URLRequestThrottlerEntry* entry);
+
+ // Explicitly erases an entry.
+ // This is useful to remove those entries which have got infinite lifetime and
+ // thus won't be garbage collected.
+ // It is only used by unit tests.
+ void EraseEntryForTests(const GURL& url);
+
+ protected:
+ URLRequestThrottlerManager();
+ ~URLRequestThrottlerManager();
+
+ // Method that allows us to transform a URL into an ID that can be used in our
+ // map. Resulting IDs will be lowercase and consist of the scheme, host, port
+ // and path (without query string, fragment, etc.).
+ // If the URL is invalid, the invalid spec will be returned, without any
+ // transformation.
+ std::string GetIdFromUrl(const GURL& url) const;
+
+ // Method that ensures the map gets cleaned from time to time. The period at
+ // which garbage collecting happens is adjustable with the
+ // kRequestBetweenCollecting constant.
+ void GarbageCollectEntriesIfNecessary();
+ // Method that does the actual work of garbage collecting.
+ void GarbageCollectEntries();
+
+ // Used by tests.
+ int GetNumberOfEntriesForTests() const { return url_entries_.size(); }
+
+ private:
+ friend struct DefaultSingletonTraits<URLRequestThrottlerManager>;
+
+ // From each URL we generate an ID composed of the scheme, host, port and path
+ // that allows us to uniquely map an entry to it.
+ typedef std::map<std::string, scoped_refptr<URLRequestThrottlerEntry> >
+ UrlEntryMap;
+
+ // Maximum number of entries that we are willing to collect in our map.
+ static const unsigned int kMaximumNumberOfEntries;
+ // Number of requests that will be made between garbage collection.
+ static const unsigned int kRequestsBetweenCollecting;
+
+ // Map that contains a list of URL ID and their matching
+ // URLRequestThrottlerEntry.
+ UrlEntryMap url_entries_;
+
+ // This keeps track of how many requests have been made. Used with
+ // GarbageCollectEntries.
+ unsigned int requests_since_last_gc_;
+
+ mutable scoped_ptr<GURL::Replacements> url_id_replacements_;
+
+ DISALLOW_COPY_AND_ASSIGN(URLRequestThrottlerManager);
+};
+
+} // namespace net
+
+#endif // NET_URL_REQUEST_URL_REQUEST_THROTTLER_MANAGER_H_
diff --git a/net/url_request/url_request_throttler_unittest.cc b/net/url_request/url_request_throttler_unittest.cc
new file mode 100644
index 0000000..0683f91
--- /dev/null
+++ b/net/url_request/url_request_throttler_unittest.cc
@@ -0,0 +1,346 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/pickle.h"
+#include "base/scoped_ptr.h"
+#include "base/string_number_conversions.h"
+#include "base/time.h"
+#include "net/base/test_completion_callback.h"
+#include "net/url_request/url_request_context.h"
+#include "net/url_request/url_request_throttler_header_interface.h"
+#include "net/url_request/url_request_throttler_manager.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using base::TimeDelta;
+using base::TimeTicks;
+
+namespace {
+class MockURLRequestThrottlerManager;
+
+class MockURLRequestThrottlerEntry : public net::URLRequestThrottlerEntry {
+ public :
+ MockURLRequestThrottlerEntry() {}
+ MockURLRequestThrottlerEntry(
+ const TimeTicks& exponential_backoff_release_time,
+ const TimeTicks& sliding_window_release_time,
+ const TimeTicks& fake_now)
+ : fake_time_now_(fake_now) {
+ set_exponential_backoff_release_time(exponential_backoff_release_time);
+ set_sliding_window_release_time(sliding_window_release_time);
+ }
+ virtual ~MockURLRequestThrottlerEntry() {}
+
+ void ResetToBlank(const TimeTicks& time_now) {
+ fake_time_now_ = time_now;
+ set_exponential_backoff_release_time(time_now);
+ set_failure_count(0);
+ set_sliding_window_release_time(time_now);
+ }
+
+ // Overridden for tests.
+ virtual TimeTicks GetTimeNow() const { return fake_time_now_; }
+
+ void set_exponential_backoff_release_time(
+ const base::TimeTicks& release_time) {
+ net::URLRequestThrottlerEntry::set_exponential_backoff_release_time(
+ release_time);
+ }
+
+ base::TimeTicks sliding_window_release_time() const {
+ return net::URLRequestThrottlerEntry::sliding_window_release_time();
+ }
+
+ void set_sliding_window_release_time(
+ const base::TimeTicks& release_time) {
+ net::URLRequestThrottlerEntry::set_sliding_window_release_time(
+ release_time);
+ }
+
+ TimeTicks fake_time_now_;
+};
+
+class MockURLRequestThrottlerHeaderAdapter
+ : public net::URLRequestThrottlerHeaderInterface {
+ public:
+ MockURLRequestThrottlerHeaderAdapter()
+ : fake_retry_value_("0.0"),
+ fake_response_code_(0) {
+ }
+
+ MockURLRequestThrottlerHeaderAdapter(const std::string& retry_value,
+ int response_code)
+ : fake_retry_value_(retry_value),
+ fake_response_code_(response_code) {
+ }
+
+ virtual ~MockURLRequestThrottlerHeaderAdapter() {}
+
+ virtual std::string GetNormalizedValue(const std::string& key) const {
+ if (key == MockURLRequestThrottlerEntry::kRetryHeaderName)
+ return fake_retry_value_;
+ return "";
+ }
+
+ virtual int GetResponseCode() const { return fake_response_code_; }
+
+ std::string fake_retry_value_;
+ int fake_response_code_;
+};
+
+class MockURLRequestThrottlerManager : public net::URLRequestThrottlerManager {
+ public:
+ MockURLRequestThrottlerManager() : create_entry_index_(0) {}
+
+ // Method to process the URL using URLRequestThrottlerManager protected
+ // method.
+ std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
+
+ // Method to use the garbage collecting method of URLRequestThrottlerManager.
+ void DoGarbageCollectEntries() { GarbageCollectEntries(); }
+
+ // Returns the number of entries in the map.
+ int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
+
+ void CreateEntry(bool is_outdated) {
+ TimeTicks time = TimeTicks::Now();
+ if (is_outdated) {
+ time -= TimeDelta::FromMilliseconds(
+ MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
+ }
+ std::string fake_url_string("http://www.fakeurl.com/");
+ fake_url_string.append(base::IntToString(create_entry_index_++));
+ GURL fake_url(fake_url_string);
+ OverrideEntryForTests(
+ fake_url,
+ new MockURLRequestThrottlerEntry(time, TimeTicks::Now(),
+ TimeTicks::Now()));
+ }
+
+ private:
+ int create_entry_index_;
+};
+
+struct TimeAndBool {
+ TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
+ time = time_value;
+ result = expected;
+ line = line_num;
+ }
+ TimeTicks time;
+ bool result;
+ int line;
+};
+
+struct GurlAndString {
+ GurlAndString(const GURL& url_value,
+ const std::string& expected,
+ int line_num) {
+ url = url_value;
+ result = expected;
+ line = line_num;
+ }
+ GURL url;
+ std::string result;
+ int line;
+};
+
+} // namespace
+
+class URLRequestThrottlerEntryTest : public testing::Test {
+ protected:
+ virtual void SetUp();
+ TimeTicks now_;
+ scoped_refptr<MockURLRequestThrottlerEntry> entry_;
+};
+
+void URLRequestThrottlerEntryTest::SetUp() {
+ now_ = TimeTicks::Now();
+ entry_ = new MockURLRequestThrottlerEntry();
+ entry_->ResetToBlank(now_);
+}
+
+std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
+ return out << time.ToInternalValue();
+}
+
+TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
+ entry_->set_exponential_backoff_release_time(
+ entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
+ EXPECT_TRUE(entry_->IsDuringExponentialBackoff());
+}
+
+TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
+ entry_->set_exponential_backoff_release_time(entry_->fake_time_now_);
+ EXPECT_FALSE(entry_->IsDuringExponentialBackoff());
+ entry_->set_exponential_backoff_release_time(
+ entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1));
+ EXPECT_FALSE(entry_->IsDuringExponentialBackoff());
+}
+
+TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateRetryAfter) {
+ // If the response we received has a retry-after field,
+ // the request should be delayed.
+ MockURLRequestThrottlerHeaderAdapter header_w_delay_header("5.5", 200);
+ entry_->UpdateWithResponse(&header_w_delay_header);
+ EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
+ << "When the server put a positive value in retry-after we should "
+ "increase release_time";
+
+ entry_->ResetToBlank(now_);
+ header_w_delay_header.fake_retry_value_ = "-5.5";
+ EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
+ << "When given a negative value, it should not change the release_time";
+}
+
+TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
+ MockURLRequestThrottlerHeaderAdapter failure_response("0", 505);
+ entry_->UpdateWithResponse(&failure_response);
+ EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
+ << "A failure should increase the release_time";
+}
+
+TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
+ MockURLRequestThrottlerHeaderAdapter success_response("0", 200);
+ entry_->UpdateWithResponse(&success_response);
+ EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
+ << "A success should not add any delay";
+}
+
+TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
+ MockURLRequestThrottlerHeaderAdapter failure_response("0", 500);
+ MockURLRequestThrottlerHeaderAdapter success_response("0", 200);
+ entry_->UpdateWithResponse(&success_response);
+ entry_->UpdateWithResponse(&failure_response);
+ EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
+ << "This scenario should add delay";
+}
+
+TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
+ TimeDelta lifetime = TimeDelta::FromMilliseconds(
+ MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
+ const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
+
+ TimeAndBool test_values[] = {
+ TimeAndBool(now_, false, __LINE__),
+ TimeAndBool(now_ - kFiveMs, false, __LINE__),
+ TimeAndBool(now_ + kFiveMs, false, __LINE__),
+ TimeAndBool(now_ - lifetime, false, __LINE__),
+ TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
+
+ for (unsigned int i = 0; i < arraysize(test_values); ++i) {
+ entry_->set_exponential_backoff_release_time(test_values[i].time);
+ EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
+ "Test case #" << i << " line " << test_values[i].line << " failed";
+ }
+}
+
+TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
+ for (int i = 0; i < 30; ++i) {
+ MockURLRequestThrottlerHeaderAdapter response_adapter("0.0", 505);
+ entry_->UpdateWithResponse(&response_adapter);
+ }
+
+ TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
+ EXPECT_EQ(delay.InMilliseconds(),
+ MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
+}
+
+TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
+ MockURLRequestThrottlerHeaderAdapter response_adapter("0.0", 505);
+ for (int i = 0; i < 5; ++i)
+ entry_->UpdateWithResponse(&response_adapter);
+
+ TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
+
+ // Inform the entry that a response body was malformed, which is supposed to
+ // increase the back-off time.
+ entry_->ReceivedContentWasMalformed();
+ EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
+}
+
+TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
+ int max_send = net::URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
+ int sliding_window =
+ net::URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
+
+ TimeTicks time_1 = entry_->fake_time_now_ +
+ TimeDelta::FromMilliseconds(sliding_window / 3);
+ TimeTicks time_2 = entry_->fake_time_now_ +
+ TimeDelta::FromMilliseconds(2 * sliding_window / 3);
+ TimeTicks time_3 = entry_->fake_time_now_ +
+ TimeDelta::FromMilliseconds(sliding_window);
+ TimeTicks time_4 = entry_->fake_time_now_ +
+ TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3);
+
+ entry_->set_exponential_backoff_release_time(time_1);
+
+ for (int i = 0; i < max_send / 2; ++i) {
+ EXPECT_EQ(2 * sliding_window / 3,
+ entry_->ReserveSendingTimeForNextRequest(time_2));
+ }
+ EXPECT_EQ(time_2, entry_->sliding_window_release_time());
+
+ entry_->fake_time_now_ = time_3;
+
+ for (int i = 0; i < (max_send + 1) / 2; ++i)
+ EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
+
+ EXPECT_EQ(time_4, entry_->sliding_window_release_time());
+}
+
+TEST(URLRequestThrottlerManager, IsUrlStandardised) {
+ MockURLRequestThrottlerManager manager;
+ GurlAndString test_values[] = {
+ GurlAndString(GURL("http://www.example.com"),
+ std::string("http://www.example.com/"), __LINE__),
+ GurlAndString(GURL("http://www.Example.com"),
+ std::string("http://www.example.com/"), __LINE__),
+ GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
+ std::string("http://www.ex4mple.com/pr4c71c41"), __LINE__),
+ GurlAndString(GURL("http://www.example.com/0/token/false"),
+ std::string("http://www.example.com/0/token/false"),
+ __LINE__),
+ GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
+ std::string("http://www.example.com/index.php"), __LINE__),
+ GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
+ std::string("http://www.example.com/index.php"),
+ __LINE__),
+ GurlAndString(GURL("http://www.example.com:1234/"),
+ std::string("http://www.example.com:1234/"), __LINE__)};
+
+ for (unsigned int i = 0; i < arraysize(test_values); ++i) {
+ std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
+ EXPECT_EQ(temp, test_values[i].result) <<
+ "Test case #" << i << " line " << test_values[i].line << " failed";
+ }
+}
+
+TEST(URLRequestThrottlerManager, AreEntriesBeingCollected) {
+ MockURLRequestThrottlerManager manager;
+
+ manager.CreateEntry(true); // true = Entry is outdated.
+ manager.CreateEntry(true);
+ manager.CreateEntry(true);
+ manager.DoGarbageCollectEntries();
+ EXPECT_EQ(0, manager.GetNumberOfEntries());
+
+ manager.CreateEntry(false);
+ manager.CreateEntry(false);
+ manager.CreateEntry(false);
+ manager.CreateEntry(true);
+ manager.DoGarbageCollectEntries();
+ EXPECT_EQ(3, manager.GetNumberOfEntries());
+}
+
+TEST(URLRequestThrottlerManager, IsHostBeingRegistered) {
+ MockURLRequestThrottlerManager manager;
+
+ manager.RegisterRequestUrl(GURL("http://www.example.com/"));
+ manager.RegisterRequestUrl(GURL("http://www.google.com/"));
+ manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
+ manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
+ manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
+
+ EXPECT_EQ(3, manager.GetNumberOfEntries());
+}