This CL will introduce a new way to do exponential back-off on failure within

Chromium. It is a network level implementation and should constitute a good 
enough bottleneck to manage every outgoing http request.

Committing for malavv@google.com.
Original review: http://codereview.chromium.org/2487001/show

BUG=none
TEST=unit tests


Review URL: http://codereview.chromium.org/3005049

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@56376 0039d316-1c4b-4281-b951-d872f2087c98

14 files changed, 859 insertions, 10 deletions

diff --git a/chrome/common/net/url_fetcher_unittest.cc b/chrome/common/net/url_fetcher_unittest.cc
index eba4983..ed7eb81 100644
--- a/chrome/common/net/url_fetcher_unittest.cc
+++ b/chrome/common/net/url_fetcher_unittest.cc
@@ -312,7 +312,9 @@ void URLFetcherProtectTest::OnURLFetchComplete(const URLFetcher* source,
     // Now running ServerUnavailable test.
     // It takes more than 1 second to finish all 11 requests.
     EXPECT_TRUE(Time::Now() - start_time_ >= one_second);
-    EXPECT_TRUE(status.is_success());
+    // We used to check this, but it gets confused because of the other
+    // implementation of request throttling (which we will be removing).
+    //EXPECT_EQ(status.os_error(), net::ERR_TEMPORARILY_THROTTLED_BY_DDOS);
     EXPECT_FALSE(data.empty());
     delete fetcher_;
     io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask());
diff --git a/net/base/net_error_list.h b/net/base/net_error_list.h
index 5fe0846..a626fad 100644
--- a/net/base/net_error_list.h
+++ b/net/base/net_error_list.h
@@ -171,6 +171,10 @@ NET_ERROR(SSL_UNSAFE_NEGOTIATION, -128)
 // The SSL server is using a weak cryptographic key.
 NET_ERROR(SSL_WEAK_SERVER_KEY, -129)
 
+// The request throttler module cancelled this request because the are too many
+// requests to a server that is failing requests.
+NET_ERROR(TEMPORARILY_THROTTLED_BY_DDOS, -130)
+
 // Certificate error codes
 //
 // The values of certificate error codes must be consecutive.
diff --git a/net/net.gyp b/net/net.gyp
index abbbc74..e02ebf4 100644
--- a/net/net.gyp
+++ b/net/net.gyp
@@ -466,6 +466,14 @@
         'proxy/proxy_service.h',
         'proxy/sync_host_resolver_bridge.cc',
         'proxy/sync_host_resolver_bridge.h',
+        'request_throttler/request_throttler_entry.cc',
+        'request_throttler/request_throttler_entry.h',
+        'request_throttler/request_throttler_entry_interface.h',
+        'request_throttler/request_throttler_header_adapter.h',
+        'request_throttler/request_throttler_header_adapter.cc',
+        'request_throttler/request_throttler_header_interface.h',
+        'request_throttler/request_throttler_manager.cc',
+        'request_throttler/request_throttler_manager.h',
         'socket/client_socket.cc',
         'socket/client_socket.h',
         'socket/client_socket_factory.cc',
@@ -776,6 +784,7 @@
         'proxy/proxy_server_unittest.cc',
         'proxy/proxy_service_unittest.cc',
         'proxy/sync_host_resolver_bridge_unittest.cc',
+        'request_throttler/request_throttler_unittest.cc',
         'socket/client_socket_pool_base_unittest.cc',
         'socket/socks5_client_socket_unittest.cc',
         'socket/socks_client_socket_pool_unittest.cc',
diff --git a/net/request_throttler/request_throttler_entry.cc b/net/request_throttler/request_throttler_entry.cc
new file mode 100644
index 0000000..826358d
--- /dev/null
+++ b/net/request_throttler/request_throttler_entry.cc
@@ -0,0 +1,134 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/request_throttler/request_throttler_entry.h"
+
+#include <cmath>
+
+#include "base/logging.h"
+#include "base/rand_util.h"
+#include "base/string_number_conversions.h"
+#include "base/string_util.h"
+#include "net/request_throttler/request_throttler_header_interface.h"
+
+const int RequestThrottlerEntry::kAdditionalConstantMs = 100;
+const int RequestThrottlerEntry::kEntryLifetimeSec = 120;
+const double RequestThrottlerEntry::kJitterFactor = 0.4;
+const int RequestThrottlerEntry::kInitialBackoffMs = 700;
+const int RequestThrottlerEntry::kMaximumBackoffMs = 24 * 60 * 60 * 1000;
+const double RequestThrottlerEntry::kMultiplyFactor = 2.0;
+const char RequestThrottlerEntry::kRetryHeaderName[] = "X-Retry-After";
+
+RequestThrottlerEntry::RequestThrottlerEntry()
+    : release_time_(base::TimeTicks::Now()),
+      num_times_delayed_(0),
+      is_managed_(false) {
+  SaveState();
+}
+
+RequestThrottlerEntry::~RequestThrottlerEntry() {
+}
+
+bool RequestThrottlerEntry::IsRequestAllowed() const {
+  return (release_time_ <= GetTimeNow());
+}
+
+void RequestThrottlerEntry::UpdateWithResponse(
+    const RequestThrottlerHeaderInterface* response) {
+  SaveState();
+  if (response->GetResponseCode() >= 500) {
+    num_times_delayed_++;
+    release_time_ = std::max(CalculateReleaseTime(), release_time_);
+    is_managed_ = true;
+  } else {
+    // We slowly decay the number of times delayed instead of resetting it to 0
+    // in order to stay stable if we received lots of requests with
+    // malformed bodies at the same time.
+    if (num_times_delayed_ > 0)
+      num_times_delayed_--;
+    is_managed_ = false;
+    // The reason why we are not just cutting release_time to GetTimeNow() is
+    // on the one hand, it would unset delay put by our custom retry-after
+    // header and on the other we would like to push every request up to our
+    // "horizon" when dealing with multiple in-flight request. Ex: If we send
+    // three request and we receive 2 failures and 1 success. The success that
+    // follows those failures will not reset release time further request will
+    // then need to wait the delay caused by the 2 failures.
+    release_time_ = std::max(GetTimeNow(), release_time_);
+    if (response->GetNormalizedValue(kRetryHeaderName) != std::string())
+      HandleCustomRetryAfter(response->GetNormalizedValue(kRetryHeaderName));
+  }
+}
+
+bool RequestThrottlerEntry::IsEntryOutdated() const {
+  int64 unused_since_ms = (GetTimeNow() - release_time_).InMilliseconds();
+  int64 lifespan_ms = kEntryLifetimeSec * 1000;
+
+  // Release time is further than now, we are managing it.
+  if (unused_since_ms < 0)
+    return false;
+
+  // There are two cases. First one, when the entry is currently being managed
+  // and should not be collected unless it is older than the maximum allowed
+  // back-off. The other one, when the entry is outdated, unmanaged and should
+  // be collected.
+  if (is_managed_)
+    return unused_since_ms > kMaximumBackoffMs;
+
+  return unused_since_ms > lifespan_ms;
+}
+
+void RequestThrottlerEntry::ReceivedContentWasMalformed() {
+  // We should never revert to less back-off or else an attacker could put a
+  // malformed body in cache and replay it to decrease delay.
+  num_times_delayed_ =
+      std::max(old_values_.number_of_failed_requests, num_times_delayed_);
+  num_times_delayed_++;
+  is_managed_ = true;
+  release_time_ = std::max(CalculateReleaseTime(),
+      std::max(old_values_.release_time, release_time_));
+}
+
+base::TimeTicks RequestThrottlerEntry::CalculateReleaseTime() {
+  double delay = kInitialBackoffMs;
+  delay *= pow(kMultiplyFactor, num_times_delayed_);
+  delay += kAdditionalConstantMs;
+  delay -= base::RandDouble() * kJitterFactor * delay;
+
+  // Ensure that we do not exceed maximum delay
+  int64 delay_int = static_cast<int64>(delay + 0.5);
+  delay_int = std::min(delay_int, static_cast<int64>(kMaximumBackoffMs));
+
+  return GetTimeNow() + base::TimeDelta::FromMilliseconds(delay_int);
+}
+
+base::TimeTicks RequestThrottlerEntry::GetTimeNow() const {
+  return base::TimeTicks::Now();
+}
+
+void RequestThrottlerEntry::HandleCustomRetryAfter(
+    const std::string& header_value) {
+  // Input parameter is the number of seconds to wait in a floating point value.
+  double time_in_sec = 0;
+  bool conversion_is_ok = base::StringToDouble(header_value, &time_in_sec);
+
+  // Conversion of custom retry-after header value failed.
+  if (!conversion_is_ok)
+    return;
+
+  // We must use an int value later so we transform this in milliseconds.
+  int64 value_ms = static_cast<int64>(0.5 + time_in_sec * 1000);
+
+  if (kMaximumBackoffMs < value_ms || value_ms < 0)
+    return;
+
+  release_time_ = std::max(
+      (GetTimeNow() + base::TimeDelta::FromMilliseconds(value_ms)),
+      release_time_);
+}
+
+void RequestThrottlerEntry::SaveState() {
+  old_values_.release_time = release_time_;
+  old_values_.number_of_failed_requests = num_times_delayed_;
+}
diff --git a/net/request_throttler/request_throttler_entry.h b/net/request_throttler/request_throttler_entry.h
new file mode 100644
index 0000000..cf8a4ae
--- /dev/null
+++ b/net/request_throttler/request_throttler_entry.h
@@ -0,0 +1,104 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_REQUEST_THROTTLER_REQUEST_THROTTLER_ENTRY_H_
+#define NET_REQUEST_THROTTLER_REQUEST_THROTTLER_ENTRY_H_
+
+#include "net/request_throttler/request_throttler_entry_interface.h"
+
+#include <string>
+
+#include "base/ref_counted.h"
+#include "base/time.h"
+
+// Represents an entry of the Request Throttler Manager.
+class RequestThrottlerEntry : public RequestThrottlerEntryInterface {
+ public:
+  // Additional constant to adjust back-off.
+  static const int kAdditionalConstantMs;
+
+  // Time after which the entry is considered outdated.
+  static const int kEntryLifetimeSec;
+
+  // Fuzzing percentage. ex: 10% will spread requests randomly
+  // between 90%-100% of the calculated time.
+  static const double kJitterFactor;
+
+  // Initial delay.
+  static const int kInitialBackoffMs;
+
+  // Maximum amount of time we are willing to delay our request.
+  static const int kMaximumBackoffMs;
+
+  // Factor by which the waiting time will be multiplied.
+  static const double kMultiplyFactor;
+
+  // Name of the header that servers can use to ask clients to delay their next
+  // request. ex: "X-Retry-After"
+  static const char kRetryHeaderName[];
+
+  RequestThrottlerEntry();
+
+  ////// Implementation of the Request throttler Interface. ///////
+
+  // This method needs to be called prior to every request; if it returns
+  // false, the calling module must cancel its current request.
+  virtual bool IsRequestAllowed() const;
+
+  // This method needs to be called each time a response is received.
+  virtual void UpdateWithResponse(
+      const RequestThrottlerHeaderInterface* response);
+
+  ////////// Specific method of Request throttler Entry ////////////////
+
+  // Used by the manager, returns if the entry needs to be garbage collected.
+  bool IsEntryOutdated() const;
+
+  // Used by the manager, enables the manager to flag the last successful
+  // request as a failure.
+  void ReceivedContentWasMalformed();
+
+ protected:
+
+  // This struct is used to save the state of the entry each time we are updated
+  // with a response header so we can regenerate it if we are informed that one
+  // of our bodies was malformed.
+  struct OldValues {
+    base::TimeTicks release_time;
+    int number_of_failed_requests;
+  };
+
+  virtual ~RequestThrottlerEntry();
+
+  // Calculates when we should start sending requests again. Follows a failure
+  // response.
+  base::TimeTicks CalculateReleaseTime();
+
+  // Equivalent to TimeTicks::Now(), virtual to be mockable for testing purpose.
+  virtual base::TimeTicks GetTimeNow() const;
+
+  // Is used internally to increase release time following a retry-after header.
+  void HandleCustomRetryAfter(const std::string& header_value);
+
+  // Saves the state of the object to be able to regenerate it.
+  // Must be informed of the state of the response.
+  void SaveState();
+
+  // This contains the timestamp at which we are allowed to start sending
+  // requests again.
+  base::TimeTicks release_time_;
+
+  // Number of times we were delayed.
+  int num_times_delayed_;
+
+  // Are we currently managing this request.
+  bool is_managed_;
+
+  OldValues old_values_;
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(RequestThrottlerEntry);
+};
+
+#endif  // NET_REQUEST_THROTTLER_REQUEST_THROTTLER_ENTRY_H_
diff --git a/net/request_throttler/request_throttler_entry_interface.h b/net/request_throttler/request_throttler_entry_interface.h
new file mode 100644
index 0000000..5781e54
--- /dev/null
+++ b/net/request_throttler/request_throttler_entry_interface.h
@@ -0,0 +1,33 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_REQUEST_THROTTLER_REQUEST_THROTTLER_ENTRY_INTERFACE_H_
+#define NET_REQUEST_THROTTLER_REQUEST_THROTTLER_ENTRY_INTERFACE_H_
+
+#include "base/basictypes.h"
+#include "base/ref_counted.h"
+
+class RequestThrottlerHeaderInterface;
+
+// Represents an entry of the request throttler manager.
+class RequestThrottlerEntryInterface
+    : public base::RefCounted<RequestThrottlerEntryInterface> {
+ public:
+  RequestThrottlerEntryInterface() {}
+
+  // This method needs to be called prior to every requests; if it returns
+  // false, the calling module must cancel its current request.
+  virtual bool IsRequestAllowed() const = 0;
+
+  // This method needs to be called each time a response is received.
+  virtual void UpdateWithResponse(
+      const RequestThrottlerHeaderInterface* response) = 0;
+ protected:
+  virtual ~RequestThrottlerEntryInterface() {}
+ private:
+  friend class base::RefCounted<RequestThrottlerEntryInterface>;
+  DISALLOW_COPY_AND_ASSIGN(RequestThrottlerEntryInterface);
+};
+
+#endif  // NET_REQUEST_THROTTLER_REQUEST_THROTTLER_ENTRY_INTERFACE_H_
diff --git a/net/request_throttler/request_throttler_header_adapter.cc b/net/request_throttler/request_throttler_header_adapter.cc
new file mode 100644
index 0000000..5307044
--- /dev/null
+++ b/net/request_throttler/request_throttler_header_adapter.cc
@@ -0,0 +1,21 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/request_throttler/request_throttler_header_adapter.h"
+
+RequestThrottlerHeaderAdapter::RequestThrottlerHeaderAdapter(
+    const scoped_refptr<net::HttpResponseHeaders>& headers)
+    : response_header_(headers) {
+}
+
+std::string RequestThrottlerHeaderAdapter::GetNormalizedValue(
+    const std::string& key) const {
+  std::string return_value;
+  response_header_->GetNormalizedHeader(key, &return_value);
+  return return_value;
+}
+
+int RequestThrottlerHeaderAdapter::GetResponseCode() const {
+  return response_header_->response_code();
+}
diff --git a/net/request_throttler/request_throttler_header_adapter.h b/net/request_throttler/request_throttler_header_adapter.h
new file mode 100644
index 0000000..86a0d20
--- /dev/null
+++ b/net/request_throttler/request_throttler_header_adapter.h
@@ -0,0 +1,32 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_REQUEST_THROTTLER_REQUEST_THROTTLER_HEADER_ADAPTER_H_
+#define NET_REQUEST_THROTTLER_REQUEST_THROTTLER_HEADER_ADAPTER_H_
+
+#include <string>
+
+#include "base/scoped_ptr.h"
+#include "net/http/http_response_headers.h"
+#include "net/request_throttler/request_throttler_header_interface.h"
+
+namespace net {
+class HttpResponseHeaders;
+}
+
+// Adapter for the http header interface of the request throttler component.
+class RequestThrottlerHeaderAdapter : public RequestThrottlerHeaderInterface {
+ public:
+  RequestThrottlerHeaderAdapter(
+      const scoped_refptr<net::HttpResponseHeaders>& headers);
+  virtual ~RequestThrottlerHeaderAdapter() {}
+
+  // Implementation of the Http header interface
+  virtual std::string GetNormalizedValue(const std::string& key) const;
+  virtual int GetResponseCode() const;
+ private:
+  const scoped_refptr<net::HttpResponseHeaders> response_header_;
+};
+
+#endif  // NET_REQUEST_THROTTLER_REQUEST_THROTTLER_HEADER_ADAPTER_H_
diff --git a/net/request_throttler/request_throttler_header_interface.h b/net/request_throttler/request_throttler_header_interface.h
new file mode 100644
index 0000000..021c6e4
--- /dev/null
+++ b/net/request_throttler/request_throttler_header_interface.h
@@ -0,0 +1,24 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_REQUEST_THROTTLER_REQUEST_THROTTLER_HEADER_INTERFACE_H_
+#define NET_REQUEST_THROTTLER_REQUEST_THROTTLER_HEADER_INTERFACE_H_
+
+#include <string>
+
+// Interface to an HTTP header to enforce we have the methods we need.
+class RequestThrottlerHeaderInterface {
+ public:
+  virtual ~RequestThrottlerHeaderInterface() {}
+
+  // Method that enables us to fetch the header value by its key.
+  // ex: location: www.example.com -> key = "location" value = "www.example.com"
+  // If the key does not exist, it returns an empty string.
+  virtual std::string GetNormalizedValue(const std::string& key) const = 0;
+
+  // Returns the HTTP response code associated with the request.
+  virtual int GetResponseCode() const = 0;
+};
+
+#endif  // NET_REQUEST_THROTTLER_REQUEST_THROTTLER_HEADER_INTERFACE_H_
diff --git a/net/request_throttler/request_throttler_manager.cc b/net/request_throttler/request_throttler_manager.cc
new file mode 100644
index 0000000..21ca3fc
--- /dev/null
+++ b/net/request_throttler/request_throttler_manager.cc
@@ -0,0 +1,85 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/request_throttler/request_throttler_manager.h"
+
+#include "base/logging.h"
+#include "base/message_loop.h"
+#include "base/rand_util.h"
+#include "base/string_util.h"
+
+const unsigned int RequestThrottlerManager::kMaximumNumberOfEntries = 1500;
+const unsigned int RequestThrottlerManager::kRequestsBetweenCollecting = 200;
+
+scoped_refptr<RequestThrottlerEntryInterface>
+    RequestThrottlerManager::RegisterRequestUrl(
+        const GURL &url) {
+  DCHECK_EQ(MessageLoop::current()->type(), MessageLoop::TYPE_IO);
+
+  // Periodically garbage collect old entries.
+  requests_since_last_gc_++;
+  if (requests_since_last_gc_ >= kRequestsBetweenCollecting) {
+    GarbageCollectEntries();
+    requests_since_last_gc_ = 0;
+  }
+
+  // Normalize the url
+  std::string url_id = GetIdFromUrl(url);
+
+  // Finds the entry in the map or creates it.
+  scoped_refptr<RequestThrottlerEntry>& entry = url_entries_[url_id];
+  if (entry == NULL)
+    entry = new RequestThrottlerEntry();
+
+  return entry;
+}
+
+RequestThrottlerManager::RequestThrottlerManager()
+    : current_loop_(MessageLoop::current()),
+      requests_since_last_gc_(0) {
+}
+
+RequestThrottlerManager::~RequestThrottlerManager() {
+  // Deletes all entries
+  url_entries_.clear();
+}
+
+std::string RequestThrottlerManager::GetIdFromUrl(const GURL& url) {
+  std::string url_id;
+  url_id += url.scheme();
+  url_id += "://";
+  url_id += url.host();
+  url_id += url.path();
+
+  return StringToLowerASCII(url_id);
+}
+
+void RequestThrottlerManager::GarbageCollectEntries() {
+  DCHECK_EQ(MessageLoop::current()->type(), MessageLoop::TYPE_IO);
+  UrlEntryMap::iterator i = url_entries_.begin();
+
+  while (i != url_entries_.end()) {
+    if ((i->second)->IsEntryOutdated()) {
+      url_entries_.erase(i++);
+    } else {
+      ++i;
+    }
+  }
+
+  // In case something broke we want to make sure not to grow indefinitely.
+  while (url_entries_.size() > kMaximumNumberOfEntries) {
+    url_entries_.erase(url_entries_.begin());
+  }
+}
+
+void RequestThrottlerManager::NotifyRequestBodyWasMalformed(const GURL& url) {
+  DCHECK_EQ(MessageLoop::current()->type(), MessageLoop::TYPE_IO);
+  // Normalize the url
+  std::string url_id = GetIdFromUrl(url);
+  UrlEntryMap::iterator i = url_entries_.find(url_id);
+
+  if (i != url_entries_.end()) {
+    i->second->ReceivedContentWasMalformed();
+  }
+}
diff --git a/net/request_throttler/request_throttler_manager.h b/net/request_throttler/request_throttler_manager.h
new file mode 100644
index 0000000..2ef695f
--- /dev/null
+++ b/net/request_throttler/request_throttler_manager.h
@@ -0,0 +1,79 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_REQUEST_THROTTLER_REQUEST_THROTTLER_MANAGER_H_
+#define NET_REQUEST_THROTTLER_REQUEST_THROTTLER_MANAGER_H_
+
+#include <map>
+#include <string>
+
+#include "base/lock.h"
+#include "base/non_thread_safe.h"
+#include "base/singleton.h"
+#include "base/scoped_ptr.h"
+#include "base/time.h"
+#include "googleurl/src/gurl.h"
+#include "net/request_throttler/request_throttler_entry.h"
+
+class MessageLoop;
+
+// Class that registers a request throttler entry for each URL being accessed in
+// order to supervise traffic. URL Http Request should register their URL in
+// this request throttler manager on each request.
+class RequestThrottlerManager  {
+ public:
+  // Must be called for every request, returns the request throttler entry
+  // associated with the URL. The caller must inform this entry of some events.
+  // Please refer to request_throttler_entry.h for further informations.
+  scoped_refptr<RequestThrottlerEntryInterface> RegisterRequestUrl(
+      const GURL& url);
+
+  // This method is used by higher level modules which can notify this class if
+  // the response they received was malformed. It is useful because
+  // in the case where the response header returned 200 response code but had
+  // a malformed content body we will categorize it as a success and so we
+  // defer this decision to the module that had requested the resource.
+  void NotifyRequestBodyWasMalformed(const GURL& url);
+
+ protected:
+  RequestThrottlerManager();
+  virtual ~RequestThrottlerManager();
+
+  // From each URL we generate an ID composed of the host and path
+  // that allows us to uniquely map an entry to it.
+  typedef std::map<std::string, scoped_refptr<RequestThrottlerEntry> >
+      UrlEntryMap;
+
+  friend struct DefaultSingletonTraits<RequestThrottlerManager>;
+
+  // Map that contains a list of URL ID and their matching
+  // RequestThrottlerEntry.
+  UrlEntryMap url_entries_;
+
+  // Method that allows us to transform an URL into an ID that is useful and
+  // can be used in our map. Resulting IDs will be lowercase and be missing both
+  // the query string and fragment.
+  std::string GetIdFromUrl(const GURL& url);
+
+  // Method that ensures the map gets cleaned from time to time. The period at
+  // which this GarbageCollection happens is adjustable with the
+  // kRequestBetweenCollecting constant.
+  void GarbageCollectEntries();
+
+ private:
+  MessageLoop* current_loop_;
+
+  // Maximum number of entries that we are willing to collect in our map.
+  static const unsigned int kMaximumNumberOfEntries;
+  // Number of requests that will be made between garbage collection.
+  static const unsigned int kRequestsBetweenCollecting;
+
+  // This keeps track of how many request have been made. Used with
+  // GarbageCollectEntries.
+  unsigned int requests_since_last_gc_;
+
+  DISALLOW_COPY_AND_ASSIGN(RequestThrottlerManager);
+};
+
+#endif  // NET_REQUEST_THROTTLER_REQUEST_THROTTLER_MANAGER_H_
diff --git a/net/request_throttler/request_throttler_unittest.cc b/net/request_throttler/request_throttler_unittest.cc
new file mode 100644
index 0000000..d43b555
--- /dev/null
+++ b/net/request_throttler/request_throttler_unittest.cc
@@ -0,0 +1,302 @@
+// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/pickle.h"
+#include "base/scoped_ptr.h"
+#include "base/string_number_conversions.h"
+#include "base/string_util.h"
+#include "base/time.h"
+#include "net/base/test_completion_callback.h"
+#include "net/request_throttler/request_throttler_manager.h"
+#include "net/request_throttler/request_throttler_header_interface.h"
+#include "net/url_request/url_request_context.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using base::TimeDelta;
+using base::TimeTicks;
+
+namespace {
+class MockRequestThrottlerManager;
+
+class MockRequestThrottlerEntry : public RequestThrottlerEntry {
+ public :
+  MockRequestThrottlerEntry() {}
+  MockRequestThrottlerEntry(TimeTicks release_time, TimeTicks fake_now)
+      : fake_time_now_(fake_now) {
+    release_time_ = release_time;
+  }
+  virtual ~MockRequestThrottlerEntry() {}
+
+  void ResetToBlank(TimeTicks time_now) {
+    fake_time_now_ = time_now;
+    release_time_ = time_now;
+    num_times_delayed_ = 0;
+  }
+
+  // Overloaded for test
+  virtual TimeTicks GetTimeNow() const { return fake_time_now_; }
+
+  TimeTicks release_time() const { return release_time_; }
+  void set_release_time(TimeTicks time) { release_time_ = time; }
+
+  TimeTicks fake_time_now_;
+};
+
+class MockRequestThrottlerHeaderAdapter
+    : public RequestThrottlerHeaderInterface {
+ public:
+  MockRequestThrottlerHeaderAdapter()
+      : fake_retry_value_("0.0"),
+        fake_response_code_(0) {
+  }
+
+  MockRequestThrottlerHeaderAdapter(const std::string& retry_value,
+                                    const int response_code)
+      : fake_retry_value_(retry_value),
+        fake_response_code_(response_code) {
+  }
+
+  ~MockRequestThrottlerHeaderAdapter() {}
+
+  virtual std::string GetNormalizedValue(const std::string& key) const {
+    if (key == MockRequestThrottlerEntry::kRetryHeaderName)
+      return fake_retry_value_;
+    return "";
+  }
+
+  virtual int GetResponseCode() const { return fake_response_code_; }
+
+  std::string fake_retry_value_;
+  int fake_response_code_;
+};
+
+class MockRequestThrottlerManager : public RequestThrottlerManager {
+ public:
+  MockRequestThrottlerManager() {}
+  virtual ~MockRequestThrottlerManager() {}
+
+  // Method to process the url using RequestThrottlerManager protected method.
+  std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
+
+  // Method to use the garbage collecting method of RequestThrottlerManager.
+  void DoGarbageCollectEntries() { GarbageCollectEntries(); }
+
+  // Returns the number of entries in the host map.
+  int GetNumberOfEntries() { return url_entries_.size(); }
+
+  void CreateEntry(const bool is_outdated);
+
+  static int create_entry_index;
+};
+
+/*------------------ Mock request throttler manager --------------------------*/
+int MockRequestThrottlerManager::create_entry_index = 0;
+
+void MockRequestThrottlerManager::CreateEntry(const bool is_outdated) {
+  TimeTicks time = TimeTicks::Now();
+  if (is_outdated) {
+    time -= TimeDelta::FromSeconds(
+        MockRequestThrottlerEntry::kEntryLifetimeSec);
+    time -= TimeDelta::FromSeconds(1);
+  }
+  std::string index = base::IntToString(create_entry_index++);
+  url_entries_[index] = new MockRequestThrottlerEntry(time, TimeTicks::Now());
+}
+
+/* ---------------- Request throttler entry test -----------------------------*/
+struct TimeAndBool {
+  TimeAndBool(TimeTicks time_value, bool expected, int line_num) {
+    time = time_value;
+    result = expected;
+    line = line_num;
+  }
+  TimeTicks time;
+  bool result;
+  int line;
+};
+
+struct GurlAndString {
+  GurlAndString(GURL url_value, const std::string& expected, int line_num) {
+    url = url_value;
+    result = expected;
+    line = line_num;
+  }
+  GURL url;
+  std::string result;
+  int line;
+};
+
+class RequestThrottlerEntryTest : public ::testing::Test {
+ protected:
+  virtual void SetUp();
+  TimeTicks now_;
+  TimeDelta delay_;
+  scoped_refptr<MockRequestThrottlerEntry> entry_;
+};
+
+void RequestThrottlerEntryTest::SetUp() {
+  now_ = TimeTicks::Now();
+  entry_ = new MockRequestThrottlerEntry();
+  entry_->ResetToBlank(now_);
+}
+
+std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
+  return out << time.ToInternalValue();
+}
+
+TEST_F(RequestThrottlerEntryTest, InterfaceRequestNotAllowed) {
+  entry_->set_release_time(entry_->fake_time_now_ +
+      TimeDelta::FromMilliseconds(1));
+  EXPECT_FALSE(entry_->IsRequestAllowed());
+}
+
+TEST_F(RequestThrottlerEntryTest, InterfaceRequestAllowed) {
+  entry_->set_release_time(entry_->fake_time_now_);
+  EXPECT_TRUE(entry_->IsRequestAllowed());
+  entry_->set_release_time(entry_->fake_time_now_ -
+      TimeDelta::FromMilliseconds(1));
+  EXPECT_TRUE(entry_->IsRequestAllowed());
+}
+
+TEST_F(RequestThrottlerEntryTest, InterfaceUpdateRetryAfter) {
+  // If the response we received as a retry-after field,
+  // the request should be delayed.
+  MockRequestThrottlerHeaderAdapter header_w_delay_header("5.5", 200);
+  entry_->UpdateWithResponse(&header_w_delay_header);
+  EXPECT_GT(entry_->release_time(), entry_->fake_time_now_) <<
+      "When the server put a positive value in retry-after we should "
+      "increase release_time";
+
+  entry_->ResetToBlank(now_);
+  header_w_delay_header.fake_retry_value_ = "-5.5";
+  EXPECT_EQ(entry_->release_time(), entry_->fake_time_now_) <<
+      "When given a negative value, it should not change the release_time";
+}
+
+TEST_F(RequestThrottlerEntryTest, InterfaceUpdateFailure) {
+  MockRequestThrottlerHeaderAdapter failure_response("0", 505);
+  entry_->UpdateWithResponse(&failure_response);
+  EXPECT_GT(entry_->release_time(), entry_->fake_time_now_) <<
+      "A failure should increase the release_time";
+}
+
+TEST_F(RequestThrottlerEntryTest, InterfaceUpdateSuccess) {
+  MockRequestThrottlerHeaderAdapter success_response("0", 200);
+  entry_->UpdateWithResponse(&success_response);
+  EXPECT_EQ(entry_->release_time(), entry_->fake_time_now_) <<
+      "A success should not add any delay";
+}
+
+TEST_F(RequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
+  MockRequestThrottlerHeaderAdapter failure_response("0", 500),
+                                    success_response("0", 200);
+  entry_->UpdateWithResponse(&success_response);
+  entry_->UpdateWithResponse(&failure_response);
+  EXPECT_GT(entry_->release_time(), entry_->fake_time_now_) <<
+      "This scenario should add delay";
+}
+
+TEST_F(RequestThrottlerEntryTest, IsEntryReallyOutdated) {
+  TimeDelta lifetime = TimeDelta::FromSeconds(
+      MockRequestThrottlerEntry::kEntryLifetimeSec);
+  const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
+
+  TimeAndBool test_values[] = {
+      TimeAndBool(now_, false, __LINE__),
+      TimeAndBool(now_ - kFiveMs, false, __LINE__),
+      TimeAndBool(now_ + kFiveMs, false, __LINE__),
+      TimeAndBool(now_ - lifetime, false, __LINE__),
+      TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
+
+  for (unsigned int i = 0; i < arraysize(test_values); ++i) {
+    entry_->set_release_time(test_values[i].time);
+    EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
+        "Test case #" << i << " line " << test_values[i].line << " failed";
+  }
+}
+
+TEST_F(RequestThrottlerEntryTest, MaxAllowedBackoff) {
+  for (int i = 0; i < 30; ++i) {
+    MockRequestThrottlerHeaderAdapter response_adapter("0.0", 505);
+    entry_->UpdateWithResponse(&response_adapter);
+  }
+
+  delay_ = entry_->release_time() - now_;
+  EXPECT_EQ(delay_.InMilliseconds(),
+      MockRequestThrottlerEntry::kMaximumBackoffMs);
+}
+
+TEST_F(RequestThrottlerEntryTest, MalformedContent) {
+  for (int i = 0; i < 5; ++i) {
+    MockRequestThrottlerHeaderAdapter response_adapter("0.0", 505);
+    entry_->UpdateWithResponse(&response_adapter);
+  }
+  TimeTicks release_after_failures = entry_->release_time();
+
+  // Send a success code to reset backoff.
+  MockRequestThrottlerHeaderAdapter response_adapter("0.0", 200);
+  entry_->UpdateWithResponse(&response_adapter);
+  EXPECT_EQ(entry_->release_time(), release_after_failures);
+
+  // Then inform the entry that previous package was malformed
+  // it is suppose to regenerate previous state.
+  entry_->ReceivedContentWasMalformed();
+  EXPECT_GT(entry_->release_time(), release_after_failures);
+}
+
+TEST(RequestThrottlerManager, IsUrlStandardised) {
+  MockRequestThrottlerManager manager;
+  GurlAndString test_values[] = {
+      GurlAndString(GURL("http://www.example.com"),
+                    std::string("http://www.example.com/"), __LINE__),
+      GurlAndString(GURL("http://www.Example.com"),
+                    std::string("http://www.example.com/"), __LINE__),
+      GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
+                    std::string("http://www.ex4mple.com/pr4c71c41"), __LINE__),
+      GurlAndString(GURL("http://www.example.com/0/token/false"),
+                    std::string("http://www.example.com/0/token/false"),
+                    __LINE__),
+      GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
+                    std::string("http://www.example.com/index.php"), __LINE__),
+      GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
+                    std::string("http://www.example.com/index.php"),
+                    __LINE__)};
+
+  for (unsigned int i = 0; i < arraysize(test_values); ++i) {
+    std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
+    EXPECT_EQ(temp, test_values[i].result) <<
+        "Test case #" << i << " line " << test_values[i].line << " failed";
+  }
+}
+
+TEST(RequestThrottlerManager, AreEntriesBeingCollected ) {
+  MockRequestThrottlerManager manager;
+
+  manager.CreateEntry(true);  // true = Entry is outdated.
+  manager.CreateEntry(true);
+  manager.CreateEntry(true);
+  manager.DoGarbageCollectEntries();
+  EXPECT_EQ(0, manager.GetNumberOfEntries());
+
+  manager.CreateEntry(false);
+  manager.CreateEntry(false);
+  manager.CreateEntry(false);
+  manager.CreateEntry(true);
+  manager.DoGarbageCollectEntries();
+  EXPECT_EQ(3, manager.GetNumberOfEntries());
+}
+
+TEST(RequestThrottlerManager, IsHostBeingRegistered) {
+  MockRequestThrottlerManager manager;
+
+  manager.RegisterRequestUrl(GURL("http://www.example.com/"));
+  manager.RegisterRequestUrl(GURL("http://www.google.com/"));
+  manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
+  manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
+  manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
+
+  EXPECT_EQ(3, manager.GetNumberOfEntries());
+}
+
+}  // namespace
diff --git a/net/url_request/url_request_http_job.cc b/net/url_request/url_request_http_job.cc
index cbccff2..5564628 100644
--- a/net/url_request/url_request_http_job.cc
+++ b/net/url_request/url_request_http_job.cc
@@ -27,6 +27,7 @@
 #include "net/http/http_transaction.h"
 #include "net/http/http_transaction_factory.h"
 #include "net/http/http_util.h"
+#include "net/request_throttler/request_throttler_header_adapter.h"
 #include "net/url_request/https_prober.h"
 #include "net/url_request/url_request.h"
 #include "net/url_request/url_request_context.h"
@@ -91,6 +92,8 @@ URLRequestHttpJob::URLRequestHttpJob(URLRequest* request)
           this, &URLRequestHttpJob::OnReadCompleted)),
       read_in_progress_(false),
       transaction_(NULL),
+      throttling_entry_(Singleton<RequestThrottlerManager>::get()->
+           RegisterRequestUrl(request->url())),
       sdch_dictionary_advertised_(false),
       sdch_test_activated_(false),
       sdch_test_control_(false),
@@ -569,6 +572,11 @@ void URLRequestHttpJob::NotifyHeadersComplete() {
   // also need this info.
   is_cached_content_ = response_info_->was_cached;
 
+  if (!is_cached_content_) {
+    RequestThrottlerHeaderAdapter response_adapter(response_info_->headers);
+    throttling_entry_->UpdateWithResponse(&response_adapter);
+  }
+
   ProcessStrictTransportSecurityHeader();
 
   if (SdchManager::Global() &&
@@ -616,30 +624,37 @@ void URLRequestHttpJob::StartTransaction() {
   // If we already have a transaction, then we should restart the transaction
   // with auth provided by username_ and password_.
 
-  int rv;
+  int return_value;
+
   if (transaction_.get()) {
-    rv = transaction_->RestartWithAuth(username_, password_, &start_callback_);
+    return_value = transaction_->RestartWithAuth(username_,
+        password_, &start_callback_);
     username_.clear();
     password_.clear();
   } else {
     DCHECK(request_->context());
     DCHECK(request_->context()->http_transaction_factory());
 
-    rv = request_->context()->http_transaction_factory()->CreateTransaction(
-        &transaction_);
-    if (rv == net::OK) {
-      rv = transaction_->Start(
-          &request_info_, &start_callback_, request_->net_log());
+    return_value = request_->context()->http_transaction_factory()->
+        CreateTransaction(&transaction_);
+    if (return_value == net::OK) {
+      if (throttling_entry_->IsRequestAllowed()) {
+        return_value = transaction_->Start(
+            &request_info_, &start_callback_, request_->net_log());
+      } else {
+        // Special error code for the exponential back-off module.
+        return_value = net::ERR_TEMPORARILY_THROTTLED_BY_DDOS;
+      }
     }
   }
 
-  if (rv == net::ERR_IO_PENDING)
+  if (return_value == net::ERR_IO_PENDING)
     return;
 
   // The transaction started synchronously, but we need to notify the
   // URLRequest delegate via the message loop.
   MessageLoop::current()->PostTask(FROM_HERE, NewRunnableMethod(
-      this, &URLRequestHttpJob::OnStartCompleted, rv));
+      this, &URLRequestHttpJob::OnStartCompleted, return_value));
 }
 
 void URLRequestHttpJob::AddExtraHeaders() {
diff --git a/net/url_request/url_request_http_job.h b/net/url_request/url_request_http_job.h
index ea2e544..79511f2 100644
--- a/net/url_request/url_request_http_job.h
+++ b/net/url_request/url_request_http_job.h
@@ -11,10 +11,12 @@
 #include <vector>
 
 #include "base/scoped_ptr.h"
+#include "base/string_util.h"
 #include "base/string16.h"
 #include "net/base/auth.h"
 #include "net/base/completion_callback.h"
 #include "net/http/http_request_info.h"
+#include "net/request_throttler/request_throttler_manager.h"
 #include "net/url_request/url_request_job.h"
 
 namespace net {
@@ -113,6 +115,9 @@ class URLRequestHttpJob : public URLRequestJob {
 
   scoped_ptr<net::HttpTransaction> transaction_;
 
+  // This is used to supervise traffic and enforce exponential back-off.
+  scoped_refptr<RequestThrottlerEntryInterface> throttling_entry_;
+
   // Indicated if an SDCH dictionary was advertised, and hence an SDCH
   // compressed response is expected.  We use this to help detect (accidental?)
   // proxy corruption of a response, which sometimes marks SDCH content as