Implement exponential back-off mechanism.

Contributed by yzshen@google.com, original review http://codereview.chromium.org/4194001/

Implement exponential back-off mechanism. Enforce it at the URLRequestHttpJob level for all outgoing HTTP requests.

The reason why to make this change is that we need back-off logic at a lower enough level to manage all outgoing HTTP traffic, so that the browser won't cause any DDoS attack.

This change:
1) patches http://codereview.chromium.org/2487001/show, which is the exponential back-off implementation.
2) resolves conflicts with URLFetcher, by removing its own back-off logic:
-- removes url_fetcher_protect.{h,cc};
-- integrates the sliding window mechanism of URLFetcherProtectEntry into RequestThrottlerEntry.
3) resolves conflicts with CloudPrintURLFetcher.
4) makes unit tests of CloudPrintURLFetcher, URLFetcher and URLRequest work.

BUG=none
TEST=pass all existing tests and also the newly-added request_throttler_unittest.cc

Review URL: http://codereview.chromium.org/5276007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@67375 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 57 insertions, 107 deletions

diff --git a/chrome/service/cloud_print/cloud_print_consts.cc b/chrome/service/cloud_print/cloud_print_consts.cc
index 8227466..cf62f92 100644
--- a/chrome/service/cloud_print/cloud_print_consts.cc
+++ b/chrome/service/cloud_print/cloud_print_consts.cc
@@ -39,10 +39,6 @@ const char kCloudPrintPushNotificationsSource[] = "cloudprint.google.com";
 // certain requests.
 const char kChromeCloudPrintProxyHeader[] = "X-Google-CloudPrint-Proxy: Chrome";
 
-// The request retry policy names. These strings are not valid hostnames, they
-// are just string keys.
-const char kCloudPrintAPIRetryPolicy[] = "cloudprint.google.com/api";
-const char kJobDataRetryPolicy[] = "cloudprint.google.com/jobdata";
 // The string to be appended to the user-agent for cloudprint requests.
 const char kCloudPrintUserAgent[] = "GoogleCloudPrintProxy";
 
diff --git a/chrome/service/cloud_print/cloud_print_consts.h b/chrome/service/cloud_print/cloud_print_consts.h
index 748a300..6c150b7 100644
--- a/chrome/service/cloud_print/cloud_print_consts.h
+++ b/chrome/service/cloud_print/cloud_print_consts.h
@@ -36,8 +36,6 @@ extern const char kCloudPrintGaiaServiceId[];
 extern const char kSyncGaiaServiceId[];
 extern const char kCloudPrintPushNotificationsSource[];
 extern const char kChromeCloudPrintProxyHeader[];
-extern const char kCloudPrintAPIRetryPolicy[];
-extern const char kJobDataRetryPolicy[];
 extern const char kCloudPrintUserAgent[];
 extern const char kJobFetchReasonStartup[];
 extern const char kJobFetchReasonPoll[];
@@ -46,8 +44,8 @@ extern const char kJobFetchReasonQueryMore[];
 
 // Max retry count for job data fetch requests.
 const int kJobDataMaxRetryCount = 5;
-// Look at CloudPrintProxyBackend::Core::CreateDefaultRetryPolicy for default
-// values of the request retry policy.
+// Max retry count (infinity) for API fetch requests.
+const int kCloudPrintAPIMaxRetryCount = -1;
 
 // When we don't have XMPP notifications available, we resort to polling for
 // print jobs. We choose a random interval in seconds between these 2 values.
diff --git a/chrome/service/cloud_print/cloud_print_proxy_backend.cc b/chrome/service/cloud_print/cloud_print_proxy_backend.cc
index 7dac83e..dbb374f 100644
--- a/chrome/service/cloud_print/cloud_print_proxy_backend.cc
+++ b/chrome/service/cloud_print/cloud_print_proxy_backend.cc
@@ -15,7 +15,6 @@
 #include "base/string_util.h"
 #include "base/utf_string_conversions.h"
 #include "base/values.h"
-#include "chrome/common/net/url_fetcher_protect.h"
 #include "chrome/service/cloud_print/cloud_print_consts.h"
 #include "chrome/service/cloud_print/cloud_print_helpers.h"
 #include "chrome/service/cloud_print/cloud_print_url_fetcher.h"
@@ -119,8 +118,6 @@ class CloudPrintProxyBackend::Core
     const std::string& email);
   void NotifyAuthenticationFailed();
 
-  URLFetcherProtectEntry* CreateDefaultRetryPolicy();
-
   // Starts a new printer registration process.
   void StartRegistration();
   // Ends the printer registration process.
@@ -357,36 +354,9 @@ void CloudPrintProxyBackend::Core::DoInitializeWithToken(
 
   proxy_id_ = proxy_id;
 
-  // Register the request retry policies for cloud print APIs and job data
-  // requests.
-  URLFetcherProtectManager::GetInstance()->Register(
-      kCloudPrintAPIRetryPolicy, CreateDefaultRetryPolicy());
-  URLFetcherProtectManager::GetInstance()->Register(
-      kJobDataRetryPolicy, CreateDefaultRetryPolicy())->SetMaxRetries(
-          kJobDataMaxRetryCount);
-
   StartRegistration();
 }
 
-URLFetcherProtectEntry*
-CloudPrintProxyBackend::Core::CreateDefaultRetryPolicy() {
-  // Times are in milliseconds.
-  const int kSlidingWindowPeriod = 2000;
-  const int kMaxSendThreshold = 20;
-  const int kMaxRetries = -1;
-  const int kInitialTimeout = 100;
-  const double kMultiplier = 2.0;
-  const int kConstantFactor = 100;
-  const int kMaximumTimeout = 5*60*1000;
-  return new URLFetcherProtectEntry(kSlidingWindowPeriod,
-                                    kMaxSendThreshold,
-                                    kMaxRetries,
-                                    kInitialTimeout,
-                                    kMultiplier,
-                                    kConstantFactor,
-                                    kMaximumTimeout);
-}
-
 void CloudPrintProxyBackend::Core::StartRegistration() {
   DCHECK(MessageLoop::current() == backend_->core_thread_.message_loop());
   printer_list_.clear();
@@ -444,7 +414,7 @@ void CloudPrintProxyBackend::Core::GetRegisteredPrinters() {
       &CloudPrintProxyBackend::Core::HandlePrinterListResponse;
   request_ = new CloudPrintURLFetcher;
   request_->StartGetRequest(printer_list_url, this, auth_token_,
-                            kCloudPrintAPIRetryPolicy);
+                            kCloudPrintAPIMaxRetryCount);
 }
 
 void CloudPrintProxyBackend::Core::RegisterNextPrinter() {
@@ -511,7 +481,7 @@ void CloudPrintProxyBackend::Core::RegisterNextPrinter() {
           &CloudPrintProxyBackend::Core::HandleRegisterPrinterResponse;
       request_ = new CloudPrintURLFetcher;
       request_->StartPostRequest(register_url, this, auth_token_,
-                                 kCloudPrintAPIRetryPolicy, mime_type,
+                                 kCloudPrintAPIMaxRetryCount, mime_type,
                                  post_data);
 
     } else {
diff --git a/chrome/service/cloud_print/cloud_print_url_fetcher.cc b/chrome/service/cloud_print/cloud_print_url_fetcher.cc
index 8996a0a..2e4fc55 100644
--- a/chrome/service/cloud_print/cloud_print_url_fetcher.cc
+++ b/chrome/service/cloud_print/cloud_print_url_fetcher.cc
@@ -7,7 +7,6 @@
 #include "base/string_util.h"
 #include "base/values.h"
 #include "chrome/common/net/http_return.h"
-#include "chrome/common/net/url_fetcher_protect.h"
 #include "chrome/service/cloud_print/cloud_print_consts.h"
 #include "chrome/service/cloud_print/cloud_print_helpers.h"
 #include "chrome/service/net/service_url_request_context.h"
@@ -16,15 +15,14 @@
 
 CloudPrintURLFetcher::CloudPrintURLFetcher()
     : delegate_(NULL),
-      protect_entry_(NULL),
       num_retries_(0) {
 }
 
 void CloudPrintURLFetcher::StartGetRequest(const GURL& url,
                                            Delegate* delegate,
                                            const std::string& auth_token,
-                                           const std::string& retry_policy) {
-  StartRequestHelper(url, URLFetcher::GET, delegate, auth_token, retry_policy,
+                                           int max_retries) {
+  StartRequestHelper(url, URLFetcher::GET, delegate, auth_token, max_retries,
                      std::string(), std::string());
 }
 
@@ -32,10 +30,10 @@ void CloudPrintURLFetcher::StartPostRequest(
     const GURL& url,
     Delegate* delegate,
     const std::string& auth_token,
-    const std::string& retry_policy,
+    int max_retries,
     const std::string& post_data_mime_type,
     const std::string& post_data) {
-  StartRequestHelper(url, URLFetcher::POST, delegate, auth_token, retry_policy,
+  StartRequestHelper(url, URLFetcher::POST, delegate, auth_token, max_retries,
                      post_data_mime_type, post_data);
 }
 
@@ -88,23 +86,23 @@ void CloudPrintURLFetcher::OnURLFetchComplete(
   }
   // Retry the request if needed.
   if (action == RETRY_REQUEST) {
-    int64 back_off_time =
-        protect_entry_->UpdateBackoff(URLFetcherProtectEntry::FAILURE);
+    // If the response code is greater than or equal to 500, then the back-off
+    // period has been increased at the network level; otherwise, explicitly
+    // call ReceivedContentWasMalformed() to count the current request as a
+    // failure and increase the back-off period.
+    if (response_code < 500)
+      request_->ReceivedContentWasMalformed();
+
     ++num_retries_;
-    int max_retries = protect_entry_->max_retries();
-    if ((-1 != max_retries) && (num_retries_ > max_retries)) {
+    if ((-1 != source->max_retries()) &&
+        (num_retries_ > source->max_retries())) {
       // Retry limit reached. Give up.
       delegate_->OnRequestGiveUp();
     } else {
       // Either no retry limit specified or retry limit has not yet been
       // reached. Try again.
-      MessageLoop::current()->PostDelayedTask(
-          FROM_HERE,
-          NewRunnableMethod(this, &CloudPrintURLFetcher::StartRequestNow),
-          back_off_time);
+      request_->Start();
     }
-  } else {
-    protect_entry_->UpdateBackoff(URLFetcherProtectEntry::SUCCESS);
   }
 }
 
@@ -113,7 +111,7 @@ void CloudPrintURLFetcher::StartRequestHelper(
     URLFetcher::RequestType request_type,
     Delegate* delegate,
     const std::string& auth_token,
-    const std::string& retry_policy,
+    int max_retries,
     const std::string& post_data_mime_type,
     const std::string& post_data) {
   DCHECK(delegate);
@@ -121,6 +119,7 @@ void CloudPrintURLFetcher::StartRequestHelper(
   request_->set_request_context(GetRequestContextGetter());
   // Since we implement our own retry logic, disable the retry in URLFetcher.
   request_->set_automatically_retry_on_5xx(false);
+  request_->set_max_retries(max_retries);
   delegate_ = delegate;
   std::string headers = "Authorization: GoogleLogin auth=";
   headers += auth_token;
@@ -130,16 +129,7 @@ void CloudPrintURLFetcher::StartRequestHelper(
   if (request_type == URLFetcher::POST) {
     request_->set_upload_data(post_data_mime_type, post_data);
   }
-  // Initialize the retry policy for this request.
-  protect_entry_ =
-      URLFetcherProtectManager::GetInstance()->Register(retry_policy);
-  MessageLoop::current()->PostDelayedTask(
-      FROM_HERE,
-      NewRunnableMethod(this, &CloudPrintURLFetcher::StartRequestNow),
-      protect_entry_->UpdateBackoff(URLFetcherProtectEntry::SEND));
-}
 
-void CloudPrintURLFetcher::StartRequestNow() {
   request_->Start();
 }
 
diff --git a/chrome/service/cloud_print/cloud_print_url_fetcher.h b/chrome/service/cloud_print/cloud_print_url_fetcher.h
index cbf9bf2..68499b4 100644
--- a/chrome/service/cloud_print/cloud_print_url_fetcher.h
+++ b/chrome/service/cloud_print/cloud_print_url_fetcher.h
@@ -13,16 +13,13 @@
 
 class DictionaryValue;
 class GURL;
-class URLFetcherProtectEntry;
 class URLRequestStatus;
 
 // A wrapper around URLFetcher for CloudPrint. URLFetcher applies retry logic
 // only on HTTP response codes >= 500. In the cloud print case, we want to
 // retry on all network errors. In addition, we want to treat non-JSON responses
 // (for all CloudPrint APIs that expect JSON responses) as errors and they
-// must also be retried. Also URLFetcher uses the host name of the URL as the
-// key for applying the retry policy. In our case, we want to apply one global
-// policy for many requests (not necessarily scoped by hostname).
+// must also be retried.
 class CloudPrintURLFetcher
     : public base::RefCountedThreadSafe<CloudPrintURLFetcher>,
       public URLFetcher::Delegate {
@@ -80,11 +77,11 @@ class CloudPrintURLFetcher
   void StartGetRequest(const GURL& url,
                        Delegate* delegate,
                        const std::string& auth_token,
-                       const std::string& retry_policy);
+                       int max_retries);
   void StartPostRequest(const GURL& url,
                         Delegate* delegate,
                         const std::string& auth_token,
-                        const std::string& retry_policy,
+                        int max_retries,
                         const std::string& post_data_mime_type,
                         const std::string& post_data);
 
@@ -103,14 +100,12 @@ class CloudPrintURLFetcher
                           URLFetcher::RequestType request_type,
                           Delegate* delegate,
                           const std::string& auth_token,
-                          const std::string& retry_policy,
+                          int max_retries,
                           const std::string& post_data_mime_type,
                           const std::string& post_data);
-  void StartRequestNow();
 
   scoped_ptr<URLFetcher> request_;
   Delegate* delegate_;
-  URLFetcherProtectEntry* protect_entry_;
   int num_retries_;
 };
 
diff --git a/chrome/service/cloud_print/cloud_print_url_fetcher_unittest.cc b/chrome/service/cloud_print/cloud_print_url_fetcher_unittest.cc
index e3ca15b..9380eaa 100644
--- a/chrome/service/cloud_print/cloud_print_url_fetcher_unittest.cc
+++ b/chrome/service/cloud_print/cloud_print_url_fetcher_unittest.cc
@@ -7,14 +7,14 @@
 #include "base/ref_counted.h"
 #include "base/thread.h"
 #include "base/waitable_event.h"
-#include "chrome/common/net/url_fetcher_protect.h"
 #include "chrome/common/net/url_request_context_getter.h"
 #include "chrome/service/service_process.h"
 #include "chrome/service/cloud_print/cloud_print_url_fetcher.h"
 #include "googleurl/src/gurl.h"
 #include "net/test/test_server.h"
-#include "net/url_request/url_request_unittest.h"
 #include "net/url_request/url_request_status.h"
+#include "net/url_request/url_request_throttler_manager.h"
+#include "net/url_request/url_request_unittest.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 using base::Time;
@@ -69,10 +69,10 @@ class TestCloudPrintURLFetcher : public CloudPrintURLFetcher {
 class CloudPrintURLFetcherTest : public testing::Test,
                                  public CloudPrintURLFetcher::Delegate {
  public:
-  CloudPrintURLFetcherTest() : fetcher_(NULL) { }
+  CloudPrintURLFetcherTest() : max_retries_(0), fetcher_(NULL) { }
 
   // Creates a URLFetcher, using the program's main thread to do IO.
-  virtual void CreateFetcher(const GURL& url, const std::string& retry_policy);
+  virtual void CreateFetcher(const GURL& url, int max_retries);
 
   // CloudPrintURLFetcher::Delegate
   virtual CloudPrintURLFetcher::ResponseAction HandleRawResponse(
@@ -113,7 +113,7 @@ class CloudPrintURLFetcherTest : public testing::Test,
   // a UI thread, we dispatch a worker thread to do so.
   MessageLoopForIO io_loop_;
   scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
-  std::string retry_policy_;
+  int max_retries_;
   Time start_time_;
   scoped_refptr<CloudPrintURLFetcher> fetcher_;
 };
@@ -188,12 +188,11 @@ class CloudPrintURLFetcherRetryBackoffTest : public CloudPrintURLFetcherTest {
 };
 
 
-void CloudPrintURLFetcherTest::CreateFetcher(const GURL& url,
-                                             const std::string& retry_policy) {
+void CloudPrintURLFetcherTest::CreateFetcher(const GURL& url, int max_retries) {
   fetcher_ = new TestCloudPrintURLFetcher(io_message_loop_proxy());
-  retry_policy_ = retry_policy;
+  max_retries_ = max_retries;
   start_time_ = Time::Now();
-  fetcher_->StartGetRequest(url, this, "", retry_policy_);
+  fetcher_->StartGetRequest(url, this, "", max_retries_);
 }
 
 CloudPrintURLFetcher::ResponseAction
@@ -265,10 +264,10 @@ CloudPrintURLFetcherOverloadTest::HandleRawData(const URLFetcher* source,
   const TimeDelta one_second = TimeDelta::FromMilliseconds(1000);
   response_count_++;
   if (response_count_ < 20) {
-    fetcher_->StartGetRequest(url, this, "", retry_policy_);
+    fetcher_->StartGetRequest(url, this, "", max_retries_);
   } else {
     // We have already sent 20 requests continuously. And we expect that
-    // it takes more than 1 second due to the overload pretection settings.
+    // it takes more than 1 second due to the overload protection settings.
     EXPECT_TRUE(Time::Now() - start_time_ >= one_second);
     io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask());
   }
@@ -297,7 +296,7 @@ TEST_F(CloudPrintURLFetcherBasicTest, HandleRawResponse) {
   ASSERT_TRUE(test_server.Start());
   SetHandleRawResponse(true);
 
-  CreateFetcher(test_server.GetURL("echo"), "DummyRetryPolicy");
+  CreateFetcher(test_server.GetURL("echo"), 0);
   MessageLoop::current()->Run();
 }
 
@@ -307,7 +306,7 @@ TEST_F(CloudPrintURLFetcherBasicTest, FLAKY_HandleRawData) {
   ASSERT_TRUE(test_server.Start());
 
   SetHandleRawData(true);
-  CreateFetcher(test_server.GetURL("echo"), "DummyRetryPolicy");
+  CreateFetcher(test_server.GetURL("echo"), 0);
   MessageLoop::current()->Run();
 }
 
@@ -319,15 +318,16 @@ TEST_F(CloudPrintURLFetcherOverloadTest, Protect) {
 
   // Registers an entry for test url. It only allows 3 requests to be sent
   // in 200 milliseconds.
-  std::string retry_policy = "OverloadTestPolicy";
-  URLFetcherProtectManager* manager = URLFetcherProtectManager::GetInstance();
-  URLFetcherProtectEntry* entry =
-      new URLFetcherProtectEntry(200, 3, 11, 1, 2.0, 0, 256);
-  manager->Register(retry_policy, entry);
+  scoped_refptr<net::URLRequestThrottlerEntry> entry(
+      new net::URLRequestThrottlerEntry(200, 3, 1, 0, 2.0, 0.0, 256));
+  net::URLRequestThrottlerManager::GetInstance()->OverrideEntryForTests(
+      url, entry);
 
-  CreateFetcher(url, retry_policy);
+  CreateFetcher(url, 11);
 
   MessageLoop::current()->Run();
+
+  net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url);
 }
 
 // http://code.google.com/p/chromium/issues/detail?id=62758
@@ -341,15 +341,16 @@ TEST_F(CloudPrintURLFetcherRetryBackoffTest, FLAKY_GiveUp) {
   //     new_backoff = 2.0 * old_backoff + 0
   // and maximum backoff time is 256 milliseconds.
   // Maximum retries allowed is set to 11.
-  std::string retry_policy = "BackoffTestPolicy";
-  URLFetcherProtectManager* manager = URLFetcherProtectManager::GetInstance();
-  URLFetcherProtectEntry* entry =
-      new URLFetcherProtectEntry(200, 3, 11, 1, 2.0, 0, 256);
-  manager->Register(retry_policy, entry);
+  scoped_refptr<net::URLRequestThrottlerEntry> entry(
+      new net::URLRequestThrottlerEntry(200, 3, 1, 0, 2.0, 0.0, 256));
+  net::URLRequestThrottlerManager::GetInstance()->OverrideEntryForTests(
+      url, entry);
 
-  CreateFetcher(url, retry_policy);
+  CreateFetcher(url, 11);
 
   MessageLoop::current()->Run();
+
+  net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url);
 }
 
 }  // namespace.
diff --git a/chrome/service/cloud_print/job_status_updater.cc b/chrome/service/cloud_print/job_status_updater.cc
index 4719a6d..f717f29 100644
--- a/chrome/service/cloud_print/job_status_updater.cc
+++ b/chrome/service/cloud_print/job_status_updater.cc
@@ -62,7 +62,7 @@ void JobStatusUpdater::UpdateStatus() {
       request_->StartGetRequest(
           CloudPrintHelpers::GetUrlForJobStatusUpdate(
               cloud_print_server_url_, job_id_, last_job_details_),
-          this, auth_token_, kCloudPrintAPIRetryPolicy);
+          this, auth_token_, kCloudPrintAPIMaxRetryCount);
     }
   }
 }
diff --git a/chrome/service/cloud_print/printer_job_handler.cc b/chrome/service/cloud_print/printer_job_handler.cc
index e705762..d285594 100644
--- a/chrome/service/cloud_print/printer_job_handler.cc
+++ b/chrome/service/cloud_print/printer_job_handler.cc
@@ -91,7 +91,7 @@ void PrinterJobHandler::Start() {
         request_->StartGetRequest(
             CloudPrintHelpers::GetUrlForPrinterDelete(
                 cloud_print_server_url_, printer_info_cloud_.printer_id),
-            this, auth_token_, kCloudPrintAPIRetryPolicy);
+            this, auth_token_, kCloudPrintAPIMaxRetryCount);
       }
       if (!task_in_progress_ && printer_update_pending_) {
         printer_update_pending_ = false;
@@ -107,7 +107,7 @@ void PrinterJobHandler::Start() {
             CloudPrintHelpers::GetUrlForJobFetch(
                 cloud_print_server_url_, printer_info_cloud_.printer_id,
                 job_fetch_reason_),
-            this, auth_token_, kCloudPrintAPIRetryPolicy);
+            this, auth_token_, kCloudPrintAPIMaxRetryCount);
         last_job_fetch_time_ = base::TimeTicks::Now();
         VLOG(1) << "Last job fetch time for printer "
                 << printer_info_.printer_name.c_str() << " is "
@@ -214,7 +214,7 @@ bool PrinterJobHandler::UpdatePrinterInfo() {
     request_->StartPostRequest(
         CloudPrintHelpers::GetUrlForPrinterUpdate(
             cloud_print_server_url_, printer_info_cloud_.printer_id),
-        this, auth_token_, kCloudPrintAPIRetryPolicy, mime_type, post_data);
+        this, auth_token_, kCloudPrintAPIMaxRetryCount, mime_type, post_data);
     ret = true;
   }
   return ret;
@@ -355,7 +355,7 @@ PrinterJobHandler::HandleJobMetadataResponse(
         request_->StartGetRequest(GURL(print_ticket_url.c_str()),
                                   this,
                                   auth_token_,
-                                  kCloudPrintAPIRetryPolicy);
+                                  kCloudPrintAPIMaxRetryCount);
       }
     }
   }
@@ -379,7 +379,7 @@ PrinterJobHandler::HandlePrintTicketResponse(const URLFetcher* source,
     request_->StartGetRequest(GURL(print_data_url_.c_str()),
                               this,
                               auth_token_,
-                              kJobDataRetryPolicy);
+                              kJobDataMaxRetryCount);
   } else {
     // The print ticket was not valid. We are done here.
     FailedFetchingJobData();
@@ -527,7 +527,7 @@ void PrinterJobHandler::UpdateJobStatus(cloud_print::PrintJobStatus status,
                                                       status),
           this,
           auth_token_,
-          kCloudPrintAPIRetryPolicy);
+          kCloudPrintAPIMaxRetryCount);
     }
   }
 }