Add a constraint on how many requests can be outstanding for any given render process (browser-side).

Once the constraint is reached, subsequent requests will fail with net::ERR_INSUFFICIENT_RESOURCES.

The bound is defined as "25 MB", which represents the amount of private bytes we expect  the pending requests to consume in the browser. This number translates into around 6000 typical requests.

Note that the upload data of a request is not currently considered part of the request's in-memory cost -- more data is needed on the average/maximum upload sizes of users before deciding what a compatible limit is.

BUG=5688
Review URL: http://codereview.chromium.org/18541

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@9298 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 328 insertions, 19 deletions

diff --git a/chrome/browser/renderer_host/resource_dispatcher_host.cc b/chrome/browser/renderer_host/resource_dispatcher_host.cc
index 586e823..a778bf3 100644
--- a/chrome/browser/renderer_host/resource_dispatcher_host.cc
+++ b/chrome/browser/renderer_host/resource_dispatcher_host.cc
@@ -72,6 +72,11 @@ static const int kUpdateLoadStatesIntervalMsec = 100;
 // given time for a given request.
 static const int kMaxPendingDataMessages = 20;
 
+// Maximum byte "cost" of all the outstanding requests for a renderer.
+// See delcaration of |max_outstanding_requests_cost_per_process_| for details.
+// This bound is 25MB, which allows for around 6000 outstanding requests.
+static const int kMaxOutstandingRequestsCostPerProcess = 26214400;
+
 // A ShutdownTask proxies a shutdown task from the UI thread to the IO thread.
 // It should be constructed on the UI thread and run in the IO thread.
 class ResourceDispatcherHost::ShutdownTask : public Task {
@@ -135,7 +140,8 @@ ResourceDispatcherHost::ResourceDispatcherHost(MessageLoop* io_loop)
       request_id_(-1),
       plugin_service_(PluginService::GetInstance()),
       method_runner_(this),
-      is_shutdown_(false) {
+      is_shutdown_(false),
+      max_outstanding_requests_cost_per_process_(kMaxOutstandingRequestsCostPerProcess) {
 }
 
 ResourceDispatcherHost::~ResourceDispatcherHost() {
@@ -568,6 +574,14 @@ void ResourceDispatcherHost::PauseRequest(int render_process_host_id,
   }
 }
 
+int ResourceDispatcherHost::GetOutstandingRequestsMemoryCost(
+    int render_process_host_id) const {
+  OutstandingRequestsMemoryCostMap::const_iterator entry =
+      outstanding_requests_memory_cost_map_.find(render_process_host_id);
+  return (entry == outstanding_requests_memory_cost_map_.end()) ?
+      0 : entry->second;
+}
+
 void ResourceDispatcherHost::OnClosePageACK(int render_process_host_id,
                                             int request_id) {
   GlobalRequestID global_id(render_process_host_id, request_id);
@@ -674,8 +688,14 @@ void ResourceDispatcherHost::RemovePendingRequest(int render_process_host_id,
 
 void ResourceDispatcherHost::RemovePendingRequest(
     const PendingRequestList::iterator& iter) {
-  // Notify the login handler that this request object is going away.
   ExtraRequestInfo* info = ExtraInfoForRequest(iter->second);
+
+  // Remove the memory credit that we added when pushing the request onto
+  // the pending list.
+  IncrementOutstandingRequestsMemoryCost(-1 * info->memory_cost,
+                                         info->render_process_host_id);
+
+  // Notify the login handler that this request object is going away.
   if (info && info->login_handler)
     info->login_handler->OnRequestCancelled();
 
@@ -823,10 +843,89 @@ bool ResourceDispatcherHost::CompleteResponseStarted(URLRequest* request) {
                                                    response.get());
 }
 
+int ResourceDispatcherHost::IncrementOutstandingRequestsMemoryCost(
+    int cost, int render_process_host_id) {
+  // Retrieve the previous value (defaulting to 0 if not found).
+  OutstandingRequestsMemoryCostMap::iterator prev_entry =
+      outstanding_requests_memory_cost_map_.find(render_process_host_id);
+  int new_cost = 0;
+  if (prev_entry != outstanding_requests_memory_cost_map_.end())
+    new_cost = prev_entry->second;
+
+  // Insert/update the total; delete entries when their value reaches 0.
+  new_cost += cost;
+  CHECK(new_cost >= 0);
+  if (new_cost == 0)
+    outstanding_requests_memory_cost_map_.erase(prev_entry);
+  else
+    outstanding_requests_memory_cost_map_[render_process_host_id] = new_cost;
+
+  return new_cost;
+}
+
+// static
+int ResourceDispatcherHost::CalculateApproximateMemoryCost(
+    URLRequest* request) {
+  // The following fields should be a minor size contribution (experimentally
+  // on the order of 100). However since they are variable length, it could
+  // in theory be a sizeable contribution.
+  int strings_cost = request->extra_request_headers().size() +
+                     request->original_url().spec().size() +
+                     request->referrer().size() +
+                     request->method().size();
+
+  int upload_cost = 0;
+
+  // TODO(eroman): don't enable the upload throttling until we have data
+  // showing what a reasonable limit is (limiting to 25MB of uploads may
+  // be too restrictive).
+#if 0
+  // Sum all the (non-file) upload data attached to the request, if any.
+  if (request->has_upload()) {
+    const std::vector<net::UploadData::Element>& uploads =
+        request->get_upload()->elements();
+    std::vector<net::UploadData::Element>::const_iterator iter;
+    for (iter = uploads.begin(); iter != uploads.end(); ++iter) {
+      if (iter->type() == net::UploadData::TYPE_BYTES) {
+        int64 element_size = iter->GetContentLength();
+        // This cast should not result in truncation.
+        upload_cost += static_cast<int>(element_size);
+      }
+    }
+  }
+#endif
+
+  // Note that this expression will typically be dominated by:
+  // |kAvgBytesPerOutstandingRequest|.
+  return kAvgBytesPerOutstandingRequest + strings_cost + upload_cost;
+}
+
 void ResourceDispatcherHost::BeginRequestInternal(URLRequest* request,
                                                   bool mixed_content) {
+  DCHECK(!request->is_pending());
   ExtraRequestInfo* info = ExtraInfoForRequest(request);
 
+  // Add the memory estimate that starting this request will consume.
+  info->memory_cost = CalculateApproximateMemoryCost(request);
+  int memory_cost = IncrementOutstandingRequestsMemoryCost(
+      info->memory_cost,
+      info->render_process_host_id);
+
+  // If enqueing/starting this request will exceed our per-process memory
+  // bound, abort it right away.
+  if (memory_cost > max_outstanding_requests_cost_per_process_) {
+    // We call "CancelWithError()" as a way of setting the URLRequest's
+    // status -- it has no effect beyond this, since the request hasn't started.
+    request->CancelWithError(net::ERR_INSUFFICIENT_RESOURCES);
+
+    // TODO(eroman): this is kinda funky -- we insert the unstarted request into
+    // |pending_requests_| simply to please OnResponseCompleted().
+    GlobalRequestID global_id(info->render_process_host_id, info->request_id);
+    pending_requests_[global_id] = request;
+    OnResponseCompleted(request);
+    return;
+  }
+
   std::pair<int, int> pair_id(info->render_process_host_id,
                               info->render_view_id);
   BlockedRequestMap::const_iterator iter = blocked_requests_map_.find(pair_id);
@@ -1317,6 +1416,11 @@ void ResourceDispatcherHost::ProcessBlockedRequestsForRenderView(
 
   for (BlockedRequestsList::iterator req_iter = requests->begin();
        req_iter != requests->end(); ++req_iter) {
+    // Remove the memory credit that we added when pushing the request onto
+    // the blocked list.
+    ExtraRequestInfo* info = ExtraInfoForRequest(req_iter->url_request);
+    IncrementOutstandingRequestsMemoryCost(-1 * info->memory_cost,
+                                           info->render_process_host_id);
     if (cancel_requests)
       delete req_iter->url_request;
     else
diff --git a/chrome/browser/renderer_host/resource_dispatcher_host.h b/chrome/browser/renderer_host/resource_dispatcher_host.h
index f2dba57..6dfdd6d 100644
--- a/chrome/browser/renderer_host/resource_dispatcher_host.h
+++ b/chrome/browser/renderer_host/resource_dispatcher_host.h
@@ -81,6 +81,7 @@ class ResourceDispatcherHost : public URLRequest::Delegate {
           upload_size(upload_size),
           last_upload_position(0),
           waiting_for_upload_progress_ack(false),
+          memory_cost(0),
           is_paused(false),
           has_started_reading(false),
           paused_read_bytes(0) {
@@ -135,6 +136,10 @@ class ResourceDispatcherHost : public URLRequest::Delegate {
 
     bool waiting_for_upload_progress_ack;
 
+    // The approximate in-memory size (bytes) that we credited this request
+    // as consuming in |outstanding_requests_memory_cost_map_|.
+    int memory_cost;
+
    private:
     // Request is temporarily not handling network data. Should be used only
     // by the ResourceDispatcherHost, not the event handlers.
@@ -246,6 +251,19 @@ class ResourceDispatcherHost : public URLRequest::Delegate {
   int pending_requests() const {
     return static_cast<int>(pending_requests_.size());
   }
+  
+  // Intended for unit-tests only. Returns the memory cost of all the
+  // outstanding requests (pending and blocked) for |render_process_host_id|.
+  int GetOutstandingRequestsMemoryCost(int render_process_host_id) const;
+
+  // Intended for unit-tests only. Overrides the outstanding requests bound.
+  void set_max_outstanding_requests_cost_per_process(int limit) {
+    max_outstanding_requests_cost_per_process_ = limit;
+  }
+
+  // The average private bytes increase of the browser for each new pending
+  // request. Experimentally obtained.
+  static const int kAvgBytesPerOutstandingRequest = 4400;
 
   DownloadFileManager* download_file_manager() const {
     return download_file_manager_;
@@ -341,6 +359,11 @@ class ResourceDispatcherHost : public URLRequest::Delegate {
 
  private:
   FRIEND_TEST(ResourceDispatcherHostTest, TestBlockedRequestsProcessDies);
+  FRIEND_TEST(ResourceDispatcherHostTest,
+              IncrementOutstandingRequestsMemoryCost);
+  FRIEND_TEST(ResourceDispatcherHostTest,
+              CalculateApproximateMemoryCost);
+
   class ShutdownTask;
 
   friend class ShutdownTask;
@@ -388,6 +411,18 @@ class ResourceDispatcherHost : public URLRequest::Delegate {
   // Helper function for regular and download requests.
   void BeginRequestInternal(URLRequest* request, bool mixed_content);
 
+  // Updates the "cost" of outstanding requests for |render_process_host_id|.
+  // The "cost" approximates how many bytes are consumed by all the in-memory
+  // data structures supporting this request (URLRequest object,
+  // HttpNetworkTransaction, etc...).
+  // The value of |cost| is added to the running total, and the resulting
+  // sum is returned. 
+  int IncrementOutstandingRequestsMemoryCost(int cost,
+                                             int render_process_host_id);
+
+  // Estimate how much heap space |request| will consume to run.
+  static int CalculateApproximateMemoryCost(URLRequest* request);
+
   // The list of all requests that we have pending. This list is not really
   // optimized, and assumes that we have relatively few requests pending at once
   // since some operations require brute-force searching of the list.
@@ -477,6 +512,20 @@ class ResourceDispatcherHost : public URLRequest::Delegate {
   typedef std::map<ProcessRendererIDs, BlockedRequestsList*> BlockedRequestMap;
   BlockedRequestMap blocked_requests_map_;
 
+  // Maps the render_process_host_ids to the approximate number of bytes 
+  // being used to service its resource requests. No entry implies 0 cost.
+  typedef std::map<int, int> OutstandingRequestsMemoryCostMap;
+  OutstandingRequestsMemoryCostMap outstanding_requests_memory_cost_map_;
+
+  // |max_outstanding_requests_cost_per_process_| is the upper bound on how
+  // many outstanding requests can be issued per render process host.
+  // The constraint is expressed in terms of bytes (where the cost of
+  // individual requests is given by CalculateApproximateMemoryCost).
+  // The total number of outstanding requests is roughly:
+  //   (max_outstanding_requests_cost_per_process_ /
+  //       kAvgBytesPerOutstandingRequest)
+  int max_outstanding_requests_cost_per_process_;
+
   DISALLOW_COPY_AND_ASSIGN(ResourceDispatcherHost);
 };
 
diff --git a/chrome/browser/renderer_host/resource_dispatcher_host_unittest.cc b/chrome/browser/renderer_host/resource_dispatcher_host_unittest.cc
index ae8ae7d..df293c2 100644
--- a/chrome/browser/renderer_host/resource_dispatcher_host_unittest.cc
+++ b/chrome/browser/renderer_host/resource_dispatcher_host_unittest.cc
@@ -104,7 +104,7 @@ class ResourceDispatcherHostTest : public testing::Test,
     URLRequest::RegisterProtocolFactory("test", NULL);
     RendererSecurityPolicy::GetInstance()->Remove(0);
 
-    // The plugin lib is automatically loaded during these test 
+    // The plugin lib is automatically loaded during these test
     // and we want a clean environment for other tests.
     ChromePluginLib::UnloadAllPlugins();
 
@@ -194,6 +194,8 @@ void CheckSuccessfulRequest(const std::vector<IPC::Message>& messages,
 
 // Tests whether many messages get dispatched properly.
 TEST_F(ResourceDispatcherHostTest, TestMany) {
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
   MakeTestRequest(0, 0, 1, URLRequestTestJob::test_url_1());
   MakeTestRequest(0, 0, 2, URLRequestTestJob::test_url_2());
   MakeTestRequest(0, 0, 3, URLRequestTestJob::test_url_3());
@@ -201,6 +203,8 @@ TEST_F(ResourceDispatcherHostTest, TestMany) {
   // flush all the pending requests
   while (URLRequestTestJob::ProcessOnePendingMessage());
 
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
   // sorts out all the messages we saw by request
   ResourceIPCAccumulator::ClassifiedMessages msgs;
   accum_.GetClassifiedMessages(&msgs);
@@ -218,6 +222,8 @@ TEST_F(ResourceDispatcherHostTest, TestMany) {
 TEST_F(ResourceDispatcherHostTest, Cancel) {
   ResourceDispatcherHost host(NULL);
 
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
   MakeTestRequest(0, 0, 1, URLRequestTestJob::test_url_1());
   MakeTestRequest(0, 0, 2, URLRequestTestJob::test_url_2());
   MakeTestRequest(0, 0, 3, URLRequestTestJob::test_url_3());
@@ -225,6 +231,9 @@ TEST_F(ResourceDispatcherHostTest, Cancel) {
 
   // flush all the pending requests
   while (URLRequestTestJob::ProcessOnePendingMessage());
+  MessageLoop::current()->RunAllPending();
+
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
 
   ResourceIPCAccumulator::ClassifiedMessages msgs;
   accum_.GetClassifiedMessages(&msgs);
@@ -235,28 +244,19 @@ TEST_F(ResourceDispatcherHostTest, Cancel) {
   CheckSuccessfulRequest(msgs[0], URLRequestTestJob::test_data_1());
   CheckSuccessfulRequest(msgs[2], URLRequestTestJob::test_data_3());
 
-  // Check that request 2 got canceled before it finished reading, which gives
-  // us 1 ReceivedResponse message.
-  ASSERT_EQ(1, msgs[1].size());
+  // Check that request 2 got canceled.
+  ASSERT_EQ(2, msgs[1].size());
   ASSERT_EQ(ViewMsg_Resource_ReceivedResponse::ID, msgs[1][0].type());
+  ASSERT_EQ(ViewMsg_Resource_RequestComplete::ID, msgs[1][1].type());
 
-  // TODO(mbelshe):
-  // Now that the async IO path is in place, the IO always completes on the
-  // initial call; so the cancel doesn't arrive until after we finished.
-  // This basically means the test doesn't work.
-#if 0
   int request_id;
   URLRequestStatus status;
 
-  // The message should be all data received with an error.
-  ASSERT_EQ(ViewMsg_Resource_RequestComplete::ID, msgs[1][2].type());
-
   void* iter = NULL;
-  ASSERT_TRUE(IPC::ReadParam(&msgs[1][2], &iter, &request_id));
-  ASSERT_TRUE(IPC::ReadParam(&msgs[1][2], &iter, &status));
+  ASSERT_TRUE(IPC::ReadParam(&msgs[1][1], &iter, &request_id));
+  ASSERT_TRUE(IPC::ReadParam(&msgs[1][1], &iter, &status));
 
   EXPECT_EQ(URLRequestStatus::CANCELED, status.status());
-#endif
 }
 
 // Tests CancelRequestsForProcess
@@ -270,7 +270,7 @@ TEST_F(ResourceDispatcherHostTest, TestProcessCancel) {
     virtual bool Send(IPC::Message* msg) {
       // no messages should be received when the process has been canceled
       if (has_canceled_)
-        received_after_canceled_ ++;
+        received_after_canceled_++;
       delete msg;
       return true;
     }
@@ -283,6 +283,8 @@ TEST_F(ResourceDispatcherHostTest, TestProcessCancel) {
   ViewHostMsg_Resource_Request request =
       CreateResourceRequest("GET", URLRequestTestJob::test_url_1());
 
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
   host_.BeginRequest(&test_receiver, GetCurrentProcess(), 0, MSG_ROUTING_NONE,
                      1, request, NULL, NULL);
   KickOffRequest();
@@ -296,7 +298,7 @@ TEST_F(ResourceDispatcherHostTest, TestProcessCancel) {
                      3, request, NULL, NULL);
   KickOffRequest();
 
-  // TODO: mbelshe
+  // TODO(mbelshe):
   // Now that the async IO path is in place, the IO always completes on the
   // initial call; so the requests have already completed.  This basically
   // breaks the whole test.
@@ -314,6 +316,7 @@ TEST_F(ResourceDispatcherHostTest, TestProcessCancel) {
   while (URLRequestTestJob::ProcessOnePendingMessage());
 
   EXPECT_EQ(0, host_.pending_requests());
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
 
   // the test delegate should not have gotten any messages after being canceled
   ASSERT_EQ(0, test_receiver.received_after_canceled_);
@@ -327,6 +330,8 @@ TEST_F(ResourceDispatcherHostTest, TestProcessCancel) {
 
 // Tests blocking and resuming requests.
 TEST_F(ResourceDispatcherHostTest, TestBlockingResumingRequests) {
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
   host_.BlockRequestsForRenderView(0, 1);
   host_.BlockRequestsForRenderView(0, 2);
   host_.BlockRequestsForRenderView(0, 3);
@@ -376,6 +381,8 @@ TEST_F(ResourceDispatcherHostTest, TestBlockingResumingRequests) {
   KickOffRequest();
   while (URLRequestTestJob::ProcessOnePendingMessage());
 
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
   msgs.clear();
   accum_.GetClassifiedMessages(&msgs);
   ASSERT_EQ(2, msgs.size());
@@ -385,6 +392,8 @@ TEST_F(ResourceDispatcherHostTest, TestBlockingResumingRequests) {
 
 // Tests blocking and canceling requests.
 TEST_F(ResourceDispatcherHostTest, TestBlockingCancelingRequests) {
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
   host_.BlockRequestsForRenderView(0, 1);
 
   MakeTestRequest(0, 0, 1, URLRequestTestJob::test_url_1());
@@ -409,6 +418,9 @@ TEST_F(ResourceDispatcherHostTest, TestBlockingCancelingRequests) {
   host_.CancelBlockedRequestsForRenderView(0, 1);
   KickOffRequest();
   while (URLRequestTestJob::ProcessOnePendingMessage());
+
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
   msgs.clear();
   accum_.GetClassifiedMessages(&msgs);
   ASSERT_EQ(0, msgs.size());
@@ -416,6 +428,9 @@ TEST_F(ResourceDispatcherHostTest, TestBlockingCancelingRequests) {
 
 // Tests that blocked requests are canceled if their associated process dies.
 TEST_F(ResourceDispatcherHostTest, TestBlockedRequestsProcessDies) {
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(1));
+
   host_.BlockRequestsForRenderView(1, 0);
 
   MakeTestRequest(0, 0, 1, URLRequestTestJob::test_url_1());
@@ -429,6 +444,9 @@ TEST_F(ResourceDispatcherHostTest, TestBlockedRequestsProcessDies) {
   // Flush all the pending requests.
   while (URLRequestTestJob::ProcessOnePendingMessage());
 
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(1));
+
   // Sort out all the messages we saw by request.
   ResourceIPCAccumulator::ClassifiedMessages msgs;
   accum_.GetClassifiedMessages(&msgs);
@@ -461,3 +479,141 @@ TEST_F(ResourceDispatcherHostTest, TestBlockedRequestsDontLeak) {
   // Flush all the pending requests.
   while (URLRequestTestJob::ProcessOnePendingMessage());
 }
+
+// Test the private helper method "CalculateApproximateMemoryCost()".
+TEST_F(ResourceDispatcherHostTest, CalculateApproximateMemoryCost) {
+  URLRequest req(GURL("http://www.google.com"), NULL);
+  EXPECT_EQ(4425, ResourceDispatcherHost::CalculateApproximateMemoryCost(&req));
+
+  // Add 9 bytes of referrer.
+  req.set_referrer("123456789");
+  EXPECT_EQ(4434, ResourceDispatcherHost::CalculateApproximateMemoryCost(&req));
+
+  // Add 33 bytes of upload content.
+  std::string upload_content;
+  upload_content.resize(33);
+  std::fill(upload_content.begin(), upload_content.end(), 'x');
+  req.AppendBytesToUpload(upload_content.data(), upload_content.size());
+
+  // Since the upload throttling is disabled, this has no effect on the cost.
+  EXPECT_EQ(4434, ResourceDispatcherHost::CalculateApproximateMemoryCost(&req));
+
+  // Add a file upload -- should have no effect.
+  req.AppendFileToUpload(L"does-not-exist.png");
+  EXPECT_EQ(4434, ResourceDispatcherHost::CalculateApproximateMemoryCost(&req));
+}
+
+// Test the private helper method "IncrementOutstandingRequestsMemoryCost()".
+TEST_F(ResourceDispatcherHostTest, IncrementOutstandingRequestsMemoryCost) {
+  ResourceDispatcherHost host(NULL);
+
+  // Add some counts for render_process_host=7
+  EXPECT_EQ(0, host.GetOutstandingRequestsMemoryCost(7));
+  EXPECT_EQ(1, host.IncrementOutstandingRequestsMemoryCost(1, 7));
+  EXPECT_EQ(2, host.IncrementOutstandingRequestsMemoryCost(1, 7));
+  EXPECT_EQ(3, host.IncrementOutstandingRequestsMemoryCost(1, 7));
+
+  // Add some counts for render_process_host=3
+  EXPECT_EQ(0, host.GetOutstandingRequestsMemoryCost(3));
+  EXPECT_EQ(1, host.IncrementOutstandingRequestsMemoryCost(1, 3));
+  EXPECT_EQ(2, host.IncrementOutstandingRequestsMemoryCost(1, 3));
+
+  // Remove all the counts for render_process_host=7
+  EXPECT_EQ(3, host.GetOutstandingRequestsMemoryCost(7));
+  EXPECT_EQ(2, host.IncrementOutstandingRequestsMemoryCost(-1, 7));
+  EXPECT_EQ(1, host.IncrementOutstandingRequestsMemoryCost(-1, 7));
+  EXPECT_EQ(0, host.IncrementOutstandingRequestsMemoryCost(-1, 7));
+  EXPECT_EQ(0, host.GetOutstandingRequestsMemoryCost(7));
+
+  // Remove all the counts for render_process_host=3
+  EXPECT_EQ(2, host.GetOutstandingRequestsMemoryCost(3));
+  EXPECT_EQ(1, host.IncrementOutstandingRequestsMemoryCost(-1, 3));
+  EXPECT_EQ(0, host.IncrementOutstandingRequestsMemoryCost(-1, 3));
+  EXPECT_EQ(0, host.GetOutstandingRequestsMemoryCost(3));
+
+  // When an entry reaches 0, it should be deleted.
+  EXPECT_TRUE(host.outstanding_requests_memory_cost_map_.end() ==
+      host.outstanding_requests_memory_cost_map_.find(7));
+  EXPECT_TRUE(host.outstanding_requests_memory_cost_map_.end() ==
+      host.outstanding_requests_memory_cost_map_.find(3));
+}
+
+// Test that when too many requests are outstanding for a particular
+// render_process_host_id, any subsequent request from it fails.
+TEST_F(ResourceDispatcherHostTest, TooManyOutstandingRequests) {
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
+  // Expected cost of each request as measured by
+  // ResourceDispatcherHost::CalculateApproximateMemoryCost().
+  int kMemoryCostOfTest2Req =
+      ResourceDispatcherHost::kAvgBytesPerOutstandingRequest + 
+      std::string("GET").size() +
+      URLRequestTestJob::test_url_2().spec().size();
+
+  // Tighten the bound on the ResourceDispatcherHost, to speed things up.
+  int kMaxCostPerProcess = 440000;
+  host_.set_max_outstanding_requests_cost_per_process(kMaxCostPerProcess);
+
+  // Determine how many instance of test_url_2() we can request before
+  // throttling kicks in.
+  int kMaxRequests = kMaxCostPerProcess / kMemoryCostOfTest2Req;
+
+  // Saturate the number of outstanding requests for process 0.
+  for (int i = 0; i < kMaxRequests; ++i)
+    MakeTestRequest(0, 0, i + 1, URLRequestTestJob::test_url_2());
+
+  // Issue two more requests for process 0 -- these should fail immediately.
+  MakeTestRequest(0, 0, kMaxRequests + 1, URLRequestTestJob::test_url_2());
+  MakeTestRequest(0, 0, kMaxRequests + 2, URLRequestTestJob::test_url_2());
+
+  // Issue two requests for process 1 -- these should succeed since
+  // it is just process 0 that is saturated.
+  MakeTestRequest(1, 0, kMaxRequests + 3, URLRequestTestJob::test_url_2());
+  MakeTestRequest(1, 0, kMaxRequests + 4, URLRequestTestJob::test_url_2());
+
+  // Flush all the pending requests.
+  while (URLRequestTestJob::ProcessOnePendingMessage());
+  MessageLoop::current()->RunAllPending();
+
+  EXPECT_EQ(0, host_.GetOutstandingRequestsMemoryCost(0));
+
+  // Sorts out all the messages we saw by request.
+  ResourceIPCAccumulator::ClassifiedMessages msgs;
+  accum_.GetClassifiedMessages(&msgs);
+
+  // We issued (kMaxRequests + 4) total requests.
+  ASSERT_EQ(kMaxRequests + 4, msgs.size());
+
+  // Check that the first kMaxRequests succeeded.
+  for (int i = 0; i < kMaxRequests; ++i)
+    CheckSuccessfulRequest(msgs[i], URLRequestTestJob::test_data_2());
+
+  // Check that the subsequent two requests (kMaxRequests + 1) and
+  // (kMaxRequests + 2) were failed, since the per-process bound was reached.
+  for (int i = 0; i < 2; ++i) {
+    // Should have sent a single RequestComplete message.
+    int index = kMaxRequests + i;
+    EXPECT_EQ(1, msgs[index].size());
+    EXPECT_EQ(ViewMsg_Resource_RequestComplete::ID, msgs[index][0].type());
+
+    // The RequestComplete message should have had status
+    // (CANCELLED, ERR_INSUFFICIENT_RESOURCES).
+    int request_id;
+    URLRequestStatus status;
+
+    void* iter = NULL;
+    EXPECT_TRUE(IPC::ReadParam(&msgs[index][0], &iter, &request_id));
+    EXPECT_TRUE(IPC::ReadParam(&msgs[index][0], &iter, &status));
+
+    EXPECT_EQ(index + 1, request_id);
+    EXPECT_EQ(URLRequestStatus::CANCELED, status.status());
+    EXPECT_EQ(net::ERR_INSUFFICIENT_RESOURCES, status.os_error());
+  }
+
+  // The final 2 requests should have succeeded.
+  CheckSuccessfulRequest(msgs[kMaxRequests + 2],
+                         URLRequestTestJob::test_data_2());
+  CheckSuccessfulRequest(msgs[kMaxRequests + 3],
+                         URLRequestTestJob::test_data_2());
+}
+