DNS Host resolver changes with retry logic. Fix for

bug Chromium cannot recover from a state when its 
DNS requests have been dropped.

Whenever we try to resolve the host, we post a delayed task
to check if host resolution (OnLookupComplete) is completed 
or not. If the original ateempt hasn't completed, then we start
another attempt to resolve for the same request.

We take the results from the attempt that finishes first and
leave all other attempts as orphaned.

BUG=73327
TEST=dns host resolver tests
R=eroman,jar
Review URL: http://codereview.chromium.org/6782001

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@83641 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 412 insertions, 49 deletions

diff --git a/base/metrics/histogram_unittest.cc b/base/metrics/histogram_unittest.cc
index 496ff63..9d671df 100644
--- a/base/metrics/histogram_unittest.cc
+++ b/base/metrics/histogram_unittest.cc
@@ -288,6 +288,28 @@ TEST(HistogramTest, BoundsTest) {
   EXPECT_EQ(kBucketCount, array_size);
   EXPECT_EQ(0, sample.counts(array_size - 2));
   EXPECT_EQ(2, sample.counts(array_size - 1));
+
+  std::vector<int> custom_ranges;
+  custom_ranges.push_back(10);
+  custom_ranges.push_back(50);
+  custom_ranges.push_back(100);
+  Histogram* test_custom_histogram(CustomHistogram::FactoryGet(
+      "TestCustomRangeBoundedHistogram", custom_ranges, Histogram::kNoFlags));
+
+  // Put two samples "out of bounds" above and below.
+  test_custom_histogram->Add(5);
+  test_custom_histogram->Add(-50);
+  test_custom_histogram->Add(100);
+  test_custom_histogram->Add(1000);
+
+  // Verify they landed in the underflow, and overflow buckets.
+  Histogram::SampleSet custom_sample;
+  test_custom_histogram->SnapshotSample(&custom_sample);
+  EXPECT_EQ(2, custom_sample.counts(0));
+  EXPECT_EQ(0, custom_sample.counts(1));
+  size_t custom_array_size = test_custom_histogram->bucket_count();
+  EXPECT_EQ(0, custom_sample.counts(custom_array_size - 2));
+  EXPECT_EQ(2, custom_sample.counts(custom_array_size - 1));
 }
 
 // Check to be sure samples land as expected is "correct" buckets.
diff --git a/net/base/host_resolver_impl.cc b/net/base/host_resolver_impl.cc
index b512561..f606c65 100644
--- a/net/base/host_resolver_impl.cc
+++ b/net/base/host_resolver_impl.cc
@@ -24,6 +24,7 @@
 #include "base/stl_util-inl.h"
 #include "base/string_util.h"
 #include "base/synchronization/lock.h"
+#include "base/task.h"
 #include "base/threading/worker_pool.h"
 #include "base/time.h"
 #include "base/utf_string_conversions.h"
@@ -71,6 +72,7 @@ HostCache* CreateDefaultCache() {
 
 }  // anonymous namespace
 
+// static
 HostResolver* CreateSystemHostResolver(size_t max_concurrent_resolves,
                                        NetLog* net_log) {
   // Maximum of 8 concurrent resolver threads.
@@ -352,8 +354,10 @@ class HostResolverImpl::Job
        resolver_(resolver),
        origin_loop_(MessageLoop::current()),
        resolver_proc_(resolver->effective_resolver_proc()),
-       error_(OK),
-       os_error_(0),
+       unresponsive_delay_(resolver->unresponsive_delay()),
+       attempt_number_(0),
+       completed_attempt_number_(0),
+       completed_attempt_error_(ERR_UNEXPECTED),
        had_non_speculative_request_(false),
        net_log_(BoundNetLog::Make(net_log,
                                   NetLog::SOURCE_HOST_RESOLVER_IMPL_JOB)) {
@@ -380,23 +384,44 @@ class HostResolverImpl::Job
 
   // Called from origin loop.
   void Start() {
-    start_time_ = base::TimeTicks::Now();
+    StartLookupAttempt();
+  }
 
-    // Dispatch the job to a worker thread.
-    if (!base::WorkerPool::PostTask(FROM_HERE,
-            NewRunnableMethod(this, &Job::DoLookup), true)) {
+  // Called from origin loop.
+  void StartLookupAttempt() {
+    base::TimeTicks start_time = base::TimeTicks::Now();
+    ++attempt_number_;
+    // Dispatch the lookup attempt to a worker thread.
+    if (!base::WorkerPool::PostTask(
+            FROM_HERE,
+            NewRunnableMethod(this, &Job::DoLookup, start_time,
+                              attempt_number_),
+            true)) {
       NOTREACHED();
 
       // Since we could be running within Resolve() right now, we can't just
       // call OnLookupComplete().  Instead we must wait until Resolve() has
       // returned (IO_PENDING).
-      error_ = ERR_UNEXPECTED;
       MessageLoop::current()->PostTask(
-          FROM_HERE, NewRunnableMethod(this, &Job::OnLookupComplete));
+          FROM_HERE,
+          NewRunnableMethod(this, &Job::OnLookupComplete, AddressList(),
+                            start_time, attempt_number_, ERR_UNEXPECTED, 0));
+      return;
     }
-  }
-
-  // Cancels the current job. Callable from origin thread.
+    // Post a task to check if we get the results within a given time.
+    // OnCheckForComplete has the potential for starting a new attempt on a
+    // different worker thread if none of our outstanding attempts have
+    // completed yet.
+    MessageLoop::current()->PostDelayedTask(
+        FROM_HERE,
+        NewRunnableMethod(this, &Job::OnCheckForComplete),
+        unresponsive_delay_.InMilliseconds());
+  }
+
+  // Cancels the current job. The Job will be orphaned. Any outstanding resolve
+  // attempts running on worker threads will continue running. Only once all the
+  // attempts complete will the final reference to this Job be released.
+  // Callable from origin thread.
   void Cancel() {
     net_log_.AddEvent(NetLog::TYPE_CANCELLED, NULL);
 
@@ -430,6 +455,11 @@ class HostResolverImpl::Job
   }
 
   // Called from origin thread.
+  bool was_completed() const {
+    return completed_attempt_number_ > 0;
+  }
+
+  // Called from origin thread.
   const Key& key() const {
     return key_;
   }
@@ -438,10 +468,6 @@ class HostResolverImpl::Job
     return id_;
   }
 
-  base::TimeTicks start_time() const {
-    return start_time_;
-  }
-
   // Called from origin thread.
   const RequestsList& requests() const {
     return requests_;
@@ -470,49 +496,100 @@ class HostResolverImpl::Job
   // WARNING: This code runs inside a worker pool. The shutdown code cannot
   // wait for it to finish, so we must be very careful here about using other
   // objects (like MessageLoops, Singletons, etc). During shutdown these objects
-  // may no longer exist.
-  void DoLookup() {
+  // may no longer exist. Multiple DoLookups() could be running in parallel, so
+  // any state inside of |this| must not mutate .
+  void DoLookup(const base::TimeTicks& start_time,
+                const uint32 attempt_number) {
+    AddressList results;
+    int os_error;
     // Running on the worker thread
-    error_ = ResolveAddrInfo(resolver_proc_,
-                             key_.hostname,
-                             key_.address_family,
-                             key_.host_resolver_flags,
-                             &results_,
-                             &os_error_);
+    int error = ResolveAddrInfo(resolver_proc_,
+                                key_.hostname,
+                                key_.address_family,
+                                key_.host_resolver_flags,
+                                &results,
+                                &os_error);
 
     // The origin loop could go away while we are trying to post to it, so we
     // need to call its PostTask method inside a lock.  See ~HostResolver.
     {
       base::AutoLock locked(origin_loop_lock_);
       if (origin_loop_) {
-        origin_loop_->PostTask(FROM_HERE,
-                               NewRunnableMethod(this, &Job::OnLookupComplete));
+        origin_loop_->PostTask(
+            FROM_HERE,
+            NewRunnableMethod(this, &Job::OnLookupComplete, results, start_time,
+                              attempt_number, error, os_error));
       }
     }
   }
 
+  // Callback to see if DoLookup has finished or not (runs on origin thread).
+  void OnCheckForComplete() {
+    if (was_cancelled() || was_completed())
+      return;
+
+    DCHECK(resolver_);
+    base::TimeDelta unresponsive_delay =
+        unresponsive_delay_ * resolver_->retry_factor();
+    if (unresponsive_delay >= resolver_->maximum_unresponsive_delay())
+      return;
+
+    unresponsive_delay_ = unresponsive_delay;
+    StartLookupAttempt();
+  }
+
   // Callback for when DoLookup() completes (runs on origin thread).
-  void OnLookupComplete() {
+  void OnLookupComplete(const AddressList& results,
+                        const base::TimeTicks& start_time,
+                        const uint32 attempt_number,
+                        int error,
+                        const int os_error) {
     // Should be running on origin loop.
     // TODO(eroman): this is being hit by URLRequestTest.CancelTest*,
     // because MessageLoop::current() == NULL.
     //DCHECK_EQ(origin_loop_, MessageLoop::current());
-    DCHECK(error_ || results_.head());
+    DCHECK(error || results.head());
+
+    bool was_retry_attempt = attempt_number > 1;
+
+    if (!was_cancelled()) {
+      // If host is already resolved, then record data and return.
+      if (was_completed()) {
+        // If this is the first attempt that is finishing later, then record
+        // data for the first attempt. Won't contaminate with retry attempt's
+        // data.
+        if (!was_retry_attempt)
+          RecordPerformanceHistograms(start_time, error, os_error);
+
+        RecordAttemptHistograms(start_time, attempt_number, error, os_error);
+        return;
+      }
+
+      // Copy the results from the first worker thread that resolves the host.
+      results_ = results;
+      completed_attempt_number_ = attempt_number;
+      completed_attempt_error_ = error;
+    }
 
     // Ideally the following code would be part of host_resolver_proc.cc,
     // however it isn't safe to call NetworkChangeNotifier from worker
     // threads. So we do it here on the IO thread instead.
-    if (error_ != OK && NetworkChangeNotifier::IsOffline())
-      error_ = ERR_INTERNET_DISCONNECTED;
+    if (error != OK && NetworkChangeNotifier::IsOffline())
+      error = ERR_INTERNET_DISCONNECTED;
+
+    // We will record data for the first attempt. Don't contaminate with retry
+    // attempt's data.
+    if (!was_retry_attempt)
+      RecordPerformanceHistograms(start_time, error, os_error);
 
-    RecordPerformanceHistograms();
+    RecordAttemptHistograms(start_time, attempt_number, error, os_error);
 
     if (was_cancelled())
       return;
 
     scoped_refptr<NetLog::EventParameters> params;
-    if (error_ != OK) {
-      params = new HostResolveFailedParams(error_, os_error_);
+    if (error != OK) {
+      params = new HostResolveFailedParams(error, os_error);
     } else {
       params = new AddressListNetLogParam(results_);
     }
@@ -524,13 +601,15 @@ class HostResolverImpl::Job
     DCHECK(!requests_.empty());
 
      // Use the port number of the first request.
-    if (error_ == OK)
+    if (error == OK)
       results_.SetPort(requests_[0]->port());
 
-    resolver_->OnJobComplete(this, error_, os_error_, results_);
+    resolver_->OnJobComplete(this, error, os_error, results_);
   }
 
-  void RecordPerformanceHistograms() const {
+  void RecordPerformanceHistograms(const base::TimeTicks& start_time,
+                                   const int error,
+                                   const int os_error) const {
     enum Category {  // Used in HISTOGRAM_ENUMERATION.
       RESOLVE_SUCCESS,
       RESOLVE_FAIL,
@@ -540,8 +619,8 @@ class HostResolverImpl::Job
     };
     int category = RESOLVE_MAX;  // Illegal value for later DCHECK only.
 
-    base::TimeDelta duration = base::TimeTicks::Now() - start_time_;
-    if (error_ == OK) {
+    base::TimeDelta duration = base::TimeTicks::Now() - start_time;
+    if (error == OK) {
       if (had_non_speculative_request_) {
         category = RESOLVE_SUCCESS;
         DNS_HISTOGRAM("DNS.ResolveSuccess", duration);
@@ -558,7 +637,7 @@ class HostResolverImpl::Job
         DNS_HISTOGRAM("DNS.ResolveSpeculativeFail", duration);
       }
       UMA_HISTOGRAM_CUSTOM_ENUMERATION(kOSErrorsForGetAddrinfoHistogramName,
-                                       std::abs(os_error_),
+                                       std::abs(os_error),
                                        GetAllGetAddrinfoOSErrors());
     }
     DCHECK_LT(category, static_cast<int>(RESOLVE_MAX));  // Be sure it was set.
@@ -591,7 +670,47 @@ class HostResolverImpl::Job
     }
   }
 
+  void RecordAttemptHistograms(const base::TimeTicks& start_time,
+                               const uint32 attempt_number,
+                               const int error,
+                               const int os_error) const {
+    bool first_attempt_to_complete =
+      completed_attempt_number_ == attempt_number;
+
+    if (first_attempt_to_complete) {
+      // If this was first attempt to complete, then record the resolution
+      // status of the attempt.
+      if (completed_attempt_error_ == OK) {
+        UMA_HISTOGRAM_ENUMERATION(
+            "DNS.AttemptFirstSuccess", attempt_number, 100);
+      } else {
+        UMA_HISTOGRAM_ENUMERATION(
+            "DNS.AttemptFirstFailure", attempt_number, 100);
+      }
+    }
+
+    if (error == OK)
+      UMA_HISTOGRAM_ENUMERATION("DNS.AttemptSuccess", attempt_number, 100);
+    else
+      UMA_HISTOGRAM_ENUMERATION("DNS.AttemptFailure", attempt_number, 100);
+
+    if (was_cancelled() || !first_attempt_to_complete) {
+      // Count those attempts which completed after the job was already canceled
+      // OR after the job was already completed by an earlier attempt (so in
+      // effect).
+      UMA_HISTOGRAM_ENUMERATION("DNS.AttemptDiscarded", attempt_number, 100);
+
+      // Record if job is cancelled.
+      if (was_cancelled())
+        UMA_HISTOGRAM_ENUMERATION("DNS.AttemptCancelled", attempt_number, 100);
+    }
 
+    base::TimeDelta duration = base::TimeTicks::Now() - start_time;
+    if (error == OK)
+      DNS_HISTOGRAM("DNS.AttemptSuccessDuration", duration);
+    else
+      DNS_HISTOGRAM("DNS.AttemptFailDuration", duration);
+  }
 
   // Immutable. Can be read from either thread,
   const int id_;
@@ -613,9 +732,21 @@ class HostResolverImpl::Job
   // reference ensures that it remains valid until we are done.
   scoped_refptr<HostResolverProc> resolver_proc_;
 
-  // Assigned on the worker thread, read on the origin thread.
-  int error_;
-  int os_error_;
+  // The amount of time after starting a resolution attempt until deciding to
+  // retry.
+  base::TimeDelta unresponsive_delay_;
+
+  // Keeps track of the number of attempts we have made so far to resolve the
+  // host. Whenever we start an attempt to resolve the host, we increase this
+  // number.
+  uint32 attempt_number_;
+
+  // The index of the attempt which finished first (or 0 if the job is still in
+  // progress).
+  uint32 completed_attempt_number_;
+
+  // The result (a net error code) from the first attempt to complete.
+  int completed_attempt_error_;
 
   // True if a non-speculative request was ever attached to this job
   // (regardless of whether or not it was later cancelled.
@@ -625,9 +756,6 @@ class HostResolverImpl::Job
 
   AddressList results_;
 
-  // The time when the job was started.
-  base::TimeTicks start_time_;
-
   BoundNetLog net_log_;
 
   DISALLOW_COPY_AND_ASSIGN(Job);
@@ -903,6 +1031,9 @@ HostResolverImpl::HostResolverImpl(
     NetLog* net_log)
     : cache_(cache),
       max_jobs_(max_jobs),
+      unresponsive_delay_(base::TimeDelta::FromMilliseconds(6000)),
+      retry_factor_(2),
+      maximum_unresponsive_delay_(base::TimeDelta::FromMilliseconds(60000)),
       next_request_id_(0),
       next_job_id_(0),
       resolver_proc_(resolver_proc),
diff --git a/net/base/host_resolver_impl.h b/net/base/host_resolver_impl.h
index 54e0b9e..140afa3 100644
--- a/net/base/host_resolver_impl.h
+++ b/net/base/host_resolver_impl.h
@@ -8,8 +8,11 @@
 
 #include <vector>
 
+#include "base/basictypes.h"
+#include "base/gtest_prod_util.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/threading/non_thread_safe.h"
+#include "base/time.h"
 #include "net/base/capturing_net_log.h"
 #include "net/base/host_cache.h"
 #include "net/base/host_resolver.h"
@@ -48,6 +51,12 @@ namespace net {
 // threads.
 //
 // Requests are ordered in the queue based on their priority.
+//
+// Whenever we try to resolve the host, we post a delayed task to check if host
+// resolution (OnLookupComplete) is completed or not. If the original attempt
+// hasn't completed, then we start another attempt for host resolution. We take
+// the results from the first attempt that finishes and ignore the results from
+// all other attempts.
 
 class HostResolverImpl : public HostResolver,
                          public base::NonThreadSafe,
@@ -74,7 +83,16 @@ class HostResolverImpl : public HostResolver,
   // |resolver_proc| is NULL then the default host resolver procedure is
   // used (which is SystemHostResolverProc except if overridden).
   // |max_jobs| specifies the maximum number of threads that the host resolver
-  // will use. Use SetPoolConstraints() to specify finer-grain settings.
+  // will use (not counting potential duplicate attempts). Use
+  // SetPoolConstraints() to specify finer-grain settings.
+  //
+  // For each attempt, we could start another attempt if host is not resolved
+  // within unresponsive_delay_ time. We keep attempting to resolve the host
+  // until retry interval reaches maximum_unresponsive_delay_ time. For every
+  // retry attempt, we grow the unresponsive_delay_ by the retry_factor_ amount
+  // (that is retry interval is multiplied by the retry factor each time). Once
+  // retry interval exceeds maximum_unresponsive_delay_ time, we give up on
+  // additional attempts.
   //
   // |net_log| must remain valid for the life of the HostResolverImpl.
   HostResolverImpl(HostResolverProc* resolver_proc,
@@ -129,6 +147,12 @@ class HostResolverImpl : public HostResolver,
   virtual void Shutdown();
 
  private:
+  // Allow tests to access our innards for testing purposes.
+  friend class LookupAttemptHostResolverProc;
+
+  // Allow tests to access our innards for testing purposes.
+  FRIEND_TEST_ALL_PREFIXES(HostResolverImplTest, MultipleAttempts);
+
   class Job;
   class JobPool;
   class IPv6ProbeJob;
@@ -226,15 +250,53 @@ class HostResolverImpl : public HostResolver,
   // NetworkChangeNotifier::IPAddressObserver methods:
   virtual void OnIPAddressChanged();
 
+  // Helper methods for unit tests to get and set unresponsive_delay_.
+  base::TimeDelta unresponsive_delay() const { return unresponsive_delay_; }
+  void set_unresponsive_delay(const base::TimeDelta& unresponsive_delay) {
+    unresponsive_delay_ = unresponsive_delay;
+  }
+
+  // Helper methods to get and set retry_factor.
+  uint32 retry_factor() const {
+    return retry_factor_;
+  }
+  void set_retry_factor(const uint32 retry_factor) {
+    retry_factor_ = retry_factor;
+  }
+
+  // Helper methods for unit tests to get and set maximum_unresponsive_delay_.
+  base::TimeDelta maximum_unresponsive_delay() const {
+    return maximum_unresponsive_delay_;
+  }
+  void set_maximum_unresponsive_delay(
+      const base::TimeDelta& maximum_unresponsive_delay) {
+    maximum_unresponsive_delay_ = maximum_unresponsive_delay;
+  }
+
   // Cache of host resolution results.
   scoped_ptr<HostCache> cache_;
 
   // Map from hostname to outstanding job.
   JobMap jobs_;
 
-  // Maximum number of concurrent jobs allowed, across all pools.
+  // Maximum number of concurrent jobs allowed, across all pools. Each job may
+  // create multiple concurrent resolve attempts for the hostname.
   size_t max_jobs_;
 
+  // This is the limit after which we make another attempt to resolve the host
+  // if the worker thread has not responded yet. Allow unit tests to change the
+  // value.
+  base::TimeDelta unresponsive_delay_;
+
+  // Factor to grow unresponsive_delay_ when we re-re-try. Allow unit tests to
+  // change the value.
+  uint32 retry_factor_;
+
+  // This is the limit on how large we grow the retry interval. Once it exceeds
+  // this, we give up on additional attempts. Allow unit tests to change the
+  // value.
+  base::TimeDelta maximum_unresponsive_delay_;
+
   // The information to track pending requests for a JobPool, as well as
   // how many outstanding jobs the pool already has, and its constraints.
   JobPool* job_pools_[POOL_COUNT];
diff --git a/net/base/host_resolver_impl_unittest.cc b/net/base/host_resolver_impl_unittest.cc
index 1093cdc..9893028 100644
--- a/net/base/host_resolver_impl_unittest.cc
+++ b/net/base/host_resolver_impl_unittest.cc
@@ -11,6 +11,9 @@
 #include "base/message_loop.h"
 #include "base/string_util.h"
 #include "base/stringprintf.h"
+#include "base/synchronization/condition_variable.h"
+#include "base/synchronization/lock.h"
+#include "base/time.h"
 #include "net/base/address_list.h"
 #include "net/base/completion_callback.h"
 #include "net/base/mock_host_resolver.h"
@@ -27,7 +30,8 @@
 
 namespace net {
 
-namespace {
+using base::TimeDelta;
+using base::TimeTicks;
 
 HostCache* CreateDefaultCache() {
   return new HostCache(
@@ -150,6 +154,107 @@ class EchoingHostResolverProc : public HostResolverProc {
   }
 };
 
+// Using LookupAttemptHostResolverProc simulate very long lookups, and control
+// which attempt resolves the host.
+class LookupAttemptHostResolverProc : public HostResolverProc {
+ public:
+  LookupAttemptHostResolverProc(HostResolverProc* previous,
+                                int attempt_number_to_resolve,
+                                int total_attempts)
+      : HostResolverProc(previous),
+        attempt_number_to_resolve_(attempt_number_to_resolve),
+        current_attempt_number_(0),
+        total_attempts_(total_attempts),
+        total_attempts_resolved_(0),
+        resolved_attempt_number_(0),
+        all_done_(&lock_) {
+  }
+
+  // Test harness will wait for all attempts to finish before checking the
+  // results.
+  void WaitForAllAttemptsToFinish(const TimeDelta& wait_time) {
+    TimeTicks end_time = TimeTicks::Now() + wait_time;
+    {
+      base::AutoLock auto_lock(lock_);
+      while (total_attempts_resolved_ != total_attempts_ &&
+             TimeTicks::Now() < end_time) {
+        all_done_.TimedWait(end_time - TimeTicks::Now());
+      }
+    }
+  }
+
+  // All attempts will wait for an attempt to resolve the host.
+  void WaitForAnAttemptToComplete() {
+    TimeDelta wait_time = TimeDelta::FromMilliseconds(60000);
+    TimeTicks end_time = TimeTicks::Now() + wait_time;
+    {
+      base::AutoLock auto_lock(lock_);
+      while (resolved_attempt_number_ == 0 && TimeTicks::Now() < end_time)
+        all_done_.TimedWait(end_time - TimeTicks::Now());
+    }
+    all_done_.Broadcast();  // Tell all waiting attempts to proceed.
+  }
+
+  // Returns the number of attempts that have finished the Resolve() method.
+  int total_attempts_resolved() { return total_attempts_resolved_; }
+
+  // Returns the first attempt that that has resolved the host.
+  int resolved_attempt_number() { return resolved_attempt_number_; }
+
+  // HostResolverProc methods.
+  virtual int Resolve(const std::string& host,
+                      AddressFamily address_family,
+                      HostResolverFlags host_resolver_flags,
+                      AddressList* addrlist,
+                      int* os_error) {
+    bool wait_for_right_attempt_to_complete = true;
+    {
+      base::AutoLock auto_lock(lock_);
+      ++current_attempt_number_;
+      if (current_attempt_number_ == attempt_number_to_resolve_) {
+        resolved_attempt_number_ = current_attempt_number_;
+        wait_for_right_attempt_to_complete = false;
+      }
+    }
+
+    if (wait_for_right_attempt_to_complete)
+      // Wait for the attempt_number_to_resolve_ attempt to resolve.
+      WaitForAnAttemptToComplete();
+
+    int result = ResolveUsingPrevious(host, address_family, host_resolver_flags,
+                                      addrlist, os_error);
+
+    {
+      base::AutoLock auto_lock(lock_);
+      ++total_attempts_resolved_;
+    }
+
+    all_done_.Broadcast();  // Tell all attempts to proceed.
+
+    // Since any negative number is considered a network error, with -1 having
+    // special meaning (ERR_IO_PENDING). We could return the attempt that has
+    // resolved the host as a negative number. For example, if attempt number 3
+    // resolves the host, then this method returns -4.
+    if (result == OK)
+      return -1 - resolved_attempt_number_;
+    else
+      return result;
+  }
+
+ private:
+  virtual ~LookupAttemptHostResolverProc() {}
+
+  int attempt_number_to_resolve_;
+  int current_attempt_number_;  // Incremented whenever Resolve is called.
+  int total_attempts_;
+  int total_attempts_resolved_;
+  int resolved_attempt_number_;
+
+  // All attempts wait for right attempt to be resolve.
+  base::Lock lock_;
+  base::ConditionVariable all_done_;
+};
+
 // Helper that represents a single Resolve() result, used to inspect all the
 // resolve results by forwarding them to Delegate.
 class ResolveRequest {
@@ -1694,8 +1799,51 @@ TEST_F(HostResolverImplTest, DisallowNonCachedResponses) {
   EXPECT_TRUE(htons(kPortnum) == sa_in->sin_port);
   EXPECT_TRUE(htonl(0xc0a8012a) == sa_in->sin_addr.s_addr);
 }
-// TODO(cbentzel): Test a mix of requests with different HostResolverFlags.
 
-}  // namespace
+// Test the retry attempts simulating host resolver proc that takes too long.
+TEST_F(HostResolverImplTest, MultipleAttempts) {
+  // Total number of attempts would be 3 and we want the 3rd attempt to resolve
+  // the host. First and second attempt will be forced to sleep until they get
+  // word that a resolution has completed. The 3rd resolution attempt will try
+  // to get done ASAP, and won't sleep..
+  int kAttemptNumberToResolve = 3;
+  int kTotalAttempts = 3;
+
+  scoped_refptr<LookupAttemptHostResolverProc> resolver_proc(
+      new LookupAttemptHostResolverProc(
+          NULL, kAttemptNumberToResolve, kTotalAttempts));
+  HostCache* cache = CreateDefaultCache();
+  scoped_ptr<HostResolverImpl> host_resolver(
+      new HostResolverImpl(resolver_proc, cache, kMaxJobs, NULL));
+
+  // Specify smaller interval for unresponsive_delay_ and
+  // maximum_unresponsive_delay_ for HostResolverImpl so that unit test
+  // runs faster. For example, this test finishes in 1.5 secs (500ms * 3).
+  TimeDelta kUnresponsiveTime = TimeDelta::FromMilliseconds(500);
+  TimeDelta kMaximumUnresponsiveTime = TimeDelta::FromMilliseconds(2500);
+
+  host_resolver->set_unresponsive_delay(kUnresponsiveTime);
+  host_resolver->set_maximum_unresponsive_delay(kMaximumUnresponsiveTime);
+
+  // Resolve "host1".
+  HostResolver::RequestInfo info(HostPortPair("host1", 70));
+  TestCompletionCallback callback;
+  AddressList addrlist;
+  int rv = host_resolver->Resolve(info, &addrlist, &callback, NULL,
+                                  BoundNetLog());
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+
+  // Resolve returns -4 to indicate that 3rd attempt has resolved the host.
+  EXPECT_EQ(-4, callback.WaitForResult());
+
+  resolver_proc->WaitForAllAttemptsToFinish(TimeDelta::FromMilliseconds(60000));
+  MessageLoop::current()->RunAllPending();
+
+  EXPECT_EQ(resolver_proc->total_attempts_resolved(), kTotalAttempts);
+  EXPECT_EQ(resolver_proc->resolved_attempt_number(), kAttemptNumberToResolve);
+}
+
+
+// TODO(cbentzel): Test a mix of requests with different HostResolverFlags.
 
 }  // namespace net