Revert of 83710 which reverted 83641 - DNS Host resolver changes

with retry logic. Fix for bug Chromium cannot recover from a state
when its DNS requests have been dropped.

Whenever we try to resolve the host, we post a delayed task
to check if host resolution (OnLookupComplete) is completed 
or not. If it hasn't completed, then we start a new job 
to resolve for the same request.

BUG=73327
TEST=dns host resolver tests
R=eroman,jar
Review URL: http://codereview.chromium.org/6976006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@84908 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 388 insertions, 48 deletions

diff --git a/net/base/host_resolver_impl.cc b/net/base/host_resolver_impl.cc
index 8c9d17f..008fd14 100644
--- a/net/base/host_resolver_impl.cc
+++ b/net/base/host_resolver_impl.cc
@@ -23,6 +23,7 @@
 #include "base/metrics/histogram.h"
 #include "base/stl_util-inl.h"
 #include "base/string_util.h"
+#include "base/task.h"
 #include "base/threading/worker_pool.h"
 #include "base/time.h"
 #include "base/utf_string_conversions.h"
@@ -70,6 +71,7 @@ HostCache* CreateDefaultCache() {
 
 }  // anonymous namespace
 
+// static
 HostResolver* CreateSystemHostResolver(size_t max_concurrent_resolves,
                                        NetLog* net_log) {
   // Maximum of 8 concurrent resolver threads.
@@ -351,8 +353,10 @@ class HostResolverImpl::Job
        resolver_(resolver),
        origin_loop_(base::MessageLoopProxy::CreateForCurrentThread()),
        resolver_proc_(resolver->effective_resolver_proc()),
-       error_(OK),
-       os_error_(0),
+       unresponsive_delay_(resolver->unresponsive_delay()),
+       attempt_number_(0),
+       completed_attempt_number_(0),
+       completed_attempt_error_(ERR_UNEXPECTED),
        had_non_speculative_request_(false),
        net_log_(BoundNetLog::Make(net_log,
                                   NetLog::SOURCE_HOST_RESOLVER_IMPL_JOB)) {
@@ -381,23 +385,43 @@ class HostResolverImpl::Job
 
   void Start() {
     DCHECK(origin_loop_->BelongsToCurrentThread());
-    start_time_ = base::TimeTicks::Now();
+    StartLookupAttempt();
+  }
 
-    // Dispatch the job to a worker thread.
-    if (!base::WorkerPool::PostTask(FROM_HERE,
-            NewRunnableMethod(this, &Job::DoLookup), true)) {
+  void StartLookupAttempt() {
+    DCHECK(origin_loop_->BelongsToCurrentThread());
+    base::TimeTicks start_time = base::TimeTicks::Now();
+    ++attempt_number_;
+    // Dispatch the lookup attempt to a worker thread.
+    if (!base::WorkerPool::PostTask(
+            FROM_HERE,
+            NewRunnableMethod(this, &Job::DoLookup, start_time,
+                              attempt_number_),
+            true)) {
       NOTREACHED();
 
       // Since we could be running within Resolve() right now, we can't just
       // call OnLookupComplete().  Instead we must wait until Resolve() has
       // returned (IO_PENDING).
-      error_ = ERR_UNEXPECTED;
       origin_loop_->PostTask(
-          FROM_HERE, NewRunnableMethod(this, &Job::OnLookupComplete));
+          FROM_HERE,
+          NewRunnableMethod(this, &Job::OnLookupComplete, AddressList(),
+                            start_time, attempt_number_, ERR_UNEXPECTED, 0));
+      return;
     }
+    // Post a task to check if we get the results within a given time.
+    // OnCheckForComplete has the potential for starting a new attempt on a
+    // different worker thread if none of our outstanding attempts have
+    // completed yet.
+    origin_loop_->PostDelayedTask(
+        FROM_HERE,
+        NewRunnableMethod(this, &Job::OnCheckForComplete),
+        unresponsive_delay_.InMilliseconds());
   }
 
-  // Cancels the current job.
+  // Cancels the current job. The Job will be orphaned. Any outstanding resolve
+  // attempts running on worker threads will continue running. Only once all the
+  // attempts complete will the final reference to this Job be released.
   void Cancel() {
     DCHECK(origin_loop_->BelongsToCurrentThread());
     net_log_.AddEvent(NetLog::TYPE_CANCELLED, NULL);
@@ -424,6 +448,11 @@ class HostResolverImpl::Job
     return resolver_ == NULL;
   }
 
+  bool was_completed() const {
+    DCHECK(origin_loop_->BelongsToCurrentThread());
+    return completed_attempt_number_ > 0;
+  }
+
   const Key& key() const {
     DCHECK(origin_loop_->BelongsToCurrentThread());
     return key_;
@@ -434,11 +463,6 @@ class HostResolverImpl::Job
     return id_;
   }
 
-  base::TimeTicks start_time() const {
-    DCHECK(origin_loop_->BelongsToCurrentThread());
-    return start_time_;
-  }
-
   const RequestsList& requests() const {
     DCHECK(origin_loop_->BelongsToCurrentThread());
     return requests_;
@@ -468,39 +492,92 @@ class HostResolverImpl::Job
   // WARNING: This code runs inside a worker pool. The shutdown code cannot
   // wait for it to finish, so we must be very careful here about using other
   // objects (like MessageLoops, Singletons, etc). During shutdown these objects
-  // may no longer exist.
-  void DoLookup() {
+  // may no longer exist. Multiple DoLookups() could be running in parallel, so
+  // any state inside of |this| must not mutate .
+  void DoLookup(const base::TimeTicks& start_time,
+                const uint32 attempt_number) {
+    AddressList results;
+    int os_error = 0;
     // Running on the worker thread
-    error_ = ResolveAddrInfo(resolver_proc_,
-                             key_.hostname,
-                             key_.address_family,
-                             key_.host_resolver_flags,
-                             &results_,
-                             &os_error_);
+    int error = ResolveAddrInfo(resolver_proc_,
+                                key_.hostname,
+                                key_.address_family,
+                                key_.host_resolver_flags,
+                                &results,
+                                &os_error);
+
+    origin_loop_->PostTask(
+        FROM_HERE,
+        NewRunnableMethod(this, &Job::OnLookupComplete, results, start_time,
+                          attempt_number, error, os_error));
+  }
+
+  // Callback to see if DoLookup() has finished or not (runs on origin thread).
+  void OnCheckForComplete() {
+    DCHECK(origin_loop_->BelongsToCurrentThread());
+
+    if (was_cancelled() || was_completed())
+      return;
+
+    DCHECK(resolver_);
+    base::TimeDelta unresponsive_delay =
+        unresponsive_delay_ * resolver_->retry_factor();
+    if (unresponsive_delay >= resolver_->maximum_unresponsive_delay())
+      return;
 
-    origin_loop_->PostTask(FROM_HERE,
-                           NewRunnableMethod(this, &Job::OnLookupComplete));
+    unresponsive_delay_ = unresponsive_delay;
+    StartLookupAttempt();
   }
 
   // Callback for when DoLookup() completes (runs on origin thread).
-  void OnLookupComplete() {
+  void OnLookupComplete(const AddressList& results,
+                        const base::TimeTicks& start_time,
+                        const uint32 attempt_number,
+                        int error,
+                        const int os_error) {
     DCHECK(origin_loop_->BelongsToCurrentThread());
-    DCHECK(error_ || results_.head());
+    DCHECK(error || results.head());
+
+    bool was_retry_attempt = attempt_number > 1;
+
+    if (!was_cancelled()) {
+      // If host is already resolved, then record data and return.
+      if (was_completed()) {
+        // If this is the first attempt that is finishing later, then record
+        // data for the first attempt. Won't contaminate with retry attempt's
+        // data.
+        if (!was_retry_attempt)
+          RecordPerformanceHistograms(start_time, error, os_error);
+
+        RecordAttemptHistograms(start_time, attempt_number, error, os_error);
+        return;
+      }
+
+      // Copy the results from the first worker thread that resolves the host.
+      results_ = results;
+      completed_attempt_number_ = attempt_number;
+      completed_attempt_error_ = error;
+    }
 
     // Ideally the following code would be part of host_resolver_proc.cc,
     // however it isn't safe to call NetworkChangeNotifier from worker
     // threads. So we do it here on the IO thread instead.
-    if (error_ != OK && NetworkChangeNotifier::IsOffline())
-      error_ = ERR_INTERNET_DISCONNECTED;
+    if (error != OK && NetworkChangeNotifier::IsOffline())
+      error = ERR_INTERNET_DISCONNECTED;
+
+    // We will record data for the first attempt. Don't contaminate with retry
+    // attempt's data.
+    if (!was_retry_attempt)
+      RecordPerformanceHistograms(start_time, error, os_error);
 
-    RecordPerformanceHistograms();
+    RecordAttemptHistograms(start_time, attempt_number, error, os_error);
 
     if (was_cancelled())
       return;
 
     scoped_refptr<NetLog::EventParameters> params;
-    if (error_ != OK) {
-      params = new HostResolveFailedParams(error_, os_error_);
+    if (error != OK) {
+      params = new HostResolveFailedParams(error, os_error);
     } else {
       params = new AddressListNetLogParam(results_);
     }
@@ -512,13 +589,15 @@ class HostResolverImpl::Job
     DCHECK(!requests_.empty());
 
      // Use the port number of the first request.
-    if (error_ == OK)
+    if (error == OK)
       results_.SetPort(requests_[0]->port());
 
-    resolver_->OnJobComplete(this, error_, os_error_, results_);
+    resolver_->OnJobComplete(this, error, os_error, results_);
   }
 
-  void RecordPerformanceHistograms() const {
+  void RecordPerformanceHistograms(const base::TimeTicks& start_time,
+                                   const int error,
+                                   const int os_error) const {
     DCHECK(origin_loop_->BelongsToCurrentThread());
     enum Category {  // Used in HISTOGRAM_ENUMERATION.
       RESOLVE_SUCCESS,
@@ -529,8 +608,8 @@ class HostResolverImpl::Job
     };
     int category = RESOLVE_MAX;  // Illegal value for later DCHECK only.
 
-    base::TimeDelta duration = base::TimeTicks::Now() - start_time_;
-    if (error_ == OK) {
+    base::TimeDelta duration = base::TimeTicks::Now() - start_time;
+    if (error == OK) {
       if (had_non_speculative_request_) {
         category = RESOLVE_SUCCESS;
         DNS_HISTOGRAM("DNS.ResolveSuccess", duration);
@@ -547,7 +626,7 @@ class HostResolverImpl::Job
         DNS_HISTOGRAM("DNS.ResolveSpeculativeFail", duration);
       }
       UMA_HISTOGRAM_CUSTOM_ENUMERATION(kOSErrorsForGetAddrinfoHistogramName,
-                                       std::abs(os_error_),
+                                       std::abs(os_error),
                                        GetAllGetAddrinfoOSErrors());
     }
     DCHECK_LT(category, static_cast<int>(RESOLVE_MAX));  // Be sure it was set.
@@ -578,7 +657,47 @@ class HostResolverImpl::Job
     }
   }
 
+  void RecordAttemptHistograms(const base::TimeTicks& start_time,
+                               const uint32 attempt_number,
+                               const int error,
+                               const int os_error) const {
+    bool first_attempt_to_complete =
+        completed_attempt_number_ == attempt_number;
 
+    if (first_attempt_to_complete) {
+      // If this was first attempt to complete, then record the resolution
+      // status of the attempt.
+      if (completed_attempt_error_ == OK) {
+        UMA_HISTOGRAM_ENUMERATION(
+            "DNS.AttemptFirstSuccess", attempt_number, 100);
+      } else {
+        UMA_HISTOGRAM_ENUMERATION(
+            "DNS.AttemptFirstFailure", attempt_number, 100);
+      }
+    }
+
+    if (error == OK)
+      UMA_HISTOGRAM_ENUMERATION("DNS.AttemptSuccess", attempt_number, 100);
+    else
+      UMA_HISTOGRAM_ENUMERATION("DNS.AttemptFailure", attempt_number, 100);
+
+    if (was_cancelled() || !first_attempt_to_complete) {
+      // Count those attempts which completed after the job was already canceled
+      // OR after the job was already completed by an earlier attempt (so in
+      // effect).
+      UMA_HISTOGRAM_ENUMERATION("DNS.AttemptDiscarded", attempt_number, 100);
+
+      // Record if job is cancelled.
+      if (was_cancelled())
+        UMA_HISTOGRAM_ENUMERATION("DNS.AttemptCancelled", attempt_number, 100);
+    }
+
+    base::TimeDelta duration = base::TimeTicks::Now() - start_time;
+    if (error == OK)
+      DNS_HISTOGRAM("DNS.AttemptSuccessDuration", duration);
+    else
+      DNS_HISTOGRAM("DNS.AttemptFailDuration", duration);
+  }
 
   // Immutable. Can be read from either thread,
   const int id_;
@@ -599,9 +718,21 @@ class HostResolverImpl::Job
   // reference ensures that it remains valid until we are done.
   scoped_refptr<HostResolverProc> resolver_proc_;
 
-  // Assigned on the worker thread, read on the origin thread.
-  int error_;
-  int os_error_;
+  // The amount of time after starting a resolution attempt until deciding to
+  // retry.
+  base::TimeDelta unresponsive_delay_;
+
+  // Keeps track of the number of attempts we have made so far to resolve the
+  // host. Whenever we start an attempt to resolve the host, we increase this
+  // number.
+  uint32 attempt_number_;
+
+  // The index of the attempt which finished first (or 0 if the job is still in
+  // progress).
+  uint32 completed_attempt_number_;
+
+  // The result (a net error code) from the first attempt to complete.
+  int completed_attempt_error_;
 
   // True if a non-speculative request was ever attached to this job
   // (regardless of whether or not it was later cancelled.
@@ -611,9 +742,6 @@ class HostResolverImpl::Job
 
   AddressList results_;
 
-  // The time when the job was started.
-  base::TimeTicks start_time_;
-
   BoundNetLog net_log_;
 
   DISALLOW_COPY_AND_ASSIGN(Job);
@@ -862,6 +990,9 @@ HostResolverImpl::HostResolverImpl(
     NetLog* net_log)
     : cache_(cache),
       max_jobs_(max_jobs),
+      unresponsive_delay_(base::TimeDelta::FromMilliseconds(6000)),
+      retry_factor_(2),
+      maximum_unresponsive_delay_(base::TimeDelta::FromMilliseconds(60000)),
       next_request_id_(0),
       next_job_id_(0),
       resolver_proc_(resolver_proc),
diff --git a/net/base/host_resolver_impl.h b/net/base/host_resolver_impl.h
index 54e0b9e..140afa3 100644
--- a/net/base/host_resolver_impl.h
+++ b/net/base/host_resolver_impl.h
@@ -8,8 +8,11 @@
 
 #include <vector>
 
+#include "base/basictypes.h"
+#include "base/gtest_prod_util.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/threading/non_thread_safe.h"
+#include "base/time.h"
 #include "net/base/capturing_net_log.h"
 #include "net/base/host_cache.h"
 #include "net/base/host_resolver.h"
@@ -48,6 +51,12 @@ namespace net {
 // threads.
 //
 // Requests are ordered in the queue based on their priority.
+//
+// Whenever we try to resolve the host, we post a delayed task to check if host
+// resolution (OnLookupComplete) is completed or not. If the original attempt
+// hasn't completed, then we start another attempt for host resolution. We take
+// the results from the first attempt that finishes and ignore the results from
+// all other attempts.
 
 class HostResolverImpl : public HostResolver,
                          public base::NonThreadSafe,
@@ -74,7 +83,16 @@ class HostResolverImpl : public HostResolver,
   // |resolver_proc| is NULL then the default host resolver procedure is
   // used (which is SystemHostResolverProc except if overridden).
   // |max_jobs| specifies the maximum number of threads that the host resolver
-  // will use. Use SetPoolConstraints() to specify finer-grain settings.
+  // will use (not counting potential duplicate attempts). Use
+  // SetPoolConstraints() to specify finer-grain settings.
+  //
+  // For each attempt, we could start another attempt if host is not resolved
+  // within unresponsive_delay_ time. We keep attempting to resolve the host
+  // until retry interval reaches maximum_unresponsive_delay_ time. For every
+  // retry attempt, we grow the unresponsive_delay_ by the retry_factor_ amount
+  // (that is retry interval is multiplied by the retry factor each time). Once
+  // retry interval exceeds maximum_unresponsive_delay_ time, we give up on
+  // additional attempts.
   //
   // |net_log| must remain valid for the life of the HostResolverImpl.
   HostResolverImpl(HostResolverProc* resolver_proc,
@@ -129,6 +147,12 @@ class HostResolverImpl : public HostResolver,
   virtual void Shutdown();
 
  private:
+  // Allow tests to access our innards for testing purposes.
+  friend class LookupAttemptHostResolverProc;
+
+  // Allow tests to access our innards for testing purposes.
+  FRIEND_TEST_ALL_PREFIXES(HostResolverImplTest, MultipleAttempts);
+
   class Job;
   class JobPool;
   class IPv6ProbeJob;
@@ -226,15 +250,53 @@ class HostResolverImpl : public HostResolver,
   // NetworkChangeNotifier::IPAddressObserver methods:
   virtual void OnIPAddressChanged();
 
+  // Helper methods for unit tests to get and set unresponsive_delay_.
+  base::TimeDelta unresponsive_delay() const { return unresponsive_delay_; }
+  void set_unresponsive_delay(const base::TimeDelta& unresponsive_delay) {
+    unresponsive_delay_ = unresponsive_delay;
+  }
+
+  // Helper methods to get and set retry_factor.
+  uint32 retry_factor() const {
+    return retry_factor_;
+  }
+  void set_retry_factor(const uint32 retry_factor) {
+    retry_factor_ = retry_factor;
+  }
+
+  // Helper methods for unit tests to get and set maximum_unresponsive_delay_.
+  base::TimeDelta maximum_unresponsive_delay() const {
+    return maximum_unresponsive_delay_;
+  }
+  void set_maximum_unresponsive_delay(
+      const base::TimeDelta& maximum_unresponsive_delay) {
+    maximum_unresponsive_delay_ = maximum_unresponsive_delay;
+  }
+
   // Cache of host resolution results.
   scoped_ptr<HostCache> cache_;
 
   // Map from hostname to outstanding job.
   JobMap jobs_;
 
-  // Maximum number of concurrent jobs allowed, across all pools.
+  // Maximum number of concurrent jobs allowed, across all pools. Each job may
+  // create multiple concurrent resolve attempts for the hostname.
   size_t max_jobs_;
 
+  // This is the limit after which we make another attempt to resolve the host
+  // if the worker thread has not responded yet. Allow unit tests to change the
+  // value.
+  base::TimeDelta unresponsive_delay_;
+
+  // Factor to grow unresponsive_delay_ when we re-re-try. Allow unit tests to
+  // change the value.
+  uint32 retry_factor_;
+
+  // This is the limit on how large we grow the retry interval. Once it exceeds
+  // this, we give up on additional attempts. Allow unit tests to change the
+  // value.
+  base::TimeDelta maximum_unresponsive_delay_;
+
   // The information to track pending requests for a JobPool, as well as
   // how many outstanding jobs the pool already has, and its constraints.
   JobPool* job_pools_[POOL_COUNT];
diff --git a/net/base/host_resolver_impl_unittest.cc b/net/base/host_resolver_impl_unittest.cc
index 1093cdc..7b72d1d 100644
--- a/net/base/host_resolver_impl_unittest.cc
+++ b/net/base/host_resolver_impl_unittest.cc
@@ -11,6 +11,9 @@
 #include "base/message_loop.h"
 #include "base/string_util.h"
 #include "base/stringprintf.h"
+#include "base/synchronization/condition_variable.h"
+#include "base/synchronization/lock.h"
+#include "base/time.h"
 #include "net/base/address_list.h"
 #include "net/base/completion_callback.h"
 #include "net/base/mock_host_resolver.h"
@@ -27,7 +30,8 @@
 
 namespace net {
 
-namespace {
+using base::TimeDelta;
+using base::TimeTicks;
 
 HostCache* CreateDefaultCache() {
   return new HostCache(
@@ -150,6 +154,107 @@ class EchoingHostResolverProc : public HostResolverProc {
   }
 };
 
+// Using LookupAttemptHostResolverProc simulate very long lookups, and control
+// which attempt resolves the host.
+class LookupAttemptHostResolverProc : public HostResolverProc {
+ public:
+  LookupAttemptHostResolverProc(HostResolverProc* previous,
+                                int attempt_number_to_resolve,
+                                int total_attempts)
+      : HostResolverProc(previous),
+        attempt_number_to_resolve_(attempt_number_to_resolve),
+        current_attempt_number_(0),
+        total_attempts_(total_attempts),
+        total_attempts_resolved_(0),
+        resolved_attempt_number_(0),
+        all_done_(&lock_) {
+  }
+
+  // Test harness will wait for all attempts to finish before checking the
+  // results.
+  void WaitForAllAttemptsToFinish(const TimeDelta& wait_time) {
+    TimeTicks end_time = TimeTicks::Now() + wait_time;
+    {
+      base::AutoLock auto_lock(lock_);
+      while (total_attempts_resolved_ != total_attempts_ &&
+             TimeTicks::Now() < end_time) {
+        all_done_.TimedWait(end_time - TimeTicks::Now());
+      }
+    }
+  }
+
+  // All attempts will wait for an attempt to resolve the host.
+  void WaitForAnAttemptToComplete() {
+    TimeDelta wait_time = TimeDelta::FromMilliseconds(60000);
+    TimeTicks end_time = TimeTicks::Now() + wait_time;
+    {
+      base::AutoLock auto_lock(lock_);
+      while (resolved_attempt_number_ == 0 && TimeTicks::Now() < end_time)
+        all_done_.TimedWait(end_time - TimeTicks::Now());
+    }
+    all_done_.Broadcast();  // Tell all waiting attempts to proceed.
+  }
+
+  // Returns the number of attempts that have finished the Resolve() method.
+  int total_attempts_resolved() { return total_attempts_resolved_; }
+
+  // Returns the first attempt that that has resolved the host.
+  int resolved_attempt_number() { return resolved_attempt_number_; }
+
+  // HostResolverProc methods.
+  virtual int Resolve(const std::string& host,
+                      AddressFamily address_family,
+                      HostResolverFlags host_resolver_flags,
+                      AddressList* addrlist,
+                      int* os_error) {
+    bool wait_for_right_attempt_to_complete = true;
+    {
+      base::AutoLock auto_lock(lock_);
+      ++current_attempt_number_;
+      if (current_attempt_number_ == attempt_number_to_resolve_) {
+        resolved_attempt_number_ = current_attempt_number_;
+        wait_for_right_attempt_to_complete = false;
+      }
+    }
+
+    if (wait_for_right_attempt_to_complete)
+      // Wait for the attempt_number_to_resolve_ attempt to resolve.
+      WaitForAnAttemptToComplete();
+
+    int result = ResolveUsingPrevious(host, address_family, host_resolver_flags,
+                                      addrlist, os_error);
+
+    {
+      base::AutoLock auto_lock(lock_);
+      ++total_attempts_resolved_;
+    }
+
+    all_done_.Broadcast();  // Tell all attempts to proceed.
+
+    // Since any negative number is considered a network error, with -1 having
+    // special meaning (ERR_IO_PENDING). We could return the attempt that has
+    // resolved the host as a negative number. For example, if attempt number 3
+    // resolves the host, then this method returns -4.
+    if (result == OK)
+      return -1 - resolved_attempt_number_;
+    else
+      return result;
+  }
+
+ private:
+  virtual ~LookupAttemptHostResolverProc() {}
+
+  int attempt_number_to_resolve_;
+  int current_attempt_number_;  // Incremented whenever Resolve is called.
+  int total_attempts_;
+  int total_attempts_resolved_;
+  int resolved_attempt_number_;
+
+  // All attempts wait for right attempt to be resolve.
+  base::Lock lock_;
+  base::ConditionVariable all_done_;
+};
+
 // Helper that represents a single Resolve() result, used to inspect all the
 // resolve results by forwarding them to Delegate.
 class ResolveRequest {
@@ -1694,8 +1799,50 @@ TEST_F(HostResolverImplTest, DisallowNonCachedResponses) {
   EXPECT_TRUE(htons(kPortnum) == sa_in->sin_port);
   EXPECT_TRUE(htonl(0xc0a8012a) == sa_in->sin_addr.s_addr);
 }
-// TODO(cbentzel): Test a mix of requests with different HostResolverFlags.
 
-}  // namespace
+// Test the retry attempts simulating host resolver proc that takes too long.
+TEST_F(HostResolverImplTest, MultipleAttempts) {
+  // Total number of attempts would be 3 and we want the 3rd attempt to resolve
+  // the host. First and second attempt will be forced to sleep until they get
+  // word that a resolution has completed. The 3rd resolution attempt will try
+  // to get done ASAP, and won't sleep..
+  int kAttemptNumberToResolve = 3;
+  int kTotalAttempts = 3;
+
+  scoped_refptr<LookupAttemptHostResolverProc> resolver_proc(
+      new LookupAttemptHostResolverProc(
+          NULL, kAttemptNumberToResolve, kTotalAttempts));
+  HostCache* cache = CreateDefaultCache();
+  scoped_ptr<HostResolverImpl> host_resolver(
+      new HostResolverImpl(resolver_proc, cache, kMaxJobs, NULL));
+
+  // Specify smaller interval for unresponsive_delay_ and
+  // maximum_unresponsive_delay_ for HostResolverImpl so that unit test
+  // runs faster. For example, this test finishes in 1.5 secs (500ms * 3).
+  TimeDelta kUnresponsiveTime = TimeDelta::FromMilliseconds(500);
+  TimeDelta kMaximumUnresponsiveTime = TimeDelta::FromMilliseconds(2500);
+
+  host_resolver->set_unresponsive_delay(kUnresponsiveTime);
+  host_resolver->set_maximum_unresponsive_delay(kMaximumUnresponsiveTime);
+
+  // Resolve "host1".
+  HostResolver::RequestInfo info(HostPortPair("host1", 70));
+  TestCompletionCallback callback;
+  AddressList addrlist;
+  int rv = host_resolver->Resolve(info, &addrlist, &callback, NULL,
+                                  BoundNetLog());
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+
+  // Resolve returns -4 to indicate that 3rd attempt has resolved the host.
+  EXPECT_EQ(-4, callback.WaitForResult());
+
+  resolver_proc->WaitForAllAttemptsToFinish(TimeDelta::FromMilliseconds(60000));
+  MessageLoop::current()->RunAllPending();
+
+  EXPECT_EQ(resolver_proc->total_attempts_resolved(), kTotalAttempts);
+  EXPECT_EQ(resolver_proc->resolved_attempt_number(), kAttemptNumberToResolve);
+}
+
+// TODO(cbentzel): Test a mix of requests with different HostResolverFlags.
 
 }  // namespace net