DNS Host resolver changes with retry logic. Fix for

bug Chromium cannot recover from a state when its 
DNS requests have been dropped.

Whenever we try to resolve the host, we post a delayed task
to check if host resolution (OnLookupComplete) is completed 
or not. If the original ateempt hasn't completed, then we start
another attempt to resolve for the same request.

We take the results from the attempt that finishes first and
leave all other attempts as orphaned.

BUG=73327
TEST=dns host resolver tests
R=eroman,jar
Review URL: http://codereview.chromium.org/6782001

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@83641 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 151 insertions, 3 deletions

diff --git a/net/base/host_resolver_impl_unittest.cc b/net/base/host_resolver_impl_unittest.cc
index 1093cdc..9893028 100644
--- a/net/base/host_resolver_impl_unittest.cc
+++ b/net/base/host_resolver_impl_unittest.cc
@@ -11,6 +11,9 @@
 #include "base/message_loop.h"
 #include "base/string_util.h"
 #include "base/stringprintf.h"
+#include "base/synchronization/condition_variable.h"
+#include "base/synchronization/lock.h"
+#include "base/time.h"
 #include "net/base/address_list.h"
 #include "net/base/completion_callback.h"
 #include "net/base/mock_host_resolver.h"
@@ -27,7 +30,8 @@
 
 namespace net {
 
-namespace {
+using base::TimeDelta;
+using base::TimeTicks;
 
 HostCache* CreateDefaultCache() {
   return new HostCache(
@@ -150,6 +154,107 @@ class EchoingHostResolverProc : public HostResolverProc {
   }
 };
 
+// Using LookupAttemptHostResolverProc simulate very long lookups, and control
+// which attempt resolves the host.
+class LookupAttemptHostResolverProc : public HostResolverProc {
+ public:
+  LookupAttemptHostResolverProc(HostResolverProc* previous,
+                                int attempt_number_to_resolve,
+                                int total_attempts)
+      : HostResolverProc(previous),
+        attempt_number_to_resolve_(attempt_number_to_resolve),
+        current_attempt_number_(0),
+        total_attempts_(total_attempts),
+        total_attempts_resolved_(0),
+        resolved_attempt_number_(0),
+        all_done_(&lock_) {
+  }
+
+  // Test harness will wait for all attempts to finish before checking the
+  // results.
+  void WaitForAllAttemptsToFinish(const TimeDelta& wait_time) {
+    TimeTicks end_time = TimeTicks::Now() + wait_time;
+    {
+      base::AutoLock auto_lock(lock_);
+      while (total_attempts_resolved_ != total_attempts_ &&
+             TimeTicks::Now() < end_time) {
+        all_done_.TimedWait(end_time - TimeTicks::Now());
+      }
+    }
+  }
+
+  // All attempts will wait for an attempt to resolve the host.
+  void WaitForAnAttemptToComplete() {
+    TimeDelta wait_time = TimeDelta::FromMilliseconds(60000);
+    TimeTicks end_time = TimeTicks::Now() + wait_time;
+    {
+      base::AutoLock auto_lock(lock_);
+      while (resolved_attempt_number_ == 0 && TimeTicks::Now() < end_time)
+        all_done_.TimedWait(end_time - TimeTicks::Now());
+    }
+    all_done_.Broadcast();  // Tell all waiting attempts to proceed.
+  }
+
+  // Returns the number of attempts that have finished the Resolve() method.
+  int total_attempts_resolved() { return total_attempts_resolved_; }
+
+  // Returns the first attempt that that has resolved the host.
+  int resolved_attempt_number() { return resolved_attempt_number_; }
+
+  // HostResolverProc methods.
+  virtual int Resolve(const std::string& host,
+                      AddressFamily address_family,
+                      HostResolverFlags host_resolver_flags,
+                      AddressList* addrlist,
+                      int* os_error) {
+    bool wait_for_right_attempt_to_complete = true;
+    {
+      base::AutoLock auto_lock(lock_);
+      ++current_attempt_number_;
+      if (current_attempt_number_ == attempt_number_to_resolve_) {
+        resolved_attempt_number_ = current_attempt_number_;
+        wait_for_right_attempt_to_complete = false;
+      }
+    }
+
+    if (wait_for_right_attempt_to_complete)
+      // Wait for the attempt_number_to_resolve_ attempt to resolve.
+      WaitForAnAttemptToComplete();
+
+    int result = ResolveUsingPrevious(host, address_family, host_resolver_flags,
+                                      addrlist, os_error);
+
+    {
+      base::AutoLock auto_lock(lock_);
+      ++total_attempts_resolved_;
+    }
+
+    all_done_.Broadcast();  // Tell all attempts to proceed.
+
+    // Since any negative number is considered a network error, with -1 having
+    // special meaning (ERR_IO_PENDING). We could return the attempt that has
+    // resolved the host as a negative number. For example, if attempt number 3
+    // resolves the host, then this method returns -4.
+    if (result == OK)
+      return -1 - resolved_attempt_number_;
+    else
+      return result;
+  }
+
+ private:
+  virtual ~LookupAttemptHostResolverProc() {}
+
+  int attempt_number_to_resolve_;
+  int current_attempt_number_;  // Incremented whenever Resolve is called.
+  int total_attempts_;
+  int total_attempts_resolved_;
+  int resolved_attempt_number_;
+
+  // All attempts wait for right attempt to be resolve.
+  base::Lock lock_;
+  base::ConditionVariable all_done_;
+};
+
 // Helper that represents a single Resolve() result, used to inspect all the
 // resolve results by forwarding them to Delegate.
 class ResolveRequest {
@@ -1694,8 +1799,51 @@ TEST_F(HostResolverImplTest, DisallowNonCachedResponses) {
   EXPECT_TRUE(htons(kPortnum) == sa_in->sin_port);
   EXPECT_TRUE(htonl(0xc0a8012a) == sa_in->sin_addr.s_addr);
 }
-// TODO(cbentzel): Test a mix of requests with different HostResolverFlags.
 
-}  // namespace
+// Test the retry attempts simulating host resolver proc that takes too long.
+TEST_F(HostResolverImplTest, MultipleAttempts) {
+  // Total number of attempts would be 3 and we want the 3rd attempt to resolve
+  // the host. First and second attempt will be forced to sleep until they get
+  // word that a resolution has completed. The 3rd resolution attempt will try
+  // to get done ASAP, and won't sleep..
+  int kAttemptNumberToResolve = 3;
+  int kTotalAttempts = 3;
+
+  scoped_refptr<LookupAttemptHostResolverProc> resolver_proc(
+      new LookupAttemptHostResolverProc(
+          NULL, kAttemptNumberToResolve, kTotalAttempts));
+  HostCache* cache = CreateDefaultCache();
+  scoped_ptr<HostResolverImpl> host_resolver(
+      new HostResolverImpl(resolver_proc, cache, kMaxJobs, NULL));
+
+  // Specify smaller interval for unresponsive_delay_ and
+  // maximum_unresponsive_delay_ for HostResolverImpl so that unit test
+  // runs faster. For example, this test finishes in 1.5 secs (500ms * 3).
+  TimeDelta kUnresponsiveTime = TimeDelta::FromMilliseconds(500);
+  TimeDelta kMaximumUnresponsiveTime = TimeDelta::FromMilliseconds(2500);
+
+  host_resolver->set_unresponsive_delay(kUnresponsiveTime);
+  host_resolver->set_maximum_unresponsive_delay(kMaximumUnresponsiveTime);
+
+  // Resolve "host1".
+  HostResolver::RequestInfo info(HostPortPair("host1", 70));
+  TestCompletionCallback callback;
+  AddressList addrlist;
+  int rv = host_resolver->Resolve(info, &addrlist, &callback, NULL,
+                                  BoundNetLog());
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+
+  // Resolve returns -4 to indicate that 3rd attempt has resolved the host.
+  EXPECT_EQ(-4, callback.WaitForResult());
+
+  resolver_proc->WaitForAllAttemptsToFinish(TimeDelta::FromMilliseconds(60000));
+  MessageLoop::current()->RunAllPending();
+
+  EXPECT_EQ(resolver_proc->total_attempts_resolved(), kTotalAttempts);
+  EXPECT_EQ(resolver_proc->resolved_attempt_number(), kAttemptNumberToResolve);
+}
+
+
+// TODO(cbentzel): Test a mix of requests with different HostResolverFlags.
 
 }  // namespace net