During appcache update, copy response from newest complete cache if http caching semantics allow rather than make a network request.

TEST=new tests added
BUG=none


Review URL: http://codereview.chromium.org/491064

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@34727 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 269 insertions, 41 deletions

diff --git a/webkit/appcache/appcache_update_job.cc b/webkit/appcache/appcache_update_job.cc
index 8d7ddc1..de7b73c 100644
--- a/webkit/appcache/appcache_update_job.cc
+++ b/webkit/appcache/appcache_update_job.cc
@@ -512,11 +512,13 @@ void AppCacheUpdateJob::HandleUrlFetchCompleted(URLRequest* request) {
     } else if (response_code == 404 || response_code == 410) {
       // Entry is skipped.  They are dropped from the cache.
     } else if (update_type_ == UPGRADE_ATTEMPT) {
-      // Copy the resource and its metadata from the newest complete cache.
+      // Copy the response from the newest complete cache.
       AppCache* cache = group_->newest_complete_cache();
       AppCacheEntry* copy = cache->GetEntry(url);
-      if (copy)
-        CopyEntryToCache(url, *copy, &entry);
+      if (copy) {
+        entry.set_response_id(copy->response_id());
+        inprogress_cache_->AddOrModifyEntry(url, entry);
+      }
     }
   }
 
@@ -803,7 +805,7 @@ void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
 
   if (ret.second)
-    urls_to_fetch_.push_back(UrlToFetch(url, false));
+    urls_to_fetch_.push_back(UrlsToFetch(url, false));
   else
     ret.first->second.add_types(type);  // URL already exists. Merge types.
 }
@@ -1002,41 +1004,62 @@ bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
 
   AppCache* newest = group_->newest_complete_cache();
   AppCacheEntry* copy_me = newest->GetEntry(url);
-  if (!copy_me)
+  if (!copy_me || !copy_me->has_response_id())
     return false;
 
-  // TODO(jennb): load HTTP headers for copy_me and wait for callback
-  // In callback:
-  // if HTTP caching semantics for entry allows its use,
-  //   CopyEntryData(url, copy_me, entry)
-  //   ++urls_fetches_completed_;
-  // Else, add url back to front of urls_to_fetch and call FetchUrls().
-  //   flag url somehow so that FetchUrls() doesn't end up here again.
-  // For now: post a message to pretend entry could not be copied
-  MessageLoop::current()->PostTask(FROM_HERE,
-      method_factory_.NewRunnableMethod(
-          &AppCacheUpdateJob::SimulateFailedLoadFromNewestCache, url));
+  // Load HTTP headers for entry from newest cache.
+  loading_responses_.insert(
+      LoadingResponses::value_type(copy_me->response_id(), url));
+  service_->storage()->LoadResponseInfo(manifest_url_, copy_me->response_id(),
+                                        this);
+  // Async: wait for OnResponseInfoLoaded to complete.
   return true;
 }
 
-// TODO(jennb): delete this after have real storage code
-void AppCacheUpdateJob::SimulateFailedLoadFromNewestCache(const GURL& url) {
+void AppCacheUpdateJob::OnResponseInfoLoaded(
+    AppCacheResponseInfo* response_info, int64 response_id) {
+  LoadingResponses::iterator found = loading_responses_.find(response_id);
+  DCHECK(found != loading_responses_.end());
+  const GURL& url = found->second;
+
+  if (!response_info) {
+    LoadFromNewestCacheFailed(url);  // no response found
+  } else {
+    // Check if response can be re-used according to HTTP caching semantics.
+    // Responses with a "vary" header get treated as expired.
+    const net::HttpResponseInfo* http_info =
+        response_info->http_response_info();
+    const std::string name = "vary";
+    std::string value;
+    void* iter = NULL;
+    if (http_info->headers->RequiresValidation(http_info->request_time,
+                                               http_info->response_time,
+                                               base::Time::Now()) ||
+        http_info->headers->EnumerateHeader(&iter, name, &value)) {
+      LoadFromNewestCacheFailed(url);
+    } else {
+      AppCache::EntryMap::iterator it = url_file_list_.find(url);
+      DCHECK(it != url_file_list_.end());
+      AppCacheEntry& entry = it->second;
+      entry.set_response_id(response_id);
+      inprogress_cache_->AddOrModifyEntry(url, entry);
+      ++url_fetches_completed_;
+    }
+  }
+  loading_responses_.erase(found);
+
+  MaybeCompleteUpdate();
+}
+
+void AppCacheUpdateJob::LoadFromNewestCacheFailed(const GURL& url) {
   if (internal_state_ == CACHE_FAILURE)
     return;
 
   // Re-insert url at front of fetch list. Indicate storage has been checked.
-  urls_to_fetch_.push_front(AppCacheUpdateJob::UrlToFetch(url, true));
+  urls_to_fetch_.push_front(AppCacheUpdateJob::UrlsToFetch(url, true));
   FetchUrls();
 }
 
-void AppCacheUpdateJob::CopyEntryToCache(const GURL& url,
-                                         const AppCacheEntry& src,
-                                         AppCacheEntry* dest) {
-  DCHECK(dest);
-  dest->set_response_id(src.response_id());
-  inprogress_cache_->AddOrModifyEntry(url, *dest);
-}
-
 void AppCacheUpdateJob::MaybeCompleteUpdate() {
   // Must wait for any pending master entries or url fetches to complete.
   if (master_entries_completed_ != pending_master_entries_.size() ||
diff --git a/webkit/appcache/appcache_update_job.h b/webkit/appcache/appcache_update_job.h
index 304df03..840840d 100644
--- a/webkit/appcache/appcache_update_job.h
+++ b/webkit/appcache/appcache_update_job.h
@@ -48,7 +48,8 @@ class AppCacheUpdateJob : public URLRequest::Delegate,
   typedef std::vector<AppCacheHost*> PendingHosts;
   typedef std::map<GURL, PendingHosts> PendingMasters;
   typedef std::map<GURL, URLRequest*> PendingUrlFetches;
-  typedef std::pair<GURL, bool> UrlToFetch;  // flag TRUE if storage checked
+  typedef std::pair<GURL, bool> UrlsToFetch;  // flag TRUE if storage checked
+  typedef std::map<int64, GURL> LoadingResponses;
 
   static const int kRerunDelayMs = 1000;
 
@@ -79,6 +80,8 @@ class AppCacheUpdateJob : public URLRequest::Delegate,
   // TODO(jennb): any other delegate callbacks to handle? certificate?
 
   // Methods for AppCacheStorage::Delegate.
+  void OnResponseInfoLoaded(AppCacheResponseInfo* response_info,
+                            int64 response_id);
   void OnGroupAndNewestCacheStored(AppCacheGroup* group, bool success);
   void OnGroupMadeObsolete(AppCacheGroup* group, bool success);
 
@@ -152,14 +155,7 @@ class AppCacheUpdateJob : public URLRequest::Delegate,
   // Returns false if immediately obvious that data cannot be loaded from
   // newest complete cache.
   bool MaybeLoadFromNewestCache(const GURL& url, AppCacheEntry& entry);
-
-  // TODO(jennb): delete me after storage code added
-  void SimulateFailedLoadFromNewestCache(const GURL& url);
-
-  // Copies the data from src entry to dest entry and adds the modified
-  // entry to the inprogress cache.
-  void CopyEntryToCache(const GURL& url, const AppCacheEntry& src,
-                        AppCacheEntry* dest);
+  void LoadFromNewestCacheFailed(const GURL& url);
 
   // Does nothing if update process is still waiting for pending master
   // entries or URL fetches to complete downloading. Otherwise, completes
@@ -216,7 +212,11 @@ class AppCacheUpdateJob : public URLRequest::Delegate,
   // Helper container to track which urls have not been fetched yet. URLs are
   // removed when the fetch is initiated. Flag indicates whether an attempt
   // to load the URL from storage has already been tried and failed.
-  std::deque<UrlToFetch> urls_to_fetch_;
+  std::deque<UrlsToFetch> urls_to_fetch_;
+
+  // Helper container to track which urls are being loaded from response
+  // storage.
+  LoadingResponses loading_responses_;
 
   // Keep track of pending URL requests so we can cancel them if necessary.
   URLRequest* manifest_url_request_;
diff --git a/webkit/appcache/appcache_update_job_unittest.cc b/webkit/appcache/appcache_update_job_unittest.cc
index 5679ef3..72698e2 100644
--- a/webkit/appcache/appcache_update_job_unittest.cc
+++ b/webkit/appcache/appcache_update_job_unittest.cc
@@ -631,14 +631,14 @@ class AppCacheUpdateJobTest : public testing::Test,
         new net::StringIOBuffer(seed_data);
     write_callback_.reset(
         new net::CompletionCallbackImpl<AppCacheUpdateJobTest>(this,
-            &AppCacheUpdateJobTest::StartUpdateAfterSeedingManifestData));
+            &AppCacheUpdateJobTest::StartUpdateAfterSeedingStorageData));
     response_writer_->WriteData(io_buffer, seed_data.length(),
                                 write_callback_.get());
 
     // Start update after data write completes asynchronously.
   }
 
-  void StartUpdateAfterSeedingManifestData(int result) {
+  void StartUpdateAfterSeedingStorageData(int result) {
     ASSERT_GT(result, 0);
     write_callback_.reset();
     response_writer_.reset();
@@ -724,13 +724,191 @@ class AppCacheUpdateJobTest : public testing::Test,
         new net::StringIOBuffer(seed_data);
     write_callback_.reset(
         new net::CompletionCallbackImpl<AppCacheUpdateJobTest>(this,
-            &AppCacheUpdateJobTest::StartUpdateAfterSeedingManifestData));
+            &AppCacheUpdateJobTest::StartUpdateAfterSeedingStorageData));
     response_writer_->WriteData(io_buffer, seed_data.length(),
                                 write_callback_.get());
 
     // Start update after data write completes asynchronously.
   }
 
+  void UpgradeLoadFromNewestCacheTest() {
+    ASSERT_EQ(MessageLoop::TYPE_IO, MessageLoop::current()->type());
+
+    MakeService();
+    group_ = new AppCacheGroup(
+        service_.get(), http_server_->TestServerPage("files/manifest1"),
+        service_->storage()->NewGroupId());
+    AppCacheUpdateJob* update = new AppCacheUpdateJob(service_.get(), group_);
+    group_->update_job_ = update;
+
+    AppCache* cache = MakeCacheForGroup(service_->storage()->NewCacheId(), 42);
+    MockFrontend* frontend = MakeMockFrontend();
+    AppCacheHost* host = MakeHost(1, frontend);
+    host->AssociateCache(cache);
+
+    // Give the newest cache an entry that is in storage.
+    response_writer_.reset(
+        service_->storage()->CreateResponseWriter(group_->manifest_url()));
+    cache->AddEntry(http_server_->TestServerPage("files/explicit1"),
+                    AppCacheEntry(AppCacheEntry::EXPLICIT,
+                                  response_writer_->response_id()));
+
+    // Set up checks for when update job finishes.
+    do_checks_after_update_finished_ = true;
+    expect_group_obsolete_ = false;
+    expect_group_has_cache_ = true;
+    expect_old_cache_ = cache;
+    expect_response_ids_.insert(
+        std::map<GURL, int64>::value_type(
+            http_server_->TestServerPage("files/explicit1"),
+            response_writer_->response_id()));
+    tested_manifest_ = MANIFEST1;
+    MockFrontend::HostIds ids(1, host->host_id());
+    frontend->AddExpectedEvent(ids, CHECKING_EVENT);
+    frontend->AddExpectedEvent(ids, DOWNLOADING_EVENT);
+    frontend->AddExpectedEvent(ids, PROGRESS_EVENT);
+    frontend->AddExpectedEvent(ids, PROGRESS_EVENT);
+    frontend->AddExpectedEvent(ids, UPDATE_READY_EVENT);
+
+    // Seed storage with expected http response info for entry. Allow reuse.
+    const char data[] =
+        "HTTP/1.1 200 OK\0"
+        "Cache-Control: max-age=8675309\0"
+        "\0";
+    net::HttpResponseHeaders* headers =
+        new net::HttpResponseHeaders(std::string(data, arraysize(data)));
+    net::HttpResponseInfo* response_info = new net::HttpResponseInfo();
+    response_info->request_time = base::Time::Now();
+    response_info->response_time = base::Time::Now();
+    response_info->headers = headers;  // adds ref to headers
+    scoped_refptr<HttpResponseInfoIOBuffer> io_buffer =
+        new HttpResponseInfoIOBuffer(response_info);  // adds ref to info
+    write_callback_.reset(
+        new net::CompletionCallbackImpl<AppCacheUpdateJobTest>(this,
+            &AppCacheUpdateJobTest::StartUpdateAfterSeedingStorageData));
+    response_writer_->WriteInfo(io_buffer, write_callback_.get());
+
+    // Start update after data write completes asynchronously.
+  }
+
+  void UpgradeNoLoadFromNewestCacheTest() {
+    ASSERT_EQ(MessageLoop::TYPE_IO, MessageLoop::current()->type());
+
+    MakeService();
+    group_ = new AppCacheGroup(
+        service_.get(), http_server_->TestServerPage("files/manifest1"),
+        service_->storage()->NewGroupId());
+    AppCacheUpdateJob* update = new AppCacheUpdateJob(service_.get(), group_);
+    group_->update_job_ = update;
+
+    AppCache* cache = MakeCacheForGroup(service_->storage()->NewCacheId(), 42);
+    MockFrontend* frontend = MakeMockFrontend();
+    AppCacheHost* host = MakeHost(1, frontend);
+    host->AssociateCache(cache);
+
+    // Give the newest cache an entry that is in storage.
+    response_writer_.reset(
+        service_->storage()->CreateResponseWriter(group_->manifest_url()));
+    cache->AddEntry(http_server_->TestServerPage("files/explicit1"),
+                    AppCacheEntry(AppCacheEntry::EXPLICIT,
+                                  response_writer_->response_id()));
+
+    // Set up checks for when update job finishes.
+    do_checks_after_update_finished_ = true;
+    expect_group_obsolete_ = false;
+    expect_group_has_cache_ = true;
+    expect_old_cache_ = cache;
+    tested_manifest_ = MANIFEST1;
+    MockFrontend::HostIds ids(1, host->host_id());
+    frontend->AddExpectedEvent(ids, CHECKING_EVENT);
+    frontend->AddExpectedEvent(ids, DOWNLOADING_EVENT);
+    frontend->AddExpectedEvent(ids, PROGRESS_EVENT);
+    frontend->AddExpectedEvent(ids, PROGRESS_EVENT);
+    // Extra progress event for re-attempt to fetch explicit1 from network.
+    frontend->AddExpectedEvent(ids, PROGRESS_EVENT);
+    frontend->AddExpectedEvent(ids, UPDATE_READY_EVENT);
+
+    // Seed storage with expected http response info for entry. Do NOT
+    // allow reuse by setting an expires header in the past.
+    const char data[] =
+        "HTTP/1.1 200 OK\0"
+        "Expires: Thu, 01 Dec 1994 16:00:00 GMT\0"
+        "\0";
+    net::HttpResponseHeaders* headers =
+        new net::HttpResponseHeaders(std::string(data, arraysize(data)));
+    net::HttpResponseInfo* response_info = new net::HttpResponseInfo();
+    response_info->request_time = base::Time::Now();
+    response_info->response_time = base::Time::Now();
+    response_info->headers = headers;  // adds ref to headers
+    scoped_refptr<HttpResponseInfoIOBuffer> io_buffer =
+        new HttpResponseInfoIOBuffer(response_info);  // adds ref to info
+    write_callback_.reset(
+        new net::CompletionCallbackImpl<AppCacheUpdateJobTest>(this,
+            &AppCacheUpdateJobTest::StartUpdateAfterSeedingStorageData));
+    response_writer_->WriteInfo(io_buffer, write_callback_.get());
+
+    // Start update after data write completes asynchronously.
+  }
+
+  void UpgradeLoadFromNewestCacheVaryHeaderTest() {
+    ASSERT_EQ(MessageLoop::TYPE_IO, MessageLoop::current()->type());
+
+    MakeService();
+    group_ = new AppCacheGroup(
+        service_.get(), http_server_->TestServerPage("files/manifest1"),
+        service_->storage()->NewGroupId());
+    AppCacheUpdateJob* update = new AppCacheUpdateJob(service_.get(), group_);
+    group_->update_job_ = update;
+
+    AppCache* cache = MakeCacheForGroup(service_->storage()->NewCacheId(), 42);
+    MockFrontend* frontend = MakeMockFrontend();
+    AppCacheHost* host = MakeHost(1, frontend);
+    host->AssociateCache(cache);
+
+    // Give the newest cache an entry that is in storage.
+    response_writer_.reset(
+        service_->storage()->CreateResponseWriter(group_->manifest_url()));
+    cache->AddEntry(http_server_->TestServerPage("files/explicit1"),
+                    AppCacheEntry(AppCacheEntry::EXPLICIT,
+                                  response_writer_->response_id()));
+
+    // Set up checks for when update job finishes.
+    do_checks_after_update_finished_ = true;
+    expect_group_obsolete_ = false;
+    expect_group_has_cache_ = true;
+    expect_old_cache_ = cache;
+    tested_manifest_ = MANIFEST1;
+    MockFrontend::HostIds ids(1, host->host_id());
+    frontend->AddExpectedEvent(ids, CHECKING_EVENT);
+    frontend->AddExpectedEvent(ids, DOWNLOADING_EVENT);
+    frontend->AddExpectedEvent(ids, PROGRESS_EVENT);
+    frontend->AddExpectedEvent(ids, PROGRESS_EVENT);
+    // Extra progress event for re-attempt to fetch explicit1 from network.
+    frontend->AddExpectedEvent(ids, PROGRESS_EVENT);
+    frontend->AddExpectedEvent(ids, UPDATE_READY_EVENT);
+
+    // Seed storage with expected http response info for entry: a vary header.
+    const char data[] =
+        "HTTP/1.1 200 OK\0"
+        "Cache-Control: max-age=8675309\0"
+        "Vary: blah\0"
+        "\0";
+    net::HttpResponseHeaders* headers =
+        new net::HttpResponseHeaders(std::string(data, arraysize(data)));
+    net::HttpResponseInfo* response_info = new net::HttpResponseInfo();
+    response_info->request_time = base::Time::Now();
+    response_info->response_time = base::Time::Now();
+    response_info->headers = headers;  // adds ref to headers
+    scoped_refptr<HttpResponseInfoIOBuffer> io_buffer =
+        new HttpResponseInfoIOBuffer(response_info);  // adds ref to info
+    write_callback_.reset(
+        new net::CompletionCallbackImpl<AppCacheUpdateJobTest>(this,
+            &AppCacheUpdateJobTest::StartUpdateAfterSeedingStorageData));
+    response_writer_->WriteInfo(io_buffer, write_callback_.get());
+
+    // Start update after data write completes asynchronously.
+  }
+
   void UpgradeSuccessMergedTypesTest() {
     ASSERT_EQ(MessageLoop::TYPE_IO, MessageLoop::current()->type());
 
@@ -896,6 +1074,8 @@ class AppCacheUpdateJobTest : public testing::Test,
     expect_extra_entries_.insert(AppCache::EntryMap::value_type(
         http_server_->TestServerPage("files/servererror"),
         AppCacheEntry(AppCacheEntry::MASTER)));
+    expect_response_ids_.insert(std::map<GURL, int64>::value_type(
+        http_server_->TestServerPage("files/servererror"), 444));  // copied
     MockFrontend::HostIds ids1(1, host1->host_id());
     frontend1->AddExpectedEvent(ids1, CHECKING_EVENT);
     frontend1->AddExpectedEvent(ids1, DOWNLOADING_EVENT);
@@ -2042,6 +2222,18 @@ class AppCacheUpdateJobTest : public testing::Test,
         for (AppCache::EntryMap::const_iterator it = entries.begin();
              it != entries.end(); ++it) {
           EXPECT_NE(kNoResponseId, it->second.response_id());
+
+          // Check that any copied entries have the expected response id
+          // and that entries that are not copied have a different response id.
+          std::map<GURL, int64>::iterator found =
+              expect_response_ids_.find(it->first);
+          if (found != expect_response_ids_.end()) {
+            EXPECT_EQ(found->second, it->second.response_id());
+          } else if (expect_old_cache_) {
+            AppCacheEntry* old_entry = expect_old_cache_->GetEntry(it->first);
+            if (old_entry)
+              EXPECT_NE(old_entry->response_id(), it->second.response_id());
+          }
         }
       }
     } else {
@@ -2124,7 +2316,6 @@ class AppCacheUpdateJobTest : public testing::Test,
       entry = cache->GetEntry(i->first);
       ASSERT_TRUE(entry);
       EXPECT_EQ(i->second.types(), entry->types());
-      // TODO(jennb): if copied, check storage id in entry is as expected
     }
 
     expected = 1;
@@ -2273,6 +2464,7 @@ class AppCacheUpdateJobTest : public testing::Test,
   std::vector<MockFrontend*> frontends_;  // to check expected events
   TestedManifest tested_manifest_;
   AppCache::EntryMap expect_extra_entries_;
+  std::map<GURL, int64> expect_response_ids_;
 
   bool registered_factory_;
   URLRequest::ProtocolFactory* old_factory_;
@@ -2396,6 +2588,19 @@ TEST_F(AppCacheUpdateJobTest, BasicUpgradeSuccess) {
   RunTestOnIOThread(&AppCacheUpdateJobTest::BasicUpgradeSuccessTest);
 }
 
+TEST_F(AppCacheUpdateJobTest, UpgradeLoadFromNewestCache) {
+  RunTestOnIOThread(&AppCacheUpdateJobTest::UpgradeLoadFromNewestCacheTest);
+}
+
+TEST_F(AppCacheUpdateJobTest, UpgradeNoLoadFromNewestCache) {
+  RunTestOnIOThread(&AppCacheUpdateJobTest::UpgradeNoLoadFromNewestCacheTest);
+}
+
+TEST_F(AppCacheUpdateJobTest, UpgradeLoadFromNewestCacheVaryHeader) {
+  RunTestOnIOThread(
+      &AppCacheUpdateJobTest::UpgradeLoadFromNewestCacheVaryHeaderTest);
+}
+
 TEST_F(AppCacheUpdateJobTest, UpgradeSuccessMergedTypes) {
   RunTestOnIOThread(&AppCacheUpdateJobTest::UpgradeSuccessMergedTypesTest);
 }