Trigger the deletion of unused responses when caches are updated and made obsolete (deleted).

BUG=none
TEST=not yet

Review URL: http://codereview.chromium.org/542021

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@36166 0039d316-1c4b-4281-b951-d872f2087c98

12 files changed, 191 insertions, 36 deletions

diff --git a/webkit/appcache/appcache_database.cc b/webkit/appcache/appcache_database.cc
index ddfa34d..8b9da0a 100644
--- a/webkit/appcache/appcache_database.cc
+++ b/webkit/appcache/appcache_database.cc
@@ -94,6 +94,11 @@ const struct {
     "(cache_id, url)",
     true },
 
+  { "EntriesResponseIdIndex",
+    kEntriesTable,
+    "(response_id)",
+    true },
+
   { "FallbackNameSpacesCacheIndex",
     kFallbackNameSpacesTable,
     "(cache_id)",
@@ -179,26 +184,33 @@ bool AppCacheDatabase::FindLastStorageIds(
 
   const char* kMaxGroupIdSql = "SELECT MAX(group_id) FROM Groups";
   const char* kMaxCacheIdSql = "SELECT MAX(cache_id) FROM Caches";
+  const char* kMaxResponseIdFromEntriesSql =
+      "SELECT MAX(response_id) FROM Entries";
+  const char* kMaxResponseIdFromDeletablesSql =
+      "SELECT MAX(response_id) FROM DeletableResponseIds";
   const char* kMaxDeletableResponseRowIdSql =
       "SELECT MAX(rowid) FROM DeletableResponseIds";
-  int64 group_id;
-  int64 cache_id;
-  int64 deletable_response_rowid;
-  if (!RunUniqueStatementWithInt64Result(kMaxGroupIdSql, &group_id) ||
-      !RunUniqueStatementWithInt64Result(kMaxCacheIdSql, &cache_id) ||
+  int64 max_group_id;
+  int64 max_cache_id;
+  int64 max_response_id_from_entries;
+  int64 max_response_id_from_deletables;
+  int64 max_deletable_response_rowid;
+  if (!RunUniqueStatementWithInt64Result(kMaxGroupIdSql, &max_group_id) ||
+      !RunUniqueStatementWithInt64Result(kMaxCacheIdSql, &max_cache_id) ||
+      !RunUniqueStatementWithInt64Result(kMaxResponseIdFromEntriesSql,
+                                         &max_response_id_from_entries) ||
+      !RunUniqueStatementWithInt64Result(kMaxResponseIdFromDeletablesSql,
+                                         &max_response_id_from_deletables) ||
       !RunUniqueStatementWithInt64Result(kMaxDeletableResponseRowIdSql,
-                                         &deletable_response_rowid)) {
+                                         &max_deletable_response_rowid)) {
     return false;
   }
 
-  // TODO(michaeln): SELECT MAX(responseId) FROM somewhere,
-  // or retrieve from the meta_table.
-  int64 response_id = 0;
-
-  *last_group_id = group_id;
-  *last_cache_id = cache_id;
-  *last_response_id = response_id;
-  *last_deletable_response_rowid = deletable_response_rowid;
+  *last_group_id = max_group_id;
+  *last_cache_id = max_cache_id;
+  *last_response_id = std::max(max_response_id_from_entries,
+                               max_response_id_from_deletables);
+  *last_deletable_response_rowid = max_deletable_response_rowid;
   return true;
 }
 
@@ -812,6 +824,33 @@ bool AppCacheDatabase::PrepareCachedStatement(
   return true;
 }
 
+bool AppCacheDatabase::FindResponseIdsForCacheHelper(
+    int64 cache_id, std::vector<int64>* ids_vector,
+    std::set<int64>* ids_set) {
+  DCHECK(ids_vector || ids_set);
+  DCHECK(!(ids_vector && ids_set));
+  if (!LazyOpen(false))
+    return false;
+
+  const char* kSql =
+      "SELECT response_id FROM Entries WHERE cache_id = ?";
+
+  sql::Statement statement;
+  if (!PrepareCachedStatement(SQL_FROM_HERE, kSql, &statement))
+    return false;
+
+  statement.BindInt64(0, cache_id);
+  while (statement.Step()) {
+    int64 id = statement.ColumnInt64(0);
+    if (ids_set)
+      ids_set->insert(id);
+    else
+      ids_vector->push_back(id);
+  }
+
+  return statement.Succeeded();
+}
+
 void AppCacheDatabase::ReadGroupRecord(
     const sql::Statement& statement, GroupRecord* record) {
   record->group_id = statement.ColumnInt64(0);
diff --git a/webkit/appcache/appcache_database.h b/webkit/appcache/appcache_database.h
index c492cae..0b9d1ac 100644
--- a/webkit/appcache/appcache_database.h
+++ b/webkit/appcache/appcache_database.h
@@ -99,6 +99,14 @@ class AppCacheDatabase {
   bool DeleteEntriesForCache(int64 cache_id);
   bool AddEntryFlags(const GURL& entry_url, int64 cache_id,
                      int additional_flags);
+  bool FindResponseIdsForCacheAsVector(
+      int64 cache_id, std::vector<int64>* response_ids) {
+    return FindResponseIdsForCacheHelper(cache_id, response_ids, NULL);
+  }
+  bool FindResponseIdsForCacheAsSet(
+      int64 cache_id, std::set<int64>* response_ids) {
+    return FindResponseIdsForCacheHelper(cache_id, NULL, response_ids);
+  }
 
   bool FindFallbackNameSpacesForOrigin(
       const GURL& origin, std::vector<FallbackNameSpaceRecord>* records);
@@ -137,6 +145,10 @@ class AppCacheDatabase {
   bool PrepareCachedStatement(
       const sql::StatementID& id, const char* sql, sql::Statement* statement);
 
+  bool FindResponseIdsForCacheHelper(
+      int64 cache_id, std::vector<int64>* ids_vector,
+      std::set<int64>* ids_set);
+
   // Record retrieval helpers
   void ReadGroupRecord(const sql::Statement& statement, GroupRecord* record);
   void ReadCacheRecord(const sql::Statement& statement, CacheRecord* record);
diff --git a/webkit/appcache/appcache_group.cc b/webkit/appcache/appcache_group.cc
index e5511a6..e1906ab 100644
--- a/webkit/appcache/appcache_group.cc
+++ b/webkit/appcache/appcache_group.cc
@@ -60,6 +60,8 @@ AppCacheGroup::~AppCacheGroup() {
   DCHECK_EQ(IDLE, update_status_);
 
   service_->storage()->working_set()->RemoveGroup(this);
+  service_->storage()->DeleteResponses(
+      manifest_url_, newly_deletable_response_ids_);
 }
 
 void AppCacheGroup::AddUpdateObserver(UpdateObserver* observer) {
@@ -111,16 +113,44 @@ void AppCacheGroup::RemoveCache(AppCache* cache) {
     newest_complete_cache_ = NULL;
     tmp_cache->set_owning_group(NULL);  // may cause this group to be deleted
   } else {
+    scoped_refptr<AppCacheGroup> protect(this);
+
     Caches::iterator it =
         std::find(old_caches_.begin(), old_caches_.end(), cache);
     if (it != old_caches_.end()) {
       AppCache* tmp_cache = *it;
       old_caches_.erase(it);
-      tmp_cache->set_owning_group(NULL);  // may cause group to be deleted
+      tmp_cache->set_owning_group(NULL);  // may cause group to be released
+    }
+
+    if (!is_obsolete() && old_caches_.empty() &&
+        !newly_deletable_response_ids_.empty()) {
+      service_->storage()->DeleteResponses(
+          manifest_url_, newly_deletable_response_ids_);
+      newly_deletable_response_ids_.clear();
     }
   }
 }
 
+void AppCacheGroup::AddNewlyDeletableResponseIds(
+    std::vector<int64>* response_ids) {
+  if (!is_obsolete() && old_caches_.empty()) {
+    service_->storage()->DeleteResponses(
+        manifest_url_, *response_ids);
+    response_ids->clear();
+    return;
+  }
+
+  if (newly_deletable_response_ids_.empty()) {
+    newly_deletable_response_ids_.swap(*response_ids);
+    return;
+  }
+  newly_deletable_response_ids_.insert(
+      newly_deletable_response_ids_.end(),
+      response_ids->begin(), response_ids->end());
+  response_ids->clear();
+}
+
 void AppCacheGroup::StartUpdateWithNewMasterEntry(
     AppCacheHost* host, const GURL& new_master_resource) {
   if (!update_job_)
diff --git a/webkit/appcache/appcache_group.h b/webkit/appcache/appcache_group.h
index e6f8972..3392eea 100644
--- a/webkit/appcache/appcache_group.h
+++ b/webkit/appcache/appcache_group.h
@@ -61,6 +61,8 @@ class AppCacheGroup : public base::RefCounted<AppCacheGroup> {
   void RemoveCache(AppCache* cache);
   bool HasCache() const { return newest_complete_cache_ != NULL; }
 
+  void AddNewlyDeletableResponseIds(std::vector<int64>* response_ids);
+
   UpdateStatus update_status() const { return update_status_; }
 
   // Starts an update via update() javascript API.
@@ -111,6 +113,7 @@ class AppCacheGroup : public base::RefCounted<AppCacheGroup> {
   const GURL manifest_url_;
   UpdateStatus update_status_;
   bool is_obsolete_;
+  std::vector<int64> newly_deletable_response_ids_;
 
   // Old complete app caches.
   Caches old_caches_;
diff --git a/webkit/appcache/appcache_response.cc b/webkit/appcache/appcache_response.cc
index 5f92439..c16bd5c 100644
--- a/webkit/appcache/appcache_response.cc
+++ b/webkit/appcache/appcache_response.cc
@@ -263,8 +263,15 @@ void AppCacheResponseWriter::OnIOComplete(int result) {
 }
 
 bool AppCacheResponseWriter::CreateEntryIfNeeded() {
-  if (!entry_)
-    disk_cache_->CreateEntry(response_key(response_id_), &entry_);
+  if (entry_)
+    return true;
+  std::string key(response_key(response_id_));
+  if (!disk_cache_->CreateEntry(key, &entry_)) {
+    // We may try to overrite existing entries.
+    DCHECK(!entry_);
+    disk_cache_->DoomEntry(key);
+    disk_cache_->CreateEntry(key, &entry_);
+  }
   return entry_ ? true : false;
 }
 
diff --git a/webkit/appcache/appcache_storage.h b/webkit/appcache/appcache_storage.h
index 7c31ca5..8f10b78 100644
--- a/webkit/appcache/appcache_storage.h
+++ b/webkit/appcache/appcache_storage.h
@@ -140,15 +140,18 @@ class AppCacheStorage {
   virtual AppCacheResponseWriter* CreateResponseWriter(
       const GURL& manifest_url) = 0;
 
-  // Schedules the imminent deletion of many responses.
+  // Schedules the lazy deletion of responses and saves the ids
+  // persistently such that the responses will be deleted upon restart
+  // if they aren't deleted prior to shutdown.
   virtual void DoomResponses(
       const GURL& manifest_url, const std::vector<int64>& response_ids) = 0;
 
-  // Schedules the imminent deletion of a single response.
-  void DoomResponse(const GURL& manifest_url, int64 response_id) {
-    std::vector<int64> response_ids(1, response_id);
-    DoomResponses(manifest_url, response_ids);
-  }
+  // Schedules the lazy deletion of responses without persistently saving
+  // the response ids.
+  virtual void DeleteResponses(
+      const GURL& manifest_url, const std::vector<int64>& response_ids) = 0;
+
+  virtual void PurgeMemory() = 0;
 
   // Generates unique storage ids for different object types.
   int64 NewCacheId() {
diff --git a/webkit/appcache/appcache_storage_impl.cc b/webkit/appcache/appcache_storage_impl.cc
index a7bbdea..58f64cc 100644
--- a/webkit/appcache/appcache_storage_impl.cc
+++ b/webkit/appcache/appcache_storage_impl.cc
@@ -299,6 +299,7 @@ class AppCacheStorageImpl::StoreGroupAndCacheTask : public StoreOrLoadTask {
   scoped_refptr<AppCacheGroup> group_;
   scoped_refptr<AppCache> cache_;
   bool success_;
+  std::vector<int64> newly_deletable_response_ids_;
 };
 
 AppCacheStorageImpl::StoreGroupAndCacheTask::StoreGroupAndCacheTask(
@@ -335,12 +336,32 @@ void AppCacheStorageImpl::StoreGroupAndCacheTask::Run() {
 
     AppCacheDatabase::CacheRecord cache;
     if (database_->FindCacheForGroup(group_record_.group_id, &cache)) {
+      // Get the set of response ids in the old cache.
+      std::set<int64> existing_response_ids;
+      database_->FindResponseIdsForCacheAsSet(cache.cache_id,
+                                              &existing_response_ids);
+
+      // Remove those that remain in the new cache.
+      std::vector<AppCacheDatabase::EntryRecord>::const_iterator entry_iter =
+          entry_records_.begin();
+      while (entry_iter != entry_records_.end()) {
+        existing_response_ids.erase(entry_iter->response_id);
+        ++entry_iter;
+      }
+
+      // The rest are deletable.
+      std::set<int64>::const_iterator id_iter = existing_response_ids.begin();
+      while (id_iter != existing_response_ids.end()) {
+        newly_deletable_response_ids_.push_back(*id_iter);
+        ++id_iter;
+      }
+
       success_ =
           database_->DeleteCache(cache.cache_id) &&
           database_->DeleteEntriesForCache(cache.cache_id) &&
           database_->DeleteFallbackNameSpacesForCache(cache.cache_id) &&
-          database_->DeleteOnlineWhiteListForCache(cache.cache_id);
-      // TODO(michaeln): schedule to purge unused responses from the disk cache
+          database_->DeleteOnlineWhiteListForCache(cache.cache_id) &&
+          database_->InsertDeletableResponseIds(newly_deletable_response_ids_);
     } else {
       NOTREACHED() << "A existing group without a cache is unexpected";
     }
@@ -360,6 +381,7 @@ void AppCacheStorageImpl::StoreGroupAndCacheTask::RunCompleted() {
     storage_->origins_with_groups_.insert(group_->manifest_url().GetOrigin());
     if (cache_ != group_->newest_complete_cache())
       group_->AddCache(cache_);
+    group_->AddNewlyDeletableResponseIds(&newly_deletable_response_ids_);
   }
   FOR_EACH_DELEGATE(delegates_, OnGroupAndNewestCacheStored(group_, success_));
   group_ = NULL;
@@ -544,6 +566,7 @@ class AppCacheStorageImpl::MakeGroupObsoleteTask : public DatabaseTask {
   int64 group_id_;
   bool success_;
   std::set<GURL> origins_with_groups_;
+  std::vector<int64> newly_deletable_response_ids_;
 };
 
 AppCacheStorageImpl::MakeGroupObsoleteTask::MakeGroupObsoleteTask(
@@ -571,12 +594,15 @@ void AppCacheStorageImpl::MakeGroupObsoleteTask::Run() {
 
   AppCacheDatabase::CacheRecord cache_record;
   if (database_->FindCacheForGroup(group_id_, &cache_record)) {
+    database_->FindResponseIdsForCacheAsVector(cache_record.cache_id,
+                                               &newly_deletable_response_ids_);
     success_ =
         database_->DeleteGroup(group_id_) &&
         database_->DeleteCache(cache_record.cache_id) &&
         database_->DeleteEntriesForCache(cache_record.cache_id) &&
         database_->DeleteFallbackNameSpacesForCache(cache_record.cache_id) &&
-        database_->DeleteOnlineWhiteListForCache(cache_record.cache_id);
+        database_->DeleteOnlineWhiteListForCache(cache_record.cache_id) &&
+        database_->InsertDeletableResponseIds(newly_deletable_response_ids_);
   } else {
     NOTREACHED() << "A existing group without a cache is unexpected";
     success_ = database_->DeleteGroup(group_id_);
@@ -585,14 +611,13 @@ void AppCacheStorageImpl::MakeGroupObsoleteTask::Run() {
   success_ = success_ &&
              database_->FindOriginsWithGroups(&origins_with_groups_) &&
              transaction.Commit();
-
-  // TODO(michaeln): schedule to purge unused responses from the disk cache
 }
 
 void AppCacheStorageImpl::MakeGroupObsoleteTask::RunCompleted() {
   if (success_) {
     storage_->origins_with_groups_.swap(origins_with_groups_);
     group_->set_obsolete(true);
+    group_->AddNewlyDeletableResponseIds(&newly_deletable_response_ids_);
 
     // Also remove from the working set, caches for an 'obsolete' group
     // may linger in use, but the group itself cannot be looked up by
@@ -884,7 +909,7 @@ void AppCacheStorageImpl::DoomResponses(
   if (response_ids.empty())
     return;
 
-  // Start deleting them from the disk cache incrementally.
+  // Start deleting them from the disk cache lazily.
   StartDeletingResponses(response_ids);
 
   // Also schedule a database task to record these ids in the
@@ -898,6 +923,18 @@ void AppCacheStorageImpl::DoomResponses(
   task->Schedule();
 }
 
+void AppCacheStorageImpl::DeleteResponses(
+    const GURL& manifest_url, const std::vector<int64>& response_ids) {
+  if (response_ids.empty())
+    return;
+  StartDeletingResponses(response_ids);
+}
+
+void AppCacheStorageImpl::PurgeMemory() {
+  scoped_refptr<CloseConnectionTask> task = new CloseConnectionTask(this);
+  task->Schedule();
+}
+
 void AppCacheStorageImpl::DelayedStartDeletingUnusedResponses() {
   // Only if we haven't already begun.
   if (!did_start_deleting_responses_) {
diff --git a/webkit/appcache/appcache_storage_impl.h b/webkit/appcache/appcache_storage_impl.h
index dc28396..c7bae95 100644
--- a/webkit/appcache/appcache_storage_impl.h
+++ b/webkit/appcache/appcache_storage_impl.h
@@ -42,6 +42,9 @@ class AppCacheStorageImpl : public AppCacheStorage {
       const GURL& manifest_url);
   virtual void DoomResponses(
       const GURL& manifest_url, const std::vector<int64>& response_ids);
+  virtual void DeleteResponses(
+      const GURL& manifest_url, const std::vector<int64>& response_ids);
+  virtual void PurgeMemory();
 
  private:
   friend class AppCacheStorageImplTest;
diff --git a/webkit/appcache/appcache_update_job.cc b/webkit/appcache/appcache_update_job.cc
index a1f4ec5..f138f4d 100644
--- a/webkit/appcache/appcache_update_job.cc
+++ b/webkit/appcache/appcache_update_job.cc
@@ -542,7 +542,7 @@ void AppCacheUpdateJob::HandleUrlFetchCompleted(URLRequest* request) {
     entry.set_response_size(info->response_writer_->amount_written());
 
     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
-      service_->storage()->DoomResponse(manifest_url_, entry.response_id());
+      duplicate_response_ids_.push_back(entry.response_id());
 
     // Foreign entries will be detected during cache selection.
     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
@@ -612,10 +612,8 @@ void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(URLRequest* request) {
     AppCacheEntry master_entry(AppCacheEntry::MASTER,
                                info->response_writer_->response_id(),
                                info->response_writer_->amount_written());
-    if (!cache->AddOrModifyEntry(url, master_entry)) {
-      service_->storage()->DoomResponse(
-          manifest_url_, master_entry.response_id());
-    }
+    if (!cache->AddOrModifyEntry(url, master_entry))
+      duplicate_response_ids_.push_back(master_entry.response_id());
 
     // In no-update case, associate host with the newest cache.
     if (!inprogress_cache_) {
@@ -711,7 +709,7 @@ void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
         manifest_response_writer_->response_id(),
         manifest_response_writer_->amount_written());
     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
-      service_->storage()->DoomResponse(manifest_url_, entry.response_id());
+      duplicate_response_ids_.push_back(entry.response_id());
     CompleteInprogressCache();
   } else {
     // Treat storage failure as if refetch of manifest failed.
@@ -1145,6 +1143,7 @@ void AppCacheUpdateJob::MaybeCompleteUpdate() {
     case NO_UPDATE:
       // 6.9.4 steps 7.3-7.7.
       NotifyAllAssociatedHosts(NO_UPDATE_EVENT);
+      DiscardDuplicateResponses();
       internal_state_ = COMPLETED;
       break;
     case DOWNLOADING:
@@ -1156,6 +1155,7 @@ void AppCacheUpdateJob::MaybeCompleteUpdate() {
         NotifyAllAssociatedHosts(CACHED_EVENT);
       else
         NotifyAllAssociatedHosts(UPDATE_READY_EVENT);
+      DiscardDuplicateResponses();
       internal_state_ = COMPLETED;
       break;
     case CACHE_FAILURE:
@@ -1239,6 +1239,10 @@ void AppCacheUpdateJob::DiscardInprogressCache() {
   inprogress_cache_ = NULL;
 }
 
+void AppCacheUpdateJob::DiscardDuplicateResponses() {
+  service_->storage()->DoomResponses(manifest_url_, duplicate_response_ids_);
+}
+
 void AppCacheUpdateJob::DeleteSoon() {
   ClearPendingMasterEntries();
   manifest_response_writer_.reset();
diff --git a/webkit/appcache/appcache_update_job.h b/webkit/appcache/appcache_update_job.h
index e4d50ca..1eec1a4 100644
--- a/webkit/appcache/appcache_update_job.h
+++ b/webkit/appcache/appcache_update_job.h
@@ -174,6 +174,7 @@ class AppCacheUpdateJob : public URLRequest::Delegate,
   void Cancel();
   void ClearPendingMasterEntries();
   void DiscardInprogressCache();
+  void DiscardDuplicateResponses();
 
   // Deletes this object after letting the stack unwind.
   void DeleteSoon();
@@ -240,6 +241,13 @@ class AppCacheUpdateJob : public URLRequest::Delegate,
   // error conditions.
   std::vector<int64> stored_response_ids_;
 
+  // In some cases we fetch the same resource multiple times, and then
+  // have to delete the duplicates upon successful update. These ids
+  // are also in the stored_response_ids_ collection so we only schedule
+  // these for deletion on success.
+  // TODO(michaeln): Rework when we no longer fetches master entries directly.
+  std::vector<int64> duplicate_response_ids_;
+
   net::CompletionCallbackImpl<AppCacheUpdateJob> manifest_info_write_callback_;
   net::CompletionCallbackImpl<AppCacheUpdateJob> manifest_data_write_callback_;
   net::CompletionCallbackImpl<AppCacheUpdateJob> manifest_data_read_callback_;
diff --git a/webkit/appcache/mock_appcache_storage.cc b/webkit/appcache/mock_appcache_storage.cc
index 3c93479..750a8ce 100644
--- a/webkit/appcache/mock_appcache_storage.cc
+++ b/webkit/appcache/mock_appcache_storage.cc
@@ -149,8 +149,13 @@ AppCacheResponseWriter* MockAppCacheStorage::CreateResponseWriter(
 
 void MockAppCacheStorage::DoomResponses(
     const GURL& manifest_url, const std::vector<int64>& response_ids) {
+  DeleteResponses(manifest_url, response_ids);
+}
+
+void MockAppCacheStorage::DeleteResponses(
+    const GURL& manifest_url, const std::vector<int64>& response_ids) {
   // We don't bother with actually removing responses from the disk-cache,
-  // just keep track of which ids have been doomed.
+  // just keep track of which ids have been doomed or deleted
   std::vector<int64>::const_iterator it = response_ids.begin();
   while (it != response_ids.end()) {
     doomed_response_ids_.insert(*it);
diff --git a/webkit/appcache/mock_appcache_storage.h b/webkit/appcache/mock_appcache_storage.h
index 0337d495..48834bd 100644
--- a/webkit/appcache/mock_appcache_storage.h
+++ b/webkit/appcache/mock_appcache_storage.h
@@ -7,6 +7,7 @@
 
 #include <deque>
 #include <map>
+#include <vector>
 
 #include "base/hash_tables.h"
 #include "base/scoped_ptr.h"
@@ -43,6 +44,9 @@ class MockAppCacheStorage : public AppCacheStorage {
   virtual AppCacheResponseWriter* CreateResponseWriter(const GURL& origin);
   virtual void DoomResponses(
       const GURL& manifest_url, const std::vector<int64>& response_ids);
+  virtual void DeleteResponses(
+      const GURL& manifest_url, const std::vector<int64>& response_ids);
+  virtual void PurgeMemory() {}
 
  private:
   friend class AppCacheRequestHandlerTest;