drive: Add ResourceMetadataStorage::UpgradeOldDB

UpgradeOldDB is responsible to DB version backward compatibility.
When encountering old version DB, it erases all entries except cache entries, canonicalizes IDs and add resource-ID-to-local-ID mapping entries.

Fix ResourceMetadata::AddEntry and ResourceMetadataStorage::CheckValidity to allow resource-ID-to-local-ID entry without corresponding ResourceEntry.
Remove FileCache::CanonicalizeIDs.
Add a new UMA histogram "Drive.InputDBVersion"

BUG=309597
TEST=unit_tests
R=asvitkine@chromium.org, kinaba@chromium.org

Review URL: https://codereview.chromium.org/33533003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@230625 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 147 insertions, 103 deletions

diff --git a/chrome/browser/chromeos/drive/drive_integration_service.cc b/chrome/browser/chromeos/drive/drive_integration_service.cc
index 9e4dbd1c..9589f065 100644
--- a/chrome/browser/chromeos/drive/drive_integration_service.cc
+++ b/chrome/browser/chromeos/drive/drive_integration_service.cc
@@ -115,12 +115,15 @@ FileError InitializeMetadata(
       file_util::FILE_PERMISSION_EXECUTE_BY_GROUP |
       file_util::FILE_PERMISSION_EXECUTE_BY_OTHERS);
 
+  internal::ResourceMetadataStorage::UpgradeOldDB(
+      metadata_storage->directory_path(), id_canonicalizer);
+
   if (!metadata_storage->Initialize()) {
     LOG(WARNING) << "Failed to initialize the metadata storage.";
     return FILE_ERROR_FAILED;
   }
 
-  if (!cache->Initialize() || !cache->CanonicalizeIDs(id_canonicalizer)) {
+  if (!cache->Initialize()) {
     LOG(WARNING) << "Failed to initialize the cache.";
     return FILE_ERROR_FAILED;
   }
diff --git a/chrome/browser/chromeos/drive/file_cache.cc b/chrome/browser/chromeos/drive/file_cache.cc
index ca614c32..f19abd0 100644
--- a/chrome/browser/chromeos/drive/file_cache.cc
+++ b/chrome/browser/chromeos/drive/file_cache.cc
@@ -420,21 +420,6 @@ void FileCache::Destroy() {
       base::Bind(&FileCache::DestroyOnBlockingPool, base::Unretained(this)));
 }
 
-bool FileCache::CanonicalizeIDs(
-    const ResourceIdCanonicalizer& id_canonicalizer) {
-  scoped_ptr<Iterator> it = GetIterator();
-  for (; !it->IsAtEnd(); it->Advance()) {
-    const std::string id_canonicalized = id_canonicalizer.Run(it->GetID());
-    if (id_canonicalized != it->GetID()) {
-      // Replace the existing entry.
-      if (!storage_->RemoveCacheEntry(it->GetID()) ||
-          !storage_->PutCacheEntry(id_canonicalized, it->GetValue()))
-        return false;
-    }
-  }
-  return !it->HasError();
-}
-
 void FileCache::DestroyOnBlockingPool() {
   AssertOnSequencedWorkerPool();
   delete this;
diff --git a/chrome/browser/chromeos/drive/file_cache.h b/chrome/browser/chromeos/drive/file_cache.h
index ba6aa1a..e6a752a 100644
--- a/chrome/browser/chromeos/drive/file_cache.h
+++ b/chrome/browser/chromeos/drive/file_cache.h
@@ -15,7 +15,6 @@
 #include "base/memory/weak_ptr.h"
 #include "chrome/browser/chromeos/drive/file_errors.h"
 #include "chrome/browser/chromeos/drive/resource_metadata_storage.h"
-#include "chrome/browser/drive/drive_service_interface.h"
 
 namespace base {
 class SequencedTaskRunner;
@@ -156,10 +155,6 @@ class FileCache {
   // Must be called on the UI thread.
   void Destroy();
 
-  // Converts entry IDs and cache file names to the desired format.
-  // TODO(hashimoto): Remove this method at some point.
-  bool CanonicalizeIDs(const ResourceIdCanonicalizer& id_canonicalizer);
-
  private:
   friend class FileCacheTest;
   friend class FileCacheTestOnUIThread;
diff --git a/chrome/browser/chromeos/drive/file_cache_unittest.cc b/chrome/browser/chromeos/drive/file_cache_unittest.cc
index d2c8810..b3fa3ed 100644
--- a/chrome/browser/chromeos/drive/file_cache_unittest.cc
+++ b/chrome/browser/chromeos/drive/file_cache_unittest.cc
@@ -12,7 +12,6 @@
 #include "base/files/scoped_temp_dir.h"
 #include "base/md5.h"
 #include "base/run_loop.h"
-#include "base/strings/string_util.h"
 #include "base/threading/sequenced_worker_pool.h"
 #include "chrome/browser/chromeos/drive/drive.pb.h"
 #include "chrome/browser/chromeos/drive/fake_free_disk_space_getter.h"
@@ -895,30 +894,6 @@ TEST_F(FileCacheTest, GetFile) {
   EXPECT_EQ(src_contents, contents);
 }
 
-TEST_F(FileCacheTest, CanonicalizeIDs) {
-  ResourceIdCanonicalizer id_canonicalizer = base::Bind(
-      (ResourceIdCanonicalizer::RunType*)(&StringToUpperASCII));
-  const std::string id("abc");
-  const std::string md5("abcdef0123456789");
-
-  const base::FilePath file_directory =
-      temp_dir_.path().AppendASCII(kCacheFileDirectory);
-
-  // Store a file to the cache.
-  base::FilePath file;
-  EXPECT_TRUE(file_util::CreateTemporaryFileInDir(temp_dir_.path(), &file));
-  EXPECT_EQ(FILE_ERROR_OK,
-            cache_->Store(id, md5, file, FileCache::FILE_OPERATION_COPY));
-
-  // Canonicalize IDs.
-  EXPECT_TRUE(cache_->CanonicalizeIDs(id_canonicalizer));
-
-  const std::string canonicalized_id = id_canonicalizer.Run(id);
-  FileCacheEntry entry;
-  EXPECT_FALSE(cache_->GetCacheEntry(id, &entry));
-  EXPECT_TRUE(cache_->GetCacheEntry(canonicalized_id, &entry));
-}
-
 TEST_F(FileCacheTest, RenameCacheFilesToNewFormat) {
   const base::FilePath file_directory =
       temp_dir_.path().AppendASCII(kCacheFileDirectory);
diff --git a/chrome/browser/chromeos/drive/resource_metadata.cc b/chrome/browser/chromeos/drive/resource_metadata.cc
index b0e5f87..742bb48 100644
--- a/chrome/browser/chromeos/drive/resource_metadata.cc
+++ b/chrome/browser/chromeos/drive/resource_metadata.cc
@@ -167,24 +167,22 @@ FileError ResourceMetadata::AddEntry(const ResourceEntry& entry,
   if (!EnoughDiskSpaceIsAvailableForDBOperation(storage_->directory_path()))
     return FILE_ERROR_NO_LOCAL_SPACE;
 
-  // Multiple entries with the same resource ID should not be present.
-  std::string existing_entry_id;
-  if (!entry.resource_id().empty() &&
-      GetIdByResourceId(entry.resource_id(),
-                        &existing_entry_id) == FILE_ERROR_OK)
-    return FILE_ERROR_EXISTS;
-
   ResourceEntry parent;
   if (!storage_->GetEntry(entry.parent_local_id(), &parent) ||
       !parent.file_info().is_directory())
     return FILE_ERROR_NOT_FOUND;
 
-  // Generate unique local ID.
+  // Multiple entries with the same resource ID should not be present.
   std::string local_id;
   ResourceEntry existing_entry;
-  do {
+  if (!entry.resource_id().empty() &&
+      storage_->GetIdByResourceId(entry.resource_id(), &local_id) &&
+      storage_->GetEntry(local_id, &existing_entry))
+    return FILE_ERROR_EXISTS;
+
+  // Generate unique local ID when needed.
+  while (local_id.empty() || storage_->GetEntry(local_id, &existing_entry))
     local_id = base::GenerateGUID();
-  } while (storage_->GetEntry(local_id, &existing_entry));
 
   ResourceEntry new_entry(entry);
   new_entry.set_local_id(local_id);
diff --git a/chrome/browser/chromeos/drive/resource_metadata_storage.cc b/chrome/browser/chromeos/drive/resource_metadata_storage.cc
index 948e737..b266b30 100644
--- a/chrome/browser/chromeos/drive/resource_metadata_storage.cc
+++ b/chrome/browser/chromeos/drive/resource_metadata_storage.cc
@@ -9,6 +9,7 @@
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/metrics/histogram.h"
+#include "base/metrics/sparse_histogram.h"
 #include "base/sequenced_task_runner.h"
 #include "base/threading/thread_restrictions.h"
 #include "chrome/browser/chromeos/drive/drive.pb.h"
@@ -80,6 +81,15 @@ bool IsCacheEntryKey(const leveldb::Slice& key) {
   return key_substring.compare(expected_suffix) == 0;
 }
 
+// Returns ID extracted from a cache entry key.
+std::string GetIdFromCacheEntryKey(const leveldb::Slice& key) {
+  DCHECK(IsCacheEntryKey(key));
+  // Drop the suffix |kDBKeyDelimeter + kCacheEntryKeySuffix| from the key.
+  const size_t kSuffixLength = arraysize(kCacheEntryKeySuffix) - 1;
+  const int id_length = key.size() - 1 - kSuffixLength;
+  return std::string(key.data(), id_length);
+}
+
 // Returns a string to be used as a key for a resource-ID-to-local-ID entry.
 std::string GetIdEntryKey(const std::string& resource_id) {
   std::string key;
@@ -279,15 +289,85 @@ void ResourceMetadataStorage::CacheEntryIterator::AdvanceInternal() {
     // TODO(hashimoto): Broken entries should be cleaned up at some point.
     if (IsCacheEntryKey(it_->key()) &&
         entry_.ParseFromArray(it_->value().data(), it_->value().size())) {
-      // Drop the suffix |kDBKeyDelimeter + kCacheEntryKeySuffix| from the key.
-      const size_t kSuffixLength = arraysize(kCacheEntryKeySuffix) - 1;
-      const int id_length = it_->key().size() - 1 - kSuffixLength;
-      id_.assign(it_->key().data(), id_length);
+      id_ = GetIdFromCacheEntryKey(it_->key());
       break;
     }
   }
 }
 
+// static
+bool ResourceMetadataStorage::UpgradeOldDB(
+    const base::FilePath& directory_path,
+    const ResourceIdCanonicalizer& id_canonicalizer) {
+  base::ThreadRestrictions::AssertIOAllowed();
+  COMPILE_ASSERT(
+      kDBVersion == 11,
+      db_version_and_this_function_should_be_updated_at_the_same_time);
+
+  // Open DB.
+  leveldb::DB* db = NULL;
+  leveldb::Options options;
+  options.max_open_files = 0;  // Use minimum.
+  options.create_if_missing = false;
+  if (!leveldb::DB::Open(
+          options,
+          directory_path.Append(kResourceMapDBName).AsUTF8Unsafe(), &db).ok())
+    return false;
+  scoped_ptr<leveldb::DB> resource_map(db);
+
+  // Check DB version.
+  std::string serialized_header;
+  ResourceMetadataHeader header;
+  if (!resource_map->Get(leveldb::ReadOptions(),
+                         leveldb::Slice(GetHeaderDBKey()),
+                         &serialized_header).ok() ||
+      !header.ParseFromString(serialized_header))
+    return false;
+  UMA_HISTOGRAM_SPARSE_SLOWLY("Drive.MetadataDBVersionBeforeUpgradeCheck",
+                              header.version());
+
+  if (header.version() == kDBVersion) {  // Nothing to do.
+    return true;
+  } else if (header.version() < 6) {  // Too old, nothing can be done.
+    return false;
+  } else if (header.version() < 11) {  // Cache entries can be reused.
+    leveldb::ReadOptions options;
+    options.verify_checksums = true;
+    scoped_ptr<leveldb::Iterator> it(resource_map->NewIterator(options));
+
+    leveldb::WriteBatch batch;
+    for (it->SeekToFirst(); it->Valid(); it->Next()) {
+      if (IsCacheEntryKey(it->key())) {
+        // The resource ID might be in old WAPI format. We need to canonicalize
+        // to the format of API service currently in use.
+        const std::string& id = GetIdFromCacheEntryKey(it->key());
+        const std::string& id_new = id_canonicalizer.Run(id);
+        if (id != id_new) {
+          batch.Delete(it->key());
+          batch.Put(GetCacheEntryKey(id_new), it->value());
+        }
+        // Before v11, resource ID was directly used as local ID. Such entries
+        // can be migrated by adding an identity ID mapping.
+        batch.Put(GetIdEntryKey(id_new), id_new);
+      } else {  // Remove all entries except cache entries.
+        batch.Delete(it->key());
+      }
+    }
+    if (!it->status().ok())
+      return false;
+
+    // Put header with the latest version number.
+    std::string serialized_header;
+    if (!GetDefaultHeaderEntry().SerializeToString(&serialized_header))
+      return false;
+    batch.Put(GetHeaderDBKey(), serialized_header);
+
+    return resource_map->Write(leveldb::WriteOptions(), &batch).ok();
+  }
+  LOG(WARNING) << "Unexpected DB version: " << header.version();
+  return false;
+}
+
 ResourceMetadataStorage::ResourceMetadataStorage(
     const base::FilePath& directory_path,
     base::SequencedTaskRunner* blocking_task_runner)
@@ -336,25 +416,6 @@ bool ResourceMetadataStorage::Initialize() {
     bool should_discard_db = true;
     if (db_version != kDBVersion) {
       open_existing_result = DB_INIT_INCOMPATIBLE;
-
-      // We can reuse cache entries when appropriate.
-      if (6 <= db_version && db_version < kDBVersion) {
-        // Remove all entries except cache entries.
-        leveldb::ReadOptions options;
-        options.verify_checksums = true;
-        scoped_ptr<leveldb::Iterator> it(resource_map_->NewIterator(options));
-
-        leveldb::WriteBatch batch;
-        for (it->SeekToFirst(); it->Valid(); it->Next()) {
-          if (!IsCacheEntryKey(it->key()))
-            batch.Delete(it->key());
-        }
-
-        should_discard_db =
-            !it->status().ok() ||
-            !resource_map_->Write(leveldb::WriteOptions(), &batch).ok() ||
-            !PutHeader(GetDefaultHeaderEntry());
-      }
       LOG(INFO) << "Reject incompatible DB.";
     } else if (!CheckValidity()) {
       open_existing_result = DB_INIT_BROKEN;
@@ -731,13 +792,16 @@ bool ResourceMetadataStorage::CheckValidity() {
     // Check if resource-ID-to-local-ID mapping is stored correctly.
     if (IsIdEntryKey(it->key())) {
       leveldb::Status status = resource_map_->Get(
-          options,
-          it->value(),
-          &serialized_entry);
-      if (!status.ok() ||
-          !entry.ParseFromString(serialized_entry) ||
-          entry.resource_id().empty() ||
-          leveldb::Slice(GetIdEntryKey(entry.resource_id())) != it->key()) {
+          options, it->value(), &serialized_entry);
+      // Resource-ID-to-local-ID mapping without entry for the local ID is ok.
+      if (status.IsNotFound())
+        continue;
+      // When the entry exists, its resource ID must be consistent.
+      const bool ok = status.ok() &&
+          entry.ParseFromString(serialized_entry) &&
+          !entry.resource_id().empty() &&
+          leveldb::Slice(GetIdEntryKey(entry.resource_id())) == it->key();
+      if (!ok) {
         DLOG(ERROR) << "Broken ID entry. status = " << status.ToString();
         return false;
       }
diff --git a/chrome/browser/chromeos/drive/resource_metadata_storage.h b/chrome/browser/chromeos/drive/resource_metadata_storage.h
index 04a2224..b19d977 100644
--- a/chrome/browser/chromeos/drive/resource_metadata_storage.h
+++ b/chrome/browser/chromeos/drive/resource_metadata_storage.h
@@ -13,6 +13,7 @@
 #include "base/memory/ref_counted.h"
 #include "base/memory/scoped_ptr.h"
 #include "chrome/browser/chromeos/drive/drive.pb.h"
+#include "chrome/browser/drive/drive_service_interface.h"
 
 namespace base {
 class SequencedTaskRunner;
@@ -104,6 +105,10 @@ class ResourceMetadataStorage {
     DISALLOW_COPY_AND_ASSIGN(CacheEntryIterator);
   };
 
+  // Returns true if the DB was successfully upgraded to the newest version.
+  static bool UpgradeOldDB(const base::FilePath& directory_path,
+                           const ResourceIdCanonicalizer& id_canonicalizer);
+
   ResourceMetadataStorage(const base::FilePath& directory_path,
                           base::SequencedTaskRunner* blocking_task_runner);
 
diff --git a/chrome/browser/chromeos/drive/resource_metadata_storage_unittest.cc b/chrome/browser/chromeos/drive/resource_metadata_storage_unittest.cc
index f4cbfb6..7d73eb5 100644
--- a/chrome/browser/chromeos/drive/resource_metadata_storage_unittest.cc
+++ b/chrome/browser/chromeos/drive/resource_metadata_storage_unittest.cc
@@ -10,9 +10,11 @@
 #include "base/files/scoped_temp_dir.h"
 #include "chrome/browser/chromeos/drive/drive.pb.h"
 #include "chrome/browser/chromeos/drive/test_util.h"
+#include "chrome/browser/drive/drive_api_util.h"
 #include "content/public/test/test_browser_thread_bundle.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/leveldatabase/src/include/leveldb/db.h"
+#include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
 
 namespace drive {
 namespace internal {
@@ -39,6 +41,8 @@ class ResourceMetadataStorageTest : public testing::Test {
     return storage_->CheckValidity();
   }
 
+  leveldb::DB* resource_map() { return storage_->resource_map_.get(); }
+
   // Puts a child entry.
   void PutChild(const std::string& parent_id,
                 const std::string& child_base_name,
@@ -358,30 +362,44 @@ TEST_F(ResourceMetadataStorageTest, OpenExistingDB) {
   EXPECT_EQ(child_id1, storage_->GetChild(parent_id1, child_name1));
 }
 
-TEST_F(ResourceMetadataStorageTest, IncompatibleDB_Old) {
+TEST_F(ResourceMetadataStorageTest, IncompatibleDB_M29) {
   const int64 kLargestChangestamp = 1234567890;
-  const std::string key1 = "abcd";
 
-  // Put some data.
+  // Construct M29 version DB.
+  SetDBVersion(6);
   EXPECT_TRUE(storage_->SetLargestChangestamp(kLargestChangestamp));
+
+  leveldb::WriteBatch batch;
+
+  // Put a file entry and its cache entry.
   ResourceEntry entry;
-  entry.set_local_id(key1);
-  EXPECT_TRUE(storage_->PutEntry(entry));
-  EXPECT_TRUE(storage_->GetEntry(key1, &entry));
+  std::string serialized_entry;
+  entry.set_resource_id("file:abcd");
+  EXPECT_TRUE(entry.SerializeToString(&serialized_entry));
+  batch.Put("file:abcd", serialized_entry);
+
   FileCacheEntry cache_entry;
-  EXPECT_TRUE(storage_->PutCacheEntry(key1, FileCacheEntry()));
-  EXPECT_TRUE(storage_->GetCacheEntry(key1, &cache_entry));
+  EXPECT_TRUE(cache_entry.SerializeToString(&serialized_entry));
+  batch.Put(std::string("file:abcd") + '\0' + "CACHE", serialized_entry);
+
+  EXPECT_TRUE(resource_map()->Write(leveldb::WriteOptions(), &batch).ok());
 
-  // Set older version and reopen DB.
-  SetDBVersion(ResourceMetadataStorage::kDBVersion - 1);
+  // Upgrade and reopen.
+  storage_.reset();
+  EXPECT_TRUE(ResourceMetadataStorage::UpgradeOldDB(
+      temp_dir_.path(), base::Bind(&util::CanonicalizeResourceId)));
   storage_.reset(new ResourceMetadataStorage(
       temp_dir_.path(), base::MessageLoopProxy::current().get()));
   ASSERT_TRUE(storage_->Initialize());
 
-  // Data is erased, except cache entries, because of the incompatible version.
+  // Resource-ID-to-local-ID mapping is added.
+  std::string id;
+  EXPECT_TRUE(storage_->GetIdByResourceId("abcd", &id));  // "file:" is dropped.
+
+  // Data is erased, except cache entries.
   EXPECT_EQ(0, storage_->GetLargestChangestamp());
-  EXPECT_FALSE(storage_->GetEntry(key1, &entry));
-  EXPECT_TRUE(storage_->GetCacheEntry(key1, &cache_entry));
+  EXPECT_FALSE(storage_->GetEntry(id, &entry));
+  EXPECT_TRUE(storage_->GetCacheEntry(id, &cache_entry));
 }
 
 TEST_F(ResourceMetadataStorageTest, IncompatibleDB_Unknown) {
@@ -393,13 +411,14 @@ TEST_F(ResourceMetadataStorageTest, IncompatibleDB_Unknown) {
   ResourceEntry entry;
   entry.set_local_id(key1);
   EXPECT_TRUE(storage_->PutEntry(entry));
-  EXPECT_TRUE(storage_->GetEntry(key1, &entry));
   FileCacheEntry cache_entry;
-  EXPECT_TRUE(storage_->PutCacheEntry(key1, FileCacheEntry()));
-  EXPECT_TRUE(storage_->GetCacheEntry(key1, &cache_entry));
+  EXPECT_TRUE(storage_->PutCacheEntry(key1, cache_entry));
 
-  // Set newer version and reopen DB.
+  // Set newer version, upgrade and reopen DB.
   SetDBVersion(ResourceMetadataStorage::kDBVersion + 1);
+  storage_.reset();
+  EXPECT_FALSE(ResourceMetadataStorage::UpgradeOldDB(
+      temp_dir_.path(), base::Bind(&util::CanonicalizeResourceId)));
   storage_.reset(new ResourceMetadataStorage(
       temp_dir_.path(), base::MessageLoopProxy::current().get()));
   ASSERT_TRUE(storage_->Initialize());