drive: Rename CacheMetadata and friends to to FileBlah

This is to make them consistent with FileCache
introduced in crrev.com/197568

BUG=231807
TEST=none
R=kinaba@chromium.org

Review URL: https://codereview.chromium.org/14751003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@197571 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 576 insertions, 0 deletions

diff --git a/chrome/browser/chromeos/drive/file_cache_metadata.cc b/chrome/browser/chromeos/drive/file_cache_metadata.cc
new file mode 100644
index 0000000..ec02a01
--- /dev/null
+++ b/chrome/browser/chromeos/drive/file_cache_metadata.cc
@@ -0,0 +1,576 @@
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/chromeos/drive/file_cache_metadata.h"
+
+#include "base/callback.h"
+#include "base/file_util.h"
+#include "base/metrics/histogram.h"
+#include "base/sequenced_task_runner.h"
+#include "chrome/browser/chromeos/drive/drive.pb.h"
+#include "chrome/browser/chromeos/drive/file_cache.h"
+#include "chrome/browser/chromeos/drive/file_system_util.h"
+#include "third_party/leveldatabase/src/include/leveldb/db.h"
+
+namespace drive {
+
+namespace {
+
+enum DBOpenStatus {
+  DB_OPEN_SUCCESS,
+  DB_OPEN_FAILURE_CORRUPTION,
+  DB_OPEN_FAILURE_OTHER,
+  DB_OPEN_FAILURE_UNRECOVERABLE,
+  DB_OPEN_MAX_VALUE,
+};
+
+// A map table of resource ID to file path.
+typedef std::map<std::string, base::FilePath> ResourceIdToFilePathMap;
+
+// Returns true if |file_path| is a valid symbolic link as |sub_dir_type|.
+// Otherwise, returns false with the reason.
+bool IsValidSymbolicLink(const base::FilePath& file_path,
+                         FileCache::CacheSubDirectoryType sub_dir_type,
+                         const std::vector<base::FilePath>& cache_paths,
+                         std::string* reason) {
+  DCHECK_EQ(FileCache::CACHE_TYPE_OUTGOING, sub_dir_type);
+
+  base::FilePath destination;
+  if (!file_util::ReadSymbolicLink(file_path, &destination)) {
+    *reason = "failed to read the symlink (maybe not a symlink)";
+    return false;
+  }
+
+  if (!file_util::PathExists(destination)) {
+    *reason = "pointing to a non-existent file";
+    return false;
+  }
+
+  // The destination file should be in the persistent directory.
+  if (!cache_paths[FileCache::CACHE_TYPE_PERSISTENT].IsParent(destination)) {
+    *reason = "pointing to a file outside of persistent directory";
+    return false;
+  }
+
+  return true;
+}
+
+// Scans cache subdirectory and build or update |cache_map|
+// with found file blobs or symlinks.
+//
+// The resource IDs and file paths of discovered files are collected as a
+// ResourceIdToFilePathMap, if these are processed properly.
+void ScanCacheDirectory(
+    const std::vector<base::FilePath>& cache_paths,
+    FileCache::CacheSubDirectoryType sub_dir_type,
+    FileCacheMetadata::CacheMap* cache_map,
+    ResourceIdToFilePathMap* processed_file_map) {
+  DCHECK(cache_map);
+  DCHECK(processed_file_map);
+
+  file_util::FileEnumerator enumerator(
+      cache_paths[sub_dir_type],
+      false,  // not recursive
+      file_util::FileEnumerator::FILES |
+      file_util::FileEnumerator::SHOW_SYM_LINKS,
+      util::kWildCard);
+  for (base::FilePath current = enumerator.Next(); !current.empty();
+       current = enumerator.Next()) {
+    // Extract resource_id and md5 from filename.
+    std::string resource_id;
+    std::string md5;
+    std::string extra_extension;
+    util::ParseCacheFilePath(current, &resource_id, &md5, &extra_extension);
+
+    // Determine cache state.
+    FileCacheEntry cache_entry;
+    cache_entry.set_md5(md5);
+    if (sub_dir_type == FileCache::CACHE_TYPE_OUTGOING) {
+      std::string reason;
+      if (!IsValidSymbolicLink(current, sub_dir_type, cache_paths, &reason)) {
+        LOG(WARNING) << "Removing an invalid symlink: " << current.value()
+                     << ": " << reason;
+        file_util::Delete(current, false);
+        continue;
+      }
+
+      // If we're scanning outgoing directory, entry must exist and be dirty.
+      // Otherwise, it's a logic error from previous execution, remove this
+      // outgoing symlink and move on.
+      FileCacheMetadata::CacheMap::iterator iter =
+          cache_map->find(resource_id);
+      if (iter == cache_map->end() || !iter->second.is_dirty()) {
+        LOG(WARNING) << "Removing an symlink to a non-dirty file: "
+                     << current.value();
+        file_util::Delete(current, false);
+        continue;
+      }
+
+      processed_file_map->insert(std::make_pair(resource_id, current));
+      continue;
+    } else if (sub_dir_type == FileCache::CACHE_TYPE_PERSISTENT ||
+               sub_dir_type == FileCache::CACHE_TYPE_TMP) {
+      if (sub_dir_type == FileCache::CACHE_TYPE_PERSISTENT)
+        cache_entry.set_is_persistent(true);
+
+      if (file_util::IsLink(current)) {
+        LOG(WARNING) << "Removing a symlink in persistent/tmp directory"
+                     << current.value();
+        file_util::Delete(current, false);
+        continue;
+      }
+      if (extra_extension == util::kMountedArchiveFileExtension) {
+        // Mounted archives in cache should be unmounted upon logout/shutdown.
+        // But if we encounter a mounted file at start, delete it and create an
+        // entry with not PRESENT state.
+        DCHECK(sub_dir_type == FileCache::CACHE_TYPE_PERSISTENT);
+        file_util::Delete(current, false);
+      } else {
+        // The cache file is present.
+        cache_entry.set_is_present(true);
+
+        // Adds the dirty bit if |md5| indicates that the file is dirty, and
+        // the file is in the persistent directory.
+        if (md5 == util::kLocallyModifiedFileExtension) {
+          if (sub_dir_type == FileCache::CACHE_TYPE_PERSISTENT) {
+            cache_entry.set_is_dirty(true);
+          } else {
+            LOG(WARNING) << "Removing a dirty file in tmp directory: "
+                         << current.value();
+            file_util::Delete(current, false);
+            continue;
+          }
+        }
+      }
+    } else {
+      NOTREACHED() << "Unexpected sub directory type: " << sub_dir_type;
+    }
+
+    // Create and insert new entry into cache map.
+    cache_map->insert(std::make_pair(resource_id, cache_entry));
+    processed_file_map->insert(std::make_pair(resource_id, current));
+  }
+}
+
+void ScanCachePaths(const std::vector<base::FilePath>& cache_paths,
+                    FileCacheMetadata::CacheMap* cache_map) {
+  DVLOG(1) << "Scanning directories";
+
+  // Scan cache persistent and tmp directories to enumerate all files and create
+  // corresponding entries for cache map.
+  ResourceIdToFilePathMap persistent_file_map;
+  ScanCacheDirectory(cache_paths,
+                     FileCache::CACHE_TYPE_PERSISTENT,
+                     cache_map,
+                     &persistent_file_map);
+  ResourceIdToFilePathMap tmp_file_map;
+  ScanCacheDirectory(cache_paths,
+                     FileCache::CACHE_TYPE_TMP,
+                     cache_map,
+                     &tmp_file_map);
+
+  // Then scan outgoing directory to check if dirty-files are committed
+  // properly (i.e. symlinks created in outgoing directory).
+  ResourceIdToFilePathMap outgoing_file_map;
+  ScanCacheDirectory(cache_paths,
+                     FileCache::CACHE_TYPE_OUTGOING,
+                     cache_map,
+                     &outgoing_file_map);
+
+  // On DB corruption, keep only dirty-and-committed files in persistent
+  // directory. Other files are deleted or moved to temporary directory.
+  for (ResourceIdToFilePathMap::const_iterator iter =
+           persistent_file_map.begin();
+       iter != persistent_file_map.end(); ++iter) {
+    const std::string& resource_id = iter->first;
+    const base::FilePath& file_path = iter->second;
+
+    FileCacheMetadata::CacheMap::iterator cache_map_iter =
+        cache_map->find(resource_id);
+    if (cache_map_iter != cache_map->end()) {
+      FileCacheEntry* cache_entry = &cache_map_iter->second;
+      const bool is_dirty = cache_entry->is_dirty();
+      const bool is_committed = outgoing_file_map.count(resource_id) != 0;
+      if (!is_dirty && !is_committed) {
+        // If the file is not dirty nor committed, move to temporary directory.
+        base::FilePath new_file_path =
+            cache_paths[FileCache::CACHE_TYPE_TMP].Append(
+                file_path.BaseName());
+        DLOG(WARNING) << "Moving: " << file_path.value()
+                      << " to: " << new_file_path.value();
+        file_util::Move(file_path, new_file_path);
+        cache_entry->set_is_persistent(false);
+      } else if (!is_dirty || !is_committed) {
+        // If the file is not dirty-and-committed, remove it.
+        DLOG(WARNING) << "Removing: " << file_path.value();
+        file_util::Delete(file_path, false);
+        cache_map->erase(cache_map_iter);
+      }
+    }
+  }
+  DVLOG(1) << "Directory scan finished";
+}
+
+// Returns true if |md5| matches the one in |cache_entry| with some
+// exceptions. See the function definition for details.
+bool CheckIfMd5Matches(
+    const std::string& md5,
+    const FileCacheEntry& cache_entry) {
+  if (cache_entry.is_dirty()) {
+    // If the entry is dirty, its MD5 may have been replaced by "local"
+    // during cache initialization, so we don't compare MD5.
+    return true;
+  } else if (cache_entry.is_pinned() && cache_entry.md5().empty()) {
+    // If the entry is pinned, it's ok for the entry to have an empty
+    // MD5. This can happen if the pinned file is not fetched. MD5 for pinned
+    // files are collected from files in "persistent" directory, but the
+    // persistent files do not exist if these are not fetched yet.
+    return true;
+  } else if (md5.empty()) {
+    // If the MD5 matching is not requested, don't check MD5.
+    return true;
+  } else if (md5 == cache_entry.md5()) {
+    // Otherwise, compare the MD5.
+    return true;
+  }
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// FileCacheMetadata implementation with std::map.
+// Used for testing.
+
+class FakeCacheMetadata : public FileCacheMetadata {
+ public:
+  explicit FakeCacheMetadata(
+      base::SequencedTaskRunner* blocking_task_runner);
+
+ private:
+  virtual ~FakeCacheMetadata();
+
+  // FileCacheMetadata overrides:
+  virtual bool Initialize(
+      const std::vector<base::FilePath>& cache_paths) OVERRIDE;
+  virtual void AddOrUpdateCacheEntry(
+      const std::string& resource_id,
+      const FileCacheEntry& cache_entry) OVERRIDE;
+  virtual void RemoveCacheEntry(const std::string& resource_id) OVERRIDE;
+  virtual bool GetCacheEntry(const std::string& resource_id,
+                             const std::string& md5,
+                             FileCacheEntry* cache_entry) OVERRIDE;
+  virtual void RemoveTemporaryFiles() OVERRIDE;
+  virtual void Iterate(const CacheIterateCallback& callback) OVERRIDE;
+
+  CacheMap cache_map_;
+
+  DISALLOW_COPY_AND_ASSIGN(FakeCacheMetadata);
+};
+
+FakeCacheMetadata::FakeCacheMetadata(
+    base::SequencedTaskRunner* blocking_task_runner)
+    : FileCacheMetadata(blocking_task_runner) {
+  AssertOnSequencedWorkerPool();
+}
+
+FakeCacheMetadata::~FakeCacheMetadata() {
+  AssertOnSequencedWorkerPool();
+}
+
+bool FakeCacheMetadata::Initialize(
+    const std::vector<base::FilePath>& cache_paths) {
+  AssertOnSequencedWorkerPool();
+
+  ScanCachePaths(cache_paths, &cache_map_);
+  return true;
+}
+
+void FakeCacheMetadata::AddOrUpdateCacheEntry(
+    const std::string& resource_id,
+    const FileCacheEntry& cache_entry) {
+  AssertOnSequencedWorkerPool();
+
+  CacheMap::iterator iter = cache_map_.find(resource_id);
+  if (iter == cache_map_.end()) {  // New resource, create new entry.
+    cache_map_.insert(std::make_pair(resource_id, cache_entry));
+  } else {  // Resource exists.
+    cache_map_[resource_id] = cache_entry;
+  }
+}
+
+void FakeCacheMetadata::RemoveCacheEntry(const std::string& resource_id) {
+  AssertOnSequencedWorkerPool();
+
+  CacheMap::iterator iter = cache_map_.find(resource_id);
+  if (iter != cache_map_.end()) {
+    // Delete the FileCacheEntry and remove it from the map.
+    cache_map_.erase(iter);
+  }
+}
+
+bool FakeCacheMetadata::GetCacheEntry(const std::string& resource_id,
+                                      const std::string& md5,
+                                      FileCacheEntry* entry) {
+  DCHECK(entry);
+  AssertOnSequencedWorkerPool();
+
+  CacheMap::iterator iter = cache_map_.find(resource_id);
+  if (iter == cache_map_.end()) {
+    DVLOG(1) << "Can't find " << resource_id << " in cache map";
+    return false;
+  }
+
+  const FileCacheEntry& cache_entry = iter->second;
+
+  if (!CheckIfMd5Matches(md5, cache_entry)) {
+    return false;
+  }
+
+  *entry = cache_entry;
+  return true;
+}
+
+void FakeCacheMetadata::RemoveTemporaryFiles() {
+  AssertOnSequencedWorkerPool();
+
+  CacheMap::iterator iter = cache_map_.begin();
+  while (iter != cache_map_.end()) {
+    if (!iter->second.is_persistent()) {
+      // Post-increment the iterator to avoid iterator invalidation.
+      cache_map_.erase(iter++);
+    } else {
+      ++iter;
+    }
+  }
+}
+
+void FakeCacheMetadata::Iterate(const CacheIterateCallback& callback) {
+  AssertOnSequencedWorkerPool();
+
+  for (CacheMap::const_iterator iter = cache_map_.begin();
+       iter != cache_map_.end(); ++iter) {
+    callback.Run(iter->first, iter->second);
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// FileCacheMetadata implementation with level::db.
+
+class FileCacheMetadataDB : public FileCacheMetadata {
+ public:
+  explicit FileCacheMetadataDB(
+      base::SequencedTaskRunner* blocking_task_runner);
+
+ private:
+  virtual ~FileCacheMetadataDB();
+
+  // FileCacheMetadata overrides:
+  virtual bool Initialize(
+      const std::vector<base::FilePath>& cache_paths) OVERRIDE;
+  virtual void AddOrUpdateCacheEntry(
+      const std::string& resource_id,
+      const FileCacheEntry& cache_entry) OVERRIDE;
+  virtual void RemoveCacheEntry(const std::string& resource_id) OVERRIDE;
+  virtual bool GetCacheEntry(const std::string& resource_id,
+                             const std::string& md5,
+                             FileCacheEntry* cache_entry) OVERRIDE;
+  virtual void RemoveTemporaryFiles() OVERRIDE;
+  virtual void Iterate(const CacheIterateCallback& callback) OVERRIDE;
+
+  // Helper function to insert |cache_map| entries into the database.
+  void InsertMapIntoDB(const CacheMap& cache_map);
+
+  scoped_ptr<leveldb::DB> level_db_;
+
+  DISALLOW_COPY_AND_ASSIGN(FileCacheMetadataDB);
+};
+
+FileCacheMetadataDB::FileCacheMetadataDB(
+    base::SequencedTaskRunner* blocking_task_runner)
+    : FileCacheMetadata(blocking_task_runner) {
+  AssertOnSequencedWorkerPool();
+}
+
+FileCacheMetadataDB::~FileCacheMetadataDB() {
+  AssertOnSequencedWorkerPool();
+}
+
+bool FileCacheMetadataDB::Initialize(
+    const std::vector<base::FilePath>& cache_paths) {
+  AssertOnSequencedWorkerPool();
+
+  const base::FilePath db_path =
+      cache_paths[FileCache::CACHE_TYPE_META].Append(
+          kCacheMetadataDBPath);
+  DVLOG(1) << "db path=" << db_path.value();
+
+  bool scan_cache = !file_util::PathExists(db_path);
+
+  leveldb::DB* level_db = NULL;
+  leveldb::Options options;
+  options.create_if_missing = true;
+  leveldb::Status db_status = leveldb::DB::Open(options, db_path.AsUTF8Unsafe(),
+                                                &level_db);
+
+  // Delete the db and scan the physical cache. This will fix a corrupt db, but
+  // perhaps not other causes of failed DB::Open.
+  DBOpenStatus uma_status = DB_OPEN_SUCCESS;
+  if (!db_status.ok()) {
+    LOG(WARNING) << "Cache db failed to open: " << db_status.ToString();
+    uma_status = db_status.IsCorruption() ?
+        DB_OPEN_FAILURE_CORRUPTION : DB_OPEN_FAILURE_OTHER;
+    const bool deleted = file_util::Delete(db_path, true);
+    DCHECK(deleted);
+    db_status = leveldb::DB::Open(options, db_path.value(), &level_db);
+    if (!db_status.ok()) {
+      LOG(WARNING) << "Still failed to open: " << db_status.ToString();
+      UMA_HISTOGRAM_ENUMERATION("Drive.CacheDBOpenStatus",
+                                DB_OPEN_FAILURE_UNRECOVERABLE,
+                                DB_OPEN_MAX_VALUE);
+      // Failed to open the cache metadata DB. Drive will be disabled.
+      return false;
+    }
+
+    scan_cache = true;
+  }
+  UMA_HISTOGRAM_ENUMERATION("Drive.CacheDBOpenStatus", uma_status,
+                            DB_OPEN_MAX_VALUE);
+  DCHECK(level_db);
+  level_db_.reset(level_db);
+
+  // We scan the cache directories to initialize the cache database if we
+  // were previously using the cache map.
+  if (scan_cache) {
+    CacheMap cache_map;
+    ScanCachePaths(cache_paths, &cache_map);
+    InsertMapIntoDB(cache_map);
+  }
+
+  return true;
+}
+
+void FileCacheMetadataDB::InsertMapIntoDB(const CacheMap& cache_map) {
+  DVLOG(1) << "InsertMapIntoDB";
+  for (CacheMap::const_iterator it = cache_map.begin();
+       it != cache_map.end(); ++it) {
+    AddOrUpdateCacheEntry(it->first, it->second);
+  }
+}
+
+void FileCacheMetadataDB::AddOrUpdateCacheEntry(
+    const std::string& resource_id,
+    const FileCacheEntry& cache_entry) {
+  AssertOnSequencedWorkerPool();
+
+  DVLOG(1) << "AddOrUpdateCacheEntry, resource_id=" << resource_id;
+  std::string serialized;
+  const bool ok = cache_entry.SerializeToString(&serialized);
+  if (ok)
+    level_db_->Put(leveldb::WriteOptions(),
+                   leveldb::Slice(resource_id),
+                   leveldb::Slice(serialized));
+}
+
+void FileCacheMetadataDB::RemoveCacheEntry(const std::string& resource_id) {
+  AssertOnSequencedWorkerPool();
+
+  DVLOG(1) << "RemoveCacheEntry, resource_id=" << resource_id;
+  level_db_->Delete(leveldb::WriteOptions(), leveldb::Slice(resource_id));
+}
+
+bool FileCacheMetadataDB::GetCacheEntry(const std::string& resource_id,
+                                        const std::string& md5,
+                                        FileCacheEntry* entry) {
+  DCHECK(entry);
+  AssertOnSequencedWorkerPool();
+
+  std::string serialized;
+  const leveldb::Status status = level_db_->Get(
+      leveldb::ReadOptions(),
+      leveldb::Slice(resource_id), &serialized);
+  if (!status.ok()) {
+    DVLOG(1) << "Can't find " << resource_id << " in cache db";
+    return false;
+  }
+
+  FileCacheEntry cache_entry;
+  const bool ok = cache_entry.ParseFromString(serialized);
+  if (!ok) {
+    LOG(ERROR) << "Failed to parse " << serialized;
+    return false;
+  }
+
+  if (!CheckIfMd5Matches(md5, cache_entry)) {
+    return false;
+  }
+
+  *entry = cache_entry;
+  return true;
+}
+
+void FileCacheMetadataDB::RemoveTemporaryFiles() {
+  AssertOnSequencedWorkerPool();
+
+  scoped_ptr<leveldb::Iterator> iter(level_db_->NewIterator(
+      leveldb::ReadOptions()));
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    FileCacheEntry cache_entry;
+    const bool ok = cache_entry.ParseFromArray(iter->value().data(),
+                                               iter->value().size());
+    if (ok && !cache_entry.is_persistent())
+      level_db_->Delete(leveldb::WriteOptions(), iter->key());
+  }
+}
+
+void FileCacheMetadataDB::Iterate(const CacheIterateCallback& callback) {
+  AssertOnSequencedWorkerPool();
+
+  scoped_ptr<leveldb::Iterator> iter(level_db_->NewIterator(
+      leveldb::ReadOptions()));
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    FileCacheEntry cache_entry;
+    const bool ok = cache_entry.ParseFromArray(iter->value().data(),
+                                               iter->value().size());
+    if (ok)
+      callback.Run(iter->key().ToString(), cache_entry);
+  }
+}
+
+}  // namespace
+
+// static
+const base::FilePath::CharType* FileCacheMetadata::kCacheMetadataDBPath =
+    FILE_PATH_LITERAL("cache_metadata.db");
+
+
+FileCacheMetadata::FileCacheMetadata(
+    base::SequencedTaskRunner* blocking_task_runner)
+    : blocking_task_runner_(blocking_task_runner) {
+  AssertOnSequencedWorkerPool();
+}
+
+FileCacheMetadata::~FileCacheMetadata() {
+  AssertOnSequencedWorkerPool();
+}
+
+// static
+scoped_ptr<FileCacheMetadata> FileCacheMetadata::CreateCacheMetadata(
+    base::SequencedTaskRunner* blocking_task_runner) {
+  return scoped_ptr<FileCacheMetadata>(
+      new FileCacheMetadataDB(blocking_task_runner));
+}
+
+// static
+scoped_ptr<FileCacheMetadata>
+FileCacheMetadata::CreateCacheMetadataForTesting(
+    base::SequencedTaskRunner* blocking_task_runner) {
+  return scoped_ptr<FileCacheMetadata>(
+      new FakeCacheMetadata(blocking_task_runner));
+}
+
+void FileCacheMetadata::AssertOnSequencedWorkerPool() {
+  DCHECK(!blocking_task_runner_ ||
+         blocking_task_runner_->RunsTasksOnCurrentThread());
+}
+
+}  // namespace drive