Copy ldb files in ChromiumEnv for the purpose of backups.

If it is later detected that an ldb file has gone missing it
will be restored from its backup.

BUG=239999
R=isherman@chromium.org, jsbell@chromium.org

Review URL: https://codereview.chromium.org/27022005

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@228952 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 201 insertions, 15 deletions

diff --git a/third_party/leveldatabase/env_chromium.cc b/third_party/leveldatabase/env_chromium.cc
index b297425..331e6e9 100644
--- a/third_party/leveldatabase/env_chromium.cc
+++ b/third_party/leveldatabase/env_chromium.cc
@@ -48,6 +48,10 @@ namespace leveldb_env {
 
 namespace {
 
+const base::FilePath::CharType backup_table_extension[] =
+    FILE_PATH_LITERAL(".bak");
+const base::FilePath::CharType table_extension[] = FILE_PATH_LITERAL(".ldb");
+
 #if (defined(OS_POSIX) && !defined(OS_LINUX)) || defined(OS_WIN)
 // The following are glibc-specific
 
@@ -256,7 +260,10 @@ class Retrier {
 
 class IDBEnv : public ChromiumEnv {
  public:
-  IDBEnv() : ChromiumEnv() { name_ = "LevelDBEnv.IDB"; }
+  IDBEnv() : ChromiumEnv() {
+    name_ = "LevelDBEnv.IDB";
+    make_backup_ = true;
+  }
 };
 
 ::base::LazyInstance<IDBEnv>::Leaky idb_env = LAZY_INSTANCE_INITIALIZER;
@@ -412,13 +419,20 @@ std::string FilePathToString(const base::FilePath& file_path) {
 ChromiumWritableFile::ChromiumWritableFile(const std::string& fname,
                                            FILE* f,
                                            const UMALogger* uma_logger,
-                                           WriteTracker* tracker)
-    : filename_(fname), file_(f), uma_logger_(uma_logger), tracker_(tracker) {
+                                           WriteTracker* tracker,
+                                           bool make_backup)
+    : filename_(fname),
+      file_(f),
+      uma_logger_(uma_logger),
+      tracker_(tracker),
+      file_type_(kOther),
+      make_backup_(make_backup) {
   base::FilePath path = base::FilePath::FromUTF8Unsafe(fname);
-  is_manifest_ =
-      FilePathToString(path.BaseName()).find("MANIFEST") !=
-      std::string::npos;
-  if (!is_manifest_)
+  if (FilePathToString(path.BaseName()).find("MANIFEST") == 0)
+    file_type_ = kManifest;
+  else if (path.MatchesExtension(table_extension))
+    file_type_ = kTable;
+  if (file_type_ != kManifest)
     tracker_->DidCreateNewFile(filename_);
   parent_dir_ = FilePathToString(CreateFilePath(fname).DirName());
 }
@@ -453,7 +467,7 @@ Status ChromiumWritableFile::SyncParent() {
 }
 
 Status ChromiumWritableFile::Append(const Slice& data) {
-  if (is_manifest_ && tracker_->DoesDirNeedSync(filename_)) {
+  if (file_type_ == kManifest && tracker_->DoesDirNeedSync(filename_)) {
     Status s = SyncParent();
     if (!s.ok())
       return s;
@@ -491,6 +505,13 @@ Status ChromiumWritableFile::Flush() {
   return result;
 }
 
+static bool MakeBackup(const std::string& fname) {
+  base::FilePath original_table_name = CreateFilePath(fname);
+  base::FilePath backup_table_name =
+      original_table_name.ReplaceExtension(backup_table_extension);
+  return base::CopyFile(original_table_name, backup_table_name);
+}
+
 Status ChromiumWritableFile::Sync() {
   TRACE_EVENT0("leveldb", "ChromiumEnv::Sync");
   Status result;
@@ -506,12 +527,16 @@ Status ChromiumWritableFile::Sync() {
   if (error) {
     result = MakeIOError(filename_, strerror(error), kWritableFileSync, error);
     uma_logger_->RecordErrorAt(kWritableFileSync);
+  } else if (make_backup_ && file_type_ == kTable) {
+    bool success = MakeBackup(filename_);
+    uma_logger_->RecordBackupResult(success);
   }
   return result;
 }
 
 ChromiumEnv::ChromiumEnv()
     : name_("LevelDBEnv"),
+      make_backup_(false),
       bgsignal_(&mu_),
       started_bgthread_(false),
       kMaxRetryTimeMillis(1000) {
@@ -581,7 +606,7 @@ Status ChromiumEnv::NewWritableFile(const std::string& fname,
     return MakeIOError(
         fname, strerror(saved_errno), kNewWritableFile, saved_errno);
   } else {
-    *result = new ChromiumWritableFile(fname, f, this, this);
+    *result = new ChromiumWritableFile(fname, f, this, this, make_backup_);
     return Status::OK();
   }
 }
@@ -590,9 +615,65 @@ bool ChromiumEnv::FileExists(const std::string& fname) {
   return ::base::PathExists(CreateFilePath(fname));
 }
 
+base::FilePath ChromiumEnv::RestoreFromBackup(const base::FilePath& base_name) {
+  base::FilePath table_name =
+      base_name.AddExtension(table_extension);
+  bool result = base::CopyFile(base_name.AddExtension(backup_table_extension),
+                               table_name);
+  std::string uma_name(name_);
+  uma_name.append(".TableRestore");
+  base::BooleanHistogram::FactoryGet(
+      uma_name, base::Histogram::kUmaTargetedHistogramFlag)->AddBoolean(result);
+  return table_name;
+}
+
+void ChromiumEnv::RestoreIfNecessary(const std::string& dir,
+                                     std::vector<std::string>* result) {
+  std::set<base::FilePath> tables_found;
+  std::set<base::FilePath> backups_found;
+  for (std::vector<std::string>::iterator it = result->begin();
+       it != result->end();
+       ++it) {
+    base::FilePath current = CreateFilePath(*it);
+    if (current.MatchesExtension(table_extension))
+      tables_found.insert(current.RemoveExtension());
+    if (current.MatchesExtension(backup_table_extension))
+      backups_found.insert(current.RemoveExtension());
+  }
+  std::set<base::FilePath> backups_only;
+  std::set_difference(backups_found.begin(),
+                      backups_found.end(),
+                      tables_found.begin(),
+                      tables_found.end(),
+                      std::inserter(backups_only, backups_only.begin()));
+  if (backups_only.size()) {
+    std::string uma_name(name_);
+    uma_name.append(".MissingFiles");
+    int num_missing_files =
+        backups_only.size() > INT_MAX ? INT_MAX : backups_only.size();
+    base::Histogram::FactoryGet(uma_name,
+                                1 /*min*/,
+                                100 /*max*/,
+                                8 /*num_buckets*/,
+                                base::Histogram::kUmaTargetedHistogramFlag)
+        ->Add(num_missing_files);
+  }
+  base::FilePath dir_filepath = base::FilePath::FromUTF8Unsafe(dir);
+  for (std::set<base::FilePath>::iterator it = backups_only.begin();
+       it != backups_only.end();
+       ++it) {
+    base::FilePath restored_table_name =
+        RestoreFromBackup(dir_filepath.Append(*it));
+    result->push_back(FilePathToString(restored_table_name.BaseName()));
+  }
+}
+
 Status ChromiumEnv::GetChildren(const std::string& dir,
                                 std::vector<std::string>* result) {
   result->clear();
+  // TODO(jorlow): Unfortunately, the FileEnumerator swallows errors, so
+  //               we'll always return OK. Maybe manually check for error
+  //               conditions like the file not existing?
   base::FileEnumerator iter(
       CreateFilePath(dir), false, base::FileEnumerator::FILES);
   base::FilePath current = iter.Next();
@@ -600,19 +681,23 @@ Status ChromiumEnv::GetChildren(const std::string& dir,
     result->push_back(FilePathToString(current.BaseName()));
     current = iter.Next();
   }
-  // TODO(jorlow): Unfortunately, the FileEnumerator swallows errors, so
-  //               we'll always return OK. Maybe manually check for error
-  //               conditions like the file not existing?
+  if (make_backup_)
+    RestoreIfNecessary(dir, result);
   return Status::OK();
 }
 
 Status ChromiumEnv::DeleteFile(const std::string& fname) {
   Status result;
+  base::FilePath fname_filepath = CreateFilePath(fname);
   // TODO(jorlow): Should we assert this is a file?
-  if (!::base::DeleteFile(CreateFilePath(fname), false)) {
+  if (!::base::DeleteFile(fname_filepath, false)) {
     result = MakeIOError(fname, "Could not delete file.", kDeleteFile);
     RecordErrorAt(kDeleteFile);
   }
+  if (make_backup_ && fname_filepath.MatchesExtension(table_extension)) {
+    base::DeleteFile(fname_filepath.ReplaceExtension(backup_table_extension),
+                     false);
+  }
   return result;
 }
 
@@ -822,6 +907,13 @@ void ChromiumEnv::RecordOSError(MethodID method, int error) const {
   GetOSErrorHistogram(method, ERANGE + 1)->Add(error);
 }
 
+void ChromiumEnv::RecordBackupResult(bool result) const {
+  std::string uma_name(name_);
+  uma_name.append(".TableBackup");
+  base::BooleanHistogram::FactoryGet(
+      uma_name, base::Histogram::kUmaTargetedHistogramFlag)->AddBoolean(result);
+}
+
 base::HistogramBase* ChromiumEnv::GetOSErrorHistogram(MethodID method,
                                                       int limit) const {
   std::string uma_name(name_);
diff --git a/third_party/leveldatabase/env_chromium.h b/third_party/leveldatabase/env_chromium.h
index 7b7b55e..740a1e0 100644
--- a/third_party/leveldatabase/env_chromium.h
+++ b/third_party/leveldatabase/env_chromium.h
@@ -78,6 +78,7 @@ class UMALogger {
   virtual void RecordOSError(MethodID method, int saved_errno) const = 0;
   virtual void RecordOSError(MethodID method,
                              base::PlatformFileError error) const = 0;
+  virtual void RecordBackupResult(bool success) const = 0;
 };
 
 class RetrierProvider {
@@ -100,7 +101,8 @@ class ChromiumWritableFile : public leveldb::WritableFile {
   ChromiumWritableFile(const std::string& fname,
                        FILE* f,
                        const UMALogger* uma_logger,
-                       WriteTracker* tracker);
+                       WriteTracker* tracker,
+                       bool make_backup);
   virtual ~ChromiumWritableFile();
   virtual leveldb::Status Append(const leveldb::Slice& data);
   virtual leveldb::Status Close();
@@ -108,14 +110,20 @@ class ChromiumWritableFile : public leveldb::WritableFile {
   virtual leveldb::Status Sync();
 
  private:
+  enum Type {
+    kManifest,
+    kTable,
+    kOther
+  };
   leveldb::Status SyncParent();
 
   std::string filename_;
   FILE* file_;
   const UMALogger* uma_logger_;
   WriteTracker* tracker_;
-  bool is_manifest_;
+  Type file_type_;
   std::string parent_dir_;
+  bool make_backup_;
 };
 
 class ChromiumEnv : public leveldb::Env,
@@ -159,6 +167,7 @@ class ChromiumEnv : public leveldb::Env,
   virtual void DidSyncDir(const std::string& fname);
 
   std::string name_;
+  bool make_backup_;
 
  private:
   // File locks may not be exclusive within a process (e.g. on POSIX). Track
@@ -192,6 +201,10 @@ class ChromiumEnv : public leveldb::Env,
   virtual void RecordOSError(MethodID method, int saved_errno) const;
   virtual void RecordOSError(MethodID method,
                              base::PlatformFileError error) const;
+  virtual void RecordBackupResult(bool result) const;
+  void RestoreIfNecessary(const std::string& dir,
+                          std::vector<std::string>* children);
+  base::FilePath RestoreFromBackup(const base::FilePath& base_name);
   void RecordOpenFilesLimit(const std::string& type);
   void RecordLockFileAncestors(int num_missing_ancestors) const;
   base::HistogramBase* GetOSErrorHistogram(MethodID method, int limit) const;
diff --git a/third_party/leveldatabase/env_chromium_unittest.cc b/third_party/leveldatabase/env_chromium_unittest.cc
index 0789b3c..e952d81 100644
--- a/third_party/leveldatabase/env_chromium_unittest.cc
+++ b/third_party/leveldatabase/env_chromium_unittest.cc
@@ -2,15 +2,21 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include "base/file_util.h"
+#include "base/files/file_enumerator.h"
 #include "base/files/file_path.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/test/test_suite.h"
 #include "env_chromium.h"
 #include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/leveldatabase/env_idb.h"
+#include "third_party/leveldatabase/src/include/leveldb/db.h"
 
 using namespace leveldb_env;
 using namespace leveldb;
 
+#define FPL FILE_PATH_LITERAL
+
 TEST(ErrorEncoding, OnlyAMethod) {
   const MethodID in_method = kSequentialFileRead;
   const Status s = MakeIOError("Somefile.txt", "message", in_method);
@@ -104,4 +110,79 @@ TEST(ChromiumEnv, DirectorySyncing) {
   EXPECT_EQ(1, env.directory_syncs());
 }
 
+int CountFilesWithExtension(const base::FilePath& dir,
+                            const base::FilePath::StringType& extension) {
+  int matching_files = 0;
+  base::FileEnumerator dir_reader(
+      dir, false, base::FileEnumerator::FileType::FILES);
+  for (base::FilePath fname = dir_reader.Next(); !fname.empty();
+       fname = dir_reader.Next()) {
+    if (fname.MatchesExtension(extension))
+      matching_files++;
+  }
+  return matching_files;
+}
+
+bool GetFirstLDBFile(const base::FilePath& dir, base::FilePath* ldb_file) {
+  base::FileEnumerator dir_reader(
+      dir, false, base::FileEnumerator::FileType::FILES);
+  for (base::FilePath fname = dir_reader.Next(); !fname.empty();
+       fname = dir_reader.Next()) {
+    if (fname.MatchesExtension(FPL(".ldb"))) {
+      *ldb_file = fname;
+      return true;
+    }
+  }
+  return false;
+}
+
+TEST(ChromiumEnv, BackupTables) {
+  Options options;
+  options.create_if_missing = true;
+  options.env = IDBEnv();
+
+  base::ScopedTempDir scoped_temp_dir;
+  scoped_temp_dir.CreateUniqueTempDir();
+  base::FilePath dir = scoped_temp_dir.path();
+
+  DB* db;
+  Status status = DB::Open(options, dir.AsUTF8Unsafe(), &db);
+  EXPECT_TRUE(status.ok()) << status.ToString();
+  status = db->Put(WriteOptions(), "key", "value");
+  EXPECT_TRUE(status.ok()) << status.ToString();
+  Slice a = "a";
+  Slice z = "z";
+  db->CompactRange(&a, &z);
+  int ldb_files = CountFilesWithExtension(dir, FPL(".ldb"));
+  int bak_files = CountFilesWithExtension(dir, FPL(".bak"));
+  EXPECT_GT(ldb_files, 0);
+  EXPECT_EQ(ldb_files, bak_files);
+  base::FilePath ldb_file;
+  EXPECT_TRUE(GetFirstLDBFile(dir, &ldb_file));
+  EXPECT_TRUE(base::DeleteFile(ldb_file, false));
+  EXPECT_EQ(ldb_files - 1, CountFilesWithExtension(dir, FPL(".ldb")));
+  delete db;
+
+  // The ldb file deleted above should be restored in Open.
+  status = leveldb::DB::Open(options, dir.AsUTF8Unsafe(), &db);
+  EXPECT_TRUE(status.ok()) << status.ToString();
+  std::string value;
+  status = db->Get(ReadOptions(), "key", &value);
+  EXPECT_TRUE(status.ok()) << status.ToString();
+  EXPECT_EQ("value", value);
+  delete db;
+
+  // Ensure that deleting an ldb file also deletes its backup.
+  int orig_ldb_files = CountFilesWithExtension(dir, FPL(".ldb"));
+  int orig_bak_files = CountFilesWithExtension(dir, FPL(".bak"));
+  EXPECT_GT(ldb_files, 0);
+  EXPECT_EQ(ldb_files, bak_files);
+  EXPECT_TRUE(GetFirstLDBFile(dir, &ldb_file));
+  options.env->DeleteFile(ldb_file.AsUTF8Unsafe());
+  ldb_files = CountFilesWithExtension(dir, FPL(".ldb"));
+  bak_files = CountFilesWithExtension(dir, FPL(".bak"));
+  EXPECT_EQ(orig_ldb_files - 1, ldb_files);
+  EXPECT_EQ(bak_files, ldb_files);
+}
+
 int main(int argc, char** argv) { return base::TestSuite(argc, argv).Run(); }