[sql] Scoped recovery framework.

sql::Recovery is intended to be used within a sql::Connection error
callback to either recover the database (*) or indicate that the
database is unrecoverable and should be razed.  The intend is that the
code should either progress to a valid database which is composed of
data recovered from the original (likely corrupt) database, or a valid
database which is empty.

This is just the framework without the SQLite-level data-recovery
virtual table.

BUG=109482

Review URL: https://chromiumcodereview.appspot.com/19281002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@212607 0039d316-1c4b-4281-b951-d872f2087c98

7 files changed, 637 insertions, 9 deletions

diff --git a/sql/connection.cc b/sql/connection.cc
index e7b6fe1..4550b0e 100644
--- a/sql/connection.cc
+++ b/sql/connection.cc
@@ -94,6 +94,21 @@ int BackupDatabase(sqlite3* src, sqlite3* dst, const char* db_name) {
   return rc;
 }
 
+// Be very strict on attachment point.  SQLite can handle a much wider
+// character set with appropriate quoting, but Chromium code should
+// just use clean names to start with.
+bool ValidAttachmentPoint(const char* attachment_point) {
+  for (size_t i = 0; attachment_point[i]; ++i) {
+    if (!((attachment_point[i] >= '0' && attachment_point[i] <= '9') ||
+          (attachment_point[i] >= 'a' && attachment_point[i] <= 'z') ||
+          (attachment_point[i] >= 'A' && attachment_point[i] <= 'Z') ||
+          attachment_point[i] == '_')) {
+      return false;
+    }
+  }
+  return true;
+}
+
 }  // namespace
 
 namespace sql {
@@ -206,6 +221,10 @@ bool Connection::OpenInMemory() {
   return OpenInternal(":memory:", NO_RETRY);
 }
 
+bool Connection::OpenTemporary() {
+  return OpenInternal("", NO_RETRY);
+}
+
 void Connection::CloseInternal(bool forced) {
   // TODO(shess): Calling "PRAGMA journal_mode = DELETE" at this point
   // will delete the -journal file.  For ChromiumOS or other more
@@ -442,9 +461,7 @@ bool Connection::RazeAndClose() {
   }
 
   // Raze() cannot run in a transaction.
-  while (transaction_nesting_) {
-    RollbackTransaction();
-  }
+  RollbackAllTransactions();
 
   bool result = Raze();
 
@@ -458,6 +475,21 @@ bool Connection::RazeAndClose() {
   return result;
 }
 
+void Connection::Poison() {
+  if (!db_) {
+    DLOG_IF(FATAL, !poisoned_) << "Cannot poison null db";
+    return;
+  }
+
+  RollbackAllTransactions();
+  CloseInternal(true);
+
+  // Mark the database so that future API calls fail appropriately,
+  // but don't DCHECK (because after calling this function they are
+  // expected to fail).
+  poisoned_ = true;
+}
+
 // TODO(shess): To the extent possible, figure out the optimal
 // ordering for these deletes which will prevent other connections
 // from seeing odd behavior.  For instance, it may be necessary to
@@ -543,6 +575,35 @@ bool Connection::CommitTransaction() {
   return commit.Run();
 }
 
+void Connection::RollbackAllTransactions() {
+  if (transaction_nesting_ > 0) {
+    transaction_nesting_ = 0;
+    DoRollback();
+  }
+}
+
+bool Connection::AttachDatabase(const base::FilePath& other_db_path,
+                                const char* attachment_point) {
+  DCHECK(ValidAttachmentPoint(attachment_point));
+
+  Statement s(GetUniqueStatement("ATTACH DATABASE ? AS ?"));
+#if OS_WIN
+  s.BindString16(0, other_db_path.value());
+#else
+  s.BindString(0, other_db_path.value());
+#endif
+  s.BindString(1, attachment_point);
+  return s.Run();
+}
+
+bool Connection::DetachDatabase(const char* attachment_point) {
+  DCHECK(ValidAttachmentPoint(attachment_point));
+
+  Statement s(GetUniqueStatement("DETACH DATABASE ?"));
+  s.BindString(0, attachment_point);
+  return s.Run();
+}
+
 int Connection::ExecuteAndReturnErrorCode(const char* sql) {
   AssertIOAllowed();
   if (!db_) {
diff --git a/sql/connection.h b/sql/connection.h
index 3ee91ef..24f06de 100644
--- a/sql/connection.h
+++ b/sql/connection.h
@@ -28,6 +28,7 @@ class FilePath;
 
 namespace sql {
 
+class Recovery;
 class Statement;
 
 // Uniquely identifies a statement. There are two modes of operation:
@@ -166,6 +167,12 @@ class SQL_EXPORT Connection {
   // empty. You can call this or Open.
   bool OpenInMemory() WARN_UNUSED_RESULT;
 
+  // Create a temporary on-disk database.  The database will be
+  // deleted after close.  This kind of database is similar to
+  // OpenInMemory() for small databases, but can page to disk if the
+  // database becomes large.
+  bool OpenTemporary() WARN_UNUSED_RESULT;
+
   // Returns true if the database has been successfully opened.
   bool is_open() const { return !!db_; }
 
@@ -230,13 +237,17 @@ class SQL_EXPORT Connection {
   bool RazeWithTimout(base::TimeDelta timeout);
 
   // Breaks all outstanding transactions (as initiated by
-  // BeginTransaction()), calls Raze() to destroy the database, then
-  // closes the database.  After this is called, any operations
-  // against the connections (or statements prepared by the
-  // connection) should fail safely.
+  // BeginTransaction()), closes the SQLite database, and poisons the
+  // object so that all future operations against the Connection (or
+  // its Statements) fail safely, without side effects.
   //
-  // The value from Raze() is returned, with Close() called in all
-  // cases.
+  // This is intended as an alternative to Close() in error callbacks.
+  // Close() should still be called at some point.
+  void Poison();
+
+  // Raze() the database and Poison() the handle.  Returns the return
+  // value from Raze().
+  // TODO(shess): Rename to RazeAndPoison().
   bool RazeAndClose();
 
   // Delete the underlying database files associated with |path|.
@@ -266,10 +277,27 @@ class SQL_EXPORT Connection {
   void RollbackTransaction();
   bool CommitTransaction();
 
+  // Rollback all outstanding transactions.  Use with care, there may
+  // be scoped transactions on the stack.
+  void RollbackAllTransactions();
+
   // Returns the current transaction nesting, which will be 0 if there are
   // no open transactions.
   int transaction_nesting() const { return transaction_nesting_; }
 
+  // Attached databases---------------------------------------------------------
+
+  // SQLite supports attaching multiple database files to a single
+  // handle.  Attach the database in |other_db_path| to the current
+  // handle under |attachment_point|.  |attachment_point| should only
+  // contain characters from [a-zA-Z0-9_].
+  //
+  // Note that calling attach or detach with an open transaction is an
+  // error.
+  bool AttachDatabase(const base::FilePath& other_db_path,
+                      const char* attachment_point);
+  bool DetachDatabase(const char* attachment_point);
+
   // Statements ----------------------------------------------------------------
 
   // Executes the given SQL string, returning true on success. This is
@@ -362,6 +390,9 @@ class SQL_EXPORT Connection {
   const char* GetErrorMessage() const;
 
  private:
+  // For recovery module.
+  friend class Recovery;
+
   // Allow test-support code to set/reset error ignorer.
   friend class ScopedErrorIgnorer;
 
diff --git a/sql/connection_unittest.cc b/sql/connection_unittest.cc
index f516215..58dedbc 100644
--- a/sql/connection_unittest.cc
+++ b/sql/connection_unittest.cc
@@ -749,4 +749,97 @@ TEST_F(SQLConnectionTest, UserPermission) {
 }
 #endif  // defined(OS_POSIX)
 
+// Test that errors start happening once Poison() is called.
+TEST_F(SQLConnectionTest, Poison) {
+  EXPECT_TRUE(db().Execute("CREATE TABLE x (x)"));
+
+  // Before the Poison() call, things generally work.
+  EXPECT_TRUE(db().IsSQLValid("INSERT INTO x VALUES ('x')"));
+  EXPECT_TRUE(db().Execute("INSERT INTO x VALUES ('x')"));
+  {
+    sql::Statement s(db().GetUniqueStatement("SELECT COUNT(*) FROM x"));
+    ASSERT_TRUE(s.is_valid());
+    ASSERT_TRUE(s.Step());
+  }
+
+  // Get a statement which is valid before and will exist across Poison().
+  sql::Statement valid_statement(
+      db().GetUniqueStatement("SELECT COUNT(*) FROM sqlite_master"));
+  ASSERT_TRUE(valid_statement.is_valid());
+  ASSERT_TRUE(valid_statement.Step());
+  valid_statement.Reset(true);
+
+  db().Poison();
+
+  // After the Poison() call, things fail.
+  EXPECT_FALSE(db().IsSQLValid("INSERT INTO x VALUES ('x')"));
+  EXPECT_FALSE(db().Execute("INSERT INTO x VALUES ('x')"));
+  {
+    sql::Statement s(db().GetUniqueStatement("SELECT COUNT(*) FROM x"));
+    ASSERT_FALSE(s.is_valid());
+    ASSERT_FALSE(s.Step());
+  }
+
+  // The existing statement has become invalid.
+  ASSERT_FALSE(valid_statement.is_valid());
+  ASSERT_FALSE(valid_statement.Step());
+}
+
+// Test attaching and detaching databases from the connection.
+TEST_F(SQLConnectionTest, Attach) {
+  EXPECT_TRUE(db().Execute("CREATE TABLE foo (a, b)"));
+
+  // Create a database to attach to.
+  base::FilePath attach_path =
+      db_path().DirName().AppendASCII("SQLConnectionAttach.db");
+  const char kAttachmentPoint[] = "other";
+  {
+    sql::Connection other_db;
+    ASSERT_TRUE(other_db.Open(attach_path));
+    EXPECT_TRUE(other_db.Execute("CREATE TABLE bar (a, b)"));
+    EXPECT_TRUE(other_db.Execute("INSERT INTO bar VALUES ('hello', 'world')"));
+  }
+
+  // Cannot see the attached database, yet.
+  EXPECT_FALSE(db().IsSQLValid("SELECT count(*) from other.bar"));
+
+  // Attach fails in a transaction.
+  EXPECT_TRUE(db().BeginTransaction());
+  {
+    sql::ScopedErrorIgnorer ignore_errors;
+    ignore_errors.IgnoreError(SQLITE_ERROR);
+    EXPECT_FALSE(db().AttachDatabase(attach_path, kAttachmentPoint));
+    ASSERT_TRUE(ignore_errors.CheckIgnoredErrors());
+  }
+
+  // Attach succeeds when the transaction is closed.
+  db().RollbackTransaction();
+  EXPECT_TRUE(db().AttachDatabase(attach_path, kAttachmentPoint));
+  EXPECT_TRUE(db().IsSQLValid("SELECT count(*) from other.bar"));
+
+  // Queries can touch both databases.
+  EXPECT_TRUE(db().Execute("INSERT INTO foo SELECT a, b FROM other.bar"));
+  {
+    sql::Statement s(db().GetUniqueStatement("SELECT COUNT(*) FROM foo"));
+    ASSERT_TRUE(s.Step());
+    EXPECT_EQ(1, s.ColumnInt(0));
+  }
+
+  // Detach also fails in a transaction.
+  EXPECT_TRUE(db().BeginTransaction());
+  {
+    sql::ScopedErrorIgnorer ignore_errors;
+    ignore_errors.IgnoreError(SQLITE_ERROR);
+    EXPECT_FALSE(db().DetachDatabase(kAttachmentPoint));
+    EXPECT_TRUE(db().IsSQLValid("SELECT count(*) from other.bar"));
+    ASSERT_TRUE(ignore_errors.CheckIgnoredErrors());
+  }
+
+  // Detach succeeds outside of a transaction.
+  db().RollbackTransaction();
+  EXPECT_TRUE(db().DetachDatabase(kAttachmentPoint));
+
+  EXPECT_FALSE(db().IsSQLValid("SELECT count(*) from other.bar"));
+}
+
 }  // namespace
diff --git a/sql/recovery.cc b/sql/recovery.cc
new file mode 100644
index 0000000..e929ed9
--- /dev/null
+++ b/sql/recovery.cc
@@ -0,0 +1,189 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "sql/recovery.h"
+
+#include "base/files/file_path.h"
+#include "base/logging.h"
+#include "base/metrics/sparse_histogram.h"
+#include "sql/connection.h"
+#include "third_party/sqlite/sqlite3.h"
+
+namespace sql {
+
+// static
+scoped_ptr<Recovery> Recovery::Begin(
+    Connection* connection,
+    const base::FilePath& db_path) {
+  scoped_ptr<Recovery> r(new Recovery(connection));
+  if (!r->Init(db_path)) {
+    // TODO(shess): Should Init() failure result in Raze()?
+    r->Shutdown(POISON);
+    return scoped_ptr<Recovery>();
+  }
+
+  return r.Pass();
+}
+
+// static
+bool Recovery::Recovered(scoped_ptr<Recovery> r) {
+  return r->Backup();
+}
+
+// static
+void Recovery::Unrecoverable(scoped_ptr<Recovery> r) {
+  CHECK(r->db_);
+  // ~Recovery() will RAZE_AND_POISON.
+}
+
+Recovery::Recovery(Connection* connection)
+    : db_(connection),
+      recover_db_() {
+  // Result should keep the page size specified earlier.
+  if (db_->page_size_)
+    recover_db_.set_page_size(db_->page_size_);
+
+  // TODO(shess): This may not handle cases where the default page
+  // size is used, but the default has changed.  I do not think this
+  // has ever happened.  This could be handled by using "PRAGMA
+  // page_size", at the cost of potential additional failure cases.
+}
+
+Recovery::~Recovery() {
+  Shutdown(RAZE_AND_POISON);
+}
+
+bool Recovery::Init(const base::FilePath& db_path) {
+  // Prevent the possibility of re-entering this code due to errors
+  // which happen while executing this code.
+  DCHECK(!db_->has_error_callback());
+
+  // Break any outstanding transactions on the original database to
+  // prevent deadlocks reading through the attached version.
+  // TODO(shess): A client may legitimately wish to recover from
+  // within the transaction context, because it would potentially
+  // preserve any in-flight changes.  Unfortunately, any attach-based
+  // system could not handle that.  A system which manually queried
+  // one database and stored to the other possibly could, but would be
+  // more complicated.
+  db_->RollbackAllTransactions();
+
+  if (!recover_db_.OpenTemporary())
+    return false;
+
+  // Turn on |SQLITE_RecoveryMode| for the handle, which allows
+  // reading certain broken databases.
+  if (!recover_db_.Execute("PRAGMA writable_schema=1"))
+    return false;
+
+  if (!recover_db_.AttachDatabase(db_path, "corrupt"))
+    return false;
+
+  return true;
+}
+
+bool Recovery::Backup() {
+  CHECK(db_);
+  CHECK(recover_db_.is_open());
+
+  // TODO(shess): Some of the failure cases here may need further
+  // exploration.  Just as elsewhere, persistent problems probably
+  // need to be razed, while anything which might succeed on a future
+  // run probably should be allowed to try.  But since Raze() uses the
+  // same approach, even that wouldn't work when this code fails.
+  //
+  // The documentation for the backup system indicate a relatively
+  // small number of errors are expected:
+  // SQLITE_BUSY - cannot lock the destination database.  This should
+  //               only happen if someone has another handle to the
+  //               database, Chromium generally doesn't do that.
+  // SQLITE_LOCKED - someone locked the source database.  Should be
+  //                 impossible (perhaps anti-virus could?).
+  // SQLITE_READONLY - destination is read-only.
+  // SQLITE_IOERR - since source database is temporary, probably
+  //                indicates that the destination contains blocks
+  //                throwing errors, or gross filesystem errors.
+  // SQLITE_NOMEM - out of memory, should be transient.
+  //
+  // AFAICT, SQLITE_BUSY and SQLITE_NOMEM could perhaps be considered
+  // transient, with SQLITE_LOCKED being unclear.
+  //
+  // SQLITE_READONLY and SQLITE_IOERR are probably persistent, with a
+  // strong chance that Raze() would not resolve them.  If Delete()
+  // deletes the database file, the code could then re-open the file
+  // and attempt the backup again.
+  //
+  // For now, this code attempts a best effort and records histograms
+  // to inform future development.
+
+  // Backup the original db from the recovered db.
+  const char* kMain = "main";
+  sqlite3_backup* backup = sqlite3_backup_init(db_->db_, kMain,
+                                               recover_db_.db_, kMain);
+  if (!backup) {
+    // Error code is in the destination database handle.
+    int err = sqlite3_errcode(db_->db_);
+    UMA_HISTOGRAM_SPARSE_SLOWLY("Sqlite.RecoveryHandle", err);
+    LOG(ERROR) << "sqlite3_backup_init() failed: "
+               << sqlite3_errmsg(db_->db_);
+    return false;
+  }
+
+  // -1 backs up the entire database.
+  int rc = sqlite3_backup_step(backup, -1);
+  int pages = sqlite3_backup_pagecount(backup);
+  // TODO(shess): sqlite3_backup_finish() appears to allow returning a
+  // different value from sqlite3_backup_step().  Circle back and
+  // figure out if that can usefully inform the decision of whether to
+  // retry or not.
+  sqlite3_backup_finish(backup);
+  DCHECK_GT(pages, 0);
+
+  if (rc != SQLITE_DONE) {
+    UMA_HISTOGRAM_SPARSE_SLOWLY("Sqlite.RecoveryStep", rc);
+    LOG(ERROR) << "sqlite3_backup_step() failed: "
+               << sqlite3_errmsg(db_->db_);
+  }
+
+  // The destination database was locked.  Give up, but leave the data
+  // in place.  Maybe it won't be locked next time.
+  if (rc == SQLITE_BUSY || rc == SQLITE_LOCKED) {
+    Shutdown(POISON);
+    return false;
+  }
+
+  // Running out of memory should be transient, retry later.
+  if (rc == SQLITE_NOMEM) {
+    Shutdown(POISON);
+    return false;
+  }
+
+  // TODO(shess): For now, leave the original database alone, pending
+  // results from Sqlite.RecoveryStep.  Some errors should probably
+  // route to RAZE_AND_POISON.
+  if (rc != SQLITE_DONE) {
+    Shutdown(POISON);
+    return false;
+  }
+
+  // Clean up the recovery db, and terminate the main database
+  // connection.
+  Shutdown(POISON);
+  return true;
+}
+
+void Recovery::Shutdown(Recovery::Disposition raze) {
+  if (!db_)
+    return;
+
+  recover_db_.Close();
+  if (raze == RAZE_AND_POISON) {
+    db_->RazeAndClose();
+  } else if (raze == POISON) {
+    db_->Poison();
+  }
+  db_ = NULL;
+}
+
+}  // namespace sql
diff --git a/sql/recovery.h b/sql/recovery.h
new file mode 100644
index 0000000..c0bb6da
--- /dev/null
+++ b/sql/recovery.h
@@ -0,0 +1,106 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SQL_RECOVERY_H_
+#define SQL_RECOVERY_H_
+
+#include "base/basictypes.h"
+
+#include "sql/connection.h"
+
+namespace base {
+class FilePath;
+}
+
+namespace sql {
+
+// Recovery module for sql/.  The basic idea is to create a fresh
+// database and populate it with the recovered contents of the
+// original database.  If recovery is successful, the recovered
+// database is backed up over the original database.  If recovery is
+// not successful, the original database is razed.  In either case,
+// the original handle is poisoned so that operations on the stack do
+// not accidentally disrupt the restored data.
+//
+// {
+//   scoped_ptr<sql::Recovery> r =
+//       sql::Recovery::Begin(orig_db, orig_db_path);
+//   if (r) {
+//     if (r.db()->Execute(kCreateSchemaSql) &&
+//         r.db()->Execute(kCopyDataFromOrigSql)) {
+//       sql::Recovery::Recovered(r.Pass());
+//     }
+//   }
+// }
+//
+// If Recovered() is not called, then RazeAndClose() is called on
+// orig_db.
+
+class SQL_EXPORT Recovery {
+ public:
+  ~Recovery();
+
+  // Begin the recovery process by opening a temporary database handle
+  // and attach the existing database to it at "corrupt".  To prevent
+  // deadlock, all transactions on |connection| are rolled back.
+  //
+  // Returns NULL in case of failure, with no cleanup done on the
+  // original connection (except for breaking the transactions).  The
+  // caller should Raze() or otherwise cleanup as appropriate.
+  //
+  // TODO(shess): Later versions of SQLite allow extracting the path
+  // from the connection.
+  // TODO(shess): Allow specifying the connection point?
+  static scoped_ptr<Recovery> Begin(
+      Connection* connection,
+      const base::FilePath& db_path) WARN_UNUSED_RESULT;
+
+  // Mark recovery completed by replicating the recovery database over
+  // the original database, then closing the recovery database.  The
+  // original database handle is poisoned, causing future calls
+  // against it to fail.
+  //
+  // If Recovered() is not called, the destructor will call
+  // Unrecoverable().
+  //
+  // TODO(shess): At this time, this function an fail while leaving
+  // the original database intact.  Figure out which failure cases
+  // should go to RazeAndClose() instead.
+  static bool Recovered(scoped_ptr<Recovery> r) WARN_UNUSED_RESULT;
+
+  // Indicate that the database is unrecoverable.  The original
+  // database is razed, and the handle poisoned.
+  static void Unrecoverable(scoped_ptr<Recovery> r);
+
+  // Handle to the temporary recovery database.
+  sql::Connection* db() { return &recover_db_; }
+
+ private:
+  explicit Recovery(Connection* connection);
+
+  // Setup the recovery database handle for Begin().  Returns false in
+  // case anything failed.
+  bool Init(const base::FilePath& db_path) WARN_UNUSED_RESULT;
+
+  // Copy the recovered database over the original database.
+  bool Backup() WARN_UNUSED_RESULT;
+
+  // Close the recovery database, and poison the original handle.
+  // |raze| controls whether the original database is razed or just
+  // poisoned.
+  enum Disposition {
+    RAZE_AND_POISON,
+    POISON,
+  };
+  void Shutdown(Disposition raze);
+
+  Connection* db_;         // Original database connection.
+  Connection recover_db_;  // Recovery connection.
+
+  DISALLOW_COPY_AND_ASSIGN(Recovery);
+};
+
+}  // namespace sql
+
+#endif  // SQL_RECOVERY_H_
diff --git a/sql/recovery_unittest.cc b/sql/recovery_unittest.cc
new file mode 100644
index 0000000..e91ee10
--- /dev/null
+++ b/sql/recovery_unittest.cc
@@ -0,0 +1,145 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/file_util.h"
+#include "base/files/scoped_temp_dir.h"
+#include "base/logging.h"
+#include "base/strings/stringprintf.h"
+#include "sql/connection.h"
+#include "sql/meta_table.h"
+#include "sql/recovery.h"
+#include "sql/statement.h"
+#include "sql/test/scoped_error_ignorer.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/sqlite/sqlite3.h"
+
+namespace {
+
+// Execute |sql|, and stringify the results with |column_sep| between
+// columns and |row_sep| between rows.
+// TODO(shess): Promote this to a central testing helper.
+std::string ExecuteWithResults(sql::Connection* db,
+                               const char* sql,
+                               const char* column_sep,
+                               const char* row_sep) {
+  sql::Statement s(db->GetUniqueStatement(sql));
+  std::string ret;
+  while (s.Step()) {
+    if (!ret.empty())
+      ret += row_sep;
+    for (int i = 0; i < s.ColumnCount(); ++i) {
+      if (i > 0)
+        ret += column_sep;
+      ret += s.ColumnString(i);
+    }
+  }
+  return ret;
+}
+
+// Dump consistent human-readable representation of the database
+// schema.  For tables or indices, this will contain the sql command
+// to create the table or index.  For certain automatic SQLite
+// structures with no sql, the name is used.
+std::string GetSchema(sql::Connection* db) {
+  const char kSql[] =
+      "SELECT COALESCE(sql, name) FROM sqlite_master ORDER BY 1";
+  return ExecuteWithResults(db, kSql, "|", "\n");
+}
+
+class SQLRecoveryTest : public testing::Test {
+ public:
+  SQLRecoveryTest() {}
+
+  virtual void SetUp() {
+    ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
+    ASSERT_TRUE(db_.Open(db_path()));
+  }
+
+  virtual void TearDown() {
+    db_.Close();
+  }
+
+  sql::Connection& db() { return db_; }
+
+  base::FilePath db_path() {
+    return temp_dir_.path().AppendASCII("SQLRecoveryTest.db");
+  }
+
+  bool Reopen() {
+    db_.Close();
+    return db_.Open(db_path());
+  }
+
+ private:
+  base::ScopedTempDir temp_dir_;
+  sql::Connection db_;
+};
+
+TEST_F(SQLRecoveryTest, RecoverBasic) {
+  const char kCreateSql[] = "CREATE TABLE x (t TEXT)";
+  const char kInsertSql[] = "INSERT INTO x VALUES ('This is a test')";
+  ASSERT_TRUE(db().Execute(kCreateSql));
+  ASSERT_TRUE(db().Execute(kInsertSql));
+  ASSERT_EQ("CREATE TABLE x (t TEXT)", GetSchema(&db()));
+
+  // If the Recovery handle goes out of scope without being
+  // Recovered(), the database is razed.
+  {
+    scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path());
+    ASSERT_TRUE(recovery.get());
+  }
+  EXPECT_FALSE(db().is_open());
+  ASSERT_TRUE(Reopen());
+  EXPECT_TRUE(db().is_open());
+  ASSERT_EQ("", GetSchema(&db()));
+
+  // Recreate the database.
+  ASSERT_TRUE(db().Execute(kCreateSql));
+  ASSERT_TRUE(db().Execute(kInsertSql));
+  ASSERT_EQ("CREATE TABLE x (t TEXT)", GetSchema(&db()));
+
+  // Unrecoverable() also razes.
+  {
+    scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path());
+    ASSERT_TRUE(recovery.get());
+    sql::Recovery::Unrecoverable(recovery.Pass());
+
+    // TODO(shess): Test that calls to recover.db() start failing.
+  }
+  EXPECT_FALSE(db().is_open());
+  ASSERT_TRUE(Reopen());
+  EXPECT_TRUE(db().is_open());
+  ASSERT_EQ("", GetSchema(&db()));
+
+  // Recreate the database.
+  ASSERT_TRUE(db().Execute(kCreateSql));
+  ASSERT_TRUE(db().Execute(kInsertSql));
+  ASSERT_EQ("CREATE TABLE x (t TEXT)", GetSchema(&db()));
+
+  // Recovered() replaces the original with the "recovered" version.
+  {
+    scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path());
+    ASSERT_TRUE(recovery.get());
+
+    // Create the new version of the table.
+    ASSERT_TRUE(recovery->db()->Execute(kCreateSql));
+
+    // Insert different data to distinguish from original database.
+    const char kAltInsertSql[] = "INSERT INTO x VALUES ('That was a test')";
+    ASSERT_TRUE(recovery->db()->Execute(kAltInsertSql));
+
+    // Successfully recovered.
+    ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass()));
+  }
+  EXPECT_FALSE(db().is_open());
+  ASSERT_TRUE(Reopen());
+  EXPECT_TRUE(db().is_open());
+  ASSERT_EQ("CREATE TABLE x (t TEXT)", GetSchema(&db()));
+
+  const char* kXSql = "SELECT * FROM x ORDER BY 1";
+  ASSERT_EQ(ExecuteWithResults(&db(), kXSql, "|", "\n"),
+            "That was a test");
+}
+
+}  // namespace
diff --git a/sql/sql.gyp b/sql/sql.gyp
index 2e8fc18..4b16673 100644
--- a/sql/sql.gyp
+++ b/sql/sql.gyp
@@ -26,6 +26,8 @@
         'init_status.h',
         'meta_table.cc',
         'meta_table.h',
+        'recovery.cc',
+        'recovery.h',
         'statement.cc',
         'statement.h',
         'transaction.cc',
@@ -81,6 +83,7 @@
       'sources': [
         'run_all_unittests.cc',
         'connection_unittest.cc',
+        'recovery_unittest.cc',
         'sqlite_features_unittest.cc',
         'statement_unittest.cc',
         'transaction_unittest.cc',