diff options
| -rw-r--r-- | chrome/browser/history/thumbnail_database.cc | 354 | ||||
| -rw-r--r-- | chrome/browser/history/thumbnail_database_unittest.cc | 111 | ||||
| -rw-r--r-- | sql/connection_unittest.cc | 23 | ||||
| -rw-r--r-- | sql/recovery.cc | 174 | ||||
| -rw-r--r-- | sql/recovery.h | 40 | ||||
| -rw-r--r-- | sql/recovery_unittest.cc | 332 | ||||
| -rw-r--r-- | sql/test/test_helpers.cc | 71 | ||||
| -rw-r--r-- | sql/test/test_helpers.h | 17 | ||||
| -rw-r--r-- | tools/metrics/histograms/histograms.xml | 35 |
9 files changed, 910 insertions, 247 deletions
diff --git a/chrome/browser/history/thumbnail_database.cc b/chrome/browser/history/thumbnail_database.cc index fe6e34f..080db55 100644 --- a/chrome/browser/history/thumbnail_database.cc +++ b/chrome/browser/history/thumbnail_database.cc @@ -275,6 +275,60 @@ void GenerateDiagnostics(sql::Connection* db, } } +// Create v5 schema for recovery code. +bool InitSchemaV5(sql::Connection* db) { + // This schema was derived from the strings used when v5 was in + // force. The [favicons] index and the [icon_mapping] items were + // copied from the current strings, after verifying that the + // resulting schema exactly matches the schema created by the + // original versions of those strings. This allows the linker to + // share the strings if they match, while preferring correctness of + // the current versions change. + + const char kFaviconsV5[] = + "CREATE TABLE IF NOT EXISTS favicons(" + "id INTEGER PRIMARY KEY," + "url LONGVARCHAR NOT NULL," + "last_updated INTEGER DEFAULT 0," + "image_data BLOB," + "icon_type INTEGER DEFAULT 1," + "sizes LONGVARCHAR" + ")"; + const char kFaviconsIndexV5[] = + "CREATE INDEX IF NOT EXISTS favicons_url ON favicons(url)"; + if (!db->Execute(kFaviconsV5) || !db->Execute(kFaviconsIndexV5)) + return false; + + const char kIconMappingV5[] = + "CREATE TABLE IF NOT EXISTS icon_mapping" + "(" + "id INTEGER PRIMARY KEY," + "page_url LONGVARCHAR NOT NULL," + "icon_id INTEGER" + ")"; + const char kIconMappingUrlIndexV5[] = + "CREATE INDEX IF NOT EXISTS icon_mapping_page_url_idx" + " ON icon_mapping(page_url)"; + const char kIconMappingIdIndexV5[] = + "CREATE INDEX IF NOT EXISTS icon_mapping_icon_id_idx" + " ON icon_mapping(icon_id)"; + if (!db->Execute(kIconMappingV5) || + !db->Execute(kIconMappingUrlIndexV5) || + !db->Execute(kIconMappingIdIndexV5)) { + return false; + } + + return true; +} + +// TODO(shess): Consider InitSchemaV7(). InitSchemaV5() is worthwhile +// because there appear to be 10s of thousands of marooned v5 +// databases in the wild. Once recovery reaches stable, the number of +// corrupt-but-recoverable databases should drop, possibly to the +// point where it is not worthwhile to maintain previous-version +// recovery code. +// TODO(shess): Alternately, think on a way to more cleanly represent +// versioned schema going forward. bool InitTables(sql::Connection* db) { const char kIconMappingSql[] = "CREATE TABLE IF NOT EXISTS icon_mapping" @@ -342,22 +396,31 @@ bool InitIndices(sql::Connection* db) { enum RecoveryEventType { RECOVERY_EVENT_RECOVERED = 0, RECOVERY_EVENT_FAILED_SCOPER, - RECOVERY_EVENT_FAILED_META_VERSION_ERROR, - RECOVERY_EVENT_FAILED_META_VERSION_NONE, + RECOVERY_EVENT_FAILED_META_VERSION_ERROR, // obsolete + RECOVERY_EVENT_FAILED_META_VERSION_NONE, // obsolete RECOVERY_EVENT_FAILED_META_WRONG_VERSION6, // obsolete - RECOVERY_EVENT_FAILED_META_WRONG_VERSION5, + RECOVERY_EVENT_FAILED_META_WRONG_VERSION5, // obsolete RECOVERY_EVENT_FAILED_META_WRONG_VERSION, - RECOVERY_EVENT_FAILED_RECOVER_META, + RECOVERY_EVENT_FAILED_RECOVER_META, // obsolete RECOVERY_EVENT_FAILED_META_INSERT, // obsolete RECOVERY_EVENT_FAILED_INIT, - RECOVERY_EVENT_FAILED_RECOVER_FAVICONS, - RECOVERY_EVENT_FAILED_FAVICONS_INSERT, - RECOVERY_EVENT_FAILED_RECOVER_FAVICON_BITMAPS, - RECOVERY_EVENT_FAILED_FAVICON_BITMAPS_INSERT, - RECOVERY_EVENT_FAILED_RECOVER_ICON_MAPPING, - RECOVERY_EVENT_FAILED_ICON_MAPPING_INSERT, + RECOVERY_EVENT_FAILED_RECOVER_FAVICONS, // obsolete + RECOVERY_EVENT_FAILED_FAVICONS_INSERT, // obsolete + RECOVERY_EVENT_FAILED_RECOVER_FAVICON_BITMAPS, // obsolete + RECOVERY_EVENT_FAILED_FAVICON_BITMAPS_INSERT, // obsolete + RECOVERY_EVENT_FAILED_RECOVER_ICON_MAPPING, // obsolete + RECOVERY_EVENT_FAILED_ICON_MAPPING_INSERT, // obsolete RECOVERY_EVENT_RECOVERED_VERSION6, RECOVERY_EVENT_FAILED_META_INIT, + RECOVERY_EVENT_FAILED_META_VERSION, + RECOVERY_EVENT_DEPRECATED, + RECOVERY_EVENT_FAILED_V5_INITSCHEMA, + RECOVERY_EVENT_FAILED_V5_AUTORECOVER_FAVICONS, + RECOVERY_EVENT_FAILED_V5_AUTORECOVER_ICON_MAPPING, + RECOVERY_EVENT_RECOVERED_VERSION5, + RECOVERY_EVENT_FAILED_AUTORECOVER_FAVICONS, + RECOVERY_EVENT_FAILED_AUTORECOVER_FAVICON_BITMAPS, + RECOVERY_EVENT_FAILED_AUTORECOVER_ICON_MAPPING, // Always keep this at the end. RECOVERY_EVENT_MAX, @@ -405,75 +468,104 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { return; } - // Setup the meta recovery table, and check that the version number - // is covered by the recovery code. - // TODO(shess): sql::Recovery should provide a helper to handle meta. - int version = 0; // For reporting which version was recovered. - { - const char kRecoverySql[] = - "CREATE VIRTUAL TABLE temp.recover_meta USING recover" - "(" - "corrupt.meta," - "key TEXT NOT NULL," - "value TEXT" // Really? Never int? - ")"; - if (!recovery->db()->Execute(kRecoverySql)) { - // TODO(shess): Failure to create the recover_meta table could - // mean that the main database is too corrupt to access, or that - // the meta table doesn't exist. + // Setup the meta recovery table and fetch the version number from + // the corrupt database. + int version = 0; + if (!recovery->SetupMeta() || !recovery->GetMetaVersionNumber(&version)) { + // TODO(shess): Prior histograms indicate all failures are in + // creating the recover virtual table for corrupt.meta. The table + // may not exist, or the database may be too far gone. Either + // way, unclear how to resolve. + sql::Recovery::Rollback(recovery.Pass()); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_VERSION); + return; + } + + // Recover v5 database to v5 schema. Next pass through Init() will + // migrate to v7. + if (version == 5) { + sql::MetaTable recover_meta_table; + if (!recover_meta_table.Init(recovery->db(), version, version)) { sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_RECOVER_META); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_INIT); return; } - { - const char kRecoveryVersionSql[] = - "SELECT value FROM recover_meta WHERE key = 'version'"; - sql::Statement recovery_version( - recovery->db()->GetUniqueStatement(kRecoveryVersionSql)); - if (!recovery_version.Step()) { - if (!recovery_version.Succeeded()) { - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_VERSION_ERROR); - // TODO(shess): An error while processing the statement is - // probably not recoverable. - } else { - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_VERSION_NONE); - // TODO(shess): If a positive version lock cannot be achieved, - // the database could still be recovered by optimistically - // attempting to copy things. In the limit, the schema found - // could be inspected. Less clear is whether optimistic - // recovery really makes sense. - } - recovery_version.Clear(); - sql::Recovery::Rollback(recovery.Pass()); - return; - } - version = recovery_version.ColumnInt(0); - - // Recovery code is generally schema-dependent. Version 7 and - // version 6 are very similar, so can be handled together. - // Track version 5, to see whether it's worth writing recovery - // code for. - if (version != 7 && version != 6) { - if (version == 5) { - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_WRONG_VERSION5); - } else { - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_WRONG_VERSION); - } - recovery_version.Clear(); - sql::Recovery::Rollback(recovery.Pass()); - return; - } + // TODO(shess): These tests are separate for histogram purposes, + // but once things look stable it can be tightened up. + if (!InitSchemaV5(recovery->db())) { + sql::Recovery::Rollback(recovery.Pass()); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_V5_INITSCHEMA); + return; } - // Either version 6 or version 7 recovers to current. - sql::MetaTable recover_meta_table; - if (!recover_meta_table.Init(recovery->db(), kCurrentVersionNumber, - kCompatibleVersionNumber)) { + if (!recovery->AutoRecoverTable("favicons", 0, &favicons_rows_recovered)) { sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_INIT); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_V5_AUTORECOVER_FAVICONS); + return; + } + + if (!recovery->AutoRecoverTable("icon_mapping", 0, + &icon_mapping_rows_recovered)) { + sql::Recovery::Rollback(recovery.Pass()); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_V5_AUTORECOVER_ICON_MAPPING); return; } + + ignore_result(sql::Recovery::Recovered(recovery.Pass())); + + // TODO(shess): Could this code be shared with the v6/7 code + // without requiring too much state to be carried? + + // Track the size of the recovered database relative to the size of + // the input database. The size should almost always be smaller, + // unless the input database was empty to start with. If the + // percentage results are very low, something is awry. + int64 final_size = 0; + if (original_size > 0 && + file_util::GetFileSize(db_path, &final_size) && + final_size > 0) { + int percentage = static_cast<int>(original_size * 100 / final_size); + UMA_HISTOGRAM_PERCENTAGE("History.FaviconsRecoveredPercentage", + std::max(100, percentage)); + } + + // Using 10,000 because these cases mostly care about "none + // recovered" and "lots recovered". More than 10,000 rows recovered + // probably means there's something wrong with the profile. + UMA_HISTOGRAM_COUNTS_10000("History.FaviconsRecoveredRowsFavicons", + favicons_rows_recovered); + UMA_HISTOGRAM_COUNTS_10000("History.FaviconsRecoveredRowsIconMapping", + icon_mapping_rows_recovered); + + RecordRecoveryEvent(RECOVERY_EVENT_RECOVERED_VERSION5); + return; + } + + // This code may be able to fetch versions that the regular + // deprecation path cannot. + if (version <= kDeprecatedVersionNumber) { + sql::Recovery::Unrecoverable(recovery.Pass()); + RecordRecoveryEvent(RECOVERY_EVENT_DEPRECATED); + return; + } + + // TODO(shess): Earlier versions have been handled or deprecated, + // later versions should be impossible. Unrecoverable() seems + // reasonable. + if (version != 6 && version != 7) { + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_WRONG_VERSION); + sql::Recovery::Rollback(recovery.Pass()); + return; + } + + // Both v6 and v7 recover to current schema version. + sql::MetaTable recover_meta_table; + if (!recover_meta_table.Init(recovery->db(), kCurrentVersionNumber, + kCompatibleVersionNumber)) { + sql::Recovery::Rollback(recovery.Pass()); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_INIT); + return; } // Create a fresh version of the database. The recovery code uses @@ -493,114 +585,24 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { return; } - // Setup favicons table. - { - // Version 6 had the |sizes| column, version 7 removed it. The - // recover virtual table treats more columns than expected as an - // error, but if _fewer_ columns are present, they can be treated - // as NULL. SQLite requires this because ALTER TABLE adds columns - // to the schema, but not to the actual table storage. - const char kRecoverySql[] = - "CREATE VIRTUAL TABLE temp.recover_favicons USING recover" - "(" - "corrupt.favicons," - "id ROWID," - "url TEXT NOT NULL," - "icon_type INTEGER," - "sizes TEXT" - ")"; - if (!recovery->db()->Execute(kRecoverySql)) { - // TODO(shess): Failure to create the recovery table probably - // means unrecoverable. - sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_RECOVER_FAVICONS); - return; - } - - // TODO(shess): Check if the DEFAULT 1 will just cover the - // COALESCE(). Either way, the new code has a literal 1 rather - // than a NULL, right? - const char kCopySql[] = - "INSERT OR REPLACE INTO main.favicons " - "SELECT id, url, COALESCE(icon_type, 1) FROM recover_favicons"; - if (!recovery->db()->Execute(kCopySql)) { - // TODO(shess): The recover_favicons table should mask problems - // with the source file, so this implies failure to write to the - // recovery database. - sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_FAVICONS_INSERT); - return; - } - favicons_rows_recovered = recovery->db()->GetLastChangeCount(); + // [favicons] differs because v6 had an unused [sizes] column which + // was removed in v7. + if (!recovery->AutoRecoverTable("favicons", 1, &favicons_rows_recovered)) { + sql::Recovery::Rollback(recovery.Pass()); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_AUTORECOVER_FAVICONS); + return; } - - // Setup favicons_bitmaps table. - { - const char kRecoverySql[] = - "CREATE VIRTUAL TABLE temp.recover_favicons_bitmaps USING recover" - "(" - "corrupt.favicon_bitmaps," - "id ROWID," - "icon_id INTEGER STRICT NOT NULL," - "last_updated INTEGER," - "image_data BLOB," - "width INTEGER," - "height INTEGER" - ")"; - if (!recovery->db()->Execute(kRecoverySql)) { - // TODO(shess): Failure to create the recovery table probably - // means unrecoverable. - sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_RECOVER_FAVICON_BITMAPS); - return; - } - - const char kCopySql[] = - "INSERT OR REPLACE INTO main.favicon_bitmaps " - "SELECT id, icon_id, COALESCE(last_updated, 0), image_data, " - " COALESCE(width, 0), COALESCE(height, 0) " - "FROM recover_favicons_bitmaps"; - if (!recovery->db()->Execute(kCopySql)) { - // TODO(shess): The recover_faviconbitmaps table should mask - // problems with the source file, so this implies failure to - // write to the recovery database. - sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_FAVICON_BITMAPS_INSERT); - return; - } - favicon_bitmaps_rows_recovered = recovery->db()->GetLastChangeCount(); + if (!recovery->AutoRecoverTable("favicon_bitmaps", 0, + &favicon_bitmaps_rows_recovered)) { + sql::Recovery::Rollback(recovery.Pass()); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_AUTORECOVER_FAVICON_BITMAPS); + return; } - - // Setup icon_mapping table. - { - const char kRecoverySql[] = - "CREATE VIRTUAL TABLE temp.recover_icon_mapping USING recover" - "(" - "corrupt.icon_mapping," - "id ROWID," - "page_url TEXT STRICT NOT NULL," - "icon_id INTEGER STRICT" - ")"; - if (!recovery->db()->Execute(kRecoverySql)) { - // TODO(shess): Failure to create the recovery table probably - // means unrecoverable. - sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_RECOVER_ICON_MAPPING); - return; - } - - const char kCopySql[] = - "INSERT OR REPLACE INTO main.icon_mapping " - "SELECT id, page_url, icon_id FROM recover_icon_mapping"; - if (!recovery->db()->Execute(kCopySql)) { - // TODO(shess): The recover_icon_mapping table should mask - // problems with the source file, so this implies failure to - // write to the recovery database. - sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_ICON_MAPPING_INSERT); - return; - } - icon_mapping_rows_recovered = recovery->db()->GetLastChangeCount(); + if (!recovery->AutoRecoverTable("icon_mapping", 0, + &icon_mapping_rows_recovered)) { + sql::Recovery::Rollback(recovery.Pass()); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_AUTORECOVER_ICON_MAPPING); + return; } // TODO(shess): Is it possible/likely to have broken foreign-key diff --git a/chrome/browser/history/thumbnail_database_unittest.cc b/chrome/browser/history/thumbnail_database_unittest.cc index 9d0dcaa7..5e72eb7 100644 --- a/chrome/browser/history/thumbnail_database_unittest.cc +++ b/chrome/browser/history/thumbnail_database_unittest.cc @@ -790,12 +790,7 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { { sql::Connection raw_db; EXPECT_TRUE(raw_db.Open(file_name_)); - { - sql::Statement statement( - raw_db.GetUniqueStatement("PRAGMA integrity_check")); - EXPECT_TRUE(statement.Step()); - ASSERT_EQ("ok", statement.ColumnString(0)); - } + ASSERT_EQ("ok", sql::test::IntegrityCheck(&raw_db)); const char kIndexName[] = "icon_mapping_page_url_idx"; const int idx_root_page = GetRootPage(&raw_db, kIndexName); @@ -818,10 +813,7 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { { sql::Connection raw_db; EXPECT_TRUE(raw_db.Open(file_name_)); - sql::Statement statement( - raw_db.GetUniqueStatement("PRAGMA integrity_check")); - EXPECT_TRUE(statement.Step()); - ASSERT_NE("ok", statement.ColumnString(0)); + ASSERT_NE("ok", sql::test::IntegrityCheck(&raw_db)); } // Open the database and access the corrupt index. @@ -844,10 +836,7 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { { sql::Connection raw_db; EXPECT_TRUE(raw_db.Open(file_name_)); - sql::Statement statement( - raw_db.GetUniqueStatement("PRAGMA integrity_check")); - EXPECT_TRUE(statement.Step()); - EXPECT_EQ("ok", statement.ColumnString(0)); + ASSERT_EQ("ok", sql::test::IntegrityCheck(&raw_db)); // Check that the expected tables exist. VerifyTablesAndColumns(&raw_db); @@ -867,21 +856,8 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { kIconUrl1, kLargeSize, sizeof(kBlob1), kBlob1)); } - // Corrupt the database again by making the actual file shorter than - // the header expects. - { - int64 db_size = 0; - EXPECT_TRUE(file_util::GetFileSize(file_name_, &db_size)); - { - sql::Connection raw_db; - EXPECT_TRUE(raw_db.Open(file_name_)); - EXPECT_TRUE(raw_db.Execute("CREATE TABLE t(x)")); - } - file_util::ScopedFILE file(file_util::OpenFile(file_name_, "rb+")); - ASSERT_TRUE(file.get() != NULL); - EXPECT_EQ(0, fseek(file.get(), static_cast<long>(db_size), SEEK_SET)); - EXPECT_TRUE(file_util::TruncateFile(file.get())); - } + // Corrupt the database again by adjusting the header. + EXPECT_TRUE(sql::test::CorruptSizeInHeader(file_name_)); // Database is unusable at the SQLite level. { @@ -918,23 +894,10 @@ TEST_F(ThumbnailDatabaseTest, Recovery6) { // (which would upgrade it). EXPECT_TRUE(CreateDatabaseFromSQL(file_name_, "Favicons.v6.sql")); - // Corrupt the database by making the actual file shorter than the - // SQLite header expects. This form of corruption will cause - // immediate failures during Open(), before the migration code runs, - // so the version-6 recovery will occur. - { - int64 db_size = 0; - EXPECT_TRUE(file_util::GetFileSize(file_name_, &db_size)); - { - sql::Connection raw_db; - EXPECT_TRUE(raw_db.Open(file_name_)); - EXPECT_TRUE(raw_db.Execute("CREATE TABLE t(x)")); - } - file_util::ScopedFILE file(file_util::OpenFile(file_name_, "rb+")); - ASSERT_TRUE(file.get() != NULL); - EXPECT_EQ(0, fseek(file.get(), static_cast<long>(db_size), SEEK_SET)); - EXPECT_TRUE(file_util::TruncateFile(file.get())); - } + // Corrupt the database again by adjusting the header. This form of + // corruption will cause immediate failures during Open(), before + // the migration code runs, so the version-6 recovery will occur. + EXPECT_TRUE(sql::test::CorruptSizeInHeader(file_name_)); // Database is unusable at the SQLite level. { @@ -970,10 +933,58 @@ TEST_F(ThumbnailDatabaseTest, Recovery6) { { sql::Connection raw_db; EXPECT_TRUE(raw_db.Open(file_name_)); - sql::Statement statement( - raw_db.GetUniqueStatement("PRAGMA integrity_check")); - EXPECT_TRUE(statement.Step()); - EXPECT_EQ("ok", statement.ColumnString(0)); + ASSERT_EQ("ok", sql::test::IntegrityCheck(&raw_db)); + + // Check that the expected tables exist. + VerifyTablesAndColumns(&raw_db); + } +} + +TEST_F(ThumbnailDatabaseTest, Recovery5) { + // Create an example database without loading into ThumbnailDatabase + // (which would upgrade it). + EXPECT_TRUE(CreateDatabaseFromSQL(file_name_, "Favicons.v5.sql")); + + // Corrupt the database again by adjusting the header. This form of + // corruption will cause immediate failures during Open(), before + // the migration code runs, so the version-5 recovery will occur. + EXPECT_TRUE(sql::test::CorruptSizeInHeader(file_name_)); + + // Database is unusable at the SQLite level. + { + sql::ScopedErrorIgnorer ignore_errors; + ignore_errors.IgnoreError(SQLITE_CORRUPT); + sql::Connection raw_db; + EXPECT_TRUE(raw_db.Open(file_name_)); + EXPECT_FALSE(raw_db.IsSQLValid("PRAGMA integrity_check")); + ASSERT_TRUE(ignore_errors.CheckIgnoredErrors()); + } + + // Database should be recovered during open. + { + sql::ScopedErrorIgnorer ignore_errors; + ignore_errors.IgnoreError(SQLITE_CORRUPT); + ThumbnailDatabase db; + ASSERT_EQ(sql::INIT_OK, db.Init(file_name_)); + + // Test that some data is present, copied from + // ThumbnailDatabaseTest.Version5 . + EXPECT_TRUE( + CheckPageHasIcon(&db, kPageUrl3, chrome::FAVICON, + kIconUrl1, gfx::Size(), sizeof(kBlob1), kBlob1)); + EXPECT_TRUE( + CheckPageHasIcon(&db, kPageUrl3, chrome::TOUCH_ICON, + kIconUrl3, gfx::Size(), sizeof(kBlob2), kBlob2)); + + ASSERT_TRUE(ignore_errors.CheckIgnoredErrors()); + } + + // Check that the database is recovered at a SQLite level, and that + // the current schema is in place. + { + sql::Connection raw_db; + EXPECT_TRUE(raw_db.Open(file_name_)); + ASSERT_EQ("ok", sql::test::IntegrityCheck(&raw_db)); // Check that the expected tables exist. VerifyTablesAndColumns(&raw_db); diff --git a/sql/connection_unittest.cc b/sql/connection_unittest.cc index 445db34..5ffc26a 100644 --- a/sql/connection_unittest.cc +++ b/sql/connection_unittest.cc @@ -11,6 +11,7 @@ #include "sql/statement.h" #include "sql/test/error_callback_support.h" #include "sql/test/scoped_error_ignorer.h" +#include "sql/test/test_helpers.h" #include "testing/gtest/include/gtest/gtest.h" #include "third_party/sqlite/sqlite3.h" @@ -502,32 +503,14 @@ TEST_F(SQLConnectionTest, RazeNOTADB2) { // essential for cases where the Open() can fail entirely, so the // Raze() cannot happen later. Additionally test that when the // callback does this during Open(), the open is retried and succeeds. -// -// Most corruptions seen in the wild seem to happen when two pages in -// the database were not written transactionally (the transaction -// changed both, but one wasn't successfully written for some reason). -// A special case of that is when the header indicates that the -// database contains more pages than are in the file. This breaks -// things at a very basic level, verify that Raze() can handle it. TEST_F(SQLConnectionTest, RazeCallbackReopen) { const char* kCreateSql = "CREATE TABLE foo (id INTEGER PRIMARY KEY, value)"; ASSERT_TRUE(db().Execute(kCreateSql)); ASSERT_EQ(1, SqliteMasterCount(&db())); - int page_size = 0; - { - sql::Statement s(db().GetUniqueStatement("PRAGMA page_size")); - ASSERT_TRUE(s.Step()); - page_size = s.ColumnInt(0); - } db().Close(); - // Trim a single page from the end of the file. - { - file_util::ScopedFILE file(file_util::OpenFile(db_path(), "rb+")); - ASSERT_TRUE(file.get() != NULL); - ASSERT_EQ(0, fseek(file.get(), -page_size, SEEK_END)); - ASSERT_TRUE(file_util::TruncateFile(file.get())); - } + // Corrupt the database so that nothing works, including PRAGMAs. + ASSERT_TRUE(sql::test::CorruptSizeInHeader(db_path())); // Open() will succeed, even though the PRAGMA calls within will // fail with SQLITE_CORRUPT, as will this PRAGMA. diff --git a/sql/recovery.cc b/sql/recovery.cc index c750fd0..46f609b 100644 --- a/sql/recovery.cc +++ b/sql/recovery.cc @@ -5,9 +5,13 @@ #include "sql/recovery.h" #include "base/files/file_path.h" +#include "base/format_macros.h" #include "base/logging.h" #include "base/metrics/sparse_histogram.h" +#include "base/strings/string_util.h" +#include "base/strings/stringprintf.h" #include "sql/connection.h" +#include "sql/statement.h" #include "third_party/sqlite/sqlite3.h" namespace sql { @@ -233,4 +237,174 @@ void Recovery::Shutdown(Recovery::Disposition raze) { db_ = NULL; } +bool Recovery::AutoRecoverTable(const char* table_name, + size_t extend_columns, + size_t* rows_recovered) { + // Query the info for the recovered table in database [main]. + std::string query( + base::StringPrintf("PRAGMA main.table_info(%s)", table_name)); + Statement s(db()->GetUniqueStatement(query.c_str())); + + // The columns of the recover virtual table. + std::vector<std::string> create_column_decls; + + // The columns to select from the recover virtual table when copying + // to the recovered table. + std::vector<std::string> insert_columns; + + // If PRIMARY KEY is a single INTEGER column, then it is an alias + // for ROWID. The primary key can be compound, so this can only be + // determined after processing all column data and tracking what is + // seen. |pk_column_count| counts the columns in the primary key. + // |rowid_decl| stores the ROWID version of the last INTEGER column + // seen, which is at |rowid_ofs| in |create_column_decls|. + size_t pk_column_count = 0; + size_t rowid_ofs; // Only valid if rowid_decl is set. + std::string rowid_decl; // ROWID version of column |rowid_ofs|. + + while (s.Step()) { + const std::string column_name(s.ColumnString(1)); + const std::string column_type(s.ColumnString(2)); + const bool not_null = s.ColumnBool(3); + const int default_type = s.ColumnType(4); + const bool default_is_null = (default_type == COLUMN_TYPE_NULL); + const int pk_column = s.ColumnInt(5); + + if (pk_column > 0) { + // TODO(shess): http://www.sqlite.org/pragma.html#pragma_table_info + // documents column 5 as the index of the column in the primary key + // (zero for not in primary key). I find that it is always 1 for + // columns in the primary key. Since this code is very dependent on + // that pragma, review if the implementation changes. + DCHECK_EQ(pk_column, 1); + ++pk_column_count; + } + + // Construct column declaration as "name type [optional constraint]". + std::string column_decl = column_name; + + // SQLite's affinity detection is documented at: + // http://www.sqlite.org/datatype3.html#affname + // The gist of it is that CHAR, TEXT, and INT use substring matches. + if (column_type.find("INT") != std::string::npos) { + if (pk_column == 1) { + rowid_ofs = create_column_decls.size(); + rowid_decl = column_name + " ROWID"; + } + column_decl += " INTEGER"; + } else if (column_type.find("CHAR") != std::string::npos || + column_type.find("TEXT") != std::string::npos) { + column_decl += " TEXT"; + } else if (column_type == "BLOB") { + column_decl += " BLOB"; + } else { + // TODO(shess): AFAICT, there remain: + // - contains("CLOB") -> TEXT + // - contains("REAL") -> REAL + // - contains("FLOA") -> REAL + // - contains("DOUB") -> REAL + // - other -> "NUMERIC" + // Just code those in as they come up. + NOTREACHED() << " Unsupported type " << column_type; + return false; + } + + // If column has constraint "NOT NULL", then inserting NULL into + // that column will fail. If the column has a non-NULL DEFAULT + // specified, the INSERT will handle it (see below). If the + // DEFAULT is also NULL, the row must be filtered out. + // TODO(shess): The above scenario applies to INSERT OR REPLACE, + // whereas INSERT OR IGNORE drops such rows. + // http://www.sqlite.org/lang_conflict.html + if (not_null && default_is_null) + column_decl += " NOT NULL"; + + create_column_decls.push_back(column_decl); + + // Per the NOTE in the header file, convert NULL values to the + // DEFAULT. All columns could be IFNULL(column_name,default), but + // the NULL case would require special handling either way. + if (default_is_null) { + insert_columns.push_back(column_name); + } else { + // The default value appears to be pre-quoted, as if it is + // literally from the sqlite_master CREATE statement. + std::string default_value = s.ColumnString(4); + insert_columns.push_back(base::StringPrintf( + "IFNULL(%s,%s)", column_name.c_str(), default_value.c_str())); + } + } + + // Receiving no column information implies that the table doesn't exist. + if (create_column_decls.empty()) + return false; + + // If the PRIMARY KEY was a single INTEGER column, convert it to ROWID. + if (pk_column_count == 1 && !rowid_decl.empty()) + create_column_decls[rowid_ofs] = rowid_decl; + + // Additional columns accept anything. + // TODO(shess): ignoreN isn't well namespaced. But it will fail to + // execute in case of conflicts. + for (size_t i = 0; i < extend_columns; ++i) { + create_column_decls.push_back( + base::StringPrintf("ignore%" PRIuS " ANY", i)); + } + + std::string recover_create(base::StringPrintf( + "CREATE VIRTUAL TABLE temp.recover_%s USING recover(corrupt.%s, %s)", + table_name, + table_name, + JoinString(create_column_decls, ',').c_str())); + + std::string recover_insert(base::StringPrintf( + "INSERT OR REPLACE INTO main.%s SELECT %s FROM temp.recover_%s", + table_name, + JoinString(insert_columns, ',').c_str(), + table_name)); + + std::string recover_drop(base::StringPrintf( + "DROP TABLE temp.recover_%s", table_name)); + + if (!db()->Execute(recover_create.c_str())) + return false; + + if (!db()->Execute(recover_insert.c_str())) { + ignore_result(db()->Execute(recover_drop.c_str())); + return false; + } + + *rows_recovered = db()->GetLastChangeCount(); + + // TODO(shess): Is leaving the recover table around a breaker? + return db()->Execute(recover_drop.c_str()); +} + +bool Recovery::SetupMeta() { + const char kCreateSql[] = + "CREATE VIRTUAL TABLE temp.recover_meta USING recover" + "(" + "corrupt.meta," + "key TEXT NOT NULL," + "value ANY" // Whatever is stored. + ")"; + return db()->Execute(kCreateSql); +} + +bool Recovery::GetMetaVersionNumber(int* version) { + DCHECK(version); + // TODO(shess): DCHECK(db()->DoesTableExist("temp.recover_meta")); + // Unfortunately, DoesTableExist() queries sqlite_master, not + // sqlite_temp_master. + + const char kVersionSql[] = + "SELECT value FROM temp.recover_meta WHERE key = 'version'"; + sql::Statement recovery_version(db()->GetUniqueStatement(kVersionSql)); + if (!recovery_version.Step()) + return false; + + *version = recovery_version.ColumnInt(0); + return true; +} + } // namespace sql diff --git a/sql/recovery.h b/sql/recovery.h index be23e97..2475b0f 100644 --- a/sql/recovery.h +++ b/sql/recovery.h @@ -93,6 +93,46 @@ class SQL_EXPORT Recovery { // Handle to the temporary recovery database. sql::Connection* db() { return &recover_db_; } + // Attempt to recover the named table from the corrupt database into + // the recovery database using a temporary recover virtual table. + // The virtual table schema is derived from the named table's schema + // in database [main]. Data is copied using INSERT OR REPLACE, so + // duplicates overwrite each other. + // + // |extend_columns| allows recovering tables which have excess + // columns relative to the target schema. The recover virtual table + // treats more data than specified as a sign of corruption. + // + // Returns true if all operations succeeded, with the number of rows + // recovered in |*rows_recovered|. + // + // NOTE(shess): Due to a flaw in the recovery virtual table, at this + // time this code injects the DEFAULT value of the target table in + // locations where the recovery table returns NULL. This is not + // entirely correct, because it happens both when there is a short + // row (correct) but also where there is an actual NULL value + // (incorrect). + // + // TODO(shess): Flag for INSERT OR REPLACE vs IGNORE. + // TODO(shess): Handle extended table names. + bool AutoRecoverTable(const char* table_name, + size_t extend_columns, + size_t* rows_recovered); + + // Setup a recover virtual table at temp.recover_meta, reading from + // corrupt.meta. Returns true if created. + // TODO(shess): Perhaps integrate into Begin(). + // TODO(shess): Add helpers to fetch additional items from the meta + // table as needed. + bool SetupMeta(); + + // Fetch the version number from temp.recover_meta. Returns false + // if the query fails, or if there is no version row. Otherwise + // returns true, with the version in |*version_number|. + // + // Only valid to call after successful SetupMeta(). + bool GetMetaVersionNumber(int* version_number); + private: explicit Recovery(Connection* connection); diff --git a/sql/recovery_unittest.cc b/sql/recovery_unittest.cc index cc06090..e9e77cf 100644 --- a/sql/recovery_unittest.cc +++ b/sql/recovery_unittest.cc @@ -6,6 +6,7 @@ #include "base/file_util.h" #include "base/files/scoped_temp_dir.h" #include "base/logging.h" +#include "base/strings/string_number_conversions.h" #include "base/strings/stringprintf.h" #include "sql/connection.h" #include "sql/meta_table.h" @@ -32,7 +33,15 @@ std::string ExecuteWithResults(sql::Connection* db, for (int i = 0; i < s.ColumnCount(); ++i) { if (i > 0) ret += column_sep; - ret += s.ColumnString(i); + if (s.ColumnType(i) == sql::COLUMN_TYPE_NULL) { + ret += "<null>"; + } else if (s.ColumnType(i) == sql::COLUMN_TYPE_BLOB) { + ret += "<x'"; + ret += base::HexEncode(s.ColumnBlob(i), s.ColumnByteLength(i)); + ret += "'>"; + } else { + ret += s.ColumnString(i); + } } } return ret; @@ -422,6 +431,327 @@ TEST_F(SQLRecoveryTest, RecoverCorruptTable) { const char kSelectSql[] = "SELECT v FROM x WHERE id = 0"; EXPECT_EQ("100", ExecuteWithResults(&db(), kSelectSql, "|", ",")); } + +TEST_F(SQLRecoveryTest, Meta) { + const int kVersion = 3; + const int kCompatibleVersion = 2; + + { + sql::MetaTable meta; + EXPECT_TRUE(meta.Init(&db(), kVersion, kCompatibleVersion)); + EXPECT_EQ(kVersion, meta.GetVersionNumber()); + } + + // Test expected case where everything works. + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + EXPECT_TRUE(recovery->SetupMeta()); + int version = 0; + EXPECT_TRUE(recovery->GetMetaVersionNumber(&version)); + EXPECT_EQ(kVersion, version); + + sql::Recovery::Rollback(recovery.Pass()); + } + ASSERT_TRUE(Reopen()); // Handle was poisoned. + + // Test version row missing. + EXPECT_TRUE(db().Execute("DELETE FROM meta WHERE key = 'version'")); + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + EXPECT_TRUE(recovery->SetupMeta()); + int version = 0; + EXPECT_FALSE(recovery->GetMetaVersionNumber(&version)); + EXPECT_EQ(0, version); + + sql::Recovery::Rollback(recovery.Pass()); + } + ASSERT_TRUE(Reopen()); // Handle was poisoned. + + // Test meta table missing. + EXPECT_TRUE(db().Execute("DROP TABLE meta")); + { + sql::ScopedErrorIgnorer ignore_errors; + ignore_errors.IgnoreError(SQLITE_CORRUPT); // From virtual table. + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + EXPECT_FALSE(recovery->SetupMeta()); + ASSERT_TRUE(ignore_errors.CheckIgnoredErrors()); + } +} + +// Baseline AutoRecoverTable() test. +TEST_F(SQLRecoveryTest, AutoRecoverTable) { + // BIGINT and VARCHAR to test type affinity. + const char kCreateSql[] = "CREATE TABLE x (id BIGINT, t TEXT, v VARCHAR)"; + ASSERT_TRUE(db().Execute(kCreateSql)); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (11, 'This is', 'a test')")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (5, 'That was', 'a test')")); + + // Save aside a copy of the original schema and data. + const std::string orig_schema(GetSchema(&db())); + const char kXSql[] = "SELECT * FROM x ORDER BY 1"; + const std::string orig_data(ExecuteWithResults(&db(), kXSql, "|", "\n")); + + // Create a lame-duck table which will not be propagated by recovery to + // detect that the recovery code actually ran. + ASSERT_TRUE(db().Execute("CREATE TABLE y (c TEXT)")); + ASSERT_NE(orig_schema, GetSchema(&db())); + + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + ASSERT_TRUE(recovery->db()->Execute(kCreateSql)); + + // Save a copy of the temp db's schema before recovering the table. + const char kTempSchemaSql[] = "SELECT name, sql FROM sqlite_temp_master"; + const std::string temp_schema( + ExecuteWithResults(recovery->db(), kTempSchemaSql, "|", "\n")); + + size_t rows = 0; + EXPECT_TRUE(recovery->AutoRecoverTable("x", 0, &rows)); + EXPECT_EQ(2u, rows); + + // Test that any additional temp tables were cleaned up. + EXPECT_EQ(temp_schema, + ExecuteWithResults(recovery->db(), kTempSchemaSql, "|", "\n")); + + ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass())); + } + + // Since the database was not corrupt, the entire schema and all + // data should be recovered. + ASSERT_TRUE(Reopen()); + ASSERT_EQ(orig_schema, GetSchema(&db())); + ASSERT_EQ(orig_data, ExecuteWithResults(&db(), kXSql, "|", "\n")); + + // Recovery fails if the target table doesn't exist. + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + ASSERT_TRUE(recovery->db()->Execute(kCreateSql)); + + // TODO(shess): Should this failure implicitly lead to Raze()? + size_t rows = 0; + EXPECT_FALSE(recovery->AutoRecoverTable("y", 0, &rows)); + + sql::Recovery::Unrecoverable(recovery.Pass()); + } +} + +// Test that default values correctly replace nulls. The recovery +// virtual table reads directly from the database, so DEFAULT is not +// interpretted at that level. +TEST_F(SQLRecoveryTest, AutoRecoverTableWithDefault) { + ASSERT_TRUE(db().Execute("CREATE TABLE x (id INTEGER)")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (5)")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (15)")); + + // ALTER effectively leaves the new columns NULL in the first two + // rows. The row with 17 will get the default injected at insert + // time, while the row with 42 will get the actual value provided. + // Embedded "'" to make sure default-handling continues to be quoted + // correctly. + ASSERT_TRUE(db().Execute("ALTER TABLE x ADD COLUMN t TEXT DEFAULT 'a''a'")); + ASSERT_TRUE(db().Execute("ALTER TABLE x ADD COLUMN b BLOB DEFAULT x'AA55'")); + ASSERT_TRUE(db().Execute("ALTER TABLE x ADD COLUMN i INT DEFAULT 93")); + ASSERT_TRUE(db().Execute("INSERT INTO x (id) VALUES (17)")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (42, 'b', x'1234', 12)")); + + // Save aside a copy of the original schema and data. + const std::string orig_schema(GetSchema(&db())); + const char kXSql[] = "SELECT * FROM x ORDER BY 1"; + const std::string orig_data(ExecuteWithResults(&db(), kXSql, "|", "\n")); + + // Create a lame-duck table which will not be propagated by recovery to + // detect that the recovery code actually ran. + ASSERT_TRUE(db().Execute("CREATE TABLE y (c TEXT)")); + ASSERT_NE(orig_schema, GetSchema(&db())); + + // Mechanically adjust the stored schema and data to allow detecting + // where the default value is coming from. The target table is just + // like the original with the default for [t] changed, to signal + // defaults coming from the recovery system. The two %5 rows should + // get the target-table default for [t], while the others should get + // the source-table default. + std::string final_schema(orig_schema); + std::string final_data(orig_data); + size_t pos; + while ((pos = final_schema.find("'a''a'")) != std::string::npos) { + final_schema.replace(pos, 6, "'c''c'"); + } + while ((pos = final_data.find("5|a'a")) != std::string::npos) { + final_data.replace(pos, 5, "5|c'c"); + } + + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + // Different default to detect which table provides the default. + ASSERT_TRUE(recovery->db()->Execute(final_schema.c_str())); + + size_t rows = 0; + EXPECT_TRUE(recovery->AutoRecoverTable("x", 0, &rows)); + EXPECT_EQ(4u, rows); + + ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass())); + } + + // Since the database was not corrupt, the entire schema and all + // data should be recovered. + ASSERT_TRUE(Reopen()); + ASSERT_EQ(final_schema, GetSchema(&db())); + ASSERT_EQ(final_data, ExecuteWithResults(&db(), kXSql, "|", "\n")); +} + +// Test that rows with NULL in a NOT NULL column are filtered +// correctly. In the wild, this would probably happen due to +// corruption, but here it is simulated by recovering a table which +// allowed nulls into a table which does not. +TEST_F(SQLRecoveryTest, AutoRecoverTableNullFilter) { + const char kOrigSchema[] = "CREATE TABLE x (id INTEGER, t TEXT)"; + const char kFinalSchema[] = "CREATE TABLE x (id INTEGER, t TEXT NOT NULL)"; + + ASSERT_TRUE(db().Execute(kOrigSchema)); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (5, null)")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (15, 'this is a test')")); + + // Create a lame-duck table which will not be propagated by recovery to + // detect that the recovery code actually ran. + ASSERT_EQ(kOrigSchema, GetSchema(&db())); + ASSERT_TRUE(db().Execute("CREATE TABLE y (c TEXT)")); + ASSERT_NE(kOrigSchema, GetSchema(&db())); + + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + ASSERT_TRUE(recovery->db()->Execute(kFinalSchema)); + + size_t rows = 0; + EXPECT_TRUE(recovery->AutoRecoverTable("x", 0, &rows)); + EXPECT_EQ(1u, rows); + + ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass())); + } + + // The schema should be the same, but only one row of data should + // have been recovered. + ASSERT_TRUE(Reopen()); + ASSERT_EQ(kFinalSchema, GetSchema(&db())); + const char kXSql[] = "SELECT * FROM x ORDER BY 1"; + ASSERT_EQ("15|this is a test", ExecuteWithResults(&db(), kXSql, "|", "\n")); +} + +// Test AutoRecoverTable with a ROWID alias. +TEST_F(SQLRecoveryTest, AutoRecoverTableWithRowid) { + // The rowid alias is almost always the first column, intentionally + // put it later. + const char kCreateSql[] = + "CREATE TABLE x (t TEXT, id INTEGER PRIMARY KEY NOT NULL)"; + ASSERT_TRUE(db().Execute(kCreateSql)); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES ('This is a test', null)")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES ('That was a test', null)")); + + // Save aside a copy of the original schema and data. + const std::string orig_schema(GetSchema(&db())); + const char kXSql[] = "SELECT * FROM x ORDER BY 1"; + const std::string orig_data(ExecuteWithResults(&db(), kXSql, "|", "\n")); + + // Create a lame-duck table which will not be propagated by recovery to + // detect that the recovery code actually ran. + ASSERT_TRUE(db().Execute("CREATE TABLE y (c TEXT)")); + ASSERT_NE(orig_schema, GetSchema(&db())); + + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + ASSERT_TRUE(recovery->db()->Execute(kCreateSql)); + + size_t rows = 0; + EXPECT_TRUE(recovery->AutoRecoverTable("x", 0, &rows)); + EXPECT_EQ(2u, rows); + + ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass())); + } + + // Since the database was not corrupt, the entire schema and all + // data should be recovered. + ASSERT_TRUE(Reopen()); + ASSERT_EQ(orig_schema, GetSchema(&db())); + ASSERT_EQ(orig_data, ExecuteWithResults(&db(), kXSql, "|", "\n")); +} + +// Test that a compound primary key doesn't fire the ROWID code. +TEST_F(SQLRecoveryTest, AutoRecoverTableWithCompoundKey) { + const char kCreateSql[] = + "CREATE TABLE x (" + "id INTEGER NOT NULL," + "id2 TEXT NOT NULL," + "t TEXT," + "PRIMARY KEY (id, id2)" + ")"; + ASSERT_TRUE(db().Execute(kCreateSql)); + + // NOTE(shess): Do not accidentally use [id] 1, 2, 3, as those will + // be the ROWID values. + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (1, 'a', 'This is a test')")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (1, 'b', 'That was a test')")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (2, 'a', 'Another test')")); + + // Save aside a copy of the original schema and data. + const std::string orig_schema(GetSchema(&db())); + const char kXSql[] = "SELECT * FROM x ORDER BY 1"; + const std::string orig_data(ExecuteWithResults(&db(), kXSql, "|", "\n")); + + // Create a lame-duck table which will not be propagated by recovery to + // detect that the recovery code actually ran. + ASSERT_TRUE(db().Execute("CREATE TABLE y (c TEXT)")); + ASSERT_NE(orig_schema, GetSchema(&db())); + + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + ASSERT_TRUE(recovery->db()->Execute(kCreateSql)); + + size_t rows = 0; + EXPECT_TRUE(recovery->AutoRecoverTable("x", 0, &rows)); + EXPECT_EQ(3u, rows); + + ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass())); + } + + // Since the database was not corrupt, the entire schema and all + // data should be recovered. + ASSERT_TRUE(Reopen()); + ASSERT_EQ(orig_schema, GetSchema(&db())); + ASSERT_EQ(orig_data, ExecuteWithResults(&db(), kXSql, "|", "\n")); +} + +// Test |extend_columns| support. +TEST_F(SQLRecoveryTest, AutoRecoverTableExtendColumns) { + const char kCreateSql[] = "CREATE TABLE x (id INTEGER PRIMARY KEY, t0 TEXT)"; + ASSERT_TRUE(db().Execute(kCreateSql)); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (1, 'This is')")); + ASSERT_TRUE(db().Execute("INSERT INTO x VALUES (2, 'That was')")); + + // Save aside a copy of the original schema and data. + const std::string orig_schema(GetSchema(&db())); + const char kXSql[] = "SELECT * FROM x ORDER BY 1"; + const std::string orig_data(ExecuteWithResults(&db(), kXSql, "|", "\n")); + + // Modify the table to add a column, and add data to that column. + ASSERT_TRUE(db().Execute("ALTER TABLE x ADD COLUMN t1 TEXT")); + ASSERT_TRUE(db().Execute("UPDATE x SET t1 = 'a test'")); + ASSERT_NE(orig_schema, GetSchema(&db())); + ASSERT_NE(orig_data, ExecuteWithResults(&db(), kXSql, "|", "\n")); + + { + scoped_ptr<sql::Recovery> recovery = sql::Recovery::Begin(&db(), db_path()); + ASSERT_TRUE(recovery->db()->Execute(kCreateSql)); + size_t rows = 0; + EXPECT_TRUE(recovery->AutoRecoverTable("x", 1, &rows)); + EXPECT_EQ(2u, rows); + ASSERT_TRUE(sql::Recovery::Recovered(recovery.Pass())); + } + + // Since the database was not corrupt, the entire schema and all + // data should be recovered. + ASSERT_TRUE(Reopen()); + ASSERT_EQ(orig_schema, GetSchema(&db())); + ASSERT_EQ(orig_data, ExecuteWithResults(&db(), kXSql, "|", "\n")); +} #endif // !defined(USE_SYSTEM_SQLITE) } // namespace diff --git a/sql/test/test_helpers.cc b/sql/test/test_helpers.cc index de3e8f8..5438bd6 100644 --- a/sql/test/test_helpers.cc +++ b/sql/test/test_helpers.cc @@ -21,11 +21,73 @@ size_t CountSQLItemsOfType(sql::Connection* db, const char* type) { return s.ColumnInt(0); } +// Helper for reading a number from the SQLite header. +// See net/base/big_endian.h. +unsigned ReadBigEndian(unsigned char* buf, size_t bytes) { + unsigned r = buf[0]; + for (size_t i = 1; i < bytes; i++) { + r <<= 8; + r |= buf[i]; + } + return r; +} + +// Helper for writing a number to the SQLite header. +void WriteBigEndian(unsigned val, unsigned char* buf, size_t bytes) { + for (size_t i = 0; i < bytes; i++) { + buf[bytes - i - 1] = (val & 0xFF); + val >>= 8; + } +} + } // namespace namespace sql { namespace test { +bool CorruptSizeInHeader(const base::FilePath& db_path) { + // See http://www.sqlite.org/fileformat.html#database_header + const size_t kHeaderSize = 100; + const size_t kPageSizeOffset = 16; + const size_t kFileChangeCountOffset = 24; + const size_t kPageCountOffset = 28; + const size_t kVersionValidForOffset = 92; // duplicate kFileChangeCountOffset + + unsigned char header[kHeaderSize]; + + file_util::ScopedFILE file(file_util::OpenFile(db_path, "rb+")); + if (!file.get()) + return false; + + if (0 != fseek(file.get(), 0, SEEK_SET)) + return false; + if (1u != fread(header, sizeof(header), 1, file.get())) + return false; + + int64 db_size = 0; + if (!file_util::GetFileSize(db_path, &db_size)) + return false; + + const unsigned page_size = ReadBigEndian(header + kPageSizeOffset, 2); + + // One larger than the expected size. + const unsigned page_count = (db_size + page_size) / page_size; + WriteBigEndian(page_count, header + kPageCountOffset, 4); + + // Update change count so outstanding readers know the info changed. + // Both spots must match for the page count to be considered valid. + unsigned change_count = ReadBigEndian(header + kFileChangeCountOffset, 4); + WriteBigEndian(change_count + 1, header + kFileChangeCountOffset, 4); + WriteBigEndian(change_count + 1, header + kVersionValidForOffset, 4); + + if (0 != fseek(file.get(), 0, SEEK_SET)) + return false; + if (1u != fwrite(header, sizeof(header), 1, file.get())) + return false; + + return true; +} + size_t CountSQLTables(sql::Connection* db) { return CountSQLItemsOfType(db, "table"); } @@ -91,5 +153,14 @@ bool CreateDatabaseFromSQL(const base::FilePath& db_path, return db.Execute(sql.c_str()); } +std::string IntegrityCheck(sql::Connection* db) { + sql::Statement statement(db->GetUniqueStatement("PRAGMA integrity_check")); + + // SQLite should always return a row of data. + EXPECT_TRUE(statement.Step()); + + return statement.ColumnString(0); +} + } // namespace test } // namespace sql diff --git a/sql/test/test_helpers.h b/sql/test/test_helpers.h index 330f59a..b9d5e9b 100644 --- a/sql/test/test_helpers.h +++ b/sql/test/test_helpers.h @@ -5,6 +5,8 @@ #ifndef SQL_TEST_TEST_HELPERS_H_ #define SQL_TEST_TEST_HELPERS_H_ +#include <string> + #include "base/basictypes.h" #include "base/compiler_specific.h" @@ -21,6 +23,16 @@ class Connection; namespace sql { namespace test { +// SQLite stores the database size in the header, and if the actual +// OS-derived size is smaller, the database is considered corrupt. +// [This case is actually a common form of corruption in the wild.] +// This helper sets the in-header size to one page larger than the +// actual file size. The resulting file will return SQLITE_CORRUPT +// for most operations unless PRAGMA writable_schema is turned ON. +// +// Returns false if any error occurs accessing the file. +bool CorruptSizeInHeader(const base::FilePath& db_path) WARN_UNUSED_RESULT; + // Return the number of tables in sqlite_master. size_t CountSQLTables(sql::Connection* db) WARN_UNUSED_RESULT; @@ -43,6 +55,11 @@ bool CountTableRows(sql::Connection* db, const char* table, size_t* count); bool CreateDatabaseFromSQL(const base::FilePath& db_path, const base::FilePath& sql_path) WARN_UNUSED_RESULT; +// Return the results of running "PRAGMA integrity_check" on |db|. +// TODO(shess): sql::Connection::IntegrityCheck() is basically the +// same, but not as convenient for testing. Maybe combine. +std::string IntegrityCheck(sql::Connection* db) WARN_UNUSED_RESULT; + } // namespace test } // namespace sql diff --git a/tools/metrics/histograms/histograms.xml b/tools/metrics/histograms/histograms.xml index 9510e38..603e5ad 100644 --- a/tools/metrics/histograms/histograms.xml +++ b/tools/metrics/histograms/histograms.xml @@ -18852,6 +18852,14 @@ other types of suffix sets. <summary>Errors truncating database for Raze().</summary> </histogram> +<histogram name="Sqlite.RecoveryHandle" enum="SqliteErrorCode"> + <summary>Error from sqlite3_backup_init() in sql::Recovery.</summary> +</histogram> + +<histogram name="Sqlite.RecoveryStep" enum="SqliteErrorCode"> + <summary>Error from sqlite3_backup_step() in sql::Recovery.</summary> +</histogram> + <histogram name="Sqlite.SizeKB" units="Kb"> <summary>Size in kilobytes of pre-existing database at startup.</summary> </histogram> @@ -24399,6 +24407,33 @@ other types of suffix sets. <int value="17" label="RECOVERY_EVENT_FAILED_META_INIT"> Failed sql::MetaTable::Init(). </int> + <int value="18" label="RECOVERY_EVENT_FAILED_META_VERSION"> + Failed sql::Recovery::SetupMeta() or GetMetaVersionNumber(). + </int> + <int value="19" label="RECOVERY_EVENT_DEPRECATED"> + Recovery found deprecated version and razed. + </int> + <int value="20" label="RECOVERY_EVENT_FAILED_V5_INITSCHEMA"> + Failed v5 recovery loading schema. + </int> + <int value="21" label="RECOVERY_EVENT_FAILED_V5_AUTORECOVER_FAVICONS"> + Failed v5 recovery on favicons. + </int> + <int value="22" label="RECOVERY_EVENT_FAILED_V5_AUTORECOVER_ICON_MAPPING"> + Failed v5 recovery on icon_mapping. + </int> + <int value="23" label="RECOVERY_EVENT_RECOVERED_VERSION5"> + Successful recovery of version 6 database. + </int> + <int value="24" label="RECOVERY_EVENT_FAILED_AUTORECOVER_FAVICONS"> + Failed v6/7 recovery on favicons. + </int> + <int value="25" label="RECOVERY_EVENT_FAILED_AUTORECOVER_FAVICON_BITMAPS"> + Failed v6/7 recovery on favicon_bitmaps. + </int> + <int value="26" label="RECOVERY_EVENT_FAILED_AUTORECOVER_ICON_MAPPING"> + Failed v6/7 recovery on icon_mapping. + </int> </enum> <enum name="HttpAuthCount" type="int"> |