diff options
-rw-r--r-- | chrome/browser/history/thumbnail_database.cc | 111 | ||||
-rw-r--r-- | chrome/browser/history/thumbnail_database_unittest.cc | 172 | ||||
-rw-r--r-- | sql/test/test_helpers.cc | 6 | ||||
-rw-r--r-- | tools/metrics/histograms/histograms.xml | 6 |
4 files changed, 189 insertions, 106 deletions
diff --git a/chrome/browser/history/thumbnail_database.cc b/chrome/browser/history/thumbnail_database.cc index 05a72a4..19788d9 100644 --- a/chrome/browser/history/thumbnail_database.cc +++ b/chrome/browser/history/thumbnail_database.cc @@ -67,6 +67,27 @@ namespace { +// For this database, schema migrations are deprecated after two +// years. This means that the oldest non-deprecated version should be +// two years old or greater (thus the migrations to get there are +// older). Databases containing deprecated versions will be cleared +// at startup. Since this database is a cache, losing old data is not +// fatal (in fact, very old data may be expired immediately at startup +// anyhow). + +// Version 7: 911a634d/r209424 by qsr@chromium.org on 2013-07-01 +// Version 6: 610f923b/r152367 by pkotwicz@chromium.org on 2012-08-20 +// Version 5: e2ee8ae9/r105004 by groby@chromium.org on 2011-10-12 +// Version 4: 5f104d76/r77288 by sky@chromium.org on 2011-03-08 (deprecated) +// Version 3: 09911bf3/r15 by initial.commit on 2008-07-26 (deprecated) + +// Version number of the database. +// NOTE(shess): When changing the version, add a new golden file for +// the new version and a test to verify that Init() works with it. +const int kCurrentVersionNumber = 7; +const int kCompatibleVersionNumber = 7; +const int kDeprecatedVersionNumber = 4; // and earlier. + void FillIconMapping(const sql::Statement& statement, const GURL& page_url, history::IconMapping* icon_mapping) { @@ -323,11 +344,11 @@ enum RecoveryEventType { RECOVERY_EVENT_FAILED_SCOPER, RECOVERY_EVENT_FAILED_META_VERSION_ERROR, RECOVERY_EVENT_FAILED_META_VERSION_NONE, - RECOVERY_EVENT_FAILED_META_WRONG_VERSION6, + RECOVERY_EVENT_FAILED_META_WRONG_VERSION6, // obsolete RECOVERY_EVENT_FAILED_META_WRONG_VERSION5, RECOVERY_EVENT_FAILED_META_WRONG_VERSION, RECOVERY_EVENT_FAILED_RECOVER_META, - RECOVERY_EVENT_FAILED_META_INSERT, + RECOVERY_EVENT_FAILED_META_INSERT, // obsolete RECOVERY_EVENT_FAILED_INIT, RECOVERY_EVENT_FAILED_RECOVER_FAVICONS, RECOVERY_EVENT_FAILED_FAVICONS_INSERT, @@ -335,6 +356,8 @@ enum RecoveryEventType { RECOVERY_EVENT_FAILED_FAVICON_BITMAPS_INSERT, RECOVERY_EVENT_FAILED_RECOVER_ICON_MAPPING, RECOVERY_EVENT_FAILED_ICON_MAPPING_INSERT, + RECOVERY_EVENT_RECOVERED_VERSION6, + RECOVERY_EVENT_FAILED_META_INIT, // Always keep this at the end. RECOVERY_EVENT_MAX, @@ -352,6 +375,11 @@ void RecordRecoveryEvent(RecoveryEventType recovery_event) { // opposed to just razing it and starting over whenever corruption is // detected. So this database is a good test subject. void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { + // NOTE(shess): This code is currently specific to the version + // number. I am working on simplifying things to loosen the + // dependency, meanwhile contact me if you need to bump the version. + DCHECK_EQ(7, kCurrentVersionNumber); + // TODO(shess): Reset back after? db->reset_error_callback(); @@ -373,6 +401,7 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { // Setup the meta recovery table, and check that the version number // is covered by the recovery code. // TODO(shess): sql::Recovery should provide a helper to handle meta. + int version = 0; // For reporting which version was recovered. { const char kRecoverySql[] = "CREATE VIRTUAL TABLE temp.recover_meta USING recover" @@ -411,20 +440,18 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { recovery_version.Clear(); sql::Recovery::Rollback(recovery.Pass()); return; - } else if (7 != recovery_version.ColumnInt(0)) { - // TODO(shess): Recovery code is generally schema-dependent. - // Version 6 should be easy, if the numbers warrant it. - // Version 5 is probably not warranted. - switch (recovery_version.ColumnInt(0)) { - case 6 : - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_WRONG_VERSION6); - break; - case 5 : - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_WRONG_VERSION5); - break; - default : - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_WRONG_VERSION); - break; + } + version = recovery_version.ColumnInt(0); + + // Recovery code is generally schema-dependent. Version 7 and + // version 6 are very similar, so can be handled together. + // Track version 5, to see whether it's worth writing recovery + // code for. + if (version != 7 && version != 6) { + if (version == 5) { + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_WRONG_VERSION5); + } else { + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_WRONG_VERSION); } recovery_version.Clear(); sql::Recovery::Rollback(recovery.Pass()); @@ -432,13 +459,12 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { } } - const char kCopySql[] = - "INSERT OR REPLACE INTO meta SELECT key, value FROM recover_meta"; - if (!recovery->db()->Execute(kCopySql)) { - // TODO(shess): Earlier the version was queried, unclear what - // this failure could mean. + // Either version 6 or version 7 recovers to current. + sql::MetaTable recover_meta_table; + if (!recover_meta_table.Init(recovery->db(), kCurrentVersionNumber, + kCompatibleVersionNumber)) { sql::Recovery::Rollback(recovery.Pass()); - RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_INSERT); + RecordRecoveryEvent(RECOVERY_EVENT_FAILED_META_INIT); return; } } @@ -462,13 +488,19 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { // Setup favicons table. { + // Version 6 had the |sizes| column, version 7 removed it. The + // recover virtual table treats more columns than expected as an + // error, but if _fewer_ columns are present, they can be treated + // as NULL. SQLite requires this because ALTER TABLE adds columns + // to the schema, but not to the actual table storage. const char kRecoverySql[] = "CREATE VIRTUAL TABLE temp.recover_favicons USING recover" "(" "corrupt.favicons," "id ROWID," "url TEXT NOT NULL," - "icon_type INTEGER" + "icon_type INTEGER," + "sizes TEXT" ")"; if (!recovery->db()->Execute(kRecoverySql)) { // TODO(shess): Failure to create the recovery table probably @@ -482,7 +514,7 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { // COALESCE(). Either way, the new code has a literal 1 rather // than a NULL, right? const char kCopySql[] = - "INSERT OR REPLACE INTO favicons " + "INSERT OR REPLACE INTO main.favicons " "SELECT id, url, COALESCE(icon_type, 1) FROM recover_favicons"; if (!recovery->db()->Execute(kCopySql)) { // TODO(shess): The recover_favicons table should mask problems @@ -516,7 +548,7 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { } const char kCopySql[] = - "INSERT OR REPLACE INTO favicon_bitmaps " + "INSERT OR REPLACE INTO main.favicon_bitmaps " "SELECT id, icon_id, COALESCE(last_updated, 0), image_data, " " COALESCE(width, 0), COALESCE(height, 0) " "FROM recover_favicons_bitmaps"; @@ -549,7 +581,7 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { } const char kCopySql[] = - "INSERT OR REPLACE INTO icon_mapping " + "INSERT OR REPLACE INTO main.icon_mapping " "SELECT id, page_url, icon_id FROM recover_icon_mapping"; if (!recovery->db()->Execute(kCopySql)) { // TODO(shess): The recover_icon_mapping table should mask @@ -572,7 +604,11 @@ void RecoverDatabaseOrRaze(sql::Connection* db, const base::FilePath& db_path) { // collection. ignore_result(sql::Recovery::Recovered(recovery.Pass())); - RecordRecoveryEvent(RECOVERY_EVENT_RECOVERED); + if (version == 6) { + RecordRecoveryEvent(RECOVERY_EVENT_RECOVERED_VERSION6); + } else { + RecordRecoveryEvent(RECOVERY_EVENT_RECOVERED); + } } void DatabaseErrorCallback(sql::Connection* db, @@ -615,27 +651,6 @@ void DatabaseErrorCallback(sql::Connection* db, namespace history { -// For this database, schema migrations are deprecated after two -// years. This means that the oldest non-deprecated version should be -// two years old or greater (thus the migrations to get there are -// older). Databases containing deprecated versions will be cleared -// at startup. Since this database is a cache, losing old data is not -// fatal (in fact, very old data may be expired immediately at startup -// anyhow). - -// Version 7: 911a634d/r209424 by qsr@chromium.org on 2013-07-01 -// Version 6: 610f923b/r152367 by pkotwicz@chromium.org on 2012-08-20 -// Version 5: e2ee8ae9/r105004 by groby@chromium.org on 2011-10-12 -// Version 4: 5f104d76/r77288 by sky@chromium.org on 2011-03-08 (deprecated) -// Version 3: 09911bf3/r15 by initial.commit on 2008-07-26 (deprecated) - -// Version number of the database. -// NOTE(shess): When changing the version, add a new golden file for -// the new version and a test to verify that Init() works with it. -static const int kCurrentVersionNumber = 7; -static const int kCompatibleVersionNumber = 7; -static const int kDeprecatedVersionNumber = 4; // and earlier. - ThumbnailDatabase::IconMappingEnumerator::IconMappingEnumerator() { } diff --git a/chrome/browser/history/thumbnail_database_unittest.cc b/chrome/browser/history/thumbnail_database_unittest.cc index ed269173..0d82b41 100644 --- a/chrome/browser/history/thumbnail_database_unittest.cc +++ b/chrome/browser/history/thumbnail_database_unittest.cc @@ -50,6 +50,17 @@ const GURL kIconUrl3 = GURL("http://www.google.com/touch.ico"); const gfx::Size kSmallSize = gfx::Size(16, 16); const gfx::Size kLargeSize = gfx::Size(32, 32); +// Create the test database at |db_path| from the golden file at +// |ascii_path| in the "History/" subdir of the test data dir. +WARN_UNUSED_RESULT bool CreateDatabaseFromSQL(const base::FilePath &db_path, + const char* ascii_path) { + base::FilePath sql_path; + if (!PathService::Get(chrome::DIR_TEST_DATA, &sql_path)) + return false; + sql_path = sql_path.AppendASCII("History").AppendASCII(ascii_path); + return sql::test::CreateDatabaseFromSQL(db_path, sql_path); +} + int GetPageSize(sql::Connection* db) { sql::Statement s(db->GetUniqueStatement("PRAGMA page_size")); EXPECT_TRUE(s.Step()); @@ -202,14 +213,7 @@ class ThumbnailDatabaseTest : public testing::Test { // Initialize a thumbnail database instance from the SQL file at // |golden_path| in the "History/" subdirectory of test data. scoped_ptr<ThumbnailDatabase> LoadFromGolden(const char* golden_path) { - base::FilePath sql_path; - if (!PathService::Get(chrome::DIR_TEST_DATA, &sql_path)) { - ADD_FAILURE() << "Failed loading " << golden_path; - return scoped_ptr<ThumbnailDatabase>(); - } - - sql_path = sql_path.AppendASCII("History").AppendASCII(golden_path); - if (!sql::test::CreateDatabaseFromSQL(file_name_, sql_path)) { + if (!CreateDatabaseFromSQL(file_name_, golden_path)) { ADD_FAILURE() << "Failed loading " << golden_path; return scoped_ptr<ThumbnailDatabase>(); } @@ -748,34 +752,13 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { if (!sql::Recovery::FullRecoverySupported()) return; - chrome::FaviconID id1, id2; - GURL page_url1("http://www.google.com"); - GURL page_url2("http://news.google.com"); - GURL favicon_url("http://www.google.com/favicon.png"); - // Create an example database. - // TODO(shess): Merge with the load-dump code when that lands. { - ThumbnailDatabase db; - ASSERT_EQ(sql::INIT_OK, db.Init(file_name_)); - db.BeginTransaction(); - - std::vector<unsigned char> data(kBlob1, kBlob1 + sizeof(kBlob1)); - scoped_refptr<base::RefCountedBytes> favicon( - new base::RefCountedBytes(data)); - - id1 = db.AddFavicon(favicon_url, chrome::TOUCH_ICON); - base::Time time = base::Time::Now(); - db.AddFaviconBitmap(id1, favicon, time, kSmallSize); - db.AddFaviconBitmap(id1, favicon, time, kLargeSize); - EXPECT_LT(0, db.AddIconMapping(page_url1, id1)); - EXPECT_LT(0, db.AddIconMapping(page_url2, id1)); - - id2 = db.AddFavicon(favicon_url, chrome::FAVICON); - EXPECT_NE(id1, id2); - db.AddFaviconBitmap(id2, favicon, time, kSmallSize); - EXPECT_LT(0, db.AddIconMapping(page_url1, id2)); - db.CommitTransaction(); + EXPECT_TRUE(CreateDatabaseFromSQL(file_name_, "Favicons.v7.sql")); + + sql::Connection raw_db; + EXPECT_TRUE(raw_db.Open(file_name_)); + VerifyTablesAndColumns(&raw_db); } // Test that the contents make sense after clean open. @@ -783,13 +766,16 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { ThumbnailDatabase db; ASSERT_EQ(sql::INIT_OK, db.Init(file_name_)); - std::vector<IconMapping> icon_mappings; - EXPECT_TRUE(db.GetIconMappingsForPageURL(page_url1, &icon_mappings)); - ASSERT_EQ(2u, icon_mappings.size()); - EXPECT_EQ(id1, icon_mappings[0].icon_id); - EXPECT_EQ(id2, icon_mappings[1].icon_id); + EXPECT_TRUE( + CheckPageHasIcon(&db, kPageUrl1, chrome::FAVICON, + kIconUrl1, kLargeSize, sizeof(kBlob1), kBlob1)); + EXPECT_TRUE( + CheckPageHasIcon(&db, kPageUrl2, chrome::FAVICON, + kIconUrl2, kLargeSize, sizeof(kBlob2), kBlob2)); } + // Corrupt the |icon_mapping.page_url| index by deleting an element + // from the backing table but not the index. { sql::Connection raw_db; EXPECT_TRUE(raw_db.Open(file_name_)); @@ -809,7 +795,7 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { { const char kDeleteSql[] = "DELETE FROM icon_mapping WHERE page_url = ?"; sql::Statement statement(raw_db.GetUniqueStatement(kDeleteSql)); - statement.BindString(0, URLDatabase::GURLToDatabaseURL(page_url2)); + statement.BindString(0, URLDatabase::GURLToDatabaseURL(kPageUrl2)); EXPECT_TRUE(statement.Run()); } raw_db.Close(); @@ -817,7 +803,7 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { EXPECT_TRUE(WritePage(file_name_, idx_root_page, buf.get(), page_size)); } - // Database should be corrupt. + // Database should be corrupt at the SQLite level. { sql::Connection raw_db; EXPECT_TRUE(raw_db.Open(file_name_)); @@ -834,16 +820,16 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { ThumbnailDatabase db; ASSERT_EQ(sql::INIT_OK, db.Init(file_name_)); - // Data for page_url2 was deleted, but the index entry remains, + // Data for kPageUrl2 was deleted, but the index entry remains, // this will throw SQLITE_CORRUPT. The corruption handler will // recover the database and poison the handle, so the outer call // fails. - std::vector<IconMapping> icon_mappings; - EXPECT_FALSE(db.GetIconMappingsForPageURL(page_url2, &icon_mappings)); + EXPECT_FALSE(db.GetIconMappingsForPageURL(kPageUrl2, NULL)); + ASSERT_TRUE(ignore_errors.CheckIgnoredErrors()); } - // Check that the database is recovered at a SQLite level. + // Check that the database is recovered at the SQLite level. { sql::Connection raw_db; EXPECT_TRUE(raw_db.Open(file_name_)); @@ -851,6 +837,9 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { raw_db.GetUniqueStatement("PRAGMA integrity_check")); EXPECT_TRUE(statement.Step()); EXPECT_EQ("ok", statement.ColumnString(0)); + + // Check that the expected tables exist. + VerifyTablesAndColumns(&raw_db); } // Database should also be recovered at higher levels. @@ -858,14 +847,13 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { ThumbnailDatabase db; ASSERT_EQ(sql::INIT_OK, db.Init(file_name_)); - std::vector<IconMapping> icon_mappings; - - EXPECT_FALSE(db.GetIconMappingsForPageURL(page_url2, &icon_mappings)); + // Now this fails because there is no mapping. + EXPECT_FALSE(db.GetIconMappingsForPageURL(kPageUrl2, NULL)); - EXPECT_TRUE(db.GetIconMappingsForPageURL(page_url1, &icon_mappings)); - ASSERT_EQ(2u, icon_mappings.size()); - EXPECT_EQ(id1, icon_mappings[0].icon_id); - EXPECT_EQ(id2, icon_mappings[1].icon_id); + // Other data was retained by recovery. + EXPECT_TRUE( + CheckPageHasIcon(&db, kPageUrl1, chrome::FAVICON, + kIconUrl1, kLargeSize, sizeof(kBlob1), kBlob1)); } // Corrupt the database again by making the actual file shorter than @@ -901,16 +889,84 @@ TEST_F(ThumbnailDatabaseTest, Recovery) { ThumbnailDatabase db; ASSERT_EQ(sql::INIT_OK, db.Init(file_name_)); - std::vector<IconMapping> icon_mappings; + EXPECT_FALSE(db.GetIconMappingsForPageURL(kPageUrl2, NULL)); + EXPECT_TRUE( + CheckPageHasIcon(&db, kPageUrl1, chrome::FAVICON, + kIconUrl1, kLargeSize, sizeof(kBlob1), kBlob1)); - EXPECT_FALSE(db.GetIconMappingsForPageURL(page_url2, &icon_mappings)); + ASSERT_TRUE(ignore_errors.CheckIgnoredErrors()); + } +} + +TEST_F(ThumbnailDatabaseTest, Recovery6) { + // TODO(shess): See comment at top of Recovery test. + if (!sql::Recovery::FullRecoverySupported()) + return; + + // Create an example database without loading into ThumbnailDatabase + // (which would upgrade it). + EXPECT_TRUE(CreateDatabaseFromSQL(file_name_, "Favicons.v6.sql")); + + // Corrupt the database by making the actual file shorter than the + // SQLite header expects. This form of corruption will cause + // immediate failures during Open(), before the migration code runs, + // so the version-6 recovery will occur. + { + int64 db_size = 0; + EXPECT_TRUE(file_util::GetFileSize(file_name_, &db_size)); + { + sql::Connection raw_db; + EXPECT_TRUE(raw_db.Open(file_name_)); + EXPECT_TRUE(raw_db.Execute("CREATE TABLE t(x)")); + } + file_util::ScopedFILE file(file_util::OpenFile(file_name_, "rb+")); + ASSERT_TRUE(file.get() != NULL); + EXPECT_EQ(0, fseek(file.get(), static_cast<long>(db_size), SEEK_SET)); + EXPECT_TRUE(file_util::TruncateFile(file.get())); + } + + // Database is unusable at the SQLite level. + { + sql::ScopedErrorIgnorer ignore_errors; + ignore_errors.IgnoreError(SQLITE_CORRUPT); + sql::Connection raw_db; + EXPECT_TRUE(raw_db.Open(file_name_)); + EXPECT_FALSE(raw_db.IsSQLValid("PRAGMA integrity_check")); + ASSERT_TRUE(ignore_errors.CheckIgnoredErrors()); + } + + // Database should be recovered during open. + { + sql::ScopedErrorIgnorer ignore_errors; + ignore_errors.IgnoreError(SQLITE_CORRUPT); + ThumbnailDatabase db; + ASSERT_EQ(sql::INIT_OK, db.Init(file_name_)); + + // Test that some data is present, copied from + // ThumbnailDatabaseTest.Version6 . + EXPECT_TRUE( + CheckPageHasIcon(&db, kPageUrl3, chrome::FAVICON, + kIconUrl1, kLargeSize, sizeof(kBlob1), kBlob1)); + EXPECT_TRUE( + CheckPageHasIcon(&db, kPageUrl3, chrome::TOUCH_ICON, + kIconUrl3, kLargeSize, sizeof(kBlob2), kBlob2)); - EXPECT_TRUE(db.GetIconMappingsForPageURL(page_url1, &icon_mappings)); - ASSERT_EQ(2u, icon_mappings.size()); - EXPECT_EQ(id1, icon_mappings[0].icon_id); - EXPECT_EQ(id2, icon_mappings[1].icon_id); ASSERT_TRUE(ignore_errors.CheckIgnoredErrors()); } + + // Check that the database is recovered at a SQLite level, and that + // the current schema is in place. + { + sql::Connection raw_db; + EXPECT_TRUE(raw_db.Open(file_name_)); + sql::Statement statement( + raw_db.GetUniqueStatement("PRAGMA integrity_check")); + EXPECT_TRUE(statement.Step()); + EXPECT_EQ("ok", statement.ColumnString(0)); + + // Check that the expected tables exist. + VerifyTablesAndColumns(&raw_db); + } } } // namespace history diff --git a/sql/test/test_helpers.cc b/sql/test/test_helpers.cc index 20471dc..de3e8f8 100644 --- a/sql/test/test_helpers.cc +++ b/sql/test/test_helpers.cc @@ -82,6 +82,12 @@ bool CreateDatabaseFromSQL(const base::FilePath& db_path, if (!db.Open(db_path)) return false; + // TODO(shess): Android defaults to auto_vacuum mode. + // Unfortunately, this makes certain kinds of tests which manipulate + // the raw database hard/impossible to write. + // http://crbug.com/307303 is for exploring this test issue. + ignore_result(db.Execute("PRAGMA auto_vacuum = 0")); + return db.Execute(sql.c_str()); } diff --git a/tools/metrics/histograms/histograms.xml b/tools/metrics/histograms/histograms.xml index 5dfbe96..a930f2d 100644 --- a/tools/metrics/histograms/histograms.xml +++ b/tools/metrics/histograms/histograms.xml @@ -22610,6 +22610,12 @@ other types of suffix sets. <int value="15" label="RECOVERY_EVENT_FAILED_ICON_MAPPING_INSERT"> Failed to copy recovery icon_mapping table. </int> + <int value="16" label="RECOVERY_EVENT_RECOVERED_VERSION6"> + Successful recovery of version 6 database. + </int> + <int value="17" label="RECOVERY_EVENT_FAILED_META_INIT"> + Failed sql::MetaTable::Init(). + </int> </enum> <enum name="HttpAuthCount" type="int"> |