diff options
author | mrossetti@chromium.org <mrossetti@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-08-14 02:45:14 +0000 |
---|---|---|
committer | mrossetti@chromium.org <mrossetti@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-08-14 02:45:14 +0000 |
commit | c93c7efd04977522cefbe0d0de4f20e5fa777bd5 (patch) | |
tree | 06e187b2770e0e6204ef3739aa28906c0acd8e51 | |
parent | bfe7c195f6363d41377c9a9bbd8d3fbd881aad76 (diff) | |
download | chromium_src-c93c7efd04977522cefbe0d0de4f20e5fa777bd5.zip chromium_src-c93c7efd04977522cefbe0d0de4f20e5fa777bd5.tar.gz chromium_src-c93c7efd04977522cefbe0d0de4f20e5fa777bd5.tar.bz2 |
Replace HistoryQuickProvider protobuf-based caching with an SQLite-based database.
The protobuf-based cache was only being read at startup and written at shutdown. (Except that when the cache could not be read at startup the cache would immediately be saved upon private data reconstruction from the History database.) With the new implementation using an SQLite database for the HQP cache the private data will be restored from the cache database at startup and continually updated during normal operation. There is no need to flush or write the cache at shutdown as it is constantly kept up-to-date.
Detailed comments about the changes made in this CL can be found in the document shared separately.
Reviewer suggestions:
sky: for proper thread usage. Also chrome/browser/ui/ and chrome/browser/visitedlink/.
pkasting: for whatever you'd like to check over but please look over the design document shared separately.
shess: please take a look at database usage in chrome/browser/history/in_memory_url_cache_database.h/.cc.
BUG=95686,95876,131668
TEST=New tests added, old tests updated, all tests pass.
To manually verify changes:
NOTE: For tests using chrome://omnibox be sure to check the "Show results per provider" and then look at the results for the HistoryQuickProvider.
1) New visits: Type an URL never visited before. Bring up new tab. Start typing the URL or parts of the page title or both and verify that the page is offered as a suggestion.
2) New visits: Type an URL never visited before. Do search using chrome://omnibox. New visit should show.
3) Delete visit: Visit some pages and verify they have been recorded. Bring up history and delete one of the visits. Check via chrome://omnibox that it was deleted.
4) Clear history: Visit some pages. Clear the visit history. Check via chrome://omnibox that none of the visits are still returned by the HQP.
5) Updated site: Create a local page and visit it. Search using chrome://omnibox by page title and verify it can be found. Change the page title dramatically and revisit the page. Verify via chrome://oomnibox that the page can be found by words from the new title.
Review URL: https://chromiumcodereview.appspot.com/10477018
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@151419 0039d316-1c4b-4281-b951-d872f2087c98
42 files changed, 3728 insertions, 2200 deletions
diff --git a/chrome/browser/autocomplete/history_provider.cc b/chrome/browser/autocomplete/history_provider.cc index ed8437f..3fd5d97 100644 --- a/chrome/browser/autocomplete/history_provider.cc +++ b/chrome/browser/autocomplete/history_provider.cc @@ -38,8 +38,10 @@ void HistoryProvider::DeleteMatch(const AutocompleteMatch& match) { DCHECK(history_service); DCHECK(match.destination_url.is_valid()); history_service->DeleteURL(match.destination_url); + DeleteMatchFromMatches(match); +} - // Delete the match from the current set of matches. +void HistoryProvider::DeleteMatchFromMatches(const AutocompleteMatch& match) { bool found = false; for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) { if (i->destination_url == match.destination_url && i->type == match.type) { diff --git a/chrome/browser/autocomplete/history_provider.h b/chrome/browser/autocomplete/history_provider.h index 96de5b6..24f0f02 100644 --- a/chrome/browser/autocomplete/history_provider.h +++ b/chrome/browser/autocomplete/history_provider.h @@ -50,6 +50,10 @@ class HistoryProvider : public AutocompleteProvider { // trailing whitespace, or if always_prevent_inline_autocomplete is true. bool PreventInlineAutocomplete(const AutocompleteInput& input); + // Finds and removes the match from the current collection of matches and + // backing data. + void DeleteMatchFromMatches(const AutocompleteMatch& match); + // If true, we always prevent inline autocompletions. bool always_prevent_inline_autocomplete_; }; diff --git a/chrome/browser/autocomplete/history_quick_provider.cc b/chrome/browser/autocomplete/history_quick_provider.cc index 3bb2c32..e098f67 100644 --- a/chrome/browser/autocomplete/history_quick_provider.cc +++ b/chrome/browser/autocomplete/history_quick_provider.cc @@ -147,8 +147,13 @@ void HistoryQuickProvider::Start(const AutocompleteInput& input, } } -// TODO(mrossetti): Implement this function. (Will happen in next CL.) -void HistoryQuickProvider::DeleteMatch(const AutocompleteMatch& match) {} +void HistoryQuickProvider::DeleteMatch(const AutocompleteMatch& match) { + DCHECK(match.deletable); + DCHECK(match.destination_url.is_valid()); + // Delete the match from the InMemoryURLIndex. + GetIndex()->DeleteURL(match.destination_url); + DeleteMatchFromMatches(match); +} HistoryQuickProvider::~HistoryQuickProvider() {} diff --git a/chrome/browser/autocomplete/history_quick_provider_unittest.cc b/chrome/browser/autocomplete/history_quick_provider_unittest.cc index ff225b3..9d09c5e 100644 --- a/chrome/browser/autocomplete/history_quick_provider_unittest.cc +++ b/chrome/browser/autocomplete/history_quick_provider_unittest.cc @@ -2,111 +2,29 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "chrome/browser/autocomplete/history_quick_provider.h" - -#include <algorithm> -#include <functional> -#include <set> -#include <string> -#include <vector> - -#include "base/memory/scoped_ptr.h" -#include "base/message_loop.h" #include "base/utf_string_conversions.h" -#include "chrome/browser/autocomplete/autocomplete_provider_listener.h" #include "chrome/browser/autocomplete/autocomplete_result.h" -#include "chrome/browser/history/history.h" -#include "chrome/browser/history/history_service_factory.h" -#include "chrome/browser/history/in_memory_url_index.h" -#include "chrome/browser/history/url_database.h" -#include "chrome/browser/history/url_index_private_data.h" -#include "chrome/browser/prefs/pref_service.h" -#include "chrome/common/pref_names.h" -#include "chrome/test/base/testing_browser_process.h" +#include "chrome/browser/autocomplete/history_quick_provider.h" +#include "chrome/browser/history/in_memory_url_cache_database.h" +#include "chrome/browser/history/in_memory_url_index_unittest_base.h" #include "chrome/test/base/testing_profile.h" -#include "content/public/test/test_browser_thread.h" -#include "testing/gtest/include/gtest/gtest.h" - -using base::Time; -using base::TimeDelta; - -using content::BrowserThread; - -struct TestURLInfo { - std::string url; - std::string title; - int visit_count; - int typed_count; - int days_from_now; -} quick_test_db[] = { - {"http://www.google.com/", "Google", 3, 3, 0}, - {"http://slashdot.org/favorite_page.html", "Favorite page", 200, 100, 0}, - {"http://kerneltrap.org/not_very_popular.html", "Less popular", 4, 0, 0}, - {"http://freshmeat.net/unpopular.html", "Unpopular", 1, 1, 0}, - {"http://news.google.com/?ned=us&topic=n", "Google News - U.S.", 2, 2, 0}, - {"http://news.google.com/", "Google News", 1, 1, 0}, - {"http://foo.com/", "Dir", 200, 100, 0}, - {"http://foo.com/dir/", "Dir", 2, 1, 10}, - {"http://foo.com/dir/another/", "Dir", 5, 10, 0}, - {"http://foo.com/dir/another/again/", "Dir", 5, 1, 0}, - {"http://foo.com/dir/another/again/myfile.html", "File", 3, 2, 0}, - {"http://visitedest.com/y/a", "VA", 10, 1, 20}, - {"http://visitedest.com/y/b", "VB", 9, 1, 20}, - {"http://visitedest.com/x/c", "VC", 8, 1, 20}, - {"http://visitedest.com/x/d", "VD", 7, 1, 20}, - {"http://visitedest.com/y/e", "VE", 6, 1, 20}, - {"http://typeredest.com/y/a", "TA", 3, 5, 0}, - {"http://typeredest.com/y/b", "TB", 3, 4, 0}, - {"http://typeredest.com/x/c", "TC", 3, 3, 0}, - {"http://typeredest.com/x/d", "TD", 3, 2, 0}, - {"http://typeredest.com/y/e", "TE", 3, 1, 0}, - {"http://daysagoest.com/y/a", "DA", 1, 1, 0}, - {"http://daysagoest.com/y/b", "DB", 1, 1, 1}, - {"http://daysagoest.com/x/c", "DC", 1, 1, 2}, - {"http://daysagoest.com/x/d", "DD", 1, 1, 3}, - {"http://daysagoest.com/y/e", "DE", 1, 1, 4}, - {"http://abcdefghixyzjklmnopqrstuvw.com/a", "", 3, 1, 0}, - {"http://spaces.com/path%20with%20spaces/foo.html", "Spaces", 2, 2, 0}, - {"http://abcdefghijklxyzmnopqrstuvw.com/a", "", 3, 1, 0}, - {"http://abcdefxyzghijklmnopqrstuvw.com/a", "", 3, 1, 0}, - {"http://abcxyzdefghijklmnopqrstuvw.com/a", "", 3, 1, 0}, - {"http://xyzabcdefghijklmnopqrstuvw.com/a", "", 3, 1, 0}, - {"http://cda.com/Dogs%20Cats%20Gorillas%20Sea%20Slugs%20and%20Mice", - "Dogs & Cats & Mice & Other Animals", 1, 1, 0}, - {"https://monkeytrap.org/", "", 3, 1, 0}, -}; - -class HistoryQuickProviderTest : public testing::Test, - public AutocompleteProviderListener { - public: - HistoryQuickProviderTest() - : ui_thread_(BrowserThread::UI, &message_loop_), - file_thread_(BrowserThread::FILE, &message_loop_) {} - - // AutocompleteProviderListener: - virtual void OnProviderUpdate(bool updated_matches) OVERRIDE; +class HistoryQuickProviderTest : public history::InMemoryURLIndexTestBase { protected: + virtual FilePath::StringType TestDBName() const OVERRIDE; + class SetShouldContain : public std::unary_function<const std::string&, std::set<std::string> > { public: explicit SetShouldContain(const ACMatches& matched_urls); - void operator()(const std::string& expected); - std::set<std::string> LeftOvers() const { return matches_; } private: std::set<std::string> matches_; }; - void SetUp(); - void TearDown(); - - virtual void GetTestData(size_t* data_count, TestURLInfo** test_data); - - // Fills test data into the history system. - void FillData(); + virtual void SetUp() OVERRIDE; // Runs an autocomplete query on |text| and checks to see that the returned // results' destination URLs match those provided. |expected_urls| does not @@ -116,16 +34,6 @@ class HistoryQuickProviderTest : public testing::Test, bool can_inline_top_result, string16 expected_fill_into_edit); - // Pass-through functions to simplify our friendship with URLIndexPrivateData. - bool UpdateURL(const history::URLRow& row); - - MessageLoopForUI message_loop_; - content::TestBrowserThread ui_thread_; - content::TestBrowserThread file_thread_; - - scoped_ptr<TestingProfile> profile_; - HistoryService* history_service_; - ACMatches ac_matches_; // The resulting matches after running RunTest. private: @@ -133,63 +41,14 @@ class HistoryQuickProviderTest : public testing::Test, }; void HistoryQuickProviderTest::SetUp() { - profile_.reset(new TestingProfile()); - profile_->CreateHistoryService(true, false); - profile_->CreateBookmarkModel(true); - profile_->BlockUntilBookmarkModelLoaded(); - history_service_ = - HistoryServiceFactory::GetForProfile(profile_.get(), - Profile::EXPLICIT_ACCESS); - EXPECT_TRUE(history_service_); - provider_ = new HistoryQuickProvider(this, profile_.get()); - FillData(); -} - -void HistoryQuickProviderTest::TearDown() { - provider_ = NULL; + InMemoryURLIndexTestBase::SetUp(); + LoadIndex(); + DCHECK(url_index_->index_available()); + provider_ = new HistoryQuickProvider(NULL, profile_.get()); } -bool HistoryQuickProviderTest::UpdateURL(const history::URLRow& row) { - history::InMemoryURLIndex* index = provider_->GetIndex(); - DCHECK(index); - history::URLIndexPrivateData* private_data = index->private_data(); - DCHECK(private_data); - return private_data->UpdateURL(row, index->languages_, - index->scheme_whitelist_); -} - -void HistoryQuickProviderTest::OnProviderUpdate(bool updated_matches) { - MessageLoop::current()->Quit(); -} - -void HistoryQuickProviderTest::GetTestData(size_t* data_count, - TestURLInfo** test_data) { - DCHECK(data_count); - DCHECK(test_data); - *data_count = arraysize(quick_test_db); - *test_data = &quick_test_db[0]; -} - -void HistoryQuickProviderTest::FillData() { - history::URLDatabase* db = history_service_->InMemoryDatabase(); - ASSERT_TRUE(db != NULL); - size_t data_count = 0; - TestURLInfo* test_data = NULL; - GetTestData(&data_count, &test_data); - for (size_t i = 0; i < data_count; ++i) { - const TestURLInfo& cur(test_data[i]); - const GURL current_url(cur.url); - Time visit_time = Time::Now() - TimeDelta::FromDays(cur.days_from_now); - - history::URLRow url_info(current_url); - url_info.set_id(i + 5000); - url_info.set_title(UTF8ToUTF16(cur.title)); - url_info.set_visit_count(cur.visit_count); - url_info.set_typed_count(cur.typed_count); - url_info.set_last_visit(visit_time); - url_info.set_hidden(false); - UpdateURL(url_info); - } +FilePath::StringType HistoryQuickProviderTest::TestDBName() const { + return FILE_PATH_LITERAL("history_quick_provider_test.db.txt"); } HistoryQuickProviderTest::SetShouldContain::SetShouldContain( @@ -432,52 +291,13 @@ TEST_F(HistoryQuickProviderTest, Spans) { // HQPOrderingTest ------------------------------------------------------------- -TestURLInfo ordering_test_db[] = { - {"http://www.teamliquid.net/tlpd/korean/games/21648_bisu_vs_iris", "", 6, 3, - 256}, - {"http://www.amazon.com/", "amazon.com: online shopping for electronics, " - "apparel, computers, books, dvds & more", 20, 20, 10}, - {"http://www.teamliquid.net/forum/viewmessage.php?topic_id=52045&" - "currentpage=83", "google images", 6, 6, 0}, - {"http://www.tempurpedic.com/", "tempur-pedic", 7, 7, 0}, - {"http://www.teamfortress.com/", "", 5, 5, 6}, - {"http://www.rottentomatoes.com/", "", 3, 3, 7}, - {"http://music.google.com/music/listen?u=0#start_pl", "", 3, 3, 9}, - {"https://www.emigrantdirect.com/", "high interest savings account, high " - "yield savings - emigrantdirect", 5, 5, 3}, - {"http://store.steampowered.com/", "", 6, 6, 1}, - {"http://techmeme.com/", "techmeme", 111, 110, 4}, - {"http://www.teamliquid.net/tlpd", "team liquid progaming database", 15, 15, - 2}, - {"http://store.steampowered.com/", "the steam summer camp sale", 6, 6, 1}, - {"http://www.teamliquid.net/tlpd/korean/players", "tlpd - bw korean - player " - "index", 100, 45, 219}, - {"http://slashdot.org/", "slashdot: news for nerds, stuff that matters", 3, 3, - 6}, - {"http://translate.google.com/", "google translate", 3, 3, 0}, - {"http://arstechnica.com/", "ars technica", 3, 3, 3}, - {"http://www.rottentomatoes.com/", "movies | movie trailers | reviews - " - "rotten tomatoes", 3, 3, 7}, - {"http://www.teamliquid.net/", "team liquid - starcraft 2 and brood war pro " - "gaming news", 26, 25, 3}, - {"http://metaleater.com/", "metaleater", 4, 3, 8}, - {"http://half.com/", "half.com: textbooks , books , music , movies , games , " - "video games", 4, 4, 6}, - {"http://teamliquid.net/", "team liquid - starcraft 2 and brood war pro " - "gaming news", 8, 5, 9}, -}; - class HQPOrderingTest : public HistoryQuickProviderTest { protected: - virtual void GetTestData(size_t* data_count, - TestURLInfo** test_data) OVERRIDE; + virtual FilePath::StringType TestDBName() const OVERRIDE; }; -void HQPOrderingTest::GetTestData(size_t* data_count, TestURLInfo** test_data) { - DCHECK(data_count); - DCHECK(test_data); - *data_count = arraysize(ordering_test_db); - *test_data = &ordering_test_db[0]; +FilePath::StringType HQPOrderingTest::TestDBName() const { + return FILE_PATH_LITERAL("history_quick_provider_ordering_test.db.txt"); } TEST_F(HQPOrderingTest, TEMatch) { diff --git a/chrome/browser/diagnostics/diagnostics_model.cc b/chrome/browser/diagnostics/diagnostics_model.cc index 8182514..b40ba77 100644 --- a/chrome/browser/diagnostics/diagnostics_model.cc +++ b/chrome/browser/diagnostics/diagnostics_model.cc @@ -99,6 +99,7 @@ class DiagnosticsModelWin : public DiagnosticsModelImpl { tests_.push_back(MakeSqliteHistoryDbTest()); tests_.push_back(MakeSqliteArchivedHistoryDbTest()); tests_.push_back(MakeSqliteThumbnailsDbTest()); + tests_.push_back(MakeSqliteHQPCacheDbTest()); tests_.push_back(MakeSqliteAppCacheDbTest()); tests_.push_back(MakeSqliteWebDatabaseTrackerDbTest()); } @@ -125,6 +126,7 @@ class DiagnosticsModelMac : public DiagnosticsModelImpl { tests_.push_back(MakeSqliteHistoryDbTest()); tests_.push_back(MakeSqliteArchivedHistoryDbTest()); tests_.push_back(MakeSqliteThumbnailsDbTest()); + tests_.push_back(MakeSqliteHistoryDbTest()); tests_.push_back(MakeSqliteAppCacheDbTest()); tests_.push_back(MakeSqliteWebDatabaseTrackerDbTest()); } @@ -152,6 +154,7 @@ class DiagnosticsModelPosix : public DiagnosticsModelImpl { tests_.push_back(MakeSqliteHistoryDbTest()); tests_.push_back(MakeSqliteArchivedHistoryDbTest()); tests_.push_back(MakeSqliteThumbnailsDbTest()); + tests_.push_back(MakeSqliteHistoryDbTest()); tests_.push_back(MakeSqliteAppCacheDbTest()); tests_.push_back(MakeSqliteWebDatabaseTrackerDbTest()); } diff --git a/chrome/browser/diagnostics/diagnostics_model_unittest.cc b/chrome/browser/diagnostics/diagnostics_model_unittest.cc index 823ef9d..29b15c0 100644 --- a/chrome/browser/diagnostics/diagnostics_model_unittest.cc +++ b/chrome/browser/diagnostics/diagnostics_model_unittest.cc @@ -78,11 +78,11 @@ class UTObserver: public DiagnosticsModel::Observer { // We currently have more tests operational on windows. #if defined(OS_WIN) -const int kDiagnosticsTestCount = 19; +const int kDiagnosticsTestCount = 20; #elif defined(OS_MACOSX) -const int kDiagnosticsTestCount = 16; -#elif defined(OS_POSIX) const int kDiagnosticsTestCount = 17; +#elif defined(OS_POSIX) +const int kDiagnosticsTestCount = 18; #endif // Test that the initial state is correct. diff --git a/chrome/browser/diagnostics/sqlite_diagnostics.cc b/chrome/browser/diagnostics/sqlite_diagnostics.cc index 3fba29e..2a2bf3c 100644 --- a/chrome/browser/diagnostics/sqlite_diagnostics.cc +++ b/chrome/browser/diagnostics/sqlite_diagnostics.cc @@ -99,7 +99,8 @@ class HistogramUniquifier { "Sqlite.History.Error", "Sqlite.Thumbnail.Error", "Sqlite.Text.Error", - "Sqlite.Web.Error" + "Sqlite.Web.Error", + "Sqlite.HQPCache.Error" }; return kHistogramNames[unique]; } @@ -127,6 +128,10 @@ sql::ErrorDelegate* GetErrorHandlerForWebDb() { return new sql::DiagnosticErrorDelegate<HistogramUniquifier<4> >(); } +sql::ErrorDelegate* GetErrorHandlerForHQPCacheDb() { + return new sql::DiagnosticErrorDelegate<HistogramUniquifier<5> >(); +} + DiagnosticTest* MakeSqliteWebDbTest() { return new SqliteIntegrityTest(true, ASCIIToUTF16("Web DB"), FilePath(chrome::kWebDataFilename)); @@ -152,6 +157,12 @@ DiagnosticTest* MakeSqliteThumbnailsDbTest() { FilePath(chrome::kThumbnailsFilename)); } +DiagnosticTest* MakeSqliteHQPCacheDbTest() { + return new SqliteIntegrityTest(false, + ASCIIToUTF16("History Provider Cache DB"), + FilePath(chrome::kHQPCacheDBFilename)); +} + DiagnosticTest* MakeSqliteAppCacheDbTest() { FilePath appcache_dir(content::kAppCacheDirname); FilePath appcache_db = appcache_dir.Append(appcache::kAppCacheDatabaseName); diff --git a/chrome/browser/diagnostics/sqlite_diagnostics.h b/chrome/browser/diagnostics/sqlite_diagnostics.h index c416e88..f66d4cf1 100644 --- a/chrome/browser/diagnostics/sqlite_diagnostics.h +++ b/chrome/browser/diagnostics/sqlite_diagnostics.h @@ -18,6 +18,7 @@ sql::ErrorDelegate* GetErrorHandlerForHistoryDb(); sql::ErrorDelegate* GetErrorHandlerForThumbnailDb(); sql::ErrorDelegate* GetErrorHandlerForTextDb(); sql::ErrorDelegate* GetErrorHandlerForWebDb(); +sql::ErrorDelegate* GetErrorHandlerForHQPCacheDb(); // Factories for the db integrity tests we run in diagnostic mode. DiagnosticTest* MakeSqliteWebDbTest(); @@ -25,6 +26,7 @@ DiagnosticTest* MakeSqliteCookiesDbTest(); DiagnosticTest* MakeSqliteHistoryDbTest(); DiagnosticTest* MakeSqliteArchivedHistoryDbTest(); DiagnosticTest* MakeSqliteThumbnailsDbTest(); +DiagnosticTest* MakeSqliteHQPCacheDbTest(); DiagnosticTest* MakeSqliteAppCacheDbTest(); DiagnosticTest* MakeSqliteWebDatabaseTrackerDbTest(); diff --git a/chrome/browser/history/history.cc b/chrome/browser/history/history.cc index 648de29..6d01c79 100644 --- a/chrome/browser/history/history.cc +++ b/chrome/browser/history/history.cc @@ -201,7 +201,7 @@ void HistoryService::UnloadBackend() { // Give the InMemoryURLIndex a chance to shutdown. if (in_memory_url_index_.get()) - in_memory_url_index_->ShutDown(); + in_memory_url_index_->Shutdown(); // The backend's destructor must run on the history thread since it is not // threadsafe. So this thread must not be the last thread holding a reference @@ -737,7 +737,8 @@ void HistoryService::Observe(int type, bool HistoryService::Init(const FilePath& history_dir, BookmarkService* bookmark_service, - bool no_db) { + bool no_db, + bool disable_index_cache) { if (!thread_->Start()) { Cleanup(); return false; @@ -756,7 +757,7 @@ bool HistoryService::Init(const FilePath& history_dir, profile_->GetPrefs()->GetString(prefs::kAcceptLanguages); in_memory_url_index_.reset( new history::InMemoryURLIndex(profile_, history_dir_, languages)); - in_memory_url_index_->Init(); + in_memory_url_index_->Init(disable_index_cache); } #endif // !OS_ANDROID diff --git a/chrome/browser/history/history.h b/chrome/browser/history/history.h index 69fda8b..1389057 100644 --- a/chrome/browser/history/history.h +++ b/chrome/browser/history/history.h @@ -111,7 +111,7 @@ class HistoryService : public CancelableRequestProvider, // the history files. The BookmarkService is used when deleting URLs to // test if a URL is bookmarked; it may be NULL during testing. bool Init(const FilePath& history_dir, BookmarkService* bookmark_service) { - return Init(history_dir, bookmark_service, false); + return Init(history_dir, bookmark_service, false, false); } // Triggers the backend to load if it hasn't already, and then returns whether @@ -254,7 +254,7 @@ class HistoryService : public CancelableRequestProvider, public: // Indicates that a URL is available. There will be exactly one call for // every URL in history. - virtual void OnURL(const GURL& url) = 0; + virtual void OnURL(const history::URLRow& url_row) = 0; // Indicates we are done iterating over URLs. Once called, there will be no // more callbacks made. This call is guaranteed to occur, even if there are @@ -389,8 +389,11 @@ class HistoryService : public CancelableRequestProvider, const QueryMostVisitedURLsCallback& callback); // Request the |result_count| URLs filtered and sorted based on the |filter|. - // If |extended_info| is enabled, additional data will be provided in the - // results. + // If |extended_info| is true, additional data will be provided in the + // results. Computing this additional data is expensive, likely to become + // more expensive as additional data points are added in future changes, and + // not useful in most cases. Set |extended_info| to true only if you + // explicitly require the additional data. Handle QueryFilteredURLs( int result_count, const history::VisitFilter& filter, @@ -618,6 +621,11 @@ class HistoryService : public CancelableRequestProvider, // history. We filter out some URLs such as JavaScript. static bool CanAddURL(const GURL& url); + // Returns the history backend associated with this service. + history::HistoryBackend* get_history_backend_for_testing() { + return history_backend_.get(); + } + protected: virtual ~HistoryService(); @@ -636,13 +644,14 @@ class HistoryService : public CancelableRequestProvider, #endif friend class base::RefCountedThreadSafe<HistoryService>; friend class BackendDelegate; + friend class CacheTestingProfile; friend class FaviconService; friend class history::HistoryBackend; friend class history::HistoryQueryTest; friend class HistoryOperation; friend class HistoryURLProvider; friend class HistoryURLProviderTest; - friend class history::InMemoryURLIndexTest; + friend class InMemoryURLIndexTestBase; template<typename Info, typename Callback> friend class DownloadRequest; friend class PageUsageRequest; friend class RedirectRequest; @@ -653,11 +662,14 @@ class HistoryService : public CancelableRequestProvider, const content::NotificationSource& source, const content::NotificationDetails& details) OVERRIDE; - // Low-level Init(). Same as the public version, but adds a |no_db| parameter - // that is only set by unittests which causes the backend to not init its DB. + // Low-level Init(). Same as the public version, but adds the |no_db| and + // |disable_index_cache| parameters that are only set by unittests. |no_db| + // causes the backend to not init its DB. |disable_index_cache| causes the + // InMemoryURLIndex to not create or use its cache database. bool Init(const FilePath& history_dir, BookmarkService* bookmark_service, - bool no_db); + bool no_db, + bool disable_index_cache); // Called by the HistoryURLProvider class to schedule an autocomplete, it // will be called back on the internal history thread with the history @@ -926,6 +938,7 @@ class HistoryService : public CancelableRequestProvider, // A cache of the user-typed URLs kept in memory that is used by the // autocomplete system. This will be NULL until the database has been created // on the background thread. + // TODO(mrossetti): Consider changing ownership. See http://crbug.com/138321 scoped_ptr<history::InMemoryHistoryBackend> in_memory_backend_; // The profile, may be null when testing. @@ -948,6 +961,8 @@ class HistoryService : public CancelableRequestProvider, bool needs_top_sites_migration_; // The index used for quick history lookups. + // TODO(mrossetti): Move in_memory_url_index out of history_service. + // See http://crbug.com/138321 scoped_ptr<history::InMemoryURLIndex> in_memory_url_index_; scoped_refptr<ObserverListThreadSafe<history::VisitDatabaseObserver> > diff --git a/chrome/browser/history/history_backend.cc b/chrome/browser/history/history_backend.cc index c932dd9..849de1a 100644 --- a/chrome/browser/history/history_backend.cc +++ b/chrome/browser/history/history_backend.cc @@ -993,7 +993,7 @@ void HistoryBackend::IterateURLs(HistoryService::URLEnumerator* iterator) { if (db_->InitURLEnumeratorForEverything(&e)) { URLRow info; while (e.GetNextURL(&info)) { - iterator->OnURL(info.url()); + iterator->OnURL(info); } iterator->OnComplete(true); // Success. return; diff --git a/chrome/browser/history/history_backend_unittest.cc b/chrome/browser/history/history_backend_unittest.cc index 20a59d2..82a1720 100644 --- a/chrome/browser/history/history_backend_unittest.cc +++ b/chrome/browser/history/history_backend_unittest.cc @@ -55,10 +55,10 @@ class HistoryBackendTest; // This just forwards the messages we're interested in to the test object. class HistoryBackendTestDelegate : public HistoryBackend::Delegate { public: - explicit HistoryBackendTestDelegate(HistoryBackendTest* test) : test_(test) {} + explicit HistoryBackendTestDelegate(HistoryBackendTest* test); virtual void NotifyProfileError(int backend_id, - sql::InitStatus init_status) OVERRIDE {} + sql::InitStatus init_status) OVERRIDE; virtual void SetInMemoryBackend(int backend_id, InMemoryHistoryBackend* backend) OVERRIDE; virtual void BroadcastNotifications(int type, @@ -66,7 +66,7 @@ class HistoryBackendTestDelegate : public HistoryBackend::Delegate { virtual void DBLoaded(int backend_id) OVERRIDE; virtual void StartTopSitesMigration(int backend_id) OVERRIDE; virtual void NotifyVisitDBObserversOnAddVisit( - const BriefVisitInfo& info) OVERRIDE {} + const BriefVisitInfo& info) OVERRIDE; private: // Not owned by us. @@ -78,21 +78,21 @@ class HistoryBackendTestDelegate : public HistoryBackend::Delegate { class HistoryBackendCancelableRequest : public CancelableRequestProvider, public CancelableRequestConsumerBase { public: - HistoryBackendCancelableRequest() {} + HistoryBackendCancelableRequest(); // CancelableRequestConsumerBase overrides: virtual void OnRequestAdded( CancelableRequestProvider* provider, - CancelableRequestProvider::Handle handle) OVERRIDE {} + CancelableRequestProvider::Handle handle) OVERRIDE; virtual void OnRequestRemoved( CancelableRequestProvider* provider, - CancelableRequestProvider::Handle handle) OVERRIDE {} + CancelableRequestProvider::Handle handle) OVERRIDE; virtual void WillExecute( CancelableRequestProvider* provider, - CancelableRequestProvider::Handle handle) OVERRIDE {} + CancelableRequestProvider::Handle handle) OVERRIDE; virtual void DidExecute( CancelableRequestProvider* provider, - CancelableRequestProvider::Handle handle) OVERRIDE {} + CancelableRequestProvider::Handle handle) OVERRIDE; template<class RequestType> CancelableRequestProvider::Handle MockScheduleOfRequest( @@ -104,21 +104,16 @@ class HistoryBackendCancelableRequest : public CancelableRequestProvider, class HistoryBackendTest : public testing::Test { public: - HistoryBackendTest() : bookmark_model_(NULL), loaded_(false) {} - virtual ~HistoryBackendTest() { - } + HistoryBackendTest(); + virtual ~HistoryBackendTest(); // Callback for QueryMostVisited. void OnQueryMostVisited(CancelableRequestProvider::Handle handle, - history::MostVisitedURLList data) { - most_visited_list_.swap(data); - } + history::MostVisitedURLList data); // Callback for QueryFiltered. void OnQueryFiltered(CancelableRequestProvider::Handle handle, - const history::FilteredURLList& data) { - filtered_list_ = data; - } + const history::FilteredURLList& data); const history::MostVisitedURLList& get_most_visited_list() const { return most_visited_list_; @@ -129,34 +124,12 @@ class HistoryBackendTest : public testing::Test { } protected: - scoped_refptr<HistoryBackend> backend_; // Will be NULL on init failure. - scoped_ptr<InMemoryHistoryBackend> mem_backend_; + void AddRedirectChain(const char* sequence[], int page_id); - void AddRedirectChain(const char* sequence[], int page_id) { - AddRedirectChainWithTransitionAndTime(sequence, page_id, - content::PAGE_TRANSITION_LINK, - Time::Now()); - } - - void AddRedirectChainWithTransitionAndTime( - const char* sequence[], - int page_id, - content::PageTransition transition, - base::Time time) { - history::RedirectList redirects; - for (int i = 0; sequence[i] != NULL; ++i) - redirects.push_back(GURL(sequence[i])); - - int int_scope = 1; - void* scope = 0; - memcpy(&scope, &int_scope, sizeof(int_scope)); - scoped_refptr<history::HistoryAddPageArgs> request( - new history::HistoryAddPageArgs( - redirects.back(), time, scope, page_id, GURL(), - redirects, transition, history::SOURCE_BROWSED, - true)); - backend_->AddPage(request); - } + void AddRedirectChainWithTransitionAndTime(const char* sequence[], + int page_id, + content::PageTransition transition, + base::Time time); // Adds CLIENT_REDIRECT page transition. // |url1| is the source URL and |url2| is the destination. @@ -165,88 +138,34 @@ class HistoryBackendTest : public testing::Test { // updated transition code of the visit records for |url1| and |url2| is // returned by filling in |*transition1| and |*transition2|, respectively. // |time| is a time of the redirect. - void AddClientRedirect(const GURL& url1, const GURL& url2, bool did_replace, + void AddClientRedirect(const GURL& url1, + const GURL& url2, + bool did_replace, base::Time time, - int* transition1, int* transition2) { - void* const dummy_scope = reinterpret_cast<void*>(0x87654321); - history::RedirectList redirects; - if (url1.is_valid()) - redirects.push_back(url1); - if (url2.is_valid()) - redirects.push_back(url2); - scoped_refptr<HistoryAddPageArgs> request( - new HistoryAddPageArgs(url2, time, dummy_scope, 0, url1, - redirects, content::PAGE_TRANSITION_CLIENT_REDIRECT, - history::SOURCE_BROWSED, did_replace)); - backend_->AddPage(request); - - *transition1 = getTransition(url1); - *transition2 = getTransition(url2); - } + int* transition1, + int* transition2); - int getTransition(const GURL& url) { - if (!url.is_valid()) - return 0; - URLRow row; - URLID id = backend_->db()->GetRowForURL(url, &row); - VisitVector visits; - EXPECT_TRUE(backend_->db()->GetVisitsForURL(id, &visits)); - return visits[0].transition; - } + int GetTransition(const GURL& url); - FilePath getTestDir() { - return test_dir_; - } + FilePath get_test_dir() { return test_dir_; } - FaviconID GetFavicon(const GURL& url, IconType icon_type) { - IconMapping icon_mapping; - if (backend_->thumbnail_db_->GetIconMappingForPageURL(url, icon_type, - &icon_mapping)) - return icon_mapping.icon_id; - else - return 0; - } + FaviconID GetFavicon(const GURL& url, IconType icon_type); + scoped_refptr<HistoryBackend> backend_; // Will be NULL on init failure. + scoped_ptr<InMemoryHistoryBackend> mem_backend_; BookmarkModel bookmark_model_; - - protected: bool loaded_; private: friend class HistoryBackendTestDelegate; // testing::Test - virtual void SetUp() { - if (!file_util::CreateNewTempDirectory(FILE_PATH_LITERAL("BackendTest"), - &test_dir_)) - return; - backend_ = new HistoryBackend(test_dir_, - 0, - new HistoryBackendTestDelegate(this), - &bookmark_model_); - backend_->Init(std::string(), false); - } - virtual void TearDown() { - if (backend_.get()) - backend_->Closing(); - backend_ = NULL; - mem_backend_.reset(); - file_util::Delete(test_dir_, true); - } + virtual void SetUp(); + virtual void TearDown(); - void SetInMemoryBackend(int backend_id, InMemoryHistoryBackend* backend) { - mem_backend_.reset(backend); - } - - void BroadcastNotifications(int type, - HistoryDetails* details) { - // Send the notifications directly to the in-memory database. - content::Details<HistoryDetails> det(details); - mem_backend_->Observe(type, content::Source<HistoryBackendTest>(NULL), det); + void SetInMemoryBackend(int backend_id, InMemoryHistoryBackend* backend); - // The backend passes ownership of the details pointer to us. - delete details; - } + void BroadcastNotifications(int type, HistoryDetails* details); MessageLoop message_loop_; FilePath test_dir_; @@ -254,6 +173,20 @@ class HistoryBackendTest : public testing::Test { history::FilteredURLList filtered_list_; }; +// HistoryBackendTestDelegate -------------------------------------------------- + +HistoryBackendTestDelegate::HistoryBackendTestDelegate(HistoryBackendTest* test) + : test_(test) { +} + +void HistoryBackendTestDelegate::NotifyProfileError(int backend_id, + sql::InitStatus init_status) { +} + +void HistoryBackendTestDelegate::NotifyVisitDBObserversOnAddVisit( + const BriefVisitInfo& info) { +} + void HistoryBackendTestDelegate::SetInMemoryBackend(int backend_id, InMemoryHistoryBackend* backend) { test_->SetInMemoryBackend(backend_id, backend); @@ -273,6 +206,141 @@ void HistoryBackendTestDelegate::StartTopSitesMigration(int backend_id) { test_->backend_->MigrateThumbnailsDatabase(); } +// HistoryBackendCancelableRequest --------------------------------------------- + +HistoryBackendCancelableRequest::HistoryBackendCancelableRequest() {} + +void HistoryBackendCancelableRequest::OnRequestAdded( + CancelableRequestProvider* provider, + CancelableRequestProvider::Handle handle) {} +void HistoryBackendCancelableRequest::OnRequestRemoved( + CancelableRequestProvider* provider, + CancelableRequestProvider::Handle handle) {} +void HistoryBackendCancelableRequest::WillExecute( + CancelableRequestProvider* provider, + CancelableRequestProvider::Handle handle) {} +void HistoryBackendCancelableRequest::DidExecute( + CancelableRequestProvider* provider, + CancelableRequestProvider::Handle handle) {} + +// HistoryBackendTest ---------------------------------------------------------- + +HistoryBackendTest::HistoryBackendTest() + : bookmark_model_(NULL), + loaded_(false) { +} + +HistoryBackendTest::~HistoryBackendTest() {} + +void HistoryBackendTest::OnQueryMostVisited( + CancelableRequestProvider::Handle handle, + history::MostVisitedURLList data) { + most_visited_list_.swap(data); +} + +void HistoryBackendTest::OnQueryFiltered( + CancelableRequestProvider::Handle handle, + const history::FilteredURLList& data) { + filtered_list_ = data; +} + +void HistoryBackendTest::AddRedirectChain(const char* sequence[], int page_id) { + AddRedirectChainWithTransitionAndTime(sequence, page_id, + content::PAGE_TRANSITION_LINK, + Time::Now()); +} + +void HistoryBackendTest::AddRedirectChainWithTransitionAndTime( + const char* sequence[], + int page_id, + content::PageTransition transition, + base::Time time) { + history::RedirectList redirects; + for (int i = 0; sequence[i] != NULL; ++i) + redirects.push_back(GURL(sequence[i])); + + int int_scope = 1; + void* scope = 0; + memcpy(&scope, &int_scope, sizeof(int_scope)); + scoped_refptr<history::HistoryAddPageArgs> request( + new history::HistoryAddPageArgs(redirects.back(), time, scope, page_id, + GURL(), redirects, transition, + history::SOURCE_BROWSED, true)); + backend_->AddPage(request); +} + +void HistoryBackendTest::AddClientRedirect(const GURL& url1, + const GURL& url2, + bool did_replace, + base::Time time, + int* transition1, + int* transition2) { + void* const dummy_scope = reinterpret_cast<void*>(0x87654321); + history::RedirectList redirects; + if (url1.is_valid()) + redirects.push_back(url1); + if (url2.is_valid()) + redirects.push_back(url2); + scoped_refptr<HistoryAddPageArgs> request( + new HistoryAddPageArgs(url2, time, dummy_scope, 0, url1, + redirects, content::PAGE_TRANSITION_CLIENT_REDIRECT, + history::SOURCE_BROWSED, did_replace)); + backend_->AddPage(request); + + *transition1 = GetTransition(url1); + *transition2 = GetTransition(url2); +} + +int HistoryBackendTest::GetTransition(const GURL& url) { + if (!url.is_valid()) + return 0; + URLRow row; + URLID id = backend_->db()->GetRowForURL(url, &row); + VisitVector visits; + EXPECT_TRUE(backend_->db()->GetVisitsForURL(id, &visits)); + return visits[0].transition; +} + +FaviconID HistoryBackendTest::GetFavicon(const GURL& url, IconType icon_type) { + IconMapping icon_mapping; + return backend_->thumbnail_db_->GetIconMappingForPageURL(url, icon_type, + &icon_mapping) ? icon_mapping.icon_id : 0; +} + +void HistoryBackendTest::SetUp() { + if (!file_util::CreateNewTempDirectory(FILE_PATH_LITERAL("BackendTest"), + &test_dir_)) + return; + backend_ = new HistoryBackend(test_dir_, + 0, + new HistoryBackendTestDelegate(this), + &bookmark_model_); + backend_->Init(std::string(), false); +} + +void HistoryBackendTest::TearDown() { + if (backend_.get()) + backend_->Closing(); + backend_ = NULL; + mem_backend_.reset(); + file_util::Delete(test_dir_, true); +} + +void HistoryBackendTest::SetInMemoryBackend(int backend_id, + InMemoryHistoryBackend* backend) { + mem_backend_.reset(backend); +} + +void HistoryBackendTest::BroadcastNotifications(int type, + HistoryDetails* details) { + // Send the notifications directly to the in-memory database. + content::Details<HistoryDetails> det(details); + mem_backend_->Observe(type, content::Source<HistoryBackendTest>(NULL), det); + + // The backend passes ownership of the details pointer to us. + delete details; +} + // http://crbug.com/114287 #if defined(OS_WIN) #define MAYBE_Loaded DISABLED_Loaded @@ -1026,7 +1094,7 @@ TEST_F(HistoryBackendTest, MigrationVisitSource) { // Copy history database file to current directory so that it will be deleted // in Teardown. - FilePath new_history_path(getTestDir()); + FilePath new_history_path(get_test_dir()); file_util::Delete(new_history_path, true); file_util::CreateDirectory(new_history_path); FilePath new_history_file = new_history_path.Append(chrome::kHistoryFilename); @@ -1499,7 +1567,7 @@ TEST_F(HistoryBackendTest, MigrationVisitDuration) { // Copy history database file to current directory so that it will be deleted // in Teardown. - FilePath new_history_path(getTestDir()); + FilePath new_history_path(get_test_dir()); file_util::Delete(new_history_path, true); file_util::CreateDirectory(new_history_path); FilePath new_history_file = new_history_path.Append(chrome::kHistoryFilename); diff --git a/chrome/browser/history/history_browsertest.cc b/chrome/browser/history/history_browsertest.cc index 0118d6e..acd8dee 100644 --- a/chrome/browser/history/history_browsertest.cc +++ b/chrome/browser/history/history_browsertest.cc @@ -70,8 +70,8 @@ class HistoryEnumerator : public HistoryService::URLEnumerator { content::RunMessageLoop(); } - virtual void OnURL(const GURL& url) { - urls_.push_back(url); + virtual void OnURL(const history::URLRow& url_row) { + urls_.push_back(url_row.url()); } virtual void OnComplete(bool success) { diff --git a/chrome/browser/history/history_database.h b/chrome/browser/history/history_database.h index 52b71f9..2a07206 100644 --- a/chrome/browser/history/history_database.h +++ b/chrome/browser/history/history_database.h @@ -149,13 +149,16 @@ class HistoryDatabase : public DownloadDatabase, virtual base::Time GetEarlyExpirationThreshold(); virtual void UpdateEarlyExpirationThreshold(base::Time threshold); + // Returns the database connection so that unit tests can directly access it. + sql::Connection* get_db_for_testing() { return &db_; } + private: #if defined(OS_ANDROID) // AndroidProviderBackend uses the |db_|. friend class AndroidProviderBackend; FRIEND_TEST_ALL_PREFIXES(AndroidURLsMigrationTest, MigrateToVersion22); #endif - friend class InMemoryURLIndexTest; + friend class InMemoryURLIndexTestBase; FRIEND_TEST_ALL_PREFIXES(IconMappingMigrationTest, TestIconMappingMigration); // Overridden from URLDatabase: diff --git a/chrome/browser/history/in_memory_url_cache_database.cc b/chrome/browser/history/in_memory_url_cache_database.cc new file mode 100644 index 0000000..e9da2da --- /dev/null +++ b/chrome/browser/history/in_memory_url_cache_database.cc @@ -0,0 +1,692 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/history/in_memory_url_cache_database.h" + +#include <algorithm> +#include <functional> +#include <iterator> + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/file_path.h" +#include "base/string_util.h" +#include "base/stringprintf.h" +#include "base/time.h" +#include "chrome/browser/diagnostics/sqlite_diagnostics.h" +#include "chrome/browser/history/url_database.h" +#include "chrome/browser/history/url_index_private_data.h" +#include "chrome/common/chrome_notification_types.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/notification_service.h" +#include "sql/diagnostic_error_delegate.h" +#include "sql/statement.h" +#include "sql/transaction.h" +#include "third_party/sqlite/sqlite3.h" + +using content::BrowserThread; + +namespace history { + +namespace { + +static const int kCurrentVersionNumber = 1; +static const int kCompatibleVersionNumber = 1; + +const char* kWordsTableName = "words"; +const char* kCharWordsTableName = "char_words"; +const char* kWordHistoryTableName = "word_history"; +const char* kURLsTableName = "urls"; +const char* kURLWordStartsTableName = "url_word_starts"; +const char* kTitleWordStartsTableName = "title_word_starts"; + +} // namespace + +// Public Methods -------------------------------------------------------------- + +InMemoryURLCacheDatabase::InMemoryURLCacheDatabase() + : update_error_(SQLITE_OK), + shutdown_(false) { +} + +InMemoryURLCacheDatabase::~InMemoryURLCacheDatabase() {} + +bool InMemoryURLCacheDatabase::Init( + const FilePath& file_path, + const base::SequencedWorkerPool::SequenceToken& sequence_token) { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + if (shutdown_) + return false; + sequence_token_ = sequence_token; + db_.set_error_delegate(GetErrorHandlerForHQPCacheDb()); + // Don't use very much memory caching this database. We primarily use it + // as a backing store for existing in-memory data. + db_.set_cache_size(64); + db_.set_exclusive_locking(); + if (!db_.Open(file_path)) { + UMA_HISTOGRAM_ENUMERATION("Sqlite.HQPCacheOpen.Error", db_.GetErrorCode(), + sql::kMaxSqliteError); + return false; + } + if (!InitDatabase()) { + db_.Close(); + return false; + } + return true; +} + +void InMemoryURLCacheDatabase::Shutdown() { + shutdown_ = true; + // Allow all outstanding sequenced database operations to complete. + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::ShutdownTask, this)); +} + +bool InMemoryURLCacheDatabase::RestorePrivateData(URLIndexPrivateData* data) { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + return RestoreWords(data) && RestoreCharWords(data) && + RestoreWordHistory(data) && RestoreURLs(data) && RestoreWordStarts(data); +} + +bool InMemoryURLCacheDatabase::Reset() { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + if (db_.Raze() && CreateTables()) + return true; + UMA_HISTOGRAM_ENUMERATION("Sqlite.HQPCacheCreate.Error", db_.GetErrorCode(), + sql::kMaxSqliteError); + return false; +} + +// Database Updating ----------------------------------------------------------- + +void InMemoryURLCacheDatabase::AddHistoryToURLs(history::HistoryID history_id, + const history::URLRow& row) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::AddHistoryToURLsTask, this, + history_id, row)); +} + +void InMemoryURLCacheDatabase::AddHistoryToWordHistory(WordID word_id, + HistoryID history_id) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::AddHistoryToWordHistoryTask, this, + word_id, history_id)); +} + +void InMemoryURLCacheDatabase::AddWordToWords(WordID word_id, + const string16& uni_word) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::AddWordToWordsTask, this, word_id, + uni_word)); +} + +void InMemoryURLCacheDatabase::AddWordToCharWords(char16 uni_char, + WordID word_id) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::AddWordToCharWordsTask, this, + uni_char, word_id)); +} + +void InMemoryURLCacheDatabase::AddRowWordStarts( + HistoryID history_id, + const RowWordStarts& row_word_starts) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::AddRowWordStartsTask, this, + history_id, row_word_starts)); +} + +void InMemoryURLCacheDatabase::RemoveHistoryIDFromURLs(HistoryID history_id) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::RemoveHistoryIDFromURLsTask, this, + history_id)); +} + +void InMemoryURLCacheDatabase::RemoveHistoryIDFromWordHistory( + HistoryID history_id) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::RemoveHistoryIDFromWordHistoryTask, + this, history_id)); +} + +void InMemoryURLCacheDatabase::RemoveWordFromWords(WordID word_id) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::RemoveWordFromWordsTask, this, + word_id)); +} + +void InMemoryURLCacheDatabase::RemoveWordStarts(HistoryID history_id) { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::RemoveWordStartsTask, this, + history_id)); +} + +void InMemoryURLCacheDatabase::BeginTransaction() { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::BeginTransactionTask, this)); +} + +void InMemoryURLCacheDatabase::CommitTransaction() { + PostSequencedDBTask(FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::CommitTransactionTask, this)); +} + +bool InMemoryURLCacheDatabase::Refresh(const URLIndexPrivateData& data) { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + return RefreshWords(data) && RefreshCharWords(data) && + RefreshWordHistory(data) && RefreshURLs(data) && RefreshWordStarts(data); +} + +// Private Methods ------------------------------------------------------------- + +bool InMemoryURLCacheDatabase::InitDatabase() { + db_.Preload(); // Prime the cache. + if (EnsureCurrentVersion() != sql::INIT_OK) + return false; + + // Create the tables if any do not already exist. + return VerifyTables() || Reset(); +} + +void InMemoryURLCacheDatabase::PostSequencedDBTask( + const tracked_objects::Location& from_here, + const base::Closure& task) { + BrowserThread::GetBlockingPool()->PostSequencedWorkerTask( + sequence_token_, from_here, task); +} + +void InMemoryURLCacheDatabase::ShutdownTask() { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + db_.Close(); +} + +sql::InitStatus InMemoryURLCacheDatabase::EnsureCurrentVersion() { + // We can't read databases newer than we were designed for. + if (!meta_table_.Init(&db_, kCurrentVersionNumber, kCompatibleVersionNumber)) + return sql::INIT_FAILURE; + if (meta_table_.GetCompatibleVersionNumber() > kCurrentVersionNumber) { + LOG(WARNING) << "In-memory URL Cache database is too new."; + return sql::INIT_TOO_NEW; + } + + // NOTE: Add migration code here as required. + + return sql::INIT_OK; +} + +void InMemoryURLCacheDatabase::BeginTransactionTask() { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + db_.BeginTransaction(); +} + +void InMemoryURLCacheDatabase::CommitTransactionTask() { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + if (update_error_ != SQLITE_OK) { + db_.RollbackTransaction(); + BrowserThread::PostTask( + BrowserThread::UI, FROM_HERE, + base::Bind(&InMemoryURLCacheDatabase::NotifyDatabaseFailure, + base::Unretained(this))); + return; + } + db_.CommitTransaction(); +} + +bool InMemoryURLCacheDatabase::VerifyTables() { + if (db_.DoesTableExist(kWordsTableName) && + db_.DoesTableExist(kCharWordsTableName) && + db_.DoesTableExist(kWordHistoryTableName) && + db_.DoesTableExist(kURLsTableName) && + db_.DoesTableExist(kURLWordStartsTableName) && + db_.DoesTableExist(kTitleWordStartsTableName)) + return true; + UMA_HISTOGRAM_ENUMERATION("Sqlite.HQPCacheVerify.Error", db_.GetErrorCode(), + sql::kMaxSqliteError); + return false; +} + +bool InMemoryURLCacheDatabase::CreateTables() { + std::string sql(StringPrintf( + "CREATE TABLE %s (word_id INTEGER PRIMARY KEY, word TEXT)", + kWordsTableName)); + if (!db_.Execute(sql.c_str())) + return false; + sql = StringPrintf("CREATE INDEX %s_index ON %s (word_id)", + kWordsTableName, kWordsTableName); + if (!db_.Execute(sql.c_str())) + return false; + + sql = StringPrintf("CREATE TABLE %s (char LONGCHAR, word_id INTEGER)", + kCharWordsTableName); + if (!db_.Execute(sql.c_str())) + return false; + sql = StringPrintf("CREATE INDEX %s_index ON %s (char)", + kCharWordsTableName, kCharWordsTableName); + if (!db_.Execute(sql.c_str())) + return false; + + sql = StringPrintf("CREATE TABLE %s (word_id INTEGER, history_id INTEGER)", + kWordHistoryTableName); + if (!db_.Execute(sql.c_str())) + return false; + sql = StringPrintf("CREATE INDEX %s_history_index ON %s (history_id)", + kWordHistoryTableName, kWordHistoryTableName); + if (!db_.Execute(sql.c_str())) + return false; + + sql = StringPrintf( + "CREATE TABLE %s (history_id INTEGER PRIMARY KEY, url TEXT, " + "title TEXT, visit_count INTEGER, typed_count INTEGER, " + "last_visit_time INTEGER, hidden INTEGER)", kURLsTableName); + if (!db_.Execute(sql.c_str())) + return false; + sql = StringPrintf("CREATE INDEX %s_index ON %s (history_id)", + kURLsTableName, kURLsTableName); + if (!db_.Execute(sql.c_str())) + return false; + + sql = StringPrintf("CREATE TABLE %s (history_id INTEGER, word_start INTEGER)", + kURLWordStartsTableName); + if (!db_.Execute(sql.c_str())) + return false; + sql = StringPrintf("CREATE INDEX %s_index ON %s (history_id)", + kURLWordStartsTableName, kURLWordStartsTableName); + if (!db_.Execute(sql.c_str())) + return false; + + sql = StringPrintf("CREATE TABLE %s (history_id INTEGER, word_start INTEGER)", + kTitleWordStartsTableName); + if (!db_.Execute(sql.c_str())) + return false; + sql = StringPrintf("CREATE INDEX %s_index ON %s (history_id)", + kTitleWordStartsTableName, kTitleWordStartsTableName); + return db_.Execute(sql.c_str()); +} + +void InMemoryURLCacheDatabase::NotifyDatabaseFailure() { + content::NotificationService::current()->Notify( + chrome::NOTIFICATION_IN_MEMORY_URL_CACHE_DATABASE_FAILURE, + content::Source<InMemoryURLCacheDatabase>(this), + content::NotificationService::NoDetails()); +} + +bool InMemoryURLCacheDatabase::RunStatement(sql::Statement* statement) { + if (statement->Run()) + return true; + // If a failure has already occurred don't spew more histograms and don't + // forget the very first failure code. + if (update_error_ != SQLITE_OK) + return false; + update_error_ = db_.GetErrorCode(); + UMA_HISTOGRAM_ENUMERATION("Sqlite.HQPCacheUpdate.Error", update_error_, + sql::kMaxSqliteError); + return false; +} + +// Database Additions ---------------------------------------------------------- + +void InMemoryURLCacheDatabase::AddHistoryToURLsTask( + history::HistoryID history_id, + const history::URLRow& row) { + std::string sql(StringPrintf( + "INSERT INTO %s (history_id, url, title, visit_count, typed_count, " + "last_visit_time, hidden) VALUES (?,?,?,?,?,?,?)", + kURLsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + statement.BindInt(0, history_id); + statement.BindString(1, URLDatabase::GURLToDatabaseURL(row.url())); + statement.BindString16(2, row.title()); + statement.BindInt(3, row.visit_count()); + statement.BindInt(4, row.typed_count()); + statement.BindInt64(5, row.last_visit().ToInternalValue()); + statement.BindBool(6, row.hidden()); + RunStatement(&statement); +} + +void InMemoryURLCacheDatabase::AddHistoryToWordHistoryTask( + WordID word_id, + HistoryID history_id) { + std::string sql(StringPrintf( + "INSERT INTO %s (word_id, history_id) VALUES (?,?)", + kWordHistoryTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + statement.BindInt(0, word_id); + statement.BindInt(1, history_id); + RunStatement(&statement); +} + +void InMemoryURLCacheDatabase::AddWordToWordsTask(WordID word_id, + const string16& uni_word) { + std::string sql(StringPrintf( + "INSERT INTO %s (word_id, word) VALUES (?,?)", + kWordsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + statement.BindInt(0, word_id); + statement.BindString16(1, uni_word); + RunStatement(&statement); +} + +void InMemoryURLCacheDatabase::AddWordToCharWordsTask(char16 uni_char, + WordID word_id) { + std::string sql(StringPrintf("INSERT INTO %s (char, word_id) VALUES (?,?)", + kCharWordsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + statement.BindInt(0, uni_char); + statement.BindInt(1, word_id); + RunStatement(&statement); +} + +void InMemoryURLCacheDatabase::AddRowWordStartsTask( + HistoryID history_id, + const RowWordStarts& row_word_starts) { + std::string sql_1(StringPrintf( + "INSERT INTO %s (history_id, word_start) VALUES (?,?)", + kURLWordStartsTableName)); + sql::Statement url_statement(db_.GetCachedStatement(SQL_FROM_HERE, + sql_1.c_str())); + for (WordStarts::const_iterator i = row_word_starts.url_word_starts_.begin(); + i != row_word_starts.url_word_starts_.end(); ++i) { + url_statement.Reset(true); + url_statement.BindInt(0, history_id); + url_statement.BindInt(1, *i); + RunStatement(&url_statement); + } + std::string sql_2(StringPrintf( + "INSERT INTO %s (history_id, word_start) VALUES (?,?)", + kTitleWordStartsTableName)); + sql::Statement title_statement(db_.GetCachedStatement(SQL_FROM_HERE, + sql_2.c_str())); + for (WordStarts::const_iterator i = + row_word_starts.title_word_starts_.begin(); + i != row_word_starts.title_word_starts_.end(); ++i) { + title_statement.Reset(true); + title_statement.BindInt(0, history_id); + title_statement.BindInt(1, *i); + RunStatement(&title_statement); + } +} + +// Database Removals ----------------------------------------------------------- + +void InMemoryURLCacheDatabase::RemoveHistoryIDFromURLsTask( + HistoryID history_id) { + std::string sql(StringPrintf("DELETE FROM %s WHERE history_id = ?", + kURLsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + statement.BindInt(0, history_id); + RunStatement(&statement); +} + +void InMemoryURLCacheDatabase::RemoveHistoryIDFromWordHistoryTask( + HistoryID history_id) { + std::string sql(StringPrintf("DELETE FROM %s WHERE history_id = ?", + kWordHistoryTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + statement.BindInt(0, history_id); + RunStatement(&statement); +} + +void InMemoryURLCacheDatabase::RemoveWordFromWordsTask(WordID word_id) { + std::string sql_1(StringPrintf("DELETE FROM %s WHERE word_id = ?", + kWordsTableName)); + sql::Statement statement_1(db_.GetCachedStatement(SQL_FROM_HERE, + sql_1.c_str())); + statement_1.BindInt(0, word_id); + RunStatement(&statement_1); + + std::string sql_2(StringPrintf("DELETE FROM %s WHERE word_id = ?", + kCharWordsTableName)); + sql::Statement statement_2(db_.GetCachedStatement(SQL_FROM_HERE, + sql_2.c_str())); + statement_2.BindInt(0, word_id); + RunStatement(&statement_2); +} + +void InMemoryURLCacheDatabase::RemoveWordStartsTask(HistoryID history_id) { + std::string url_sql(StringPrintf("DELETE FROM %s WHERE history_id = ?", + kURLWordStartsTableName)); + sql::Statement url_statement(db_.GetCachedStatement(SQL_FROM_HERE, + url_sql.c_str())); + url_statement.BindInt(0, history_id); + RunStatement(&url_statement); + + std::string title_sql(StringPrintf("DELETE FROM %s WHERE history_id = ?", + kTitleWordStartsTableName)); + sql::Statement title_statement(db_.GetCachedStatement(SQL_FROM_HERE, + title_sql.c_str())); + title_statement.BindInt(0, history_id); + RunStatement(&title_statement); +} + +// Database Refreshing --------------------------------------------------------- + +bool InMemoryURLCacheDatabase::RefreshWords(const URLIndexPrivateData& data) { + // Populate the words table from the contents of the word_map_. This will + // allow the restoration of the word_map_ as well as the word_list_ and + // available_words_. + std::string sql(StringPrintf("INSERT INTO %s (word_id, word) VALUES (?,?)", + kWordsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + for (WordMap::const_iterator i = data.word_map_.begin(); + i != data.word_map_.end(); ++i) { + statement.Reset(true); + statement.BindInt(0, i->second); + statement.BindString16(1, i->first); + if (!RunStatement(&statement)) + return false; + } + return true; +} + +bool InMemoryURLCacheDatabase::RefreshCharWords( + const URLIndexPrivateData& data) { + std::string sql(StringPrintf("INSERT INTO %s (char, word_id) VALUES (?,?)", + kCharWordsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + for (CharWordIDMap::const_iterator i = data.char_word_map_.begin(); + i != data.char_word_map_.end(); ++i) { + for (WordIDSet::const_iterator j = i->second.begin(); j != i->second.end(); + ++j) { + statement.Reset(true); + statement.BindInt(0, i->first); + statement.BindInt(1, *j); + if (!RunStatement(&statement)) + return false; + } + } + return true; +} + +bool InMemoryURLCacheDatabase::RefreshWordHistory( + const URLIndexPrivateData& data) { + // Populate the word_history table from the contents of the + // word_id_history_map_. This will allow the restoration of the + // word_id_history_map_ as well as the history_id_word_map_. + std::string sql(StringPrintf( + "INSERT INTO %s (word_id, history_id) VALUES (?,?)", + kWordHistoryTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + for (WordIDHistoryMap::const_iterator i = data.word_id_history_map_.begin(); + i != data.word_id_history_map_.end(); ++i) { + for (HistoryIDSet::const_iterator j = i->second.begin(); + j != i->second.end(); ++j) { + statement.Reset(true); + statement.BindInt(0, i->first); + statement.BindInt64(1, *j); + if (!RunStatement(&statement)) + return false; + } + } + return true; +} + +bool InMemoryURLCacheDatabase::RefreshURLs(const URLIndexPrivateData& data) { + std::string sql(StringPrintf( + "INSERT INTO %s (history_id, url, title, visit_count, typed_count, " + "last_visit_time, hidden) VALUES (?,?,?,?,?,?,?)", + kURLsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + for (HistoryInfoMap::const_iterator i = data.history_info_map_.begin(); + i != data.history_info_map_.end(); ++i) { + statement.Reset(true); + statement.BindInt(0, i->first); + statement.BindString(1, URLDatabase::GURLToDatabaseURL(i->second.url())); + statement.BindString16(2, i->second.title()); + statement.BindInt(3, i->second.visit_count()); + statement.BindInt(4, i->second.typed_count()); + statement.BindInt64(5, i->second.last_visit().ToInternalValue()); + statement.BindBool(6, i->second.hidden()); + if (!RunStatement(&statement)) + return false; + } + return true; +} + +bool InMemoryURLCacheDatabase::RefreshWordStarts( + const URLIndexPrivateData& data) { + std::string url_sql(StringPrintf( + "INSERT INTO %s (history_id, word_start) VALUES (?,?)", + kURLWordStartsTableName)); + sql::Statement url_statement(db_.GetCachedStatement(SQL_FROM_HERE, + url_sql.c_str())); + std::string title_sql(StringPrintf( + "INSERT INTO %s (history_id, word_start) VALUES (?,?)", + kTitleWordStartsTableName)); + sql::Statement title_statement(db_.GetCachedStatement(SQL_FROM_HERE, + title_sql.c_str())); + for (WordStartsMap::const_iterator i = data.word_starts_map_.begin(); + i != data.word_starts_map_.end(); ++i) { + for (WordStarts::const_iterator w = i->second.url_word_starts_.begin(); + w != i->second.url_word_starts_.end(); ++w) { + url_statement.Reset(true); + url_statement.BindInt(0, i->first); + url_statement.BindInt(1, *w); + if (!RunStatement(&url_statement)) + return false; + } + for (WordStarts::const_iterator w = i->second.title_word_starts_.begin(); + w != i->second.title_word_starts_.end(); ++w) { + title_statement.Reset(true); + title_statement.BindInt(0, i->first); + title_statement.BindInt(1, *w); + if (!RunStatement(&title_statement)) + return false; + } + } + return true; +} + +// Database Restoration -------------------------------------------------------- + +bool InMemoryURLCacheDatabase::RestoreWords(URLIndexPrivateData* data) { + // Rebuild word_list_, word_map_ and available_words_. + std::string sql(StringPrintf("SELECT word_id, word FROM %s ORDER BY word_id", + kWordsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + while (statement.Step()) { + WordID word_id = statement.ColumnInt(0); + // A gap in contiguous word IDs indicates word slots in the vector that are + // currently unused and available for future use as new words are indexed. + while (data->word_list_.size() < word_id) { + data->available_words_.insert(data->word_list_.size()); + data->word_list_.push_back(string16()); + } + string16 word = statement.ColumnString16(1); + DCHECK_EQ(static_cast<size_t>(word_id), data->word_list_.size()); + data->word_list_.push_back(word); + data->word_map_[word] = word_id; + } + return statement.Succeeded() && !data->word_list_.empty(); +} + +bool InMemoryURLCacheDatabase::RestoreCharWords(URLIndexPrivateData* data) { + // Rebuild char_word_map_. + std::string sql(StringPrintf("SELECT char, word_id FROM %s ORDER BY char", + kCharWordsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + CharWordIDMap& char_map(data->char_word_map_); + CharWordIDMap::iterator it = char_map.begin(); + while (statement.Step()) { + char16 next_char = statement.ColumnInt(0); + if (it == char_map.begin() || it->first != next_char) + it = char_map.insert(it, std::make_pair(next_char, WordIDSet())); + it->second.insert(statement.ColumnInt(1)); + } + return statement.Succeeded() && !char_map.empty(); +} + +bool InMemoryURLCacheDatabase::RestoreWordHistory(URLIndexPrivateData* data) { + // Rebuild word_id_history_map_ and history_id_word_map_. + std::string sql(StringPrintf( + "SELECT word_id, history_id FROM %s ORDER BY word_id", + kWordHistoryTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + WordIDHistoryMap& word_map(data->word_id_history_map_); + WordIDHistoryMap::iterator it = word_map.begin(); + while (statement.Step()) { + WordID word_id = statement.ColumnInt(0); + HistoryID history_id = statement.ColumnInt(1); + data->AddToHistoryIDWordMap(history_id, word_id); + if (it == word_map.begin() || it->first != word_id) + it = word_map.insert(it, std::make_pair(word_id, HistoryIDSet())); + it->second.insert(history_id); + } + return statement.Succeeded() && !word_map.empty(); +} + +bool InMemoryURLCacheDatabase::RestoreURLs(URLIndexPrivateData* data) { + // Rebuild history_info_map_. + std::string sql(StringPrintf( + "SELECT history_id, url, title, visit_count, typed_count, " + "last_visit_time, hidden FROM %s", + kURLsTableName)); + sql::Statement statement(db_.GetCachedStatement(SQL_FROM_HERE, sql.c_str())); + while (statement.Step()) { + HistoryID history_id = statement.ColumnInt64(0); + URLRow row(GURL(statement.ColumnString(1)), history_id); + row.set_title(statement.ColumnString16(2)); + row.set_visit_count(statement.ColumnInt(3)); + row.set_typed_count(statement.ColumnInt(4)); + row.set_last_visit(base::Time::FromInternalValue(statement.ColumnInt64(5))); + row.set_hidden(statement.ColumnInt(6) != 0); + data->history_info_map_[history_id] = row; + } + return statement.Succeeded() && !data->history_info_map_.empty(); +} + +bool InMemoryURLCacheDatabase::RestoreWordStarts(URLIndexPrivateData* data) { + std::string url_sql(StringPrintf( + "SELECT history_id, word_start FROM %s ORDER BY word_start", + kURLWordStartsTableName)); + sql::Statement url_statement(db_.GetCachedStatement(SQL_FROM_HERE, + url_sql.c_str())); + while (url_statement.Step()) { + HistoryID history_id = url_statement.ColumnInt64(0); + size_t word_start = url_statement.ColumnInt64(1); + data->word_starts_map_[history_id].url_word_starts_.push_back(word_start); + } + if (!url_statement.Succeeded()) + return false; + + std::string title_sql(StringPrintf( + "SELECT history_id, word_start FROM %s ORDER BY word_start", + kTitleWordStartsTableName)); + sql::Statement title_statement(db_.GetCachedStatement(SQL_FROM_HERE, + title_sql.c_str())); + while (title_statement.Step()) { + HistoryID history_id = title_statement.ColumnInt64(0); + size_t word_start = title_statement.ColumnInt64(1); + data->word_starts_map_[history_id].title_word_starts_.push_back(word_start); + } + + return title_statement.Succeeded() && !data->word_starts_map_.empty(); +} + +} // namespace history diff --git a/chrome/browser/history/in_memory_url_cache_database.h b/chrome/browser/history/in_memory_url_cache_database.h new file mode 100644 index 0000000..a03fceb --- /dev/null +++ b/chrome/browser/history/in_memory_url_cache_database.h @@ -0,0 +1,218 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_HISTORY_IN_MEMORY_URL_CACHE_DATABASE_H_ +#define CHROME_BROWSER_HISTORY_IN_MEMORY_URL_CACHE_DATABASE_H_ + +#include <string> + +#include "base/basictypes.h" +#include "base/callback_forward.h" +#include "base/gtest_prod_util.h" +#include "base/memory/ref_counted.h" +#include "base/threading/sequenced_worker_pool.h" +#include "chrome/browser/history/in_memory_url_index_types.h" +#include "sql/connection.h" +#include "sql/init_status.h" +#include "sql/meta_table.h" + +class FilePath; + +namespace sql { +class Statement; +} + +namespace history { + +class InMemoryURLIndex; +class URLIndexPrivateData; + +// Supports caching and restoring of the private data of the InMemoryURLIndex. +// This class has intimate knowledge of InMemoryURLIndex's URLIndexPrivateData +// class and directly pokes data therein during cache restoration. The only +// client of this class is URLIndexPrivateData. +// +// Tables in this database: +// word - Indexed words with IDs (from word_map_ and word_list_). +// char_words - Indexed characters with IDs of words containing the +// character (from char_word_map_). +// word_history - Word IDs with the IDs of history items containing the +// word (from word_id_history_map_). +// url - URLs and page titles along with visit and other metrics +// (from the history_info_map_ and word_starts_map_). +// url_word_starts - History IDs with the starting position of each word +// found in its URL. +// title_word_starts - History IDs with the starting position of each word +// found in its page title. +// +// NOTE: The history IDs in this database for URLs are identical to those in the +// main history database. +class InMemoryURLCacheDatabase + : public base::RefCountedThreadSafe<InMemoryURLCacheDatabase> { + public: + InMemoryURLCacheDatabase(); + + // Initializes the database connection. This must return true before any other + // functions on this class are called. |file_path| gives the path to where + // the cache database is located. |sequence_token| is used to coordinate all + // access to the cache database. This is normally called on the DB thread but + // may also be called in the sequenced worker pool for certain unit tests. + bool Init(const FilePath& file_path, + const base::SequencedWorkerPool::SequenceToken& sequence_token); + + // Shuts down the database connection. Posts a sequenced task that performs + // the matching ShutDownTask function below. + void Shutdown(); + + // Restore the private InMemoryURLIndex |data| from the database. Returns true + // if there was data to restore and the restore was successful. It is + // considered an error and false is returned if _any_ of the restored data + // structures in the private data, except |available_words_|, are empty after + // restoration. Technically, the case where _all_ data structures are empty + // after a restore is an error only when the user's history has _not_ been + // cleared. In the case where history _has_ been cleared and the data + // structures are legitimately empty we still return a false as the cost of + // attempting to rebuild the private data from an empty history database is + // negligible. + bool RestorePrivateData(URLIndexPrivateData* data); + + // Reset the database by dropping and recreating all database tables. Return + // true if successful. Note that a failure does not close the database. + virtual bool Reset(); + + // Adds information for a new |row| to the various database tables. These + // functions post a sequenced task that performs the matching '[name]Task' + // functions below. The client should wrap these functions in a transaction + // by the client by calling BeginTransaction(), whatever combination of these + // function as appropriate, and finally CommitTransaction(). The + // CommitTransaction() will notify the client if any failure occurs. + void AddHistoryToURLs(HistoryID history_id, const URLRow& row); + void AddHistoryToWordHistory(WordID word_id, HistoryID history_id); + void AddWordToWords(WordID word_id, const string16& uni_word); + void AddWordToCharWords(char16 uni_char, WordID word_id); + void AddRowWordStarts(HistoryID history_id, + const RowWordStarts& row_word_starts); + + // Deletes row information from the various database tables. These functions + // post a sequenced task that performs the matching '[name]Task' functions + // below. + void RemoveHistoryIDFromURLs(HistoryID history_id); + void RemoveHistoryIDFromWordHistory(HistoryID history_id); + void RemoveWordFromWords(WordID word_id); + void RemoveWordStarts(HistoryID history_id); + + // Wraps transactions on the database. We support nested transactions and only + // commit when the outermost one is committed (sqlite doesn't support true + // nested transactions). These functions post a sequenced task that performs + // the matching '[name]Task' functions below. + void BeginTransaction(); + void CommitTransaction(); + + // Completely replaces all data in the cache database with a fresh image. + // Returns true upon success. Must not be called on the main thread and and + // the caller should ensure that no other cache database update operations + // take place while this method is being performed. Returns true if the + // refresh was successful. + virtual bool Refresh(const URLIndexPrivateData& index_data); + + private: + friend class base::RefCountedThreadSafe<InMemoryURLCacheDatabase>; + friend class InMemoryURLIndexCacheTest; + friend class InMemoryURLIndexTest; + friend class InterposingCacheDatabase; + FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, CacheAddRemove); + + virtual ~InMemoryURLCacheDatabase(); + + // Initializes the database and returns true if successful. + bool InitDatabase(); + + // Posts the given |task| for sequential execution using a pool task + // coordinated by |sequence_token_|. + void PostSequencedDBTask(const tracked_objects::Location& from_here, + const base::Closure& task); + + // Closes the database. + void ShutdownTask(); + + // Makes sure the version is up-to-date, updating if necessary. Notify the + // user if the database is newer than expected. + sql::InitStatus EnsureCurrentVersion(); + + // Starts a database transaction. + void BeginTransactionTask(); + + // Commits the database transaction if no update operations failed after + // BeginTransactionTask() was called, otherwise rolls back the transaction. + // Sends a NOTIFICATION_IN_MEMORY_URL_CACHE_DATABASE_FAILURE notification + // if the transaction failed. + void CommitTransactionTask(); + + // Returns true if all tables exist. + bool VerifyTables(); + + // Creates the various tables and indexes returning true if successful. + bool CreateTables(); + + // Notifies observers that a failure occurred during a database update + // operation. + void NotifyDatabaseFailure(); + + // Executes |statement|, sets |update_error_| if the statement did not + // run successfully, and returns true if the statement was run successfully. + bool RunStatement(sql::Statement* statement); + + // Adds information about a new row to the various database tables. + void AddHistoryToURLsTask(HistoryID history_id, const URLRow& row); + void AddHistoryToWordHistoryTask(WordID word_id, HistoryID history_id); + // Note: AddWordToWordsTask is virtual so that it can be overridden by + // unit tests to simulate database failures. + virtual void AddWordToWordsTask(WordID word_id, const string16& uni_word); + void AddWordToCharWordsTask(char16 uni_char, WordID word_id); + void AddRowWordStartsTask(HistoryID history_id, + const RowWordStarts& row_word_starts); + + // Deletes row information from the various database tables. Returns true if + // successful. + void RemoveHistoryIDFromURLsTask(HistoryID history_id); + void RemoveHistoryIDFromWordHistoryTask(HistoryID history_id); + void RemoveWordFromWordsTask(WordID word_id); + void RemoveWordStartsTask(HistoryID history_id); + + // Completely replaces all data in the cache with a fresh image. Returns true + // if successful. + bool RefreshWords(const URLIndexPrivateData& index_data); + bool RefreshCharWords(const URLIndexPrivateData& index_data); + bool RefreshWordHistory(const URLIndexPrivateData& index_data); + bool RefreshURLs(const URLIndexPrivateData& index_data); + bool RefreshWordStarts(const URLIndexPrivateData& index_data); + + // Restores individual sections of the private data for the InMemoryURLIndex. + // Returns true if there was data to restore and the restore was successful. + bool RestoreWords(URLIndexPrivateData* index_data); + bool RestoreCharWords(URLIndexPrivateData* index_data); + bool RestoreWordHistory(URLIndexPrivateData* index_data); + bool RestoreURLs(URLIndexPrivateData* index_data); + bool RestoreWordStarts(URLIndexPrivateData* index_data); + + // Returns the database connection. + sql::Connection* get_db_for_testing() { return &db_; } + + // Sequence token for coordinating database tasks. + base::SequencedWorkerPool::SequenceToken sequence_token_; + + // The database. + sql::Connection db_; + sql::MetaTable meta_table_; + int update_error_; + + // Set to true once the shutdown process has begun. + bool shutdown_; + + DISALLOW_COPY_AND_ASSIGN(InMemoryURLCacheDatabase); +}; + +} // namespace history + +#endif // CHROME_BROWSER_HISTORY_IN_MEMORY_URL_CACHE_DATABASE_H_ diff --git a/chrome/browser/history/in_memory_url_index.cc b/chrome/browser/history/in_memory_url_index.cc index ace4275..fd6073a 100644 --- a/chrome/browser/history/in_memory_url_index.cc +++ b/chrome/browser/history/in_memory_url_index.cc @@ -8,72 +8,53 @@ #include "base/utf_string_conversions.h" #include "chrome/browser/history/history_notifications.h" #include "chrome/browser/history/history_service_factory.h" +#include "chrome/browser/history/in_memory_url_cache_database.h" #include "chrome/browser/history/url_database.h" #include "chrome/browser/history/url_index_private_data.h" #include "chrome/browser/profiles/profile.h" #include "chrome/common/chrome_notification_types.h" -#include "chrome/common/url_constants.h" #include "content/public/browser/browser_thread.h" #include "content/public/browser/notification_details.h" #include "content/public/browser/notification_service.h" #include "content/public/browser/notification_source.h" -using in_memory_url_index::InMemoryURLIndexCacheItem; +using content::BrowserThread; namespace history { -// Called by DoSaveToCacheFile to delete any old cache file at |path| when -// there is no private data to save. Runs on the FILE thread. -void DeleteCacheFile(const FilePath& path) { - DCHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); - file_util::Delete(path, false); -} +// InMemoryURLIndex::Observer -------------------------------------------------- -// Initializes a whitelist of URL schemes. -void InitializeSchemeWhitelist(std::set<std::string>* whitelist) { - DCHECK(whitelist); - if (!whitelist->empty()) - return; // Nothing to do, already initialized. - whitelist->insert(std::string(chrome::kAboutScheme)); - whitelist->insert(std::string(chrome::kChromeUIScheme)); - whitelist->insert(std::string(chrome::kFileScheme)); - whitelist->insert(std::string(chrome::kFtpScheme)); - whitelist->insert(std::string(chrome::kHttpScheme)); - whitelist->insert(std::string(chrome::kHttpsScheme)); - whitelist->insert(std::string(chrome::kMailToScheme)); +InMemoryURLIndex::Observer::Observer(InMemoryURLIndex* index) + : index_(index) { + DCHECK(index); + index_->AddObserver(this); } -// RefCountedBool -------------------------------------------------------------- - -RefCountedBool::~RefCountedBool() {} - -// Restore/SaveCacheObserver --------------------------------------------------- - -InMemoryURLIndex::RestoreCacheObserver::~RestoreCacheObserver() {} +InMemoryURLIndex::Observer::~Observer() { + index_->RemoveObserver(this); +} -InMemoryURLIndex::SaveCacheObserver::~SaveCacheObserver() {} +void InMemoryURLIndex::Observer::Loaded() { + MessageLoop::current()->QuitNow(); +} // RebuildPrivateDataFromHistoryDBTask ----------------------------------------- InMemoryURLIndex::RebuildPrivateDataFromHistoryDBTask:: - RebuildPrivateDataFromHistoryDBTask( - InMemoryURLIndex* index, - const std::string& languages, - const std::set<std::string>& scheme_whitelist) + RebuildPrivateDataFromHistoryDBTask(InMemoryURLIndex* index) : index_(index), - languages_(languages), - scheme_whitelist_(scheme_whitelist), succeeded_(false) { } +InMemoryURLIndex::RebuildPrivateDataFromHistoryDBTask:: + ~RebuildPrivateDataFromHistoryDBTask() { +} + bool InMemoryURLIndex::RebuildPrivateDataFromHistoryDBTask::RunOnDBThread( HistoryBackend* backend, HistoryDatabase* db) { - data_ = URLIndexPrivateData::RebuildFromHistory(db, languages_, - scheme_whitelist_); + data_ = URLIndexPrivateData::RebuildFromHistory(db, index_->private_data()); succeeded_ = data_.get() && !data_->Empty(); - if (!succeeded_ && data_.get()) - data_->Clear(); return true; } @@ -82,77 +63,96 @@ void InMemoryURLIndex::RebuildPrivateDataFromHistoryDBTask:: index_->DoneRebuidingPrivateDataFromHistoryDB(succeeded_, data_); } -InMemoryURLIndex::RebuildPrivateDataFromHistoryDBTask:: - ~RebuildPrivateDataFromHistoryDBTask() { +// IndexUpdateItem ------------------------------------------------------------- + +InMemoryURLIndex::IndexUpdateItem::IndexUpdateItem(UpdateType update_type, + URLRow row) + : update_type(update_type), + row(row) { } +InMemoryURLIndex::IndexUpdateItem::~IndexUpdateItem() {} + // InMemoryURLIndex ------------------------------------------------------------ InMemoryURLIndex::InMemoryURLIndex(Profile* profile, const FilePath& history_dir, const std::string& languages) : profile_(profile), - history_dir_(history_dir), languages_(languages), - private_data_(new URLIndexPrivateData), - restore_cache_observer_(NULL), - save_cache_observer_(NULL), - shutdown_(false), - needs_to_be_cached_(false) { - InitializeSchemeWhitelist(&scheme_whitelist_); + history_dir_(history_dir), + private_data_(new URLIndexPrivateData(history_dir, languages)), + sequence_token_(BrowserThread::GetBlockingPool()->GetSequenceToken()), + index_available_(false), + shutdown_(false) { if (profile) { - // TODO(mrossetti): Register for language change notifications. content::Source<Profile> source(profile); registrar_.Add(this, chrome::NOTIFICATION_HISTORY_URL_VISITED, source); registrar_.Add(this, chrome::NOTIFICATION_HISTORY_URLS_MODIFIED, source); registrar_.Add(this, chrome::NOTIFICATION_HISTORY_URLS_DELETED, source); } + // Note: private_data_ will be reset after rebuilding from the history + // database but the ownership of the database passes to the new + // private_data_ instance so there is no need to re-register for the + // following notification at that time. + registrar_.Add(this, + chrome::NOTIFICATION_IN_MEMORY_URL_CACHE_DATABASE_FAILURE, + content::Source<InMemoryURLCacheDatabase>( + private_data_->cache_db())); + // TODO(mrossetti): Register for language change notifications. } // Called only by unit tests. -InMemoryURLIndex::InMemoryURLIndex() +InMemoryURLIndex::InMemoryURLIndex(const FilePath& history_dir, + const std::string& languages) : profile_(NULL), - private_data_(new URLIndexPrivateData), - restore_cache_observer_(NULL), - save_cache_observer_(NULL), - shutdown_(false), - needs_to_be_cached_(false) { - InitializeSchemeWhitelist(&scheme_whitelist_); + languages_(languages), + history_dir_(history_dir), + index_available_(false), + shutdown_(false) { } -InMemoryURLIndex::~InMemoryURLIndex() { - // If there was a history directory (which there won't be for some unit tests) - // then insure that the cache has already been saved. - DCHECK(history_dir_.empty() || !needs_to_be_cached_); +InMemoryURLIndex::~InMemoryURLIndex() {} + +void InMemoryURLIndex::Init(bool disable_cache) { + if (disable_cache) { + RebuildFromHistoryIfLoaded(); + } else { + // It's safe to initialize the private data and the cache database without + // using the sequenced worker pool as no other database operations will be + // going on at the same time. + BrowserThread::PostTaskAndReplyWithResult<bool>( + BrowserThread::DB, FROM_HERE, + base::Bind(&URLIndexPrivateData::Init, private_data_, sequence_token_), + base::Bind(&InMemoryURLIndex::OnPrivateDataInitDone, AsWeakPtr())); + } } -void InMemoryURLIndex::Init() { - PostRestoreFromCacheFileTask(); +void InMemoryURLIndex::OnPrivateDataInitDone(bool succeeded) { + if (shutdown_) + return; + if (succeeded) + PostRestoreFromCacheTask(); + else + RebuildFromHistoryIfLoaded(); } -void InMemoryURLIndex::ShutDown() { +void InMemoryURLIndex::Shutdown() { + // Close down the cache database as quickly as possible. Any pending cache DB + // transactions will detect that the database is no longer there and give up. registrar_.RemoveAll(); cache_reader_consumer_.CancelAllRequests(); shutdown_ = true; - FilePath path; - if (!GetCacheFilePath(&path)) - return; - scoped_refptr<RefCountedBool> succeeded(new RefCountedBool(false)); - URLIndexPrivateData::WritePrivateDataToCacheFileTask( - private_data_, path, succeeded); - needs_to_be_cached_ = false; + private_data_->Shutdown(); } -void InMemoryURLIndex::ClearPrivateData() { - private_data_->Clear(); +void InMemoryURLIndex::AddObserver(InMemoryURLIndex::Observer* observer) { + observers_.AddObserver(observer); } -bool InMemoryURLIndex::GetCacheFilePath(FilePath* file_path) { - if (history_dir_.empty()) - return false; - *file_path = history_dir_.Append(FILE_PATH_LITERAL("History Provider Cache")); - return true; +void InMemoryURLIndex::RemoveObserver(InMemoryURLIndex::Observer* observer) { + observers_.RemoveObserver(observer); } // Querying -------------------------------------------------------------------- @@ -164,6 +164,10 @@ ScoredHistoryMatches InMemoryURLIndex::HistoryItemsForTerms( // Updating -------------------------------------------------------------------- +void InMemoryURLIndex::DeleteURL(const GURL& url) { + private_data_->DeleteURL(url); +} + void InMemoryURLIndex::Observe(int notification_type, const content::NotificationSource& source, const content::NotificationDetails& details) { @@ -184,6 +188,9 @@ void InMemoryURLIndex::Observe(int notification_type, content::Source<Profile>(profile_)); ScheduleRebuildFromHistory(); break; + case chrome::NOTIFICATION_IN_MEMORY_URL_CACHE_DATABASE_FAILURE: + RepairCacheDatabase(); + break; default: // For simplicity, the unit tests send us all notifications, even when // we haven't registered for them, so don't assert here. @@ -192,135 +199,170 @@ void InMemoryURLIndex::Observe(int notification_type, } void InMemoryURLIndex::OnURLVisited(const URLVisitedDetails* details) { - needs_to_be_cached_ |= - private_data_->UpdateURL(details->row, languages_, scheme_whitelist_); + if (index_available_) + private_data_->UpdateURL(details->row); + else + pending_updates_.push_back(IndexUpdateItem(UPDATE_VISIT, details->row)); } void InMemoryURLIndex::OnURLsModified(const URLsModifiedDetails* details) { for (URLRows::const_iterator row = details->changed_urls.begin(); - row != details->changed_urls.end(); ++row) - needs_to_be_cached_ |= - private_data_->UpdateURL(*row, languages_, scheme_whitelist_); + row != details->changed_urls.end(); ++row) { + if (index_available_) + private_data_->UpdateURL(*row); + else + pending_updates_.push_back(IndexUpdateItem(UPDATE_VISIT, *row)); + } } void InMemoryURLIndex::OnURLsDeleted(const URLsDeletedDetails* details) { if (details->all_history) { - ClearPrivateData(); - needs_to_be_cached_ = true; + PostResetPrivateDataTask(); } else { for (URLRows::const_iterator row = details->rows.begin(); - row != details->rows.end(); ++row) - needs_to_be_cached_ |= private_data_->DeleteURL(row->url()); + row != details->rows.end(); ++row) { + if (index_available_) + DeleteURL(row->url()); + else + pending_updates_.push_back(IndexUpdateItem(DELETE_VISIT, *row)); + } } } +void InMemoryURLIndex::FlushPendingUpdates() { + for (PendingUpdates::iterator i = pending_updates_.begin(); + i != pending_updates_.end(); ++i) { + if (i->update_type == UPDATE_VISIT) + private_data_->UpdateURL(i->row); + else if (i->update_type == DELETE_VISIT) + DeleteURL(i->row.url()); + } + pending_updates_.clear(); +} + // Restoring from Cache -------------------------------------------------------- -void InMemoryURLIndex::PostRestoreFromCacheFileTask() { - FilePath path; - if (!GetCacheFilePath(&path) || shutdown_) - return; - scoped_refptr<URLIndexPrivateData> restored_private_data = - new URLIndexPrivateData; - content::BrowserThread::PostTaskAndReply( - content::BrowserThread::FILE, FROM_HERE, - base::Bind(&URLIndexPrivateData::RestoreFromFileTask, path, - restored_private_data, languages_), - base::Bind(&InMemoryURLIndex::OnCacheLoadDone, AsWeakPtr(), - restored_private_data)); +void InMemoryURLIndex::PostRestoreFromCacheTask() { + // It's safe to restore from the cache database without using the sequenced + // worker pool as no other database operations will be going on at the same + // time. + BrowserThread::PostTaskAndReplyWithResult<bool>( + BrowserThread::DB, FROM_HERE, + base::Bind(&URLIndexPrivateData::RestoreFromCacheTask, private_data_), + base::Bind(&InMemoryURLIndex::OnCacheRestoreDone, AsWeakPtr())); } -void InMemoryURLIndex::OnCacheLoadDone( - scoped_refptr<URLIndexPrivateData> private_data) { - if (private_data.get() && !private_data->Empty()) { - private_data_ = private_data; - if (restore_cache_observer_) - restore_cache_observer_->OnCacheRestoreFinished(true); +void InMemoryURLIndex::OnCacheRestoreDone(bool succeeded) { + if (shutdown_) { + NotifyHasLoaded(); + return; + } + if (succeeded) { + FlushPendingUpdates(); + index_available_ = true; + NotifyHasLoaded(); } else if (profile_) { - // When unable to restore from the cache file delete the cache file, if - // it exists, and then rebuild from the history database if it's available, - // otherwise wait until the history database loaded and then rebuild. - FilePath path; - if (!GetCacheFilePath(&path) || shutdown_) - return; - content::BrowserThread::PostBlockingPoolTask( - FROM_HERE, base::Bind(DeleteCacheFile, path)); - HistoryService* service = - HistoryServiceFactory::GetForProfileWithoutCreating(profile_); - if (service && service->backend_loaded()) { - ScheduleRebuildFromHistory(); - } else { - registrar_.Add(this, chrome::NOTIFICATION_HISTORY_LOADED, - content::Source<Profile>(profile_)); - } + RebuildFromHistoryIfLoaded(); } } +void InMemoryURLIndex::NotifyHasLoaded() { + FOR_EACH_OBSERVER(InMemoryURLIndex::Observer, observers_, Loaded()); +} + // Restoring from the History DB ----------------------------------------------- +void InMemoryURLIndex::RebuildFromHistoryIfLoaded() { + // When unable to restore from the cache database, rebuild from the history + // database, if it's available, otherwise wait until the history database + // has loaded and then rebuild the index. + HistoryService* service = + HistoryServiceFactory::GetForProfileIfExists(profile_, + Profile::EXPLICIT_ACCESS); + if (service && service->backend_loaded()) { + ScheduleRebuildFromHistory(); + } else { + registrar_.Add(this, chrome::NOTIFICATION_HISTORY_LOADED, + content::Source<Profile>(profile_)); + } +} + void InMemoryURLIndex::ScheduleRebuildFromHistory() { + // It's possible that we were waiting on history to finish loading when + // the profile was told to shut down. + if (shutdown_) + return; + // Reset availability here as this function is called directly by unit tests. + index_available_ = false; HistoryService* service = HistoryServiceFactory::GetForProfile(profile_, Profile::EXPLICIT_ACCESS); + // Do not update the cache database while rebuilding. + private_data_->set_cache_enabled(false); service->ScheduleDBTask( - new InMemoryURLIndex::RebuildPrivateDataFromHistoryDBTask( - this, languages_, scheme_whitelist_), + new InMemoryURLIndex::RebuildPrivateDataFromHistoryDBTask(this), &cache_reader_consumer_); } void InMemoryURLIndex::DoneRebuidingPrivateDataFromHistoryDB( bool succeeded, scoped_refptr<URLIndexPrivateData> private_data) { - DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + BrowserThread::CurrentlyOn(BrowserThread::UI)); + if (shutdown_) + return; if (succeeded) { private_data_ = private_data; - PostSaveToCacheFileTask(); // Cache the newly rebuilt index. + private_data_->set_cache_enabled(true); + PostRefreshCacheTask(); // Cache the newly rebuilt index. } else { - private_data_->Clear(); // Dump the old private data. - // There is no need to do anything with the cache file as it was deleted - // when the rebuild from the history operation was kicked off. + private_data_->set_cache_enabled(true); + PostResetPrivateDataTask(); } - if (restore_cache_observer_) - restore_cache_observer_->OnCacheRestoreFinished(succeeded); } -void InMemoryURLIndex::RebuildFromHistory(HistoryDatabase* history_db) { - private_data_ = URLIndexPrivateData::RebuildFromHistory(history_db, - languages_, - scheme_whitelist_); +// Reset Cache ----------------------------------------------------------------- + +void InMemoryURLIndex::PostResetPrivateDataTask() { + index_available_ = false; + scoped_refptr<base::SequencedTaskRunner> runner = + BrowserThread::GetBlockingPool()->GetSequencedTaskRunner(sequence_token_); + DCHECK(runner.get()); + runner->PostTaskAndReply(FROM_HERE, + base::Bind(&URLIndexPrivateData::Reset, private_data_), + base::Bind(&InMemoryURLIndex::OnCacheRefreshOrResetDone, AsWeakPtr())); } -// Saving to Cache ------------------------------------------------------------- +// Refresh Cache --------------------------------------------------------------- -void InMemoryURLIndex::PostSaveToCacheFileTask() { - FilePath path; - if (!GetCacheFilePath(&path)) +void InMemoryURLIndex::PostRefreshCacheTask() { + scoped_refptr<base::SequencedTaskRunner> runner = + BrowserThread::GetBlockingPool()->GetSequencedTaskRunner(sequence_token_); + DCHECK(runner.get()); + runner->PostTaskAndReply(FROM_HERE, + base::Bind(&URLIndexPrivateData::RefreshCacheTask, private_data_), + base::Bind(&InMemoryURLIndex::OnCacheRefreshOrResetDone, AsWeakPtr())); +} + +void InMemoryURLIndex::OnCacheRefreshOrResetDone() { + if (shutdown_) { + NotifyHasLoaded(); return; - // If there is anything in our private data then make a copy of it and tell - // it to save itself to a file. - if (private_data_.get() && !private_data_->Empty()) { - // Note that ownership of the copy of our private data is passed to the - // completion closure below. - scoped_refptr<URLIndexPrivateData> private_data_copy = - private_data_->Duplicate(); - scoped_refptr<RefCountedBool> succeeded(new RefCountedBool(false)); - content::BrowserThread::PostTaskAndReply( - content::BrowserThread::FILE, FROM_HERE, - base::Bind(&URLIndexPrivateData::WritePrivateDataToCacheFileTask, - private_data_copy, path, succeeded), - base::Bind(&InMemoryURLIndex::OnCacheSaveDone, AsWeakPtr(), succeeded)); - } else { - // If there is no data in our index then delete any existing cache file. - content::BrowserThread::PostBlockingPoolTask( - FROM_HERE, - base::Bind(DeleteCacheFile, path)); } + FlushPendingUpdates(); + index_available_ = true; + NotifyHasLoaded(); } -void InMemoryURLIndex::OnCacheSaveDone( - scoped_refptr<RefCountedBool> succeeded) { - if (save_cache_observer_) - save_cache_observer_->OnCacheSaveFinished(succeeded->value()); +// Repair Cache ---------------------------------------------------------------- + +void InMemoryURLIndex::RepairCacheDatabase() { + // The database will disable itself when it detects an error so re-enable the + // database and try to refresh it from scratch. If that fails then the + // database will be left in a disabled state and will be rebuilt from the + // history database the next time the profile is opened. + private_data_->set_cache_enabled(true); + PostRefreshCacheTask(); // Cache the newly rebuilt index. } } // namespace history diff --git a/chrome/browser/history/in_memory_url_index.h b/chrome/browser/history/in_memory_url_index.h index 0c7724ac..55518b1 100644 --- a/chrome/browser/history/in_memory_url_index.h +++ b/chrome/browser/history/in_memory_url_index.h @@ -16,7 +16,9 @@ #include "base/gtest_prod_util.h" #include "base/memory/ref_counted.h" #include "base/memory/weak_ptr.h" +#include "base/observer_list.h" #include "base/string16.h" +#include "base/threading/sequenced_worker_pool.h" #include "chrome/browser/autocomplete/autocomplete_match.h" #include "chrome/browser/autocomplete/history_provider_util.h" #include "chrome/browser/cancelable_request.h" @@ -24,6 +26,7 @@ #include "chrome/browser/history/history_types.h" #include "chrome/browser/history/in_memory_url_index_types.h" #include "chrome/browser/history/scored_history_match.h" +#include "chrome/browser/history/url_index_private_data.h" #include "content/public/browser/notification_observer.h" #include "content/public/browser/notification_registrar.h" #include "sql/connection.h" @@ -35,19 +38,13 @@ namespace base { class Time; } -namespace in_memory_url_index { -class InMemoryURLIndexCacheItem; -} - namespace history { -namespace imui = in_memory_url_index; - -class HistoryDatabase; -class URLIndexPrivateData; -struct URLVisitedDetails; -struct URLsModifiedDetails; +class InMemoryURLIndexObserver; +class URLDatabase; struct URLsDeletedDetails; +struct URLsModifiedDetails; +struct URLVisitedDetails; // The URL history source. // Holds portions of the URL database in memory in an indexed form. Used to @@ -71,28 +68,25 @@ struct URLsDeletedDetails; class InMemoryURLIndex : public content::NotificationObserver, public base::SupportsWeakPtr<InMemoryURLIndex> { public: - // Defines an abstract class which is notified upon completion of restoring - // the index's private data either by reading from the cache file or by - // rebuilding from the history database. - class RestoreCacheObserver { + // Observer is used for blocking until the InMemoryURLIndex has finished + // loading. Usage typically follows this pattern: + // InMemoryURLIndex::Observer observer(index); // Create observer. + // MessageLoop::current()->Run(); // Blocks until loaded. + // + class Observer { public: - virtual ~RestoreCacheObserver(); + explicit Observer(InMemoryURLIndex* index); - // Callback that lets the observer know that the restore operation has - // completed. |succeeded| indicates if the restore was successful. This is - // called on the UI thread. - virtual void OnCacheRestoreFinished(bool succeeded) = 0; - }; + // Called when the InMemoryURLIndex has completed loading. + virtual void Loaded(); - // Defines an abstract class which is notified upon completion of saving - // the index's private data to the cache file. - class SaveCacheObserver { - public: - virtual ~SaveCacheObserver(); + private: + friend class InMemoryURLIndexTestBase; + virtual ~Observer(); - // Callback that lets the observer know that the save succeeded. - // This is called on the UI thread. - virtual void OnCacheSaveFinished(bool succeeded) = 0; + InMemoryURLIndex* index_; + + DISALLOW_COPY_AND_ASSIGN(Observer); }; // |profile|, which may be NULL during unit testing, is used to register for @@ -106,13 +100,17 @@ class InMemoryURLIndex : public content::NotificationObserver, virtual ~InMemoryURLIndex(); // Opens and prepares the index of historical URL visits. If the index private - // data cannot be restored from its cache file then it is rebuilt from the - // history database. - void Init(); + // data cannot be restored from its cache database then it is rebuilt from the + // history database. |disable_cache| causes the InMemoryURLIndex to not create + // or use its cache database. + void Init(bool disable_cache); + + // Signals that any outstanding initialization should be canceled. + void Shutdown(); - // Signals that any outstanding initialization should be canceled and - // flushes the cache to disk. - void ShutDown(); + // Returns true if the index has been loaded or rebuilt and so is available + // for use. + bool index_available() const { return index_available_; } // Scans the history index and returns a vector with all scored, matching // history items. This entry point simply forwards the call on to the @@ -120,32 +118,26 @@ class InMemoryURLIndex : public content::NotificationObserver, // refer to that class. ScoredHistoryMatches HistoryItemsForTerms(const string16& term_string); - // Sets the optional observers for completion of restoral and saving of the - // index's private data. - void set_restore_cache_observer( - RestoreCacheObserver* restore_cache_observer) { - restore_cache_observer_ = restore_cache_observer; - } - void set_save_cache_observer(SaveCacheObserver* save_cache_observer) { - save_cache_observer_ = save_cache_observer; - } + // Deletes the index entry, if any, for the given |url|. + void DeleteURL(const GURL& url); private: friend class ::HistoryQuickProviderTest; - friend class InMemoryURLIndexTest; + friend class InMemoryURLIndex::Observer; friend class InMemoryURLIndexCacheTest; + friend class InMemoryURLIndexTest; + friend class InMemoryURLIndexTestBase; + friend class IntercessionaryIndexTest; + FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, ExpireRow); + FRIEND_TEST_ALL_PREFIXES(IntercessionaryIndexTest, CacheDatabaseFailure); + FRIEND_TEST_ALL_PREFIXES(IntercessionaryIndexTest, + ShutdownDuringCacheRefresh); FRIEND_TEST_ALL_PREFIXES(LimitedInMemoryURLIndexTest, Initialization); - // Creating one of me without a history path is not allowed (tests excepted). - InMemoryURLIndex(); - // HistoryDBTask used to rebuild our private data from the history database. class RebuildPrivateDataFromHistoryDBTask : public HistoryDBTask { public: - explicit RebuildPrivateDataFromHistoryDBTask( - InMemoryURLIndex* index, - const std::string& languages, - const std::set<std::string>& scheme_whitelist); + explicit RebuildPrivateDataFromHistoryDBTask(InMemoryURLIndex* index); virtual bool RunOnDBThread(HistoryBackend* backend, history::HistoryDatabase* db) OVERRIDE; @@ -155,27 +147,56 @@ class InMemoryURLIndex : public content::NotificationObserver, virtual ~RebuildPrivateDataFromHistoryDBTask(); InMemoryURLIndex* index_; // Call back to this index at completion. - std::string languages_; // Languages for word-breaking. - std::set<std::string> scheme_whitelist_; // Schemes to be indexed. bool succeeded_; // Indicates if the rebuild was successful. scoped_refptr<URLIndexPrivateData> data_; // The rebuilt private data. DISALLOW_COPY_AND_ASSIGN(RebuildPrivateDataFromHistoryDBTask); }; - // Initializes all index data members in preparation for restoring the index - // from the cache or a complete rebuild from the history database. - void ClearPrivateData(); + // For unit testing only. + InMemoryURLIndex(const FilePath& history_dir, const std::string& languages); + + // Completes index initialization once the cache DB has been initialized. + void OnPrivateDataInitDone(bool succeeded); + + // Adds or removes an observer that is notified when the index has been + // loaded. + void AddObserver(InMemoryURLIndex::Observer* observer); + void RemoveObserver(InMemoryURLIndex::Observer* observer); + + // Handles notifications of history changes. + virtual void Observe(int notification_type, + const content::NotificationSource& source, + const content::NotificationDetails& details) OVERRIDE; + + // Notification handlers. + void OnURLVisited(const URLVisitedDetails* details); + void OnURLsModified(const URLsModifiedDetails* details); + void OnURLsDeleted(const URLsDeletedDetails* details); + + // Posts any outstanding updates to the index which were queued while the + // index was being initialized. + void FlushPendingUpdates(); - // Constructs a file path for the cache file within the same directory where - // the history database is kept and saves that path to |file_path|. Returns - // true if |file_path| can be successfully constructed. (This function - // provided as a hook for unit testing.) - bool GetCacheFilePath(FilePath* file_path); + // Restores the index's private data from the cache database stored in the + // profile directory. If no database is found or can be restored then look + // for an old version protobuf-based cache file. + void PostRestoreFromCacheTask(); - // Restores the index's private data from the cache file stored in the - // profile directory. - void PostRestoreFromCacheFileTask(); + // Determines if the private data was successfully restored from the cache + // database, as indicated by |succeeded|, or if the private data must be + // rebuilt from the history database. If successful, notifies any + // |restore_cache_observer_|. Otherwise, kicks off a rebuild from the history + // database. + void OnCacheRestoreDone(bool succeeded); + + // Notifies all observers that the index has been loaded or rebuilt. + void NotifyHasLoaded(); + + // Rebuilds the index from the history database if the history database has + // been loaded, otherwise registers for the history loaded notification so + // that the rebuild can take place at a later time. + void RebuildFromHistoryIfLoaded(); // Schedules a history task to rebuild our private data from the history // database. @@ -189,76 +210,67 @@ class InMemoryURLIndex : public content::NotificationObserver, bool succeeded, scoped_refptr<URLIndexPrivateData> private_data); - // Rebuilds the history index from the history database in |history_db|. - // Used for unit testing only. - void RebuildFromHistory(HistoryDatabase* history_db); - - // Determines if the private data was successfully reloaded from the cache - // file or if the private data must be rebuilt from the history database. - // |private_data_ptr|'s data will be NULL if the cache file load failed. If - // successful, sets the private data and notifies any - // |restore_cache_observer_|. Otherwise, kicks off a rebuild from the history - // database. - void OnCacheLoadDone( - scoped_refptr<URLIndexPrivateData> private_data_ptr); - - // Callback function that sets the private data from the just-restored-from- - // file |private_data|. Notifies any |restore_cache_observer_| that the - // restore has succeeded. - void OnCacheRestored(URLIndexPrivateData* private_data); + // Posts a task to completely reset the private data and the backing cache. + void PostResetPrivateDataTask(); - // Posts a task to cache the index private data and write the cache file to - // the profile directory. - void PostSaveToCacheFileTask(); - - // Saves private_data_ to the given |path|. Runs on the UI thread. - // Provided for unit testing so that a test cache file can be used. - void DoSaveToCacheFile(const FilePath& path); - - // Notifies the observer, if any, of the success of the private data caching. - // |succeeded| is true on a successful save. - void OnCacheSaveDone(scoped_refptr<RefCountedBool> succeeded); - - // Handles notifications of history changes. - virtual void Observe(int notification_type, - const content::NotificationSource& source, - const content::NotificationDetails& details) OVERRIDE; + // Posts a task to completely replace the cache database with a current + // image of the index private data. + void PostRefreshCacheTask(); - // Notification handlers. - void OnURLVisited(const URLVisitedDetails* details); - void OnURLsModified(const URLsModifiedDetails* details); - void OnURLsDeleted(const URLsDeletedDetails* details); + // Callback used by PostRefreshCacheTask and PostResetPrivateDataTask to + // notify observers that the cache database contents have been refreshed or + // reset and that the loading of the index is complete. + void OnCacheRefreshOrResetDone(); - // Sets the directory wherein the cache file will be maintained. - // For unit test usage only. - void set_history_dir(const FilePath& dir_path) { history_dir_ = dir_path; } + // Attempts to refresh the cache database in response to a notification that + // an update transaction has failed. If the refresh fails then the cache + // database is ignored and an attempt will be made to rebuild the cache + // the next time the associated profile is opened. + void RepairCacheDatabase(); - // Returns a pointer to our private data. For unit testing only. - URLIndexPrivateData* private_data() { return private_data_.get(); } + // Returns a pointer to our private data. + scoped_refptr<URLIndexPrivateData> private_data() { return private_data_; } - // Returns the set of whitelisted schemes. For unit testing only. - const std::set<std::string>& scheme_whitelist() { return scheme_whitelist_; } + // Returns the blocking pool sequence token. + base::SequencedWorkerPool::SequenceToken sequence_token_for_testing() { + return sequence_token_; + } // The profile, may be null when testing. Profile* profile_; - // Directory where cache file resides. This is, except when unit testing, - // the same directory in which the profile's history database is found. It - // should never be empty. - FilePath history_dir_; - // Languages used during the word-breaking process during indexing. std::string languages_; - // Only URLs with a whitelisted scheme are indexed. - std::set<std::string> scheme_whitelist_; + // Directory where cache database or protobuf-based cache file resides. + // This is, except when unit testing, the same directory in which the + // profile's history database is found. + FilePath history_dir_; // The index's durable private data. scoped_refptr<URLIndexPrivateData> private_data_; - // Observers to notify upon restoral or save of the private data cache. - RestoreCacheObserver* restore_cache_observer_; - SaveCacheObserver* save_cache_observer_; + // Sequence token for coordinating database tasks. This is shared with + // our private data and its cache database. + const base::SequencedWorkerPool::SequenceToken sequence_token_; + + bool index_available_; // True when index is available for updating. + + // Contains index updates queued up while the index is unavailable. This + // usually during profile startup. + enum UpdateType { UPDATE_VISIT, DELETE_VISIT }; + + struct IndexUpdateItem { + IndexUpdateItem(UpdateType update_type, URLRow row); + ~IndexUpdateItem(); + + UpdateType update_type; + URLRow row; // The row to be updated or deleted. + }; + typedef std::vector<IndexUpdateItem> PendingUpdates; + PendingUpdates pending_updates_; + + ObserverList<InMemoryURLIndex::Observer> observers_; CancelableRequestConsumer cache_reader_consumer_; content::NotificationRegistrar registrar_; @@ -266,14 +278,6 @@ class InMemoryURLIndex : public content::NotificationObserver, // Set to true once the shutdown process has begun. bool shutdown_; - // Set to true when changes to the index have been made and the index needs - // to be cached. Set to false when the index has been cached. Used as a - // temporary safety check to insure that the cache is saved before the - // index has been destructed. - // TODO(mrossetti): Eliminate once the transition to SQLite has been done. - // http://crbug.com/83659 - bool needs_to_be_cached_; - DISALLOW_COPY_AND_ASSIGN(InMemoryURLIndex); }; diff --git a/chrome/browser/history/in_memory_url_index_types.h b/chrome/browser/history/in_memory_url_index_types.h index 5183019..12af660 100644 --- a/chrome/browser/history/in_memory_url_index_types.h +++ b/chrome/browser/history/in_memory_url_index_types.h @@ -117,6 +117,7 @@ typedef std::map<string16, WordID> WordMap; // A map from character to the word_ids of words containing that character. typedef std::set<WordID> WordIDSet; // An index into the WordList. +// TODO(mrossetti): Consider using <multimap>. See http://crbug.com/138334 typedef std::map<char16, WordIDSet> CharWordIDMap; // A map from word (by word_id) to history items containing that word. @@ -142,24 +143,6 @@ struct RowWordStarts { }; typedef std::map<HistoryID, RowWordStarts> WordStartsMap; -// A RefCountedThreadSafe class that manages a bool used for passing around -// success when saving the persistent data for the InMemoryURLIndex in a cache. -class RefCountedBool : public base::RefCountedThreadSafe<RefCountedBool> { - public: - explicit RefCountedBool(bool value) : value_(value) {} - - bool value() const { return value_; } - void set_value(bool value) { value_ = value; } - - private: - friend class base::RefCountedThreadSafe<RefCountedBool>; - virtual ~RefCountedBool(); - - bool value_; - - DISALLOW_COPY_AND_ASSIGN(RefCountedBool); -}; - } // namespace history #endif // CHROME_BROWSER_HISTORY_IN_MEMORY_URL_INDEX_TYPES_H_ diff --git a/chrome/browser/history/in_memory_url_index_types_unittest.cc b/chrome/browser/history/in_memory_url_index_types_unittest.cc index 634f459b..4bd467c 100644 --- a/chrome/browser/history/in_memory_url_index_types_unittest.cc +++ b/chrome/browser/history/in_memory_url_index_types_unittest.cc @@ -35,27 +35,28 @@ TEST_F(InMemoryURLIndexTypesTest, StaticFunctions) { String16Vector string_vec = String16VectorFromString16(string_a, false, &actual_starts_a); ASSERT_EQ(7U, string_vec.size()); + // See if we got the words we expected. - EXPECT_EQ(UTF8ToUTF16("http"), string_vec[0]); - EXPECT_EQ(UTF8ToUTF16("www"), string_vec[1]); - EXPECT_EQ(UTF8ToUTF16("google"), string_vec[2]); - EXPECT_EQ(UTF8ToUTF16("com"), string_vec[3]); - EXPECT_EQ(UTF8ToUTF16("frammy"), string_vec[4]); - EXPECT_EQ(UTF8ToUTF16("the"), string_vec[5]); - EXPECT_EQ(UTF8ToUTF16("brammy"), string_vec[6]); + EXPECT_EQ(ASCIIToUTF16("http"), string_vec[0]); + EXPECT_EQ(ASCIIToUTF16("www"), string_vec[1]); + EXPECT_EQ(ASCIIToUTF16("google"), string_vec[2]); + EXPECT_EQ(ASCIIToUTF16("com"), string_vec[3]); + EXPECT_EQ(ASCIIToUTF16("frammy"), string_vec[4]); + EXPECT_EQ(ASCIIToUTF16("the"), string_vec[5]); + EXPECT_EQ(ASCIIToUTF16("brammy"), string_vec[6]); + // Verify the word starts. size_t expected_starts_a[] = {0, 7, 11, 18, 23, 31, 35}; EXPECT_TRUE(IntArraysEqual(expected_starts_a, arraysize(expected_starts_a), actual_starts_a)); - WordStarts actual_starts_b; string_vec = String16VectorFromString16(string_a, true, &actual_starts_b); ASSERT_EQ(5U, string_vec.size()); - EXPECT_EQ(UTF8ToUTF16("http://"), string_vec[0]); - EXPECT_EQ(UTF8ToUTF16("www.google.com/"), string_vec[1]); - EXPECT_EQ(UTF8ToUTF16("frammy"), string_vec[2]); - EXPECT_EQ(UTF8ToUTF16("the"), string_vec[3]); - EXPECT_EQ(UTF8ToUTF16("brammy"), string_vec[4]); + EXPECT_EQ(ASCIIToUTF16("http://"), string_vec[0]); + EXPECT_EQ(ASCIIToUTF16("www.google.com/"), string_vec[1]); + EXPECT_EQ(ASCIIToUTF16("frammy"), string_vec[2]); + EXPECT_EQ(ASCIIToUTF16("the"), string_vec[3]); + EXPECT_EQ(ASCIIToUTF16("brammy"), string_vec[4]); size_t expected_starts_b[] = {0, 7, 23, 31, 35}; EXPECT_TRUE(IntArraysEqual(expected_starts_b, arraysize(expected_starts_b), actual_starts_b)); @@ -65,6 +66,7 @@ TEST_F(InMemoryURLIndexTypesTest, StaticFunctions) { WordStarts actual_starts_c; string_vec = String16VectorFromString16(string_c, false, &actual_starts_c); ASSERT_EQ(8U, string_vec.size()); + // Note that we stop collecting words and word starts at kMaxSignificantChars. size_t expected_starts_c[] = {1, 7, 16, 22, 32, 43}; EXPECT_TRUE(IntArraysEqual(expected_starts_c, arraysize(expected_starts_c), @@ -76,12 +78,13 @@ TEST_F(InMemoryURLIndexTypesTest, StaticFunctions) { WordStarts actual_starts_d; String16Set string_set = String16SetFromString16(string_d, &actual_starts_d); EXPECT_EQ(5U, string_set.size()); + // See if we got the words we expected. - EXPECT_TRUE(string_set.find(UTF8ToUTF16("com")) != string_set.end()); - EXPECT_TRUE(string_set.find(UTF8ToUTF16("google")) != string_set.end()); - EXPECT_TRUE(string_set.find(UTF8ToUTF16("http")) != string_set.end()); - EXPECT_TRUE(string_set.find(UTF8ToUTF16("search")) != string_set.end()); - EXPECT_TRUE(string_set.find(UTF8ToUTF16("web")) != string_set.end()); + EXPECT_TRUE(string_set.find(ASCIIToUTF16("com")) != string_set.end()); + EXPECT_TRUE(string_set.find(ASCIIToUTF16("google")) != string_set.end()); + EXPECT_TRUE(string_set.find(ASCIIToUTF16("http")) != string_set.end()); + EXPECT_TRUE(string_set.find(ASCIIToUTF16("search")) != string_set.end()); + EXPECT_TRUE(string_set.find(ASCIIToUTF16("web")) != string_set.end()); size_t expected_starts_d[] = {0, 7, 11, 18, 22, 29, 36, 40}; EXPECT_TRUE(IntArraysEqual(expected_starts_d, arraysize(expected_starts_d), actual_starts_d)); @@ -113,7 +116,7 @@ TEST_F(InMemoryURLIndexTypesTest, StaticFunctions) { // Test MatchTermInString TermMatches matches_g = MatchTermInString( - UTF8ToUTF16("x"), UTF8ToUTF16("axbxcxdxex fxgx/hxixjx.kx"), 123); + ASCIIToUTF16("x"), ASCIIToUTF16("axbxcxdxex fxgx/hxixjx.kx"), 123); const size_t expected_offsets[] = { 1, 3, 5, 7, 9, 12, 14, 17, 19, 21, 24 }; ASSERT_EQ(arraysize(expected_offsets), matches_g.size()); for (size_t i = 0; i < arraysize(expected_offsets); ++i) diff --git a/chrome/browser/history/in_memory_url_index_unittest.cc b/chrome/browser/history/in_memory_url_index_unittest.cc index ddc7825..6682fcd41 100644 --- a/chrome/browser/history/in_memory_url_index_unittest.cc +++ b/chrome/browser/history/in_memory_url_index_unittest.cc @@ -2,313 +2,93 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include <algorithm> -#include <fstream> - -#include "base/file_path.h" -#include "base/file_util.h" -#include "base/message_loop.h" -#include "base/path_service.h" -#include "base/scoped_temp_dir.h" -#include "base/string16.h" +#include "base/memory/ref_counted.h" #include "base/string_util.h" +#include "base/stringprintf.h" +#include "base/threading/sequenced_worker_pool.h" #include "base/utf_string_conversions.h" #include "chrome/browser/autocomplete/autocomplete_provider.h" -#include "chrome/browser/history/history.h" -#include "chrome/browser/history/history_backend.h" -#include "chrome/browser/history/history_database.h" +#include "chrome/browser/bookmarks/bookmark_model.h" +#include "chrome/browser/bookmarks/bookmark_model_factory.h" #include "chrome/browser/history/history_notifications.h" -#include "chrome/browser/history/history_service_factory.h" -#include "chrome/browser/history/in_memory_url_index.h" +#include "chrome/browser/history/in_memory_url_cache_database.h" +#include "chrome/browser/history/in_memory_url_index_unittest_base.h" #include "chrome/browser/history/in_memory_url_index_types.h" #include "chrome/browser/history/url_index_private_data.h" +#include "chrome/common/chrome_constants.h" #include "chrome/common/chrome_notification_types.h" -#include "chrome/common/chrome_paths.h" -#include "chrome/test/base/testing_profile.h" -#include "content/public/browser/notification_details.h" +#include "chrome/test/base/ui_test_utils.h" +#include "content/public/browser/notification_service.h" #include "content/public/browser/notification_source.h" -#include "content/public/test/test_browser_thread.h" -#include "sql/transaction.h" -#include "testing/gtest/include/gtest/gtest.h" - -using content::BrowserThread; - -// The test version of the history url database table ('url') is contained in -// a database file created from a text file('url_history_provider_test.db.txt'). -// The only difference between this table and a live 'urls' table from a -// profile is that the last_visit_time column in the test table contains a -// number specifying the number of days relative to 'today' to which the -// absolute time should be set during the test setup stage. -// -// The format of the test database text file is of a SQLite .dump file. -// Note that only lines whose first character is an upper-case letter are -// processed when creating the test database. +#include "sql/connection.h" +#include "third_party/sqlite/sqlite3.h" -namespace history { - -// ----------------------------------------------------------------------------- - -// Observer class so the unit tests can wait while the cache is being saved. -class CacheFileSaverObserver : public InMemoryURLIndex::SaveCacheObserver { - public: - explicit CacheFileSaverObserver(MessageLoop* loop); - virtual void OnCacheSaveFinished(bool succeeded) OVERRIDE; +using history::String16SetFromString16; - MessageLoop* loop_; - bool succeeded_; - DISALLOW_COPY_AND_ASSIGN(CacheFileSaverObserver); -}; - -CacheFileSaverObserver::CacheFileSaverObserver(MessageLoop* loop) - : loop_(loop), - succeeded_(false) { - DCHECK(loop); -} - -void CacheFileSaverObserver::OnCacheSaveFinished(bool succeeded) { - succeeded_ = succeeded; - loop_->Quit(); -} +//------------------------------------------------------------------------------ -// Observer class so the unit tests can wait while the cache is being restored. -class CacheFileReaderObserver : public InMemoryURLIndex::RestoreCacheObserver { +// This class overrides the standard TestingProfile's disabling of the +// InMemoryURLIndexCacheDatabase. +class CacheTestingProfile : public TestingProfile { public: - explicit CacheFileReaderObserver(MessageLoop* loop); - virtual void OnCacheRestoreFinished(bool succeeded) OVERRIDE; - - MessageLoop* loop_; - bool succeeded_; - DISALLOW_COPY_AND_ASSIGN(CacheFileReaderObserver); + // Creates a testing profile but enables the cache database. + bool InitHistoryService(HistoryService* history_service, + bool no_db) OVERRIDE; }; -CacheFileReaderObserver::CacheFileReaderObserver(MessageLoop* loop) - : loop_(loop), - succeeded_(false) { - DCHECK(loop); +bool CacheTestingProfile::InitHistoryService(HistoryService* history_service, + bool no_db) { + DCHECK(history_service); + return history_service->Init(GetPath(), + reinterpret_cast<BookmarkService*>( + BookmarkModelFactory::GetForProfile(this)), + no_db, false); } -void CacheFileReaderObserver::OnCacheRestoreFinished(bool succeeded) { - succeeded_ = succeeded; - loop_->Quit(); -} +namespace history { // ----------------------------------------------------------------------------- -class InMemoryURLIndexTest : public testing::Test { - public: - InMemoryURLIndexTest(); - +// Creates a URLRow with basic data for |url|, |title|, |visit_count|, +// |typed_count| and |id|. |last_visit_ago| gives the number of days from now +// to which to set the URL's last_visit. +URLRow MakeURLRowWithID(const char* url, + const char* title, + int visit_count, + int last_visit_ago, + int typed_count, + URLID id); + +class InMemoryURLIndexTest : public InMemoryURLIndexTestBase { protected: - // Test setup. - virtual void SetUp(); - - // Allows the database containing the test data to be customized by - // subclasses. - virtual FilePath::StringType TestDBName() const; - -#if 0 - // Convenience function to create a URLRow with basic data for |url|, |title|, - // |visit_count|, and |typed_count|. |last_visit_ago| gives the number of - // days from now to set the URL's last_visit. - URLRow MakeURLRow(const char* url, - const char* title, - int visit_count, - int last_visit_ago, - int typed_count); - - // Convenience functions for easily creating vectors of search terms. - String16Vector Make1Term(const char* term) const; - String16Vector Make2Terms(const char* term_1, const char* term_2) const; - - // Convenience function for GetTopicalityScore() that builds the - // term match and word break information automatically that are needed - // to call GetTopicalityScore(). It only works for scoring a single term, - // not multiple terms. - float GetTopicalityScoreOfTermAgainstURLAndTitle(const string16& term, - const string16& url, - const string16& title); -#endif + void SetUp() OVERRIDE; + + // Provides custom test data file name. + virtual FilePath::StringType TestDBName() const OVERRIDE; // Validates that the given |term| is contained in |cache| and that it is // marked as in-use. void CheckTerm(const URLIndexPrivateData::SearchTermCacheMap& cache, string16 term) const; - - // Pass-through function to simplify our friendship with HistoryService. - sql::Connection& GetDB(); - - // Pass-through functions to simplify our friendship with InMemoryURLIndex. - URLIndexPrivateData* GetPrivateData() const; - void ClearPrivateData(); - void set_history_dir(const FilePath& dir_path); - bool GetCacheFilePath(FilePath* file_path) const; - void PostRestoreFromCacheFileTask(); - void PostSaveToCacheFileTask(); - void Observe(int notification_type, - const content::NotificationSource& source, - const content::NotificationDetails& details); - const std::set<std::string>& scheme_whitelist(); - - - // Pass-through functions to simplify our friendship with URLIndexPrivateData. - bool UpdateURL(const URLRow& row); - bool DeleteURL(const GURL& url); - - // Data verification helper functions. - void ExpectPrivateDataNotEmpty(const URLIndexPrivateData& data); - void ExpectPrivateDataEmpty(const URLIndexPrivateData& data); - void ExpectPrivateDataEqual(const URLIndexPrivateData& expected, - const URLIndexPrivateData& actual); - - MessageLoopForUI message_loop_; - content::TestBrowserThread ui_thread_; - content::TestBrowserThread file_thread_; - TestingProfile profile_; - - scoped_ptr<InMemoryURLIndex> url_index_; - HistoryDatabase* history_database_; }; -InMemoryURLIndexTest::InMemoryURLIndexTest() - : ui_thread_(content::BrowserThread::UI, &message_loop_), - file_thread_(content::BrowserThread::FILE, &message_loop_) { -} - -sql::Connection& InMemoryURLIndexTest::GetDB() { - return history_database_->GetDB(); -} - -URLIndexPrivateData* InMemoryURLIndexTest::GetPrivateData() const { - DCHECK(url_index_->private_data()); - return url_index_->private_data(); -} - -void InMemoryURLIndexTest::ClearPrivateData() { - return url_index_->ClearPrivateData(); -} - -void InMemoryURLIndexTest::set_history_dir(const FilePath& dir_path) { - return url_index_->set_history_dir(dir_path); -} - -bool InMemoryURLIndexTest::GetCacheFilePath(FilePath* file_path) const { - DCHECK(file_path); - return url_index_->GetCacheFilePath(file_path); -} - -void InMemoryURLIndexTest::PostRestoreFromCacheFileTask() { - url_index_->PostRestoreFromCacheFileTask(); -} - -void InMemoryURLIndexTest::PostSaveToCacheFileTask() { - url_index_->PostSaveToCacheFileTask(); -} - -void InMemoryURLIndexTest::Observe( - int notification_type, - const content::NotificationSource& source, - const content::NotificationDetails& details) { - url_index_->Observe(notification_type, source, details); -} - -const std::set<std::string>& InMemoryURLIndexTest::scheme_whitelist() { - return url_index_->scheme_whitelist(); -} - -bool InMemoryURLIndexTest::UpdateURL(const URLRow& row) { - return GetPrivateData()->UpdateURL(row, url_index_->languages_, - url_index_->scheme_whitelist_); -} - -bool InMemoryURLIndexTest::DeleteURL(const GURL& url) { - return GetPrivateData()->DeleteURL(url); -} - void InMemoryURLIndexTest::SetUp() { - // We cannot access the database until the backend has been loaded. - profile_.CreateHistoryService(true, false); - profile_.CreateBookmarkModel(true); - profile_.BlockUntilBookmarkModelLoaded(); - profile_.BlockUntilHistoryProcessesPendingRequests(); - HistoryService* history_service = - HistoryServiceFactory::GetForProfile(&profile_, - Profile::EXPLICIT_ACCESS); - ASSERT_TRUE(history_service); - HistoryBackend* backend = history_service->history_backend_.get(); - history_database_ = backend->db(); - - // Create and populate a working copy of the URL history database. - FilePath history_proto_path; - PathService::Get(chrome::DIR_TEST_DATA, &history_proto_path); - history_proto_path = history_proto_path.Append( - FILE_PATH_LITERAL("History")); - history_proto_path = history_proto_path.Append(TestDBName()); - EXPECT_TRUE(file_util::PathExists(history_proto_path)); - - std::ifstream proto_file(history_proto_path.value().c_str()); - static const size_t kCommandBufferMaxSize = 2048; - char sql_cmd_line[kCommandBufferMaxSize]; - - sql::Connection& db(GetDB()); - ASSERT_TRUE(db.is_open()); - { - sql::Transaction transaction(&db); - transaction.Begin(); - while (!proto_file.eof()) { - proto_file.getline(sql_cmd_line, kCommandBufferMaxSize); - if (!proto_file.eof()) { - // We only process lines which begin with a upper-case letter. - // TODO(mrossetti): Can iswupper() be used here? - if (sql_cmd_line[0] >= 'A' && sql_cmd_line[0] <= 'Z') { - std::string sql_cmd(sql_cmd_line); - sql::Statement sql_stmt(db.GetUniqueStatement(sql_cmd_line)); - EXPECT_TRUE(sql_stmt.Run()); - } - } - } - transaction.Commit(); - } - proto_file.close(); - - // Update the last_visit_time table column - // such that it represents a time relative to 'now'. - sql::Statement statement(db.GetUniqueStatement( - "SELECT" HISTORY_URL_ROW_FIELDS "FROM urls;")); - ASSERT_TRUE(statement.is_valid()); - base::Time time_right_now = base::Time::NowFromSystemTime(); - base::TimeDelta day_delta = base::TimeDelta::FromDays(1); - { - sql::Transaction transaction(&db); - transaction.Begin(); - while (statement.Step()) { - URLRow row; - history_database_->FillURLRow(statement, &row); - base::Time last_visit = time_right_now; - for (int64 i = row.last_visit().ToInternalValue(); i > 0; --i) - last_visit -= day_delta; - row.set_last_visit(last_visit); - history_database_->UpdateURLRow(row.id(), row); - } - transaction.Commit(); - } - - url_index_.reset( - new InMemoryURLIndex(&profile_, FilePath(), "en,ja,hi,zh")); - url_index_->Init(); - url_index_->RebuildFromHistory(history_database_); + // Set things up without enabling the cache database. + InMemoryURLIndexTestBase::SetUp(); + LoadIndex(); } FilePath::StringType InMemoryURLIndexTest::TestDBName() const { return FILE_PATH_LITERAL("url_history_provider_test.db.txt"); } -#if 0 -URLRow InMemoryURLIndexTest::MakeURLRow(const char* url, - const char* title, - int visit_count, - int last_visit_ago, - int typed_count) { - URLRow row(GURL(url), 0); +URLRow MakeURLRowWithID(const char* url, + const char* title, + int visit_count, + int last_visit_ago, + int typed_count, + URLID id) { + URLRow row(GURL(url), id); row.set_title(UTF8ToUTF16(title)); row.set_visit_count(visit_count); row.set_typed_count(typed_count); @@ -317,34 +97,6 @@ URLRow InMemoryURLIndexTest::MakeURLRow(const char* url, return row; } -String16Vector InMemoryURLIndexTest::Make1Term(const char* term) const { - String16Vector original_terms; - original_terms.push_back(UTF8ToUTF16(term)); - return original_terms; -} - -String16Vector InMemoryURLIndexTest::Make2Terms(const char* term_1, - const char* term_2) const { - String16Vector original_terms; - original_terms.push_back(UTF8ToUTF16(term_1)); - original_terms.push_back(UTF8ToUTF16(term_2)); - return original_terms; -} - -float InMemoryURLIndexTest::GetTopicalityScoreOfTermAgainstURLAndTitle( - const string16& term, - const string16& url, - const string16& title) { - TermMatches url_matches = MatchTermInString(term, url, 0); - TermMatches title_matches = MatchTermInString(term, title, 0); - RowWordStarts word_starts; - String16SetFromString16(url, &word_starts.url_word_starts_); - String16SetFromString16(title, &word_starts.title_word_starts_); - return ScoredHistoryMatch::GetTopicalityScore( - 1, url, url_matches, title_matches, word_starts); -} -#endif - void InMemoryURLIndexTest::CheckTerm( const URLIndexPrivateData::SearchTermCacheMap& cache, string16 term) const { @@ -357,120 +109,11 @@ void InMemoryURLIndexTest::CheckTerm( << "Cache item '" << term << "' should be marked as being in use."; } -void InMemoryURLIndexTest::ExpectPrivateDataNotEmpty( - const URLIndexPrivateData& data) { - EXPECT_FALSE(data.word_list_.empty()); - // available_words_ will be empty since we have freshly built the - // data set for these tests. - EXPECT_TRUE(data.available_words_.empty()); - EXPECT_FALSE(data.word_map_.empty()); - EXPECT_FALSE(data.char_word_map_.empty()); - EXPECT_FALSE(data.word_id_history_map_.empty()); - EXPECT_FALSE(data.history_id_word_map_.empty()); - EXPECT_FALSE(data.history_info_map_.empty()); -} - -void InMemoryURLIndexTest::ExpectPrivateDataEmpty( - const URLIndexPrivateData& data) { - EXPECT_TRUE(data.word_list_.empty()); - EXPECT_TRUE(data.available_words_.empty()); - EXPECT_TRUE(data.word_map_.empty()); - EXPECT_TRUE(data.char_word_map_.empty()); - EXPECT_TRUE(data.word_id_history_map_.empty()); - EXPECT_TRUE(data.history_id_word_map_.empty()); - EXPECT_TRUE(data.history_info_map_.empty()); -} - -// Helper function which compares two maps for equivalence. The maps' values -// are associative containers and their contents are compared as well. -template<typename T> -void ExpectMapOfContainersIdentical(const T& expected, const T& actual) { - ASSERT_EQ(expected.size(), actual.size()); - for (typename T::const_iterator expected_iter = expected.begin(); - expected_iter != expected.end(); ++expected_iter) { - typename T::const_iterator actual_iter = actual.find(expected_iter->first); - ASSERT_TRUE(actual.end() != actual_iter); - typename T::mapped_type const& expected_values(expected_iter->second); - typename T::mapped_type const& actual_values(actual_iter->second); - ASSERT_EQ(expected_values.size(), actual_values.size()); - for (typename T::mapped_type::const_iterator set_iter = - expected_values.begin(); set_iter != expected_values.end(); ++set_iter) - EXPECT_EQ(actual_values.count(*set_iter), - expected_values.count(*set_iter)); - } -} - -void InMemoryURLIndexTest::ExpectPrivateDataEqual( - const URLIndexPrivateData& expected, - const URLIndexPrivateData& actual) { - EXPECT_EQ(expected.word_list_.size(), actual.word_list_.size()); - EXPECT_EQ(expected.word_map_.size(), actual.word_map_.size()); - EXPECT_EQ(expected.char_word_map_.size(), actual.char_word_map_.size()); - EXPECT_EQ(expected.word_id_history_map_.size(), - actual.word_id_history_map_.size()); - EXPECT_EQ(expected.history_id_word_map_.size(), - actual.history_id_word_map_.size()); - EXPECT_EQ(expected.history_info_map_.size(), actual.history_info_map_.size()); - EXPECT_EQ(expected.word_starts_map_.size(), actual.word_starts_map_.size()); - // WordList must be index-by-index equal. - size_t count = expected.word_list_.size(); - for (size_t i = 0; i < count; ++i) - EXPECT_EQ(expected.word_list_[i], actual.word_list_[i]); - - ExpectMapOfContainersIdentical(expected.char_word_map_, - actual.char_word_map_); - ExpectMapOfContainersIdentical(expected.word_id_history_map_, - actual.word_id_history_map_); - ExpectMapOfContainersIdentical(expected.history_id_word_map_, - actual.history_id_word_map_); - - for (HistoryInfoMap::const_iterator expected_info = - expected.history_info_map_.begin(); - expected_info != expected.history_info_map_.end(); ++expected_info) { - HistoryInfoMap::const_iterator actual_info = - actual.history_info_map_.find(expected_info->first); - // NOTE(yfriedman): ASSERT_NE can't be used due to incompatibility between - // gtest and STLPort in the Android build. See - // http://code.google.com/p/googletest/issues/detail?id=359 - ASSERT_TRUE(actual_info != actual.history_info_map_.end()); - const URLRow& expected_row(expected_info->second); - const URLRow& actual_row(actual_info->second); - EXPECT_EQ(expected_row.visit_count(), actual_row.visit_count()); - EXPECT_EQ(expected_row.typed_count(), actual_row.typed_count()); - EXPECT_EQ(expected_row.last_visit(), actual_row.last_visit()); - EXPECT_EQ(expected_row.url(), actual_row.url()); - } - - for (WordStartsMap::const_iterator expected_starts = - expected.word_starts_map_.begin(); - expected_starts != expected.word_starts_map_.end(); - ++expected_starts) { - WordStartsMap::const_iterator actual_starts = - actual.word_starts_map_.find(expected_starts->first); - // NOTE(yfriedman): ASSERT_NE can't be used due to incompatibility between - // gtest and STLPort in the Android build. See - // http://code.google.com/p/googletest/issues/detail?id=359 - ASSERT_TRUE(actual_starts != actual.word_starts_map_.end()); - const RowWordStarts& expected_word_starts(expected_starts->second); - const RowWordStarts& actual_word_starts(actual_starts->second); - EXPECT_EQ(expected_word_starts.url_word_starts_.size(), - actual_word_starts.url_word_starts_.size()); - EXPECT_TRUE(std::equal(expected_word_starts.url_word_starts_.begin(), - expected_word_starts.url_word_starts_.end(), - actual_word_starts.url_word_starts_.begin())); - EXPECT_EQ(expected_word_starts.title_word_starts_.size(), - actual_word_starts.title_word_starts_.size()); - EXPECT_TRUE(std::equal(expected_word_starts.title_word_starts_.begin(), - expected_word_starts.title_word_starts_.end(), - actual_word_starts.title_word_starts_.begin())); - } -} - //------------------------------------------------------------------------------ class LimitedInMemoryURLIndexTest : public InMemoryURLIndexTest { protected: - FilePath::StringType TestDBName() const; + FilePath::StringType TestDBName() const OVERRIDE; }; FilePath::StringType LimitedInMemoryURLIndexTest::TestDBName() const { @@ -480,25 +123,25 @@ FilePath::StringType LimitedInMemoryURLIndexTest::TestDBName() const { TEST_F(LimitedInMemoryURLIndexTest, Initialization) { // Verify that the database contains the expected number of items, which // is the pre-filtered count, i.e. all of the items. - sql::Statement statement(GetDB().GetUniqueStatement("SELECT * FROM urls;")); + sql::Statement statement( + history_database_->get_db_for_testing()->GetUniqueStatement( + "SELECT * FROM urls;")); ASSERT_TRUE(statement.is_valid()); uint64 row_count = 0; while (statement.Step()) ++row_count; EXPECT_EQ(1U, row_count); - url_index_.reset( - new InMemoryURLIndex(&profile_, FilePath(), "en,ja,hi,zh")); - url_index_->Init(); - url_index_->RebuildFromHistory(history_database_); - URLIndexPrivateData& private_data(*GetPrivateData()); // history_info_map_ should have the same number of items as were filtered. - EXPECT_EQ(1U, private_data.history_info_map_.size()); - EXPECT_EQ(35U, private_data.char_word_map_.size()); - EXPECT_EQ(17U, private_data.word_map_.size()); + EXPECT_EQ(1U, url_index_->private_data_->history_info_map_.size()); + EXPECT_EQ(35U, url_index_->private_data_->char_word_map_.size()); + EXPECT_EQ(17U, url_index_->private_data_->word_map_.size()); } +//------------------------------------------------------------------------------ + TEST_F(InMemoryURLIndexTest, Retrieval) { // See if a very specific term gives a single result. + // Note that in each case the term will be lowercased by the search. ScoredHistoryMatches matches = url_index_->HistoryItemsForTerms(ASCIIToUTF16("DrudgeReport")); ASSERT_EQ(1U, matches.size()); @@ -650,13 +293,13 @@ TEST_F(InMemoryURLIndexTest, HugeResultSet) { ScoredHistoryMatches matches = url_index_->HistoryItemsForTerms(ASCIIToUTF16("b")); - URLIndexPrivateData& private_data(*GetPrivateData()); + const URLIndexPrivateData* private_data(GetPrivateData()); ASSERT_EQ(AutocompleteProvider::kMaxMatches, matches.size()); // There are 7 matches already in the database. - ASSERT_EQ(1008U, private_data.pre_filter_item_count_); - ASSERT_EQ(500U, private_data.post_filter_item_count_); + ASSERT_EQ(1008U, private_data->pre_filter_item_count_); + ASSERT_EQ(500U, private_data->post_filter_item_count_); ASSERT_EQ(AutocompleteProvider::kMaxMatches, - private_data.post_scoring_item_count_); + private_data->post_scoring_item_count_); } TEST_F(InMemoryURLIndexTest, TitleSearch) { @@ -746,7 +389,7 @@ TEST_F(InMemoryURLIndexTest, TypedCharacterCaching) { typedef URLIndexPrivateData::SearchTermCacheMap::iterator CacheIter; typedef URLIndexPrivateData::SearchTermCacheItem CacheItem; - URLIndexPrivateData::SearchTermCacheMap& cache( + const URLIndexPrivateData::SearchTermCacheMap& cache( GetPrivateData()->search_term_cache_); // The cache should be empty at this point. @@ -795,128 +438,6 @@ TEST_F(InMemoryURLIndexTest, TypedCharacterCaching) { CheckTerm(cache, ASCIIToUTF16("rec")); } -#if 0 -TEST_F(InMemoryURLIndexTest, Scoring) { - URLRow row_a(MakeURLRow("http://abcdef", "fedcba", 3, 30, 1)); - // We use NowFromSystemTime() because MakeURLRow uses the same function - // to calculate last visit time when building a row. - base::Time now = base::Time::NowFromSystemTime(); - RowWordStarts word_starts; - // Test scores based on position. - // TODO(mpearson): Test new_scoring if we're actually going to turn it - // on by default. This requires setting word_starts, which isn't done - // right now. - ScoredHistoryMatch scored_a(row_a, ASCIIToUTF16("abc"), Make1Term("abc"), - word_starts, now); - ScoredHistoryMatch scored_b(row_a, ASCIIToUTF16("bcd"), Make1Term("bcd"), - word_starts, now); - EXPECT_GT(scored_a.raw_score, scored_b.raw_score); - // Test scores based on length. - ScoredHistoryMatch scored_c(row_a, ASCIIToUTF16("abcd"), Make1Term("abcd"), - word_starts, now); - EXPECT_LT(scored_a.raw_score, scored_c.raw_score); - // Test scores based on order. - ScoredHistoryMatch scored_d(row_a, ASCIIToUTF16("abcdef"), - Make2Terms("abc", "def"), word_starts, now); - ScoredHistoryMatch scored_e(row_a, ASCIIToUTF16("def abc"), - Make2Terms("def", "abc"), word_starts, now); - EXPECT_GT(scored_d.raw_score, scored_e.raw_score); - // Test scores based on visit_count. - URLRow row_b(MakeURLRow("http://abcdef", "fedcba", 10, 30, 1)); - ScoredHistoryMatch scored_f(row_b, ASCIIToUTF16("abc"), Make1Term("abc"), - word_starts, now); - EXPECT_GT(scored_f.raw_score, scored_a.raw_score); - // Test scores based on last_visit. - URLRow row_c(MakeURLRow("http://abcdef", "fedcba", 3, 10, 1)); - ScoredHistoryMatch scored_g(row_c, ASCIIToUTF16("abc"), Make1Term("abc"), - word_starts, now); - EXPECT_GT(scored_g.raw_score, scored_a.raw_score); - // Test scores based on typed_count. - URLRow row_d(MakeURLRow("http://abcdef", "fedcba", 3, 30, 10)); - ScoredHistoryMatch scored_h(row_d, ASCIIToUTF16("abc"), Make1Term("abc"), - word_starts, now); - EXPECT_GT(scored_h.raw_score, scored_a.raw_score); - // Test scores based on a terms appearing multiple times. - URLRow row_i(MakeURLRow("http://csi.csi.csi/csi_csi", - "CSI Guide to CSI Las Vegas, CSI New York, CSI Provo", 3, 30, 10)); - ScoredHistoryMatch scored_i(row_i, ASCIIToUTF16("csi"), Make1Term("csi"), - word_starts, now); - EXPECT_LT(scored_i.raw_score, 1400); -} - -TEST_F(InMemoryURLIndexTest, GetTopicalityScoreTrailingSlash) { - const float hostname = GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("def"), - ASCIIToUTF16("http://abc.def.com/"), - ASCIIToUTF16("Non-Matching Title")); - const float hostname_no_slash = GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("def"), - ASCIIToUTF16("http://abc.def.com"), - ASCIIToUTF16("Non-Matching Title")); - EXPECT_EQ(hostname_no_slash, hostname); -} - -// This function only tests scoring of single terms that match exactly -// once somewhere in the URL or title. -TEST_F(InMemoryURLIndexTest, GetTopicalityScore) { - string16 url = ASCIIToUTF16("http://abc.def.com/path1/path2?" - "arg1=val1&arg2=val2#hash_component"); - string16 title = ASCIIToUTF16("here is a title"); - const float hostname_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("abc"), url, title); - const float hostname_mid_word_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("bc"), url, title); - const float path_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("path1"), url, title); - const float path_mid_word_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("ath1"), url, title); - const float arg_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("arg2"), url, title); - const float arg_mid_word_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("rg2"), url, title); - const float protocol_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("htt"), url, title); - const float protocol_mid_word_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("tt"), url, title); - const float title_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("her"), url, title); - const float title_mid_word_score = - GetTopicalityScoreOfTermAgainstURLAndTitle( - ASCIIToUTF16("er"), url, title); - // Verify hostname > path > arg, and the same for the matches at - // non-word-boundaries. - EXPECT_GT(hostname_score, path_score); - EXPECT_GT(path_score, arg_score); - EXPECT_GT(hostname_mid_word_score, path_mid_word_score); - EXPECT_GT(path_mid_word_score, arg_mid_word_score); - // Also verify that the matches at non-word-boundaries all score - // worse than the matches at word boundaries. These two sets suffice. - EXPECT_GT(arg_score, hostname_mid_word_score); - EXPECT_GT(title_score, title_mid_word_score); - // Check that title matches fit somewhere reasonable compared to the - // various types of URL matches. - EXPECT_GT(title_score, arg_score); - EXPECT_GT(arg_score, hostname_mid_word_score); - EXPECT_GT(title_mid_word_score, arg_mid_word_score); - // Finally, verify that protocol matches score worse than everything - // (except possibly mid-word matches in the ?arg section of the URL--I - // can imagine scoring those pretty harshly as well). - EXPECT_GT(path_mid_word_score, protocol_score); - EXPECT_GT(path_mid_word_score, protocol_mid_word_score); - EXPECT_GT(title_mid_word_score, protocol_score); - EXPECT_GT(title_mid_word_score, protocol_mid_word_score); -} -#endif - TEST_F(InMemoryURLIndexTest, AddNewRows) { // Verify that the row we're going to add does not already exist. URLID new_row_id = 87654321; @@ -971,9 +492,11 @@ TEST_F(InMemoryURLIndexTest, ExpireRow) { URLsDeletedDetails deleted_details; deleted_details.all_history = false; deleted_details.rows.push_back(matches[0].url_info); - Observe(chrome::NOTIFICATION_HISTORY_URLS_DELETED, - content::Source<InMemoryURLIndexTest>(this), - content::Details<history::HistoryDetails>(&deleted_details)); + content::Source<InMemoryURLIndexTest> source(this); + url_index_->Observe( + chrome::NOTIFICATION_HISTORY_URLS_DELETED, + content::Source<InMemoryURLIndexTest>(this), + content::Details<history::HistoryDetails>(&deleted_details)); EXPECT_TRUE(url_index_->HistoryItemsForTerms( ASCIIToUTF16("DrudgeReport")).empty()); } @@ -1052,113 +575,610 @@ TEST_F(InMemoryURLIndexTest, WhitelistedURLs) { { "xmpp://guest@example.com", false }, }; - URLIndexPrivateData& private_data(*GetPrivateData()); - const std::set<std::string>& whitelist(scheme_whitelist()); + std::set<std::string> whitelist; + URLIndexPrivateData::InitializeSchemeWhitelistForTesting(&whitelist); for (size_t i = 0; i < ARRAYSIZE_UNSAFE(data); ++i) { GURL url(data[i].url_spec); EXPECT_EQ(data[i].expected_is_whitelisted, - private_data.URLSchemeIsWhitelisted(url, whitelist)); + URLIndexPrivateData::URLSchemeIsWhitelisted(url, whitelist)); + } +} + +// InMemoryURLIndexCacheTest --------------------------------------------------- + +class InMemoryURLIndexCacheTest : public InMemoryURLIndexTestBase { + public: + // A test helper class which captures the state of the URL index and cache + // prior to the addition, update or delete of a history URL visit, determines + // the expected state after the URL change, and verifies the actual state. + class CacheChecker { + public: + enum ChangeType { + URL_ADDED, + URL_UPDATED, + URL_REMOVED, + }; + + // Captures the state of the index and cache and the changes which are + // expected to be made. |index| points to the test index. |change_type| + // specifies the kind of history URL visit change that will take place. + // |existing_words| lists the words that are referenced by the history item + // (both URL and page title) which are expected to already be recorded in + // the index. |added_words| gives terms which are not currently in the index + // but will be added for this history item. |removed_words| gives words in + // the index but which are not referenced by any other history item and + // which will be removed by the test action. + CacheChecker(InMemoryURLIndex* index, + ChangeType change_type, + URLID history_id, + const String16Set& existing_words, + const String16Set& added_words, + const String16Set& removed_words); + virtual ~CacheChecker(); + + // Initializes and destroys things in a way that the standard gtest + // EXPECT_xxx macros can be used since those macros cannot be used in + // constructors and destructors. + void Init(); + void Destroy(); + + private: + // A helper struct that records index current state for words in the list of + // |existing_words| and |removed_words|. + struct ExistingWord { + ExistingWord() : word_id(0), word_map_count(0), word_table_count(0) {} + ExistingWord(WordID word_id, + size_t word_map_count, + size_t word_table_count) + : word_id(word_id), + word_map_count(word_map_count), + word_table_count(word_table_count) {} + + WordID word_id; // Used for removed_words only. + size_t word_map_count; + size_t word_table_count; + }; + + // Table names. + static const char* kWordsTableName; + static const char* kWordHistoryTableName; + static const char* kURLsTableName; + static const char* kURLWordStartsTableName; + static const char* kTitleWordStartsTableName; + // Field names. + static const char* kWord; + static const char* kWordID; + static const char* kHistoryID; + static const char* kWordStart; + + // Helper functions that perform a COUNT query of the cache database with + // the given criteria and return the number of rows meeting that criteria. + // |table| specifies the name of the table in which the SELECT will be + // performed. |column| gives the column against which the desired |word|, + // |word_id| or |history_id| will be matched. + int SelectCount(const char* table); + int SelectCount(const char* table, + const char* column, + const string16& word); + int SelectCount(const char* table, + const char* column, + WordID word_id); + int SelectCount(const char* table, + const char* column, + HistoryID history_id); + + InMemoryURLIndex& index_; + sql::Connection* db_; + // Information about the changes that the test is expected to cause. + ChangeType change_type_; + URLID history_id_; + String16Set existing_words_; + String16Set added_words_; + String16Set removed_words_; + // Current state information about the index. + size_t word_list_count_; + size_t word_table_count; + size_t history_id_word_map_count_; + size_t available_words_count_; + std::map<string16, ExistingWord> existing_words_and_counts_; + std::map<string16, ExistingWord> removed_words_and_counts_; + }; + + void SetUp() OVERRIDE; + + // Provides custom test data file name. + virtual FilePath::StringType TestDBName() const OVERRIDE; +}; + +// InMemoryURLIndexCacheTest::CacheChecker ------------------------------------- + +// Cache database table names. See in_memory_url_cache_database.cc. +typedef InMemoryURLIndexCacheTest::CacheChecker CacheChecker; +const char* CacheChecker::kWordsTableName = "words"; +const char* CacheChecker::kWordHistoryTableName = "word_history"; +const char* CacheChecker::kURLsTableName = "urls"; +const char* CacheChecker::kURLWordStartsTableName = "url_word_starts"; +const char* CacheChecker::kTitleWordStartsTableName = "title_word_starts"; + +// Cache database table field names. +const char* CacheChecker::kWord = "word"; +const char* CacheChecker::kWordID = "word_id"; +const char* CacheChecker::kHistoryID = "history_id"; +const char* CacheChecker::kWordStart = "word_start"; + +CacheChecker::CacheChecker( + InMemoryURLIndex* index, + ChangeType change_type, + URLID history_id, + const String16Set& existing_words, + const String16Set& added_words, + const String16Set& removed_words) + : index_(*index), + db_(index_.private_data_->cache_db()->get_db_for_testing()), + change_type_(change_type), + history_id_(history_id), + existing_words_(existing_words), + added_words_(added_words), + removed_words_(removed_words), + // Remember the old word count, the old word table count, the old history + // ID count, and how many unused words are available for reuse. + word_list_count_(index_.private_data_->word_list_.size()), + word_table_count(SelectCount(kWordsTableName)), + history_id_word_map_count_( + index_.private_data_->history_id_word_map_.size()), + available_words_count_(index_.private_data_->available_words_.size()) { + DCHECK(db_); + Init(); +} + +void CacheChecker::Init() { + // Verify that the existing words exist and remember their counts. + URLIndexPrivateData* private_data(index_.private_data_.get()); + for (String16Set::iterator word_iter = existing_words_.begin(); + word_iter != existing_words_.end(); ++word_iter) { + string16 word(*word_iter); + ASSERT_NE(0U, private_data->word_map_.count(word)) + << "Existing word '" << word << "' not found."; + EXPECT_GT(SelectCount(kWordsTableName, kWord, word), 0) + << "Existing word '" << word << "' not found in words table."; + WordID word_id = private_data->word_map_[word]; + EXPECT_GT(SelectCount(kWordHistoryTableName, kWordID, word_id), 0) + << "Existing word '" << word << "' not found in word_history table."; + existing_words_and_counts_[word] = + ExistingWord(word_id, private_data->word_id_history_map_[word_id]. + size(), + SelectCount(kWordHistoryTableName, kWordID, word_id)); + } + + // Verify that the added words don't already exist. + for (String16Set::iterator word_iter = added_words_.begin(); + word_iter != added_words_.end(); ++word_iter) { + string16 word(*word_iter); + EXPECT_EQ(0U, private_data->word_map_.count(word)) + << "Word '" << word << "' to be added is already there."; + EXPECT_EQ(0, SelectCount(kWordsTableName, kWord, word)) + << "Word '" << word << "' to be added is already in words table."; + } + + // Verify that the removed words exist and remember their counts. + for (String16Set::iterator word_iter = removed_words_.begin(); + word_iter != removed_words_.end(); ++word_iter) { + string16 word(*word_iter); + ASSERT_EQ(1U, private_data->word_map_.count(word)) + << "Word '" << word << "' to be removed not found."; + WordID word_id = private_data->word_map_[word]; + EXPECT_GT(private_data->word_id_history_map_[word_id].size(), 0U); + EXPECT_GT(SelectCount(kWordsTableName, kWord, word), 0) + << "Word '" << word << "' to be removed not found in words table."; + EXPECT_GT(SelectCount(kWordHistoryTableName, kWordID, word_id), 0) + << "Word '" << word << "' to remove not found in word_history table."; + removed_words_and_counts_[word] = + ExistingWord(word_id, private_data->word_id_history_map_[word_id]. + size(), + SelectCount(kWordHistoryTableName, kWordID, word_id)); + EXPECT_EQ(removed_words_and_counts_[word].word_map_count, + removed_words_and_counts_[word].word_table_count); + } +} + +CacheChecker::~CacheChecker() { + Destroy(); +} + +void CacheChecker::Destroy() { + // Verify that the existing words still exist and their counts have + // incremented by 1. + URLIndexPrivateData* private_data(index_.private_data_.get()); + for (String16Set::iterator word_iter = existing_words_.begin(); + word_iter != existing_words_.end(); ++word_iter) { + string16 word(*word_iter); + const ExistingWord& existing_word(existing_words_and_counts_[word]); + size_t expected_count = existing_word.word_map_count; + if (change_type_ == URL_ADDED) + ++expected_count; + WordID word_id = private_data->word_map_[word]; + EXPECT_EQ(expected_count, + private_data->word_id_history_map_[word_id].size()) + << "Occurrence count for existing word '" << word + << "' in the word_id_history_map_ is incorrect."; + EXPECT_EQ(static_cast<int>(expected_count), + SelectCount(kWordHistoryTableName, kWordID, word_id)) + << "Occurrence count for existing word '" << word << "' in " + "the word_history database table is incorrect."; + } + + // Verify the added words have been added and their counts are 1. + for (String16Set::iterator word_iter = added_words_.begin(); + word_iter != added_words_.end(); ++word_iter) { + string16 word(*word_iter); + ASSERT_EQ(1U, private_data->word_map_.count(word)); + WordID word_id = private_data->word_map_[word]; + EXPECT_EQ(1U, private_data->word_id_history_map_[word_id].size()) + << "Count for added word '" << word << "' not 1."; + EXPECT_EQ(1, SelectCount(kWordsTableName, kWord, word)) + << "Word '" << word << "' not added to words table."; + EXPECT_EQ(1, SelectCount(kWordHistoryTableName, kWordID, word_id)) + << "Word '" << word << "' not added to word_history table."; } + + switch (change_type_) { + case URL_ADDED: + ASSERT_EQ(0U, removed_words_.size()) + << "BAD TEST DATA -- removed_words must be empty for URL_ADDED."; + // Fall through. + case URL_UPDATED: { + // There should be added-words + existing-words - removed-words entries + // in the word_history table for the history ID. + size_t word_count_for_id = existing_words_.size() + added_words_.size(); + EXPECT_EQ(word_count_for_id, + private_data->history_id_word_map_[history_id_].size()); + EXPECT_EQ(static_cast<int>(word_count_for_id), + SelectCount(kWordHistoryTableName, kHistoryID, history_id_)); + EXPECT_EQ(1U, private_data->history_id_word_map_.count(history_id_)); + EXPECT_EQ(1, SelectCount(kURLsTableName, kHistoryID, history_id_)); + } + break; + case URL_REMOVED: + // There should be no entries in the word_history table for the history + // ID. + ASSERT_EQ(0U, added_words_.size()) + << "BAD TEST DATA -- added_words must be empty for URL_REMOVED."; + EXPECT_EQ(0U, private_data->history_id_word_map_.count(history_id_)); + EXPECT_EQ(0, SelectCount(kWordHistoryTableName, kHistoryID, history_id_)); + EXPECT_EQ(0, SelectCount(kURLsTableName, kHistoryID, history_id_)); + break; + } + + // Verify that the count for removed words has been decremented or that + // they have been deleted if their count has dropped to 0. + int completely_removed = 0; + for (String16Set::iterator word_iter = removed_words_.begin(); + word_iter != removed_words_.end(); ++word_iter) { + string16 word(*word_iter); + const ExistingWord& removed_word(removed_words_and_counts_[word]); + size_t expected_word_count = removed_word.word_map_count - 1; + if (expected_word_count > 0) { + EXPECT_EQ(1, SelectCount(kWordsTableName, kWord, word)) + << "Word '" << word << "' not removed from words table."; + ASSERT_EQ(1U, private_data->word_map_.count(word)) + << "Word '" << word << "' is gone but should still be there."; + EXPECT_EQ(expected_word_count, + private_data->word_id_history_map_[removed_word.word_id].size()) + << "Count for existing word '" << word << "' not decremented. A"; + EXPECT_EQ(static_cast<int>(expected_word_count), + SelectCount(kWordHistoryTableName, kWordID, + removed_word.word_id)) + << "Count for existing word '" << word << "' not decremented. B"; + } else { + EXPECT_EQ(0, SelectCount(kWordsTableName, kWord, word)) + << "Word '" << word << "' not removed from words table."; + EXPECT_EQ(0U, private_data->word_map_.count(word)) + << "Word '" << word << "' to be removed is still there."; + ++completely_removed; + } + } + + // Verify that the size of the in-memory and on-disk database tables have + // changed as expected. + EXPECT_EQ(std::max(0, static_cast<int>(available_words_count_) + + completely_removed - static_cast<int>(added_words_.size())), + static_cast<int>(private_data->available_words_.size())); + // The in-memory table will never get smaller as we remember the freed-up + // slots and reuse them. + EXPECT_EQ(static_cast<int>(word_list_count_) + std::max(0, + static_cast<int>(added_words_.size()) - completely_removed), + static_cast<int>(private_data->word_list_.size())); + EXPECT_EQ(static_cast<int>(word_table_count) + + static_cast<int>(added_words_.size()) - completely_removed, + SelectCount(kWordsTableName)); } -TEST_F(InMemoryURLIndexTest, CacheSaveRestore) { - ScopedTempDir temp_directory; - ASSERT_TRUE(temp_directory.CreateUniqueTempDir()); - set_history_dir(temp_directory.path()); - - URLIndexPrivateData& private_data(*GetPrivateData()); - - // Ensure that there is really something there to be saved. - EXPECT_FALSE(private_data.word_list_.empty()); - // available_words_ will already be empty since we have freshly built the - // data set for this test. - EXPECT_TRUE(private_data.available_words_.empty()); - EXPECT_FALSE(private_data.word_map_.empty()); - EXPECT_FALSE(private_data.char_word_map_.empty()); - EXPECT_FALSE(private_data.word_id_history_map_.empty()); - EXPECT_FALSE(private_data.history_id_word_map_.empty()); - EXPECT_FALSE(private_data.history_info_map_.empty()); - EXPECT_FALSE(private_data.word_starts_map_.empty()); - - // Capture the current private data for later comparison to restored data. - scoped_refptr<URLIndexPrivateData> old_data(private_data.Duplicate()); - - // Save then restore our private data. - CacheFileSaverObserver save_observer(&message_loop_); - url_index_->set_save_cache_observer(&save_observer); - PostSaveToCacheFileTask(); - message_loop_.Run(); - EXPECT_TRUE(save_observer.succeeded_); - - // Clear and then prove it's clear before restoring. - ClearPrivateData(); - EXPECT_TRUE(private_data.word_list_.empty()); - EXPECT_TRUE(private_data.available_words_.empty()); - EXPECT_TRUE(private_data.word_map_.empty()); - EXPECT_TRUE(private_data.char_word_map_.empty()); - EXPECT_TRUE(private_data.word_id_history_map_.empty()); - EXPECT_TRUE(private_data.history_id_word_map_.empty()); - EXPECT_TRUE(private_data.history_info_map_.empty()); - EXPECT_TRUE(private_data.word_starts_map_.empty()); - - CacheFileReaderObserver read_observer(&message_loop_); - url_index_->set_restore_cache_observer(&read_observer); - PostRestoreFromCacheFileTask(); - message_loop_.Run(); - EXPECT_TRUE(read_observer.succeeded_); - - URLIndexPrivateData& new_data(*GetPrivateData()); - - // Compare the captured and restored for equality. - ExpectPrivateDataEqual(*old_data, new_data); +int CacheChecker::SelectCount(const char* table) { + std::string sql(StringPrintf("SELECT COUNT(*) FROM %s", table)); + sql::Statement statement(db_->GetUniqueStatement(sql.c_str())); + return (statement.Step()) ? statement.ColumnInt(0) : -1; } -class InMemoryURLIndexCacheTest : public testing::Test { +int CacheChecker::SelectCount(const char* table, + const char* column, + const string16& word) { + std::string sql(StringPrintf("SELECT COUNT(*) FROM %s WHERE %s = ?", + table, column)); + sql::Statement statement(db_->GetUniqueStatement(sql.c_str())); + statement.BindString16(0, word); + return (statement.Step()) ? statement.ColumnInt(0) : -1; +} + +int CacheChecker::SelectCount( + const char* table, + const char* column, + WordID word_id) { + std::string sql(StringPrintf("SELECT COUNT(*) FROM %s WHERE %s = ?", + table, column)); + sql::Statement statement(db_->GetUniqueStatement(sql.c_str())); + statement.BindInt(0, word_id); + return (statement.Step()) ? statement.ColumnInt(0) : -1; +} + +int CacheChecker::SelectCount(const char* table, + const char* column, + HistoryID history_id) { + std::string sql(StringPrintf("SELECT COUNT(*) FROM %s WHERE %s = ?", + table, column)); + sql::Statement statement(db_->GetUniqueStatement(sql.c_str())); + statement.BindInt(0, history_id); + return (statement.Step()) ? statement.ColumnInt(0) : -1; +} + +// InMemoryURLIndexCacheTest --------------------------------------------------- + +void InMemoryURLIndexCacheTest::SetUp() { + profile_.reset(new CacheTestingProfile); + InMemoryURLIndexTestBase::SetUp(); + LoadIndex(); + DCHECK(url_index_->index_available()); +} + +FilePath::StringType InMemoryURLIndexCacheTest::TestDBName() const { + return FILE_PATH_LITERAL("url_history_provider_test.db.txt"); +} + +TEST_F(InMemoryURLIndexCacheTest, CacheAddRemove) { + // Initialize the cache then add and remove some history items. + const URLID kNewRowID = 250; + URLRow new_row(MakeURLRowWithID("http://www.frank-and-earnest.com/", + "Frank and Earnest Go Crash in Washington", + 10, 0, 5, kNewRowID)); + + { + // Add a new URL. + String16Set existing_words(String16SetFromString16( + UTF8ToUTF16("http www and com crash in"), NULL)); + // New words: frank, earnest, go, washington. + String16Set added_words(String16SetFromString16( + UTF8ToUTF16("frank earnest go washington"), NULL)); + String16Set removed_words; + CacheChecker cache_checker(url_index_, CacheChecker::URL_ADDED, + kNewRowID, existing_words, added_words, removed_words); + UpdateURL(new_row); + } + + URLRow old_row(new_row); + { + // Update an existing URL resulting in a net addition of words. + old_row.set_title(UTF8ToUTF16( + "Frank and Earnest Go Crazy in Draper Utah USA")); + String16Set existing_words(String16SetFromString16( + UTF8ToUTF16("http www and com in frank earnest go"), NULL)); + String16Set added_words(String16SetFromString16( + UTF8ToUTF16("crazy draper utah usa"), NULL)); + String16Set removed_words(String16SetFromString16( + UTF8ToUTF16("washington crash"), NULL)); + CacheChecker cache_checker(url_index_, CacheChecker::URL_UPDATED, + kNewRowID, existing_words, added_words, removed_words); + UpdateURL(old_row); + } + + { + // Update an existing URL resulting in a net removal of words. + old_row.set_title(UTF8ToUTF16("Frank and Earnest Go Crazy Crazy")); + String16Set existing_words(String16SetFromString16( + UTF8ToUTF16("http www and com frank earnest go crazy"), NULL)); + String16Set added_words; + String16Set removed_words(String16SetFromString16( + UTF8ToUTF16("in draper utah usa"), NULL)); + CacheChecker cache_checker(url_index_, CacheChecker::URL_UPDATED, + kNewRowID, existing_words, added_words, removed_words); + UpdateURL(old_row); + } + + { + // Delete an existing URL. + old_row.set_title(UTF8ToUTF16("Frank and Earnest Go Crazy Crazy")); + String16Set existing_words; + String16Set added_words; + String16Set removed_words(String16SetFromString16( + UTF8ToUTF16("http www and com frank earnest go crazy"), NULL)); + CacheChecker cache_checker(url_index_, CacheChecker::URL_REMOVED, + kNewRowID, existing_words, added_words, removed_words); + DeleteURL(old_row.url()); + } +} + +// InterposingCacheDatabase ---------------------------------------------------- + +// This class allows certain InMemoryURLCacheDatabase methods to be intercepted +// for purposes of recording or simulating failures. +class InterposingCacheDatabase : public InMemoryURLCacheDatabase { public: - InMemoryURLIndexCacheTest() {} + InterposingCacheDatabase(); + + virtual bool Reset() OVERRIDE; + virtual bool Refresh(const URLIndexPrivateData& index_data) OVERRIDE; + virtual void AddWordToWordsTask(WordID word_id, + const string16& uni_word) OVERRIDE; + + void set_simulate_update_fail(bool fail) { simulate_update_fail_ = fail; } + void set_simulate_refresh_fail(bool fail) { simulate_refresh_fail_ = fail; } + void set_tracking_calls(bool tracking) { tracking_calls_ = tracking; } + int reset_calling_sequence() const { return reset_calling_sequence_; } + int refresh_calling_sequence() const { return refresh_calling_sequence_; } + void set_update_error(int error) { update_error_ = error; } + bool failure_occurred() const { return update_error_ != SQLITE_OK; } + + private: + virtual ~InterposingCacheDatabase() {} + + bool simulate_update_fail_; + bool simulate_refresh_fail_; + int next_calling_sequence_; + bool tracking_calls_; + int reset_calling_sequence_; + int refresh_calling_sequence_; +}; - protected: - virtual void SetUp() OVERRIDE; +InterposingCacheDatabase::InterposingCacheDatabase() + : simulate_update_fail_(false), + simulate_refresh_fail_(false), + next_calling_sequence_(0), + tracking_calls_(false), + reset_calling_sequence_(-1), + refresh_calling_sequence_(-1) { +} + +bool InterposingCacheDatabase::Reset() { + bool success = InMemoryURLCacheDatabase::Reset(); + if (tracking_calls_) + reset_calling_sequence_ = next_calling_sequence_++; + return success; +} + +bool InterposingCacheDatabase::Refresh( + const URLIndexPrivateData& index_data) { + bool success = + !simulate_refresh_fail_ && InMemoryURLCacheDatabase::Refresh(index_data); + if (tracking_calls_) + refresh_calling_sequence_ = next_calling_sequence_++; + return success; +} + +void InterposingCacheDatabase::AddWordToWordsTask( + WordID word_id, + const string16& uni_word) { + if (simulate_update_fail_) + set_update_error(SQLITE_CORRUPT); + else + InMemoryURLCacheDatabase::AddWordToWordsTask(word_id, uni_word); +} + +// IntercessionaryIndexTest ---------------------------------------------------- + +// Tests which intercede in some way with the normal operation of the index, +// its private data, and/or the cache database. +class IntercessionaryIndexTest : public InMemoryURLIndexTestBase { + public: + IntercessionaryIndexTest(); + ~IntercessionaryIndexTest() {} + + void SetUp() OVERRIDE; - // Pass-through functions to simplify our friendship with InMemoryURLIndex. - void set_history_dir(const FilePath& dir_path); - bool GetCacheFilePath(FilePath* file_path) const; + // Provides custom test data file name. + virtual FilePath::StringType TestDBName() const OVERRIDE; - ScopedTempDir temp_dir_; - scoped_ptr<InMemoryURLIndex> url_index_; + protected: + scoped_refptr<InterposingCacheDatabase> test_db_; + URLIndexPrivateData* private_data_; }; -void InMemoryURLIndexCacheTest::SetUp() { - ASSERT_TRUE(temp_dir_.CreateUniqueTempDir()); - FilePath path(temp_dir_.path()); - url_index_.reset( - new InMemoryURLIndex(NULL, path, "en,ja,hi,zh")); +IntercessionaryIndexTest::IntercessionaryIndexTest() + : test_db_(new InterposingCacheDatabase) { +} + +void IntercessionaryIndexTest::SetUp() { + // Set things up without enabling the cache database. + InMemoryURLIndexTestBase::SetUp(); + LoadIndex(); + DCHECK(url_index_->index_available()); + + // Enabled our custom database and refresh it. + FilePath dir_path; + private_data_ = GetPrivateData(); + DCHECK(private_data_->GetCacheDBPath(&dir_path)); + base::SequencedWorkerPool::SequenceToken sequence_token = + url_index_->sequence_token_for_testing(); + content::BrowserThread::GetBlockingPool()->PostWorkerTask(FROM_HERE, + base::Bind(base::IgnoreResult(&InMemoryURLCacheDatabase::Init), + test_db_, dir_path, sequence_token)); + content::BrowserThread::GetBlockingPool()->FlushForTesting(); + private_data_->SetCacheDatabaseForTesting(test_db_); + private_data_->set_cache_enabled(true); + content::BrowserThread::GetBlockingPool()->PostWorkerTask(FROM_HERE, + base::Bind(&URLIndexPrivateData::RefreshCacheTask, private_data_)); + content::BrowserThread::GetBlockingPool()->FlushForTesting(); + test_db_->set_tracking_calls(true); } -void InMemoryURLIndexCacheTest::set_history_dir(const FilePath& dir_path) { - return url_index_->set_history_dir(dir_path); +FilePath::StringType IntercessionaryIndexTest::TestDBName() const { + return FILE_PATH_LITERAL("url_history_provider_test.db.txt"); } -bool InMemoryURLIndexCacheTest::GetCacheFilePath(FilePath* file_path) const { - DCHECK(file_path); - return url_index_->GetCacheFilePath(file_path); +TEST_F(IntercessionaryIndexTest, CacheDatabaseFailure) { + // Perform an update but do not fail. + const URLID kNewRowID = 250; + URLRow new_row(MakeURLRowWithID("http://www.frank-and-earnest.com/", + "Frank and Earnest Go Crash in Washington", + 10, 0, 5, kNewRowID)); + UpdateURL(new_row); + EXPECT_EQ(-1, test_db_->reset_calling_sequence()); + EXPECT_EQ(-1, test_db_->refresh_calling_sequence()); + EXPECT_FALSE(test_db_->failure_occurred()); + EXPECT_TRUE(private_data_->cache_enabled()); + + // Perform an update that fails but remediation succeeds. + test_db_->set_simulate_update_fail(true); + URLRow old_row(new_row); + old_row.set_title(UTF8ToUTF16( + "Frank and Earnest Go Crazy in Draper Utah USA")); + content::WindowedNotificationObserver update_failure_observer( + chrome::NOTIFICATION_IN_MEMORY_URL_CACHE_DATABASE_FAILURE, + content::NotificationService::AllSources()); + UpdateURL(old_row); + update_failure_observer.Wait(); + // Wait for the pending reset and refresh to complete. + content::BrowserThread::GetBlockingPool()->FlushForTesting(); + EXPECT_EQ(0, test_db_->reset_calling_sequence()); + EXPECT_EQ(1, test_db_->refresh_calling_sequence()); + EXPECT_TRUE(test_db_->failure_occurred()); + EXPECT_TRUE(private_data_->cache_enabled()); + + // Perform an update that fails and remediation fails. + test_db_->set_simulate_update_fail(true); + test_db_->set_simulate_refresh_fail(true); + test_db_->set_update_error(SQLITE_OK); + old_row.set_title(UTF8ToUTF16( + "Frank and Earnest Light Up Dizzy World")); + content::WindowedNotificationObserver refresh_failure_observer( + chrome::NOTIFICATION_IN_MEMORY_URL_CACHE_DATABASE_FAILURE, + content::NotificationService::AllSources()); + UpdateURL(old_row); + refresh_failure_observer.Wait(); + // Wait for the pending reset and refresh to complete. + content::BrowserThread::GetBlockingPool()->FlushForTesting(); + EXPECT_EQ(2, test_db_->reset_calling_sequence()); + EXPECT_EQ(3, test_db_->refresh_calling_sequence()); + EXPECT_TRUE(test_db_->failure_occurred()); + EXPECT_FALSE(private_data_->cache_enabled()); } -TEST_F(InMemoryURLIndexCacheTest, CacheFilePath) { - FilePath expectedPath = - temp_dir_.path().Append(FILE_PATH_LITERAL("History Provider Cache")); - std::vector<FilePath::StringType> expected_parts; - expectedPath.GetComponents(&expected_parts); - FilePath full_file_path; - ASSERT_TRUE(GetCacheFilePath(&full_file_path)); - std::vector<FilePath::StringType> actual_parts; - full_file_path.GetComponents(&actual_parts); - ASSERT_EQ(expected_parts.size(), actual_parts.size()); - size_t count = expected_parts.size(); - for (size_t i = 0; i < count; ++i) - EXPECT_EQ(expected_parts[i], actual_parts[i]); - // Must clear the history_dir_ to satisfy the dtor's DCHECK. - set_history_dir(FilePath()); +TEST_F(IntercessionaryIndexTest, ShutdownDuringCacheRefresh) { + // Pretend that the cache is dysfunctional. + private_data_->set_cache_enabled(false); + // Kick off a refresh an immediately shut down. + url_index_->PostRefreshCacheTask(); + url_index_->Shutdown(); + // The index should still be viable but the cache should have been shut down. + EXPECT_FALSE(private_data_->cache_enabled()); + EXPECT_TRUE(url_index_->index_available()); } } // namespace history diff --git a/chrome/browser/history/in_memory_url_index_unittest_base.cc b/chrome/browser/history/in_memory_url_index_unittest_base.cc new file mode 100644 index 0000000..acc4c8e --- /dev/null +++ b/chrome/browser/history/in_memory_url_index_unittest_base.cc @@ -0,0 +1,131 @@ +// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/history/in_memory_url_index_unittest_base.h" + +#include "base/file_path.h" +#include "base/file_util.h" +#include "base/path_service.h" +#include "base/string_split.h" +#include "chrome/browser/history/history.h" +#include "chrome/browser/history/history_backend.h" +#include "chrome/browser/history/history_service_factory.h" +#include "chrome/browser/history/in_memory_url_cache_database.h" +#include "chrome/browser/history/in_memory_url_index.h" +#include "chrome/browser/history/url_index_private_data.h" +#include "chrome/common/chrome_paths.h" +#include "chrome/test/base/ui_test_utils.h" +#include "sql/connection.h" +#include "sql/statement.h" +#include "sql/transaction.h" + +namespace history { + +InMemoryURLIndexTestBase::InMemoryURLIndexTestBase() + : ui_thread_(content::BrowserThread::UI, &message_loop_), + db_thread_(content::BrowserThread::DB) { +} + +InMemoryURLIndexTestBase::~InMemoryURLIndexTestBase() {} + +void InMemoryURLIndexTestBase::SetUp() { + if (!profile_.get()) + profile_.reset(new TestingProfile); + db_thread_.Start(); + // We cannot access the database until the backend has been loaded. + profile_->CreateHistoryService(true, false); + HistoryService* history_service = + HistoryServiceFactory::GetForProfile(profile_.get(), + Profile::EXPLICIT_ACCESS); + ASSERT_TRUE(history_service); + url_index_ = history_service->InMemoryIndex(); + BlockUntilIndexLoaded(); + DCHECK(url_index_->index_available()); + profile_->CreateBookmarkModel(true); + HistoryBackend* backend = history_service->get_history_backend_for_testing(); + history_database_ = backend->db(); + ui_test_utils::WaitForHistoryToLoad(history_service); + DCHECK(history_service->backend_loaded()); + + // Create and populate a working copy of the URL history database from the + // data contained in the file specified by the TestDBName() function. + // TODO(mrossetti): Adopt sqlite3_ functions for performing the database + // initialization. See http://crbug.com/137352. + FilePath test_file_path; + PathService::Get(chrome::DIR_TEST_DATA, &test_file_path); + test_file_path = test_file_path.Append(FILE_PATH_LITERAL("History")); + test_file_path = test_file_path.Append(TestDBName()); + EXPECT_TRUE(file_util::PathExists(test_file_path)); + + std::string sql_command_buffer; + file_util::ReadFileToString(test_file_path, &sql_command_buffer); + std::vector<std::string> sql_commands; + base::SplitStringDontTrim(sql_command_buffer, '\n', &sql_commands); + sql::Connection* db(history_database_->get_db_for_testing()); + for (std::vector<std::string>::const_iterator i = sql_commands.begin(); + i != sql_commands.end(); ++i) { + // We only process lines which begin with a upper-case letter. + // TODO(mrossetti): Can iswupper() be used here? + const std::string& sql_command(*i); + if (sql_command[0] >= 'A' && sql_command[0] <= 'Z') + EXPECT_TRUE(db->Execute(sql_command.c_str())); + } + + // Update the last_visit_time table column such that it represents a time + // relative to 'now'. + // TODO(mrossetti): Do an UPDATE to alter the days-ago. + // See http://crbug.com/137352. + sql::Statement statement(db->GetUniqueStatement( + "SELECT" HISTORY_URL_ROW_FIELDS "FROM urls")); + EXPECT_TRUE(statement.is_valid()); + base::Time time_right_now = base::Time::NowFromSystemTime(); + base::TimeDelta day_delta = base::TimeDelta::FromDays(1); + while (statement.Step()) { + URLRow row; + history_database_->FillURLRow(statement, &row); + row.set_last_visit(time_right_now - + day_delta * row.last_visit().ToInternalValue()); + history_database_->UpdateURLRow(row.id(), row); + } +} + +void InMemoryURLIndexTestBase::TearDown() { + MessageLoop::current()->RunAllPending(); + message_loop_.RunAllPending(); + db_thread_.Stop(); +} + +void InMemoryURLIndexTestBase::LoadIndex() { + url_index_->ScheduleRebuildFromHistory(); + BlockUntilIndexLoaded(); +} + +void InMemoryURLIndexTestBase::BlockUntilIndexLoaded() { + if (url_index_->index_available()) + return; + InMemoryURLIndex::Observer observer(url_index_); + MessageLoop::current()->Run(); +} + +URLIndexPrivateData* InMemoryURLIndexTestBase::GetPrivateData() { + return url_index_->private_data(); +} + +bool InMemoryURLIndexTestBase::UpdateURL(const URLRow& row) { + bool success = GetPrivateData()->UpdateURL(row); + content::BrowserThread::GetBlockingPool()->FlushForTesting(); + return success; +} + +bool InMemoryURLIndexTestBase::DeleteURL(const GURL& url) { + bool success = GetPrivateData()->DeleteURL(url); + content::BrowserThread::GetBlockingPool()->FlushForTesting(); + return success; +} + +bool InMemoryURLIndexTestBase::GetCacheDBPath(FilePath* file_path) { + return GetPrivateData()->GetCacheDBPath(file_path); +} + +} // namespace history diff --git a/chrome/browser/history/in_memory_url_index_unittest_base.h b/chrome/browser/history/in_memory_url_index_unittest_base.h new file mode 100644 index 0000000..07b33cd --- /dev/null +++ b/chrome/browser/history/in_memory_url_index_unittest_base.h @@ -0,0 +1,88 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_BROWSER_HISTORY_IN_MEMORY_URL_INDEX_UNITTEST_BASE_H_ +#define CHROME_BROWSER_HISTORY_IN_MEMORY_URL_INDEX_UNITTEST_BASE_H_ + +#include "base/file_util.h" +#include "base/memory/scoped_ptr.h" +#include "base/message_loop.h" +#include "base/scoped_temp_dir.h" +#include "chrome/browser/history/history_database.h" +#include "chrome/browser/history/in_memory_database.h" +#include "chrome/browser/history/in_memory_url_index.h" +#include "chrome/test/base/testing_profile.h" +#include "content/public/test/test_browser_thread.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace history { + +class HistoryDatabase; + +// A base class for unit tests that exercise the InMemoryURLIndex and the +// HistoryQuickProvider. Provides initialization of the index using data +// contained in a test file. +// +// The test version of the history url database table ('url') is contained in +// a database file created from a text file as specified by the +// TestDBName() method overridden by subclasses. The only difference between +// the table specified in this test file and a live 'urls' table from a +// profile is that the last_visit_time column in the test table contains a +// number specifying the number of days relative to 'today' to which the +// visit time of the URL will be set during the test setup stage. +// +// The format of the test database text file is of a SQLite .dump file. +// Note that only lines whose first character is an upper-case letter are +// processed when creating the test database. +// +class InMemoryURLIndexTestBase : public testing::Test { + protected: + InMemoryURLIndexTestBase(); + virtual ~InMemoryURLIndexTestBase(); + + // Specifies the test data file name used by the subclass. The specified file + // must reside in the path given by chrome::DIR_TEST_DATA. + virtual FilePath::StringType TestDBName() const = 0; + + // Fills the HistoryBackend with test data from the test data file and creates + // the InMemoryURLIndex instance, but does not fill it with data. Call + // LoadIndex() after calling SetUp() To fill the InMemoryURLIndex instance + // with the test data. + // NOTE: By default, TestingProfile does not enable the cache database + // (InMemoryURLCacheDatabase). If a test relies on the cache database + // having been enabled then that test should subclass TestingProfile + // and provide an override of InitHistoryService(...) that causes + // the cache database to be created and initialized. For an example, + // see CacheTestingProfile in in_memory_url_index_unittest.cc. + virtual void SetUp() OVERRIDE; + virtual void TearDown() OVERRIDE; + + // Loads the InMemoryURLIndex instance with data from the HistoryBackend. + // Blocks until the load completes. Completion does not imply success. + void LoadIndex(); + + // Blocks the caller until the load sequence for the index has completed. + // Note that load completion does not imply success. + void BlockUntilIndexLoaded(); + + // Pass-through function to simplify our friendship with InMemoryURLIndex. + URLIndexPrivateData* GetPrivateData(); + + // Pass-through functions to simplify our friendship with URLIndexPrivateData. + bool UpdateURL(const URLRow& row); + bool DeleteURL(const GURL& url); + bool GetCacheDBPath(FilePath* file_path); + + InMemoryURLIndex* url_index_; + HistoryDatabase* history_database_; + + MessageLoopForUI message_loop_; + content::TestBrowserThread ui_thread_; + content::TestBrowserThread db_thread_; + scoped_ptr<TestingProfile> profile_; +}; + +} // namespace history + +#endif // CHROME_BROWSER_HISTORY_IN_MEMORY_URL_INDEX_UNITTEST_BASE_H_ diff --git a/chrome/browser/history/scored_history_match.h b/chrome/browser/history/scored_history_match.h index 38d12ba..e3c33da 100644 --- a/chrome/browser/history/scored_history_match.h +++ b/chrome/browser/history/scored_history_match.h @@ -22,14 +22,11 @@ struct ScoredHistoryMatch : public history::HistoryMatch { ScoredHistoryMatch(); // Required by STL. // Creates a new match with a raw score calculated for the history item given - // in |row| by first determining if all of the terms in |terms_vector| occur - // in |row| and, if so, calculating a raw score based on 1) starting position - // of each term in the user input, 2) completeness of each term's match, - // 3) ordering of the occurrence of each term (i.e. they appear in order), - // 4) last visit time (compared to |now|), and 5) number of visits. - // This raw score allows the results to be ordered and can be used - // to influence the final score calculated by the client of this - // index. + // in |row|. First determines if the row qualifies by seeing if all of the + // terms in |terms_vector| occur in |row|. If so, calculates a raw score. + // This raw score allows the results to be ordered and can be used to + // influence the final score calculated by the client of this index. + // If the row does not qualify the raw score will be 0. ScoredHistoryMatch(const URLRow& row, const string16& lower_string, const String16Vector& terms_vector, diff --git a/chrome/browser/history/scored_history_match_unittest.cc b/chrome/browser/history/scored_history_match_unittest.cc index d7c9f6b..3f80d64 100644 --- a/chrome/browser/history/scored_history_match_unittest.cc +++ b/chrome/browser/history/scored_history_match_unittest.cc @@ -14,12 +14,12 @@ namespace history { class ScoredHistoryMatchTest : public testing::Test { protected: // Convenience function to create a URLRow with basic data for |url|, |title|, - // |visit_count|, and |typed_count|. |last_visit_ago| gives the number of - // days from now to set the URL's last_visit. + // |visit_count|, and |typed_count|. |days_since_last_visit| gives the number + // of days ago to which to set the URL's last_visit. URLRow MakeURLRow(const char* url, const char* title, int visit_count, - int last_visit_ago, + int days_since_last_visit, int typed_count); // Convenience functions for easily creating vectors of search terms. @@ -38,28 +38,28 @@ class ScoredHistoryMatchTest : public testing::Test { URLRow ScoredHistoryMatchTest::MakeURLRow(const char* url, const char* title, int visit_count, - int last_visit_ago, + int days_since_last_visit, int typed_count) { URLRow row(GURL(url), 0); - row.set_title(UTF8ToUTF16(title)); + row.set_title(ASCIIToUTF16(title)); row.set_visit_count(visit_count); row.set_typed_count(typed_count); row.set_last_visit(base::Time::NowFromSystemTime() - - base::TimeDelta::FromDays(last_visit_ago)); + base::TimeDelta::FromDays(days_since_last_visit)); return row; } String16Vector ScoredHistoryMatchTest::Make1Term(const char* term) const { String16Vector original_terms; - original_terms.push_back(UTF8ToUTF16(term)); + original_terms.push_back(ASCIIToUTF16(term)); return original_terms; } String16Vector ScoredHistoryMatchTest::Make2Terms(const char* term_1, const char* term_2) const { String16Vector original_terms; - original_terms.push_back(UTF8ToUTF16(term_1)); - original_terms.push_back(UTF8ToUTF16(term_2)); + original_terms.push_back(ASCIIToUTF16(term_1)); + original_terms.push_back(ASCIIToUTF16(term_2)); return original_terms; } @@ -82,6 +82,7 @@ TEST_F(ScoredHistoryMatchTest, Scoring) { // to calculate last visit time when building a row. base::Time now = base::Time::NowFromSystemTime(); RowWordStarts word_starts; + // Test scores based on position. // TODO(mpearson): Test new_scoring if we're actually going to turn it // on by default. This requires setting word_starts, which isn't done @@ -91,31 +92,37 @@ TEST_F(ScoredHistoryMatchTest, Scoring) { ScoredHistoryMatch scored_b(row_a, ASCIIToUTF16("bcd"), Make1Term("bcd"), word_starts, now); EXPECT_GT(scored_a.raw_score, scored_b.raw_score); + // Test scores based on length. ScoredHistoryMatch scored_c(row_a, ASCIIToUTF16("abcd"), Make1Term("abcd"), word_starts, now); EXPECT_LT(scored_a.raw_score, scored_c.raw_score); + // Test scores based on order. ScoredHistoryMatch scored_d(row_a, ASCIIToUTF16("abcdef"), Make2Terms("abc", "def"), word_starts, now); ScoredHistoryMatch scored_e(row_a, ASCIIToUTF16("def abc"), Make2Terms("def", "abc"), word_starts, now); EXPECT_GT(scored_d.raw_score, scored_e.raw_score); + // Test scores based on visit_count. URLRow row_b(MakeURLRow("http://abcdef", "fedcba", 10, 30, 1)); ScoredHistoryMatch scored_f(row_b, ASCIIToUTF16("abc"), Make1Term("abc"), word_starts, now); EXPECT_GT(scored_f.raw_score, scored_a.raw_score); + // Test scores based on last_visit. URLRow row_c(MakeURLRow("http://abcdef", "fedcba", 3, 10, 1)); ScoredHistoryMatch scored_g(row_c, ASCIIToUTF16("abc"), Make1Term("abc"), word_starts, now); EXPECT_GT(scored_g.raw_score, scored_a.raw_score); + // Test scores based on typed_count. URLRow row_d(MakeURLRow("http://abcdef", "fedcba", 3, 30, 10)); ScoredHistoryMatch scored_h(row_d, ASCIIToUTF16("abc"), Make1Term("abc"), word_starts, now); EXPECT_GT(scored_h.raw_score, scored_a.raw_score); + // Test scores based on a terms appearing multiple times. URLRow row_i(MakeURLRow("http://csi.csi.csi/csi_csi", "CSI Guide to CSI Las Vegas, CSI New York, CSI Provo", 3, 30, 10)); diff --git a/chrome/browser/history/top_sites_unittest.cc b/chrome/browser/history/top_sites_unittest.cc index 4524f92..3a06338 100644 --- a/chrome/browser/history/top_sites_unittest.cc +++ b/chrome/browser/history/top_sites_unittest.cc @@ -1186,6 +1186,7 @@ TEST_F(TopSitesTest, CreateTopSitesThenHistory) { false); // Create TopSites, but not History. + HistoryServiceFactory::GetInstance()->SetTestingFactory(profile(), NULL); profile()->CreateTopSites(); WaitForTopSites(); EXPECT_FALSE(IsTopSitesLoaded()); diff --git a/chrome/browser/history/url_index_private_data.cc b/chrome/browser/history/url_index_private_data.cc index 13bb574..a8878d6 100644 --- a/chrome/browser/history/url_index_private_data.cc +++ b/chrome/browser/history/url_index_private_data.cc @@ -14,44 +14,533 @@ #include "base/basictypes.h" #include "base/file_util.h" #include "base/i18n/case_conversion.h" -#include "base/metrics/histogram.h" #include "base/string_util.h" -#include "base/time.h" #include "base/utf_string_conversions.h" #include "chrome/browser/autocomplete/autocomplete_provider.h" -#include "chrome/browser/autocomplete/url_prefix.h" #include "chrome/browser/history/history_database.h" -#include "chrome/browser/history/in_memory_url_index.h" +#include "chrome/browser/history/in_memory_url_cache_database.h" +#include "chrome/common/chrome_constants.h" +#include "chrome/common/url_constants.h" #include "content/public/browser/browser_thread.h" #include "content/public/browser/notification_details.h" #include "content/public/browser/notification_service.h" #include "content/public/browser/notification_source.h" #include "net/base/net_util.h" -#include "third_party/protobuf/src/google/protobuf/repeated_field.h" -using google::protobuf::RepeatedField; -using google::protobuf::RepeatedPtrField; -using in_memory_url_index::InMemoryURLIndexCacheItem; +using content::BrowserThread; namespace history { -typedef imui::InMemoryURLIndexCacheItem_WordListItem WordListItem; -typedef imui::InMemoryURLIndexCacheItem_WordMapItem_WordMapEntry WordMapEntry; -typedef imui::InMemoryURLIndexCacheItem_WordMapItem WordMapItem; -typedef imui::InMemoryURLIndexCacheItem_CharWordMapItem CharWordMapItem; -typedef imui::InMemoryURLIndexCacheItem_CharWordMapItem_CharWordMapEntry - CharWordMapEntry; -typedef imui::InMemoryURLIndexCacheItem_WordIDHistoryMapItem - WordIDHistoryMapItem; -typedef imui:: - InMemoryURLIndexCacheItem_WordIDHistoryMapItem_WordIDHistoryMapEntry - WordIDHistoryMapEntry; -typedef imui::InMemoryURLIndexCacheItem_HistoryInfoMapItem HistoryInfoMapItem; -typedef imui::InMemoryURLIndexCacheItem_HistoryInfoMapItem_HistoryInfoMapEntry - HistoryInfoMapEntry; -typedef imui::InMemoryURLIndexCacheItem_WordStartsMapItem WordStartsMapItem; -typedef imui::InMemoryURLIndexCacheItem_WordStartsMapItem_WordStartsMapEntry - WordStartsMapEntry; +// Comparison function for sorting search terms by descending length. +bool LengthGreater(const string16& string_a, const string16& string_b) { + return string_a.length() > string_b.length(); +} + +// Initializes a whitelist of URL schemes. +void InitializeSchemeWhitelist(std::set<std::string>* whitelist) { + DCHECK(whitelist); + if (!whitelist->empty()) + return; // Nothing to do, already initialized. + whitelist->insert(std::string(chrome::kAboutScheme)); + whitelist->insert(std::string(chrome::kChromeUIScheme)); + whitelist->insert(std::string(chrome::kFileScheme)); + whitelist->insert(std::string(chrome::kFtpScheme)); + whitelist->insert(std::string(chrome::kHttpScheme)); + whitelist->insert(std::string(chrome::kHttpsScheme)); + whitelist->insert(std::string(chrome::kMailToScheme)); +} + +// CacheTransaction ------------------------------------------------------------ + +// Simple automatic helper class encapsulating calls to BeginTransaction/ +// CommitTransaction. +class CacheTransaction { + public: + explicit CacheTransaction(InMemoryURLCacheDatabase* db); + ~CacheTransaction(); + + private: + InMemoryURLCacheDatabase* db_; + + DISALLOW_COPY_AND_ASSIGN(CacheTransaction); +}; + +CacheTransaction::CacheTransaction(InMemoryURLCacheDatabase* db) + : db_(db) { + if (db_) + db_->BeginTransaction(); +} + +CacheTransaction::~CacheTransaction() { + if (db_) + db_->CommitTransaction(); +} + +// URLIndexPrivateData Public (but only to InMemoryURLIndex) Functions --------- + +URLIndexPrivateData::URLIndexPrivateData(const FilePath& history_dir, + const std::string& languages) + : history_dir_(history_dir), + languages_(languages), + cache_enabled_(true), + initialization_lock_(), + shutdown_(false), + pre_filter_item_count_(0), + post_filter_item_count_(0), + post_scoring_item_count_(0) { + InitializeSchemeWhitelist(&scheme_whitelist_); +} + +bool URLIndexPrivateData::Init( + base::SequencedWorkerPool::SequenceToken sequence_token) { + // Since init occurs on the DB thread, it is possible that the profile was + // told to shutdown before this task had a chance to kick off. + if (shutdown_) + return false; + cache_db_ = new InMemoryURLCacheDatabase(); + FilePath dir_path; + base::AutoLock lock(initialization_lock_); + set_cache_enabled(GetCacheDBPath(&dir_path) && + cache_db_->Init(dir_path, sequence_token)); + return cache_enabled(); +} + +void URLIndexPrivateData::Reset() { + Clear(); + if (cache_enabled()) + cache_db_->Reset(); +} + +bool URLIndexPrivateData::Empty() const { + return history_info_map_.empty(); +} + +URLIndexPrivateData* URLIndexPrivateData::Snapshot() const { + return new URLIndexPrivateData(*this); +} + +void URLIndexPrivateData::Shutdown() { + shutdown_ = true; + base::AutoLock lock(initialization_lock_); + if (cache_enabled()) + cache_db_->Shutdown(); +} + +// Helper predicates for ValidateConsistency. +template <typename Pair> +struct SelectFirst : std::unary_function<Pair, typename Pair::first_type> { + const typename Pair::first_type& operator()(const Pair& x) const { + return x.first; + } +}; + +template <typename Pair> +struct SelectSecond : std::unary_function<Pair, typename Pair::second_type> { + const typename Pair::second_type& operator()(const Pair& x) const { + return x.second; + } +}; + +template <typename SourcePair, typename Target> +struct TargetInserter : std::unary_function<SourcePair, void> { + explicit TargetInserter(Target* target) : target_(target) {} + + void operator()(const SourcePair& source) { + target_->insert(source.second.begin(), source.second.end()); + } + + Target* target_; +}; + +bool URLIndexPrivateData::ValidateConsistency() const { + // Scope things so that a large data set doesn't unnecessarily accumulate and + // hang around. + { + // Make a set of WordIDs from word_map_. + WordIDSet word_id_set_a; + std::transform(word_map_.begin(), word_map_.end(), + std::inserter(word_id_set_a, word_id_set_a.begin()), + SelectSecond<WordMap::value_type>()); + + // Compare word_map_'s word set to the words from word_id_history_map_. + { + WordIDSet word_id_set_b; + std::transform(word_id_history_map_.begin(), word_id_history_map_.end(), + std::inserter(word_id_set_b, word_id_set_b.begin()), + SelectFirst<WordIDHistoryMap::value_type>()); + WordIDSet leftovers; + std::set_symmetric_difference( + word_id_set_a.begin(), word_id_set_a.end(), + word_id_set_b.begin(), word_id_set_b.end(), + std::inserter(leftovers, leftovers.begin())); + if (!leftovers.empty()) + return false; + } + + // Compare word_map_'s word set to the words from history_id_word_map_. + { + WordIDSet word_id_set_b; + std::for_each(history_id_word_map_.begin(), history_id_word_map_.end(), + TargetInserter<HistoryIDWordMap::value_type, + WordIDSet>(&word_id_set_b)); + WordIDSet leftovers; + std::set_symmetric_difference( + word_id_set_a.begin(), word_id_set_a.end(), + word_id_set_b.begin(), word_id_set_b.end(), + std::inserter(leftovers, leftovers.begin())); + if (!leftovers.empty()) + return false; + } + + // Compare word_map_'s word set to the words from char_word_map_. + { + WordIDSet word_id_set_b; + std::for_each(char_word_map_.begin(), char_word_map_.end(), + TargetInserter<CharWordIDMap::value_type, + WordIDSet>(&word_id_set_b)); + WordIDSet leftovers; + std::set_symmetric_difference( + word_id_set_a.begin(), word_id_set_a.end(), + word_id_set_b.begin(), word_id_set_b.end(), + std::inserter(leftovers, leftovers.begin())); + if (!leftovers.empty()) + return false; + } + + // Intersect available_words_ with set of WordIDs (created above from + // word_map_) and test for no common WordIDs. + { + WordIDSet leftovers; + std::set_intersection(word_id_set_a.begin(), word_id_set_a.end(), + available_words_.begin(), available_words_.end(), + std::inserter(leftovers, leftovers.begin())); + if (!leftovers.empty()) + return false; + } + } + + { + // Make a set of HistoryIDs from history_info_map_. + HistoryIDSet history_id_set_a; + std::transform(history_info_map_.begin(), history_info_map_.end(), + std::inserter(history_id_set_a, history_id_set_a.begin()), + SelectFirst<HistoryInfoMap::value_type>()); + + // Compare history_info_map_'s set to word_id_history_map_'s HistoryIDs. + { + HistoryIDSet history_id_set_b; + std::transform(history_info_map_.begin(), history_info_map_.end(), + std::inserter(history_id_set_b, history_id_set_b.begin()), + SelectFirst<HistoryInfoMap::value_type>()); + HistoryIDSet leftovers; + std::set_symmetric_difference( + history_id_set_a.begin(), history_id_set_a.end(), + history_id_set_b.begin(), history_id_set_b.end(), + std::inserter(leftovers, leftovers.begin())); + if (!leftovers.empty()) + return false; + } + + // Compare history_info_map_'s set to word_id_history_map_'s HistoryIDs. + { + HistoryIDSet history_id_set_b; + std::for_each(word_id_history_map_.begin(), word_id_history_map_.end(), + TargetInserter<WordIDHistoryMap::value_type, + HistoryIDSet>(&history_id_set_b)); + HistoryIDSet leftovers; + std::set_symmetric_difference( + history_id_set_a.begin(), history_id_set_a.end(), + history_id_set_b.begin(), history_id_set_b.end(), + std::inserter(leftovers, leftovers.begin())); + if (!leftovers.empty()) + return false; + } + + // Compare history_info_map_'s set to word_starts_map_'s HistoryIDs. + { + HistoryIDSet history_id_set_b; + std::transform(word_starts_map_.begin(), word_starts_map_.end(), + std::inserter(history_id_set_b, history_id_set_b.begin()), + SelectFirst<WordStartsMap::value_type>()); + HistoryIDSet leftovers; + std::set_symmetric_difference( + history_id_set_a.begin(), history_id_set_a.end(), + history_id_set_b.begin(), history_id_set_b.end(), + std::inserter(leftovers, leftovers.begin())); + if (!leftovers.empty()) + return false; + } + } + + // Make sets of words from word_list_ and word_map_ and test for equality. + { + String16Set word_list_set; + std::copy(word_list_.begin(), word_list_.end(), + std::inserter(word_list_set, word_list_set.end())); + String16Set word_map_set; + std::transform(word_map_.begin(), word_map_.end(), + std::inserter(word_map_set, word_map_set.begin()), + SelectFirst<WordMap::value_type>()); + if (word_list_set != word_map_set) + return false; + } + + return true; +} + +ScoredHistoryMatches URLIndexPrivateData::HistoryItemsForTerms( + const string16& search_string) { + pre_filter_item_count_ = 0; + post_filter_item_count_ = 0; + post_scoring_item_count_ = 0; + // The search string we receive may contain escaped characters. For reducing + // the index we need individual, lower-cased words, ignoring escapings. For + // the final filtering we need whitespace separated substrings possibly + // containing escaped characters. + string16 lower_raw_string(base::i18n::ToLower(search_string)); + string16 lower_unescaped_string = + net::UnescapeURLComponent(lower_raw_string, + net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS); + // Extract individual 'words' (as opposed to 'terms'; see below) from the + // search string. When the user types "colspec=ID%20Mstone Release" we get + // four 'words': "colspec", "id", "mstone" and "release". + String16Vector lower_words( + history::String16VectorFromString16(lower_unescaped_string, false, NULL)); + ScoredHistoryMatches scored_items; + + // Do nothing if we have indexed no words (probably because we've not been + // initialized yet) or the search string has no words. + if (word_list_.empty() || lower_words.empty()) { + search_term_cache_.clear(); // Invalidate the term cache. + return scored_items; + } + + // Reset used_ flags for search_term_cache_. We use a basic mark-and-sweep + // approach. + ResetSearchTermCache(); + + HistoryIDSet history_id_set = HistoryIDSetFromWords(lower_words); + + // Trim the candidate pool if it is large. Note that we do not filter out + // items that do not contain the search terms as proper substrings -- doing + // so is the performance-costly operation we are trying to avoid in order + // to maintain omnibox responsiveness. + const size_t kItemsToScoreLimit = 500; + pre_filter_item_count_ = history_id_set.size(); + // If we trim the results set we do not want to cache the results for next + // time as the user's ultimately desired result could easily be eliminated + // in this early rough filter. + bool was_trimmed = (pre_filter_item_count_ > kItemsToScoreLimit); + if (was_trimmed) { + HistoryIDVector history_ids; + std::copy(history_id_set.begin(), history_id_set.end(), + std::back_inserter(history_ids)); + // Trim down the set by sorting by typed-count, visit-count, and last + // visit. + HistoryItemFactorGreater + item_factor_functor(history_info_map_); + std::partial_sort(history_ids.begin(), + history_ids.begin() + kItemsToScoreLimit, + history_ids.end(), + item_factor_functor); + history_id_set.clear(); + std::copy(history_ids.begin(), history_ids.begin() + kItemsToScoreLimit, + std::inserter(history_id_set, history_id_set.end())); + post_filter_item_count_ = history_id_set.size(); + } + + // Pass over all of the candidates filtering out any without a proper + // substring match, inserting those which pass in order by score. Note that + // in this step we are using the raw search string complete with escaped + // URL elements. When the user has specifically typed something akin to + // "sort=pri&colspec=ID%20Mstone%20Release" we want to make sure that that + // specific substring appears in the URL or page title. + + // We call these 'terms' (as opposed to 'words'; see above) as in this case + // we only want to break up the search string on 'true' whitespace rather than + // escaped whitespace. When the user types "colspec=ID%20Mstone Release" we + // get two 'terms': "colspec=id%20mstone" and "release". + history::String16Vector lower_raw_terms; + Tokenize(lower_raw_string, kWhitespaceUTF16, &lower_raw_terms); + scored_items = std::for_each(history_id_set.begin(), history_id_set.end(), + AddHistoryMatch(*this, lower_raw_string, + lower_raw_terms, base::Time::Now())).ScoredMatches(); + + // Select and sort only the top kMaxMatches results. + if (scored_items.size() > AutocompleteProvider::kMaxMatches) { + std::partial_sort(scored_items.begin(), + scored_items.begin() + + AutocompleteProvider::kMaxMatches, + scored_items.end(), + ScoredHistoryMatch::MatchScoreGreater); + scored_items.resize(AutocompleteProvider::kMaxMatches); + } else { + std::sort(scored_items.begin(), scored_items.end(), + ScoredHistoryMatch::MatchScoreGreater); + } + post_scoring_item_count_ = scored_items.size(); + + if (was_trimmed) { + search_term_cache_.clear(); // Invalidate the term cache. + } else { + // Remove any stale SearchTermCacheItems. + for (SearchTermCacheMap::iterator cache_iter = search_term_cache_.begin(); + cache_iter != search_term_cache_.end(); ) { + if (!cache_iter->second.used_) + search_term_cache_.erase(cache_iter++); + else + ++cache_iter; + } + } + + return scored_items; +} + +bool URLIndexPrivateData::UpdateURL(const URLRow& row) { + // The row may or may not already be in our index. If it is not already + // indexed and it qualifies then it gets indexed. If it is already + // indexed and still qualifies then it gets updated, otherwise it + // is deleted from the index. + bool row_was_updated = false; + URLID row_id = row.id(); + HistoryInfoMap::iterator row_pos = history_info_map_.find(row_id); + if (row_pos == history_info_map_.end()) { + // This new row should be indexed if it qualifies. + URLRow new_row(row); + new_row.set_id(row_id); + row_was_updated = RowQualifiesAsSignificant(new_row, base::Time()) && + IndexRow(new_row); + } else if (RowQualifiesAsSignificant(row, base::Time())) { + // This indexed row still qualifies and will be re-indexed. + // The url won't have changed but the title, visit count, etc. + // might have changed. + URLRow& row_to_update = row_pos->second; + bool title_updated = row_to_update.title() != row.title(); + if (row_to_update.visit_count() != row.visit_count() || + row_to_update.typed_count() != row.typed_count() || + row_to_update.last_visit() != row.last_visit() || title_updated) { + CacheTransaction cache_transaction(cache_enabled() ? + cache_db_.get() : NULL); + row_to_update.set_visit_count(row.visit_count()); + row_to_update.set_typed_count(row.typed_count()); + row_to_update.set_last_visit(row.last_visit()); + row_to_update.set_hidden(row.hidden()); + // While the URL is guaranteed to remain stable, the title may have + // changed. If so, then update the index with the changed words. + if (title_updated) { + // Clear all words associated with this row and re-index both the + // URL and title. + RemoveRowWordsFromIndex(row_to_update); + row_to_update.set_title(row.title()); + RowWordStarts word_starts; + AddRowWordsToIndex(row_to_update, &word_starts); + word_starts_map_[row_id] = word_starts; + // Replace all word_starts for the row. + if (cache_enabled()) { + cache_db_->RemoveWordStarts(row_id); + cache_db_->AddRowWordStarts(row_id, word_starts); + } + } + if (cache_enabled()) { + cache_db_->RemoveHistoryIDFromURLs(row_id); + cache_db_->AddHistoryToURLs(row_id, row); + } + row_was_updated = true; + } + } else { + // This indexed row no longer qualifies and will be de-indexed by + // clearing all words associated with this row. + RemoveRowFromIndex(row); + row_was_updated = true; + } + if (row_was_updated) + search_term_cache_.clear(); // This invalidates the cache. + return row_was_updated; +} + +// Helper functor for DeleteURL. +class HistoryInfoMapItemHasURL { + public: + explicit HistoryInfoMapItemHasURL(const GURL& url): url_(url) {} + + bool operator()(const std::pair<const HistoryID, URLRow>& item) { + return item.second.url() == url_; + } + + private: + const GURL& url_; +}; + +bool URLIndexPrivateData::DeleteURL(const GURL& url) { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + BrowserThread::CurrentlyOn(BrowserThread::UI)); + // Find the matching entry in the history_info_map_. + HistoryInfoMap::iterator pos = std::find_if( + history_info_map_.begin(), + history_info_map_.end(), + HistoryInfoMapItemHasURL(url)); + if (pos == history_info_map_.end()) + return false; + RemoveRowFromIndex(pos->second); + search_term_cache_.clear(); // This invalidates the cache. + return true; +} + +bool URLIndexPrivateData::RestoreFromCacheTask() { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + DCHECK(cache_db_); + if (shutdown_) + return false; + Clear(); + if (RestoreFromCache(cache_db_)) + return true; + // If there was no SQLite-based cache then there might be an old, + // deprecated, protobuf-based cache file laying around. Get rid of it. + DeleteOldVersionCacheFile(); + return false; +} + +// static +scoped_refptr<URLIndexPrivateData> URLIndexPrivateData::RebuildFromHistory( + HistoryDatabase* history_db, + scoped_refptr<URLIndexPrivateData> old_data) { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + if (!history_db) + return NULL; + + scoped_refptr<URLIndexPrivateData> rebuilt_data( + new URLIndexPrivateData(*(old_data.get()))); + + // NOTE: We disable the cache database until after the replacement private + // data has been created and then we re-enable the database. Once the private + // data has been swapped in the database is refreshed with the new data. + URLDatabase::URLEnumerator history_enum; + if (!history_db->InitURLEnumeratorForSignificant(&history_enum)) + return NULL; + + for (URLRow row; !old_data->shutdown() && history_enum.GetNextURL(&row); ) + rebuilt_data->IndexRow(row); + + return rebuilt_data; +} + +void URLIndexPrivateData::RefreshCacheTask() { + DCHECK(!BrowserThread::IsWellKnownThread(BrowserThread::UI) || + !BrowserThread::CurrentlyOn(BrowserThread::UI)); + if (shutdown_ || !cache_enabled()) + return; + // Should a refresh fail for any reason we consider the database to be + // corrupt and unavailable; no further remedial action is possible. + set_cache_enabled(cache_db_->Reset() && cache_db_->Refresh(*this)); +} + +// static +void URLIndexPrivateData::InitializeSchemeWhitelistForTesting( + std::set<std::string>* whitelist) { + InitializeSchemeWhitelist(whitelist); +} // SearchTermCacheItem --------------------------------------------------------- @@ -67,21 +556,91 @@ URLIndexPrivateData::SearchTermCacheItem::SearchTermCacheItem() URLIndexPrivateData::SearchTermCacheItem::~SearchTermCacheItem() {} -// Algorithm Functions --------------------------------------------------------- +// URLIndexPrivateData::AddHistoryMatch ---------------------------------------- -// Comparison function for sorting search terms by descending length. -bool LengthGreater(const string16& string_a, const string16& string_b) { - return string_a.length() > string_b.length(); +URLIndexPrivateData::AddHistoryMatch::AddHistoryMatch( + const URLIndexPrivateData& private_data, + const string16& lower_string, + const String16Vector& lower_terms, + const base::Time now) + : private_data_(private_data), + lower_string_(lower_string), + lower_terms_(lower_terms), + now_(now) {} + +URLIndexPrivateData::AddHistoryMatch::~AddHistoryMatch() {} + +void URLIndexPrivateData::AddHistoryMatch::operator()( + const HistoryID history_id) { + HistoryInfoMap::const_iterator hist_pos = + private_data_.history_info_map_.find(history_id); + if (hist_pos != private_data_.history_info_map_.end()) { + const URLRow& hist_item = hist_pos->second; + WordStartsMap::const_iterator starts_pos = + private_data_.word_starts_map_.find(history_id); + DCHECK(starts_pos != private_data_.word_starts_map_.end()); + ScoredHistoryMatch match(hist_item, lower_string_, lower_terms_, + starts_pos->second, now_); + if (match.raw_score > 0) + scored_matches_.push_back(match); + } } -// InMemoryURLIndex's Private Data --------------------------------------------- +// URLIndexPrivateData::HistoryItemFactorGreater ------------------------------- +URLIndexPrivateData::HistoryItemFactorGreater::HistoryItemFactorGreater( + const HistoryInfoMap& history_info_map) + : history_info_map_(history_info_map) { +} + +URLIndexPrivateData::HistoryItemFactorGreater::~HistoryItemFactorGreater() {} + +bool URLIndexPrivateData::HistoryItemFactorGreater::operator()( + const HistoryID h1, + const HistoryID h2) { + HistoryInfoMap::const_iterator entry1(history_info_map_.find(h1)); + if (entry1 == history_info_map_.end()) + return false; + HistoryInfoMap::const_iterator entry2(history_info_map_.find(h2)); + if (entry2 == history_info_map_.end()) + return true; + const URLRow& r1(entry1->second); + const URLRow& r2(entry2->second); + // First cut: typed count, visit count, recency. + // TODO(mrossetti): This is too simplistic. Consider an approach which ranks + // recently visited (within the last 12/24 hours) as highly important. Get + // input from mpearson. + if (r1.typed_count() != r2.typed_count()) + return (r1.typed_count() > r2.typed_count()); + if (r1.visit_count() != r2.visit_count()) + return (r1.visit_count() > r2.visit_count()); + return (r1.last_visit() > r2.last_visit()); +} + +// URLIndexPrivateData Private Functions --------------------------------------- + +URLIndexPrivateData::URLIndexPrivateData(const URLIndexPrivateData& old_data) + : history_dir_(old_data.history_dir_), + languages_(old_data.languages_), + cache_enabled_(old_data.cache_enabled_), + initialization_lock_(), + shutdown_(old_data.shutdown_), + scheme_whitelist_(old_data.scheme_whitelist_), + pre_filter_item_count_(0), + post_filter_item_count_(0), + post_scoring_item_count_(0) { + cache_db_ = old_data.cache_db_; +} + +// Called only by unit tests. URLIndexPrivateData::URLIndexPrivateData() - : restored_cache_version_(0), - saved_cache_version_(kCurrentCacheFileVersion), + : cache_enabled_(true), + initialization_lock_(), + shutdown_(false), pre_filter_item_count_(0), post_filter_item_count_(0), post_scoring_item_count_(0) { + InitializeSchemeWhitelist(&scheme_whitelist_); } URLIndexPrivateData::~URLIndexPrivateData() {} @@ -97,48 +656,24 @@ void URLIndexPrivateData::Clear() { word_starts_map_.clear(); } -bool URLIndexPrivateData::Empty() const { - return history_info_map_.empty(); -} - -scoped_refptr<URLIndexPrivateData> URLIndexPrivateData::Duplicate() const { - scoped_refptr<URLIndexPrivateData> data_copy = new URLIndexPrivateData; - data_copy->word_list_ = word_list_; - data_copy->available_words_ = available_words_; - data_copy->word_map_ = word_map_; - data_copy->char_word_map_ = char_word_map_; - data_copy->word_id_history_map_ = word_id_history_map_; - data_copy->history_id_word_map_ = history_id_word_map_; - data_copy->history_info_map_ = history_info_map_; - data_copy->word_starts_map_ = word_starts_map_; - return data_copy; - // Not copied: - // search_term_cache_ - // pre_filter_item_count_ - // post_filter_item_count_ - // post_scoring_item_count_ -}; - // Cache Updating -------------------------------------------------------------- -bool URLIndexPrivateData::IndexRow( - const URLRow& row, - const std::string& languages, - const std::set<std::string>& scheme_whitelist) { +bool URLIndexPrivateData::IndexRow(const URLRow& row) { const GURL& gurl(row.url()); // Index only URLs with a whitelisted scheme. - if (!URLSchemeIsWhitelisted(gurl, scheme_whitelist)) + if (!URLSchemeIsWhitelisted(gurl, scheme_whitelist_)) return false; - URLID row_id = row.id(); // Strip out username and password before saving and indexing. - string16 url(net::FormatUrl(gurl, languages, + string16 url(net::FormatUrl(gurl, languages_, net::kFormatUrlOmitUsernamePassword, net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS, NULL, NULL, NULL)); + URLID row_id = row.id(); HistoryID history_id = static_cast<HistoryID>(row_id); + DCHECK(history_info_map_.find(history_id) == history_info_map_.end()); DCHECK_LT(history_id, std::numeric_limits<HistoryID>::max()); // Add the row for quick lookup in the history info store. @@ -147,22 +682,27 @@ bool URLIndexPrivateData::IndexRow( new_row.set_typed_count(row.typed_count()); new_row.set_last_visit(row.last_visit()); new_row.set_title(row.title()); + + CacheTransaction cache_transaction(cache_enabled() ? cache_db_.get() : NULL); history_info_map_[history_id] = new_row; + if (cache_enabled()) + cache_db_->AddHistoryToURLs(history_id, row); // Index the words contained in the URL and title of the row. RowWordStarts word_starts; - AddRowWordsToIndex(new_row, &word_starts, languages); + AddRowWordsToIndex(new_row, &word_starts); word_starts_map_[history_id] = word_starts; + if (cache_enabled()) + cache_db_->AddRowWordStarts(history_id, word_starts); return true; } void URLIndexPrivateData::AddRowWordsToIndex(const URLRow& row, - RowWordStarts* word_starts, - const std::string& languages) { + RowWordStarts* word_starts) { HistoryID history_id = static_cast<HistoryID>(row.id()); // Split URL into individual, unique words then add in the title words. const GURL& gurl(row.url()); - string16 url(net::FormatUrl(gurl, languages, + string16 url(net::FormatUrl(gurl, languages_, net::kFormatUrlOmitUsernamePassword, net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS, NULL, NULL, NULL)); @@ -195,9 +735,15 @@ void URLIndexPrivateData::UpdateWordHistory(WordID word_id, HistoryID history_id) { WordIDHistoryMap::iterator history_pos = word_id_history_map_.find(word_id); DCHECK(history_pos != word_id_history_map_.end()); + // There is no need to record changes to the word_id_history_map_ in the cache + // because it can be rebuilt from the history_id_word_map_'s table. HistoryIDSet& history_id_set(history_pos->second); history_id_set.insert(history_id); AddToHistoryIDWordMap(history_id, word_id); + + // Add word_id/history_id entry to the word_history table + if (cache_enabled()) + cache_db_->AddHistoryToWordHistory(word_id, history_id); } void URLIndexPrivateData::AddWordHistory(const string16& term, @@ -212,11 +758,19 @@ void URLIndexPrivateData::AddWordHistory(const string16& term, } word_map_[term] = word_id; + // Add word_id/term entry to the words table; + if (cache_enabled()) + cache_db_->AddWordToWords(word_id, term); + HistoryIDSet history_id_set; history_id_set.insert(history_id); word_id_history_map_[word_id] = history_id_set; AddToHistoryIDWordMap(history_id, word_id); + // Add word_id/history_id entry to the word_history table + if (cache_enabled()) + cache_db_->AddHistoryToWordHistory(word_id, history_id); + // For each character in the newly added word (i.e. a word that is not // already in the word index), add the word to the character index. Char16Set characters = Char16SetFromString16(term); @@ -234,14 +788,20 @@ void URLIndexPrivateData::AddWordHistory(const string16& term, word_id_set.insert(word_id); char_word_map_[uni_char] = word_id_set; } + // Add uni_char/word_id entry to the char_words database table. + if (cache_enabled()) + cache_db_->AddWordToCharWords(uni_char, word_id); } } void URLIndexPrivateData::RemoveRowFromIndex(const URLRow& row) { + CacheTransaction cache_transaction(cache_enabled() ? cache_db_.get() : NULL); RemoveRowWordsFromIndex(row); HistoryID history_id = static_cast<HistoryID>(row.id()); history_info_map_.erase(history_id); word_starts_map_.erase(history_id); + if (cache_enabled()) + cache_db_->RemoveHistoryIDFromURLs(history_id); } void URLIndexPrivateData::RemoveRowWordsFromIndex(const URLRow& row) { @@ -275,7 +835,11 @@ void URLIndexPrivateData::RemoveRowWordsFromIndex(const URLRow& row) { word_map_.erase(word); word_list_[word_id] = string16(); available_words_.insert(word_id); + if (cache_enabled()) + cache_db_->RemoveWordFromWords(word_id); } + if (cache_enabled()) + cache_db_->RemoveHistoryIDFromWordHistory(history_id); } void URLIndexPrivateData::AddToHistoryIDWordMap(HistoryID history_id, @@ -291,257 +855,6 @@ void URLIndexPrivateData::AddToHistoryIDWordMap(HistoryID history_id, } } -bool URLIndexPrivateData::UpdateURL( - const URLRow& row, - const std::string& languages, - const std::set<std::string>& scheme_whitelist) { - // The row may or may not already be in our index. If it is not already - // indexed and it qualifies then it gets indexed. If it is already - // indexed and still qualifies then it gets updated, otherwise it - // is deleted from the index. - bool row_was_updated = false; - URLID row_id = row.id(); - HistoryInfoMap::iterator row_pos = history_info_map_.find(row_id); - if (row_pos == history_info_map_.end()) { - // This new row should be indexed if it qualifies. - URLRow new_row(row); - new_row.set_id(row_id); - row_was_updated = RowQualifiesAsSignificant(new_row, base::Time()) && - IndexRow(new_row, languages, scheme_whitelist); - } else if (RowQualifiesAsSignificant(row, base::Time())) { - // This indexed row still qualifies and will be re-indexed. - // The url won't have changed but the title, visit count, etc. - // might have changed. - URLRow& row_to_update = row_pos->second; - bool title_updated = row_to_update.title() != row.title(); - if (row_to_update.visit_count() != row.visit_count() || - row_to_update.typed_count() != row.typed_count() || - row_to_update.last_visit() != row.last_visit() || title_updated) { - row_to_update.set_visit_count(row.visit_count()); - row_to_update.set_typed_count(row.typed_count()); - row_to_update.set_last_visit(row.last_visit()); - // While the URL is guaranteed to remain stable, the title may have - // changed. If so, then update the index with the changed words. - if (title_updated) { - // Clear all words associated with this row and re-index both the - // URL and title. - RemoveRowWordsFromIndex(row_to_update); - row_to_update.set_title(row.title()); - RowWordStarts word_starts; - AddRowWordsToIndex(row_to_update, &word_starts, languages); - word_starts_map_[row_id] = word_starts; - } - row_was_updated = true; - } - } else { - // This indexed row no longer qualifies and will be de-indexed by - // clearing all words associated with this row. - RemoveRowFromIndex(row); - row_was_updated = true; - } - if (row_was_updated) - search_term_cache_.clear(); // This invalidates the cache. - return row_was_updated; -} - -// Helper functor for DeleteURL. -class HistoryInfoMapItemHasURL { - public: - explicit HistoryInfoMapItemHasURL(const GURL& url): url_(url) {} - - bool operator()(const std::pair<const HistoryID, URLRow>& item) { - return item.second.url() == url_; - } - - private: - const GURL& url_; -}; - -bool URLIndexPrivateData::DeleteURL(const GURL& url) { - // Find the matching entry in the history_info_map_. - HistoryInfoMap::iterator pos = std::find_if( - history_info_map_.begin(), - history_info_map_.end(), - HistoryInfoMapItemHasURL(url)); - if (pos == history_info_map_.end()) - return false; - RemoveRowFromIndex(pos->second); - search_term_cache_.clear(); // This invalidates the cache. - return true; -} - -// URLIndexPrivateData::HistoryItemFactorGreater ------------------------------- - -URLIndexPrivateData::HistoryItemFactorGreater::HistoryItemFactorGreater( - const HistoryInfoMap& history_info_map) - : history_info_map_(history_info_map) { -} - -URLIndexPrivateData::HistoryItemFactorGreater::~HistoryItemFactorGreater() {} - -bool URLIndexPrivateData::HistoryItemFactorGreater::operator()( - const HistoryID h1, - const HistoryID h2) { - HistoryInfoMap::const_iterator entry1(history_info_map_.find(h1)); - if (entry1 == history_info_map_.end()) - return false; - HistoryInfoMap::const_iterator entry2(history_info_map_.find(h2)); - if (entry2 == history_info_map_.end()) - return true; - const URLRow& r1(entry1->second); - const URLRow& r2(entry2->second); - // First cut: typed count, visit count, recency. - // TODO(mrossetti): This is too simplistic. Consider an approach which ranks - // recently visited (within the last 12/24 hours) as highly important. Get - // input from mpearson. - if (r1.typed_count() != r2.typed_count()) - return (r1.typed_count() > r2.typed_count()); - if (r1.visit_count() != r2.visit_count()) - return (r1.visit_count() > r2.visit_count()); - return (r1.last_visit() > r2.last_visit()); -} - -// Cache Searching ------------------------------------------------------------- - -// NOTE: This is the main public search function. -ScoredHistoryMatches URLIndexPrivateData::HistoryItemsForTerms( - const string16& search_string) { - pre_filter_item_count_ = 0; - post_filter_item_count_ = 0; - post_scoring_item_count_ = 0; - // The search string we receive may contain escaped characters. For reducing - // the index we need individual, lower-cased words, ignoring escapings. For - // the final filtering we need whitespace separated substrings possibly - // containing escaped characters. - string16 lower_raw_string(base::i18n::ToLower(search_string)); - string16 lower_unescaped_string = - net::UnescapeURLComponent(lower_raw_string, - net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS); - // Extract individual 'words' (as opposed to 'terms'; see below) from the - // search string. When the user types "colspec=ID%20Mstone Release" we get - // four 'words': "colspec", "id", "mstone" and "release". - String16Vector lower_words( - history::String16VectorFromString16(lower_unescaped_string, false, NULL)); - ScoredHistoryMatches scored_items; - - // Do nothing if we have indexed no words (probably because we've not been - // initialized yet) or the search string has no words. - if (word_list_.empty() || lower_words.empty()) { - search_term_cache_.clear(); // Invalidate the term cache. - return scored_items; - } - - // Reset used_ flags for search_term_cache_. We use a basic mark-and-sweep - // approach. - ResetSearchTermCache(); - - HistoryIDSet history_id_set = HistoryIDSetFromWords(lower_words); - - // Trim the candidate pool if it is large. Note that we do not filter out - // items that do not contain the search terms as proper substrings -- doing - // so is the performance-costly operation we are trying to avoid in order - // to maintain omnibox responsiveness. - const size_t kItemsToScoreLimit = 500; - pre_filter_item_count_ = history_id_set.size(); - // If we trim the results set we do not want to cache the results for next - // time as the user's ultimately desired result could easily be eliminated - // in this early rough filter. - bool was_trimmed = (pre_filter_item_count_ > kItemsToScoreLimit); - if (was_trimmed) { - HistoryIDVector history_ids; - std::copy(history_id_set.begin(), history_id_set.end(), - std::back_inserter(history_ids)); - // Trim down the set by sorting by typed-count, visit-count, and last - // visit. - HistoryItemFactorGreater - item_factor_functor(history_info_map_); - std::partial_sort(history_ids.begin(), - history_ids.begin() + kItemsToScoreLimit, - history_ids.end(), - item_factor_functor); - history_id_set.clear(); - std::copy(history_ids.begin(), history_ids.begin() + kItemsToScoreLimit, - std::inserter(history_id_set, history_id_set.end())); - post_filter_item_count_ = history_id_set.size(); - } - - // Pass over all of the candidates filtering out any without a proper - // substring match, inserting those which pass in order by score. Note that - // in this step we are using the raw search string complete with escaped - // URL elements. When the user has specifically typed something akin to - // "sort=pri&colspec=ID%20Mstone%20Release" we want to make sure that that - // specific substring appears in the URL or page title. - - // We call these 'terms' (as opposed to 'words'; see above) as in this case - // we only want to break up the search string on 'true' whitespace rather than - // escaped whitespace. When the user types "colspec=ID%20Mstone Release" we - // get two 'terms': "colspec=id%20mstone" and "release". - history::String16Vector lower_raw_terms; - Tokenize(lower_raw_string, kWhitespaceUTF16, &lower_raw_terms); - scored_items = std::for_each(history_id_set.begin(), history_id_set.end(), - AddHistoryMatch(*this, lower_raw_string, - lower_raw_terms, base::Time::Now())).ScoredMatches(); - - // Select and sort only the top kMaxMatches results. - if (scored_items.size() > AutocompleteProvider::kMaxMatches) { - std::partial_sort(scored_items.begin(), - scored_items.begin() + - AutocompleteProvider::kMaxMatches, - scored_items.end(), - ScoredHistoryMatch::MatchScoreGreater); - scored_items.resize(AutocompleteProvider::kMaxMatches); - } else { - std::sort(scored_items.begin(), scored_items.end(), - ScoredHistoryMatch::MatchScoreGreater); - } - post_scoring_item_count_ = scored_items.size(); - - if (was_trimmed) { - search_term_cache_.clear(); // Invalidate the term cache. - } else { - // Remove any stale SearchTermCacheItems. - for (SearchTermCacheMap::iterator cache_iter = search_term_cache_.begin(); - cache_iter != search_term_cache_.end(); ) { - if (!cache_iter->second.used_) - search_term_cache_.erase(cache_iter++); - else - ++cache_iter; - } - } - - return scored_items; -} - -// URLIndexPrivateData::AddHistoryMatch ---------------------------------------- - -URLIndexPrivateData::AddHistoryMatch::AddHistoryMatch( - const URLIndexPrivateData& private_data, - const string16& lower_string, - const String16Vector& lower_terms, - const base::Time now) - : private_data_(private_data), - lower_string_(lower_string), - lower_terms_(lower_terms), - now_(now) {} - -URLIndexPrivateData::AddHistoryMatch::~AddHistoryMatch() {} - -void URLIndexPrivateData::AddHistoryMatch::operator()( - const HistoryID history_id) { - HistoryInfoMap::const_iterator hist_pos = - private_data_.history_info_map_.find(history_id); - if (hist_pos != private_data_.history_info_map_.end()) { - const URLRow& hist_item = hist_pos->second; - WordStartsMap::const_iterator starts_pos = - private_data_.word_starts_map_.find(history_id); - DCHECK(starts_pos != private_data_.word_starts_map_.end()); - ScoredHistoryMatch match(hist_item, lower_string_, lower_terms_, - starts_pos->second, now_); - if (match.raw_score > 0) - scored_matches_.push_back(match); - } -} - void URLIndexPrivateData::ResetSearchTermCache() { for (SearchTermCacheMap::iterator iter = search_term_cache_.begin(); iter != search_term_cache_.end(); ++iter) @@ -730,416 +1043,33 @@ WordIDSet URLIndexPrivateData::WordIDSetForTermChars( return word_id_set; } -// Cache Saving ---------------------------------------------------------------- - -// static -void URLIndexPrivateData::WritePrivateDataToCacheFileTask( - scoped_refptr<URLIndexPrivateData> private_data, - const FilePath& file_path, - scoped_refptr<RefCountedBool> succeeded) { - DCHECK(private_data.get()); - DCHECK(!file_path.empty()); - succeeded->set_value(private_data->SaveToFile(file_path)); -} - -bool URLIndexPrivateData::SaveToFile(const FilePath& file_path) { - base::TimeTicks beginning_time = base::TimeTicks::Now(); - InMemoryURLIndexCacheItem index_cache; - SavePrivateData(&index_cache); - std::string data; - if (!index_cache.SerializeToString(&data)) { - LOG(WARNING) << "Failed to serialize the InMemoryURLIndex cache."; - return false; - } - - int size = data.size(); - if (file_util::WriteFile(file_path, data.c_str(), size) != size) { - LOG(WARNING) << "Failed to write " << file_path.value(); +bool URLIndexPrivateData::RestoreFromCache(InMemoryURLCacheDatabase* cache_db) { + DCHECK(cache_db); + if (!cache_db->RestorePrivateData(this)) { + cache_db->Reset(); return false; } - UMA_HISTOGRAM_TIMES("History.InMemoryURLIndexSaveCacheTime", - base::TimeTicks::Now() - beginning_time); - return true; -} - -void URLIndexPrivateData::SavePrivateData( - InMemoryURLIndexCacheItem* cache) const { - DCHECK(cache); - cache->set_timestamp(base::Time::Now().ToInternalValue()); - cache->set_version(saved_cache_version_); - // history_item_count_ is no longer used but rather than change the protobuf - // definition use a placeholder. This will go away with the switch to SQLite. - cache->set_history_item_count(0); - SaveWordList(cache); - SaveWordMap(cache); - SaveCharWordMap(cache); - SaveWordIDHistoryMap(cache); - SaveHistoryInfoMap(cache); - SaveWordStartsMap(cache); + return !Empty() && ValidateConsistency(); } -void URLIndexPrivateData::SaveWordList(InMemoryURLIndexCacheItem* cache) const { - if (word_list_.empty()) +void URLIndexPrivateData::DeleteOldVersionCacheFile() const { + if (history_dir_.empty()) return; - WordListItem* list_item = cache->mutable_word_list(); - list_item->set_word_count(word_list_.size()); - for (String16Vector::const_iterator iter = word_list_.begin(); - iter != word_list_.end(); ++iter) - list_item->add_word(UTF16ToUTF8(*iter)); + FilePath path = history_dir_.Append(chrome::kHQPCacheFilename); + file_util::Delete(path, false); } -void URLIndexPrivateData::SaveWordMap(InMemoryURLIndexCacheItem* cache) const { - if (word_map_.empty()) - return; - WordMapItem* map_item = cache->mutable_word_map(); - map_item->set_item_count(word_map_.size()); - for (WordMap::const_iterator iter = word_map_.begin(); - iter != word_map_.end(); ++iter) { - WordMapEntry* map_entry = map_item->add_word_map_entry(); - map_entry->set_word(UTF16ToUTF8(iter->first)); - map_entry->set_word_id(iter->second); - } -} - -void URLIndexPrivateData::SaveCharWordMap( - InMemoryURLIndexCacheItem* cache) const { - if (char_word_map_.empty()) - return; - CharWordMapItem* map_item = cache->mutable_char_word_map(); - map_item->set_item_count(char_word_map_.size()); - for (CharWordIDMap::const_iterator iter = char_word_map_.begin(); - iter != char_word_map_.end(); ++iter) { - CharWordMapEntry* map_entry = map_item->add_char_word_map_entry(); - map_entry->set_char_16(iter->first); - const WordIDSet& word_id_set(iter->second); - map_entry->set_item_count(word_id_set.size()); - for (WordIDSet::const_iterator set_iter = word_id_set.begin(); - set_iter != word_id_set.end(); ++set_iter) - map_entry->add_word_id(*set_iter); - } -} - -void URLIndexPrivateData::SaveWordIDHistoryMap( - InMemoryURLIndexCacheItem* cache) const { - if (word_id_history_map_.empty()) - return; - WordIDHistoryMapItem* map_item = cache->mutable_word_id_history_map(); - map_item->set_item_count(word_id_history_map_.size()); - for (WordIDHistoryMap::const_iterator iter = word_id_history_map_.begin(); - iter != word_id_history_map_.end(); ++iter) { - WordIDHistoryMapEntry* map_entry = - map_item->add_word_id_history_map_entry(); - map_entry->set_word_id(iter->first); - const HistoryIDSet& history_id_set(iter->second); - map_entry->set_item_count(history_id_set.size()); - for (HistoryIDSet::const_iterator set_iter = history_id_set.begin(); - set_iter != history_id_set.end(); ++set_iter) - map_entry->add_history_id(*set_iter); - } -} - -void URLIndexPrivateData::SaveHistoryInfoMap( - InMemoryURLIndexCacheItem* cache) const { - if (history_info_map_.empty()) - return; - HistoryInfoMapItem* map_item = cache->mutable_history_info_map(); - map_item->set_item_count(history_info_map_.size()); - for (HistoryInfoMap::const_iterator iter = history_info_map_.begin(); - iter != history_info_map_.end(); ++iter) { - HistoryInfoMapEntry* map_entry = map_item->add_history_info_map_entry(); - map_entry->set_history_id(iter->first); - const URLRow& url_row(iter->second); - // Note: We only save information that contributes to the index so there - // is no need to save search_term_cache_ (not persistent). - map_entry->set_visit_count(url_row.visit_count()); - map_entry->set_typed_count(url_row.typed_count()); - map_entry->set_last_visit(url_row.last_visit().ToInternalValue()); - map_entry->set_url(url_row.url().spec()); - map_entry->set_title(UTF16ToUTF8(url_row.title())); - } -} - -void URLIndexPrivateData::SaveWordStartsMap( - InMemoryURLIndexCacheItem* cache) const { - if (word_starts_map_.empty()) - return; - // For unit testing: Enable saving of the cache as an earlier version to - // allow testing of cache file upgrading in ReadFromFile(). - // TODO(mrossetti): Instead of intruding on production code with this kind of - // test harness, save a copy of an older version cache with known results. - // Implement this when switching the caching over to SQLite. - if (saved_cache_version_ < 1) - return; - - WordStartsMapItem* map_item = cache->mutable_word_starts_map(); - map_item->set_item_count(word_starts_map_.size()); - for (WordStartsMap::const_iterator iter = word_starts_map_.begin(); - iter != word_starts_map_.end(); ++iter) { - WordStartsMapEntry* map_entry = map_item->add_word_starts_map_entry(); - map_entry->set_history_id(iter->first); - const RowWordStarts& word_starts(iter->second); - for (WordStarts::const_iterator i = word_starts.url_word_starts_.begin(); - i != word_starts.url_word_starts_.end(); ++i) - map_entry->add_url_word_starts(*i); - for (WordStarts::const_iterator i = word_starts.title_word_starts_.begin(); - i != word_starts.title_word_starts_.end(); ++i) - map_entry->add_title_word_starts(*i); - } -} - -// Cache Restoring ------------------------------------------------------------- - -// static -void URLIndexPrivateData::RestoreFromFileTask( - const FilePath& file_path, - scoped_refptr<URLIndexPrivateData> private_data, - const std::string& languages) { - private_data = URLIndexPrivateData::RestoreFromFile(file_path, languages); -} - -// static -scoped_refptr<URLIndexPrivateData> URLIndexPrivateData::RestoreFromFile( - const FilePath& file_path, - const std::string& languages) { - base::TimeTicks beginning_time = base::TimeTicks::Now(); - if (!file_util::PathExists(file_path)) - return NULL; - std::string data; - // If there is no cache file then simply give up. This will cause us to - // attempt to rebuild from the history database. - if (!file_util::ReadFileToString(file_path, &data)) - return NULL; - - scoped_refptr<URLIndexPrivateData> restored_data(new URLIndexPrivateData); - InMemoryURLIndexCacheItem index_cache; - if (!index_cache.ParseFromArray(data.c_str(), data.size())) { - LOG(WARNING) << "Failed to parse URLIndexPrivateData cache data read from " - << file_path.value(); - return restored_data; - } - - if (!restored_data->RestorePrivateData(index_cache, languages)) - return NULL; - - UMA_HISTOGRAM_TIMES("History.InMemoryURLIndexRestoreCacheTime", - base::TimeTicks::Now() - beginning_time); - UMA_HISTOGRAM_COUNTS("History.InMemoryURLHistoryItems", - restored_data->history_id_word_map_.size()); - UMA_HISTOGRAM_COUNTS("History.InMemoryURLCacheSize", data.size()); - UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLWords", - restored_data->word_map_.size()); - UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLChars", - restored_data->char_word_map_.size()); - if (restored_data->Empty()) - return NULL; // 'No data' is the same as a failed reload. - return restored_data; -} - -// static -scoped_refptr<URLIndexPrivateData> URLIndexPrivateData::RebuildFromHistory( - HistoryDatabase* history_db, - const std::string& languages, - const std::set<std::string>& scheme_whitelist) { - if (!history_db) - return NULL; - - base::TimeTicks beginning_time = base::TimeTicks::Now(); - - scoped_refptr<URLIndexPrivateData> rebuilt_data(new URLIndexPrivateData); - URLDatabase::URLEnumerator history_enum; - if (!history_db->InitURLEnumeratorForSignificant(&history_enum)) - return NULL; - for (URLRow row; history_enum.GetNextURL(&row); ) - rebuilt_data->IndexRow(row, languages, scheme_whitelist); - - UMA_HISTOGRAM_TIMES("History.InMemoryURLIndexingTime", - base::TimeTicks::Now() - beginning_time); - UMA_HISTOGRAM_COUNTS("History.InMemoryURLHistoryItems", - rebuilt_data->history_id_word_map_.size()); - UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLWords", - rebuilt_data->word_map_.size()); - UMA_HISTOGRAM_COUNTS_10000("History.InMemoryURLChars", - rebuilt_data->char_word_map_.size()); - return rebuilt_data; -} - -bool URLIndexPrivateData::RestorePrivateData( - const InMemoryURLIndexCacheItem& cache, - const std::string& languages) { - if (cache.has_version()) - restored_cache_version_ = cache.version(); - return RestoreWordList(cache) && RestoreWordMap(cache) && - RestoreCharWordMap(cache) && RestoreWordIDHistoryMap(cache) && - RestoreHistoryInfoMap(cache) && RestoreWordStartsMap(cache, languages); -} - -bool URLIndexPrivateData::RestoreWordList( - const InMemoryURLIndexCacheItem& cache) { - if (!cache.has_word_list()) - return false; - const WordListItem& list_item(cache.word_list()); - uint32 expected_item_count = list_item.word_count(); - uint32 actual_item_count = list_item.word_size(); - if (actual_item_count == 0 || actual_item_count != expected_item_count) - return false; - const RepeatedPtrField<std::string>& words(list_item.word()); - for (RepeatedPtrField<std::string>::const_iterator iter = words.begin(); - iter != words.end(); ++iter) - word_list_.push_back(UTF8ToUTF16(*iter)); - return true; -} - -bool URLIndexPrivateData::RestoreWordMap( - const InMemoryURLIndexCacheItem& cache) { - if (!cache.has_word_map()) - return false; - const WordMapItem& list_item(cache.word_map()); - uint32 expected_item_count = list_item.item_count(); - uint32 actual_item_count = list_item.word_map_entry_size(); - if (actual_item_count == 0 || actual_item_count != expected_item_count) +bool URLIndexPrivateData::GetCacheDBPath(FilePath* file_path) { + DCHECK(file_path); + if (history_dir_.empty()) return false; - const RepeatedPtrField<WordMapEntry>& entries(list_item.word_map_entry()); - for (RepeatedPtrField<WordMapEntry>::const_iterator iter = entries.begin(); - iter != entries.end(); ++iter) - word_map_[UTF8ToUTF16(iter->word())] = iter->word_id(); - return true; -} - -bool URLIndexPrivateData::RestoreCharWordMap( - const InMemoryURLIndexCacheItem& cache) { - if (!cache.has_char_word_map()) - return false; - const CharWordMapItem& list_item(cache.char_word_map()); - uint32 expected_item_count = list_item.item_count(); - uint32 actual_item_count = list_item.char_word_map_entry_size(); - if (actual_item_count == 0 || actual_item_count != expected_item_count) - return false; - const RepeatedPtrField<CharWordMapEntry>& - entries(list_item.char_word_map_entry()); - for (RepeatedPtrField<CharWordMapEntry>::const_iterator iter = - entries.begin(); iter != entries.end(); ++iter) { - expected_item_count = iter->item_count(); - actual_item_count = iter->word_id_size(); - if (actual_item_count == 0 || actual_item_count != expected_item_count) - return false; - char16 uni_char = static_cast<char16>(iter->char_16()); - WordIDSet word_id_set; - const RepeatedField<int32>& word_ids(iter->word_id()); - for (RepeatedField<int32>::const_iterator jiter = word_ids.begin(); - jiter != word_ids.end(); ++jiter) - word_id_set.insert(*jiter); - char_word_map_[uni_char] = word_id_set; - } + *file_path = history_dir_.Append(chrome::kHQPCacheDBFilename); return true; } -bool URLIndexPrivateData::RestoreWordIDHistoryMap( - const InMemoryURLIndexCacheItem& cache) { - if (!cache.has_word_id_history_map()) - return false; - const WordIDHistoryMapItem& list_item(cache.word_id_history_map()); - uint32 expected_item_count = list_item.item_count(); - uint32 actual_item_count = list_item.word_id_history_map_entry_size(); - if (actual_item_count == 0 || actual_item_count != expected_item_count) - return false; - const RepeatedPtrField<WordIDHistoryMapEntry>& - entries(list_item.word_id_history_map_entry()); - for (RepeatedPtrField<WordIDHistoryMapEntry>::const_iterator iter = - entries.begin(); iter != entries.end(); ++iter) { - expected_item_count = iter->item_count(); - actual_item_count = iter->history_id_size(); - if (actual_item_count == 0 || actual_item_count != expected_item_count) - return false; - WordID word_id = iter->word_id(); - HistoryIDSet history_id_set; - const RepeatedField<int64>& history_ids(iter->history_id()); - for (RepeatedField<int64>::const_iterator jiter = history_ids.begin(); - jiter != history_ids.end(); ++jiter) { - history_id_set.insert(*jiter); - AddToHistoryIDWordMap(*jiter, word_id); - } - word_id_history_map_[word_id] = history_id_set; - } - return true; -} - -bool URLIndexPrivateData::RestoreHistoryInfoMap( - const InMemoryURLIndexCacheItem& cache) { - if (!cache.has_history_info_map()) - return false; - const HistoryInfoMapItem& list_item(cache.history_info_map()); - uint32 expected_item_count = list_item.item_count(); - uint32 actual_item_count = list_item.history_info_map_entry_size(); - if (actual_item_count == 0 || actual_item_count != expected_item_count) - return false; - const RepeatedPtrField<HistoryInfoMapEntry>& - entries(list_item.history_info_map_entry()); - for (RepeatedPtrField<HistoryInfoMapEntry>::const_iterator iter = - entries.begin(); iter != entries.end(); ++iter) { - HistoryID history_id = iter->history_id(); - GURL url(iter->url()); - URLRow url_row(url, history_id); - url_row.set_visit_count(iter->visit_count()); - url_row.set_typed_count(iter->typed_count()); - url_row.set_last_visit(base::Time::FromInternalValue(iter->last_visit())); - if (iter->has_title()) { - string16 title(UTF8ToUTF16(iter->title())); - url_row.set_title(title); - } - history_info_map_[history_id] = url_row; - } - return true; -} - -bool URLIndexPrivateData::RestoreWordStartsMap( - const InMemoryURLIndexCacheItem& cache, - const std::string& languages) { - // Note that this function must be called after RestoreHistoryInfoMap() has - // been run as the word starts may have to be recalculated from the urls and - // page titles. - if (cache.has_word_starts_map()) { - const WordStartsMapItem& list_item(cache.word_starts_map()); - uint32 expected_item_count = list_item.item_count(); - uint32 actual_item_count = list_item.word_starts_map_entry_size(); - if (actual_item_count == 0 || actual_item_count != expected_item_count) - return false; - const RepeatedPtrField<WordStartsMapEntry>& - entries(list_item.word_starts_map_entry()); - for (RepeatedPtrField<WordStartsMapEntry>::const_iterator iter = - entries.begin(); iter != entries.end(); ++iter) { - HistoryID history_id = iter->history_id(); - RowWordStarts word_starts; - // Restore the URL word starts. - const RepeatedField<int32>& url_starts(iter->url_word_starts()); - for (RepeatedField<int32>::const_iterator jiter = url_starts.begin(); - jiter != url_starts.end(); ++jiter) - word_starts.url_word_starts_.push_back(*jiter); - // Restore the page title word starts. - const RepeatedField<int32>& title_starts(iter->title_word_starts()); - for (RepeatedField<int32>::const_iterator jiter = title_starts.begin(); - jiter != title_starts.end(); ++jiter) - word_starts.title_word_starts_.push_back(*jiter); - word_starts_map_[history_id] = word_starts; - } - } else { - // Since the cache did not contain any word starts we must rebuild then from - // the URL and page titles. - for (HistoryInfoMap::const_iterator iter = history_info_map_.begin(); - iter != history_info_map_.end(); ++iter) { - RowWordStarts word_starts; - const URLRow& row(iter->second); - string16 url(net::FormatUrl(row.url(), languages, - net::kFormatUrlOmitUsernamePassword, - net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS, - NULL, NULL, NULL)); - url = base::i18n::ToLower(url); - String16VectorFromString16(url, false, &word_starts.url_word_starts_); - String16VectorFromString16( - row.title(), false, &word_starts.title_word_starts_); - word_starts_map_[iter->first] = word_starts; - } - } - return true; +void URLIndexPrivateData::SetCacheDatabaseForTesting( + InMemoryURLCacheDatabase* test_db) { + cache_db_ = test_db; } // static diff --git a/chrome/browser/history/url_index_private_data.h b/chrome/browser/history/url_index_private_data.h index f5ed8ed..eb11d1b 100644 --- a/chrome/browser/history/url_index_private_data.h +++ b/chrome/browser/history/url_index_private_data.h @@ -11,8 +11,9 @@ #include "base/file_path.h" #include "base/gtest_prod_util.h" #include "base/memory/ref_counted.h" +#include "base/synchronization/lock.h" +#include "base/threading/sequenced_worker_pool.h" #include "chrome/browser/history/in_memory_url_index_types.h" -#include "chrome/browser/history/in_memory_url_index_cache.pb.h" #include "chrome/browser/history/scored_history_match.h" #include "content/public/browser/notification_details.h" @@ -27,34 +28,142 @@ namespace history { namespace imui = in_memory_url_index; class HistoryDatabase; +class InMemoryURLCacheDatabase; class InMemoryURLIndex; class RefCountedBool; -// Current version of the cache file. -static const int kCurrentCacheFileVersion = 1; - -// A structure describing the InMemoryURLIndex's internal data and providing for -// restoring, rebuilding and updating that internal data. +// A structure private to InMemoryURLIndex describing its internal data and +// providing for restoring, rebuilding and updating that internal data. As +// this class is for exclusive use by the InMemoryURLIndex class there should +// be no calls from any other class. +// +// All public member functions are called on the main thread unless otherwise +// annotated. class URLIndexPrivateData : public base::RefCountedThreadSafe<URLIndexPrivateData> { public: - URLIndexPrivateData(); + // Creates a new instance of private data, creating or opening the cache + // database located in |history_dir|. |languages| is used to break down + // search terms, URLs, and page titles into words and characters. + URLIndexPrivateData(const FilePath& history_dir, + const std::string& languages); + + // Initializes the private data and its cache database. Returns true if the + // database is successfully initialized. Any failures will mark the cache + // database as not enabled. |sequence_token| is used to coordinate all + // future database operations (not including those performed during this + // initialization). Called on the DB thread. + bool Init(base::SequencedWorkerPool::SequenceToken sequence_token); + + // Performs a Clear() and then erases the cache database. Called on the + // worker pool sequenced by InMemoryURLIndex's |sequence_token_|. + void Reset(); + + // Returns true if there is no data in the index. + bool Empty() const; + + // Returns a copy of the private data for archiving purposes. + URLIndexPrivateData* Snapshot() const; + + // Closes the database. + void Shutdown(); + + // Verifies that the private data is consistent. + bool ValidateConsistency() const; + + // Given a string16 in |search_string|, scans the history index and returns a + // vector with all scored, matching history items. The |search_string| is + // broken down into individual terms (words), each of which must occur in the + // candidate history item's URL or page title for the item to qualify; + // however, the terms do not necessarily have to be adjacent. Once we have + // a set of candidates, they are filtered to insure that all |search_string| + // terms, as separated by whitespace, occur within the candidate's URL + // or page title. Scores are then calculated on no more than + // |kItemsToScoreLimit| candidates, as the scoring of such a large number of + // candidates may cause perceptible typing response delays in the omnibox. + // This is likely to occur for short omnibox terms such as 'h' and 'w' which + // will be found in nearly all history candidates. Results are sorted by + // descending score. The full results set (i.e. beyond the + // |kItemsToScoreLimit| limit) will be retained and used for subsequent calls + // to this function. + ScoredHistoryMatches HistoryItemsForTerms(const string16& search_string); + + // Adds the history item in |row| to the index if it does not already already + // exist and it meets the minimum 'quick' criteria. If the row already exists + // in the index then the index will be updated if the row still meets the + // criteria, otherwise the row will be removed from the index. Returns true + // if the index was actually updated. Posts updates to the cache database + // that are run on the worker pool sequenced by InMemoryURLIndex's + // |sequence_token_|. + bool UpdateURL(const URLRow& row); + + // Deletes index data for the history item with the given |url|. + // The item may not have actually been indexed, which is the case if it did + // not previously meet minimum 'quick' criteria. Returns true if the index + // was actually updated. Posts updates to the cache database that are run on + // the worker pool sequenced by InMemoryURLIndex's |sequence_token_|. + bool DeleteURL(const GURL& url); + + // Sets if the cache database is enabled. + void set_cache_enabled(bool enabled) { cache_enabled_ = enabled; } + + // Returns the cache database. + InMemoryURLCacheDatabase* cache_db() { return cache_db_.get(); } + + // Restores the index data from the contents of the cache database. This is + // called on the DB thread during profile startup and returns true upon a + // successful restoration. Restoration will fail if there is no cache + // database or the cache database has been corrupted. All other database + // operations (i.e. updates from site visits, etc.) will be postponed while + // this task is being run. + bool RestoreFromCacheTask(); + + // Constructs a new private data object by rebuilding its contents from the + // history database in |history_db|. Returns the new URLIndexPrivateData which + // on success will contain the rebuilt data but upon failure will be empty. + // |history_dir| points to the directory in which the cache database will be + // created. |old_data| provides the cache database and the languages to be + // used for breaking down search terms, URLs and page titles. This is called + // on the DB thread during profile startup iff restoring from the cache + // database fails (see also RestoreFromCacheTask()). All other database + // operations (i.e. updates from site visits, etc.) will be postponed while + // this task is being run. + static scoped_refptr<URLIndexPrivateData> RebuildFromHistory( + HistoryDatabase* history_db, + scoped_refptr<URLIndexPrivateData> old_data); + + // Completely refreshes the contents of the cache database using the contents + // of the in-memory index data. This task is performed on the sequenced + // blocking pool using the sequence_token with which this instance was + // Init'ed. A refresh will occur 1) during profile startup if a + // RebuildFromHistory(...) is required, or 2) at any time database corruption + // is detected while updating the database in an attempt to repair the + // corruption. + void RefreshCacheTask(); + + static void InitializeSchemeWhitelistForTesting( + std::set<std::string>* whitelist); private: friend class base::RefCountedThreadSafe<URLIndexPrivateData>; - ~URLIndexPrivateData(); - friend class AddHistoryMatch; friend class ::HistoryQuickProviderTest; - friend class InMemoryURLIndex; + friend class InMemoryURLCacheDatabase; + friend class InMemoryURLIndexCacheTest; friend class InMemoryURLIndexTest; - FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, CacheSaveRestore); + friend class InMemoryURLIndexTestBase; + friend class IntercessionaryIndexTest; + friend class URLIndexOldCacheTest; + friend class URLIndexPrivateDataTest; FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, HugeResultSet); - FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, Scoring); FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, TitleSearch); FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, TypedCharacterCaching); FRIEND_TEST_ALL_PREFIXES(InMemoryURLIndexTest, WhitelistedURLs); + FRIEND_TEST_ALL_PREFIXES(IntercessionaryIndexTest, CacheDatabaseFailure); + FRIEND_TEST_ALL_PREFIXES(IntercessionaryIndexTest, + ShutdownDuringCacheRefresh); FRIEND_TEST_ALL_PREFIXES(LimitedInMemoryURLIndexTest, Initialization); + FRIEND_TEST_ALL_PREFIXES(URLIndexPrivateDataTest, CacheFetch); // Support caching of term results so that we can optimize searches which // build upon a previous search. Each entry in this map represents one @@ -123,117 +232,40 @@ class URLIndexPrivateData const history::HistoryInfoMap& history_info_map_; }; - // Given a string16 in |term_string|, scans the history index and returns a - // vector with all scored, matching history items. The |term_string| is - // broken down into individual terms (words), each of which must occur in the - // candidate history item's URL or page title for the item to qualify; - // however, the terms do not necessarily have to be adjacent. Once we have - // a set of candidates, they are filtered to insure that all |term_string| - // terms, as separated by whitespace, occur within the candidate's URL - // or page title. Scores are then calculated on no more than - // |kItemsToScoreLimit| candidates, as the scoring of such a large number of - // candidates may cause perceptible typing response delays in the omnibox. - // This is likely to occur for short omnibox terms such as 'h' and 'w' which - // will be found in nearly all history candidates. Results are sorted by - // descending score. The full results set (i.e. beyond the - // |kItemsToScoreLimit| limit) will be retained and used for subsequent calls - // to this function. - ScoredHistoryMatches HistoryItemsForTerms(const string16& term_string); - - // Creates a new URLIndexPrivateData object, populates it from the contents - // of the cache file stored in |file_path|, and assigns it to |private_data|. - // |languages| will be used to break URLs and page titles into words. - static void RestoreFromFileTask( - const FilePath& file_path, - scoped_refptr<URLIndexPrivateData> private_data, - const std::string& languages); - - // Constructs a new object by restoring its contents from the file at |path|. - // Returns the new URLIndexPrivateData which on success will contain the - // restored data but upon failure will be empty. |languages| will be used to - // break URLs and page titles into words - static scoped_refptr<URLIndexPrivateData> RestoreFromFile( - const FilePath& path, - const std::string& languages); - - // Constructs a new object by rebuilding its contents from the history - // database in |history_db|. Returns the new URLIndexPrivateData which on - // success will contain the rebuilt data but upon failure will be empty. - // |languages| gives a list of language encodings by which the URLs and page - // titles are broken down into words and characters. - static scoped_refptr<URLIndexPrivateData> RebuildFromHistory( - HistoryDatabase* history_db, - const std::string& languages, - const std::set<std::string>& scheme_whitelist); - - // Writes |private_data| as a cache file to |file_path| and returns success - // via |succeeded|. - static void WritePrivateDataToCacheFileTask( - scoped_refptr<URLIndexPrivateData> private_data, - const FilePath& file_path, - scoped_refptr<RefCountedBool> succeeded); - - // Caches the index private data and writes the cache file to the profile - // directory. Called by WritePrivateDataToCacheFileTask. - bool SaveToFile(const FilePath& file_path); - - // Initializes all index data members in preparation for restoring the index - // from the cache or a complete rebuild from the history database. - void Clear(); + // Creates a new instance of private data for purposes of rebuilding from + // the history database while simultaneously allowing continued use of an + // older private data |old_data|. The old data will still be used for + // providing search results. Any updates to the private data will be queued + // for application to the new data once it has been successfully rebuilt. + URLIndexPrivateData(const URLIndexPrivateData& old_data); - // Returns true if there is no data in the index. - bool Empty() const; + // The following constructor is for unit testing purposes only. + URLIndexPrivateData(); - // Creates a copy of ourself. - scoped_refptr<URLIndexPrivateData> Duplicate() const; + virtual ~URLIndexPrivateData(); - // Adds |word_id| to |history_id|'s entry in the history/word map, - // creating a new entry if one does not already exist. - void AddToHistoryIDWordMap(HistoryID history_id, WordID word_id); + // Returns true if Shutdown() has been called. + bool shutdown() const { return shutdown_; } - // Given a set of Char16s, finds words containing those characters. - WordIDSet WordIDSetForTermChars(const Char16Set& term_chars); + // Gets if the cache database is enabled. + bool cache_enabled() const { return cache_enabled_ && cache_db_; } + + // Initializes all index private data members in preparation for restoring, + // rebuilding or resetting the index. + void Clear(); // URL History indexing support functions. // Indexes one URL history item as described by |row|. Returns true if the - // row was actually indexed. |languages| gives a list of language encodings by - // which the URLs and page titles are broken down into words and characters. - // |scheme_whitelist| is used to filter non-qualifying schemes. - bool IndexRow(const URLRow& row, - const std::string& languages, - const std::set<std::string>& scheme_whitelist); - - // Adds the history item in |row| to the index if it does not already already - // exist and it meets the minimum 'quick' criteria. If the row already exists - // in the index then the index will be updated if the row still meets the - // criteria, otherwise the row will be removed from the index. Returns true - // if the index was actually updated. |languages| gives a list of language - // encodings by which the URLs and page titles are broken down into words and - // characters. |scheme_whitelist| is used to filter non-qualifying schemes. - bool UpdateURL(const URLRow& row, - const std::string& languages, - const std::set<std::string>& scheme_whitelist); - - // Deletes index data for the history item with the given |url|. - // The item may not have actually been indexed, which is the case if it did - // not previously meet minimum 'quick' criteria. Returns true if the index - // was actually updated. - bool DeleteURL(const GURL& url); + // row was actually indexed. + bool IndexRow(const URLRow& row); // Parses and indexes the words in the URL and page title of |row| and // calculate the word starts in each, saving the starts in |word_starts|. // |languages| gives a list of language encodings by which the URLs and page // titles are broken down into words and characters. void AddRowWordsToIndex(const URLRow& row, - RowWordStarts* word_starts, - const std::string& languages); - - // Removes |row| and all associated words and characters from the index. - void RemoveRowFromIndex(const URLRow& row); - - // Removes all words and characters associated with |row| from the index. - void RemoveRowWordsFromIndex(const URLRow& row); + RowWordStarts* word_starts); // Given a single word in |uni_word|, adds a reference for the containing // history item identified by |history_id| to the index. @@ -247,6 +279,16 @@ class URLIndexPrivateData // |history_id| as the initial element of the word's set. void AddWordHistory(const string16& uni_word, HistoryID history_id); + // Removes |row| and all associated words and characters from the index. + void RemoveRowFromIndex(const URLRow& row); + + // Removes all words and characters associated with |row| from the index. + void RemoveRowWordsFromIndex(const URLRow& row); + + // Adds |word_id| to |history_id|'s entry in the history/word map, + // creating a new entry if one does not already exist. + void AddToHistoryIDWordMap(HistoryID history_id, WordID word_id); + // Clears |used_| for each item in the search term cache. void ResetSearchTermCache(); @@ -258,41 +300,53 @@ class URLIndexPrivateData // ids for the given term given in |term|. HistoryIDSet HistoryIDsForTerm(const string16& term); - // Encode a data structure into the protobuf |cache|. - void SavePrivateData(imui::InMemoryURLIndexCacheItem* cache) const; - void SaveWordList(imui::InMemoryURLIndexCacheItem* cache) const; - void SaveWordMap(imui::InMemoryURLIndexCacheItem* cache) const; - void SaveCharWordMap(imui::InMemoryURLIndexCacheItem* cache) const; - void SaveWordIDHistoryMap(imui::InMemoryURLIndexCacheItem* cache) const; - void SaveHistoryInfoMap(imui::InMemoryURLIndexCacheItem* cache) const; - void SaveWordStartsMap(imui::InMemoryURLIndexCacheItem* cache) const; - - // Decode a data structure from the protobuf |cache|. Return false if there - // is any kind of failure. |languages| will be used to break URLs and page - // titles into words - bool RestorePrivateData(const imui::InMemoryURLIndexCacheItem& cache, - const std::string& languages); - bool RestoreWordList(const imui::InMemoryURLIndexCacheItem& cache); - bool RestoreWordMap(const imui::InMemoryURLIndexCacheItem& cache); - bool RestoreCharWordMap(const imui::InMemoryURLIndexCacheItem& cache); - bool RestoreWordIDHistoryMap(const imui::InMemoryURLIndexCacheItem& cache); - bool RestoreHistoryInfoMap(const imui::InMemoryURLIndexCacheItem& cache); - bool RestoreWordStartsMap(const imui::InMemoryURLIndexCacheItem& cache, - const std::string& languages); + // Given a set of Char16s, finds words containing those characters. + WordIDSet WordIDSetForTermChars(const Char16Set& term_chars); + + // Restores our contents from the cache database |cache_db|. + bool RestoreFromCache(InMemoryURLCacheDatabase* cache_db); + + // Deletes any old style protobuf-based cache file. + void DeleteOldVersionCacheFile() const; + + // Constructs a file path for the cache database within the same directory + // where the history database is kept and saves that path to |file_path|. + // Returns true if |file_path| can be successfully constructed. + bool GetCacheDBPath(FilePath* file_path); + + // Sets the cache database for testing. Takes ownership of |test_db|. + void SetCacheDatabaseForTesting(InMemoryURLCacheDatabase* test_db); // Determines if |gurl| has a whitelisted scheme and returns true if so. static bool URLSchemeIsWhitelisted(const GURL& gurl, const std::set<std::string>& whitelist); + // Directory where cache database (and older protobuf-based cache file) + // resides. Except when unit testing, this is the same directory in which + // the profile's history database is found. + FilePath history_dir_; + + // Languages used during the word-breaking process during indexing. + std::string languages_; + // Cache of search terms. SearchTermCacheMap search_term_cache_; - // Start of data members that are cached ------------------------------------- + // The cache database. + scoped_refptr<InMemoryURLCacheDatabase> cache_db_; + bool cache_enabled_; // true if the cache is enabled. - // The version of the cache file most recently used to restore this instance - // of the private data. If the private data was rebuilt from the history - // database this will be 0. - int restored_cache_version_; + // Guard that prevents inconsistent |cache_enabled_| state during possible + // simultaneous Init() and Shutdown() calls. There is a period of time during + // Init() when the cache database has not yet been fully initialized and thus + // is not enabled. A Shutdown() may occur during this time; it must wait for + // the initialization to complete and |cache_enabled_|'s state to be set. + base::Lock initialization_lock_; + + // Set to true once the shutdown process has begun. + bool shutdown_; + + // Start of data members that are cached ------------------------------------- // A list of all of indexed words. The index of a word in this list is the // ID of the word in the word_map_. It reduces the memory overhead by @@ -336,10 +390,8 @@ class URLIndexPrivateData // End of data members that are cached --------------------------------------- - // For unit testing only. Specifies the version of the cache file to be saved. - // Used only for testing upgrading of an older version of the cache upon - // restore. - int saved_cache_version_; + // Only URLs with a whitelisted scheme are indexed. + std::set<std::string> scheme_whitelist_; // Used for unit testing only. Records the number of candidate history items // at three stages in the index searching process. diff --git a/chrome/browser/history/url_index_private_data_unittest.cc b/chrome/browser/history/url_index_private_data_unittest.cc new file mode 100644 index 0000000..67e64fd --- /dev/null +++ b/chrome/browser/history/url_index_private_data_unittest.cc @@ -0,0 +1,283 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/file_util.h" +#include "base/path_service.h" +#include "base/scoped_temp_dir.h" +#include "base/time.h" +#include "base/utf_string_conversions.h" +#include "chrome/browser/history/in_memory_url_cache_database.h" +#include "chrome/browser/history/url_index_private_data.h" +#include "chrome/common/chrome_paths.h" +#include "content/public/browser/browser_thread.h" +#include "googleurl/src/gurl.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace history { + +struct TestURLInfo { + std::string url; + std::string title; + int visit_count; + int typed_count; + int days_from_now; +} private_data_test_db[] = { + {"http://www.google.com/", "Google", 3, 3, 0}, + {"http://slashdot.org/favorite_page.html", "Favorite page", 200, 100, 0}, + {"http://kerneltrap.org/not_very_popular.html", "Less popular", 4, 0, 0}, + {"http://freshmeat.net/unpopular.html", "Unpopular", 1, 1, 0}, + {"http://news.google.com/?ned=us&topic=n", "Google News - U.S.", 2, 2, 0}, + {"http://news.google.com/", "Google News", 1, 1, 0}, + {"http://foo.com/", "Dir", 200, 100, 0}, + {"http://foo.com/dir/", "Dir", 2, 1, 10}, + {"http://foo.com/dir/another/", "Dir", 5, 10, 0}, + {"http://foo.com/dir/another/again/", "Dir", 5, 1, 0}, + {"http://foo.com/dir/another/again/myfile.html", "File", 3, 2, 0}, + {"http://visitedest.com/y/a", "VA", 10, 1, 20}, + {"http://visitedest.com/y/b", "VB", 9, 1, 20}, + {"http://visitedest.com/x/c", "VC", 8, 1, 20}, + {"http://visitedest.com/x/d", "VD", 7, 1, 20}, + {"http://visitedest.com/y/e", "VE", 6, 1, 20}, + {"http://typeredest.com/y/a", "TA", 3, 5, 0}, + {"http://typeredest.com/y/b", "TB", 3, 4, 0}, + {"http://typeredest.com/x/c", "TC", 3, 3, 0}, + {"http://typeredest.com/x/d", "TD", 3, 2, 0}, + {"http://typeredest.com/y/e", "TE", 3, 1, 0}, + {"http://daysagoest.com/y/a", "DA", 1, 1, 0}, + {"http://daysagoest.com/y/b", "DB", 1, 1, 1}, + {"http://daysagoest.com/x/c", "DC", 1, 1, 2}, + {"http://daysagoest.com/x/d", "DD", 1, 1, 3}, + {"http://daysagoest.com/y/e", "DE", 1, 1, 4}, + {"http://abcdefghixyzjklmnopqrstuvw.com/a", "", 3, 1, 0}, + {"http://spaces.com/path%20with%20spaces/foo.html", "Spaces", 2, 2, 0}, + {"http://abcdefghijklxyzmnopqrstuvw.com/a", "", 3, 1, 0}, + {"http://abcdefxyzghijklmnopqrstuvw.com/a", "", 3, 1, 0}, + {"http://abcxyzdefghijklmnopqrstuvw.com/a", "", 3, 1, 0}, + {"http://xyzabcdefghijklmnopqrstuvw.com/a", "", 3, 1, 0}, + {"http://cda.com/Dogs%20Cats%20Gorillas%20Sea%20Slugs%20and%20Mice", + "Dogs & Cats & Mice & Other Animals", 1, 1, 0}, + {"https://monkeytrap.org/", "", 3, 1, 0}, +}; + +class URLIndexPrivateDataTest : public testing::Test { + public: + URLIndexPrivateDataTest(); + virtual ~URLIndexPrivateDataTest(); + + virtual void SetUp() OVERRIDE; + + void GetTestData(size_t* data_count, TestURLInfo** test_data); + + // Fills test data into the history system. + void FillData(); + + // Data verification helper functions. + void ExpectPrivateDataNotEmpty(const URLIndexPrivateData& data); + void ExpectPrivateDataEqual(const URLIndexPrivateData& expected, + const URLIndexPrivateData& actual); + + protected: + scoped_refptr<URLIndexPrivateData> private_data_; + ScopedTempDir temp_cache_dir_; +}; + +URLIndexPrivateDataTest::URLIndexPrivateDataTest() {} +URLIndexPrivateDataTest::~URLIndexPrivateDataTest() {} + +void URLIndexPrivateDataTest::SetUp() { + ASSERT_TRUE(temp_cache_dir_.CreateUniqueTempDir()); + private_data_ = new URLIndexPrivateData(temp_cache_dir_.path(), "en"); + private_data_->Init( + content::BrowserThread::GetBlockingPool()->GetSequenceToken()); + FillData(); +} + +void URLIndexPrivateDataTest::GetTestData(size_t* data_count, + TestURLInfo** test_data) { + DCHECK(data_count); + DCHECK(test_data); + *data_count = arraysize(private_data_test_db); + *test_data = &private_data_test_db[0]; +} + +void URLIndexPrivateDataTest::FillData() { + size_t data_count = 0; + TestURLInfo* test_data = NULL; + GetTestData(&data_count, &test_data); + for (size_t i = 0; i < data_count; ++i) { + const TestURLInfo& cur(test_data[i]); + const GURL current_url(cur.url); + base::Time visit_time = + base::Time::Now() - base::TimeDelta::FromDays(cur.days_from_now); + history::URLRow url_info(current_url); + url_info.set_id(i); + url_info.set_title(UTF8ToUTF16(cur.title)); + url_info.set_visit_count(cur.visit_count); + url_info.set_typed_count(cur.typed_count); + url_info.set_last_visit(visit_time); + url_info.set_hidden(false); + private_data_->UpdateURL(url_info); + } + // Stall until the pending operations have completed. + content::BrowserThread::GetBlockingPool()->FlushForTesting(); +} + +void URLIndexPrivateDataTest::ExpectPrivateDataNotEmpty( + const URLIndexPrivateData& data) { + EXPECT_FALSE(data.word_list_.empty()); + // available_words_ will be empty since we have freshly built the + // data set for these tests. + EXPECT_TRUE(data.available_words_.empty()); + EXPECT_FALSE(data.word_map_.empty()); + EXPECT_FALSE(data.char_word_map_.empty()); + EXPECT_FALSE(data.word_id_history_map_.empty()); + EXPECT_FALSE(data.history_id_word_map_.empty()); + EXPECT_FALSE(data.history_info_map_.empty()); + EXPECT_FALSE(data.word_starts_map_.empty()); +} + +// Helper function which compares two maps for equivalence. The maps' values +// are associative containers and their contents are compared as well. +template<typename T> +void ExpectMapOfContainersIdentical(const T& expected, const T& actual) { + ASSERT_EQ(expected.size(), actual.size()); + for (typename T::const_iterator expected_iter = expected.begin(); + expected_iter != expected.end(); ++expected_iter) { + typename T::const_iterator actual_iter = actual.find(expected_iter->first); + ASSERT_TRUE(actual_iter != actual.end()); + typename T::mapped_type const& expected_values(expected_iter->second); + typename T::mapped_type const& actual_values(actual_iter->second); + ASSERT_EQ(expected_values.size(), actual_values.size()); + for (typename T::mapped_type::const_iterator set_iter = + expected_values.begin(); set_iter != expected_values.end(); ++set_iter) + EXPECT_EQ(actual_values.count(*set_iter), + expected_values.count(*set_iter)); + } +} + +void URLIndexPrivateDataTest::ExpectPrivateDataEqual( + const URLIndexPrivateData& expected, + const URLIndexPrivateData& actual) { + EXPECT_EQ(expected.word_list_.size(), actual.word_list_.size()); + EXPECT_EQ(expected.word_map_.size(), actual.word_map_.size()); + EXPECT_EQ(expected.char_word_map_.size(), actual.char_word_map_.size()); + EXPECT_EQ(expected.word_id_history_map_.size(), + actual.word_id_history_map_.size()); + EXPECT_EQ(expected.history_id_word_map_.size(), + actual.history_id_word_map_.size()); + EXPECT_EQ(expected.history_info_map_.size(), actual.history_info_map_.size()); + EXPECT_EQ(expected.word_starts_map_.size(), actual.word_starts_map_.size()); + // WordList must be index-by-index equal. + size_t count = expected.word_list_.size(); + for (size_t i = 0; i < count; ++i) + EXPECT_EQ(expected.word_list_[i], actual.word_list_[i]); + + ExpectMapOfContainersIdentical(expected.char_word_map_, + actual.char_word_map_); + ExpectMapOfContainersIdentical(expected.word_id_history_map_, + actual.word_id_history_map_); + ExpectMapOfContainersIdentical(expected.history_id_word_map_, + actual.history_id_word_map_); + + for (HistoryInfoMap::const_iterator expected_info = + expected.history_info_map_.begin(); + expected_info != expected.history_info_map_.end(); ++expected_info) { + HistoryInfoMap::const_iterator actual_info = + actual.history_info_map_.find(expected_info->first); + // NOTE(yfriedman): ASSERT_NE can't be used due to incompatibility between + // gtest and STLPort in the Android build. See + // http://code.google.com/p/googletest/issues/detail?id=359 + ASSERT_TRUE(actual_info != actual.history_info_map_.end()); + const URLRow& expected_row(expected_info->second); + const URLRow& actual_row(actual_info->second); + EXPECT_EQ(expected_row.visit_count(), actual_row.visit_count()); + EXPECT_EQ(expected_row.typed_count(), actual_row.typed_count()); + EXPECT_EQ(expected_row.last_visit(), actual_row.last_visit()); + EXPECT_EQ(expected_row.url(), actual_row.url()); + } + + for (WordStartsMap::const_iterator expected_starts = + expected.word_starts_map_.begin(); + expected_starts != expected.word_starts_map_.end(); + ++expected_starts) { + WordStartsMap::const_iterator actual_starts = + actual.word_starts_map_.find(expected_starts->first); + // NOTE(yfriedman): ASSERT_NE can't be used due to incompatibility between + // gtest and STLPort in the Android build. See + // http://code.google.com/p/googletest/issues/detail?id=359 + ASSERT_TRUE(actual_starts != actual.word_starts_map_.end()); + const RowWordStarts& expected_word_starts(expected_starts->second); + const RowWordStarts& actual_word_starts(actual_starts->second); + EXPECT_EQ(expected_word_starts.url_word_starts_.size(), + actual_word_starts.url_word_starts_.size()); + EXPECT_TRUE(std::equal(expected_word_starts.url_word_starts_.begin(), + expected_word_starts.url_word_starts_.end(), + actual_word_starts.url_word_starts_.begin())); + EXPECT_EQ(expected_word_starts.title_word_starts_.size(), + actual_word_starts.title_word_starts_.size()); + EXPECT_TRUE(std::equal(expected_word_starts.title_word_starts_.begin(), + expected_word_starts.title_word_starts_.end(), + actual_word_starts.title_word_starts_.begin())); + } +} + +TEST_F(URLIndexPrivateDataTest, CacheFetch) { + // Compare the in-memory index data to the on-disk cached data. + const URLIndexPrivateData& expected_data(*(private_data_.get())); + ExpectPrivateDataNotEmpty(expected_data); + + // Grab the index data from the cache. + scoped_refptr<URLIndexPrivateData> cached_data = new URLIndexPrivateData; + URLIndexPrivateData* actual_data = cached_data.get(); + ASSERT_TRUE(private_data_->cache_db_->RestorePrivateData(actual_data)); + EXPECT_TRUE(actual_data->ValidateConsistency()); + ExpectPrivateDataEqual(expected_data, *actual_data); +} + +class URLIndexOldCacheTest : public testing::Test { + public: + URLIndexOldCacheTest(); + virtual ~URLIndexOldCacheTest(); + + virtual void SetUp() OVERRIDE; + + protected: + scoped_refptr<URLIndexPrivateData> private_data_; + ScopedTempDir temp_cache_dir_; +}; + +URLIndexOldCacheTest::URLIndexOldCacheTest() {} +URLIndexOldCacheTest::~URLIndexOldCacheTest() {} + +void URLIndexOldCacheTest::SetUp() { + // Create a file that looks like an old protobuf-based cache file. + ASSERT_TRUE(temp_cache_dir_.CreateUniqueTempDir()); + std::string dummy_data("DUMMY DATA"); + int size = dummy_data.size(); + FilePath path = temp_cache_dir_.path().Append( + FILE_PATH_LITERAL("History Provider Cache")); + ASSERT_EQ(size, file_util::WriteFile(path, dummy_data.c_str(), size)); + ASSERT_TRUE(file_util::PathExists(path)); + + // Continue initializing. This will attempt to restore from the SQLite cache + // but it doesn't exist so the old protobuf file should be automatically + // deleted. + private_data_ = new URLIndexPrivateData(temp_cache_dir_.path(), "en"); + private_data_->Init( + content::BrowserThread::GetBlockingPool()->GetSequenceToken()); + EXPECT_FALSE(private_data_->RestoreFromCacheTask()); +} + +TEST_F(URLIndexOldCacheTest, CacheProtobufDelete) { + // If an old, protobuf-based cache file exists then it should be being deleted + // when an attempt is made to restore the index data from the SQLite cache but + // such SQLite cache does not exist. This will happen the first time a user + // runs Chrome after the SQLite-based cache implementation has been added. + // All we have to do here is verify that the file does not exist. + FilePath path = temp_cache_dir_.path().Append( + FILE_PATH_LITERAL("History Provider Cache")); + ASSERT_FALSE(file_util::PathExists(path)); +} + +} // namespace history diff --git a/chrome/browser/visitedlink/visitedlink_master.cc b/chrome/browser/visitedlink/visitedlink_master.cc index 1c3f027..63bf461 100644 --- a/chrome/browser/visitedlink/visitedlink_master.cc +++ b/chrome/browser/visitedlink/visitedlink_master.cc @@ -147,7 +147,7 @@ class VisitedLinkMaster::TableBuilder void DisownMaster(); // HistoryService::URLEnumerator - virtual void OnURL(const GURL& url); + virtual void OnURL(const history::URLRow& url_row); virtual void OnComplete(bool succeed); private: @@ -953,7 +953,8 @@ void VisitedLinkMaster::TableBuilder::DisownMaster() { master_ = NULL; } -void VisitedLinkMaster::TableBuilder::OnURL(const GURL& url) { +void VisitedLinkMaster::TableBuilder::OnURL(const history::URLRow& url_row) { + const GURL& url((url_row.url())); if (!url.is_empty()) { fingerprints_.push_back(VisitedLinkMaster::ComputeURLFingerprint( url.spec().data(), url.spec().length(), salt_)); diff --git a/chrome/chrome_browser.gypi b/chrome/chrome_browser.gypi index 2936b55..6e0a0ef 100644 --- a/chrome/chrome_browser.gypi +++ b/chrome/chrome_browser.gypi @@ -1220,6 +1220,8 @@ 'browser/history/in_memory_database.h', 'browser/history/in_memory_history_backend.cc', 'browser/history/in_memory_history_backend.h', + 'browser/history/in_memory_url_cache_database.cc', + 'browser/history/in_memory_url_cache_database.h', 'browser/history/in_memory_url_index.cc', 'browser/history/in_memory_url_index.h', 'browser/history/in_memory_url_index_types.cc', diff --git a/chrome/chrome_tests.gypi b/chrome/chrome_tests.gypi index c730d12..f3cfb83 100644 --- a/chrome/chrome_tests.gypi +++ b/chrome/chrome_tests.gypi @@ -1320,6 +1320,8 @@ 'browser/history/history_unittest_base.h', 'browser/history/in_memory_url_index_types_unittest.cc', 'browser/history/in_memory_url_index_unittest.cc', + 'browser/history/in_memory_url_index_unittest_base.cc', + 'browser/history/in_memory_url_index_unittest_base.h', 'browser/history/query_parser_unittest.cc', 'browser/history/scored_history_match_unittest.cc', 'browser/history/shortcuts_backend_unittest.cc', @@ -1332,6 +1334,7 @@ 'browser/history/top_sites_database_unittest.cc', 'browser/history/top_sites_unittest.cc', 'browser/history/url_database_unittest.cc', + 'browser/history/url_index_private_data_unittest.cc', 'browser/history/visit_database_unittest.cc', 'browser/history/visit_filter_unittest.cc', 'browser/history/visit_tracker_unittest.cc', diff --git a/chrome/common/chrome_constants.cc b/chrome/common/chrome_constants.cc index e1a1c3b..14611c9 100644 --- a/chrome/common/chrome_constants.cc +++ b/chrome/common/chrome_constants.cc @@ -139,6 +139,9 @@ const FilePath::CharType kFaviconsFilename[] = FPL("Favicons"); const FilePath::CharType kHistoryBookmarksFileName[] = FPL("Bookmarks From History"); const FilePath::CharType kHistoryFilename[] = FPL("History"); +const FilePath::CharType kHQPCacheFilename[] = FPL("History Provider Cache"); +const FilePath::CharType kHQPCacheDBFilename[] = + FPL("History Provider Cache DB"); const FilePath::CharType kIsolatedAppStateDirname[] = FPL("Isolated Apps"); const FilePath::CharType kJumpListIconDirname[] = FPL("JumpListIcons"); const FilePath::CharType kLocalStateFilename[] = FPL("Local State"); diff --git a/chrome/common/chrome_constants.h b/chrome/common/chrome_constants.h index 180af47..d54ac2b 100644 --- a/chrome/common/chrome_constants.h +++ b/chrome/common/chrome_constants.h @@ -65,6 +65,8 @@ extern const FilePath::CharType kExtensionsCookieFilename[]; extern const FilePath::CharType kFaviconsFilename[]; extern const FilePath::CharType kHistoryBookmarksFileName[]; extern const FilePath::CharType kHistoryFilename[]; +extern const FilePath::CharType kHQPCacheFilename[]; +extern const FilePath::CharType kHQPCacheDBFilename[]; extern const FilePath::CharType kIsolatedAppStateDirname[]; extern const FilePath::CharType kJumpListIconDirname[]; extern const FilePath::CharType kLocalStateFilename[]; diff --git a/chrome/common/chrome_notification_types.h b/chrome/common/chrome_notification_types.h index 002f93f..4521730 100644 --- a/chrome/common/chrome_notification_types.h +++ b/chrome/common/chrome_notification_types.h @@ -318,6 +318,13 @@ enum NotificationType { // The details are none and the source is a Profile*. NOTIFICATION_PROFILE_URL_REQUEST_CONTEXT_GETTER_INITIALIZED, + // HistoryQuickProvider and InMemoryURLIndex ------------------------------- + + // Sent when an update operation to the InMemoryURLCacheDatabase fails + // indicating that the database is probably corrupt. The source is the + // instance of the InMemoryURLCacheDatabase. Details is not used. + NOTIFICATION_IN_MEMORY_URL_CACHE_DATABASE_FAILURE, + // TopSites ---------------------------------------------------------------- // Sent by TopSites when it finishes loading. The source is the profile the diff --git a/chrome/test/base/testing_profile.cc b/chrome/test/base/testing_profile.cc index eabf2e3..3b926c0 100644 --- a/chrome/test/base/testing_profile.cc +++ b/chrome/test/base/testing_profile.cc @@ -319,15 +319,20 @@ void TestingProfile::CreateHistoryService(bool delete_file, bool no_db) { path = path.Append(chrome::kHistoryFilename); file_util::Delete(path, false); } - // This will create and init the history service. HistoryService* history_service = static_cast<HistoryService*>( HistoryServiceFactory::GetInstance()->SetTestingFactoryAndUse( this, BuildHistoryService).get()); - if (!history_service->Init(this->GetPath(), - BookmarkModelFactory::GetForProfile(this), - no_db)) { + if (!InitHistoryService(history_service, no_db)) HistoryServiceFactory::GetInstance()->SetTestingFactoryAndUse(this, NULL); - } +} + +bool TestingProfile::InitHistoryService(HistoryService* history_service, + bool no_db) { + DCHECK(history_service); + // By default, disable the InMemoryURLIndex's cache database. + return history_service->Init(GetPath(), + BookmarkModelFactory::GetForProfile(this), + no_db, true); } void TestingProfile::DestroyHistoryService() { diff --git a/chrome/test/base/testing_profile.h b/chrome/test/base/testing_profile.h index 2cf3970..1db59a8 100644 --- a/chrome/test/base/testing_profile.h +++ b/chrome/test/base/testing_profile.h @@ -38,6 +38,7 @@ class SpecialStoragePolicy; class CommandLine; class ExtensionSpecialStoragePolicy; class FaviconService; +class HistoryService; class HostContentSettingsMap; class PrefService; class ProfileDependencyManager; @@ -140,6 +141,12 @@ class TestingProfile : public Profile { // for testing error conditions. void CreateHistoryService(bool delete_file, bool no_db); + // Initializes the history service. If |no_db| is true, the history backend + // will fail to initialize its database; this is useful for testing error + // conditions. Returns true upon success. + virtual bool InitHistoryService(HistoryService* history_service, + bool no_db); + // Shuts down and nulls out the reference to HistoryService. void DestroyHistoryService(); diff --git a/chrome/test/data/History/history_quick_provider_ordering_test.db.txt b/chrome/test/data/History/history_quick_provider_ordering_test.db.txt new file mode 100644 index 0000000..d2c8bf2 --- /dev/null +++ b/chrome/test/data/History/history_quick_provider_ordering_test.db.txt @@ -0,0 +1,49 @@ +/* + The schema of the database is defined by HISTORY_URL_ROW_FIELDS found in + url_database.h and is equivalent to: + + CREATE TABLE urls(id INTEGER PRIMARY KEY, + url LONGVARCHAR, + title LONGVARCHAR, + visit_count INTEGER DEFAULT 0 NOT NULL, + typed_count INTEGER DEFAULT 0 NOT NULL, + last_visit_time INTEGER NOT NULL, + hidden INTEGER DEFAULT 0 NOT NULL, + favicon_id INTEGER DEFAULT 0 NOT NULL); + + The quick history autocomplete provider filters out history items that: + 1) have not been visited in kLowQualityMatchAgeLimitInDays, AND + 2) for which the URL was not explicitly typed at least + kLowQualityMatchTypedLimit + 1 times, AND + 3) have not been visited at least kLowQualityMatchVisitLimit + 1 times. + So we create history items in all of those combinations. + + Note that the last_visit_time column for this test table represents the + relative number of days prior to 'today' to which the final column + value will be set during test setup. Beware: Do not set this number + to be equal to kLowQualityMatchAgeLimitInDays. + + The ordering, URLs and titles must be kept in sync with the unit tests found + in in_memory_url_index_unittest.cc. +*/ +INSERT INTO "urls" VALUES(1,'http://www.teamliquid.net/tlpd/korean/games/21648_bisu_vs_iris','',6,3,256,0,29); +INSERT INTO "urls" VALUES(2,'http://www.amazon.com/','amazon.com: online shopping for electronics, apparel, computers, books, dvds & more',20,20,10,0,29); +INSERT INTO "urls" VALUES(3,'http://www.teamliquid.net/forum/viewmessage.php?topic_id=52045¤tpage=83','google images',6,6,0,0,29); +INSERT INTO "urls" VALUES(4,'http://www.tempurpedic.com/','tempur-pedic',7,7,0,0,29); +INSERT INTO "urls" VALUES(5,'http://www.teamfortress.com/','',5,5,6,0,29); +INSERT INTO "urls" VALUES(6,'http://www.rottentomatoes.com/','',3,3,7,0,29); +INSERT INTO "urls" VALUES(7,'http://music.google.com/music/listen?u=0#start_pl','',3,3,9,0,29); +INSERT INTO "urls" VALUES(8,'https://www.emigrantdirect.com/','high interest savings account, high yield savings - emigrantdirect',5,5,3,0,29); +INSERT INTO "urls" VALUES(9,'http://store.steampowered.com/','',6,6,1,0,29); +INSERT INTO "urls" VALUES(10,'http://techmeme.com/','techmeme',111,110,4,0,29); +INSERT INTO "urls" VALUES(11,'http://www.teamliquid.net/tlpd','team liquid progaming database',15,15,2,0,29); +INSERT INTO "urls" VALUES(12,'http://store.steampowered.com/','the steam summer camp sale',6,6,1,0,29); +INSERT INTO "urls" VALUES(13,'http://www.teamliquid.net/tlpd/korean/players','tlpd - bw korean - player index',100,45,219,0,29); +INSERT INTO "urls" VALUES(14,'http://slashdot.org/','slashdot: news for nerds, stuff that matters',3,3,6,0,29); +INSERT INTO "urls" VALUES(15,'http://translate.google.com/','google translate',3,3,0,0,29); +INSERT INTO "urls" VALUES(16,'http://arstechnica.com/','ars technica',3,3,3,0,29); +INSERT INTO "urls" VALUES(17,'http://www.rottentomatoes.com/','movies | movie trailers | reviews - rotten tomatoes',3,3,7,0,29); +INSERT INTO "urls" VALUES(18,'http://www.teamliquid.net/','team liquid - starcraft 2 and brood war pro gaming news',26,25,3,0,29); +INSERT INTO "urls" VALUES(19,'http://metaleater.com/','metaleater',4,3,8,0,29); +INSERT INTO "urls" VALUES(20,'http://half.com/','half.com: textbooks , books , music , movies , games , video games',4,4,6,0,29); +INSERT INTO "urls" VALUES(21,'http://teamliquid.net/','team liquid - starcraft 2 and brood war pro gaming news',8,5,9,0,29); diff --git a/chrome/test/data/History/history_quick_provider_test.db.txt b/chrome/test/data/History/history_quick_provider_test.db.txt new file mode 100644 index 0000000..4629d43 --- /dev/null +++ b/chrome/test/data/History/history_quick_provider_test.db.txt @@ -0,0 +1,62 @@ +/* + The schema of the database is defined by HISTORY_URL_ROW_FIELDS found in + url_database.h and is equivalent to: + + CREATE TABLE urls(id INTEGER PRIMARY KEY, + url LONGVARCHAR, + title LONGVARCHAR, + visit_count INTEGER DEFAULT 0 NOT NULL, + typed_count INTEGER DEFAULT 0 NOT NULL, + last_visit_time INTEGER NOT NULL, + hidden INTEGER DEFAULT 0 NOT NULL, + favicon_id INTEGER DEFAULT 0 NOT NULL); + + The quick history autocomplete provider filters out history items that: + 1) have not been visited in kLowQualityMatchAgeLimitInDays, AND + 2) for which the URL was not explicitly typed at least + kLowQualityMatchTypedLimit + 1 times, AND + 3) have not been visited at least kLowQualityMatchVisitLimit + 1 times. + So we create history items in all of those combinations. + + Note that the last_visit_time column for this test table represents the + relative number of days prior to 'today' to which the final column + value will be set during test setup. Beware: Do not set this number + to be equal to kLowQualityMatchAgeLimitInDays. + + The ordering, URLs and titles must be kept in sync with the unit tests found + in in_memory_url_index_unittest.cc. +*/ +INSERT INTO "urls" VALUES(1,'http://www.google.com/','Google',3,3,0,0,29); +INSERT INTO "urls" VALUES(2,'http://slashdot.org/favorite_page.html','Favorite page',200,100,0,0,29); +INSERT INTO "urls" VALUES(3,'http://kerneltrap.org/not_very_popular.html','Less popular',4,0,0,0,29); +INSERT INTO "urls" VALUES(4,'http://freshmeat.net/unpopular.html','Unpopular',1,1,0,0,29); +INSERT INTO "urls" VALUES(5,'http://news.google.com/?ned=us&topic=n','Google News - U.S.',2,2,0,0,29); +INSERT INTO "urls" VALUES(6,'http://news.google.com/','Google News',1,1,0,0,29); +INSERT INTO "urls" VALUES(7,'http://foo.com/','Dir',200,100,0,0,29); +INSERT INTO "urls" VALUES(8,'http://foo.com/dir/','Dir',2,1,10,0,29); +INSERT INTO "urls" VALUES(9,'http://foo.com/dir/another/','Dir',5,10,0,0,29); +INSERT INTO "urls" VALUES(10,'http://foo.com/dir/another/again/','Dir',5,1,0,0,29); +INSERT INTO "urls" VALUES(11,'http://foo.com/dir/another/again/myfile.html','File',3,2,0,0,29); +INSERT INTO "urls" VALUES(12,'http://visitedest.com/y/a','VA',10,1,20,0,29); +INSERT INTO "urls" VALUES(13,'http://visitedest.com/y/b','VB',9,1,20,0,29); +INSERT INTO "urls" VALUES(14,'http://visitedest.com/x/c','VC',8,1,20,0,29); +INSERT INTO "urls" VALUES(15,'http://visitedest.com/x/d','VD',7,1,20,0,29); +INSERT INTO "urls" VALUES(16,'http://visitedest.com/y/e','VE',6,1,20,0,29); +INSERT INTO "urls" VALUES(17,'http://typeredest.com/y/a','TA',3,5,0,0,29); +INSERT INTO "urls" VALUES(18,'http://typeredest.com/y/b','TB',3,4,0,0,29); +INSERT INTO "urls" VALUES(19,'http://typeredest.com/x/c','TC',3,3,0,0,29); +INSERT INTO "urls" VALUES(20,'http://typeredest.com/x/d','TD',3,2,0,0,29); +INSERT INTO "urls" VALUES(21,'http://typeredest.com/y/e','TE',3,1,0,0,29); +INSERT INTO "urls" VALUES(22,'http://daysagoest.com/y/a','DA',1,1,0,0,29); +INSERT INTO "urls" VALUES(23,'http://daysagoest.com/y/b','DB',1,1,1,0,29); +INSERT INTO "urls" VALUES(24,'http://daysagoest.com/x/c','DC',1,1,2,0,29); +INSERT INTO "urls" VALUES(25,'http://daysagoest.com/x/d','DD',1,1,3,0,29); +INSERT INTO "urls" VALUES(26,'http://daysagoest.com/y/e','DE',1,1,4,0,29); +INSERT INTO "urls" VALUES(27,'http://abcdefghixyzjklmnopqrstuvw.com/a','',3,1,0,0,29); +INSERT INTO "urls" VALUES(28,'http://spaces.com/path%20with%20spaces/foo.html','Spaces',2,2,0,0,29); +INSERT INTO "urls" VALUES(29,'http://abcdefghijklxyzmnopqrstuvw.com/a','',3,1,0,0,29); +INSERT INTO "urls" VALUES(30,'http://abcdefxyzghijklmnopqrstuvw.com/a','',3,1,0,0,29); +INSERT INTO "urls" VALUES(31,'http://abcxyzdefghijklmnopqrstuvw.com/a','',3,1,0,0,29); +INSERT INTO "urls" VALUES(32,'http://xyzabcdefghijklmnopqrstuvw.com/a','',3,1,0,0,29); +INSERT INTO "urls" VALUES(33,'http://cda.com/Dogs%20Cats%20Gorillas%20Sea%20Slugs%20and%20Mice','Dogs & Cats & Mice & Other Animals',1,1,0,0,29); +INSERT INTO "urls" VALUES(34,'https://monkeytrap.org/','',3,1,0,0,29); diff --git a/chrome/test/data/History/url_history_provider_test.db.txt b/chrome/test/data/History/url_history_provider_test.db.txt index 9ea9e5f..61884c8 100644 --- a/chrome/test/data/History/url_history_provider_test.db.txt +++ b/chrome/test/data/History/url_history_provider_test.db.txt @@ -26,35 +26,35 @@ The ordering, URLs and titles must be kept in sync with the unit tests found in in_memory_url_index_unittest.cc. */ -INSERT INTO "urls" VALUES(1,'http://www.reuters.com/article/idUSN0839880620100708','UPDATE 1-US 30-yr mortgage rate drops to new record low | Reuters',3,1,2,0,29); // Qualifies -INSERT INTO "urls" VALUES(2,'http://www.golfweek.com/news/2010/jul/08/goydos-opens-john-deere-classic-59/','Goydos opens John Deere Classic with 59',3,1,4,0,27); // Qualifies -INSERT INTO "urls" VALUES(3,'http://www.businessandmedia.org/articles/2010/20100708120415.aspx','LeBronomics: Could High Taxes Influence James'' Team Decision?',4,1,2,0,28); // Qualifies -INSERT INTO "urls" VALUES(4,'http://www.realclearmarkets.com/articles/2010/07/08/diversity_in_the_financial_sector_98562.html','RealClearMarkets - Racial, Gender Quotas in the Financial Bill?',4,1,4,0,0); // Qualifies -INSERT INTO "urls" VALUES(5,'http://drudgereport.com/','DRUDGE REPORT 2010',3,2,2,0,0); // Qualifies -INSERT INTO "urls" VALUES(6,'http://totalfinder.binaryage.com/','TotalFinder brings tabs to your native Finder and more!',3,2,4,0,26); // Qualifies -INSERT INTO "urls" VALUES(7,'http://getsharekit.com/','ShareKit : Drop-in Share Features for all iOS Apps',4,2,4,0,20); // Qualifies +INSERT INTO "urls" VALUES(1,'http://www.reuters.com/article/idUSN0839880620100708','UPDATE 1-US 30-yr mortgage rate drops to new record low | Reuters',3,1,2,0,29); /* Qualifies */ +INSERT INTO "urls" VALUES(2,'http://www.golfweek.com/news/2010/jul/08/goydos-opens-john-deere-classic-59/','Goydos opens John Deere Classic with 59',3,1,4,0,27); /* Qualifies */ +INSERT INTO "urls" VALUES(3,'http://www.businessandmedia.org/articles/2010/20100708120415.aspx','LeBronomics: Could High Taxes Influence James'' Team Decision?',4,1,2,0,28); /* Qualifies */ +INSERT INTO "urls" VALUES(4,'http://www.realclearmarkets.com/articles/2010/07/08/diversity_in_the_financial_sector_98562.html','RealClearMarkets - Racial, Gender Quotas in the Financial Bill?',4,1,4,0,0); /* Qualifies */ +INSERT INTO "urls" VALUES(5,'http://drudgereport.com/','DRUDGE REPORT 2010',3,2,2,0,0); /* Qualifies */ +INSERT INTO "urls" VALUES(6,'http://totalfinder.binaryage.com/','TotalFinder brings tabs to your native Finder and more!',3,2,4,0,26); /* Qualifies */ +INSERT INTO "urls" VALUES(7,'http://getsharekit.com/','ShareKit : Drop-in Share Features for all iOS Apps',4,2,4,0,20); /* Qualifies */ INSERT INTO "urls" VALUES(8,'http://getsharekit.com/index.html','ShareKit : Drop-in Share Features for all iOS Apps',3,0,4,0,20); INSERT INTO "urls" VALUES(9,'http://en.wikipedia.org/wiki/Control-Z','Control-Z - Wikipedia, the free encyclopedia',0,0,6,0,0); INSERT INTO "urls" VALUES(10,'http://vmware.com/info?id=724','VMware Account Management Login',1,0,6,0,0); -INSERT INTO "urls" VALUES(11,'http://www.tech-recipes.com/rx/2621/os_x_change_path_environment_variable/','OS X: Change your PATH environment variable | Mac system administration | Tech-Recipes',0,1,6,0,14); // Qualifies -INSERT INTO "urls" VALUES(12,'http://view.atdmt.com/PPJ/iview/194841301/direct;wi.160;hi.600/01?click=','',6,6,0,1,0); // Qualifies -INSERT INTO "urls" VALUES(15,'http://www.cnn.com/','CNN.com International - Breaking, World, Business, Sports, Entertainment and Video News',6,6,0,0,89); // Qualifies -INSERT INTO "urls" VALUES(16,'http://www.zdnet.com/','Technology News, Analysis, Comments and Product Reviews for IT Professionals | ZDNet',6,6,0,0,652); // Qualifies -INSERT INTO "urls" VALUES(17,'http://www.crash.net/','Crash.Net | Formula 1 & MotoGP | Motorsport News',6,6,0,0,239); // Qualifies -INSERT INTO "urls" VALUES(18,'http://www.theinquirer.net/','THE INQUIRER - Microprocessor, Server, Memory, PCS, Graphics, Networking, Storage',6,6,0,0,79); // Qualifies -INSERT INTO "urls" VALUES(19,'http://www.theregister.co.uk/','The Register: Sci/Tech News for the World',6,6,0,0,74); // Qualifies -INSERT INTO "urls" VALUES(20,'http://blogs.technet.com/markrussinovich/','Mark''s Blog - Site Home - TechNet Blogs',6,6,0,0,685); // Qualifies -INSERT INTO "urls" VALUES(21,'http://www.icu-project.org/','ICU Home Page (ICU - International Components for Unicode)',6,6,0,0,445); // Qualifies -INSERT INTO "urls" VALUES(22,'http://site.icu-project.org/','ICU Home Page (ICU - International Components for Unicode)',6,6,0,0,445); // Qualifies -INSERT INTO "urls" VALUES(23,'http://icu-project.org/apiref/icu4c/','ICU 4.2: Main Page',6,6,0,0,212); // Qualifies -INSERT INTO "urls" VALUES(24,'http://www.danilatos.com/event-test/ExperimentTest.html','Experimentation Harness',6,6,0,0,0); // Qualifies -INSERT INTO "urls" VALUES(25,'http://www.codeguru.com/','CodeGuru : codeguru',6,6,0,0,110); // Qualifies -INSERT INTO "urls" VALUES(26,'http://www.codeproject.com/','Your Development Resource - CodeProject',6,6,0,0,369); // Qualifies -INSERT INTO "urls" VALUES(27,'http://www.tomshardware.com/us/#redir','Tom''s Hardware: Hardware News, Tests and Reviews',6,6,0,0,65); // Qualifies -INSERT INTO "urls" VALUES(28,'http://www.ddj.com/windows/184416623','Dr. ABRACADABRA''s | Avoiding the Visual C++ Runtime Library | 2 1, 2003',6,6,0,0,0); // Qualifies -INSERT INTO "urls" VALUES(29,'http://svcs.cnn.com/weather/getForecast?time=34&mode=json_html&zipCode=336736767676&locCode=EGLL&celcius=true&csiID=csi2','',6,6,0,1,0); // Qualifies -INSERT INTO "urls" VALUES(30,'http://www.drudgery.com/Dogs%20and%20Mice','Life in the Slow Lane',8,2,2,0,0); // Qualifies +INSERT INTO "urls" VALUES(11,'http://www.tech-recipes.com/rx/2621/os_x_change_path_environment_variable/','OS X: Change your PATH environment variable | Mac system administration | Tech-Recipes',0,1,6,0,14); /* Qualifies */ +INSERT INTO "urls" VALUES(12,'http://view.atdmt.com/PPJ/iview/194841301/direct;wi.160;hi.600/01?click=','',6,6,0,1,0); /* Qualifies */ +INSERT INTO "urls" VALUES(15,'http://www.cnn.com/','CNN.com International - Breaking, World, Business, Sports, Entertainment and Video News',6,6,0,0,89); /* Qualifies */ +INSERT INTO "urls" VALUES(16,'http://www.zdnet.com/','Technology News, Analysis, Comments and Product Reviews for IT Professionals | ZDNet',6,6,0,0,652); /* Qualifies */ +INSERT INTO "urls" VALUES(17,'http://www.crash.net/','Crash.Net | Formula 1 & MotoGP | Motorsport News',6,6,0,0,239); /* Qualifies */ +INSERT INTO "urls" VALUES(18,'http://www.theinquirer.net/','THE INQUIRER - Microprocessor, Server, Memory, PCS, Graphics, Networking, Storage',6,6,0,0,79); /* Qualifies */ +INSERT INTO "urls" VALUES(19,'http://www.theregister.co.uk/','The Register: Sci/Tech News for the World',6,6,0,0,74); /* Qualifies */ +INSERT INTO "urls" VALUES(20,'http://blogs.technet.com/markrussinovich/','Mark''s Blog - Site Home - TechNet Blogs',6,6,0,0,685); /* Qualifies */ +INSERT INTO "urls" VALUES(21,'http://www.icu-project.org/','ICU Home Page (ICU - International Components for Unicode)',6,6,0,0,445); /* Qualifies */ +INSERT INTO "urls" VALUES(22,'http://site.icu-project.org/','ICU Home Page (ICU - International Components for Unicode)',6,6,0,0,445); /* Qualifies */ +INSERT INTO "urls" VALUES(23,'http://icu-project.org/apiref/icu4c/','ICU 4.2: Main Page',6,6,0,0,212); /* Qualifies */ +INSERT INTO "urls" VALUES(24,'http://www.danilatos.com/event-test/ExperimentTest.html','Experimentation Harness',6,6,0,0,0); /* Qualifies */ +INSERT INTO "urls" VALUES(25,'http://www.codeguru.com/','CodeGuru : codeguru',6,6,0,0,110); /* Qualifies */ +INSERT INTO "urls" VALUES(26,'http://www.codeproject.com/','Your Development Resource - CodeProject',6,6,0,0,369); /* Qualifies */ +INSERT INTO "urls" VALUES(27,'http://www.tomshardware.com/us/#redir','Tom''s Hardware: Hardware News, Tests and Reviews',6,6,0,0,65); /* Qualifies */ +INSERT INTO "urls" VALUES(28,'http://www.ddj.com/windows/184416623','Dr. ABRACADABRA''s | Avoiding the Visual C++ Runtime Library | 2 1, 2003',6,6,0,0,0); /* Qualifies */ +INSERT INTO "urls" VALUES(29,'http://svcs.cnn.com/weather/getForecast?time=34&mode=json_html&zipCode=336736767676&locCode=EGLL&celcius=true&csiID=csi2','',6,6,0,1,0); /* Qualifies */ +INSERT INTO "urls" VALUES(30,'http://www.drudgery.com/Dogs%20and%20Mice','Life in the Slow Lane',8,2,2,0,0); /* Qualifies */ INSERT INTO "urls" VALUES(31,'http://www.redrudgerydo.com/','Music of the Wild Landscape',0,0,6,0,0); -INSERT INTO "urls" VALUES(32,'https://NearlyPerfectResult.com/','Practically Perfect Search Result',99,99,0,0,0); // Qualifies -INSERT INTO "urls" VALUES(33,'http://QuiteUselessSearchResultxyz.com/','Practically Useless Search Result',4,0,99,0,0); // Qualifies -INSERT INTO "urls" VALUES(34,'http://FubarFubarAndFubar.com/','Situation Normal -- FUBARED',99,99,0,0,0); // Qualifies +INSERT INTO "urls" VALUES(32,'https://NearlyPerfectResult.com/','Practically Perfect Search Result',99,99,0,0,0); /* Qualifies */ +INSERT INTO "urls" VALUES(33,'http://QuiteUselessSearchResultxyz.com/','Practically Useless Search Result',4,0,99,0,0); /* Qualifies */ +INSERT INTO "urls" VALUES(34,'http://FubarFubarAndFubar.com/','Situation Normal -- FUBARED',99,99,0,0,0); /* Qualifies */ diff --git a/chrome/test/data/History/url_history_provider_test_limited.db.txt b/chrome/test/data/History/url_history_provider_test_limited.db.txt index f40215f..8b9665e 100644 --- a/chrome/test/data/History/url_history_provider_test_limited.db.txt +++ b/chrome/test/data/History/url_history_provider_test_limited.db.txt @@ -1,4 +1,4 @@ /* See url_history_provider_test.db.txt for a schema definition. */ -INSERT INTO "urls" VALUES(1, 'http://def.ghi?jkl=mno&pqr0123456789stu#vwx!yz=words', 'This example has 19 WORDS and 35 unique characters', 3, 1, 2, 0, 29); +INSERT INTO "urls" VALUES(1, 'http://def.ghi?jkl=mno&pqr0123456789stu#vwx!yz=words', 'This example has 17 unique WORDS and 35 unique characters', 3, 1, 2, 0, 29); diff --git a/sql/diagnostic_error_delegate.h b/sql/diagnostic_error_delegate.h index e0c42ba..022e6a5 100644 --- a/sql/diagnostic_error_delegate.h +++ b/sql/diagnostic_error_delegate.h @@ -12,6 +12,10 @@ namespace sql { +// The histogram values from sqlite result codes currently range from 1 to 26 +// but 50 gives them room to grow. +static const int kMaxSqliteError = 50; + // This class handles the exceptional sqlite errors that we might encounter // if for example the db is corrupted. Right now we just generate a UMA // histogram for release and an assert for debug builds. @@ -39,9 +43,7 @@ class DiagnosticErrorDelegate : public ErrorDelegate { // Trim off the extended error codes. error &= 0xff; - // The histogram values from sqlite result codes go currently from 1 to - // 26 currently but 50 gives them room to grow. - UMA_HISTOGRAM_ENUMERATION(UniqueT::name(), error, 50); + UMA_HISTOGRAM_ENUMERATION(UniqueT::name(), error, kMaxSqliteError); } }; |