diff options
author | mathp@chromium.org <mathp@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-06-18 16:22:30 +0000 |
---|---|---|
committer | mathp@chromium.org <mathp@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2014-06-18 16:22:30 +0000 |
commit | 2b894b8dae83b4bada9199713504aeb52cdfc8ed (patch) | |
tree | f527ed157c42675be946635c08a81288863b4ef5 | |
parent | ab4f95a66b2e01a329562d2f7a2b4445b0e4ee47 (diff) | |
download | chromium_src-2b894b8dae83b4bada9199713504aeb52cdfc8ed.zip chromium_src-2b894b8dae83b4bada9199713504aeb52cdfc8ed.tar.gz chromium_src-2b894b8dae83b4bada9199713504aeb52cdfc8ed.tar.bz2 |
Extract protobuf database into a new 'leveldb_proto' component
Code extracted from components/dom_distiller/core/dom_distiller_database.*
Slight API change: callers to UpdateEntries now have to pass a vector of (string, proto) as key and value, instead of just a vector of protos where key is derived.
Ran clang-format on the files I touched so you may see some diffs.
Note: Implementations are in proto_database_impl.h and fake_db.h for proper linking.
BUG=385747
TBR=jochen,dgrogan
TEST=DomDistiller*,ProtoDatabaseImplTest
Review URL: https://codereview.chromium.org/330833002
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@278096 0039d316-1c4b-4281-b951-d872f2087c98
31 files changed, 897 insertions, 776 deletions
diff --git a/chrome/browser/DEPS b/chrome/browser/DEPS index fd4490b..f313199 100644 --- a/chrome/browser/DEPS +++ b/chrome/browser/DEPS @@ -31,6 +31,7 @@ include_rules = [ "+components/invalidation", "+components/keyed_service", "+components/language_usage_metrics", + "+components/leveldb_proto", "+components/nacl/browser", "+components/nacl/common", "+components/navigation_interception", diff --git a/chrome/browser/dom_distiller/dom_distiller_service_factory.cc b/chrome/browser/dom_distiller/dom_distiller_service_factory.cc index acee37d..0d9d5e3 100644 --- a/chrome/browser/dom_distiller/dom_distiller_service_factory.cc +++ b/chrome/browser/dom_distiller/dom_distiller_service_factory.cc @@ -6,9 +6,12 @@ #include "base/threading/sequenced_worker_pool.h" #include "components/dom_distiller/content/distiller_page_web_contents.h" +#include "components/dom_distiller/core/article_entry.h" #include "components/dom_distiller/core/distiller.h" #include "components/dom_distiller/core/dom_distiller_store.h" #include "components/keyed_service/content/browser_context_dependency_manager.h" +#include "components/leveldb_proto/proto_database.h" +#include "components/leveldb_proto/proto_database_impl.h" #include "content/public/browser/browser_context.h" #include "content/public/browser/browser_thread.h" @@ -18,10 +21,8 @@ DomDistillerContextKeyedService::DomDistillerContextKeyedService( scoped_ptr<DomDistillerStoreInterface> store, scoped_ptr<DistillerFactory> distiller_factory, scoped_ptr<DistillerPageFactory> distiller_page_factory) - : DomDistillerService(store.Pass(), - distiller_factory.Pass(), - distiller_page_factory.Pass()) { -} + : DomDistillerService(store.Pass(), distiller_factory.Pass(), + distiller_page_factory.Pass()) {} // static DomDistillerServiceFactory* DomDistillerServiceFactory::GetInstance() { @@ -49,14 +50,15 @@ KeyedService* DomDistillerServiceFactory::BuildServiceInstanceFor( content::BrowserThread::GetBlockingPool()->GetSequencedTaskRunner( content::BrowserThread::GetBlockingPool()->GetSequenceToken()); - scoped_ptr<DomDistillerDatabase> db( - new DomDistillerDatabase(background_task_runner)); + scoped_ptr<leveldb_proto::ProtoDatabaseImpl<ArticleEntry> > db( + new leveldb_proto::ProtoDatabaseImpl<ArticleEntry>( + background_task_runner)); base::FilePath database_dir( profile->GetPath().Append(FILE_PATH_LITERAL("Articles"))); scoped_ptr<DomDistillerStore> dom_distiller_store(new DomDistillerStore( - db.PassAs<DomDistillerDatabaseInterface>(), database_dir)); + db.PassAs<leveldb_proto::ProtoDatabase<ArticleEntry> >(), database_dir)); scoped_ptr<DistillerPageFactory> distiller_page_factory( new DistillerPageWebContentsFactory(profile)); @@ -65,9 +67,8 @@ KeyedService* DomDistillerServiceFactory::BuildServiceInstanceFor( dom_distiller::proto::DomDistillerOptions options; if (VLOG_IS_ON(1)) { - options.set_debug_level( - logging::GetVlogLevelHelper(FROM_HERE.file_name(), - ::strlen(FROM_HERE.file_name()))); + options.set_debug_level(logging::GetVlogLevelHelper( + FROM_HERE.file_name(), ::strlen(FROM_HERE.file_name()))); } scoped_ptr<DistillerFactory> distiller_factory( new DistillerFactoryImpl(distiller_url_fetcher_factory.Pass(), options)); @@ -75,8 +76,7 @@ KeyedService* DomDistillerServiceFactory::BuildServiceInstanceFor( DomDistillerContextKeyedService* service = new DomDistillerContextKeyedService( dom_distiller_store.PassAs<DomDistillerStoreInterface>(), - distiller_factory.Pass(), - distiller_page_factory.Pass()); + distiller_factory.Pass(), distiller_page_factory.Pass()); return service; } diff --git a/chrome/browser/dom_distiller/dom_distiller_viewer_source_browsertest.cc b/chrome/browser/dom_distiller/dom_distiller_viewer_source_browsertest.cc index 37f4093..d36e45b 100644 --- a/chrome/browser/dom_distiller/dom_distiller_viewer_source_browsertest.cc +++ b/chrome/browser/dom_distiller/dom_distiller_viewer_source_browsertest.cc @@ -17,16 +17,17 @@ #include "chrome/test/base/in_process_browser_test.h" #include "chrome/test/base/ui_test_utils.h" #include "components/dom_distiller/content/dom_distiller_viewer_source.h" +#include "components/dom_distiller/core/article_entry.h" #include "components/dom_distiller/core/distiller.h" #include "components/dom_distiller/core/dom_distiller_service.h" #include "components/dom_distiller/core/dom_distiller_store.h" #include "components/dom_distiller/core/dom_distiller_test_util.h" -#include "components/dom_distiller/core/fake_db.h" #include "components/dom_distiller/core/fake_distiller.h" #include "components/dom_distiller/core/fake_distiller_page.h" #include "components/dom_distiller/core/task_tracker.h" #include "components/dom_distiller/core/url_constants.h" #include "components/dom_distiller/core/url_utils.h" +#include "components/leveldb_proto/testing/fake_db.h" #include "content/public/browser/render_view_host.h" #include "content/public/browser/url_data_source.h" #include "content/public/browser/web_contents.h" @@ -37,7 +38,7 @@ namespace dom_distiller { -using test::FakeDB; +using leveldb_proto::test::FakeDB; using test::FakeDistiller; using test::MockDistillerPage; using test::MockDistillerFactory; @@ -56,7 +57,7 @@ const char kGetContent[] = "window.domAutomationController.send(" "document.getElementById('content').innerHTML)"; -void AddEntry(const ArticleEntry& e, FakeDB::EntryMap* map) { +void AddEntry(const ArticleEntry& e, FakeDB<ArticleEntry>::EntryMap* map) { (*map)[e.entry_id()] = e; } @@ -78,7 +79,7 @@ class DomDistillerViewerSourceBrowserTest : public InProcessBrowserTest { virtual ~DomDistillerViewerSourceBrowserTest() {} virtual void SetUpOnMainThread() OVERRIDE { - database_model_ = new FakeDB::EntryMap; + database_model_ = new FakeDB<ArticleEntry>::EntryMap; } virtual void CleanUpOnMainThread() OVERRIDE { delete database_model_; } @@ -88,14 +89,15 @@ class DomDistillerViewerSourceBrowserTest : public InProcessBrowserTest { } static KeyedService* Build(content::BrowserContext* context) { - FakeDB* fake_db = new FakeDB(database_model_); + FakeDB<ArticleEntry>* fake_db = new FakeDB<ArticleEntry>(database_model_); distiller_factory_ = new MockDistillerFactory(); MockDistillerPageFactory* distiller_page_factory_ = new MockDistillerPageFactory(); DomDistillerContextKeyedService* service = new DomDistillerContextKeyedService( scoped_ptr<DomDistillerStoreInterface>( - CreateStoreWithFakeDB(fake_db, FakeDB::EntryMap())), + CreateStoreWithFakeDB(fake_db, + FakeDB<ArticleEntry>::EntryMap())), scoped_ptr<DistillerFactory>(distiller_factory_), scoped_ptr<DistillerPageFactory>(distiller_page_factory_)); fake_db->InitCallback(true); @@ -118,13 +120,14 @@ class DomDistillerViewerSourceBrowserTest : public InProcessBrowserTest { void ViewSingleDistilledPage(const GURL& url, const std::string& expected_mime_type); // Database entries. - static FakeDB::EntryMap* database_model_; + static FakeDB<ArticleEntry>::EntryMap* database_model_; static bool expect_distillation_; static bool expect_distiller_page_; static MockDistillerFactory* distiller_factory_; }; -FakeDB::EntryMap* DomDistillerViewerSourceBrowserTest::database_model_; +FakeDB<ArticleEntry>::EntryMap* + DomDistillerViewerSourceBrowserTest::database_model_; bool DomDistillerViewerSourceBrowserTest::expect_distillation_ = false; bool DomDistillerViewerSourceBrowserTest::expect_distiller_page_ = false; MockDistillerFactory* DomDistillerViewerSourceBrowserTest::distiller_factory_ = diff --git a/chrome/browser/extensions/api/reading_list_private/reading_list_private_apitest.cc b/chrome/browser/extensions/api/reading_list_private/reading_list_private_apitest.cc index a9faef6..fac874c 100644 --- a/chrome/browser/extensions/api/reading_list_private/reading_list_private_apitest.cc +++ b/chrome/browser/extensions/api/reading_list_private/reading_list_private_apitest.cc @@ -6,14 +6,15 @@ #include "chrome/browser/extensions/api/reading_list_private/reading_list_private_api.h" #include "chrome/browser/extensions/extension_apitest.h" #include "chrome/browser/profiles/profile.h" +#include "components/dom_distiller/core/article_entry.h" #include "components/dom_distiller/core/dom_distiller_service.h" #include "components/dom_distiller/core/dom_distiller_store.h" #include "components/dom_distiller/core/dom_distiller_test_util.h" -#include "components/dom_distiller/core/fake_db.h" #include "components/dom_distiller/core/fake_distiller.h" #include "components/dom_distiller/core/fake_distiller_page.h" +#include "components/leveldb_proto/testing/fake_db.h" -using dom_distiller::test::FakeDB; +using dom_distiller::ArticleEntry; using dom_distiller::test::FakeDistiller; using dom_distiller::test::util::CreateStoreWithFakeDB; using dom_distiller::DomDistillerContextKeyedService; @@ -24,11 +25,13 @@ using dom_distiller::DomDistillerStoreInterface; using dom_distiller::test::MockDistillerFactory; using dom_distiller::test::MockDistillerPage; using dom_distiller::test::MockDistillerPageFactory; +using leveldb_proto::test::FakeDB; class ReadingListPrivateApiTest : public ExtensionApiTest { public: static KeyedService* Build(content::BrowserContext* context) { - FakeDB* fake_db = new FakeDB(new FakeDB::EntryMap); + FakeDB<ArticleEntry>* fake_db = + new FakeDB<ArticleEntry>(new FakeDB<ArticleEntry>::EntryMap); FakeDistiller* distiller = new FakeDistiller(true); MockDistillerPage* distiller_page = new MockDistillerPage(); MockDistillerFactory* distiller_factory = new MockDistillerFactory(); @@ -37,7 +40,8 @@ class ReadingListPrivateApiTest : public ExtensionApiTest { DomDistillerContextKeyedService* service = new DomDistillerContextKeyedService( scoped_ptr<DomDistillerStoreInterface>( - CreateStoreWithFakeDB(fake_db, FakeDB::EntryMap())), + CreateStoreWithFakeDB(fake_db, + FakeDB<ArticleEntry>::EntryMap())), scoped_ptr<DistillerFactory>(distiller_factory), scoped_ptr<DistillerPageFactory>(distiller_page_factory)); fake_db->InitCallback(true); diff --git a/components/OWNERS b/components/OWNERS index fa24ddc..979f2c2 100644 --- a/components/OWNERS +++ b/components/OWNERS @@ -71,6 +71,9 @@ per-file json_schema.gypi=kalman@chromium.org per-file json_schema.gypi=koz@chromium.org per-file json_schema.gypi=mpcomplete@chromium.org +per-file leveldb_proto*=cjhopman@chromium.org +per-file leveldb_proto*=mathp@chromium.org + per-file metrics*=asvitkine@chromium.org per-file metrics*=mpearson@chromium.org per-file metrics*=jar@chromium.org diff --git a/components/components.gyp b/components/components.gyp index 0430bfa..b0ee53c 100644 --- a/components/components.gyp +++ b/components/components.gyp @@ -30,6 +30,7 @@ 'json_schema.gypi', 'keyed_service.gypi', 'language_usage_metrics.gypi', + 'leveldb_proto.gypi', 'metrics.gypi', 'navigation_metrics.gypi', 'network_time.gypi', diff --git a/components/components_tests.gyp b/components/components_tests.gyp index 1ec98df..826d09e 100644 --- a/components/components_tests.gyp +++ b/components/components_tests.gyp @@ -1,4 +1,4 @@ -# Copyright 2012 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -84,7 +84,6 @@ 'dom_distiller/core/distilled_content_store_unittest.cc', 'dom_distiller/core/distiller_unittest.cc', 'dom_distiller/core/distiller_url_fetcher_unittest.cc', - 'dom_distiller/core/dom_distiller_database_unittest.cc', 'dom_distiller/core/dom_distiller_model_unittest.cc', 'dom_distiller/core/dom_distiller_service_unittest.cc', 'dom_distiller/core/dom_distiller_store_unittest.cc', @@ -115,6 +114,7 @@ 'keyed_service/content/browser_context_dependency_manager_unittest.cc', 'keyed_service/core/dependency_graph_unittest.cc', 'language_usage_metrics/language_usage_metrics_unittest.cc', + 'leveldb_proto/proto_database_impl_unittest.cc', 'metrics/compression_utils_unittest.cc', 'metrics/machine_id_provider_win_unittest.cc', 'metrics/metrics_hashes_unittest.cc', @@ -286,6 +286,11 @@ # Dependencies of language_usage_metrics 'components.gyp:language_usage_metrics', + # Dependencies of leveldb_proto + '../third_party/leveldatabase/leveldatabase.gyp:leveldatabase', + 'components.gyp:leveldb_proto', + 'components.gyp:leveldb_proto_test_support', + # Dependencies of metrics 'components.gyp:metrics', 'components.gyp:metrics_net', @@ -408,6 +413,7 @@ ['include', '^json_schema/'], ['include', '^keyed_service/core/'], ['include', '^language_usage_metrics/'], + ['include', '^leveldb_proto/'], ['include', '^metrics/'], ['include', '^network_time/'], ['include', '^password_manager/'], diff --git a/components/dom_distiller.gypi b/components/dom_distiller.gypi index cc7bae5..80e1f4e 100644 --- a/components/dom_distiller.gypi +++ b/components/dom_distiller.gypi @@ -37,8 +37,7 @@ '../skia/skia.gyp:skia', '../sync/sync.gyp:sync', '../third_party/dom_distiller_js/dom_distiller_js.gyp:dom_distiller_js_proto', - '../third_party/leveldatabase/leveldatabase.gyp:leveldatabase', - '../third_party/protobuf/protobuf.gyp:protobuf_lite', + 'components.gyp:leveldb_proto', 'components_resources.gyp:components_resources', 'components_strings.gyp:components_strings', 'distilled_page_proto', @@ -67,8 +66,6 @@ 'dom_distiller/core/distiller_url_fetcher.h', 'dom_distiller/core/dom_distiller_constants.cc', 'dom_distiller/core/dom_distiller_constants.h', - 'dom_distiller/core/dom_distiller_database.cc', - 'dom_distiller/core/dom_distiller_database.h', 'dom_distiller/core/dom_distiller_model.cc', 'dom_distiller/core/dom_distiller_model.h', 'dom_distiller/core/dom_distiller_observer.h', @@ -102,6 +99,7 @@ 'type': 'static_library', 'dependencies': [ 'dom_distiller_core', + 'components.gyp:leveldb_proto_test_support', '../sync/sync.gyp:sync', '../testing/gmock.gyp:gmock', ], @@ -111,8 +109,6 @@ 'sources': [ 'dom_distiller/core/dom_distiller_test_util.cc', 'dom_distiller/core/dom_distiller_test_util.h', - 'dom_distiller/core/fake_db.cc', - 'dom_distiller/core/fake_db.h', 'dom_distiller/core/fake_distiller.cc', 'dom_distiller/core/fake_distiller.h', 'dom_distiller/core/fake_distiller_page.cc', diff --git a/components/dom_distiller/DEPS b/components/dom_distiller/DEPS index b81b29f..cb7ec79 100644 --- a/components/dom_distiller/DEPS +++ b/components/dom_distiller/DEPS @@ -1,11 +1,11 @@ include_rules = [ + "+components/leveldb_proto", "+google", # For third_party/protobuf. "+grit", # For generated headers. "+jni", "+sync/api", "+sync/protocol", "+third_party/dom_distiller_js", - "+third_party/leveldatabase/src/include", "+net/base", "+net/http", "+net/url_request", diff --git a/components/dom_distiller/core/dom_distiller_database.cc b/components/dom_distiller/core/dom_distiller_database.cc deleted file mode 100644 index a799a62..0000000 --- a/components/dom_distiller/core/dom_distiller_database.cc +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/dom_distiller/core/dom_distiller_database.h" - -#include "base/bind.h" -#include "base/file_util.h" -#include "base/message_loop/message_loop.h" -#include "base/sequenced_task_runner.h" -#include "base/strings/string_util.h" -#include "base/threading/sequenced_worker_pool.h" -#include "components/dom_distiller/core/article_entry.h" -#include "third_party/leveldatabase/src/include/leveldb/db.h" -#include "third_party/leveldatabase/src/include/leveldb/iterator.h" -#include "third_party/leveldatabase/src/include/leveldb/options.h" -#include "third_party/leveldatabase/src/include/leveldb/slice.h" -#include "third_party/leveldatabase/src/include/leveldb/status.h" -#include "third_party/leveldatabase/src/include/leveldb/write_batch.h" - -using base::MessageLoop; -using base::SequencedTaskRunner; - -namespace dom_distiller { - -DomDistillerDatabase::LevelDB::LevelDB() {} - -DomDistillerDatabase::LevelDB::~LevelDB() { - DFAKE_SCOPED_LOCK(thread_checker_); -} - -bool DomDistillerDatabase::LevelDB::Init(const base::FilePath& database_dir) { - DFAKE_SCOPED_LOCK(thread_checker_); - - leveldb::Options options; - options.create_if_missing = true; - options.max_open_files = 0; // Use minimum. - - std::string path = database_dir.AsUTF8Unsafe(); - - leveldb::DB* db = NULL; - leveldb::Status status = leveldb::DB::Open(options, path, &db); - if (status.IsCorruption()) { - base::DeleteFile(database_dir, true); - status = leveldb::DB::Open(options, path, &db); - } - - if (status.ok()) { - CHECK(db); - db_.reset(db); - return true; - } - - LOG(WARNING) << "Unable to open " << database_dir.value() << ": " - << status.ToString(); - return false; -} - -bool DomDistillerDatabase::LevelDB::Save(const EntryVector& entries_to_save, - const EntryVector& entries_to_remove) { - DFAKE_SCOPED_LOCK(thread_checker_); - - leveldb::WriteBatch updates; - for (EntryVector::const_iterator it = entries_to_save.begin(); - it != entries_to_save.end(); - ++it) { - updates.Put(leveldb::Slice(it->entry_id()), - leveldb::Slice(it->SerializeAsString())); - } - for (EntryVector::const_iterator it = entries_to_remove.begin(); - it != entries_to_remove.end(); - ++it) { - updates.Delete(leveldb::Slice(it->entry_id())); - } - - leveldb::WriteOptions options; - options.sync = true; - leveldb::Status status = db_->Write(options, &updates); - if (status.ok()) - return true; - - DLOG(WARNING) << "Failed writing dom_distiller entries: " - << status.ToString(); - return false; -} - -bool DomDistillerDatabase::LevelDB::Load(EntryVector* entries) { - DFAKE_SCOPED_LOCK(thread_checker_); - - leveldb::ReadOptions options; - scoped_ptr<leveldb::Iterator> db_iterator(db_->NewIterator(options)); - for (db_iterator->SeekToFirst(); db_iterator->Valid(); db_iterator->Next()) { - leveldb::Slice value_slice = db_iterator->value(); - - ArticleEntry entry; - if (!entry.ParseFromArray(value_slice.data(), value_slice.size())) { - DLOG(WARNING) << "Unable to parse dom_distiller entry " - << db_iterator->key().ToString(); - // TODO(cjhopman): Decide what to do about un-parseable entries. - } - entries->push_back(entry); - } - return true; -} - -namespace { - -void RunInitCallback(DomDistillerDatabaseInterface::InitCallback callback, - const bool* success) { - callback.Run(*success); -} - -void RunUpdateCallback(DomDistillerDatabaseInterface::UpdateCallback callback, - const bool* success) { - callback.Run(*success); -} - -void RunLoadCallback(DomDistillerDatabaseInterface::LoadCallback callback, - const bool* success, - scoped_ptr<EntryVector> entries) { - callback.Run(*success, entries.Pass()); -} - -void InitFromTaskRunner(DomDistillerDatabase::Database* database, - const base::FilePath& database_dir, - bool* success) { - DCHECK(success); - - // TODO(cjhopman): Histogram for database size. - *success = database->Init(database_dir); -} - -void UpdateEntriesFromTaskRunner(DomDistillerDatabase::Database* database, - scoped_ptr<EntryVector> entries_to_save, - scoped_ptr<EntryVector> entries_to_remove, - bool* success) { - DCHECK(success); - *success = database->Save(*entries_to_save, *entries_to_remove); -} - -void LoadEntriesFromTaskRunner(DomDistillerDatabase::Database* database, - EntryVector* entries, - bool* success) { - DCHECK(success); - DCHECK(entries); - - entries->clear(); - *success = database->Load(entries); -} - -} // namespace - -DomDistillerDatabase::DomDistillerDatabase( - scoped_refptr<base::SequencedTaskRunner> task_runner) - : task_runner_(task_runner) { -} - -void DomDistillerDatabase::Init(const base::FilePath& database_dir, - InitCallback callback) { - DCHECK(thread_checker_.CalledOnValidThread()); - InitWithDatabase(scoped_ptr<Database>(new LevelDB()), database_dir, callback); -} - -void DomDistillerDatabase::InitWithDatabase(scoped_ptr<Database> database, - const base::FilePath& database_dir, - InitCallback callback) { - DCHECK(thread_checker_.CalledOnValidThread()); - DCHECK(!db_); - DCHECK(database); - db_.reset(database.release()); - bool* success = new bool(false); - task_runner_->PostTaskAndReply( - FROM_HERE, - base::Bind(InitFromTaskRunner, - base::Unretained(db_.get()), - database_dir, - success), - base::Bind(RunInitCallback, callback, base::Owned(success))); -} - -void DomDistillerDatabase::UpdateEntries( - scoped_ptr<EntryVector> entries_to_save, - scoped_ptr<EntryVector> entries_to_remove, - UpdateCallback callback) { - DCHECK(thread_checker_.CalledOnValidThread()); - bool* success = new bool(false); - task_runner_->PostTaskAndReply( - FROM_HERE, - base::Bind(UpdateEntriesFromTaskRunner, - base::Unretained(db_.get()), - base::Passed(&entries_to_save), - base::Passed(&entries_to_remove), - success), - base::Bind(RunUpdateCallback, callback, base::Owned(success))); -} - -void DomDistillerDatabase::LoadEntries(LoadCallback callback) { - DCHECK(thread_checker_.CalledOnValidThread()); - bool* success = new bool(false); - - scoped_ptr<EntryVector> entries(new EntryVector()); - // Get this pointer before entries is base::Passed() so we can use it below. - EntryVector* entries_ptr = entries.get(); - - task_runner_->PostTaskAndReply( - FROM_HERE, - base::Bind(LoadEntriesFromTaskRunner, - base::Unretained(db_.get()), - entries_ptr, - success), - base::Bind(RunLoadCallback, - callback, - base::Owned(success), - base::Passed(&entries))); -} - -DomDistillerDatabase::~DomDistillerDatabase() { - DCHECK(thread_checker_.CalledOnValidThread()); - if (!task_runner_->DeleteSoon(FROM_HERE, db_.release())) { - DLOG(WARNING) << "DOM distiller database will not be deleted."; - } -} - -} // namespace dom_distiller diff --git a/components/dom_distiller/core/dom_distiller_database.h b/components/dom_distiller/core/dom_distiller_database.h deleted file mode 100644 index ca3d80b..0000000 --- a/components/dom_distiller/core/dom_distiller_database.h +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_DOM_DISTILLER_CORE_DOM_DISTILLER_DATABASE_H_ -#define COMPONENTS_DOM_DISTILLER_CORE_DOM_DISTILLER_DATABASE_H_ - -#include <string> -#include <vector> - -#include "base/callback.h" -#include "base/files/file_path.h" -#include "base/memory/ref_counted.h" -#include "base/memory/scoped_ptr.h" -#include "base/memory/scoped_vector.h" -#include "base/memory/weak_ptr.h" -#include "base/threading/thread_checker.h" -#include "base/threading/thread_collision_warner.h" -#include "components/dom_distiller/core/article_entry.h" - -namespace base { -class SequencedTaskRunner; -class MessageLoop; -} - -namespace leveldb { -class DB; -} - -namespace dom_distiller { - -typedef std::vector<ArticleEntry> EntryVector; - -// Interface for classes providing persistent storage of DomDistiller entries. -class DomDistillerDatabaseInterface { - public: - typedef std::vector<std::string> ArticleEntryIds; - typedef base::Callback<void(bool success)> InitCallback; - typedef base::Callback<void(bool success)> UpdateCallback; - typedef base::Callback<void(bool success, scoped_ptr<EntryVector>)> - LoadCallback; - - virtual ~DomDistillerDatabaseInterface() {} - - // Asynchronously initializes the object. |callback| will be invoked on the UI - // thread when complete. - virtual void Init(const base::FilePath& database_dir, - InitCallback callback) = 0; - - // Asynchronously saves |entries_to_save| and deletes entries from - // |entries_to_remove| from the database. |callback| will be invoked on the UI - // thread when complete. - virtual void UpdateEntries(scoped_ptr<EntryVector> entries_to_save, - scoped_ptr<EntryVector> entries_to_remove, - UpdateCallback callback) = 0; - - // Asynchronously loads all entries from the database and invokes |callback| - // when complete. - virtual void LoadEntries(LoadCallback callback) = 0; -}; - -// When the DomDistillerDatabase instance is deleted, in-progress asynchronous -// operations will be completed and the corresponding callbacks will be called. -class DomDistillerDatabase - : public DomDistillerDatabaseInterface { - public: - // The underlying database. Calls to this type may be blocking. - class Database { - public: - virtual bool Init(const base::FilePath& database_dir) = 0; - virtual bool Save(const EntryVector& entries_to_save, - const EntryVector& entries_to_remove) = 0; - virtual bool Load(EntryVector* entries) = 0; - virtual ~Database() {} - }; - - // Once constructed, function calls and destruction should all occur on the - // same thread (not necessarily the same as the constructor). - class LevelDB : public Database { - public: - LevelDB(); - virtual ~LevelDB(); - virtual bool Init(const base::FilePath& database_dir) OVERRIDE; - virtual bool Save(const EntryVector& entries_to_save, - const EntryVector& entries_to_remove) OVERRIDE; - virtual bool Load(EntryVector* entries) OVERRIDE; - - private: - DFAKE_MUTEX(thread_checker_); - scoped_ptr<leveldb::DB> db_; - }; - - explicit DomDistillerDatabase( - scoped_refptr<base::SequencedTaskRunner> task_runner); - - virtual ~DomDistillerDatabase(); - - // DomDistillerDatabaseInterface implementation. - virtual void Init(const base::FilePath& database_dir, - InitCallback callback) OVERRIDE; - virtual void UpdateEntries(scoped_ptr<EntryVector> entries_to_save, - scoped_ptr<EntryVector> entries_to_remove, - UpdateCallback callback) OVERRIDE; - virtual void LoadEntries(LoadCallback callback) OVERRIDE; - - // Allow callers to provide their own Database implementation. - void InitWithDatabase(scoped_ptr<Database> database, - const base::FilePath& database_dir, - InitCallback callback); - - private: - base::ThreadChecker thread_checker_; - - // Used to run blocking tasks in-order. - scoped_refptr<base::SequencedTaskRunner> task_runner_; - - scoped_ptr<Database> db_; - - DISALLOW_COPY_AND_ASSIGN(DomDistillerDatabase); -}; - -} // namespace dom_distiller - -#endif // COMPONENTS_DOM_DISTILLER_CORE_DOM_DISTILLER_DATABASE_H_ diff --git a/components/dom_distiller/core/dom_distiller_service_unittest.cc b/components/dom_distiller/core/dom_distiller_service_unittest.cc index abb8031..777e3fe 100644 --- a/components/dom_distiller/core/dom_distiller_service_unittest.cc +++ b/components/dom_distiller/core/dom_distiller_service_unittest.cc @@ -14,13 +14,14 @@ #include "components/dom_distiller/core/dom_distiller_model.h" #include "components/dom_distiller/core/dom_distiller_store.h" #include "components/dom_distiller/core/dom_distiller_test_util.h" -#include "components/dom_distiller/core/fake_db.h" #include "components/dom_distiller/core/fake_distiller.h" #include "components/dom_distiller/core/fake_distiller_page.h" #include "components/dom_distiller/core/task_tracker.h" +#include "components/leveldb_proto/testing/fake_db.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" +using leveldb_proto::test::FakeDB; using testing::Invoke; using testing::Return; using testing::_; @@ -79,9 +80,10 @@ class DomDistillerServiceTest : public testing::Test { public: virtual void SetUp() { main_loop_.reset(new base::MessageLoop()); - FakeDB* fake_db = new FakeDB(&db_model_); - FakeDB::EntryMap store_model; - store_ = test::util::CreateStoreWithFakeDB(fake_db, store_model); + FakeDB<ArticleEntry>* fake_db = new FakeDB<ArticleEntry>(&db_model_); + FakeDB<ArticleEntry>::EntryMap store_model; + store_ = + test::util::CreateStoreWithFakeDB(fake_db, store_model); distiller_factory_ = new MockDistillerFactory(); distiller_page_factory_ = new MockDistillerPageFactory(); service_.reset(new DomDistillerService( @@ -106,7 +108,7 @@ class DomDistillerServiceTest : public testing::Test { MockDistillerPageFactory* distiller_page_factory_; scoped_ptr<DomDistillerService> service_; scoped_ptr<base::MessageLoop> main_loop_; - FakeDB::EntryMap db_model_; + FakeDB<ArticleEntry>::EntryMap db_model_; }; TEST_F(DomDistillerServiceTest, TestViewEntry) { @@ -244,8 +246,7 @@ TEST_F(DomDistillerServiceTest, TestAddAndRemoveEntry) { EXPECT_CALL(article_cb, DistillationCompleted(true)); std::string entry_id = - service_->AddToList(url, - service_->CreateDefaultDistillerPage().Pass(), + service_->AddToList(url, service_->CreateDefaultDistillerPage().Pass(), ArticleCallback(&article_cb)); ASSERT_FALSE(distiller->GetArticleCallback().is_null()); @@ -276,8 +277,7 @@ TEST_F(DomDistillerServiceTest, TestCancellation) { GURL url("http://www.example.com/p1"); std::string entry_id = - service_->AddToList(url, - service_->CreateDefaultDistillerPage().Pass(), + service_->AddToList(url, service_->CreateDefaultDistillerPage().Pass(), ArticleCallback(&article_cb)); // Remove entry will cause the |article_cb| to be called with false value. @@ -309,9 +309,8 @@ TEST_F(DomDistillerServiceTest, TestMultipleObservers) { expected_updates.push_back(update); for (int i = 0; i < kObserverCount; ++i) { - EXPECT_CALL( - observers[i], - ArticleEntriesUpdated(util::HasExpectedUpdates(expected_updates))); + EXPECT_CALL(observers[i], ArticleEntriesUpdated( + util::HasExpectedUpdates(expected_updates))); } scoped_ptr<DistilledArticleProto> proto = CreateDefaultArticle(); @@ -322,9 +321,8 @@ TEST_F(DomDistillerServiceTest, TestMultipleObservers) { expected_updates.clear(); expected_updates.push_back(update); for (int i = 0; i < kObserverCount; ++i) { - EXPECT_CALL( - observers[i], - ArticleEntriesUpdated(util::HasExpectedUpdates(expected_updates))); + EXPECT_CALL(observers[i], ArticleEntriesUpdated( + util::HasExpectedUpdates(expected_updates))); } service_->RemoveEntry(entry_id); @@ -341,16 +339,14 @@ TEST_F(DomDistillerServiceTest, TestMultipleCallbacks) { // Adding a URL and then distilling calls all clients. GURL url("http://www.example.com/p1"); const std::string entry_id = - service_->AddToList(url, - service_->CreateDefaultDistillerPage().Pass(), + service_->AddToList(url, service_->CreateDefaultDistillerPage().Pass(), ArticleCallback(&article_cb[0])); EXPECT_CALL(article_cb[0], DistillationCompleted(true)); for (int i = 1; i < kClientsCount; ++i) { - EXPECT_EQ(entry_id, - service_->AddToList(url, - service_->CreateDefaultDistillerPage().Pass(), - ArticleCallback(&article_cb[i]))); + EXPECT_EQ(entry_id, service_->AddToList( + url, service_->CreateDefaultDistillerPage().Pass(), + ArticleCallback(&article_cb[i]))); EXPECT_CALL(article_cb[i], DistillationCompleted(true)); } @@ -360,10 +356,9 @@ TEST_F(DomDistillerServiceTest, TestMultipleCallbacks) { // Add the same url again, all callbacks should be called with true. for (int i = 0; i < kClientsCount; ++i) { EXPECT_CALL(article_cb[i], DistillationCompleted(true)); - EXPECT_EQ(entry_id, - service_->AddToList(url, - service_->CreateDefaultDistillerPage().Pass(), - ArticleCallback(&article_cb[i]))); + EXPECT_EQ(entry_id, service_->AddToList( + url, service_->CreateDefaultDistillerPage().Pass(), + ArticleCallback(&article_cb[i]))); } base::RunLoop().RunUntilIdle(); @@ -380,16 +375,14 @@ TEST_F(DomDistillerServiceTest, TestMultipleCallbacksOnRemove) { // called with false. GURL url("http://www.example.com/p1"); const std::string entry_id = - service_->AddToList(url, - service_->CreateDefaultDistillerPage().Pass(), + service_->AddToList(url, service_->CreateDefaultDistillerPage().Pass(), ArticleCallback(&article_cb[0])); EXPECT_CALL(article_cb[0], DistillationCompleted(false)); for (int i = 1; i < kClientsCount; ++i) { - EXPECT_EQ(entry_id, - service_->AddToList(url, - service_->CreateDefaultDistillerPage().Pass(), - ArticleCallback(&article_cb[i]))); + EXPECT_EQ(entry_id, service_->AddToList( + url, service_->CreateDefaultDistillerPage().Pass(), + ArticleCallback(&article_cb[i]))); EXPECT_CALL(article_cb[i], DistillationCompleted(false)); } @@ -413,10 +406,9 @@ TEST_F(DomDistillerServiceTest, TestMultiplePageArticle) { MockArticleAvailableCallback article_cb; EXPECT_CALL(article_cb, DistillationCompleted(true)); - std::string entry_id = - service_->AddToList(pages_url[0], - service_->CreateDefaultDistillerPage().Pass(), - ArticleCallback(&article_cb)); + std::string entry_id = service_->AddToList( + pages_url[0], service_->CreateDefaultDistillerPage().Pass(), + ArticleCallback(&article_cb)); ArticleEntry entry; ASSERT_FALSE(distiller->GetArticleCallback().is_null()); diff --git a/components/dom_distiller/core/dom_distiller_store.cc b/components/dom_distiller/core/dom_distiller_store.cc index b8d7fbb..c6def8d 100644 --- a/components/dom_distiller/core/dom_distiller_store.cc +++ b/components/dom_distiller/core/dom_distiller_store.cc @@ -11,6 +11,7 @@ #include "sync/protocol/article_specifics.pb.h" #include "sync/protocol/sync.pb.h" +using leveldb_proto::ProtoDatabase; using sync_pb::ArticleSpecifics; using sync_pb::EntitySpecifics; using syncer::ModelType; @@ -24,27 +25,25 @@ using syncer::SyncMergeResult; namespace dom_distiller { DomDistillerStore::DomDistillerStore( - scoped_ptr<DomDistillerDatabaseInterface> database, + scoped_ptr<ProtoDatabase<ArticleEntry> > database, const base::FilePath& database_dir) : database_(database.Pass()), database_loaded_(false), weak_ptr_factory_(this) { - database_->Init(database_dir, - base::Bind(&DomDistillerStore::OnDatabaseInit, - weak_ptr_factory_.GetWeakPtr())); + database_->Init(database_dir, base::Bind(&DomDistillerStore::OnDatabaseInit, + weak_ptr_factory_.GetWeakPtr())); } DomDistillerStore::DomDistillerStore( - scoped_ptr<DomDistillerDatabaseInterface> database, + scoped_ptr<ProtoDatabase<ArticleEntry> > database, const std::vector<ArticleEntry>& initial_data, const base::FilePath& database_dir) : database_(database.Pass()), database_loaded_(false), model_(initial_data), weak_ptr_factory_(this) { - database_->Init(database_dir, - base::Bind(&DomDistillerStore::OnDatabaseInit, - weak_ptr_factory_.GetWeakPtr())); + database_->Init(database_dir, base::Bind(&DomDistillerStore::OnDatabaseInit, + weak_ptr_factory_.GetWeakPtr())); } DomDistillerStore::~DomDistillerStore() {} @@ -59,12 +58,10 @@ bool DomDistillerStore::GetEntryById(const std::string& entry_id, return model_.GetEntryById(entry_id, entry); } -bool DomDistillerStore::GetEntryByUrl(const GURL& url, - ArticleEntry* entry) { +bool DomDistillerStore::GetEntryByUrl(const GURL& url, ArticleEntry* entry) { return model_.GetEntryByUrl(url, entry); } - bool DomDistillerStore::AddEntry(const ArticleEntry& entry) { if (!database_loaded_) { return false; @@ -165,8 +162,7 @@ std::vector<ArticleEntry> DomDistillerStore::GetEntries() const { // syncer::SyncableService implementation. SyncMergeResult DomDistillerStore::MergeDataAndStartSyncing( - ModelType type, - const SyncDataList& initial_sync_data, + ModelType type, const SyncDataList& initial_sync_data, scoped_ptr<syncer::SyncChangeProcessor> sync_processor, scoped_ptr<syncer::SyncErrorFactory> error_handler) { DCHECK_EQ(syncer::ARTICLES, type); @@ -210,8 +206,7 @@ void DomDistillerStore::NotifyObservers(const syncer::SyncChangeList& changes) { if (observers_.might_have_observers() && changes.size() > 0) { std::vector<DomDistillerObserver::ArticleUpdate> article_changes; for (SyncChangeList::const_iterator it = changes.begin(); - it != changes.end(); - ++it) { + it != changes.end(); ++it) { DomDistillerObserver::ArticleUpdate article_update; switch (it->change_type()) { case SyncChange::ACTION_ADD: @@ -233,16 +228,14 @@ void DomDistillerStore::NotifyObservers(const syncer::SyncChangeList& changes) { article_update.entry_id = entry.entry_id(); article_changes.push_back(article_update); } - FOR_EACH_OBSERVER(DomDistillerObserver, - observers_, + FOR_EACH_OBSERVER(DomDistillerObserver, observers_, ArticleEntriesUpdated(article_changes)); } } -void DomDistillerStore::ApplyChangesToModel( - const SyncChangeList& changes, - SyncChangeList* changes_applied, - SyncChangeList* changes_missing) { +void DomDistillerStore::ApplyChangesToModel(const SyncChangeList& changes, + SyncChangeList* changes_applied, + SyncChangeList* changes_missing) { model_.ApplyChangesToModel(changes, changes_applied, changes_missing); NotifyObservers(*changes_applied); } @@ -313,27 +306,29 @@ bool DomDistillerStore::ApplyChangesToDatabase( if (change_list.empty()) { return true; } - scoped_ptr<EntryVector> entries_to_save(new EntryVector()); - scoped_ptr<EntryVector> entries_to_remove(new EntryVector()); + scoped_ptr<ProtoDatabase<ArticleEntry>::KeyEntryVector> entries_to_save( + new ProtoDatabase<ArticleEntry>::KeyEntryVector()); + scoped_ptr<std::vector<std::string> > keys_to_remove( + new std::vector<std::string>()); for (SyncChangeList::const_iterator it = change_list.begin(); - it != change_list.end(); - ++it) { - if (it->change_type() == SyncChange::ACTION_DELETE) - entries_to_remove->push_back(GetEntryFromChange(*it)); - else - entries_to_save->push_back(GetEntryFromChange(*it)); + it != change_list.end(); ++it) { + if (it->change_type() == SyncChange::ACTION_DELETE) { + ArticleEntry entry = GetEntryFromChange(*it); + keys_to_remove->push_back(entry.entry_id()); + } else { + ArticleEntry entry = GetEntryFromChange(*it); + entries_to_save->push_back(std::make_pair(entry.entry_id(), entry)); + } } - database_->UpdateEntries(entries_to_save.Pass(), - entries_to_remove.Pass(), + database_->UpdateEntries(entries_to_save.Pass(), keys_to_remove.Pass(), base::Bind(&DomDistillerStore::OnDatabaseSave, weak_ptr_factory_.GetWeakPtr())); return true; } SyncMergeResult DomDistillerStore::MergeDataWithModel( - const SyncDataList& data, - SyncChangeList* changes_applied, + const SyncDataList& data, SyncChangeList* changes_applied, SyncChangeList* changes_missing) { DCHECK(changes_applied); DCHECK(changes_missing); @@ -349,8 +344,7 @@ SyncMergeResult DomDistillerStore::MergeDataWithModel( int num_added = 0; int num_modified = 0; for (SyncChangeList::const_iterator it = changes_applied->begin(); - it != changes_applied->end(); - ++it) { + it != changes_applied->end(); ++it) { DCHECK(it->IsValid()); switch (it->change_type()) { case SyncChange::ACTION_ADD: diff --git a/components/dom_distiller/core/dom_distiller_store.h b/components/dom_distiller/core/dom_distiller_store.h index 407e968..7e24f38 100644 --- a/components/dom_distiller/core/dom_distiller_store.h +++ b/components/dom_distiller/core/dom_distiller_store.h @@ -12,9 +12,9 @@ #include "base/memory/weak_ptr.h" #include "base/observer_list.h" #include "components/dom_distiller/core/article_entry.h" -#include "components/dom_distiller/core/dom_distiller_database.h" #include "components/dom_distiller/core/dom_distiller_model.h" #include "components/dom_distiller/core/dom_distiller_observer.h" +#include "components/leveldb_proto/proto_database.h" #include "sync/api/sync_change.h" #include "sync/api/sync_data.h" #include "sync/api/sync_error.h" @@ -76,17 +76,21 @@ class DomDistillerStoreInterface { class DomDistillerStore : public syncer::SyncableService, public DomDistillerStoreInterface { public: + typedef std::vector<ArticleEntry> EntryVector; + // Creates storage using the given database for local storage. Initializes the // database with |database_dir|. - DomDistillerStore(scoped_ptr<DomDistillerDatabaseInterface> database, - const base::FilePath& database_dir); + DomDistillerStore( + scoped_ptr<leveldb_proto::ProtoDatabase<ArticleEntry> > database, + const base::FilePath& database_dir); // Creates storage using the given database for local storage. Initializes the // database with |database_dir|. Also initializes the internal model to // |initial_model|. - DomDistillerStore(scoped_ptr<DomDistillerDatabaseInterface> database, - const std::vector<ArticleEntry>& initial_data, - const base::FilePath& database_dir); + DomDistillerStore( + scoped_ptr<leveldb_proto::ProtoDatabase<ArticleEntry> > database, + const std::vector<ArticleEntry>& initial_data, + const base::FilePath& database_dir); virtual ~DomDistillerStore(); @@ -104,24 +108,23 @@ class DomDistillerStore : public syncer::SyncableService, // syncer::SyncableService implementation. virtual syncer::SyncMergeResult MergeDataAndStartSyncing( - syncer::ModelType type, - const syncer::SyncDataList& initial_sync_data, + syncer::ModelType type, const syncer::SyncDataList& initial_sync_data, scoped_ptr<syncer::SyncChangeProcessor> sync_processor, scoped_ptr<syncer::SyncErrorFactory> error_handler) OVERRIDE; virtual void StopSyncing(syncer::ModelType type) OVERRIDE; - virtual syncer::SyncDataList GetAllSyncData(syncer::ModelType type) const - OVERRIDE; + virtual syncer::SyncDataList GetAllSyncData( + syncer::ModelType type) const OVERRIDE; virtual syncer::SyncError ProcessSyncChanges( const tracked_objects::Location& from_here, const syncer::SyncChangeList& change_list) OVERRIDE; + private: void OnDatabaseInit(bool success); void OnDatabaseLoad(bool success, scoped_ptr<EntryVector> entries); void OnDatabaseSave(bool success); syncer::SyncMergeResult MergeDataWithModel( - const syncer::SyncDataList& data, - syncer::SyncChangeList* changes_applied, + const syncer::SyncDataList& data, syncer::SyncChangeList* changes_applied, syncer::SyncChangeList* changes_missing); // Convert a SyncDataList to a SyncChangeList of add or update changes based @@ -145,7 +148,7 @@ class DomDistillerStore : public syncer::SyncableService, scoped_ptr<syncer::SyncChangeProcessor> sync_processor_; scoped_ptr<syncer::SyncErrorFactory> error_factory_; - scoped_ptr<DomDistillerDatabaseInterface> database_; + scoped_ptr<leveldb_proto::ProtoDatabase<ArticleEntry> > database_; bool database_loaded_; ObserverList<DomDistillerObserver> observers_; diff --git a/components/dom_distiller/core/dom_distiller_store_unittest.cc b/components/dom_distiller/core/dom_distiller_store_unittest.cc index a80a01b..286fc36 100644 --- a/components/dom_distiller/core/dom_distiller_store_unittest.cc +++ b/components/dom_distiller/core/dom_distiller_store_unittest.cc @@ -12,7 +12,7 @@ #include "base/time/time.h" #include "components/dom_distiller/core/article_entry.h" #include "components/dom_distiller/core/dom_distiller_test_util.h" -#include "components/dom_distiller/core/fake_db.h" +#include "components/leveldb_proto/testing/fake_db.h" #include "sync/api/attachments/attachment_id.h" #include "sync/api/attachments/attachment_service_proxy_for_test.h" #include "sync/protocol/sync.pb.h" @@ -20,7 +20,7 @@ #include "testing/gtest/include/gtest/gtest.h" using base::Time; -using dom_distiller::test::FakeDB; +using leveldb_proto::test::FakeDB; using sync_pb::EntitySpecifics; using syncer::ModelType; using syncer::SyncChange; @@ -59,8 +59,8 @@ class FakeSyncChangeProcessor : public syncer::SyncChangeProcessor { public: explicit FakeSyncChangeProcessor(EntryMap* model) : model_(model) {} - virtual syncer::SyncDataList GetAllSyncData(syncer::ModelType type) const - OVERRIDE { + virtual syncer::SyncDataList GetAllSyncData( + syncer::ModelType type) const OVERRIDE { ADD_FAILURE() << "FakeSyncChangeProcessor::GetAllSyncData not implemented."; return syncer::SyncDataList(); } @@ -69,8 +69,7 @@ class FakeSyncChangeProcessor : public syncer::SyncChangeProcessor { const tracked_objects::Location&, const syncer::SyncChangeList& changes) OVERRIDE { for (SyncChangeList::const_iterator it = changes.begin(); - it != changes.end(); - ++it) { + it != changes.end(); ++it) { AddEntry(GetEntryFromChange(*it), model_); } return SyncError(); @@ -80,10 +79,8 @@ class FakeSyncChangeProcessor : public syncer::SyncChangeProcessor { EntryMap* model_; }; -ArticleEntry CreateEntry(std::string entry_id, - std::string page_url1, - std::string page_url2, - std::string page_url3) { +ArticleEntry CreateEntry(std::string entry_id, std::string page_url1, + std::string page_url2, std::string page_url3) { ArticleEntry entry; entry.set_entry_id(entry_id); if (!page_url1.empty()) { @@ -111,7 +108,8 @@ ArticleEntry GetSampleEntry(int id) { CreateEntry("entry5", "rock.example.com/p1", "rock.example.com/p2", ""), CreateEntry("entry7", "example.com/entry7/1", "example.com/entry7/2", ""), CreateEntry("entry8", "example.com/entry8/1", "", ""), - CreateEntry("entry9", "example.com/entry9/all", "", ""), }; + CreateEntry("entry9", "example.com/entry9/all", "", ""), + }; EXPECT_LT(id, 9); return entries[id % 9]; } @@ -142,7 +140,7 @@ class DomDistillerStoreTest : public testing::Test { // Creates a simple DomDistillerStore initialized with |store_model_| and // with a FakeDB backed by |db_model_|. void CreateStore() { - fake_db_ = new FakeDB(&db_model_); + fake_db_ = new FakeDB<ArticleEntry>(&db_model_); store_.reset(test::util::CreateStoreWithFakeDB(fake_db_, store_model_)); } @@ -150,8 +148,7 @@ class DomDistillerStoreTest : public testing::Test { fake_sync_processor_ = new FakeSyncChangeProcessor(&sync_model_); store_->MergeDataAndStartSyncing( - kDomDistillerModelType, - SyncDataFromEntryMap(sync_model_), + kDomDistillerModelType, SyncDataFromEntryMap(sync_model_), make_scoped_ptr<SyncChangeProcessor>(fake_sync_processor_), scoped_ptr<SyncErrorFactory>(new FakeSyncErrorFactory())); } @@ -160,9 +157,7 @@ class DomDistillerStoreTest : public testing::Test { SyncData CreateSyncData(const ArticleEntry& entry) { EntitySpecifics specifics = SpecificsFromEntry(entry); return SyncData::CreateRemoteData( - next_sync_id_++, - specifics, - Time::UnixEpoch(), + next_sync_id_++, specifics, Time::UnixEpoch(), syncer::AttachmentIdList(), syncer::AttachmentServiceProxyForTest::Create()); } @@ -179,25 +174,25 @@ class DomDistillerStoreTest : public testing::Test { EntryMap db_model_; EntryMap sync_model_; - FakeDB::EntryMap store_model_; + FakeDB<ArticleEntry>::EntryMap store_model_; scoped_ptr<DomDistillerStore> store_; // Both owned by |store_|. - FakeDB* fake_db_; + FakeDB<ArticleEntry>* fake_db_; FakeSyncChangeProcessor* fake_sync_processor_; int64 next_sync_id_; }; -AssertionResult AreEntriesEqual(const EntryVector& entries, +AssertionResult AreEntriesEqual(const DomDistillerStore::EntryVector& entries, EntryMap expected_entries) { if (entries.size() != expected_entries.size()) return AssertionFailure() << "Expected " << expected_entries.size() << " entries but found " << entries.size(); - for (EntryVector::const_iterator it = entries.begin(); it != entries.end(); - ++it) { + for (DomDistillerStore::EntryVector::const_iterator it = entries.begin(); + it != entries.end(); ++it) { EntryMap::iterator expected_it = expected_entries.find(it->entry_id()); if (expected_it == expected_entries.end()) { return AssertionFailure() << "Found unexpected entry with id <" @@ -213,7 +208,7 @@ AssertionResult AreEntriesEqual(const EntryVector& entries, } AssertionResult AreEntryMapsEqual(const EntryMap& left, const EntryMap& right) { - EntryVector entries; + DomDistillerStore::EntryVector entries; for (EntryMap::const_iterator it = left.begin(); it != left.end(); ++it) { entries.push_back(it->second); } @@ -228,7 +223,8 @@ TEST_F(DomDistillerStoreTest, TestDatabaseLoad) { CreateStore(); fake_db_->InitCallback(true); - EXPECT_EQ(fake_db_->GetDirectory(), FakeDB::DirectoryForTestDB()); + EXPECT_EQ(fake_db_->GetDirectory(), + FakeDB<ArticleEntry>::DirectoryForTestDB()); fake_db_->LoadCallback(true); EXPECT_TRUE(AreEntriesEqual(store_->GetEntries(), db_model_)); @@ -372,7 +368,7 @@ TEST_F(DomDistillerStoreTest, TestGetAllSyncData) { StartSyncing(); SyncDataList data = store_->GetAllSyncData(kDomDistillerModelType); - EntryVector entries; + DomDistillerStore::EntryVector entries; for (SyncDataList::iterator it = data.begin(); it != data.end(); ++it) { entries.push_back(EntryFromSpecifics(it->GetSpecifics())); } @@ -396,10 +392,10 @@ TEST_F(DomDistillerStoreTest, TestProcessSyncChanges) { StartSyncing(); SyncChangeList changes; - changes.push_back(SyncChange( - FROM_HERE, SyncChange::ACTION_ADD, CreateSyncData(GetSampleEntry(2)))); - changes.push_back(SyncChange( - FROM_HERE, SyncChange::ACTION_ADD, CreateSyncData(GetSampleEntry(3)))); + changes.push_back(SyncChange(FROM_HERE, SyncChange::ACTION_ADD, + CreateSyncData(GetSampleEntry(2)))); + changes.push_back(SyncChange(FROM_HERE, SyncChange::ACTION_ADD, + CreateSyncData(GetSampleEntry(3)))); store_->ProcessSyncChanges(FROM_HERE, changes); @@ -426,9 +422,10 @@ TEST_F(DomDistillerStoreTest, TestSyncMergeWithSecondDomDistillerStore) { fake_db_->InitCallback(true); fake_db_->LoadCallback(true); - FakeDB* other_fake_db = new FakeDB(&other_db_model); + FakeDB<ArticleEntry>* other_fake_db = + new FakeDB<ArticleEntry>(&other_db_model); scoped_ptr<DomDistillerStore> owned_other_store(new DomDistillerStore( - scoped_ptr<DomDistillerDatabaseInterface>(other_fake_db), + scoped_ptr<leveldb_proto::ProtoDatabase<ArticleEntry> >(other_fake_db), std::vector<ArticleEntry>(), base::FilePath(FILE_PATH_LITERAL("/fake/other/path")))); DomDistillerStore* other_store = owned_other_store.get(); @@ -441,8 +438,7 @@ TEST_F(DomDistillerStoreTest, TestSyncMergeWithSecondDomDistillerStore) { FakeSyncErrorFactory* other_error_factory = new FakeSyncErrorFactory(); store_->MergeDataAndStartSyncing( - kDomDistillerModelType, - SyncDataFromEntryMap(other_db_model), + kDomDistillerModelType, SyncDataFromEntryMap(other_db_model), owned_other_store.PassAs<SyncChangeProcessor>(), make_scoped_ptr<SyncErrorFactory>(other_error_factory)); @@ -461,18 +457,16 @@ TEST_F(DomDistillerStoreTest, TestObserver) { update.entry_id = GetSampleEntry(0).entry_id(); update.update_type = DomDistillerObserver::ArticleUpdate::ADD; expected_updates.push_back(update); - EXPECT_CALL( - observer, - ArticleEntriesUpdated(test::util::HasExpectedUpdates(expected_updates))); + EXPECT_CALL(observer, ArticleEntriesUpdated( + test::util::HasExpectedUpdates(expected_updates))); store_->AddEntry(GetSampleEntry(0)); expected_updates.clear(); update.entry_id = GetSampleEntry(1).entry_id(); update.update_type = DomDistillerObserver::ArticleUpdate::ADD; expected_updates.push_back(update); - EXPECT_CALL( - observer, - ArticleEntriesUpdated(test::util::HasExpectedUpdates(expected_updates))); + EXPECT_CALL(observer, ArticleEntriesUpdated( + test::util::HasExpectedUpdates(expected_updates))); store_->AddEntry(GetSampleEntry(1)); expected_updates.clear(); @@ -480,9 +474,8 @@ TEST_F(DomDistillerStoreTest, TestObserver) { update.update_type = DomDistillerObserver::ArticleUpdate::REMOVE; expected_updates.clear(); expected_updates.push_back(update); - EXPECT_CALL( - observer, - ArticleEntriesUpdated(test::util::HasExpectedUpdates(expected_updates))); + EXPECT_CALL(observer, ArticleEntriesUpdated( + test::util::HasExpectedUpdates(expected_updates))); store_->RemoveEntry(GetSampleEntry(0)); // Add entry_id = 3 and update entry_id = 1. @@ -498,17 +491,15 @@ TEST_F(DomDistillerStoreTest, TestObserver) { update.entry_id = GetSampleEntry(1).entry_id(); update.update_type = DomDistillerObserver::ArticleUpdate::UPDATE; expected_updates.push_back(update); - EXPECT_CALL( - observer, - ArticleEntriesUpdated(test::util::HasExpectedUpdates(expected_updates))); + EXPECT_CALL(observer, ArticleEntriesUpdated( + test::util::HasExpectedUpdates(expected_updates))); FakeSyncErrorFactory* fake_error_factory = new FakeSyncErrorFactory(); EntryMap fake_model; FakeSyncChangeProcessor* fake_sync_change_processor = new FakeSyncChangeProcessor(&fake_model); store_->MergeDataAndStartSyncing( - kDomDistillerModelType, - change_data, + kDomDistillerModelType, change_data, make_scoped_ptr<SyncChangeProcessor>(fake_sync_change_processor), make_scoped_ptr<SyncErrorFactory>(fake_error_factory)); } diff --git a/components/dom_distiller/core/dom_distiller_test_util.cc b/components/dom_distiller/core/dom_distiller_test_util.cc index f9164fd..45ddf86 100644 --- a/components/dom_distiller/core/dom_distiller_test_util.cc +++ b/components/dom_distiller/core/dom_distiller_test_util.cc @@ -7,17 +7,19 @@ #include "components/dom_distiller/core/dom_distiller_store.h" #include "components/dom_distiller/core/fake_distiller.h" +using leveldb_proto::test::FakeDB; + namespace dom_distiller { namespace test { namespace util { namespace { -std::vector<ArticleEntry> EntryMapToList(const FakeDB::EntryMap& entries) { +std::vector<ArticleEntry> EntryMapToList( + const FakeDB<ArticleEntry>::EntryMap& entries) { std::vector<ArticleEntry> entry_list; - for (FakeDB::EntryMap::const_iterator it = entries.begin(); - it != entries.end(); - ++it) { + for (FakeDB<ArticleEntry>::EntryMap::const_iterator it = entries.begin(); + it != entries.end(); ++it) { entry_list.push_back(it->second); } return entry_list; @@ -39,8 +41,7 @@ bool ObserverUpdatesMatcher::MatchAndExplain( std::vector<DomDistillerObserver::ArticleUpdate>::const_iterator expected, actual; for (expected = expected_updates_.begin(), actual = actual_updates.begin(); - expected != expected_updates_.end(); - ++expected, ++actual) { + expected != expected_updates_.end(); ++expected, ++actual) { if (expected->entry_id != actual->entry_id) { *listener << " Mismatched entry id. Expected: " << expected->entry_id << " actual: " << actual->entry_id; @@ -60,8 +61,7 @@ void ObserverUpdatesMatcher::DescribeUpdates(std::ostream* os) const { bool start = true; for (std::vector<DomDistillerObserver::ArticleUpdate>::const_iterator i = expected_updates_.begin(); - i != expected_updates_.end(); - ++i) { + i != expected_updates_.end(); ++i) { if (start) { start = false; } else { @@ -90,12 +90,12 @@ HasExpectedUpdates( } // static -DomDistillerStore* CreateStoreWithFakeDB(FakeDB* fake_db, - const FakeDB::EntryMap& store_model) { +DomDistillerStore* CreateStoreWithFakeDB( + FakeDB<ArticleEntry>* fake_db, + const FakeDB<ArticleEntry>::EntryMap& store_model) { return new DomDistillerStore( - scoped_ptr<DomDistillerDatabaseInterface>(fake_db), - EntryMapToList(store_model), - FakeDB::DirectoryForTestDB()); + scoped_ptr<leveldb_proto::ProtoDatabase<ArticleEntry> >(fake_db), + EntryMapToList(store_model), FakeDB<ArticleEntry>::DirectoryForTestDB()); } } // namespace util diff --git a/components/dom_distiller/core/dom_distiller_test_util.h b/components/dom_distiller/core/dom_distiller_test_util.h index d0e05e4..dca8fcb 100644 --- a/components/dom_distiller/core/dom_distiller_test_util.h +++ b/components/dom_distiller/core/dom_distiller_test_util.h @@ -8,7 +8,7 @@ #include <vector> #include "components/dom_distiller/core/dom_distiller_observer.h" -#include "components/dom_distiller/core/fake_db.h" +#include "components/leveldb_proto/testing/fake_db.h" #include "testing/gmock/include/gmock/gmock.h" namespace dom_distiller { @@ -42,8 +42,9 @@ testing::Matcher<const std::vector<DomDistillerObserver::ArticleUpdate>&> // Creates a simple DomDistillerStore backed by |fake_db| and initialized // with |store_model|. -DomDistillerStore* CreateStoreWithFakeDB(FakeDB* fake_db, - const FakeDB::EntryMap& store_model); +DomDistillerStore* CreateStoreWithFakeDB( + leveldb_proto::test::FakeDB<ArticleEntry>* fake_db, + const leveldb_proto::test::FakeDB<ArticleEntry>::EntryMap& store_model); } // namespace util } // namespace test diff --git a/components/dom_distiller/core/fake_db.cc b/components/dom_distiller/core/fake_db.cc deleted file mode 100644 index 5ada8f0..0000000 --- a/components/dom_distiller/core/fake_db.cc +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "components/dom_distiller/core/fake_db.h" - -#include "base/bind.h" - -namespace dom_distiller { -namespace test { - -FakeDB::FakeDB(EntryMap* db) : db_(db) {} - -FakeDB::~FakeDB() {} - -void FakeDB::Init(const base::FilePath& database_dir, - DomDistillerDatabaseInterface::InitCallback callback) { - dir_ = database_dir; - init_callback_ = callback; -} - -void FakeDB::UpdateEntries( - scoped_ptr<EntryVector> entries_to_save, - scoped_ptr<EntryVector> entries_to_remove, - DomDistillerDatabaseInterface::UpdateCallback callback) { - for (EntryVector::iterator it = entries_to_save->begin(); - it != entries_to_save->end(); - ++it) { - (*db_)[it->entry_id()] = *it; - } - for (EntryVector::iterator it = entries_to_remove->begin(); - it != entries_to_remove->end(); - ++it) { - (*db_).erase(it->entry_id()); - } - update_callback_ = callback; -} - -void FakeDB::LoadEntries(DomDistillerDatabaseInterface::LoadCallback callback) { - scoped_ptr<EntryVector> entries(new EntryVector()); - for (EntryMap::iterator it = db_->begin(); it != db_->end(); ++it) { - entries->push_back(it->second); - } - load_callback_ = - base::Bind(RunLoadCallback, callback, base::Passed(&entries)); -} - -base::FilePath& FakeDB::GetDirectory() { return dir_; } - -void FakeDB::InitCallback(bool success) { - init_callback_.Run(success); - init_callback_.Reset(); -} - -void FakeDB::LoadCallback(bool success) { - load_callback_.Run(success); - load_callback_.Reset(); -} - -void FakeDB::UpdateCallback(bool success) { - update_callback_.Run(success); - update_callback_.Reset(); -} - -// static -void FakeDB::RunLoadCallback( - DomDistillerDatabaseInterface::LoadCallback callback, - scoped_ptr<EntryVector> entries, - bool success) { - callback.Run(success, entries.Pass()); -} - -// static -base::FilePath FakeDB::DirectoryForTestDB() { - return base::FilePath(FILE_PATH_LITERAL("/fake/path")); -} - -} // namespace test -} // namespace dom_distiller diff --git a/components/dom_distiller/core/fake_db.h b/components/dom_distiller/core/fake_db.h deleted file mode 100644 index 8d03d98..0000000 --- a/components/dom_distiller/core/fake_db.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef COMPONENTS_DOM_DISTILLER_CORE_FAKE_DB_H_ -#define COMPONENTS_DOM_DISTILLER_CORE_FAKE_DB_H_ - -#include <string> - -#include "components/dom_distiller/core/dom_distiller_database.h" - -namespace dom_distiller { -namespace test { - -class FakeDB : public DomDistillerDatabaseInterface { - typedef base::Callback<void(bool)> Callback; - - public: - typedef base::hash_map<std::string, ArticleEntry> EntryMap; - - explicit FakeDB(EntryMap* db); - virtual ~FakeDB(); - - virtual void Init(const base::FilePath& database_dir, - DomDistillerDatabaseInterface::InitCallback callback) - OVERRIDE; - - virtual void UpdateEntries( - scoped_ptr<EntryVector> entries_to_save, - scoped_ptr<EntryVector> entries_to_remove, - DomDistillerDatabaseInterface::UpdateCallback callback) OVERRIDE; - - virtual void LoadEntries(DomDistillerDatabaseInterface::LoadCallback callback) - OVERRIDE; - base::FilePath& GetDirectory(); - - void InitCallback(bool success); - - void LoadCallback(bool success); - - void UpdateCallback(bool success); - - static base::FilePath DirectoryForTestDB(); - - private: - static void RunLoadCallback( - DomDistillerDatabaseInterface::LoadCallback callback, - scoped_ptr<EntryVector> entries, - bool success); - - base::FilePath dir_; - EntryMap* db_; - - Callback init_callback_; - Callback load_callback_; - Callback update_callback_; -}; - -} // namespace test -} // namespace dom_distiller - -#endif // COMPONENTS_DOM_DISTILLER_CORE_FAKE_DB_H_ diff --git a/components/dom_distiller/standalone/content_extractor.cc b/components/dom_distiller/standalone/content_extractor.cc index 0b609ed..7f7a216 100644 --- a/components/dom_distiller/standalone/content_extractor.cc +++ b/components/dom_distiller/standalone/content_extractor.cc @@ -12,13 +12,15 @@ #include "base/strings/string_number_conversions.h" #include "base/strings/string_split.h" #include "components/dom_distiller/content/distiller_page_web_contents.h" +#include "components/dom_distiller/core/article_entry.h" #include "components/dom_distiller/core/distiller.h" -#include "components/dom_distiller/core/dom_distiller_database.h" #include "components/dom_distiller/core/dom_distiller_service.h" #include "components/dom_distiller/core/dom_distiller_store.h" #include "components/dom_distiller/core/proto/distilled_article.pb.h" #include "components/dom_distiller/core/proto/distilled_page.pb.h" #include "components/dom_distiller/core/task_tracker.h" +#include "components/leveldb_proto/proto_database.h" +#include "components/leveldb_proto/proto_database_impl.h" #include "content/public/browser/browser_context.h" #include "content/public/browser/browser_thread.h" #include "content/public/test/content_browser_test.h" @@ -69,10 +71,11 @@ scoped_ptr<DomDistillerService> CreateDomDistillerService( // TODO(cjhopman): use an in-memory database instead of an on-disk one with // temporary directory. - scoped_ptr<DomDistillerDatabase> db( - new DomDistillerDatabase(background_task_runner)); + scoped_ptr<leveldb_proto::ProtoDatabaseImpl<ArticleEntry> > db( + new leveldb_proto::ProtoDatabaseImpl<ArticleEntry>( + background_task_runner)); scoped_ptr<DomDistillerStore> dom_distiller_store(new DomDistillerStore( - db.PassAs<DomDistillerDatabaseInterface>(), db_path)); + db.PassAs<leveldb_proto::ProtoDatabase<ArticleEntry> >(), db_path)); scoped_ptr<DistillerPageFactory> distiller_page_factory( new DistillerPageWebContentsFactory(context)); diff --git a/components/leveldb_proto.gypi b/components/leveldb_proto.gypi new file mode 100644 index 0000000..b90ad90 --- /dev/null +++ b/components/leveldb_proto.gypi @@ -0,0 +1,44 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +{ + 'targets': [ + { + 'target_name': 'leveldb_proto', + 'type': 'static_library', + 'include_dirs': [ + '..', + ], + 'dependencies': [ + '../base/base.gyp:base', + '../third_party/leveldatabase/leveldatabase.gyp:leveldatabase', + ], + 'sources': [ + 'leveldb_proto/leveldb_database.cc', + 'leveldb_proto/leveldb_database.h', + 'leveldb_proto/proto_database.h', + 'leveldb_proto/proto_database_impl.h', + ] + }, + { + 'target_name': 'leveldb_proto_test_support', + 'type': 'static_library', + 'dependencies': [ + 'leveldb_proto', + ], + 'include_dirs': [ + '..', + ], + 'sources': [ + 'leveldb_proto/testing/fake_db.h', + 'leveldb_proto/testing/proto/test.proto', + ], + 'variables': { + 'proto_in_dir': 'leveldb_proto/testing/proto', + 'proto_out_dir': 'components/leveldb_proto/testing/proto', + }, + 'includes': [ '../build/protoc.gypi' ] + }, + ], +} diff --git a/components/leveldb_proto/DEPS b/components/leveldb_proto/DEPS new file mode 100644 index 0000000..19109d2 --- /dev/null +++ b/components/leveldb_proto/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+third_party/leveldatabase/src/include", +] diff --git a/components/leveldb_proto/OWNERS b/components/leveldb_proto/OWNERS new file mode 100644 index 0000000..480350b --- /dev/null +++ b/components/leveldb_proto/OWNERS @@ -0,0 +1,2 @@ +cjhopman@chromium.org +mathp@chromium.org diff --git a/components/leveldb_proto/README b/components/leveldb_proto/README new file mode 100644 index 0000000..6d255eb --- /dev/null +++ b/components/leveldb_proto/README @@ -0,0 +1,6 @@ +This component provides a database that can be used to store Protocol Buffers. + +It is based on LevelDB[1]. In this implementation, keys are strings chosen by +the developer, and values are serialized Protocol Buffers. + +[1] https://code.google.com/p/leveldb/ diff --git a/components/leveldb_proto/leveldb_database.cc b/components/leveldb_proto/leveldb_database.cc new file mode 100644 index 0000000..811ae1e --- /dev/null +++ b/components/leveldb_proto/leveldb_database.cc @@ -0,0 +1,93 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/leveldb_proto/leveldb_database.h" + +#include <string> +#include <vector> + +#include "base/file_util.h" +#include "base/files/file_path.h" +#include "base/memory/scoped_ptr.h" +#include "base/threading/thread_checker.h" +#include "third_party/leveldatabase/src/include/leveldb/db.h" +#include "third_party/leveldatabase/src/include/leveldb/iterator.h" +#include "third_party/leveldatabase/src/include/leveldb/options.h" +#include "third_party/leveldatabase/src/include/leveldb/slice.h" +#include "third_party/leveldatabase/src/include/leveldb/status.h" +#include "third_party/leveldatabase/src/include/leveldb/write_batch.h" + +namespace leveldb_proto { + +LevelDB::LevelDB() {} + +LevelDB::~LevelDB() { DFAKE_SCOPED_LOCK(thread_checker_); } + +bool LevelDB::Init(const base::FilePath& database_dir) { + DFAKE_SCOPED_LOCK(thread_checker_); + + leveldb::Options options; + options.create_if_missing = true; + options.max_open_files = 0; // Use minimum. + + std::string path = database_dir.AsUTF8Unsafe(); + + leveldb::DB* db = NULL; + leveldb::Status status = leveldb::DB::Open(options, path, &db); + if (status.IsCorruption()) { + base::DeleteFile(database_dir, true); + status = leveldb::DB::Open(options, path, &db); + } + + if (status.ok()) { + CHECK(db); + db_.reset(db); + return true; + } + + LOG(WARNING) << "Unable to open " << database_dir.value() << ": " + << status.ToString(); + return false; +} + +bool LevelDB::Save( + const std::vector<std::pair<std::string, std::string> >& entries_to_save, + const std::vector<std::string>& keys_to_remove) { + DFAKE_SCOPED_LOCK(thread_checker_); + + leveldb::WriteBatch updates; + for (std::vector<std::pair<std::string, std::string> >::const_iterator it = + entries_to_save.begin(); + it != entries_to_save.end(); ++it) { + updates.Put(leveldb::Slice(it->first), leveldb::Slice(it->second)); + } + for (std::vector<std::string>::const_iterator it = keys_to_remove.begin(); + it != keys_to_remove.end(); ++it) { + updates.Delete(leveldb::Slice(*it)); + } + + leveldb::WriteOptions options; + options.sync = true; + leveldb::Status status = db_->Write(options, &updates); + if (status.ok()) return true; + + DLOG(WARNING) << "Failed writing leveldb_proto entries: " + << status.ToString(); + return false; +} + +bool LevelDB::Load(std::vector<std::string>* entries) { + DFAKE_SCOPED_LOCK(thread_checker_); + + leveldb::ReadOptions options; + scoped_ptr<leveldb::Iterator> db_iterator(db_->NewIterator(options)); + for (db_iterator->SeekToFirst(); db_iterator->Valid(); db_iterator->Next()) { + leveldb::Slice value_slice = db_iterator->value(); + std::string entry(value_slice.data(), value_slice.size()); + entries->push_back(entry); + } + return true; +} + +} // namespace leveldb_proto diff --git a/components/leveldb_proto/leveldb_database.h b/components/leveldb_proto/leveldb_database.h new file mode 100644 index 0000000..16d3c30 --- /dev/null +++ b/components/leveldb_proto/leveldb_database.h @@ -0,0 +1,42 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_LEVELDB_PROTO_LEVELDB_DATABASE_H_ +#define COMPONENTS_LEVELDB_PROTO_LEVELDB_DATABASE_H_ + +#include <string> +#include <vector> + +#include "base/files/file_path.h" +#include "base/memory/scoped_ptr.h" +#include "base/threading/thread_collision_warner.h" + +namespace leveldb { +class DB; +} // namespace leveldb + +namespace leveldb_proto { + +// Interacts with the LevelDB third party module. +// Once constructed, function calls and destruction should all occur on the +// same thread (not necessarily the same as the constructor). +class LevelDB { + public: + LevelDB(); + virtual ~LevelDB(); + + virtual bool Init(const base::FilePath& database_dir); + virtual bool Save( + const std::vector<std::pair<std::string, std::string> >& pairs_to_save, + const std::vector<std::string>& keys_to_remove); + virtual bool Load(std::vector<std::string>* entries); + + private: + DFAKE_MUTEX(thread_checker_); + scoped_ptr<leveldb::DB> db_; +}; + +} // namespace leveldb_proto + +#endif // COMPONENTS_LEVELDB_PROTO_LEVELDB_DATABASE_H_ diff --git a/components/leveldb_proto/proto_database.h b/components/leveldb_proto/proto_database.h new file mode 100644 index 0000000..34ee799 --- /dev/null +++ b/components/leveldb_proto/proto_database.h @@ -0,0 +1,52 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_LEVELDB_PROTO_PROTO_DATABASE_H_ +#define COMPONENTS_LEVELDB_PROTO_PROTO_DATABASE_H_ + +#include <string> +#include <utility> +#include <vector> + +#include "base/callback.h" +#include "base/files/file_path.h" +#include "base/memory/scoped_ptr.h" + +namespace leveldb_proto { + +// Interface for classes providing persistent storage of Protocol Buffer +// entries (T must be a Proto type extending MessageLite). +template <typename T> +class ProtoDatabase { + public: + typedef base::Callback<void(bool success)> InitCallback; + typedef base::Callback<void(bool success)> UpdateCallback; + typedef base::Callback<void(bool success, scoped_ptr<std::vector<T> >)> + LoadCallback; + // A list of key-value (string, T) tuples. + typedef std::vector<std::pair<std::string, T> > KeyEntryVector; + + virtual ~ProtoDatabase() {} + + // Asynchronously initializes the object. |callback| will be invoked on the + // calling thread when complete. + virtual void Init(const base::FilePath& database_dir, + InitCallback callback) = 0; + + // Asynchronously saves |entries_to_save| and deletes entries from + // |keys_to_remove| from the database. |callback| will be invoked on the + // calling thread when complete. + virtual void UpdateEntries( + scoped_ptr<KeyEntryVector> entries_to_save, + scoped_ptr<std::vector<std::string> > keys_to_remove, + UpdateCallback callback) = 0; + + // Asynchronously loads all entries from the database and invokes |callback| + // when complete. + virtual void LoadEntries(LoadCallback callback) = 0; +}; + +} // namespace leveldb_proto + +#endif // COMPONENTS_LEVELDB_PROTO_PROTO_DATABASE_H_ diff --git a/components/leveldb_proto/proto_database_impl.h b/components/leveldb_proto/proto_database_impl.h new file mode 100644 index 0000000..1b1c740 --- /dev/null +++ b/components/leveldb_proto/proto_database_impl.h @@ -0,0 +1,204 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_LEVELDB_PROTO_PROTO_DATABASE_IMPL_H_ +#define COMPONENTS_LEVELDB_PROTO_PROTO_DATABASE_IMPL_H_ + +#include <string> +#include <vector> + +#include "base/bind.h" +#include "base/files/file_path.h" +#include "base/memory/scoped_ptr.h" +#include "base/message_loop/message_loop.h" +#include "base/sequenced_task_runner.h" +#include "base/strings/string_util.h" +#include "base/threading/sequenced_worker_pool.h" +#include "base/threading/thread_checker.h" +#include "components/leveldb_proto/leveldb_database.h" +#include "components/leveldb_proto/proto_database.h" + +namespace leveldb_proto { + +typedef std::vector<std::pair<std::string, std::string> > KeyValueVector; +typedef std::vector<std::string> KeyVector; + +// When the ProtoDatabaseImpl instance is deleted, in-progress asynchronous +// operations will be completed and the corresponding callbacks will be called. +// Construction/calls/destruction should all happen on the same thread. +template <typename T> +class ProtoDatabaseImpl : public ProtoDatabase<T> { + public: + // All blocking calls/disk access will happen on the provided |task_runner|. + explicit ProtoDatabaseImpl( + scoped_refptr<base::SequencedTaskRunner> task_runner); + + virtual ~ProtoDatabaseImpl(); + + // ProtoDatabase implementation. + // TODO(cjhopman): Perhaps Init() shouldn't be exposed to users and not just + // part of the constructor + virtual void Init(const base::FilePath& database_dir, + typename ProtoDatabase<T>::InitCallback callback) OVERRIDE; + virtual void UpdateEntries( + scoped_ptr<typename ProtoDatabase<T>::KeyEntryVector> entries_to_save, + scoped_ptr<KeyVector> keys_to_remove, + typename ProtoDatabase<T>::UpdateCallback callback) OVERRIDE; + virtual void LoadEntries( + typename ProtoDatabase<T>::LoadCallback callback) OVERRIDE; + + // Allow callers to provide their own Database implementation. + void InitWithDatabase(scoped_ptr<LevelDB> database, + const base::FilePath& database_dir, + typename ProtoDatabase<T>::InitCallback callback); + + private: + base::ThreadChecker thread_checker_; + + // Used to run blocking tasks in-order. + scoped_refptr<base::SequencedTaskRunner> task_runner_; + + scoped_ptr<LevelDB> db_; + + DISALLOW_COPY_AND_ASSIGN(ProtoDatabaseImpl); +}; + +namespace { + +template <typename T> +void RunInitCallback(typename ProtoDatabase<T>::InitCallback callback, + const bool* success) { + callback.Run(*success); +} + +template <typename T> +void RunUpdateCallback(typename ProtoDatabase<T>::UpdateCallback callback, + const bool* success) { + callback.Run(*success); +} + +template <typename T> +void RunLoadCallback(typename ProtoDatabase<T>::LoadCallback callback, + const bool* success, scoped_ptr<std::vector<T> > entries) { + callback.Run(*success, entries.Pass()); +} + +void InitFromTaskRunner(LevelDB* database, const base::FilePath& database_dir, + bool* success) { + DCHECK(success); + + // TODO(cjhopman): Histogram for database size. + *success = database->Init(database_dir); +} + +template <typename T> +void UpdateEntriesFromTaskRunner( + LevelDB* database, + scoped_ptr<typename ProtoDatabase<T>::KeyEntryVector> entries_to_save, + scoped_ptr<KeyVector> keys_to_remove, bool* success) { + DCHECK(success); + // Serialize the values from Proto to string before passing on to database. + KeyValueVector pairs_to_save; + for (typename ProtoDatabase<T>::KeyEntryVector::iterator it = + entries_to_save->begin(); + it != entries_to_save->end(); ++it) { + pairs_to_save.push_back( + std::make_pair(it->first, it->second.SerializeAsString())); + } + *success = database->Save(pairs_to_save, *keys_to_remove); +} + +template <typename T> +void LoadEntriesFromTaskRunner(LevelDB* database, std::vector<T>* entries, + bool* success) { + DCHECK(success); + DCHECK(entries); + + entries->clear(); + std::vector<std::string> loaded_entries; + *success = database->Load(&loaded_entries); + for (std::vector<std::string>::iterator it = loaded_entries.begin(); + it != loaded_entries.end(); ++it) { + T entry; + if (!entry.ParseFromString(*it)) { + DLOG(WARNING) << "Unable to parse leveldb_proto entry " << *it; + // TODO(cjhopman): Decide what to do about un-parseable entries. + } + entries->push_back(entry); + } +} + +} // namespace + +template <typename T> +ProtoDatabaseImpl<T>::ProtoDatabaseImpl( + scoped_refptr<base::SequencedTaskRunner> task_runner) + : task_runner_(task_runner) {} + +template <typename T> +ProtoDatabaseImpl<T>::~ProtoDatabaseImpl() { + DCHECK(thread_checker_.CalledOnValidThread()); + if (!task_runner_->DeleteSoon(FROM_HERE, db_.release())) { + DLOG(WARNING) << "DOM distiller database will not be deleted."; + } +} + +template <typename T> +void ProtoDatabaseImpl<T>::Init( + const base::FilePath& database_dir, + typename ProtoDatabase<T>::InitCallback callback) { + DCHECK(thread_checker_.CalledOnValidThread()); + InitWithDatabase(scoped_ptr<LevelDB>(new LevelDB()), database_dir, callback); +} + +template <typename T> +void ProtoDatabaseImpl<T>::InitWithDatabase( + scoped_ptr<LevelDB> database, const base::FilePath& database_dir, + typename ProtoDatabase<T>::InitCallback callback) { + DCHECK(thread_checker_.CalledOnValidThread()); + DCHECK(!db_); + DCHECK(database); + db_.reset(database.release()); + bool* success = new bool(false); + task_runner_->PostTaskAndReply( + FROM_HERE, base::Bind(InitFromTaskRunner, base::Unretained(db_.get()), + database_dir, success), + base::Bind(RunInitCallback<T>, callback, base::Owned(success))); +} + +template <typename T> +void ProtoDatabaseImpl<T>::UpdateEntries( + scoped_ptr<typename ProtoDatabase<T>::KeyEntryVector> entries_to_save, + scoped_ptr<KeyVector> keys_to_remove, + typename ProtoDatabase<T>::UpdateCallback callback) { + DCHECK(thread_checker_.CalledOnValidThread()); + bool* success = new bool(false); + task_runner_->PostTaskAndReply( + FROM_HERE, + base::Bind(UpdateEntriesFromTaskRunner<T>, base::Unretained(db_.get()), + base::Passed(&entries_to_save), base::Passed(&keys_to_remove), + success), + base::Bind(RunUpdateCallback<T>, callback, base::Owned(success))); +} + +template <typename T> +void ProtoDatabaseImpl<T>::LoadEntries( + typename ProtoDatabase<T>::LoadCallback callback) { + DCHECK(thread_checker_.CalledOnValidThread()); + bool* success = new bool(false); + + scoped_ptr<std::vector<T> > entries(new std::vector<T>()); + // Get this pointer before entries is base::Passed() so we can use it below. + std::vector<T>* entries_ptr = entries.get(); + + task_runner_->PostTaskAndReply( + FROM_HERE, base::Bind(LoadEntriesFromTaskRunner<T>, + base::Unretained(db_.get()), entries_ptr, success), + base::Bind(RunLoadCallback<T>, callback, base::Owned(success), + base::Passed(&entries))); +} + +} // namespace leveldb_proto + +#endif // COMPONENTS_LEVELDB_PROTO_PROTO_DATABASE_IMPL_H_ diff --git a/components/dom_distiller/core/dom_distiller_database_unittest.cc b/components/leveldb_proto/proto_database_impl_unittest.cc index 54c9043..073328b 100644 --- a/components/dom_distiller/core/dom_distiller_database_unittest.cc +++ b/components/leveldb_proto/proto_database_impl_unittest.cc @@ -1,8 +1,8 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. +// Copyright 2014 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "components/dom_distiller/core/dom_distiller_database.h" +#include "components/leveldb_proto/proto_database_impl.h" #include <map> @@ -11,7 +11,8 @@ #include "base/files/scoped_temp_dir.h" #include "base/run_loop.h" #include "base/threading/thread.h" -#include "components/dom_distiller/core/article_entry.h" +#include "components/leveldb_proto/leveldb_database.h" +#include "components/leveldb_proto/testing/proto/test.pb.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" @@ -21,35 +22,33 @@ using testing::Invoke; using testing::Return; using testing::_; -namespace dom_distiller { +namespace leveldb_proto { namespace { -typedef std::map<std::string, ArticleEntry> EntryMap; +typedef std::map<std::string, TestProto> EntryMap; -class MockDB : public DomDistillerDatabase::Database { +class MockDB : public LevelDB { public: MOCK_METHOD1(Init, bool(const base::FilePath&)); - MOCK_METHOD2(Save, bool(const EntryVector&, const EntryVector&)); - MOCK_METHOD1(Load, bool(EntryVector*)); + MOCK_METHOD2(Save, bool(const KeyValueVector&, const KeyVector&)); + MOCK_METHOD1(Load, bool(std::vector<std::string>*)); MockDB() { ON_CALL(*this, Init(_)).WillByDefault(Return(true)); ON_CALL(*this, Save(_, _)).WillByDefault(Return(true)); ON_CALL(*this, Load(_)).WillByDefault(Return(true)); } - - bool LoadEntries(EntryVector* entries); }; class MockDatabaseCaller { public: MOCK_METHOD1(InitCallback, void(bool)); MOCK_METHOD1(SaveCallback, void(bool)); - void LoadCallback(bool success, scoped_ptr<EntryVector> entries) { + void LoadCallback(bool success, scoped_ptr<std::vector<TestProto> > entries) { LoadCallback1(success, entries.get()); } - MOCK_METHOD2(LoadCallback1, void(bool, EntryVector*)); + MOCK_METHOD2(LoadCallback1, void(bool, std::vector<TestProto>*)); }; } // namespace @@ -57,25 +56,23 @@ class MockDatabaseCaller { EntryMap GetSmallModel() { EntryMap model; - model["key0"].set_entry_id("key0"); - model["key0"].add_pages()->set_url("http://foo.com/1"); - model["key0"].add_pages()->set_url("http://foo.com/2"); - model["key0"].add_pages()->set_url("http://foo.com/3"); + model["0"].set_id("0"); + model["0"].set_data("http://foo.com/1"); - model["key1"].set_entry_id("key1"); - model["key1"].add_pages()->set_url("http://bar.com/all"); + model["1"].set_id("1"); + model["1"].set_data("http://bar.com/all"); - model["key2"].set_entry_id("key2"); - model["key2"].add_pages()->set_url("http://baz.com/1"); + model["2"].set_id("2"); + model["2"].set_data("http://baz.com/1"); return model; } -void ExpectEntryPointersEquals(EntryMap expected, const EntryVector& actual) { +void ExpectEntryPointersEquals(EntryMap expected, + const std::vector<TestProto>& actual) { EXPECT_EQ(expected.size(), actual.size()); for (size_t i = 0; i < actual.size(); i++) { - EntryMap::iterator expected_it = - expected.find(std::string(actual[i].entry_id())); + EntryMap::iterator expected_it = expected.find(actual[i].id()); EXPECT_TRUE(expected_it != expected.end()); std::string serialized_expected = expected_it->second.SerializeAsString(); std::string serialized_actual = actual[i].SerializeAsString(); @@ -84,11 +81,12 @@ void ExpectEntryPointersEquals(EntryMap expected, const EntryVector& actual) { } } -class DomDistillerDatabaseTest : public testing::Test { +class ProtoDatabaseImplTest : public testing::Test { public: virtual void SetUp() { main_loop_.reset(new MessageLoop()); - db_.reset(new DomDistillerDatabase(main_loop_->message_loop_proxy())); + db_.reset( + new ProtoDatabaseImpl<TestProto>(main_loop_->message_loop_proxy())); } virtual void TearDown() { @@ -97,13 +95,13 @@ class DomDistillerDatabaseTest : public testing::Test { main_loop_.reset(); } - scoped_ptr<DomDistillerDatabase> db_; + scoped_ptr<ProtoDatabaseImpl<TestProto> > db_; scoped_ptr<MessageLoop> main_loop_; }; -// Test that DomDistillerDatabase calls Init on the underlying database and that +// Test that ProtoDatabaseImpl calls Init on the underlying database and that // the caller's InitCallback is called with the correct value. -TEST_F(DomDistillerDatabaseTest, TestDBInitSuccess) { +TEST_F(ProtoDatabaseImplTest, TestDBInitSuccess) { base::FilePath path(FILE_PATH_LITERAL("/fake/path")); MockDB* mock_db = new MockDB(); @@ -113,14 +111,13 @@ TEST_F(DomDistillerDatabaseTest, TestDBInitSuccess) { EXPECT_CALL(caller, InitCallback(true)); db_->InitWithDatabase( - scoped_ptr<DomDistillerDatabase::Database>(mock_db), - base::FilePath(path), + scoped_ptr<LevelDB>(mock_db), base::FilePath(path), base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); base::RunLoop().RunUntilIdle(); } -TEST_F(DomDistillerDatabaseTest, TestDBInitFailure) { +TEST_F(ProtoDatabaseImplTest, TestDBInitFailure) { base::FilePath path(FILE_PATH_LITERAL("/fake/path")); MockDB* mock_db = new MockDB(); @@ -130,31 +127,30 @@ TEST_F(DomDistillerDatabaseTest, TestDBInitFailure) { EXPECT_CALL(caller, InitCallback(false)); db_->InitWithDatabase( - scoped_ptr<DomDistillerDatabase::Database>(mock_db), - base::FilePath(path), + scoped_ptr<LevelDB>(mock_db), base::FilePath(path), base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); base::RunLoop().RunUntilIdle(); } ACTION_P(AppendLoadEntries, model) { - EntryVector* output = arg0; + std::vector<std::string>* output = arg0; for (EntryMap::const_iterator it = model.begin(); it != model.end(); ++it) { - output->push_back(it->second); + output->push_back(it->second.SerializeAsString()); } return true; } ACTION_P(VerifyLoadEntries, expected) { - EntryVector* actual = arg1; + std::vector<TestProto>* actual = arg1; ExpectEntryPointersEquals(expected, *actual); } -// Test that DomDistillerDatabase calls Load on the underlying database and that +// Test that ProtoDatabaseImpl calls Load on the underlying database and that // the caller's LoadCallback is called with the correct success value. Also // confirms that on success, the expected entries are passed to the caller's // LoadCallback. -TEST_F(DomDistillerDatabaseTest, TestDBLoadSuccess) { +TEST_F(ProtoDatabaseImplTest, TestDBLoadSuccess) { base::FilePath path(FILE_PATH_LITERAL("/fake/path")); MockDB* mock_db = new MockDB(); @@ -164,8 +160,7 @@ TEST_F(DomDistillerDatabaseTest, TestDBLoadSuccess) { EXPECT_CALL(*mock_db, Init(_)); EXPECT_CALL(caller, InitCallback(_)); db_->InitWithDatabase( - scoped_ptr<DomDistillerDatabase::Database>(mock_db), - base::FilePath(path), + scoped_ptr<LevelDB>(mock_db), base::FilePath(path), base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); EXPECT_CALL(*mock_db, Load(_)).WillOnce(AppendLoadEntries(model)); @@ -177,7 +172,7 @@ TEST_F(DomDistillerDatabaseTest, TestDBLoadSuccess) { base::RunLoop().RunUntilIdle(); } -TEST_F(DomDistillerDatabaseTest, TestDBLoadFailure) { +TEST_F(ProtoDatabaseImplTest, TestDBLoadFailure) { base::FilePath path(FILE_PATH_LITERAL("/fake/path")); MockDB* mock_db = new MockDB(); @@ -186,8 +181,7 @@ TEST_F(DomDistillerDatabaseTest, TestDBLoadFailure) { EXPECT_CALL(*mock_db, Init(_)); EXPECT_CALL(caller, InitCallback(_)); db_->InitWithDatabase( - scoped_ptr<DomDistillerDatabase::Database>(mock_db), - base::FilePath(path), + scoped_ptr<LevelDB>(mock_db), base::FilePath(path), base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); EXPECT_CALL(*mock_db, Load(_)).WillOnce(Return(false)); @@ -199,15 +193,24 @@ TEST_F(DomDistillerDatabaseTest, TestDBLoadFailure) { } ACTION_P(VerifyUpdateEntries, expected) { - const EntryVector& actual = arg0; - ExpectEntryPointersEquals(expected, actual); + const KeyValueVector actual = arg0; + // Create a vector of TestProto from |actual| to reuse the comparison + // function. + std::vector<TestProto> extracted_entries; + for (KeyValueVector::const_iterator it = actual.begin(); it != actual.end(); + ++it) { + TestProto entry; + entry.ParseFromString(it->second); + extracted_entries.push_back(entry); + } + ExpectEntryPointersEquals(expected, extracted_entries); return true; } -// Test that DomDistillerDatabase calls Save on the underlying database with the +// Test that ProtoDatabaseImpl calls Save on the underlying database with the // correct entries to save and that the caller's SaveCallback is called with the // correct success value. -TEST_F(DomDistillerDatabaseTest, TestDBSaveSuccess) { +TEST_F(ProtoDatabaseImplTest, TestDBSaveSuccess) { base::FilePath path(FILE_PATH_LITERAL("/fake/path")); MockDB* mock_db = new MockDB(); @@ -217,61 +220,53 @@ TEST_F(DomDistillerDatabaseTest, TestDBSaveSuccess) { EXPECT_CALL(*mock_db, Init(_)); EXPECT_CALL(caller, InitCallback(_)); db_->InitWithDatabase( - scoped_ptr<DomDistillerDatabase::Database>(mock_db), - base::FilePath(path), + scoped_ptr<LevelDB>(mock_db), base::FilePath(path), base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); - scoped_ptr<EntryVector> entries(new EntryVector()); + scoped_ptr<ProtoDatabase<TestProto>::KeyEntryVector> entries( + new ProtoDatabase<TestProto>::KeyEntryVector()); for (EntryMap::iterator it = model.begin(); it != model.end(); ++it) { - entries->push_back(it->second); + entries->push_back(std::make_pair(it->second.id(), it->second)); } - scoped_ptr<EntryVector> entries_to_remove(new EntryVector()); + scoped_ptr<KeyVector> keys_to_remove(new KeyVector()); EXPECT_CALL(*mock_db, Save(_, _)).WillOnce(VerifyUpdateEntries(model)); EXPECT_CALL(caller, SaveCallback(true)); db_->UpdateEntries( - entries.Pass(), - entries_to_remove.Pass(), + entries.Pass(), keys_to_remove.Pass(), base::Bind(&MockDatabaseCaller::SaveCallback, base::Unretained(&caller))); base::RunLoop().RunUntilIdle(); } -TEST_F(DomDistillerDatabaseTest, TestDBSaveFailure) { +TEST_F(ProtoDatabaseImplTest, TestDBSaveFailure) { base::FilePath path(FILE_PATH_LITERAL("/fake/path")); MockDB* mock_db = new MockDB(); MockDatabaseCaller caller; - scoped_ptr<EntryVector> entries(new EntryVector()); - scoped_ptr<EntryVector> entries_to_remove(new EntryVector()); + scoped_ptr<ProtoDatabase<TestProto>::KeyEntryVector> entries( + new ProtoDatabase<TestProto>::KeyEntryVector()); + scoped_ptr<KeyVector> keys_to_remove(new KeyVector()); EXPECT_CALL(*mock_db, Init(_)); EXPECT_CALL(caller, InitCallback(_)); db_->InitWithDatabase( - scoped_ptr<DomDistillerDatabase::Database>(mock_db), - base::FilePath(path), + scoped_ptr<LevelDB>(mock_db), base::FilePath(path), base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); EXPECT_CALL(*mock_db, Save(_, _)).WillOnce(Return(false)); EXPECT_CALL(caller, SaveCallback(false)); db_->UpdateEntries( - entries.Pass(), - entries_to_remove.Pass(), + entries.Pass(), keys_to_remove.Pass(), base::Bind(&MockDatabaseCaller::SaveCallback, base::Unretained(&caller))); base::RunLoop().RunUntilIdle(); } -ACTION_P(VerifyRemoveEntries, expected) { - const EntryVector& actual = arg1; - ExpectEntryPointersEquals(expected, actual); - return true; -} - -// Test that DomDistillerDatabase calls Save on the underlying database with the +// Test that ProtoDatabaseImpl calls Save on the underlying database with the // correct entries to delete and that the caller's SaveCallback is called with // the correct success value. -TEST_F(DomDistillerDatabaseTest, TestDBRemoveSuccess) { +TEST_F(ProtoDatabaseImplTest, TestDBRemoveSuccess) { base::FilePath path(FILE_PATH_LITERAL("/fake/path")); MockDB* mock_db = new MockDB(); @@ -281,55 +276,53 @@ TEST_F(DomDistillerDatabaseTest, TestDBRemoveSuccess) { EXPECT_CALL(*mock_db, Init(_)); EXPECT_CALL(caller, InitCallback(_)); db_->InitWithDatabase( - scoped_ptr<DomDistillerDatabase::Database>(mock_db), - base::FilePath(path), + scoped_ptr<LevelDB>(mock_db), base::FilePath(path), base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); - scoped_ptr<EntryVector> entries(new EntryVector()); - scoped_ptr<EntryVector> entries_to_remove(new EntryVector()); + scoped_ptr<ProtoDatabase<TestProto>::KeyEntryVector> entries( + new ProtoDatabase<TestProto>::KeyEntryVector()); + scoped_ptr<KeyVector> keys_to_remove(new KeyVector()); for (EntryMap::iterator it = model.begin(); it != model.end(); ++it) { - entries_to_remove->push_back(it->second); + keys_to_remove->push_back(it->second.id()); } - EXPECT_CALL(*mock_db, Save(_, _)).WillOnce(VerifyRemoveEntries(model)); + KeyVector keys_copy(*keys_to_remove.get()); + EXPECT_CALL(*mock_db, Save(_, keys_copy)).WillOnce(Return(true)); EXPECT_CALL(caller, SaveCallback(true)); db_->UpdateEntries( - entries.Pass(), - entries_to_remove.Pass(), + entries.Pass(), keys_to_remove.Pass(), base::Bind(&MockDatabaseCaller::SaveCallback, base::Unretained(&caller))); base::RunLoop().RunUntilIdle(); } -TEST_F(DomDistillerDatabaseTest, TestDBRemoveFailure) { +TEST_F(ProtoDatabaseImplTest, TestDBRemoveFailure) { base::FilePath path(FILE_PATH_LITERAL("/fake/path")); MockDB* mock_db = new MockDB(); MockDatabaseCaller caller; - scoped_ptr<EntryVector> entries(new EntryVector()); - scoped_ptr<EntryVector> entries_to_remove(new EntryVector()); + scoped_ptr<ProtoDatabase<TestProto>::KeyEntryVector> entries( + new ProtoDatabase<TestProto>::KeyEntryVector()); + scoped_ptr<KeyVector> keys_to_remove(new KeyVector()); EXPECT_CALL(*mock_db, Init(_)); EXPECT_CALL(caller, InitCallback(_)); db_->InitWithDatabase( - scoped_ptr<DomDistillerDatabase::Database>(mock_db), - base::FilePath(path), + scoped_ptr<LevelDB>(mock_db), base::FilePath(path), base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); EXPECT_CALL(*mock_db, Save(_, _)).WillOnce(Return(false)); EXPECT_CALL(caller, SaveCallback(false)); db_->UpdateEntries( - entries.Pass(), - entries_to_remove.Pass(), + entries.Pass(), keys_to_remove.Pass(), base::Bind(&MockDatabaseCaller::SaveCallback, base::Unretained(&caller))); base::RunLoop().RunUntilIdle(); } - // This tests that normal usage of the real database does not cause any // threading violations. -TEST(DomDistillerDatabaseThreadingTest, TestDBDestruction) { +TEST(ProtoDatabaseImplThreadingTest, TestDBDestruction) { base::MessageLoop main_loop; ScopedTempDir temp_dir; @@ -338,14 +331,13 @@ TEST(DomDistillerDatabaseThreadingTest, TestDBDestruction) { base::Thread db_thread("dbthread"); ASSERT_TRUE(db_thread.Start()); - scoped_ptr<DomDistillerDatabase> db( - new DomDistillerDatabase(db_thread.message_loop_proxy())); + scoped_ptr<ProtoDatabaseImpl<TestProto> > db( + new ProtoDatabaseImpl<TestProto>(db_thread.message_loop_proxy())); MockDatabaseCaller caller; EXPECT_CALL(caller, InitCallback(_)); - db->Init( - temp_dir.path(), - base::Bind(&MockDatabaseCaller::InitCallback, base::Unretained(&caller))); + db->Init(temp_dir.path(), base::Bind(&MockDatabaseCaller::InitCallback, + base::Unretained(&caller))); db.reset(); @@ -363,35 +355,43 @@ void TestLevelDBSaveAndLoad(bool close_after_save) { ASSERT_TRUE(temp_dir.CreateUniqueTempDir()); EntryMap model = GetSmallModel(); - EntryVector save_entries; - EntryVector load_entries; - EntryVector remove_entries; + + KeyValueVector save_entries; + std::vector<std::string> load_entries; + KeyVector remove_keys; for (EntryMap::iterator it = model.begin(); it != model.end(); ++it) { - save_entries.push_back(it->second); + save_entries.push_back( + std::make_pair(it->second.id(), it->second.SerializeAsString())); } - scoped_ptr<DomDistillerDatabase::LevelDB> db( - new DomDistillerDatabase::LevelDB()); + scoped_ptr<LevelDB> db(new LevelDB()); EXPECT_TRUE(db->Init(temp_dir.path())); - EXPECT_TRUE(db->Save(save_entries, remove_entries)); + EXPECT_TRUE(db->Save(save_entries, remove_keys)); if (close_after_save) { - db.reset(new DomDistillerDatabase::LevelDB()); + db.reset(new LevelDB()); EXPECT_TRUE(db->Init(temp_dir.path())); } EXPECT_TRUE(db->Load(&load_entries)); - - ExpectEntryPointersEquals(model, load_entries); + // Convert the strings back to TestProto. + std::vector<TestProto> loaded_protos; + for (std::vector<std::string>::iterator it = load_entries.begin(); + it != load_entries.end(); ++it) { + TestProto entry; + entry.ParseFromString(*it); + loaded_protos.push_back(entry); + } + ExpectEntryPointersEquals(model, loaded_protos); } -TEST(DomDistillerDatabaseLevelDBTest, TestDBSaveAndLoad) { +TEST(ProtoDatabaseImplLevelDBTest, TestDBSaveAndLoad) { TestLevelDBSaveAndLoad(false); } -TEST(DomDistillerDatabaseLevelDBTest, TestDBCloseAndReopen) { +TEST(ProtoDatabaseImplLevelDBTest, TestDBCloseAndReopen) { TestLevelDBSaveAndLoad(true); } -} // namespace dom_distiller +} // namespace leveldb_proto diff --git a/components/leveldb_proto/testing/fake_db.h b/components/leveldb_proto/testing/fake_db.h new file mode 100644 index 0000000..8e57e0e --- /dev/null +++ b/components/leveldb_proto/testing/fake_db.h @@ -0,0 +1,146 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_LEVELDB_PROTO_TESTING_FAKE_DB_H_ +#define COMPONENTS_LEVELDB_PROTO_TESTING_FAKE_DB_H_ + +#include <string> +#include <vector> + +#include "base/bind.h" +#include "base/callback.h" +#include "base/files/file_path.h" +#include "base/memory/scoped_ptr.h" +#include "components/leveldb_proto/proto_database.h" + +namespace leveldb_proto { +namespace test { + +template <typename T> +class FakeDB : public ProtoDatabase<T> { + typedef base::Callback<void(bool)> Callback; + + public: + typedef typename base::hash_map<std::string, T> EntryMap; + + explicit FakeDB(EntryMap* db); + virtual ~FakeDB(); + + virtual void Init(const base::FilePath& database_dir, + typename ProtoDatabase<T>::InitCallback callback) + OVERRIDE; + + virtual void UpdateEntries( + scoped_ptr<typename ProtoDatabase<T>::KeyEntryVector> entries_to_save, + scoped_ptr<std::vector<std::string> > keys_to_remove, + typename ProtoDatabase<T>::UpdateCallback callback) OVERRIDE; + + virtual void LoadEntries(typename ProtoDatabase<T>::LoadCallback callback) + OVERRIDE; + base::FilePath& GetDirectory(); + + void InitCallback(bool success); + + void LoadCallback(bool success); + + void UpdateCallback(bool success); + + static base::FilePath DirectoryForTestDB(); + + private: + static void RunLoadCallback( + typename ProtoDatabase<T>::LoadCallback callback, + scoped_ptr<typename std::vector<T> > entries, + bool success); + + base::FilePath dir_; + EntryMap* db_; + + Callback init_callback_; + Callback load_callback_; + Callback update_callback_; +}; + +template <typename T> +FakeDB<T>::FakeDB(EntryMap* db) + : db_(db) {} + +template <typename T> +FakeDB<T>::~FakeDB() {} + +template <typename T> +void FakeDB<T>::Init(const base::FilePath& database_dir, + typename ProtoDatabase<T>::InitCallback callback) { + dir_ = database_dir; + init_callback_ = callback; +} + +template <typename T> +void FakeDB<T>::UpdateEntries( + scoped_ptr<typename ProtoDatabase<T>::KeyEntryVector> entries_to_save, + scoped_ptr<std::vector<std::string> > keys_to_remove, + typename ProtoDatabase<T>::UpdateCallback callback) { + for (typename ProtoDatabase<T>::KeyEntryVector::iterator it = + entries_to_save->begin(); + it != entries_to_save->end(); ++it) { + (*db_)[it->first] = it->second; + } + for (std::vector<std::string>::iterator it = keys_to_remove->begin(); + it != keys_to_remove->end(); ++it) { + (*db_).erase(*it); + } + update_callback_ = callback; +} + +template <typename T> +void FakeDB<T>::LoadEntries(typename ProtoDatabase<T>::LoadCallback callback) { + scoped_ptr<std::vector<T> > entries(new std::vector<T>()); + for (typename EntryMap::iterator it = db_->begin(); it != db_->end(); ++it) { + entries->push_back(it->second); + } + load_callback_ = + base::Bind(RunLoadCallback, callback, base::Passed(&entries)); +} + +template <typename T> +base::FilePath& FakeDB<T>::GetDirectory() { + return dir_; +} + +template <typename T> +void FakeDB<T>::InitCallback(bool success) { + init_callback_.Run(success); + init_callback_.Reset(); +} + +template <typename T> +void FakeDB<T>::LoadCallback(bool success) { + load_callback_.Run(success); + load_callback_.Reset(); +} + +template <typename T> +void FakeDB<T>::UpdateCallback(bool success) { + update_callback_.Run(success); + update_callback_.Reset(); +} + +// static +template <typename T> +void FakeDB<T>::RunLoadCallback( + typename ProtoDatabase<T>::LoadCallback callback, + scoped_ptr<typename std::vector<T> > entries, bool success) { + callback.Run(success, entries.Pass()); +} + +// static +template <typename T> +base::FilePath FakeDB<T>::DirectoryForTestDB() { + return base::FilePath(FILE_PATH_LITERAL("/fake/path")); +} + +} // namespace test +} // namespace leveldb_proto + +#endif // COMPONENTS_LEVELDB_PROTO_TESTING_FAKE_DB_H_ diff --git a/components/leveldb_proto/testing/proto/test.proto b/components/leveldb_proto/testing/proto/test.proto new file mode 100644 index 0000000..8498105 --- /dev/null +++ b/components/leveldb_proto/testing/proto/test.proto @@ -0,0 +1,20 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +option optimize_for = LITE_RUNTIME; + +package leveldb_proto; + +// A proto to test leveldb_proto::ProtoDatabaseImpl. +// +// Next tag: 3 +message TestProto { + // Arbitary id. + optional string id = 1; + + // Arbitrary data. + optional string data = 2; +} |