diff options
author | noelutz@google.com <noelutz@google.com@0039d316-1c4b-4281-b951-d872f2087c98> | 2011-09-21 20:28:41 +0000 |
---|---|---|
committer | noelutz@google.com <noelutz@google.com@0039d316-1c4b-4281-b951-d872f2087c98> | 2011-09-21 20:28:41 +0000 |
commit | befc386449cb7be884668f8c7b1651dd905e560c (patch) | |
tree | 92bf66b10120aef111a96ea91cad6ff1ef08b2c1 | |
parent | 790e2fdbb6ca0ce407b4af36584442ee3dd41243 (diff) | |
download | chromium_src-befc386449cb7be884668f8c7b1651dd905e560c.zip chromium_src-befc386449cb7be884668f8c7b1651dd905e560c.tar.gz chromium_src-befc386449cb7be884668f8c7b1651dd905e560c.tar.bz2 |
Add a whitelist for the new binary download protection.
BUG=None
TEST=SafeBrowsingDatabaseTest
Review URL: http://codereview.chromium.org/7863006
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@102158 0039d316-1c4b-4281-b951-d872f2087c98
10 files changed, 367 insertions, 120 deletions
diff --git a/chrome/browser/safe_browsing/safe_browsing_database.cc b/chrome/browser/safe_browsing/safe_browsing_database.cc index e035e22..d3276d4 100644 --- a/chrome/browser/safe_browsing/safe_browsing_database.cc +++ b/chrome/browser/safe_browsing/safe_browsing_database.cc @@ -33,6 +33,9 @@ const FilePath::CharType kDownloadDBFile[] = FILE_PATH_LITERAL(" Download"); // Filename suffix for client-side phishing detection whitelist store. const FilePath::CharType kCsdWhitelistDBFile[] = FILE_PATH_LITERAL(" Csd Whitelist"); +// Filename suffix for the download whitelist store. +const FilePath::CharType kDownloadWhitelistDBFile[] = + FILE_PATH_LITERAL(" Download Whitelist"); // Filename suffix for browse store. // TODO(lzheng): change to a better name when we change the file format. const FilePath::CharType kBrowseDBFile[] = FILE_PATH_LITERAL(" Bloom"); @@ -40,15 +43,15 @@ const FilePath::CharType kBrowseDBFile[] = FILE_PATH_LITERAL(" Bloom"); // The maximum staleness for a cached entry. const int kMaxStalenessMinutes = 45; -// Maximum number of entries we allow in the client-side phishing detection -// whitelist. If the whitelist on disk contains more entries then -// ContainsCsdWhitelistedUrl will always return true. -const size_t kMaxCsdWhitelistSize = 5000; +// Maximum number of entries we allow in any of the whitelists. +// If a whitelist on disk contains more entries then all lookups to +// the whitelist will be considered a match. +const size_t kMaxWhitelistSize = 5000; -// If the hash of this exact expression is on the csd whitelist then -// ContainsCsdWhitelistedUrl will always return true. -const char kCsdKillSwitchUrl[] = - "sb-ssl.google.com/safebrowsing/csd/killswitch"; +// If the hash of this exact expression is on a whitelist then all +// lookups to this whitelist will be considered a match. +const char kWhitelistKillSwitchUrl[] = + "sb-ssl.google.com/safebrowsing/csd/killswitch"; // Don't change this! // To save space, the incoming |chunk_id| and |list_id| are combined // into an |encoded_chunk_id| for storage by shifting the |list_id| @@ -71,7 +74,7 @@ int EncodeChunkId(const int chunk, const int list_id) { // |include_whitelist_hashes| is true we will generate additional path-prefixes // to match against the csd whitelist. E.g., if the path-prefix /foo is on the // whitelist it should also match /foo/bar which is not the case for all the -// other lists. +// other lists. We'll also always add a pattern for the empty path. // TODO(shess): This function is almost the same as // |CompareFullHashes()| in safe_browsing_util.cc, except that code // does an early exit on match. Since match should be the infrequent @@ -395,11 +398,13 @@ class SafeBrowsingDatabaseFactoryImpl : public SafeBrowsingDatabaseFactory { public: virtual SafeBrowsingDatabase* CreateSafeBrowsingDatabase( bool enable_download_protection, - bool enable_client_side_whitelist) { + bool enable_client_side_whitelist, + bool enable_download_whitelist) { return new SafeBrowsingDatabaseNew( new SafeBrowsingStoreFile, enable_download_protection ? new SafeBrowsingStoreFile : NULL, - enable_client_side_whitelist ? new SafeBrowsingStoreFile : NULL); + enable_client_side_whitelist ? new SafeBrowsingStoreFile : NULL, + enable_download_whitelist ? new SafeBrowsingStoreFile : NULL); } SafeBrowsingDatabaseFactoryImpl() { } @@ -418,11 +423,13 @@ SafeBrowsingDatabaseFactory* SafeBrowsingDatabase::factory_ = NULL; // callers just construct things directly. SafeBrowsingDatabase* SafeBrowsingDatabase::Create( bool enable_download_protection, - bool enable_client_side_whitelist) { + bool enable_client_side_whitelist, + bool enable_download_whitelist) { if (!factory_) factory_ = new SafeBrowsingDatabaseFactoryImpl(); return factory_->CreateSafeBrowsingDatabase(enable_download_protection, - enable_client_side_whitelist); + enable_client_side_whitelist, + enable_download_whitelist); } SafeBrowsingDatabase::~SafeBrowsingDatabase() { @@ -452,6 +459,12 @@ FilePath SafeBrowsingDatabase::CsdWhitelistDBFilename( return FilePath(db_filename.value() + kCsdWhitelistDBFile); } +// static +FilePath SafeBrowsingDatabase::DownloadWhitelistDBFilename( + const FilePath& db_filename) { + return FilePath(db_filename.value() + kDownloadWhitelistDBFile); +} + SafeBrowsingStore* SafeBrowsingDatabaseNew::GetStore(const int list_id) { if (list_id == safe_browsing_util::PHISH || list_id == safe_browsing_util::MALWARE) { @@ -461,6 +474,8 @@ SafeBrowsingStore* SafeBrowsingDatabaseNew::GetStore(const int list_id) { return download_store_.get(); } else if (list_id == safe_browsing_util::CSDWHITELIST) { return csd_whitelist_store_.get(); + } else if (list_id == safe_browsing_util::DOWNLOADWHITELIST) { + return download_whitelist_store_.get(); } return NULL; } @@ -476,20 +491,24 @@ SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew() browse_store_(new SafeBrowsingStoreFile), download_store_(NULL), csd_whitelist_store_(NULL), + download_whitelist_store_(NULL), ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)) { DCHECK(browse_store_.get()); DCHECK(!download_store_.get()); DCHECK(!csd_whitelist_store_.get()); + DCHECK(!download_whitelist_store_.get()); } SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew( SafeBrowsingStore* browse_store, SafeBrowsingStore* download_store, - SafeBrowsingStore* csd_whitelist_store) + SafeBrowsingStore* csd_whitelist_store, + SafeBrowsingStore* download_whitelist_store) : creation_loop_(MessageLoop::current()), browse_store_(browse_store), download_store_(download_store), csd_whitelist_store_(csd_whitelist_store), + download_whitelist_store_(download_whitelist_store), ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)), corruption_detected_(false) { DCHECK(browse_store_.get()); @@ -505,6 +524,7 @@ void SafeBrowsingDatabaseNew::Init(const FilePath& filename_base) { DCHECK(browse_filename_.empty()); DCHECK(download_filename_.empty()); DCHECK(csd_whitelist_filename_.empty()); + DCHECK(download_whitelist_filename_.empty()); browse_filename_ = BrowseDBFilename(filename_base); bloom_filter_filename_ = BloomFilterForFilename(browse_filename_); @@ -541,12 +561,29 @@ void SafeBrowsingDatabaseNew::Init(const FilePath& filename_base) { DVLOG(1) << "Init csd whitelist store: " << csd_whitelist_filename_.value(); std::vector<SBAddFullHash> full_hashes; if (csd_whitelist_store_->GetAddFullHashes(&full_hashes)) { - LoadCsdWhitelist(full_hashes); + LoadWhitelist(full_hashes, &csd_whitelist_); + } else { + WhitelistEverything(&csd_whitelist_); + } + } else { + WhitelistEverything(&csd_whitelist_); // Just to be safe. + } + + if (download_whitelist_store_.get()) { + download_whitelist_filename_ = DownloadWhitelistDBFilename(filename_base); + download_whitelist_store_->Init( + download_whitelist_filename_, + NewCallback(this, &SafeBrowsingDatabaseNew::HandleCorruptDatabase)); + DVLOG(1) << "Init download whitelist store: " + << download_whitelist_filename_.value(); + std::vector<SBAddFullHash> full_hashes; + if (download_whitelist_store_->GetAddFullHashes(&full_hashes)) { + LoadWhitelist(full_hashes, &download_whitelist_); } else { - CsdWhitelistAllUrls(); + WhitelistEverything(&download_whitelist_); } } else { - CsdWhitelistAllUrls(); // Just to be safe. + WhitelistEverything(&download_whitelist_); // Just to be safe. } } @@ -573,7 +610,8 @@ bool SafeBrowsingDatabaseNew::ResetDatabase() { prefix_set_.reset(new safe_browsing::PrefixSet(std::vector<SBPrefix>())); } // Wants to acquire the lock itself. - CsdWhitelistAllUrls(); + WhitelistEverything(&csd_whitelist_); + WhitelistEverything(&download_whitelist_); return true; } @@ -713,15 +751,35 @@ bool SafeBrowsingDatabaseNew::ContainsCsdWhitelistedUrl(const GURL& url) { // This method is theoretically thread-safe but we expect all calls to // originate from the IO thread. DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); - base::AutoLock l(lookup_lock_); - if (csd_whitelist_all_urls_) - return true; + std::vector<SBFullHash> full_hashes; + BrowseFullHashesToCheck(url, true, &full_hashes); + return ContainsWhitelistedHashes(csd_whitelist_, full_hashes); +} +bool SafeBrowsingDatabaseNew::ContainsDownloadWhitelistedUrl(const GURL& url) { std::vector<SBFullHash> full_hashes; BrowseFullHashesToCheck(url, true, &full_hashes); - for (std::vector<SBFullHash>::const_iterator it = full_hashes.begin(); - it != full_hashes.end(); ++it) { - if (std::binary_search(csd_whitelist_.begin(), csd_whitelist_.end(), *it)) + return ContainsWhitelistedHashes(download_whitelist_, full_hashes); +} + +bool SafeBrowsingDatabaseNew::ContainsDownloadWhitelistedString( + const std::string& str) { + SBFullHash hash; + crypto::SHA256HashString(str, &hash, sizeof(hash)); + std::vector<SBFullHash> hashes; + hashes.push_back(hash); + return ContainsWhitelistedHashes(download_whitelist_, hashes); +} + +bool SafeBrowsingDatabaseNew::ContainsWhitelistedHashes( + const SBWhitelist& whitelist, + const std::vector<SBFullHash>& hashes) { + base::AutoLock l(lookup_lock_); + if (whitelist.second) + return true; + for (std::vector<SBFullHash>::const_iterator it = hashes.begin(); + it != hashes.end(); ++it) { + if (std::binary_search(whitelist.first.begin(), whitelist.first.end(), *it)) return true; } return false; @@ -979,7 +1037,14 @@ bool SafeBrowsingDatabaseNew::UpdateStarted( } if (csd_whitelist_store_.get() && !csd_whitelist_store_->BeginUpdate()) { - RecordFailure(FAILURE_CSD_WHITELIST_DATABASE_UPDATE_BEGIN); + RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN); + HandleCorruptDatabase(); + return false; + } + + if (download_whitelist_store_.get() && + !download_whitelist_store_->BeginUpdate()) { + RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN); HandleCorruptDatabase(); return false; } @@ -1003,6 +1068,14 @@ bool SafeBrowsingDatabaseNew::UpdateStarted( csd_whitelist_listnames, lists); } + if (download_whitelist_store_.get()) { + std::vector<std::string> download_whitelist_listnames; + download_whitelist_listnames.push_back( + safe_browsing_util::kDownloadWhiteList); + UpdateChunkRanges(download_whitelist_store_.get(), + download_whitelist_listnames, lists); + } + corruption_detected_ = false; change_detected_ = false; return true; @@ -1025,6 +1098,8 @@ void SafeBrowsingDatabaseNew::UpdateFinished(bool update_succeeded) { download_store_->CancelUpdate(); if (csd_whitelist_store_.get()) csd_whitelist_store_->CancelUpdate(); + if (download_whitelist_store_.get()) + download_whitelist_store_->CancelUpdate(); return; } @@ -1032,39 +1107,45 @@ void SafeBrowsingDatabaseNew::UpdateFinished(bool update_succeeded) { UpdateDownloadStore(); // for browsing UpdateBrowseStore(); - // for csd whitelist - UpdateCsdWhitelistStore(); + // for csd and download whitelists. + UpdateWhitelistStore(csd_whitelist_filename_, + csd_whitelist_store_.get(), + &csd_whitelist_); + UpdateWhitelistStore(download_whitelist_filename_, + download_whitelist_store_.get(), + &download_whitelist_); } -void SafeBrowsingDatabaseNew::UpdateCsdWhitelistStore() { - if (!csd_whitelist_store_.get()) +void SafeBrowsingDatabaseNew::UpdateWhitelistStore( + const FilePath& store_filename, + SafeBrowsingStore* store, + SBWhitelist* whitelist) { + if (!store) return; - // For the csd whitelist, we don't cache and save full hashes since all + // For the whitelists, we don't cache and save full hashes since all // hashes are already full. std::vector<SBAddFullHash> empty_add_hashes; - // Not needed for the csd whitelist. + // Not needed for the whitelists. std::set<SBPrefix> empty_miss_cache; // Note: prefixes will not be empty. The current data store implementation // stores all full-length hashes as both full and prefix hashes. std::vector<SBAddPrefix> prefixes; std::vector<SBAddFullHash> full_hashes; - if (!csd_whitelist_store_->FinishUpdate(empty_add_hashes, - empty_miss_cache, - &prefixes, - &full_hashes)) { - RecordFailure(FAILURE_CSD_WHITELIST_DATABASE_UPDATE_FINISH); - CsdWhitelistAllUrls(); + if (!store->FinishUpdate(empty_add_hashes, empty_miss_cache, &prefixes, + &full_hashes)) { + RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_FINISH); + WhitelistEverything(whitelist); return; } #if defined(OS_MACOSX) - base::mac::SetFileBackupExclusion(csd_whitelist_filename_); + base::mac::SetFileBackupExclusion(store_filename); #endif - LoadCsdWhitelist(full_hashes); + LoadWhitelist(full_hashes, whitelist); } void SafeBrowsingDatabaseNew::UpdateDownloadStore() { @@ -1279,10 +1360,15 @@ bool SafeBrowsingDatabaseNew::Delete() { if (!r3) RecordFailure(FAILURE_DATABASE_STORE_DELETE); - const bool r4 = file_util::Delete(bloom_filter_filename_, false); + const bool r4 = download_whitelist_store_.get() ? + download_whitelist_store_->Delete() : true; if (!r4) + RecordFailure(FAILURE_DATABASE_STORE_DELETE); + + const bool r5 = file_util::Delete(bloom_filter_filename_, false); + if (!r5) RecordFailure(FAILURE_DATABASE_FILTER_DELETE); - return r1 && r2 && r3 && r4; + return r1 && r2 && r3 && r4 && r5; } void SafeBrowsingDatabaseNew::WriteBloomFilter() { @@ -1304,37 +1390,38 @@ void SafeBrowsingDatabaseNew::WriteBloomFilter() { #endif } -void SafeBrowsingDatabaseNew::CsdWhitelistAllUrls() { +void SafeBrowsingDatabaseNew::WhitelistEverything(SBWhitelist* whitelist) { base::AutoLock locked(lookup_lock_); - csd_whitelist_all_urls_ = true; - csd_whitelist_.clear(); + whitelist->second = true; + whitelist->first.clear(); } -void SafeBrowsingDatabaseNew::LoadCsdWhitelist( - const std::vector<SBAddFullHash>& full_hashes) { +void SafeBrowsingDatabaseNew::LoadWhitelist( + const std::vector<SBAddFullHash>& full_hashes, + SBWhitelist* whitelist) { DCHECK_EQ(creation_loop_, MessageLoop::current()); - if (full_hashes.size() > kMaxCsdWhitelistSize) { - CsdWhitelistAllUrls(); + if (full_hashes.size() > kMaxWhitelistSize) { + WhitelistEverything(whitelist); return; } - std::vector<SBFullHash> new_csd_whitelist; + std::vector<SBFullHash> new_whitelist; for (std::vector<SBAddFullHash>::const_iterator it = full_hashes.begin(); it != full_hashes.end(); ++it) { - new_csd_whitelist.push_back(it->full_hash); + new_whitelist.push_back(it->full_hash); } - std::sort(new_csd_whitelist.begin(), new_csd_whitelist.end()); + std::sort(new_whitelist.begin(), new_whitelist.end()); SBFullHash kill_switch; - crypto::SHA256HashString(kCsdKillSwitchUrl, &kill_switch, + crypto::SHA256HashString(kWhitelistKillSwitchUrl, &kill_switch, sizeof(kill_switch)); - if (std::binary_search(new_csd_whitelist.begin(), new_csd_whitelist.end(), + if (std::binary_search(new_whitelist.begin(), new_whitelist.end(), kill_switch)) { // The kill switch is whitelisted hence we whitelist all URLs. - CsdWhitelistAllUrls(); + WhitelistEverything(whitelist); } else { base::AutoLock locked(lookup_lock_); - csd_whitelist_all_urls_ = false; - csd_whitelist_.swap(new_csd_whitelist); + whitelist->second = false; + whitelist->first.swap(new_whitelist); } } diff --git a/chrome/browser/safe_browsing/safe_browsing_database.h b/chrome/browser/safe_browsing/safe_browsing_database.h index eeb83e0..3a885e8 100644 --- a/chrome/browser/safe_browsing/safe_browsing_database.h +++ b/chrome/browser/safe_browsing/safe_browsing_database.h @@ -37,22 +37,24 @@ class SafeBrowsingDatabaseFactory { virtual ~SafeBrowsingDatabaseFactory() { } virtual SafeBrowsingDatabase* CreateSafeBrowsingDatabase( bool enable_download_protection, - bool enable_client_side_whitelist) = 0; + bool enable_client_side_whitelist, + bool enable_download_whitelist) = 0; private: DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseFactory); }; // Encapsulates on-disk databases that for safebrowsing. There are -// three databases: browse, download and client-side detection (csd) -// whitelist databases. The browse database contains information -// about phishing and malware urls. The download database contains +// four databases: browse, download, download whitelist and +// client-side detection (csd) whitelist databases. The browse database contains +// information about phishing and malware urls. The download database contains // URLs for bad binaries (e.g: those containing virus) and hash of -// these downloaded contents. The csd whitelist database contains URLs -// that will never be considered as phishing by the client-side -// phishing detection. These on-disk databases are shared among all -// profiles, as it doesn't contain user-specific data. This object is -// not thread-safe, i.e. all its methods should be used on the same +// these downloaded contents. The download whitelist contains whitelisted +// download hosting sites as well as whitelisted binary signing certificates +// etc. The csd whitelist database contains URLs that will never be considered +// as phishing by the client-side phishing detection. These on-disk databases +// are shared among all profiles, as it doesn't contain user-specific data. This +// object is not thread-safe, i.e. all its methods should be used on the same // thread that it was created on. class SafeBrowsingDatabase { public: @@ -62,8 +64,11 @@ class SafeBrowsingDatabase { // feature. // |enable_client_side_whitelist| is used to control the csd whitelist // database feature. + // |enable_download_whitelist| is used to control the download whitelist + // database feature. static SafeBrowsingDatabase* Create(bool enable_download_protection, - bool enable_client_side_whitelist); + bool enable_client_side_whitelist, + bool enable_download_whitelist); // Makes the passed |factory| the factory used to instantiate // a SafeBrowsingDatabase. This is used for tests. @@ -106,6 +111,16 @@ class SafeBrowsingDatabase { // This function should only be called from the IO thread. virtual bool ContainsCsdWhitelistedUrl(const GURL& url) = 0; + // The download whitelist is used for two purposes: a white-domain list of + // sites that are considered to host only harmless binaries as well as a + // whitelist of arbitrary strings such as hashed certificate authorities that + // are considered to be trusted. The two methods below let you lookup + // the whitelist either for a URL or an arbitrary string. These methods will + // return false if no match is found and true otherwise. + // This function could ONLY be accessed from the IO thread. + virtual bool ContainsDownloadWhitelistedUrl(const GURL& url) = 0; + virtual bool ContainsDownloadWhitelistedString(const std::string& str) = 0; + // A database transaction should look like: // // std::vector<SBListChunkRanges> lists; @@ -154,6 +169,10 @@ class SafeBrowsingDatabase { static FilePath CsdWhitelistDBFilename( const FilePath& csd_whitelist_base_filename); + // Filename for download whitelist databsae. + static FilePath DownloadWhitelistDBFilename( + const FilePath& download_whitelist_base_filename); + // Enumerate failures for histogramming purposes. DO NOT CHANGE THE // ORDERING OF THESE VALUES. enum FailureType { @@ -169,9 +188,8 @@ class SafeBrowsingDatabase { FAILURE_DATABASE_STORE_DELETE, FAILURE_DOWNLOAD_DATABASE_UPDATE_BEGIN, FAILURE_DOWNLOAD_DATABASE_UPDATE_FINISH, - FAILURE_CSD_WHITELIST_DATABASE_UPDATE_BEGIN, - FAILURE_CSD_WHITELIST_DATABASE_UPDATE_FINISH, - + FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN, + FAILURE_WHITELIST_DATABASE_UPDATE_FINISH, // Memory space for histograms is determined by the max. ALWAYS // ADD NEW VALUES BEFORE THIS ONE. FAILURE_DATABASE_MAX @@ -188,14 +206,15 @@ class SafeBrowsingDatabase { class SafeBrowsingDatabaseNew : public SafeBrowsingDatabase { public: - // Create a database with a browse store, download store and - // csd_whitelist_store. Takes ownership of browse_store, download_store and - // csd_whitelist_store. When |download_store| is NULL, the database - // will ignore any operations related download (url hashes and - // binary hashes). Same for the |csd_whitelist_store|. + // Create a database with a browse, download, download whitelist and + // csd whitelist store objects. Takes ownership of all the store objects. + // When |download_store| is NULL, the database will ignore any operations + // related download (url hashes and binary hashes). The same is true for + // the |csd_whitelist_store| and |download_whitelist_store|. SafeBrowsingDatabaseNew(SafeBrowsingStore* browse_store, SafeBrowsingStore* download_store, - SafeBrowsingStore* csd_whitelist_store); + SafeBrowsingStore* csd_whitelist_store, + SafeBrowsingStore* download_whitelist_store); // Create a database with a browse store. This is a legacy interface that // useds Sqlite. @@ -215,6 +234,8 @@ class SafeBrowsingDatabaseNew : public SafeBrowsingDatabase { std::vector<SBPrefix>* prefix_hits); virtual bool ContainsDownloadHashPrefix(const SBPrefix& prefix); virtual bool ContainsCsdWhitelistedUrl(const GURL& url); + virtual bool ContainsDownloadWhitelistedUrl(const GURL& url); + virtual bool ContainsDownloadWhitelistedString(const std::string& str); virtual bool UpdateStarted(std::vector<SBListChunkRanges>* lists); virtual void InsertChunks(const std::string& list_name, const SBChunkList& chunks); @@ -227,8 +248,18 @@ class SafeBrowsingDatabaseNew : public SafeBrowsingDatabase { friend class SafeBrowsingDatabaseTest; FRIEND_TEST(SafeBrowsingDatabaseTest, HashCaching); - // Return the browse_store_, download_store_ or csd_whitelist_store_ - // based on list_id. + // A SafeBrowsing whitelist contains a list of whitelisted full-hashes (stored + // in a sorted vector) as well as a boolean flag indicating whether all + // lookups in the whitelist should be considered matches for safety. + typedef std::pair<std::vector<SBFullHash>, bool> SBWhitelist; + + // Returns true if the whitelist is disabled or if any of the given hashes + // matches the whitelist. + bool ContainsWhitelistedHashes(const SBWhitelist& whitelist, + const std::vector<SBFullHash>& hashes); + + // Return the browse_store_, download_store_, download_whitelist_store or + // csd_whitelist_store_ based on list_id. SafeBrowsingStore* GetStore(int list_id); // Deletes the files on disk. @@ -240,14 +271,15 @@ class SafeBrowsingDatabaseNew : public SafeBrowsingDatabase { // Writes the current bloom filter to disk. void WriteBloomFilter(); - // Loads the given full-length hashes to the csd whitelist. If the number + // Loads the given full-length hashes to the given whitelist. If the number // of hashes is too large or if the kill switch URL is on the whitelist - // we will whitelist all URLs. - void LoadCsdWhitelist(const std::vector<SBAddFullHash>& full_hashes); + // we will whitelist everything. + void LoadWhitelist(const std::vector<SBAddFullHash>& full_hashes, + SBWhitelist* whitelist); - // Call this method if an error occured with the csd whitelist. This will - // result in all calls to ContainsCsdWhitelistedUrl() to returning true. - void CsdWhitelistAllUrls(); + // Call this method if an error occured with the given whitelist. This will + // result in all lookups to the whitelist to return true. + void WhitelistEverything(SBWhitelist* whitelist); // Helpers for handling database corruption. // |OnHandleCorruptDatabase()| runs |ResetDatabase()| and sets @@ -267,7 +299,9 @@ class SafeBrowsingDatabaseNew : public SafeBrowsingDatabase { void UpdateDownloadStore(); void UpdateBrowseStore(); - void UpdateCsdWhitelistStore(); + void UpdateWhitelistStore(const FilePath& store_filename, + SafeBrowsingStore* store, + SBWhitelist* whitelist); // Helper function to compare addprefixes in download_store_ with |prefixes|. // The |list_bit| indicates which list (download url or download hash) @@ -302,15 +336,13 @@ class SafeBrowsingDatabaseNew : public SafeBrowsingDatabase { FilePath csd_whitelist_filename_; scoped_ptr<SafeBrowsingStore> csd_whitelist_store_; - // All the client-side phishing detection whitelist entries are loaded in - // a sorted vector. - std::vector<SBFullHash> csd_whitelist_; + // For the download whitelist chunks and full-length hashes. This list only + // contains 256 bit hashes. + FilePath download_whitelist_filename_; + scoped_ptr<SafeBrowsingStore> download_whitelist_store_; - // If true, ContainsCsdWhitelistedUrl will always return true for all URLs. - // This is set to true if the csd whitelist is too large to be stored in - // memory, if the kill switch URL is on the csd whitelist or if there was - // an error during the most recent update. - bool csd_whitelist_all_urls_; + SBWhitelist csd_whitelist_; + SBWhitelist download_whitelist_; // Bloom filter generated from the add-prefixes in |browse_store_|. // Only browse_store_ requires the BloomFilter for fast query. diff --git a/chrome/browser/safe_browsing/safe_browsing_database_unittest.cc b/chrome/browser/safe_browsing/safe_browsing_database_unittest.cc index 634797e..088701b 100644 --- a/chrome/browser/safe_browsing/safe_browsing_database_unittest.cc +++ b/chrome/browser/safe_browsing/safe_browsing_database_unittest.cc @@ -366,9 +366,11 @@ TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) { SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile(); SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile(); SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile(); + SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile(); database_.reset(new SafeBrowsingDatabaseNew(browse_store, download_store, - csd_whitelist_store)); + csd_whitelist_store, + download_whitelist_store)); database_->Init(database_filename_); SBChunkList chunks; @@ -410,10 +412,18 @@ TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) { chunks.push_back(chunk); database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks); + chunk.hosts.clear(); + InsertAddChunkHostFullHashes(&chunk, 6, "www.download.com/", + "www.download.com/"); + + chunks.clear(); + chunks.push_back(chunk); + database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks); + database_->UpdateFinished(true); GetListsInfo(&lists); - EXPECT_EQ(5U, lists.size()); + EXPECT_EQ(6U, lists.size()); EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); EXPECT_EQ(lists[0].adds, "1"); EXPECT_TRUE(lists[0].subs.empty()); @@ -429,6 +439,9 @@ TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) { EXPECT_TRUE(lists[4].name == safe_browsing_util::kCsdWhiteList); EXPECT_EQ(lists[4].adds, "5"); EXPECT_TRUE(lists[4].subs.empty()); + EXPECT_TRUE(lists[5].name == safe_browsing_util::kDownloadWhiteList); + EXPECT_EQ(lists[5].adds, "6"); + EXPECT_TRUE(lists[5].subs.empty()); database_.reset(); } @@ -1052,7 +1065,7 @@ TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) { database_.reset(); MessageLoop loop(MessageLoop::TYPE_DEFAULT); SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile(); - database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL)); + database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL, NULL)); database_->Init(database_filename_); // This will cause an empty database to be created. @@ -1124,7 +1137,8 @@ TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) { SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile(); database_.reset(new SafeBrowsingDatabaseNew(browse_store, download_store, - csd_whitelist_store)); + csd_whitelist_store, + NULL)); database_->Init(database_filename_); const char kEvil1Host[] = "www.evil1.com/"; @@ -1216,25 +1230,30 @@ TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) { database_.reset(); } -// Checks that the csd-whitelist is handled properly. -TEST_F(SafeBrowsingDatabaseTest, CsdWhitelist) { +// Checks that the whitelists are handled properly. +TEST_F(SafeBrowsingDatabaseTest, Whitelists) { database_.reset(); MessageLoop loop(MessageLoop::TYPE_DEFAULT); - // We expect all calls to ContainsCsdWhitelistedUrl to be made from the IO - // thread. + // We expect all calls to ContainsCsdWhitelistedUrl in particular to be made + // from the IO thread. In general the whitelist lookups are thread-safe. BrowserThread io_thread(BrowserThread::IO, &loop); // If the whitelist is disabled everything should match the whitelist. database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(), - NULL, NULL)); + NULL, NULL, NULL)); database_->Init(database_filename_); - EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( - GURL(std::string("http://www.phishig.com/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://www.phishing.com/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://www.phishing.com/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf")); SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile(); SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile(); + SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile(); database_.reset(new SafeBrowsingDatabaseNew(browse_store, NULL, - csd_whitelist_store)); + csd_whitelist_store, + download_whitelist_store)); database_->Init(database_filename_); const char kGood1Host[] = "www.good1.com/"; @@ -1244,20 +1263,41 @@ TEST_F(SafeBrowsingDatabaseTest, CsdWhitelist) { const char kGood2Host[] = "www.good2.com/"; const char kGood2Url1[] = "www.good2.com/c"; // Should match '/c/bla'. - SBChunkList chunks; + // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist. + const char kGood3Host[] = "good3.com/"; + const char kGood3Url1[] = "good3.com/"; + + const char kGoodString[] = "good_string"; + + SBChunkList download_chunks, csd_chunks; SBChunk chunk; // Add two simple chunks to the csd whitelist. InsertAddChunkHost2FullHashes(&chunk, 1, kGood1Host, kGood1Url1, kGood1Url2); - chunks.push_back(chunk); + csd_chunks.push_back(chunk); chunk.hosts.clear(); InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1); - chunks.push_back(chunk); + csd_chunks.push_back(chunk); + + chunk.hosts.clear(); + InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1); + download_chunks.push_back(chunk); + + chunk.hosts.clear(); + InsertAddChunkHostFullHashes(&chunk, 3, kGoodString, kGoodString); + download_chunks.push_back(chunk); + + chunk.hosts.clear(); + InsertAddChunkHostFullHashes(&chunk, 4, kGood3Host, kGood3Url1); + download_chunks.push_back(chunk); std::vector<SBListChunkRanges> lists; EXPECT_TRUE(database_->UpdateStarted(&lists)); - database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks); + database_->InsertChunks(safe_browsing_util::kCsdWhiteList, + csd_chunks); + database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, + download_chunks); database_->UpdateFinished(true); EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl( @@ -1286,16 +1326,42 @@ TEST_F(SafeBrowsingDatabaseTest, CsdWhitelist) { EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl( GURL(std::string("http://www.google.com/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://") + kGood2Url1 + "/c"))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://") + kGood2Url1 + "/c?bla"))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://") + kGood2Url1 + "/c/bla"))); + + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://good3.com/a/b/c/d/e/f/g/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://a.b.good3.com/")))); + + EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf")); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString)); + + EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://www.google.com/")))); + // Test that the kill-switch works as intended. - chunks.clear(); + csd_chunks.clear(); + download_chunks.clear(); lists.clear(); - SBChunk chunk2; - InsertAddChunkHostFullHashes(&chunk2, 3, "sb-ssl.google.com/", + chunk.hosts.clear(); + InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/", "sb-ssl.google.com/safebrowsing/csd/killswitch"); - chunks.push_back(chunk2); + csd_chunks.push_back(chunk); + + chunk.hosts.clear(); + InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/", + "sb-ssl.google.com/safebrowsing/csd/killswitch"); + download_chunks.push_back(chunk); EXPECT_TRUE(database_->UpdateStarted(&lists)); - database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks); + database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks); + database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, + download_chunks); database_->UpdateFinished(true); EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( @@ -1305,17 +1371,36 @@ TEST_F(SafeBrowsingDatabaseTest, CsdWhitelist) { EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( GURL(std::string("http://www.phishing_url.com/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("https://") + kGood1Url2 + "/c.html"))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://www.google.com/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://www.phishing_url.com/")))); + + EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf")); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString)); + // Remove the kill-switch and verify that we can recover. - chunks.clear(); + csd_chunks.clear(); + download_chunks.clear(); lists.clear(); SBChunk sub_chunk; - InsertSubChunkHostFullHash(&sub_chunk, 1, 3, + InsertSubChunkHostFullHash(&sub_chunk, 1, 5, + "sb-ssl.google.com/", + "sb-ssl.google.com/safebrowsing/csd/killswitch"); + csd_chunks.push_back(sub_chunk); + + sub_chunk.hosts.clear(); + InsertSubChunkHostFullHash(&sub_chunk, 1, 5, "sb-ssl.google.com/", "sb-ssl.google.com/safebrowsing/csd/killswitch"); - chunks.push_back(sub_chunk); + download_chunks.push_back(sub_chunk); EXPECT_TRUE(database_->UpdateStarted(&lists)); - database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks); + database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks); + database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, + download_chunks); database_->UpdateFinished(true); EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( @@ -1327,6 +1412,17 @@ TEST_F(SafeBrowsingDatabaseTest, CsdWhitelist) { EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl( GURL(std::string("http://www.phishing_url.com/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("https://") + kGood2Url1 + "/c/bla"))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("https://good3.com/")))); + EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString)); + EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://www.google.com/")))); + EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl( + GURL(std::string("http://www.phishing_url.com/")))); + EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf")); + database_.reset(); } @@ -1439,7 +1535,7 @@ TEST_F(SafeBrowsingDatabaseTest, BinHashInsertLookup) { SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile(); database_.reset(new SafeBrowsingDatabaseNew(browse_store, download_store, - NULL)); + NULL, NULL)); database_->Init(database_filename_); SBChunkList chunks; diff --git a/chrome/browser/safe_browsing/safe_browsing_service.cc b/chrome/browser/safe_browsing/safe_browsing_service.cc index d640f1c..ca0c508 100644 --- a/chrome/browser/safe_browsing/safe_browsing_service.cc +++ b/chrome/browser/safe_browsing/safe_browsing_service.cc @@ -161,6 +161,7 @@ SafeBrowsingService::SafeBrowsingService() enabled_(false), enable_download_protection_(false), enable_csd_whitelist_(false), + enable_download_whitelist_(false), update_in_progress_(false), database_update_in_progress_(false), closing_database_(false), @@ -687,7 +688,8 @@ SafeBrowsingDatabase* SafeBrowsingService::GetDatabase() { SafeBrowsingDatabase* database = SafeBrowsingDatabase::Create(enable_download_protection_, - enable_csd_whitelist_); + enable_csd_whitelist_, + enable_download_whitelist_); database->Init(path); { @@ -907,6 +909,9 @@ void SafeBrowsingService::Start() { local_state->GetBoolean(prefs::kMetricsReportingEnabled)))); #endif + enable_download_whitelist_ = cmdline->HasSwitch( + switches::kEnableImprovedDownloadProtection); + BrowserThread::PostTask( BrowserThread::IO, FROM_HERE, NewRunnableMethod( diff --git a/chrome/browser/safe_browsing/safe_browsing_service.h b/chrome/browser/safe_browsing/safe_browsing_service.h index 0b94938..66a4952 100644 --- a/chrome/browser/safe_browsing/safe_browsing_service.h +++ b/chrome/browser/safe_browsing/safe_browsing_service.h @@ -495,6 +495,9 @@ class SafeBrowsingService // or not. bool enable_csd_whitelist_; + // Indicate if the download whitelist should be enabled or not. + bool enable_download_whitelist_; + // The SafeBrowsing thread that runs database operations. // // Note: Functions that run on this thread should run synchronously and return diff --git a/chrome/browser/safe_browsing/safe_browsing_service_browsertest.cc b/chrome/browser/safe_browsing/safe_browsing_service_browsertest.cc index ea5743a..72989fb 100644 --- a/chrome/browser/safe_browsing/safe_browsing_service_browsertest.cc +++ b/chrome/browser/safe_browsing/safe_browsing_service_browsertest.cc @@ -84,6 +84,12 @@ class TestSafeBrowsingDatabase : public SafeBrowsingDatabase { virtual bool ContainsCsdWhitelistedUrl(const GURL& url) { return true; } + virtual bool ContainsDownloadWhitelistedString(const std::string& str) { + return true; + } + virtual bool ContainsDownloadWhitelistedUrl(const GURL& url) { + return true; + } virtual bool UpdateStarted(std::vector<SBListChunkRanges>* lists) { ADD_FAILURE() << "Not implemented."; return false; @@ -167,7 +173,8 @@ class TestSafeBrowsingDatabaseFactory : public SafeBrowsingDatabaseFactory { virtual SafeBrowsingDatabase* CreateSafeBrowsingDatabase( bool enable_download_protection, - bool enable_client_side_whitelist) { + bool enable_client_side_whitelist, + bool enable_download_whitelist) { db_ = new TestSafeBrowsingDatabase(); return db_; } diff --git a/chrome/browser/safe_browsing/safe_browsing_util.cc b/chrome/browser/safe_browsing/safe_browsing_util.cc index bb13a00..8503913 100644 --- a/chrome/browser/safe_browsing/safe_browsing_util.cc +++ b/chrome/browser/safe_browsing/safe_browsing_util.cc @@ -170,6 +170,7 @@ const char kPhishingList[] = "goog-phish-shavar"; const char kBinUrlList[] = "goog-badbinurl-shavar"; const char kBinHashList[] = "goog-badbin-digestvar"; const char kCsdWhiteList[] = "goog-csdwhite-sha256"; +const char kDownloadWhiteList[] = "goog-downloadwhite-digest256"; int GetListId(const std::string& name) { int id; @@ -183,6 +184,8 @@ int GetListId(const std::string& name) { id = BINHASH; } else if (name == safe_browsing_util::kCsdWhiteList) { id = CSDWHITELIST; + } else if (name == safe_browsing_util::kDownloadWhiteList) { + id = DOWNLOADWHITELIST; } else { id = INVALID; } @@ -206,6 +209,9 @@ bool GetListName(int list_id, std::string* list) { case CSDWHITELIST: *list = safe_browsing_util::kCsdWhiteList; break; + case DOWNLOADWHITELIST: + *list = safe_browsing_util::kDownloadWhiteList; + break; default: return false; } diff --git a/chrome/browser/safe_browsing/safe_browsing_util.h b/chrome/browser/safe_browsing/safe_browsing_util.h index 2911f46..577ffab 100644 --- a/chrome/browser/safe_browsing/safe_browsing_util.h +++ b/chrome/browser/safe_browsing/safe_browsing_util.h @@ -266,6 +266,8 @@ extern const char kBinUrlList[]; extern const char kBinHashList[]; // SafeBrowsing client-side detection whitelist list name. extern const char kCsdWhiteList[]; +// SafeBrowsing download whitelist list name. +extern const char kDownloadWhiteList[]; enum ListType { INVALID = -1, @@ -274,6 +276,10 @@ enum ListType { BINURL = 2, BINHASH = 3, CSDWHITELIST = 4, + // SafeBrowsing lists are stored in pairs. Keep ListType 5 + // available for a potential second list that we would store in the + // csd-whitelist store file. + DOWNLOADWHITELIST = 6, }; // Maps a list name to ListType. diff --git a/chrome/common/chrome_switches.cc b/chrome/common/chrome_switches.cc index a413f11..28b9076 100644 --- a/chrome/common/chrome_switches.cc +++ b/chrome/common/chrome_switches.cc @@ -446,6 +446,10 @@ const char kEnableFastback[] = "enable-fastback"; // testing, for example page cycler and layout tests. See bug 1157243. const char kEnableFileCookies[] = "enable-file-cookies"; +// Enable improved SafeBrowsing download protection. +const char kEnableImprovedDownloadProtection[] = + "enable-improved-download-protection"; + // Enable the in-browser thumbnailing, which is more efficient than the // in-renderer thumbnailing, as we can use more information to determine // if we need to update thumbnails. diff --git a/chrome/common/chrome_switches.h b/chrome/common/chrome_switches.h index 732e45b..f82923e 100644 --- a/chrome/common/chrome_switches.h +++ b/chrome/common/chrome_switches.h @@ -130,6 +130,7 @@ extern const char kEnableExperimentalExtensionApis[]; extern const char kEnableExtensionTimelineApi[]; extern const char kEnableFastback[]; extern const char kEnableFileCookies[]; +extern const char kEnableImprovedDownloadProtection[]; extern const char kEnableInBrowserThumbnailing[]; extern const char kEnableIPv6[]; extern const char kEnableIPCFuzzing[]; |