From ddcb01e6116caefe5b89c589455df887ba18b1cd Mon Sep 17 00:00:00 2001 From: vakh Date: Fri, 25 Mar 2016 18:15:54 -0700 Subject: v4_update_protocol_manager: Basic implementation with TODOs BUG=543161 Review URL: https://codereview.chromium.org/1727033003 Cr-Commit-Position: refs/heads/master@{#383433} --- components/components_tests.gyp | 1 + components/safe_browsing_db.gypi | 2 + components/safe_browsing_db/BUILD.gn | 45 +++- .../v4_get_hash_protocol_manager.cc | 25 +- .../v4_get_hash_protocol_manager.h | 30 --- .../safe_browsing_db/v4_protocol_manager_util.cc | 19 ++ .../safe_browsing_db/v4_protocol_manager_util.h | 58 ++++- .../safe_browsing_db/v4_update_protocol_manager.cc | 277 +++++++++++++++++++++ .../safe_browsing_db/v4_update_protocol_manager.h | 178 +++++++++++++ .../v4_update_protocol_manager_unittest.cc | 199 +++++++++++++++ tools/metrics/histograms/histograms.xml | 46 +++- 11 files changed, 815 insertions(+), 65 deletions(-) create mode 100644 components/safe_browsing_db/v4_update_protocol_manager.cc create mode 100644 components/safe_browsing_db/v4_update_protocol_manager.h create mode 100644 components/safe_browsing_db/v4_update_protocol_manager_unittest.cc diff --git a/components/components_tests.gyp b/components/components_tests.gyp index be9d5dd..5c08282 100644 --- a/components/components_tests.gyp +++ b/components/components_tests.gyp @@ -640,6 +640,7 @@ 'safe_browsing_db/util_unittest.cc', 'safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc', 'safe_browsing_db/v4_protocol_manager_util_unittest.cc', + 'safe_browsing_db/v4_update_protocol_manager_unittest.cc', ], 'safe_json_unittest_sources': [ 'safe_json/json_sanitizer_unittest.cc', diff --git a/components/safe_browsing_db.gypi b/components/safe_browsing_db.gypi index 7636168d..bd62fca 100644 --- a/components/safe_browsing_db.gypi +++ b/components/safe_browsing_db.gypi @@ -28,6 +28,8 @@ 'safe_browsing_db/v4_protocol_manager_util.cc', 'safe_browsing_db/v4_get_hash_protocol_manager.h', 'safe_browsing_db/v4_get_hash_protocol_manager.cc', + 'safe_browsing_db/v4_update_protocol_manager.h', + 'safe_browsing_db/v4_update_protocol_manager.cc', ], 'include_dirs': [ '..', diff --git a/components/safe_browsing_db/BUILD.gn b/components/safe_browsing_db/BUILD.gn index f1154f7..975861a 100644 --- a/components/safe_browsing_db/BUILD.gn +++ b/components/safe_browsing_db/BUILD.gn @@ -51,6 +51,7 @@ source_set("database_manager") { ":hit_report", ":util", ":v4_get_hash_protocol_manager", + ":v4_update_protocol_manager", "//base:base", "//content/public/browser", "//content/public/common", @@ -123,6 +124,18 @@ source_set("safe_browsing_api_handler_util") { ] } +source_set("test_database_manager") { + sources = [ + "test_database_manager.cc", + "test_database_manager.h", + ] + deps = [ + ":database_manager", + "//base:base", + "//net", + ] +} + source_set("util") { sources = [ "util.cc", @@ -130,8 +143,10 @@ source_set("util") { ] deps = [ "//base", + "//base:base", "//crypto", "//net", + "//url:url", ] if (is_win) { # TODO(jschuh): crbug.com/167187 fix size_t to int truncations. @@ -139,18 +154,6 @@ source_set("util") { } } -source_set("test_database_manager") { - sources = [ - "test_database_manager.cc", - "test_database_manager.h", - ] - deps = [ - ":database_manager", - "//base:base", - "//net", - ] -} - source_set("v4_get_hash_protocol_manager") { sources = [ "v4_get_hash_protocol_manager.cc", @@ -174,6 +177,22 @@ source_set("v4_protocol_manager_util") { "v4_protocol_manager_util.h", ] deps = [ + ":proto", + "//base", + "//net", + "//url:url", + ] +} + +source_set("v4_update_protocol_manager") { + sources = [ + "v4_update_protocol_manager.cc", + "v4_update_protocol_manager.h", + ] + deps = [ + ":proto", + ":util", + ":v4_protocol_manager_util", "//base", "//net", "//url:url", @@ -187,6 +206,7 @@ source_set("unit_tests") { "util_unittest.cc", "v4_get_hash_protocol_manager_unittest.cc", "v4_protocol_manager_util_unittest.cc", + "v4_update_protocol_manager_unittest.cc", ] deps = [ ":prefix_set", @@ -194,6 +214,7 @@ source_set("unit_tests") { ":util", ":v4_get_hash_protocol_manager", ":v4_protocol_manager_util", + ":v4_update_protocol_manager", "//base", "//net", "//net:test_support", diff --git a/components/safe_browsing_db/v4_get_hash_protocol_manager.cc b/components/safe_browsing_db/v4_get_hash_protocol_manager.cc index 55196ba..b0a30bc 100644 --- a/components/safe_browsing_db/v4_get_hash_protocol_manager.cc +++ b/components/safe_browsing_db/v4_get_hash_protocol_manager.cc @@ -55,6 +55,13 @@ void RecordParseGetHashResult(ParseResultType result_type) { PARSE_RESULT_TYPE_MAX); } +// Record a GetHash result. +void RecordGetHashResult(safe_browsing::V4OperationResult result) { + UMA_HISTOGRAM_ENUMERATION( + "SafeBrowsing.GetV4HashResult", result, + safe_browsing::V4OperationResult::OPERATION_RESULT_MAX); +} + } // namespace namespace safe_browsing { @@ -108,12 +115,6 @@ V4GetHashProtocolManager::V4GetHashProtocolManager( url_fetcher_id_(0) { } -// static -void V4GetHashProtocolManager::RecordGetHashResult(ResultType result_type) { - UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.GetV4HashResult", result_type, - GET_HASH_RESULT_MAX); -} - V4GetHashProtocolManager::~V4GetHashProtocolManager() { // Delete in-progress SafeBrowsing requests. STLDeleteContainerPairFirstPointers(hash_requests_.begin(), @@ -248,9 +249,9 @@ void V4GetHashProtocolManager::GetFullHashes( // (i.e. treat the page as safe). if (Time::Now() <= next_gethash_time_) { if (gethash_error_count_) { - RecordGetHashResult(GET_HASH_BACKOFF_ERROR); + RecordGetHashResult(V4OperationResult::BACKOFF_ERROR); } else { - RecordGetHashResult(GET_HASH_MIN_WAIT_DURATION_ERROR); + RecordGetHashResult(V4OperationResult::MIN_WAIT_DURATION_ERROR); } std::vector full_hashes; callback.Run(full_hashes, base::TimeDelta()); @@ -301,13 +302,13 @@ void V4GetHashProtocolManager::OnURLFetchComplete( std::vector full_hashes; base::TimeDelta negative_cache_duration; if (status.is_success() && response_code == net::HTTP_OK) { - RecordGetHashResult(GET_HASH_STATUS_200); + RecordGetHashResult(V4OperationResult::STATUS_200); ResetGetHashErrors(); std::string data; source->GetResponseAsString(&data); if (!ParseHashResponse(data, &full_hashes, &negative_cache_duration)) { full_hashes.clear(); - RecordGetHashResult(GET_HASH_PARSE_ERROR); + RecordGetHashResult(V4OperationResult::PARSE_ERROR); } } else { HandleGetHashError(Time::Now()); @@ -317,9 +318,9 @@ void V4GetHashProtocolManager::OnURLFetchComplete( << " and response code: " << response_code; if (status.status() == net::URLRequestStatus::FAILED) { - RecordGetHashResult(GET_HASH_NETWORK_ERROR); + RecordGetHashResult(V4OperationResult::NETWORK_ERROR); } else { - RecordGetHashResult(GET_HASH_HTTP_ERROR); + RecordGetHashResult(V4OperationResult::HTTP_ERROR); } } diff --git a/components/safe_browsing_db/v4_get_hash_protocol_manager.h b/components/safe_browsing_db/v4_get_hash_protocol_manager.h index f0bfc74..407a7b6 100644 --- a/components/safe_browsing_db/v4_get_hash_protocol_manager.h +++ b/components/safe_browsing_db/v4_get_hash_protocol_manager.h @@ -78,36 +78,6 @@ class V4GetHashProtocolManager : public net::URLFetcherDelegate, virtual void GetFullHashesWithApis(const std::vector& prefixes, FullHashCallback callback); - // Enumerate failures for histogramming purposes. DO NOT CHANGE THE - // ORDERING OF THESE VALUES. - enum ResultType { - // 200 response code means that the server recognized the hash - // prefix. - GET_HASH_STATUS_200 = 0, - - // Subset of successful responses where the response body wasn't parsable. - GET_HASH_PARSE_ERROR = 1, - - // Gethash request failed (network error). - GET_HASH_NETWORK_ERROR = 2, - - // Gethash request returned HTTP result code other than 200. - GET_HASH_HTTP_ERROR = 3, - - // Gethash attempted during error backoff, no request sent. - GET_HASH_BACKOFF_ERROR = 4, - - // Gethash attempted before min wait duration elapsed, no request sent. - GET_HASH_MIN_WAIT_DURATION_ERROR = 5, - - // Memory space for histograms is determined by the max. ALWAYS - // ADD NEW VALUES BEFORE THIS ONE. - GET_HASH_RESULT_MAX = 6 - }; - - // Record a GetHash result. - static void RecordGetHashResult(ResultType result_type); - protected: // Constructs a V4GetHashProtocolManager that issues // network requests using |request_context_getter|. diff --git a/components/safe_browsing_db/v4_protocol_manager_util.cc b/components/safe_browsing_db/v4_protocol_manager_util.cc index 5f91bfba..2dddc16 100644 --- a/components/safe_browsing_db/v4_protocol_manager_util.cc +++ b/components/safe_browsing_db/v4_protocol_manager_util.cc @@ -18,6 +18,25 @@ namespace safe_browsing { // The Safe Browsing V4 server URL prefix. const char kSbV4UrlPrefix[] = "https://safebrowsing.googleapis.com/v4"; +bool UpdateListIdentifier::operator==(const UpdateListIdentifier& other) const { + return platform_type == other.platform_type && + threat_entry_type == other.threat_entry_type && + threat_type == other.threat_type; +} + +bool UpdateListIdentifier::operator!=(const UpdateListIdentifier& other) const { + return !operator==(other); +} + +size_t UpdateListIdentifier::hash() const { + std::size_t first = std::hash()(platform_type); + std::size_t second = std::hash()(threat_entry_type); + std::size_t third = std::hash()(threat_type); + + std::size_t interim = base::HashInts(first, second); + return base::HashInts(interim, third); +} + // static // Backoff interval is MIN(((2^(n-1))*15 minutes) * (RAND + 1), 24 hours) where // n is the number of consecutive errors. diff --git a/components/safe_browsing_db/v4_protocol_manager_util.h b/components/safe_browsing_db/v4_protocol_manager_util.h index f4665f0..6671555 100644 --- a/components/safe_browsing_db/v4_protocol_manager_util.h +++ b/components/safe_browsing_db/v4_protocol_manager_util.h @@ -11,11 +11,12 @@ #include #include "base/gtest_prod_util.h" +#include "base/hash.h" +#include "components/safe_browsing_db/safebrowsing.pb.h" #include "net/url_request/url_request_status.h" #include "url/gurl.h" namespace safe_browsing { - // Config passed to the constructor of a V4 protocol manager. struct V4ProtocolConfig { // The safe browsing client name sent in each request. @@ -28,6 +29,52 @@ struct V4ProtocolConfig { std::string key_param; }; +// The information required to uniquely identify each list the client is +// interested in maintaining and downloading from the SafeBrowsing servers. +// For example, for digests of Malware binaries on Windows: +// platform_type = WINDOWS, +// threat_entry_type = BINARY_DIGEST, +// threat_type = MALWARE +struct UpdateListIdentifier { + PlatformType platform_type; + ThreatEntryType threat_entry_type; + ThreatType threat_type; + + bool operator==(const UpdateListIdentifier& other) const; + bool operator!=(const UpdateListIdentifier& other) const; + size_t hash() const; +}; + +// Enumerate failures for histogramming purposes. DO NOT CHANGE THE +// ORDERING OF THESE VALUES. +enum V4OperationResult { + // 200 response code means that the server recognized the request. + STATUS_200 = 0, + + // Subset of successful responses where the response body wasn't parsable. + PARSE_ERROR = 1, + + // Operation request failed (network error). + NETWORK_ERROR = 2, + + // Operation request returned HTTP result code other than 200. + HTTP_ERROR = 3, + + // Operation attempted during error backoff, no request sent. + BACKOFF_ERROR = 4, + + // Operation attempted before min wait duration elapsed, no request sent. + MIN_WAIT_DURATION_ERROR = 5, + + // Identical operation already pending. + ALREADY_PENDING_ERROR = 6, + + // Memory space for histograms is determined by the max. ALWAYS + // ADD NEW VALUES BEFORE THIS ONE. + OPERATION_RESULT_MAX = 7 +}; + +// A class that provides static methods related to the Pver4 protocol. class V4ProtocolManagerUtil { public: // Record HTTP response code when there's no error in fetching an HTTP @@ -76,4 +123,13 @@ class V4ProtocolManagerUtil { } // namespace safe_browsing +namespace std { +template <> +struct hash { + std::size_t operator()(const safe_browsing::UpdateListIdentifier& s) const { + return s.hash(); + } +}; +} + #endif // COMPONENTS_SAFE_BROWSING_DB_V4_PROTOCOL_MANAGER_UTIL_H_ diff --git a/components/safe_browsing_db/v4_update_protocol_manager.cc b/components/safe_browsing_db/v4_update_protocol_manager.cc new file mode 100644 index 0000000..c5dbfb5 --- /dev/null +++ b/components/safe_browsing_db/v4_update_protocol_manager.cc @@ -0,0 +1,277 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/safe_browsing_db/v4_update_protocol_manager.h" + +#include + +#include "base/base64.h" +#include "base/macros.h" +#include "base/metrics/histogram_macros.h" +#include "base/timer/timer.h" +#include "components/safe_browsing_db/safebrowsing.pb.h" +#include "net/base/load_flags.h" +#include "net/http/http_response_headers.h" +#include "net/http/http_status_code.h" +#include "net/url_request/url_fetcher.h" +#include "net/url_request/url_request_context_getter.h" + +using base::Time; +using base::TimeDelta; + +namespace { + +// Enumerate parsing failures for histogramming purposes. DO NOT CHANGE +// THE ORDERING OF THESE VALUES. +enum ParseResultType { + // Error parsing the protocol buffer from a string. + PARSE_FROM_STRING_ERROR = 0, + + // No platform_type set in the response. + NO_PLATFORM_TYPE_ERROR = 1, + + // No threat_entry_type set in the response. + NO_THREAT_ENTRY_TYPE_ERROR = 2, + + // No threat_type set in the response. + NO_THREAT_TYPE_ERROR = 3, + + // No state set in the response for one or more lists. + NO_STATE_ERROR = 4, + + // Memory space for histograms is determined by the max. ALWAYS + // ADD NEW VALUES BEFORE THIS ONE. + PARSE_RESULT_TYPE_MAX = 5 +}; + +// Record parsing errors of an update result. +void RecordParseUpdateResult(ParseResultType result_type) { + UMA_HISTOGRAM_ENUMERATION("SafeBrowsing.ParseV4UpdateResult", result_type, + PARSE_RESULT_TYPE_MAX); +} + +void RecordUpdateResult(safe_browsing::V4OperationResult result) { + UMA_HISTOGRAM_ENUMERATION( + "SafeBrowsing.V4UpdateResult", result, + safe_browsing::V4OperationResult::OPERATION_RESULT_MAX); +} + +} // namespace + +namespace safe_browsing { + +// The default V4UpdateProtocolManagerFactory. +class V4UpdateProtocolManagerFactoryImpl + : public V4UpdateProtocolManagerFactory { + public: + V4UpdateProtocolManagerFactoryImpl() {} + ~V4UpdateProtocolManagerFactoryImpl() override {} + V4UpdateProtocolManager* CreateProtocolManager( + net::URLRequestContextGetter* request_context_getter, + const V4ProtocolConfig& config) override { + return new V4UpdateProtocolManager(request_context_getter, config); + } + + private: + DISALLOW_COPY_AND_ASSIGN(V4UpdateProtocolManagerFactoryImpl); +}; + +// V4UpdateProtocolManager implementation -------------------------------- + +// static +V4UpdateProtocolManagerFactory* V4UpdateProtocolManager::factory_ = NULL; + +// static +V4UpdateProtocolManager* V4UpdateProtocolManager::Create( + net::URLRequestContextGetter* request_context_getter, + const V4ProtocolConfig& config) { + if (!factory_) + factory_ = new V4UpdateProtocolManagerFactoryImpl(); + return factory_->CreateProtocolManager(request_context_getter, config); +} + +void V4UpdateProtocolManager::ResetUpdateErrors() { + update_error_count_ = 0; + update_back_off_mult_ = 1; +} + +V4UpdateProtocolManager::V4UpdateProtocolManager( + net::URLRequestContextGetter* request_context_getter, + const V4ProtocolConfig& config) + : update_error_count_(0), + update_back_off_mult_(1), + next_update_time_(Time::Now()), + config_(config), + request_context_getter_(request_context_getter), + url_fetcher_id_(0) {} + +V4UpdateProtocolManager::~V4UpdateProtocolManager() { +} + +std::string V4UpdateProtocolManager::GetUpdateRequest( + const base::hash_set& lists_to_update, + const base::hash_map& + current_list_states) { + // Build the request. Client info and client states are not added to the + // request protocol buffer. Client info is passed as params in the url. + FetchThreatListUpdatesRequest request; + for (const auto& list_to_update : lists_to_update) { + ListUpdateRequest* list_update_request = request.add_list_update_requests(); + list_update_request->set_platform_type(list_to_update.platform_type); + list_update_request->set_threat_entry_type( + list_to_update.threat_entry_type); + list_update_request->set_threat_type(list_to_update.threat_type); + + // If the current state of the list is available, add that to the proto. + base::hash_map::const_iterator + list_iter = current_list_states.find(list_to_update); + if (list_iter != current_list_states.end()) { + list_update_request->set_state(list_iter->second); + } + } + + // Serialize and Base64 encode. + std::string req_data, req_base64; + request.SerializeToString(&req_data); + base::Base64Encode(req_data, &req_base64); + + return req_base64; +} + +bool V4UpdateProtocolManager::ParseUpdateResponse( + const std::string& data, + std::vector* list_update_responses) { + FetchThreatListUpdatesResponse response; + + if (!response.ParseFromString(data)) { + RecordParseUpdateResult(PARSE_FROM_STRING_ERROR); + return false; + } + + if (response.has_minimum_wait_duration()) { + // Seconds resolution is good enough so we ignore the nanos field. + base::TimeDelta next_update_interval = base::TimeDelta::FromSeconds( + response.minimum_wait_duration().seconds()); + next_update_time_ = Time::Now() + next_update_interval; + } + + // TODO(vakh): Do something useful with this response. + for (const ListUpdateResponse& list_update_response : + response.list_update_responses()) { + if (!list_update_response.has_platform_type()) { + RecordParseUpdateResult(NO_PLATFORM_TYPE_ERROR); + } else if (!list_update_response.has_threat_entry_type()) { + RecordParseUpdateResult(NO_THREAT_ENTRY_TYPE_ERROR); + } else if (!list_update_response.has_threat_type()) { + RecordParseUpdateResult(NO_THREAT_TYPE_ERROR); + } else if (!list_update_response.has_new_client_state()) { + RecordParseUpdateResult(NO_STATE_ERROR); + } else { + list_update_responses->push_back(list_update_response); + } + } + return true; +} + +void V4UpdateProtocolManager::GetUpdates( + const base::hash_set& lists_to_update, + const base::hash_map& + current_list_states, + UpdateCallback callback) { + DCHECK(CalledOnValidThread()); + + // If an update request is already pending, return an empty result. + if (request_) { + RecordUpdateResult(V4OperationResult::ALREADY_PENDING_ERROR); + std::vector list_update_responses; + callback.Run(list_update_responses); + return; + } + + // We need to wait the minimum waiting duration, and if we are in backoff, + // we need to check if we're past the next allowed time. If we are, we can + // proceed with the request. If not, we are required to return empty results. + if (Time::Now() <= next_update_time_) { + if (update_error_count_) { + RecordUpdateResult(V4OperationResult::BACKOFF_ERROR); + } else { + RecordUpdateResult(V4OperationResult::MIN_WAIT_DURATION_ERROR); + } + std::vector list_update_responses; + callback.Run(list_update_responses); + return; + } + + std::string req_base64 = + GetUpdateRequest(lists_to_update, current_list_states); + GURL update_url = GetUpdateUrl(req_base64); + + request_.reset(net::URLFetcher::Create(url_fetcher_id_++, update_url, + net::URLFetcher::GET, this) + .release()); + callback_ = callback; + + request_->SetLoadFlags(net::LOAD_DISABLE_CACHE); + request_->SetRequestContext(request_context_getter_.get()); + request_->Start(); + //TODO(vakh): Handle request timeout. +} + +// net::URLFetcherDelegate implementation ---------------------------------- + +// SafeBrowsing request responses are handled here. +void V4UpdateProtocolManager::OnURLFetchComplete( + const net::URLFetcher* source) { + DCHECK(CalledOnValidThread()); + + int response_code = source->GetResponseCode(); + net::URLRequestStatus status = source->GetStatus(); + V4ProtocolManagerUtil::RecordHttpResponseOrErrorCode( + "SafeBrowsing.V4UpdateHttpResponseOrErrorCode", status, response_code); + + std::vector list_update_responses; + if (status.is_success() && response_code == net::HTTP_OK) { + RecordUpdateResult(V4OperationResult::STATUS_200); + ResetUpdateErrors(); + std::string data; + source->GetResponseAsString(&data); + if (!ParseUpdateResponse(data, &list_update_responses)) { + list_update_responses.clear(); + RecordUpdateResult(V4OperationResult::PARSE_ERROR); + } + } else { + HandleUpdateError(Time::Now()); + + DVLOG(1) << "SafeBrowsing GetEncodedUpdates request for: " + << source->GetURL() << " failed with error: " << status.error() + << " and response code: " << response_code; + + if (status.status() == net::URLRequestStatus::FAILED) { + RecordUpdateResult(V4OperationResult::NETWORK_ERROR); + } else { + RecordUpdateResult(V4OperationResult::HTTP_ERROR); + } + } + + // Invoke the callback with list_update_responses, even if there was a parse + // error or an error response code (in which case list_update_responses will + // be empty). The caller can't be blocked indefinitely. + callback_.Run(list_update_responses); + request_.reset(); +} + +void V4UpdateProtocolManager::HandleUpdateError(const Time& now) { + DCHECK(CalledOnValidThread()); + base::TimeDelta next = V4ProtocolManagerUtil::GetNextBackOffInterval( + &update_error_count_, &update_back_off_mult_); + next_update_time_ = now + next; +} + +GURL V4UpdateProtocolManager::GetUpdateUrl( + const std::string& req_base64) const { + return V4ProtocolManagerUtil::GetRequestUrl(req_base64, "encodedUpdates", + config_); +} + +} // namespace safe_browsing diff --git a/components/safe_browsing_db/v4_update_protocol_manager.h b/components/safe_browsing_db/v4_update_protocol_manager.h new file mode 100644 index 0000000..19e3234 --- /dev/null +++ b/components/safe_browsing_db/v4_update_protocol_manager.h @@ -0,0 +1,178 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_SAFE_BROWSING_DB_V4_UPDATE_PROTOCOL_MANAGER_H_ +#define COMPONENTS_SAFE_BROWSING_DB_V4_UPDATE_PROTOCOL_MANAGER_H_ + +// A class that implements Chrome's interface with the SafeBrowsing V4 update +// protocol. +// +// The V4UpdateProtocolManager handles formatting and making requests of, and +// handling responses from, Google's SafeBrowsing servers. The purpose of this +// class is to get hash prefixes from the SB server for the given set of lists. + +#include +#include + +#include "base/gtest_prod_util.h" +#include "base/macros.h" +#include "base/memory/scoped_ptr.h" +#include "base/threading/non_thread_safe.h" +#include "base/time/time.h" +#include "base/timer/timer.h" +#include "components/safe_browsing_db/safebrowsing.pb.h" +#include "components/safe_browsing_db/util.h" +#include "components/safe_browsing_db/v4_protocol_manager_util.h" +#include "net/url_request/url_fetcher_delegate.h" +#include "url/gurl.h" + +namespace net { +class URLFetcher; +class URLRequestContextGetter; +} // namespace net + +namespace safe_browsing { + +class V4UpdateProtocolManagerFactory; + +class V4UpdateProtocolManager : public net::URLFetcherDelegate, + public base::NonThreadSafe { + public: + typedef FetchThreatListUpdatesRequest::ListUpdateRequest ListUpdateRequest; + typedef FetchThreatListUpdatesResponse::ListUpdateResponse ListUpdateResponse; + + // UpdateCallback is invoked when GetUpdates completes. + // Parameters: + // - The vector of update response protobufs received from the server for + // each list type. + // The caller can then use this vector to re-build the current_list_states. + typedef base::Callback&)> + UpdateCallback; + + ~V4UpdateProtocolManager() override; + + // Makes the passed |factory| the factory used to instantiate + // a V4UpdateProtocolManager. Useful for tests. + static void RegisterFactory(V4UpdateProtocolManagerFactory* factory) { + factory_ = factory; + } + + // Create an instance of the safe browsing v4 protocol manager. + static V4UpdateProtocolManager* Create( + net::URLRequestContextGetter* request_context_getter, + const V4ProtocolConfig& config); + + // net::URLFetcherDelegate interface. + void OnURLFetchComplete(const net::URLFetcher* source) override; + + // Retrieve the hash prefix update, and invoke the callback argument when the + // results are retrieved. The callback may be invoked synchronously. + // Parameters: + // - The set of lists to fetch the updates for. + // - The last known state for each of the known lists. + // It is valid to have one or more lists in lists_to_update set that have no + // corresponding value in the current_list_states map. This corresponds to the + // initial state for those lists. + virtual void GetUpdates( + const base::hash_set& lists_to_update, + const base::hash_map& + current_list_states, + UpdateCallback callback); + + protected: + // Constructs a V4UpdateProtocolManager that issues network requests using + // |request_context_getter|. + V4UpdateProtocolManager(net::URLRequestContextGetter* request_context_getter, + const V4ProtocolConfig& config); + + private: + FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest, + TestGetUpdatesErrorHandlingNetwork); + FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest, + TestGetUpdatesErrorHandlingResponseCode); + FRIEND_TEST_ALL_PREFIXES(V4UpdateProtocolManagerTest, TestGetUpdatesNoError); + friend class V4UpdateProtocolManagerFactoryImpl; + + // The method to generate the URL for the request to be sent to the server. + // |request_base64| is the base64 encoded form of an instance of the protobuf + // FetchThreatListUpdatesRequest. + GURL GetUpdateUrl(const std::string& request_base64) const; + + // Fills a FetchThreatListUpdatesRequest protocol buffer for a request. + // Returns the serialized and base 64 encoded request as a string. + std::string GetUpdateRequest( + const base::hash_set& lists_to_update, + const base::hash_map& + current_list_states); + + // Parses the base64 encoded response received from the server as a + // FetchThreatListUpdatesResponse protobuf and returns each of the + // ListUpdateResponse protobufs contained in it as a vector. + // Returns true if parsing is successful, false otherwise. + bool ParseUpdateResponse( + const std::string& data_base64, + std::vector* list_update_responses); + + // Resets the update error counter and multiplier. + void ResetUpdateErrors(); + + // Updates internal update and backoff state for each update response error, + // assuming that the current time is |now|. + void HandleUpdateError(const base::Time& now); + + // The factory that controls the creation of V4UpdateProtocolManager. + // This is used by tests. + static V4UpdateProtocolManagerFactory* factory_; + + // The number of HTTP response errors since the the last successful HTTP + // response, used for request backoff timing. + size_t update_error_count_; + + // Multiplier for the backoff error after the second. + size_t update_back_off_mult_; + + // The time before which the next update request may not be sent. + // It is set to: + // the backoff time, if the last response was an error, or + // the minimum wait time, if the last response was successful. + base::Time next_update_time_; + + // The config of the client making Pver4 requests. + const V4ProtocolConfig config_; + + // The context we use to issue network requests. + scoped_refptr request_context_getter_; + + // ID for URLFetchers for testing. + int url_fetcher_id_; + + // True if there's a request pending. + bool update_request_pending_; + + // The callback that's called when GetUpdates completes. + UpdateCallback callback_; + + // The pending update request. The request must be canceled when the object is + // destroyed. + scoped_ptr request_; + + DISALLOW_COPY_AND_ASSIGN(V4UpdateProtocolManager); +}; + +// Interface of a factory to create V4UpdateProtocolManager. Useful for tests. +class V4UpdateProtocolManagerFactory { + public: + V4UpdateProtocolManagerFactory() {} + virtual ~V4UpdateProtocolManagerFactory() {} + virtual V4UpdateProtocolManager* CreateProtocolManager( + net::URLRequestContextGetter* request_context_getter, + const V4ProtocolConfig& config) = 0; + + private: + DISALLOW_COPY_AND_ASSIGN(V4UpdateProtocolManagerFactory); +}; + +} // namespace safe_browsing + +#endif // COMPONENTS_SAFE_BROWSING_DB_V4_UPDATE_PROTOCOL_MANAGER_H_ diff --git a/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc b/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc new file mode 100644 index 0000000..2a9a239 --- /dev/null +++ b/components/safe_browsing_db/v4_update_protocol_manager_unittest.cc @@ -0,0 +1,199 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include + +#include "base/base64.h" +#include "base/memory/scoped_ptr.h" +#include "base/strings/stringprintf.h" +#include "base/time/time.h" +#include "components/safe_browsing_db/safebrowsing.pb.h" +#include "components/safe_browsing_db/util.h" +#include "components/safe_browsing_db/v4_update_protocol_manager.h" +#include "net/base/escape.h" +#include "net/base/load_flags.h" +#include "net/base/net_errors.h" +#include "net/url_request/test_url_fetcher_factory.h" +#include "testing/gtest/include/gtest/gtest.h" + +using base::Time; +using base::TimeDelta; + +namespace { + +const char kClient[] = "unittest"; +const char kAppVer[] = "1.0"; +const char kKeyParam[] = "test_key_param"; + +} // namespace + +namespace safe_browsing { + +typedef V4UpdateProtocolManager::ListUpdateRequest ListUpdateRequest; +typedef V4UpdateProtocolManager::ListUpdateResponse ListUpdateResponse; + +class V4UpdateProtocolManagerTest : public testing::Test { + protected: + scoped_ptr CreateProtocolManager() { + V4ProtocolConfig config; + config.client_name = kClient; + config.version = kAppVer; + config.key_param = kKeyParam; + return scoped_ptr( + V4UpdateProtocolManager::Create(NULL, config)); + } + + void SetupListsToUpdate( + base::hash_set* lists_to_update) { + UpdateListIdentifier list_identifier; + list_identifier.platform_type = WINDOWS_PLATFORM; + list_identifier.threat_entry_type = URL_EXPRESSION; + list_identifier.threat_type = MALWARE_THREAT; + lists_to_update->insert(list_identifier); + + list_identifier.platform_type = WINDOWS_PLATFORM; + list_identifier.threat_entry_type = URL_EXPRESSION; + list_identifier.threat_type = UNWANTED_SOFTWARE; + lists_to_update->insert(list_identifier); + + list_identifier.platform_type = WINDOWS_PLATFORM; + list_identifier.threat_entry_type = BINARY_DIGEST; + list_identifier.threat_type = MALWARE_THREAT; + lists_to_update->insert(list_identifier); + } + + void ClearListsToUpdate( + base::hash_set* lists_to_update) { + lists_to_update->clear(); + } + + void SetupCurrentListStates( + const base::hash_set& lists_to_update, + base::hash_map* current_list_states) { + // TODO(vakh): Implement this to test the cases when we have an existing + // state for some of the lists. + } + + std::string GetStockV4UpdateResponse() { + FetchThreatListUpdatesResponse response; + + ListUpdateResponse* lur = response.add_list_update_responses(); + lur->set_platform_type(WINDOWS_PLATFORM); + lur->set_response_type(ListUpdateResponse::PARTIAL_UPDATE); + lur->set_threat_entry_type(URL_EXPRESSION); + lur->set_threat_type(MALWARE_THREAT); + + lur = response.add_list_update_responses(); + lur->set_platform_type(WINDOWS_PLATFORM); + lur->set_response_type(ListUpdateResponse::PARTIAL_UPDATE); + lur->set_threat_entry_type(URL_EXPRESSION); + lur->set_threat_type(UNWANTED_SOFTWARE); + + lur = response.add_list_update_responses(); + lur->set_platform_type(WINDOWS_PLATFORM); + lur->set_response_type(ListUpdateResponse::FULL_UPDATE); + lur->set_threat_entry_type(BINARY_DIGEST); + lur->set_threat_type(MALWARE_THREAT); + + // Serialize. + std::string res_data; + response.SerializeToString(&res_data); + + return res_data; + } +}; + +void ValidateGetUpdatesResults( + const std::vector& expected_lurs, + const std::vector& list_update_responses) { + ASSERT_EQ(expected_lurs.size(), list_update_responses.size()); + + for (unsigned int i = 0; i < list_update_responses.size(); ++i) { + const ListUpdateResponse& expected = expected_lurs[i]; + const ListUpdateResponse& actual = list_update_responses[i]; + + EXPECT_EQ(expected.platform_type(), actual.platform_type()); + EXPECT_EQ(expected.response_type(), actual.response_type()); + EXPECT_EQ(expected.threat_entry_type(), actual.threat_entry_type()); + EXPECT_EQ(expected.threat_type(), actual.threat_type()); + + // TODO(vakh): Test more fields from the proto. + } +} + +// TODO(vakh): Add many more tests. + +TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesErrorHandlingNetwork) { + net::TestURLFetcherFactory factory; + scoped_ptr pm(CreateProtocolManager()); + + const std::vector expected_lurs; + const base::hash_set lists_to_update; + const base::hash_map current_list_states; + pm->GetUpdates(lists_to_update, current_list_states, + base::Bind(&ValidateGetUpdatesResults, expected_lurs)); + + net::TestURLFetcher* fetcher = factory.GetFetcherByID(0); + DCHECK(fetcher); + // Failed request status should result in error. + fetcher->set_status(net::URLRequestStatus(net::URLRequestStatus::FAILED, + net::ERR_CONNECTION_RESET)); + fetcher->delegate()->OnURLFetchComplete(fetcher); + + // Should have recorded one error, but back off multiplier is unchanged. + EXPECT_EQ(1ul, pm->update_error_count_); + EXPECT_EQ(1ul, pm->update_back_off_mult_); +} + +TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesErrorHandlingResponseCode) { + net::TestURLFetcherFactory factory; + scoped_ptr pm(CreateProtocolManager()); + + const std::vector expected_lurs; + const base::hash_set lists_to_update; + const base::hash_map current_list_states; + pm->GetUpdates(lists_to_update, current_list_states, + base::Bind(&ValidateGetUpdatesResults, expected_lurs)); + + + net::TestURLFetcher* fetcher = factory.GetFetcherByID(0); + DCHECK(fetcher); + fetcher->set_status(net::URLRequestStatus()); + // Response code of anything other than 200 should result in error. + fetcher->set_response_code(204); + fetcher->SetResponseString(GetStockV4UpdateResponse()); + fetcher->delegate()->OnURLFetchComplete(fetcher); + + // Should have recorded one error, but back off multiplier is unchanged. + EXPECT_EQ(1ul, pm->update_error_count_); + EXPECT_EQ(1ul, pm->update_back_off_mult_); +} + +TEST_F(V4UpdateProtocolManagerTest, TestGetUpdatesNoError) { + net::TestURLFetcherFactory factory; + scoped_ptr pm(CreateProtocolManager()); + + + const std::vector expected_lurs; + base::hash_set lists_to_update; + SetupListsToUpdate(&lists_to_update); + base::hash_map current_list_states; + SetupCurrentListStates(lists_to_update, ¤t_list_states); + pm->GetUpdates(lists_to_update, current_list_states, + base::Bind(&ValidateGetUpdatesResults, expected_lurs)); + ClearListsToUpdate(&lists_to_update); + + net::TestURLFetcher* fetcher = factory.GetFetcherByID(0); + DCHECK(fetcher); + fetcher->set_status(net::URLRequestStatus()); + fetcher->set_response_code(200); + fetcher->SetResponseString(GetStockV4UpdateResponse()); + fetcher->delegate()->OnURLFetchComplete(fetcher); + + // No error, back off multiplier is unchanged. + EXPECT_EQ(0ul, pm->update_error_count_); + EXPECT_EQ(1ul, pm->update_back_off_mult_); +} + +} // namespace safe_browsing diff --git a/tools/metrics/histograms/histograms.xml b/tools/metrics/histograms/histograms.xml index 4404169e..360ede5 100644 --- a/tools/metrics/histograms/histograms.xml +++ b/tools/metrics/histograms/histograms.xml @@ -42578,7 +42578,7 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries. + enum="SafeBrowsingV4OperationResult"> kcarattini@chromium.org Track return status from V4 GetHash attempts. The buckets of this histogram @@ -42603,6 +42603,14 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries. + + vakh@chromium.org + + Track the parsing results of a status 200 GetV4Update request. + + + nparker@chromium.org @@ -42668,6 +42676,15 @@ http://cs/file:chrome/histograms.xml - but prefer this file for new entries. + + vakh@chromium.org + + Track return status from V4 update attempts. The buckets of this histogram + overlap, so the counts cannot be used as percentages. + + + Has not been generated for years (7/8/14). @@ -79861,15 +79878,6 @@ To add a new entry, add it with any value and run test to compute valid value. - - - - - - - - - @@ -79879,6 +79887,24 @@ To add a new entry, add it with any value and run test to compute valid value. + + + + + + + + + + + + + + + + + + -- cgit v1.1