diff options
author | xunjieli <xunjieli@chromium.org> | 2015-06-16 10:15:43 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2015-06-16 17:16:17 +0000 |
commit | 413a687892c42560986bf952590155bc65afc978 (patch) | |
tree | 82451120f2ab9998479210c13226d2ad5d0e85b0 | |
parent | 6d491c39b882fa2b3e27ddc785847fd21884fdce (diff) | |
download | chromium_src-413a687892c42560986bf952590155bc65afc978.zip chromium_src-413a687892c42560986bf952590155bc65afc978.tar.gz chromium_src-413a687892c42560986bf952590155bc65afc978.tar.bz2 |
Make a ResourceThrottle for extensions on top of net::URLRequestThrottlerManager.
net::URLRequestThrottlerManager is used for throttling extensions originated
requests. We would like to move it out of net/ and place it in a more appropriate
directory. Since content::ResourceThrottle is the standard way to throttle
requests, this CL makes URLRequestThrottlerManager into a extensions
ResourceThrottle and place it in extensions/browser/. Followup CLs will clean up
URLRequestThrottlerManager usage from net/.
BUG=484241
Committed: https://crrev.com/6a47d0e298a4c3050d95505f5ee18b122fdc213b
Cr-Commit-Position: refs/heads/master@{#334457}
Review URL: https://codereview.chromium.org/1171983003
Cr-Commit-Position: refs/heads/master@{#334624}
28 files changed, 2670 insertions, 12 deletions
diff --git a/chrome/browser/extensions/extension_request_limiting_throttle_browsertest.cc b/chrome/browser/extensions/extension_request_limiting_throttle_browsertest.cc new file mode 100644 index 0000000..4cc8efc --- /dev/null +++ b/chrome/browser/extensions/extension_request_limiting_throttle_browsertest.cc @@ -0,0 +1,210 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/bind.h" +#include "base/memory/scoped_ptr.h" +#include "base/strings/string_util.h" +#include "base/strings/stringprintf.h" +#include "chrome/browser/extensions/extension_browsertest.h" +#include "chrome/browser/profiles/profile_io_data.h" +#include "chrome/common/chrome_switches.h" +#include "chrome/test/base/ui_test_utils.h" +#include "extensions/browser/extension_throttle_manager.h" +#include "extensions/test/result_catcher.h" +#include "net/base/url_util.h" +#include "net/dns/mock_host_resolver.h" +#include "net/test/embedded_test_server/embedded_test_server.h" +#include "net/test/embedded_test_server/http_request.h" +#include "net/test/embedded_test_server/http_response.h" + +namespace extensions { + +namespace { + +scoped_ptr<net::test_server::HttpResponse> HandleRequest( + bool set_cache_header_redirect_page, + bool set_cache_header_test_throttle_page, + const net::test_server::HttpRequest& request) { + if (base::StartsWithASCII(request.relative_url, "/redirect", true)) { + scoped_ptr<net::test_server::BasicHttpResponse> http_response( + new net::test_server::BasicHttpResponse()); + http_response->set_code(net::HTTP_FOUND); + http_response->set_content("Redirecting..."); + http_response->set_content_type("text/plain"); + http_response->AddCustomHeader("Location", "/test_throttle"); + if (set_cache_header_redirect_page) + http_response->AddCustomHeader("Cache-Control", "max-age=3600"); + return http_response.Pass(); + } + + if (base::StartsWithASCII(request.relative_url, "/test_throttle", true)) { + scoped_ptr<net::test_server::BasicHttpResponse> http_response( + new net::test_server::BasicHttpResponse()); + http_response->set_code(net::HTTP_SERVICE_UNAVAILABLE); + http_response->set_content("The server is overloaded right now."); + http_response->set_content_type("text/plain"); + if (set_cache_header_test_throttle_page) + http_response->AddCustomHeader("Cache-Control", "max-age=3600"); + return http_response.Pass(); + } + + // Unhandled requests result in the Embedded test server sending a 404. + return scoped_ptr<net::test_server::BasicHttpResponse>(); +} + +} // namespace + +class ExtensionRequestLimitingThrottleBrowserTest + : public ExtensionBrowserTest { + public: + void SetUpOnMainThread() override { + ExtensionBrowserTest::SetUpOnMainThread(); + ProfileIOData* io_data = + ProfileIOData::FromResourceContext(profile()->GetResourceContext()); + ExtensionThrottleManager* manager = io_data->GetExtensionThrottleManager(); + if (manager) { + // Requests issued within within |kUserGestureWindowMs| of a user gesture + // are also considered as user gestures (see + // resource_dispatcher_host_impl.cc), so these tests need to bypass the + // checking of the net::LOAD_MAYBE_USER_GESTURE load flag in the manager + // in order to test the throttling logic. + manager->SetIgnoreUserGestureLoadFlagForTests(true); + } + // Requests to 127.0.0.1 bypass throttling, so set up a host resolver rule + // to use a fake domain. + host_resolver()->AddRule("www.example.com", "127.0.0.1"); + ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady()); + extension_ = + LoadExtension(test_data_dir_.AppendASCII("extension_throttle")); + ASSERT_TRUE(extension_); + } + + void RunTest(const std::string& file_path, const std::string& request_url) { + ResultCatcher catcher; + GURL test_url = net::AppendQueryParameter( + extension_->GetResourceURL(file_path), "url", request_url); + ui_test_utils::NavigateToURL(browser(), test_url); + ASSERT_TRUE(catcher.GetNextResult()); + } + + private: + const Extension* extension_; +}; + +class ExtensionRequestLimitingThrottleCommandLineBrowserTest + : public ExtensionRequestLimitingThrottleBrowserTest { + public: + void SetUpCommandLine(base::CommandLine* command_line) override { + ExtensionRequestLimitingThrottleBrowserTest::SetUpCommandLine(command_line); + command_line->AppendSwitch(switches::kDisableExtensionsHttpThrottling); + } +}; + +// Tests that if the same URL is requested repeatedly by an extension, it will +// eventually be throttled. +IN_PROC_BROWSER_TEST_F(ExtensionRequestLimitingThrottleBrowserTest, + ThrottleRequest) { + embedded_test_server()->RegisterRequestHandler( + base::Bind(&HandleRequest, false, false)); + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_eventually_throttled.html", + base::StringPrintf("http://www.example.com:%d/test_throttle", + embedded_test_server()->port()))); +} + +// Tests that if the same URL is repeatedly requested by an extension, and the +// response is served from the cache, it will not be throttled. +IN_PROC_BROWSER_TEST_F(ExtensionRequestLimitingThrottleBrowserTest, + DoNotThrottleCachedResponse) { + embedded_test_server()->RegisterRequestHandler( + base::Bind(&HandleRequest, false, true)); + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_not_throttled.html", + base::StringPrintf("http://www.example.com:%d/test_throttle", + embedded_test_server()->port()))); +} + +// Tests that the redirected request is also being throttled. +IN_PROC_BROWSER_TEST_F(ExtensionRequestLimitingThrottleBrowserTest, + ThrottleRequest_Redirect) { + embedded_test_server()->RegisterRequestHandler( + base::Bind(&HandleRequest, false, false)); + // Issue a bunch of requests to a url which gets redirected to a new url that + // generates 503. + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_eventually_throttled.html", + base::StringPrintf("http://www.example.com:%d/redirect", + embedded_test_server()->port()))); + + // Now requests to both URLs should be throttled. Explicitly validate that the + // second URL is throttled. + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_throttled_on_first_try.html", + base::StringPrintf("http://www.example.com:%d/test_throttle", + embedded_test_server()->port()))); +} + +// Tests that if both redirect (302) and non-redirect (503) responses are +// served from cache, the extension throttle does not throttle the request. +IN_PROC_BROWSER_TEST_F(ExtensionRequestLimitingThrottleBrowserTest, + DoNotThrottleCachedResponse_Redirect) { + embedded_test_server()->RegisterRequestHandler( + base::Bind(&HandleRequest, true, true)); + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_not_throttled.html", + base::StringPrintf("http://www.example.com:%d/redirect", + embedded_test_server()->port()))); +} + +// Tests that if the redirect (302) is served from cache, but the non-redirect +// (503) is not, the extension throttle throttles the requests for the second +// url. +IN_PROC_BROWSER_TEST_F(ExtensionRequestLimitingThrottleBrowserTest, + ThrottleRequest_RedirectCached) { + embedded_test_server()->RegisterRequestHandler( + base::Bind(&HandleRequest, true, false)); + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_eventually_throttled.html", + base::StringPrintf("http://www.example.com:%d/redirect", + embedded_test_server()->port()))); + + // Explicitly validate that the second URL is throttled. + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_throttled_on_first_try.html", + base::StringPrintf("http://www.example.com:%d/test_throttle", + embedded_test_server()->port()))); +} + +// Tests that if the redirect (302) is not served from cache, but the +// non-redirect (503) is, the extension throttle only throttles requests to the +// redirect URL. +IN_PROC_BROWSER_TEST_F(ExtensionRequestLimitingThrottleBrowserTest, + DoNotThrottleCachedResponse_NonRedirectCached) { + embedded_test_server()->RegisterRequestHandler( + base::Bind(&HandleRequest, false, true)); + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_not_throttled.html", + base::StringPrintf("http://www.example.com:%d/redirect", + embedded_test_server()->port()))); + + // Explicitly validate that the second URL is not throttled. + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_not_throttled.html", + base::StringPrintf("http://www.example.com:%d/test_throttle", + embedded_test_server()->port()))); +} + +// Tests that if switches::kDisableExtensionsHttpThrottling is set on the +// command line, throttling is disabled. +IN_PROC_BROWSER_TEST_F(ExtensionRequestLimitingThrottleCommandLineBrowserTest, + ThrottleRequestDisabled) { + embedded_test_server()->RegisterRequestHandler( + base::Bind(&HandleRequest, false, false)); + ASSERT_NO_FATAL_FAILURE( + RunTest("test_request_not_throttled.html", + base::StringPrintf("http://www.example.com:%d/test_throttle", + embedded_test_server()->port()))); +} + +} // namespace extensions diff --git a/chrome/browser/net/chrome_network_delegate.cc b/chrome/browser/net/chrome_network_delegate.cc index 737317e..6cba8ba 100644 --- a/chrome/browser/net/chrome_network_delegate.cc +++ b/chrome/browser/net/chrome_network_delegate.cc @@ -678,14 +678,7 @@ bool ChromeNetworkDelegate::OnCanAccessFile(const net::URLRequest& request, bool ChromeNetworkDelegate::OnCanThrottleRequest( const net::URLRequest& request) const { -#if defined(ENABLE_EXTENSIONS) - if (g_never_throttle_requests_) - return false; - return request.first_party_for_cookies().scheme() == - extensions::kExtensionScheme; -#else return false; -#endif } bool ChromeNetworkDelegate::OnCanEnablePrivacyMode( diff --git a/chrome/browser/net/chrome_network_delegate_unittest.cc b/chrome/browser/net/chrome_network_delegate_unittest.cc index bd9fcb55..4d4b070 100644 --- a/chrome/browser/net/chrome_network_delegate_unittest.cc +++ b/chrome/browser/net/chrome_network_delegate_unittest.cc @@ -91,7 +91,7 @@ class ChromeNetworkDelegateThrottlingTest : public testing::Test { web_page_request->set_first_party_for_cookies( GURL("http://example.com/helloworld.html")); - ASSERT_TRUE(delegate->OnCanThrottleRequest(*extension_request)); + ASSERT_FALSE(delegate->OnCanThrottleRequest(*extension_request)); ASSERT_FALSE(delegate->OnCanThrottleRequest(*web_page_request)); delegate->NeverThrottleRequests(); diff --git a/chrome/browser/profiles/profile_io_data.cc b/chrome/browser/profiles/profile_io_data.cc index 5e3185e..a3da86c 100644 --- a/chrome/browser/profiles/profile_io_data.cc +++ b/chrome/browser/profiles/profile_io_data.cc @@ -95,6 +95,7 @@ #include "chrome/browser/extensions/extension_resource_protocols.h" #include "extensions/browser/extension_protocols.h" #include "extensions/browser/extension_system.h" +#include "extensions/browser/extension_throttle_manager.h" #include "extensions/browser/info_map.h" #include "extensions/common/constants.h" #endif @@ -800,7 +801,17 @@ extensions::InfoMap* ProfileIOData::GetExtensionInfoMap() const { #if defined(ENABLE_EXTENSIONS) return extension_info_map_.get(); #else - return NULL; + return nullptr; +#endif +} + +extensions::ExtensionThrottleManager* +ProfileIOData::GetExtensionThrottleManager() const { + DCHECK(initialized_) << "ExtensionSystem not initialized"; +#if defined(ENABLE_EXTENSIONS) + return extension_throttle_manager_.get(); +#else + return nullptr; #endif } @@ -1007,7 +1018,12 @@ void ProfileIOData::Init( #if defined(ENABLE_EXTENSIONS) network_delegate->set_extension_info_map( profile_params_->extension_info_map.get()); + if (!command_line.HasSwitch(switches::kDisableExtensionsHttpThrottling)) { + extension_throttle_manager_.reset( + new extensions::ExtensionThrottleManager()); + } #endif + #if defined(ENABLE_CONFIGURATION_POLICY) network_delegate->set_url_blacklist_manager(url_blacklist_manager_.get()); #endif diff --git a/chrome/browser/profiles/profile_io_data.h b/chrome/browser/profiles/profile_io_data.h index 95b8a92..f63dd0f 100644 --- a/chrome/browser/profiles/profile_io_data.h +++ b/chrome/browser/profiles/profile_io_data.h @@ -50,6 +50,7 @@ class DataReductionProxyIOData; } namespace extensions { +class ExtensionThrottleManager; class InfoMap; } @@ -132,6 +133,7 @@ class ProfileIOData { // with a content::ResourceContext, and they want access to Chrome data for // that profile. extensions::InfoMap* GetExtensionInfoMap() const; + extensions::ExtensionThrottleManager* GetExtensionThrottleManager() const; CookieSettings* GetCookieSettings() const; HostContentSettingsMap* GetHostContentSettingsMap() const; @@ -577,6 +579,12 @@ class ProfileIOData { supervised_user_url_filter_; #endif +#if defined(ENABLE_EXTENSIONS) + // Is NULL if switches::kDisableExtensionsHttpThrottling is on. + mutable scoped_ptr<extensions::ExtensionThrottleManager> + extension_throttle_manager_; +#endif + mutable scoped_ptr<DevToolsNetworkController> network_controller_; // TODO(jhawkins): Remove once crbug.com/102004 is fixed. diff --git a/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc b/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc index 5375e1e..094ad36 100644 --- a/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc +++ b/chrome/browser/renderer_host/chrome_resource_dispatcher_host_delegate.cc @@ -68,6 +68,7 @@ #include "chrome/browser/apps/ephemeral_app_throttle.h" #include "chrome/browser/extensions/api/streams_private/streams_private_api.h" #include "chrome/browser/extensions/user_script_listener.h" +#include "extensions/browser/extension_throttle_manager.h" #include "extensions/browser/guest_view/web_view/web_view_renderer_state.h" #include "extensions/browser/info_map.h" #include "extensions/common/constants.h" @@ -586,11 +587,20 @@ void ChromeResourceDispatcherHostDelegate::AppendStandardResourceThrottles( #endif #if defined(ENABLE_EXTENSIONS) - content::ResourceThrottle* throttle = + content::ResourceThrottle* wait_for_extensions_init_throttle = user_script_listener_->CreateResourceThrottle(request->url(), resource_type); - if (throttle) - throttles->push_back(throttle); + if (wait_for_extensions_init_throttle) + throttles->push_back(wait_for_extensions_init_throttle); + + extensions::ExtensionThrottleManager* extension_throttle_manager = + io_data->GetExtensionThrottleManager(); + if (extension_throttle_manager) { + scoped_ptr<content::ResourceThrottle> extension_throttle = + extension_throttle_manager->MaybeCreateThrottle(request); + if (extension_throttle) + throttles->push_back(extension_throttle.release()); + } #endif const ResourceRequestInfo* info = ResourceRequestInfo::ForRequest(request); diff --git a/chrome/chrome_tests.gypi b/chrome/chrome_tests.gypi index e8a43e0..8588643 100644 --- a/chrome/chrome_tests.gypi +++ b/chrome/chrome_tests.gypi @@ -250,6 +250,7 @@ 'browser/extensions/extension_management_test_util.h', 'browser/extensions/extension_messages_apitest.cc', 'browser/extensions/extension_override_apitest.cc', + 'browser/extensions/extension_request_limiting_throttle_browsertest.cc', 'browser/extensions/extension_resource_request_policy_apitest.cc', 'browser/extensions/extension_startup_browsertest.cc', 'browser/extensions/extension_storage_apitest.cc', diff --git a/chrome/test/data/extensions/extension_throttle/background.js b/chrome/test/data/extensions/extension_throttle/background.js new file mode 100644 index 0000000..52c8e34 --- /dev/null +++ b/chrome/test/data/extensions/extension_throttle/background.js @@ -0,0 +1,13 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { + if (message.type == "xhr") { + var xhr = new XMLHttpRequest(); + xhr.open(message.method, message.url); + xhr.send(); + } else { + console.error("Unknown message: " + JSON.stringify(message)); + } +}); diff --git a/chrome/test/data/extensions/extension_throttle/manifest.json b/chrome/test/data/extensions/extension_throttle/manifest.json new file mode 100644 index 0000000..70285c6 --- /dev/null +++ b/chrome/test/data/extensions/extension_throttle/manifest.json @@ -0,0 +1,9 @@ +{ + "name": "Extension tests which issue XHRs from the background page.", + "version": "1.0", + "manifest_version": 2, + "background": { + "scripts": ["background.js"] + }, + "permissions": ["webRequest", "<all_urls>"] +} diff --git a/chrome/test/data/extensions/extension_throttle/test_request_eventually_throttled.html b/chrome/test/data/extensions/extension_throttle/test_request_eventually_throttled.html new file mode 100644 index 0000000..89594b7 --- /dev/null +++ b/chrome/test/data/extensions/extension_throttle/test_request_eventually_throttled.html @@ -0,0 +1,6 @@ +<!-- + * Copyright 2015 The Chromium Authors. All rights reserved. Use of this + * source code is governed by a BSD-style license that can be found in the + * LICENSE file. +--> +<script src="test_request_eventually_throttled.js"></script> diff --git a/chrome/test/data/extensions/extension_throttle/test_request_eventually_throttled.js b/chrome/test/data/extensions/extension_throttle/test_request_eventually_throttled.js new file mode 100644 index 0000000..a262da5 --- /dev/null +++ b/chrome/test/data/extensions/extension_throttle/test_request_eventually_throttled.js @@ -0,0 +1,24 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// TODO(xunjieli): When URLSearchParams is stable and implemented, switch this +// (and a lot of other test code) to it. https://crbug.com/303152 +var url = decodeURIComponent(/url=([^&]*)/.exec(location.search)[1]); +var filter = {urls: ['http://www.example.com/*'], types: ['xmlhttprequest']}; +var numRequests = 0; + +chrome.webRequest.onCompleted.addListener(function(details) { + chrome.test.assertEq(503, details.statusCode); + numRequests++; + chrome.runtime.sendMessage({type: 'xhr', method: 'GET', url: url}); +}, filter); + +chrome.webRequest.onErrorOccurred.addListener(function(details) { + // Should not throttle the first request. + chrome.test.assertTrue(numRequests > 1); + chrome.test.assertEq('net::ERR_TEMPORARILY_THROTTLED', details.error); + chrome.test.notifyPass(); +}, filter); + +chrome.runtime.sendMessage({type: 'xhr', method: 'GET', url: url}); diff --git a/chrome/test/data/extensions/extension_throttle/test_request_not_throttled.html b/chrome/test/data/extensions/extension_throttle/test_request_not_throttled.html new file mode 100644 index 0000000..9b239a2 --- /dev/null +++ b/chrome/test/data/extensions/extension_throttle/test_request_not_throttled.html @@ -0,0 +1,6 @@ +<!-- + * Copyright 2015 The Chromium Authors. All rights reserved. Use of this + * source code is governed by a BSD-style license that can be found in the + * LICENSE file. +--> +<script src="test_request_not_throttled.js"></script> diff --git a/chrome/test/data/extensions/extension_throttle/test_request_not_throttled.js b/chrome/test/data/extensions/extension_throttle/test_request_not_throttled.js new file mode 100644 index 0000000..457beb8 --- /dev/null +++ b/chrome/test/data/extensions/extension_throttle/test_request_not_throttled.js @@ -0,0 +1,25 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// TODO(xunjieli): When URLSearchParams is stable and implemented, switch this +// (and a lot of other test code) to it. https://crbug.com/303152 +var url = decodeURIComponent(/url=([^&]*)/.exec(location.search)[1]); +var filter = {urls: ['http://www.example.com/*'], types: ['xmlhttprequest']}; +var numRequests = 0; + +chrome.webRequest.onCompleted.addListener(function(details) { + chrome.test.assertEq(503, details.statusCode); + if (numRequests == 20) { + chrome.test.notifyPass(); + } else { + numRequests++; + chrome.runtime.sendMessage({type: 'xhr', method: 'GET', url: url}); + } +}, filter); + +chrome.webRequest.onErrorOccurred.addListener(function(details) { + chrome.test.notifyFail('Unexpected error'); +}, filter); + +chrome.runtime.sendMessage({type: 'xhr', method: 'GET', url: url}); diff --git a/chrome/test/data/extensions/extension_throttle/test_request_throttled_on_first_try.html b/chrome/test/data/extensions/extension_throttle/test_request_throttled_on_first_try.html new file mode 100644 index 0000000..1833c20 --- /dev/null +++ b/chrome/test/data/extensions/extension_throttle/test_request_throttled_on_first_try.html @@ -0,0 +1,6 @@ +<!-- + * Copyright 2015 The Chromium Authors. All rights reserved. Use of this + * source code is governed by a BSD-style license that can be found in the + * LICENSE file. +--> +<script src="test_request_throttled_on_first_try.js"></script> diff --git a/chrome/test/data/extensions/extension_throttle/test_request_throttled_on_first_try.js b/chrome/test/data/extensions/extension_throttle/test_request_throttled_on_first_try.js new file mode 100644 index 0000000..eca6518 --- /dev/null +++ b/chrome/test/data/extensions/extension_throttle/test_request_throttled_on_first_try.js @@ -0,0 +1,19 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// TODO(xunjieli): When URLSearchParams is stable and implemented, switch this +// (and a lot of other test code) to it. https://crbug.com/303152 +var url = decodeURIComponent(/url=([^&]*)/.exec(location.search)[1]); +var filter = {urls: ['http://www.example.com/*'], types: ['xmlhttprequest']}; + +chrome.webRequest.onCompleted.addListener(function(details) { + chrome.test.notifyFail(); +}, filter); + +chrome.webRequest.onErrorOccurred.addListener(function(details) { + chrome.test.assertEq('net::ERR_TEMPORARILY_THROTTLED', details.error); + chrome.test.notifyPass(); +}, filter); + +chrome.runtime.sendMessage({type: 'xhr', method: 'GET', url: url}); diff --git a/extensions/browser/extension_request_limiting_throttle.cc b/extensions/browser/extension_request_limiting_throttle.cc new file mode 100644 index 0000000..05d0410 --- /dev/null +++ b/extensions/browser/extension_request_limiting_throttle.cc @@ -0,0 +1,58 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "extensions/browser/extension_request_limiting_throttle.h" + +#include "base/logging.h" +#include "content/public/browser/resource_controller.h" +#include "extensions/browser/extension_throttle_entry.h" +#include "extensions/browser/extension_throttle_manager.h" +#include "net/base/net_errors.h" +#include "net/url_request/redirect_info.h" +#include "net/url_request/url_request.h" + +namespace extensions { + +ExtensionRequestLimitingThrottle::ExtensionRequestLimitingThrottle( + const net::URLRequest* request, + ExtensionThrottleManager* manager) + : request_(request), manager_(manager) { + DCHECK(manager_); +} + +ExtensionRequestLimitingThrottle::~ExtensionRequestLimitingThrottle() { +} + +void ExtensionRequestLimitingThrottle::WillStartRequest(bool* defer) { + throttling_entry_ = manager_->RegisterRequestUrl(request_->url()); + if (throttling_entry_->ShouldRejectRequest(*request_)) + controller()->CancelWithError(net::ERR_TEMPORARILY_THROTTLED); +} + +void ExtensionRequestLimitingThrottle::WillRedirectRequest( + const net::RedirectInfo& redirect_info, + bool* defer) { + DCHECK_EQ(manager_->GetIdFromUrl(request_->url()), + throttling_entry_->GetURLIdForDebugging()); + + throttling_entry_->UpdateWithResponse(redirect_info.status_code); + + throttling_entry_ = manager_->RegisterRequestUrl(redirect_info.new_url); + if (throttling_entry_->ShouldRejectRequest(*request_)) + controller()->CancelWithError(net::ERR_TEMPORARILY_THROTTLED); +} + +void ExtensionRequestLimitingThrottle::WillProcessResponse(bool* defer) { + DCHECK_EQ(manager_->GetIdFromUrl(request_->url()), + throttling_entry_->GetURLIdForDebugging()); + + if (!request_->was_cached()) + throttling_entry_->UpdateWithResponse(request_->GetResponseCode()); +} + +const char* ExtensionRequestLimitingThrottle::GetNameForLogging() const { + return "ExtensionRequestLimitingThrottle"; +} + +} // namespace extensions diff --git a/extensions/browser/extension_request_limiting_throttle.h b/extensions/browser/extension_request_limiting_throttle.h new file mode 100644 index 0000000..bd9dd4c --- /dev/null +++ b/extensions/browser/extension_request_limiting_throttle.h @@ -0,0 +1,52 @@ +// Copyright 2015 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef EXTENSIONS_BROWSER_EXTENSION_REQUEST_LIMITING_THROTTLE_H_ +#define EXTENSIONS_BROWSER_EXTENSION_REQUEST_LIMITING_THROTTLE_H_ + +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "content/public/browser/resource_throttle.h" + +namespace net { +struct RedirectInfo; +class URLRequest; +} + +namespace extensions { + +class ExtensionThrottleEntryInterface; +class ExtensionThrottleManager; + +// This class monitors requests issued by extensions and throttles the request +// if there are too many requests made within a short time to urls with the same +// scheme, host, port and path. For the exact criteria for throttling, please +// also see extension_throttle_manager.cc. +class ExtensionRequestLimitingThrottle : public content::ResourceThrottle { + public: + ExtensionRequestLimitingThrottle(const net::URLRequest* request, + ExtensionThrottleManager* manager); + ~ExtensionRequestLimitingThrottle() override; + + // content::ResourceThrottle implementation (called on IO thread): + void WillStartRequest(bool* defer) override; + void WillRedirectRequest(const net::RedirectInfo& redirect_info, + bool* defer) override; + void WillProcessResponse(bool* defer) override; + + const char* GetNameForLogging() const override; + + private: + const net::URLRequest* request_; + ExtensionThrottleManager* manager_; + + // This is used to supervise traffic and enforce exponential back-off. + scoped_refptr<ExtensionThrottleEntryInterface> throttling_entry_; + + DISALLOW_COPY_AND_ASSIGN(ExtensionRequestLimitingThrottle); +}; + +} // namespace extensions + +#endif // EXTENSIONS_BROWSER_EXTENSION_REQUEST_LIMITING_THROTTLE_H_ diff --git a/extensions/browser/extension_throttle_entry.cc b/extensions/browser/extension_throttle_entry.cc new file mode 100644 index 0000000..1e22d94 --- /dev/null +++ b/extensions/browser/extension_throttle_entry.cc @@ -0,0 +1,297 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "extensions/browser/extension_throttle_entry.h" + +#include <cmath> + +#include "base/logging.h" +#include "base/rand_util.h" +#include "base/strings/string_number_conversions.h" +#include "base/values.h" +#include "extensions/browser/extension_throttle_manager.h" +#include "net/base/load_flags.h" +#include "net/log/net_log.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context.h" + +namespace extensions { + +const int ExtensionThrottleEntry::kDefaultSlidingWindowPeriodMs = 2000; +const int ExtensionThrottleEntry::kDefaultMaxSendThreshold = 20; + +// This set of back-off parameters will (at maximum values, i.e. without +// the reduction caused by jitter) add 0-41% (distributed uniformly +// in that range) to the "perceived downtime" of the remote server, once +// exponential back-off kicks in and is throttling requests for more than +// about a second at a time. Once the maximum back-off is reached, the added +// perceived downtime decreases rapidly, percentage-wise. +// +// Another way to put it is that the maximum additional perceived downtime +// with these numbers is a couple of seconds shy of 15 minutes, and such +// a delay would not occur until the remote server has been actually +// unavailable at the end of each back-off period for a total of about +// 48 minutes. +// +// Ignoring the first couple of errors is just a conservative measure to +// avoid false positives. It should help avoid back-off from kicking in e.g. +// on flaky connections. +const int ExtensionThrottleEntry::kDefaultNumErrorsToIgnore = 2; +const int ExtensionThrottleEntry::kDefaultInitialDelayMs = 700; +const double ExtensionThrottleEntry::kDefaultMultiplyFactor = 1.4; +const double ExtensionThrottleEntry::kDefaultJitterFactor = 0.4; +const int ExtensionThrottleEntry::kDefaultMaximumBackoffMs = 15 * 60 * 1000; +const int ExtensionThrottleEntry::kDefaultEntryLifetimeMs = 2 * 60 * 1000; + +// Returns NetLog parameters when a request is rejected by throttling. +scoped_ptr<base::Value> NetLogRejectedRequestCallback( + const std::string* url_id, + int num_failures, + const base::TimeDelta& release_after, + net::NetLogCaptureMode /* capture_mode */) { + scoped_ptr<base::DictionaryValue> dict(new base::DictionaryValue()); + dict->SetString("url", *url_id); + dict->SetInteger("num_failures", num_failures); + dict->SetInteger("release_after_ms", + static_cast<int>(release_after.InMilliseconds())); + return dict.Pass(); +} + +ExtensionThrottleEntry::ExtensionThrottleEntry( + ExtensionThrottleManager* manager, + const std::string& url_id) + : ExtensionThrottleEntry(manager, url_id, false) { +} + +ExtensionThrottleEntry::ExtensionThrottleEntry( + ExtensionThrottleManager* manager, + const std::string& url_id, + bool ignore_user_gesture_load_flag_for_tests) + : sliding_window_period_( + base::TimeDelta::FromMilliseconds(kDefaultSlidingWindowPeriodMs)), + max_send_threshold_(kDefaultMaxSendThreshold), + is_backoff_disabled_(false), + backoff_entry_(&backoff_policy_), + manager_(manager), + url_id_(url_id), + net_log_(net::BoundNetLog::Make( + manager->net_log(), + net::NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING)), + ignore_user_gesture_load_flag_for_tests_( + ignore_user_gesture_load_flag_for_tests) { + DCHECK(manager_); + Initialize(); +} + +ExtensionThrottleEntry::ExtensionThrottleEntry( + ExtensionThrottleManager* manager, + const std::string& url_id, + int sliding_window_period_ms, + int max_send_threshold, + int initial_backoff_ms, + double multiply_factor, + double jitter_factor, + int maximum_backoff_ms) + : sliding_window_period_( + base::TimeDelta::FromMilliseconds(sliding_window_period_ms)), + max_send_threshold_(max_send_threshold), + is_backoff_disabled_(false), + backoff_entry_(&backoff_policy_), + manager_(manager), + url_id_(url_id), + ignore_user_gesture_load_flag_for_tests_(false) { + DCHECK_GT(sliding_window_period_ms, 0); + DCHECK_GT(max_send_threshold_, 0); + DCHECK_GE(initial_backoff_ms, 0); + DCHECK_GT(multiply_factor, 0); + DCHECK_GE(jitter_factor, 0.0); + DCHECK_LT(jitter_factor, 1.0); + DCHECK_GE(maximum_backoff_ms, 0); + DCHECK(manager_); + + Initialize(); + backoff_policy_.initial_delay_ms = initial_backoff_ms; + backoff_policy_.multiply_factor = multiply_factor; + backoff_policy_.jitter_factor = jitter_factor; + backoff_policy_.maximum_backoff_ms = maximum_backoff_ms; + backoff_policy_.entry_lifetime_ms = -1; + backoff_policy_.num_errors_to_ignore = 0; + backoff_policy_.always_use_initial_delay = false; +} + +bool ExtensionThrottleEntry::IsEntryOutdated() const { + // This function is called by the ExtensionThrottleManager to determine + // whether entries should be discarded from its url_entries_ map. We + // want to ensure that it does not remove entries from the map while there + // are clients (objects other than the manager) holding references to + // the entry, otherwise separate clients could end up holding separate + // entries for a request to the same URL, which is undesirable. Therefore, + // if an entry has more than one reference (the map will always hold one), + // it should not be considered outdated. + // + // We considered whether to make ExtensionThrottleEntry objects + // non-refcounted, but since any means of knowing whether they are + // currently in use by others than the manager would be more or less + // equivalent to a refcount, we kept them refcounted. + if (!HasOneRef()) + return false; + + // If there are send events in the sliding window period, we still need this + // entry. + if (!send_log_.empty() && + send_log_.back() + sliding_window_period_ > ImplGetTimeNow()) { + return false; + } + + return GetBackoffEntry()->CanDiscard(); +} + +void ExtensionThrottleEntry::DisableBackoffThrottling() { + is_backoff_disabled_ = true; +} + +void ExtensionThrottleEntry::DetachManager() { + manager_ = NULL; +} + +bool ExtensionThrottleEntry::ShouldRejectRequest( + const net::URLRequest& request) const { + bool reject_request = false; + if (!is_backoff_disabled_ && (ignore_user_gesture_load_flag_for_tests_ || + !ExplicitUserRequest(request.load_flags())) && + GetBackoffEntry()->ShouldRejectRequest()) { + net_log_.AddEvent(net::NetLog::TYPE_THROTTLING_REJECTED_REQUEST, + base::Bind(&NetLogRejectedRequestCallback, &url_id_, + GetBackoffEntry()->failure_count(), + GetBackoffEntry()->GetTimeUntilRelease())); + reject_request = true; + } + return reject_request; +} + +int64 ExtensionThrottleEntry::ReserveSendingTimeForNextRequest( + const base::TimeTicks& earliest_time) { + base::TimeTicks now = ImplGetTimeNow(); + + // If a lot of requests were successfully made recently, + // sliding_window_release_time_ may be greater than + // exponential_backoff_release_time_. + base::TimeTicks recommended_sending_time = + std::max(std::max(now, earliest_time), + std::max(GetBackoffEntry()->GetReleaseTime(), + sliding_window_release_time_)); + + DCHECK(send_log_.empty() || recommended_sending_time >= send_log_.back()); + // Log the new send event. + send_log_.push(recommended_sending_time); + + sliding_window_release_time_ = recommended_sending_time; + + // Drop the out-of-date events in the event list. + // We don't need to worry that the queue may become empty during this + // operation, since the last element is sliding_window_release_time_. + while ((send_log_.front() + sliding_window_period_ <= + sliding_window_release_time_) || + send_log_.size() > static_cast<unsigned>(max_send_threshold_)) { + send_log_.pop(); + } + + // Check if there are too many send events in recent time. + if (send_log_.size() == static_cast<unsigned>(max_send_threshold_)) + sliding_window_release_time_ = send_log_.front() + sliding_window_period_; + + return (recommended_sending_time - now).InMillisecondsRoundedUp(); +} + +base::TimeTicks ExtensionThrottleEntry::GetExponentialBackoffReleaseTime() + const { + // If a site opts out, it's likely because they have problems that trigger + // the back-off mechanism when it shouldn't be triggered, in which case + // returning the calculated back-off release time would probably be the + // wrong thing to do (i.e. it would likely be too long). Therefore, we + // return "now" so that retries are not delayed. + if (is_backoff_disabled_) + return ImplGetTimeNow(); + + return GetBackoffEntry()->GetReleaseTime(); +} + +void ExtensionThrottleEntry::UpdateWithResponse(int status_code) { + GetBackoffEntry()->InformOfRequest(IsConsideredSuccess(status_code)); +} + +void ExtensionThrottleEntry::ReceivedContentWasMalformed(int response_code) { + // A malformed body can only occur when the request to fetch a resource + // was successful. Therefore, in such a situation, we will receive one + // call to ReceivedContentWasMalformed() and one call to + // UpdateWithResponse() with a response categorized as "good". To end + // up counting one failure, we need to count two failures here against + // the one success in UpdateWithResponse(). + // + // We do nothing for a response that is already being considered an error + // based on its status code (otherwise we would count 3 errors instead of 1). + if (IsConsideredSuccess(response_code)) { + GetBackoffEntry()->InformOfRequest(false); + GetBackoffEntry()->InformOfRequest(false); + } +} + +const std::string& ExtensionThrottleEntry::GetURLIdForDebugging() const { + return url_id_; +} + +ExtensionThrottleEntry::~ExtensionThrottleEntry() { +} + +void ExtensionThrottleEntry::Initialize() { + sliding_window_release_time_ = base::TimeTicks::Now(); + backoff_policy_.num_errors_to_ignore = kDefaultNumErrorsToIgnore; + backoff_policy_.initial_delay_ms = kDefaultInitialDelayMs; + backoff_policy_.multiply_factor = kDefaultMultiplyFactor; + backoff_policy_.jitter_factor = kDefaultJitterFactor; + backoff_policy_.maximum_backoff_ms = kDefaultMaximumBackoffMs; + backoff_policy_.entry_lifetime_ms = kDefaultEntryLifetimeMs; + backoff_policy_.always_use_initial_delay = false; +} + +bool ExtensionThrottleEntry::IsConsideredSuccess(int response_code) { + // We throttle only for the status codes most likely to indicate the server + // is failing because it is too busy or otherwise are likely to be + // because of DDoS. + // + // 500 is the generic error when no better message is suitable, and + // as such does not necessarily indicate a temporary state, but + // other status codes cover most of the permanent error states. + // 503 is explicitly documented as a temporary state where the server + // is either overloaded or down for maintenance. + // 509 is the (non-standard but widely implemented) Bandwidth Limit Exceeded + // status code, which might indicate DDoS. + // + // We do not back off on 502 or 504, which are reported by gateways + // (proxies) on timeouts or failures, because in many cases these requests + // have not made it to the destination server and so we do not actually + // know that it is down or busy. One degenerate case could be a proxy on + // localhost, where you are not actually connected to the network. + return !(response_code == 500 || response_code == 503 || + response_code == 509); +} + +base::TimeTicks ExtensionThrottleEntry::ImplGetTimeNow() const { + return base::TimeTicks::Now(); +} + +const net::BackoffEntry* ExtensionThrottleEntry::GetBackoffEntry() const { + return &backoff_entry_; +} + +net::BackoffEntry* ExtensionThrottleEntry::GetBackoffEntry() { + return &backoff_entry_; +} + +// static +bool ExtensionThrottleEntry::ExplicitUserRequest(const int load_flags) { + return (load_flags & net::LOAD_MAYBE_USER_GESTURE) != 0; +} + +} // namespace extensions diff --git a/extensions/browser/extension_throttle_entry.h b/extensions/browser/extension_throttle_entry.h new file mode 100644 index 0000000..d0f25cb --- /dev/null +++ b/extensions/browser/extension_throttle_entry.h @@ -0,0 +1,168 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef EXTENSIONS_BROWSER_EXTENSION_THROTTLE_ENTRY_H_ +#define EXTENSIONS_BROWSER_EXTENSION_THROTTLE_ENTRY_H_ + +#include <queue> +#include <string> + +#include "base/basictypes.h" +#include "base/time/time.h" +#include "extensions/browser/extension_throttle_entry_interface.h" +#include "net/base/backoff_entry.h" +#include "net/log/net_log.h" + +namespace extensions { + +class ExtensionThrottleManager; + +// ExtensionThrottleEntry represents an entry of ExtensionThrottleManager. +// It analyzes requests of a specific URL over some period of time, in order to +// deduce the back-off time for every request. +// The back-off algorithm consists of two parts. Firstly, exponential back-off +// is used when receiving 5XX server errors or malformed response bodies. +// The exponential back-off rule is enforced by URLRequestHttpJob. Any +// request sent during the back-off period will be cancelled. +// Secondly, a sliding window is used to count recent requests to a given +// destination and provide guidance (to the application level only) on whether +// too many requests have been sent and when a good time to send the next one +// would be. This is never used to deny requests at the network level. +class ExtensionThrottleEntry : public ExtensionThrottleEntryInterface { + public: + // Sliding window period. + static const int kDefaultSlidingWindowPeriodMs; + + // Maximum number of requests allowed in sliding window period. + static const int kDefaultMaxSendThreshold; + + // Number of initial errors to ignore before starting exponential back-off. + static const int kDefaultNumErrorsToIgnore; + + // Initial delay for exponential back-off. + static const int kDefaultInitialDelayMs; + + // Factor by which the waiting time will be multiplied. + static const double kDefaultMultiplyFactor; + + // Fuzzing percentage. ex: 10% will spread requests randomly + // between 90%-100% of the calculated time. + static const double kDefaultJitterFactor; + + // Maximum amount of time we are willing to delay our request. + static const int kDefaultMaximumBackoffMs; + + // Time after which the entry is considered outdated. + static const int kDefaultEntryLifetimeMs; + + // The manager object's lifetime must enclose the lifetime of this object. + ExtensionThrottleEntry(ExtensionThrottleManager* manager, + const std::string& url_id); + + // Same as above, but exposes the option to ignore + // net::LOAD_MAYBE_USER_GESTURE flag of the request. + ExtensionThrottleEntry(ExtensionThrottleManager* manager, + const std::string& url_id, + bool ignore_user_gesture_load_flag_for_tests); + + // The life span of instances created with this constructor is set to + // infinite, and the number of initial errors to ignore is set to 0. + // It is only used by unit tests. + ExtensionThrottleEntry(ExtensionThrottleManager* manager, + const std::string& url_id, + int sliding_window_period_ms, + int max_send_threshold, + int initial_backoff_ms, + double multiply_factor, + double jitter_factor, + int maximum_backoff_ms); + + // Used by the manager, returns true if the entry needs to be garbage + // collected. + bool IsEntryOutdated() const; + + // Causes this entry to never reject requests due to back-off. + void DisableBackoffThrottling(); + + // Causes this entry to NULL its manager pointer. + void DetachManager(); + + // Implementation of ExtensionThrottleEntryInterface. + bool ShouldRejectRequest(const net::URLRequest& request) const override; + int64 ReserveSendingTimeForNextRequest( + const base::TimeTicks& earliest_time) override; + base::TimeTicks GetExponentialBackoffReleaseTime() const override; + void UpdateWithResponse(int status_code) override; + void ReceivedContentWasMalformed(int response_code) override; + const std::string& GetURLIdForDebugging() const override; + + protected: + ~ExtensionThrottleEntry() override; + + void Initialize(); + + // Returns true if the given response code is considered a success for + // throttling purposes. + bool IsConsideredSuccess(int response_code); + + // Equivalent to TimeTicks::Now(), virtual to be mockable for testing purpose. + virtual base::TimeTicks ImplGetTimeNow() const; + + // Retrieves the back-off entry object we're using. Used to enable a + // unit testing seam for dependency injection in tests. + virtual const net::BackoffEntry* GetBackoffEntry() const; + virtual net::BackoffEntry* GetBackoffEntry(); + + // Returns true if |load_flags| contains a flag that indicates an + // explicit request by the user to load the resource. We never + // throttle requests with such load flags. + static bool ExplicitUserRequest(const int load_flags); + + // Used by tests. + base::TimeTicks sliding_window_release_time() const { + return sliding_window_release_time_; + } + + // Used by tests. + void set_sliding_window_release_time(const base::TimeTicks& release_time) { + sliding_window_release_time_ = release_time; + } + + // Valid and immutable after construction time. + net::BackoffEntry::Policy backoff_policy_; + + private: + // Timestamp calculated by the sliding window algorithm for when we advise + // clients the next request should be made, at the earliest. Advisory only, + // not used to deny requests. + base::TimeTicks sliding_window_release_time_; + + // A list of the recent send events. We use them to decide whether there are + // too many requests sent in sliding window. + std::queue<base::TimeTicks> send_log_; + + const base::TimeDelta sliding_window_period_; + const int max_send_threshold_; + + // True if DisableBackoffThrottling() has been called on this object. + bool is_backoff_disabled_; + + // Access it through GetBackoffEntry() to allow a unit test seam. + net::BackoffEntry backoff_entry_; + + // Weak back-reference to the manager object managing us. + ExtensionThrottleManager* manager_; + + // Canonicalized URL string that this entry is for; used for logging only. + std::string url_id_; + + net::BoundNetLog net_log_; + bool ignore_user_gesture_load_flag_for_tests_; + + DISALLOW_COPY_AND_ASSIGN(ExtensionThrottleEntry); +}; + +} // namespace extensions + +#endif // EXTENSIONS_BROWSER_EXTENSION_THROTTLE_ENTRY_H_ diff --git a/extensions/browser/extension_throttle_entry_interface.h b/extensions/browser/extension_throttle_entry_interface.h new file mode 100644 index 0000000..99989b8 --- /dev/null +++ b/extensions/browser/extension_throttle_entry_interface.h @@ -0,0 +1,76 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef EXTENSIONS_BROWSER_EXTENSION_THROTTLE_ENTRY_INTERFACE_H_ +#define EXTENSIONS_BROWSER_EXTENSION_THROTTLE_ENTRY_INTERFACE_H_ + +#include <string> + +#include "base/basictypes.h" +#include "base/memory/ref_counted.h" +#include "base/time/time.h" +#include "net/base/net_export.h" + +namespace net { +class URLRequest; +} // namespace net + +namespace extensions { + +// Interface provided on entries of the URL request throttler manager. +class ExtensionThrottleEntryInterface + : public base::RefCountedThreadSafe<ExtensionThrottleEntryInterface> { + public: + ExtensionThrottleEntryInterface() {} + + // Returns true when we have encountered server errors and are doing + // exponential back-off, unless the request has load flags that mean + // it is likely to be user-initiated, or the NetworkDelegate returns + // false for |CanThrottleRequest(request)|. + // + // URLRequestHttpJob checks this method prior to every request; it + // cancels requests if this method returns true. + virtual bool ShouldRejectRequest(const net::URLRequest& request) const = 0; + + // Calculates a recommended sending time for the next request and reserves it. + // The sending time is not earlier than the current exponential back-off + // release time or |earliest_time|. Moreover, the previous results of + // the method are taken into account, in order to make sure they are spread + // properly over time. + // Returns the recommended delay before sending the next request, in + // milliseconds. The return value is always positive or 0. + // Although it is not mandatory, respecting the value returned by this method + // is helpful to avoid traffic overload. + virtual int64 ReserveSendingTimeForNextRequest( + const base::TimeTicks& earliest_time) = 0; + + // Returns the time after which requests are allowed. + virtual base::TimeTicks GetExponentialBackoffReleaseTime() const = 0; + + // This method needs to be called each time a response is received. + virtual void UpdateWithResponse(int status_code) = 0; + + // Lets higher-level modules, that know how to parse particular response + // bodies, notify of receiving malformed content for the given URL. This will + // be handled by the throttler as if an HTTP 503 response had been received to + // the request, i.e. it will count as a failure, unless the HTTP response code + // indicated is already one of those that will be counted as an error. + virtual void ReceivedContentWasMalformed(int response_code) = 0; + + // Get the URL ID associated with his entry. Should only be used for debugging + // purpose. + virtual const std::string& GetURLIdForDebugging() const = 0; + + protected: + friend class base::RefCountedThreadSafe<ExtensionThrottleEntryInterface>; + virtual ~ExtensionThrottleEntryInterface() {} + + private: + friend class base::RefCounted<ExtensionThrottleEntryInterface>; + DISALLOW_COPY_AND_ASSIGN(ExtensionThrottleEntryInterface); +}; + +} // namespace extensions + +#endif // EXTENSIONS_BROWSER_EXTENSION_THROTTLE_ENTRY_INTERFACE_H_ diff --git a/extensions/browser/extension_throttle_manager.cc b/extensions/browser/extension_throttle_manager.cc new file mode 100644 index 0000000..a8491de --- /dev/null +++ b/extensions/browser/extension_throttle_manager.cc @@ -0,0 +1,204 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "extensions/browser/extension_throttle_manager.h" + +#include "base/logging.h" +#include "base/metrics/field_trial.h" +#include "base/metrics/histogram.h" +#include "base/strings/string_util.h" +#include "extensions/browser/extension_request_limiting_throttle.h" +#include "extensions/common/constants.h" +#include "net/base/net_util.h" +#include "net/log/net_log.h" +#include "net/url_request/url_request.h" + +namespace extensions { + +const unsigned int ExtensionThrottleManager::kMaximumNumberOfEntries = 1500; +const unsigned int ExtensionThrottleManager::kRequestsBetweenCollecting = 200; + +ExtensionThrottleManager::ExtensionThrottleManager() + : requests_since_last_gc_(0), + enable_thread_checks_(false), + logged_for_localhost_disabled_(false), + registered_from_thread_(base::kInvalidThreadId), + ignore_user_gesture_load_flag_for_tests_(false) { + url_id_replacements_.ClearPassword(); + url_id_replacements_.ClearUsername(); + url_id_replacements_.ClearQuery(); + url_id_replacements_.ClearRef(); + + net::NetworkChangeNotifier::AddIPAddressObserver(this); + net::NetworkChangeNotifier::AddConnectionTypeObserver(this); +} + +ExtensionThrottleManager::~ExtensionThrottleManager() { + net::NetworkChangeNotifier::RemoveIPAddressObserver(this); + net::NetworkChangeNotifier::RemoveConnectionTypeObserver(this); + + // Since the manager object might conceivably go away before the + // entries, detach the entries' back-pointer to the manager. + UrlEntryMap::iterator i = url_entries_.begin(); + while (i != url_entries_.end()) { + if (i->second.get() != NULL) { + i->second->DetachManager(); + } + ++i; + } + + // Delete all entries. + url_entries_.clear(); +} + +scoped_ptr<content::ResourceThrottle> +ExtensionThrottleManager::MaybeCreateThrottle(const net::URLRequest* request) { + if (request->first_party_for_cookies().scheme() != + extensions::kExtensionScheme) { + return nullptr; + } + return make_scoped_ptr( + new extensions::ExtensionRequestLimitingThrottle(request, this)); +} + +scoped_refptr<ExtensionThrottleEntryInterface> +ExtensionThrottleManager::RegisterRequestUrl(const GURL& url) { + DCHECK(!enable_thread_checks_ || CalledOnValidThread()); + + // Normalize the url. + std::string url_id = GetIdFromUrl(url); + + // Periodically garbage collect old entries. + GarbageCollectEntriesIfNecessary(); + + // Find the entry in the map or create a new NULL entry. + scoped_refptr<ExtensionThrottleEntry>& entry = url_entries_[url_id]; + + // If the entry exists but could be garbage collected at this point, we + // start with a fresh entry so that we possibly back off a bit less + // aggressively (i.e. this resets the error count when the entry's URL + // hasn't been requested in long enough). + if (entry.get() && entry->IsEntryOutdated()) { + entry = NULL; + } + + // Create the entry if needed. + if (entry.get() == NULL) { + entry = new ExtensionThrottleEntry( + this, url_id, ignore_user_gesture_load_flag_for_tests_); + + // We only disable back-off throttling on an entry that we have + // just constructed. This is to allow unit tests to explicitly override + // the entry for localhost URLs. + std::string host = url.host(); + if (net::IsLocalhost(host)) { + if (!logged_for_localhost_disabled_ && net::IsLocalhost(host)) { + logged_for_localhost_disabled_ = true; + net_log_.AddEvent(net::NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST, + net::NetLog::StringCallback("host", &host)); + } + + // TODO(joi): Once sliding window is separate from back-off throttling, + // we can simply return a dummy implementation of + // ExtensionThrottleEntryInterface here that never blocks anything. + entry->DisableBackoffThrottling(); + } + } + + return entry; +} + +void ExtensionThrottleManager::OverrideEntryForTests( + const GURL& url, + ExtensionThrottleEntry* entry) { + // Normalize the url. + std::string url_id = GetIdFromUrl(url); + + // Periodically garbage collect old entries. + GarbageCollectEntriesIfNecessary(); + + url_entries_[url_id] = entry; +} + +void ExtensionThrottleManager::EraseEntryForTests(const GURL& url) { + // Normalize the url. + std::string url_id = GetIdFromUrl(url); + url_entries_.erase(url_id); +} + +void ExtensionThrottleManager::SetIgnoreUserGestureLoadFlagForTests( + bool ignore_user_gesture_load_flag_for_tests) { + ignore_user_gesture_load_flag_for_tests_ = true; +} + +void ExtensionThrottleManager::set_enable_thread_checks(bool enable) { + enable_thread_checks_ = enable; +} + +bool ExtensionThrottleManager::enable_thread_checks() const { + return enable_thread_checks_; +} + +void ExtensionThrottleManager::set_net_log(net::NetLog* net_log) { + DCHECK(net_log); + net_log_ = net::BoundNetLog::Make( + net_log, net::NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING); +} + +net::NetLog* ExtensionThrottleManager::net_log() const { + return net_log_.net_log(); +} + +void ExtensionThrottleManager::OnIPAddressChanged() { + OnNetworkChange(); +} + +void ExtensionThrottleManager::OnConnectionTypeChanged( + net::NetworkChangeNotifier::ConnectionType type) { + OnNetworkChange(); +} + +std::string ExtensionThrottleManager::GetIdFromUrl(const GURL& url) const { + if (!url.is_valid()) + return url.possibly_invalid_spec(); + + GURL id = url.ReplaceComponents(url_id_replacements_); + return base::StringToLowerASCII(id.spec()).c_str(); +} + +void ExtensionThrottleManager::GarbageCollectEntriesIfNecessary() { + requests_since_last_gc_++; + if (requests_since_last_gc_ < kRequestsBetweenCollecting) + return; + requests_since_last_gc_ = 0; + + GarbageCollectEntries(); +} + +void ExtensionThrottleManager::GarbageCollectEntries() { + UrlEntryMap::iterator i = url_entries_.begin(); + while (i != url_entries_.end()) { + if ((i->second)->IsEntryOutdated()) { + url_entries_.erase(i++); + } else { + ++i; + } + } + + // In case something broke we want to make sure not to grow indefinitely. + while (url_entries_.size() > kMaximumNumberOfEntries) { + url_entries_.erase(url_entries_.begin()); + } +} + +void ExtensionThrottleManager::OnNetworkChange() { + // Remove all entries. Any entries that in-flight requests have a reference + // to will live until those requests end, and these entries may be + // inconsistent with new entries for the same URLs, but since what we + // want is a clean slate for the new connection type, this is OK. + url_entries_.clear(); + requests_since_last_gc_ = 0; +} + +} // namespace extensions diff --git a/extensions/browser/extension_throttle_manager.h b/extensions/browser/extension_throttle_manager.h new file mode 100644 index 0000000..d48c05f --- /dev/null +++ b/extensions/browser/extension_throttle_manager.h @@ -0,0 +1,179 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef EXTENSIONS_BROWSER_EXTENSION_THROTTLE_MANAGER_H_ +#define EXTENSIONS_BROWSER_EXTENSION_THROTTLE_MANAGER_H_ + +#include <map> +#include <string> + +#include "base/basictypes.h" +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_ptr.h" +#include "base/threading/non_thread_safe.h" +#include "base/threading/platform_thread.h" +#include "extensions/browser/extension_throttle_entry.h" +#include "net/base/net_export.h" +#include "net/base/network_change_notifier.h" +#include "url/gurl.h" + +namespace content { +class ResourceThrottle; +} + +namespace net { +class BoundNetLog; +class NetLog; +} + +namespace extensions { + +// Class that registers URL request throttler entries for URLs being accessed +// in order to supervise traffic. URL requests for HTTP contents should +// register their URLs in this manager on each request. +// +// ExtensionThrottleManager maintains a map of URL IDs to URL request +// throttler entries. It creates URL request throttler entries when new URLs +// are registered, and does garbage collection from time to time in order to +// clean out outdated entries. URL ID consists of lowercased scheme, host, port +// and path. All URLs converted to the same ID will share the same entry. +class ExtensionThrottleManager + : NON_EXPORTED_BASE(public base::NonThreadSafe), + public net::NetworkChangeNotifier::IPAddressObserver, + public net::NetworkChangeNotifier::ConnectionTypeObserver { + public: + ExtensionThrottleManager(); + ~ExtensionThrottleManager() override; + + // Creates a content::ResourceThrottle for |request| to prevent extensions + // from requesting a URL too often, if such a throttle is needed. + scoped_ptr<content::ResourceThrottle> MaybeCreateThrottle( + const net::URLRequest* request); + + // TODO(xunjieli): Remove this method and replace with + // ShouldRejectRequest(request) and UpdateWithResponse(request, status_code), + // which will also allow ExtensionThrottleEntry to no longer be reference + // counted, and ExtensionThrottleEntryInterface to be removed. + + // Must be called for every request, returns the URL request throttler entry + // associated with the URL. The caller must inform this entry of some events. + // Please refer to extension_throttle_entry_interface.h for further + // informations. + scoped_refptr<ExtensionThrottleEntryInterface> RegisterRequestUrl( + const GURL& url); + + // Registers a new entry in this service and overrides the existing entry (if + // any) for the URL. The service will hold a reference to the entry. + // It is only used by unit tests. + void OverrideEntryForTests(const GURL& url, ExtensionThrottleEntry* entry); + + // Explicitly erases an entry. + // This is useful to remove those entries which have got infinite lifetime and + // thus won't be garbage collected. + // It is only used by unit tests. + void EraseEntryForTests(const GURL& url); + + // Sets whether to ignore net::LOAD_MAYBE_USER_GESTURE of the request for + // testing. Otherwise, requests will not be throttled when they may have been + // throttled in response to a recent user gesture, though they're still + // counted for the purpose of throttling other requests. + void SetIgnoreUserGestureLoadFlagForTests( + bool ignore_user_gesture_load_flag_for_tests); + + // Turns threading model verification on or off. Any code that correctly + // uses the network stack should preferably call this function to enable + // verification of correct adherence to the network stack threading model. + void set_enable_thread_checks(bool enable); + bool enable_thread_checks() const; + + // Whether throttling is enabled or not. + void set_enforce_throttling(bool enforce); + bool enforce_throttling(); + + // Sets the net::NetLog instance to use. + void set_net_log(net::NetLog* net_log); + net::NetLog* net_log() const; + + // IPAddressObserver interface. + void OnIPAddressChanged() override; + + // ConnectionTypeObserver interface. + void OnConnectionTypeChanged( + net::NetworkChangeNotifier::ConnectionType type) override; + + // Method that allows us to transform a URL into an ID that can be used in our + // map. Resulting IDs will be lowercase and consist of the scheme, host, port + // and path (without query string, fragment, etc.). + // If the URL is invalid, the invalid spec will be returned, without any + // transformation. + std::string GetIdFromUrl(const GURL& url) const; + + // Method that ensures the map gets cleaned from time to time. The period at + // which garbage collecting happens is adjustable with the + // kRequestBetweenCollecting constant. + void GarbageCollectEntriesIfNecessary(); + + // Method that does the actual work of garbage collecting. + void GarbageCollectEntries(); + + // When we switch from online to offline or change IP addresses, we + // clear all back-off history. This is a precaution in case the change in + // online state now lets us communicate without error with servers that + // we were previously getting 500 or 503 responses from (perhaps the + // responses are from a badly-written proxy that should have returned a + // 502 or 504 because it's upstream connection was down or it had no route + // to the server). + void OnNetworkChange(); + + // Used by tests. + int GetNumberOfEntriesForTests() const { return url_entries_.size(); } + + private: + // From each URL we generate an ID composed of the scheme, host, port and path + // that allows us to uniquely map an entry to it. + typedef std::map<std::string, scoped_refptr<ExtensionThrottleEntry>> + UrlEntryMap; + + // Maximum number of entries that we are willing to collect in our map. + static const unsigned int kMaximumNumberOfEntries; + // Number of requests that will be made between garbage collection. + static const unsigned int kRequestsBetweenCollecting; + + // Map that contains a list of URL ID and their matching + // ExtensionThrottleEntry. + UrlEntryMap url_entries_; + + // This keeps track of how many requests have been made. Used with + // GarbageCollectEntries. + unsigned int requests_since_last_gc_; + + // Valid after construction. + GURL::Replacements url_id_replacements_; + + // Certain tests do not obey the net component's threading policy, so we + // keep track of whether we're being used by tests, and turn off certain + // checks. + // + // TODO(joi): See if we can fix the offending unit tests and remove this + // workaround. + bool enable_thread_checks_; + + // Initially false, switches to true once we have logged because of back-off + // being disabled for localhost. + bool logged_for_localhost_disabled_; + + // net::NetLog to use, if configured. + net::BoundNetLog net_log_; + + // Valid once we've registered for network notifications. + base::PlatformThreadId registered_from_thread_; + + bool ignore_user_gesture_load_flag_for_tests_; + + DISALLOW_COPY_AND_ASSIGN(ExtensionThrottleManager); +}; + +} // namespace extensions + +#endif // EXTENSIONS_BROWSER_EXTENSION_THROTTLE_MANAGER_H_ diff --git a/extensions/browser/extension_throttle_simulation_unittest.cc b/extensions/browser/extension_throttle_simulation_unittest.cc new file mode 100644 index 0000000..be55453 --- /dev/null +++ b/extensions/browser/extension_throttle_simulation_unittest.cc @@ -0,0 +1,743 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// The tests in this file attempt to verify the following through simulation: +// a) That a server experiencing overload will actually benefit from the +// anti-DDoS throttling logic, i.e. that its traffic spike will subside +// and be distributed over a longer period of time; +// b) That "well-behaved" clients of a server under DDoS attack actually +// benefit from the anti-DDoS throttling logic; and +// c) That the approximate increase in "perceived downtime" introduced by +// anti-DDoS throttling for various different actual downtimes is what +// we expect it to be. + +#include <cmath> +#include <limits> +#include <vector> + +#include "base/environment.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/scoped_vector.h" +#include "base/message_loop/message_loop.h" +#include "base/rand_util.h" +#include "base/time/time.h" +#include "extensions/browser/extension_throttle_manager.h" +#include "extensions/browser/extension_throttle_test_support.h" +#include "net/base/request_priority.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +using base::TimeDelta; +using base::TimeTicks; +using net::BackoffEntry; +using net::TestURLRequestContext; +using net::URLRequest; +using net::URLRequestContext; + +namespace extensions { +namespace { + +// Set this variable in your environment if you want to see verbose results +// of the simulation tests. +const char kShowSimulationVariableName[] = "SHOW_SIMULATION_RESULTS"; + +// Prints output only if a given environment variable is set. We use this +// to not print any output for human evaluation when the test is run without +// supervision. +void VerboseOut(const char* format, ...) { + static bool have_checked_environment = false; + static bool should_print = false; + if (!have_checked_environment) { + have_checked_environment = true; + scoped_ptr<base::Environment> env(base::Environment::Create()); + if (env->HasVar(kShowSimulationVariableName)) + should_print = true; + } + + if (should_print) { + va_list arglist; + va_start(arglist, format); + vprintf(format, arglist); + va_end(arglist); + } +} + +// A simple two-phase discrete time simulation. Actors are added in the order +// they should take action at every tick of the clock. Ticks of the clock +// are two-phase: +// - Phase 1 advances every actor's time to a new absolute time. +// - Phase 2 asks each actor to perform their action. +class DiscreteTimeSimulation { + public: + class Actor { + public: + virtual ~Actor() {} + virtual void AdvanceTime(const TimeTicks& absolute_time) = 0; + virtual void PerformAction() = 0; + }; + + DiscreteTimeSimulation() {} + + // Adds an |actor| to the simulation. The client of the simulation maintains + // ownership of |actor| and must ensure its lifetime exceeds that of the + // simulation. Actors should be added in the order you wish for them to + // act at each tick of the simulation. + void AddActor(Actor* actor) { actors_.push_back(actor); } + + // Runs the simulation for, pretending |time_between_ticks| passes from one + // tick to the next. The start time will be the current real time. The + // simulation will stop when the simulated duration is equal to or greater + // than |maximum_simulated_duration|. + void RunSimulation(const TimeDelta& maximum_simulated_duration, + const TimeDelta& time_between_ticks) { + TimeTicks start_time = TimeTicks(); + TimeTicks now = start_time; + while ((now - start_time) <= maximum_simulated_duration) { + for (std::vector<Actor*>::iterator it = actors_.begin(); + it != actors_.end(); ++it) { + (*it)->AdvanceTime(now); + } + + for (std::vector<Actor*>::iterator it = actors_.begin(); + it != actors_.end(); ++it) { + (*it)->PerformAction(); + } + + now += time_between_ticks; + } + } + + private: + std::vector<Actor*> actors_; + + DISALLOW_COPY_AND_ASSIGN(DiscreteTimeSimulation); +}; + +// Represents a web server in a simulation of a server under attack by +// a lot of clients. Must be added to the simulation's list of actors +// after all |Requester| objects. +class Server : public DiscreteTimeSimulation::Actor { + public: + Server(int max_queries_per_tick, double request_drop_ratio) + : max_queries_per_tick_(max_queries_per_tick), + request_drop_ratio_(request_drop_ratio), + num_overloaded_ticks_remaining_(0), + num_current_tick_queries_(0), + num_overloaded_ticks_(0), + max_experienced_queries_per_tick_(0), + mock_request_( + context_.CreateRequest(GURL(), net::DEFAULT_PRIORITY, NULL)) {} + + void SetDowntime(const TimeTicks& start_time, const TimeDelta& duration) { + start_downtime_ = start_time; + end_downtime_ = start_time + duration; + } + + void AdvanceTime(const TimeTicks& absolute_time) override { + now_ = absolute_time; + } + + void PerformAction() override { + // We are inserted at the end of the actor's list, so all Requester + // instances have already done their bit. + if (num_current_tick_queries_ > max_experienced_queries_per_tick_) + max_experienced_queries_per_tick_ = num_current_tick_queries_; + + if (num_current_tick_queries_ > max_queries_per_tick_) { + // We pretend the server fails for the next several ticks after it + // gets overloaded. + num_overloaded_ticks_remaining_ = 5; + ++num_overloaded_ticks_; + } else if (num_overloaded_ticks_remaining_ > 0) { + --num_overloaded_ticks_remaining_; + } + + requests_per_tick_.push_back(num_current_tick_queries_); + num_current_tick_queries_ = 0; + } + + // This is called by Requester. It returns the response code from + // the server. + int HandleRequest() { + ++num_current_tick_queries_; + if (!start_downtime_.is_null() && start_downtime_ < now_ && + now_ < end_downtime_) { + // For the simulation measuring the increase in perceived + // downtime, it might be interesting to count separately the + // queries seen by the server (assuming a front-end reverse proxy + // is what actually serves up the 503s in this case) so that we could + // visualize the traffic spike seen by the server when it comes up, + // which would in many situations be ameliorated by the anti-DDoS + // throttling. + return 503; + } + + if ((num_overloaded_ticks_remaining_ > 0 || + num_current_tick_queries_ > max_queries_per_tick_) && + base::RandDouble() < request_drop_ratio_) { + return 503; + } + + return 200; + } + + int num_overloaded_ticks() const { return num_overloaded_ticks_; } + + int max_experienced_queries_per_tick() const { + return max_experienced_queries_per_tick_; + } + + const URLRequest& mock_request() const { return *mock_request_.get(); } + + std::string VisualizeASCII(int terminal_width) { + // Account for | characters we place at left of graph. + terminal_width -= 1; + + VerboseOut("Overloaded for %d of %d ticks.\n", num_overloaded_ticks_, + requests_per_tick_.size()); + VerboseOut("Got maximum of %d requests in a tick.\n\n", + max_experienced_queries_per_tick_); + + VerboseOut("Traffic graph:\n\n"); + + // Printing the graph like this is a bit overkill, but was very useful + // while developing the various simulations to see if they were testing + // the corner cases we want to simulate. + + // Find the smallest number of whole ticks we need to group into a + // column that will let all ticks fit into the column width we have. + int num_ticks = requests_per_tick_.size(); + double ticks_per_column_exact = + static_cast<double>(num_ticks) / static_cast<double>(terminal_width); + int ticks_per_column = std::ceil(ticks_per_column_exact); + DCHECK_GE(ticks_per_column * terminal_width, num_ticks); + + // Sum up the column values. + int num_columns = num_ticks / ticks_per_column; + if (num_ticks % ticks_per_column) + ++num_columns; + DCHECK_LE(num_columns, terminal_width); + scoped_ptr<int[]> columns(new int[num_columns]); + for (int tx = 0; tx < num_ticks; ++tx) { + int cx = tx / ticks_per_column; + if (tx % ticks_per_column == 0) + columns[cx] = 0; + columns[cx] += requests_per_tick_[tx]; + } + + // Find the lowest integer divisor that will let the column values + // be represented in a graph of maximum height 50. + int max_value = 0; + for (int cx = 0; cx < num_columns; ++cx) + max_value = std::max(max_value, columns[cx]); + const int kNumRows = 50; + double row_divisor_exact = max_value / static_cast<double>(kNumRows); + int row_divisor = std::ceil(row_divisor_exact); + DCHECK_GE(row_divisor * kNumRows, max_value); + + // To show the overload line, we calculate the appropriate value. + int overload_value = max_queries_per_tick_ * ticks_per_column; + + // When num_ticks is not a whole multiple of ticks_per_column, the last + // column includes fewer ticks than the others. In this case, don't + // print it so that we don't show an inconsistent value. + int num_printed_columns = num_columns; + if (num_ticks % ticks_per_column) + --num_printed_columns; + + // This is a top-to-bottom traversal of rows, left-to-right per row. + std::string output; + for (int rx = 0; rx < kNumRows; ++rx) { + int range_min = (kNumRows - rx) * row_divisor; + int range_max = range_min + row_divisor; + if (range_min == 0) + range_min = -1; // Make 0 values fit in the bottom range. + output.append("|"); + for (int cx = 0; cx < num_printed_columns; ++cx) { + char block = ' '; + // Show the overload line. + if (range_min < overload_value && overload_value <= range_max) + block = '-'; + + // Preferentially, show the graph line. + if (range_min < columns[cx] && columns[cx] <= range_max) + block = '#'; + + output.append(1, block); + } + output.append("\n"); + } + output.append("|"); + output.append(num_printed_columns, '='); + + return output; + } + + const URLRequestContext& context() const { return context_; } + + private: + TimeTicks now_; + TimeTicks start_downtime_; // Can be 0 to say "no downtime". + TimeTicks end_downtime_; + const int max_queries_per_tick_; + const double request_drop_ratio_; // Ratio of requests to 503 when failing. + int num_overloaded_ticks_remaining_; + int num_current_tick_queries_; + int num_overloaded_ticks_; + int max_experienced_queries_per_tick_; + std::vector<int> requests_per_tick_; + + TestURLRequestContext context_; + scoped_ptr<URLRequest> mock_request_; + + DISALLOW_COPY_AND_ASSIGN(Server); +}; + +// Mock throttler entry used by Requester class. +class MockExtensionThrottleEntry : public ExtensionThrottleEntry { + public: + explicit MockExtensionThrottleEntry(ExtensionThrottleManager* manager) + : ExtensionThrottleEntry(manager, std::string()), + backoff_entry_(&backoff_policy_, &fake_clock_) {} + + const BackoffEntry* GetBackoffEntry() const override { + return &backoff_entry_; + } + + BackoffEntry* GetBackoffEntry() override { return &backoff_entry_; } + + TimeTicks ImplGetTimeNow() const override { return fake_clock_.NowTicks(); } + + void SetFakeNow(const TimeTicks& fake_time) { + fake_clock_.set_now(fake_time); + } + + protected: + ~MockExtensionThrottleEntry() override {} + + private: + mutable TestTickClock fake_clock_; + BackoffEntry backoff_entry_; +}; + +// Registry of results for a class of |Requester| objects (e.g. attackers vs. +// regular clients). +class RequesterResults { + public: + RequesterResults() + : num_attempts_(0), num_successful_(0), num_failed_(0), num_blocked_(0) {} + + void AddSuccess() { + ++num_attempts_; + ++num_successful_; + } + + void AddFailure() { + ++num_attempts_; + ++num_failed_; + } + + void AddBlocked() { + ++num_attempts_; + ++num_blocked_; + } + + int num_attempts() const { return num_attempts_; } + int num_successful() const { return num_successful_; } + int num_failed() const { return num_failed_; } + int num_blocked() const { return num_blocked_; } + + double GetBlockedRatio() { + DCHECK(num_attempts_); + return static_cast<double>(num_blocked_) / + static_cast<double>(num_attempts_); + } + + double GetSuccessRatio() { + DCHECK(num_attempts_); + return static_cast<double>(num_successful_) / + static_cast<double>(num_attempts_); + } + + void PrintResults(const char* class_description) { + if (num_attempts_ == 0) { + VerboseOut("No data for %s\n", class_description); + return; + } + + VerboseOut("Requester results for %s\n", class_description); + VerboseOut(" %d attempts\n", num_attempts_); + VerboseOut(" %d successes\n", num_successful_); + VerboseOut(" %d 5xx responses\n", num_failed_); + VerboseOut(" %d requests blocked\n", num_blocked_); + VerboseOut(" %.2f success ratio\n", GetSuccessRatio()); + VerboseOut(" %.2f blocked ratio\n", GetBlockedRatio()); + VerboseOut("\n"); + } + + private: + int num_attempts_; + int num_successful_; + int num_failed_; + int num_blocked_; +}; + +// Represents an Requester in a simulated DDoS situation, that periodically +// requests a specific resource. +class Requester : public DiscreteTimeSimulation::Actor { + public: + Requester(MockExtensionThrottleEntry* throttler_entry, + const TimeDelta& time_between_requests, + Server* server, + RequesterResults* results) + : throttler_entry_(throttler_entry), + time_between_requests_(time_between_requests), + last_attempt_was_failure_(false), + server_(server), + results_(results) { + DCHECK(server_); + } + + void AdvanceTime(const TimeTicks& absolute_time) override { + if (time_of_last_success_.is_null()) + time_of_last_success_ = absolute_time; + + throttler_entry_->SetFakeNow(absolute_time); + } + + void PerformAction() override { + TimeDelta effective_delay = time_between_requests_; + TimeDelta current_jitter = TimeDelta::FromMilliseconds( + request_jitter_.InMilliseconds() * base::RandDouble()); + if (base::RandInt(0, 1)) { + effective_delay -= current_jitter; + } else { + effective_delay += current_jitter; + } + + if (throttler_entry_->ImplGetTimeNow() - time_of_last_attempt_ > + effective_delay) { + if (!throttler_entry_->ShouldRejectRequest(server_->mock_request())) { + int status_code = server_->HandleRequest(); + throttler_entry_->UpdateWithResponse(status_code); + + if (status_code == 200) { + if (results_) + results_->AddSuccess(); + + if (last_attempt_was_failure_) { + last_downtime_duration_ = + throttler_entry_->ImplGetTimeNow() - time_of_last_success_; + } + + time_of_last_success_ = throttler_entry_->ImplGetTimeNow(); + last_attempt_was_failure_ = false; + } else { + if (results_) + results_->AddFailure(); + last_attempt_was_failure_ = true; + } + } else { + if (results_) + results_->AddBlocked(); + last_attempt_was_failure_ = true; + } + + time_of_last_attempt_ = throttler_entry_->ImplGetTimeNow(); + } + } + + // Adds a delay until the first request, equal to a uniformly distributed + // value between now and now + max_delay. + void SetStartupJitter(const TimeDelta& max_delay) { + int delay_ms = base::RandInt(0, max_delay.InMilliseconds()); + time_of_last_attempt_ = TimeTicks() + + TimeDelta::FromMilliseconds(delay_ms) - + time_between_requests_; + } + + void SetRequestJitter(const TimeDelta& request_jitter) { + request_jitter_ = request_jitter; + } + + TimeDelta last_downtime_duration() const { return last_downtime_duration_; } + + private: + scoped_refptr<MockExtensionThrottleEntry> throttler_entry_; + const TimeDelta time_between_requests_; + TimeDelta request_jitter_; + TimeTicks time_of_last_attempt_; + TimeTicks time_of_last_success_; + bool last_attempt_was_failure_; + TimeDelta last_downtime_duration_; + Server* const server_; + RequesterResults* const results_; // May be NULL. + + DISALLOW_COPY_AND_ASSIGN(Requester); +}; + +void SimulateAttack(Server* server, + RequesterResults* attacker_results, + RequesterResults* client_results, + bool enable_throttling) { + const size_t kNumAttackers = 50; + const size_t kNumClients = 50; + DiscreteTimeSimulation simulation; + ExtensionThrottleManager manager; + ScopedVector<Requester> requesters; + for (size_t i = 0; i < kNumAttackers; ++i) { + // Use a tiny time_between_requests so the attackers will ping the + // server at every tick of the simulation. + scoped_refptr<MockExtensionThrottleEntry> throttler_entry( + new MockExtensionThrottleEntry(&manager)); + if (!enable_throttling) + throttler_entry->DisableBackoffThrottling(); + + Requester* attacker = + new Requester(throttler_entry.get(), TimeDelta::FromMilliseconds(1), + server, attacker_results); + attacker->SetStartupJitter(TimeDelta::FromSeconds(120)); + requesters.push_back(attacker); + simulation.AddActor(attacker); + } + for (size_t i = 0; i < kNumClients; ++i) { + // Normal clients only make requests every 2 minutes, plus/minus 1 minute. + scoped_refptr<MockExtensionThrottleEntry> throttler_entry( + new MockExtensionThrottleEntry(&manager)); + if (!enable_throttling) + throttler_entry->DisableBackoffThrottling(); + + Requester* client = + new Requester(throttler_entry.get(), TimeDelta::FromMinutes(2), server, + client_results); + client->SetStartupJitter(TimeDelta::FromSeconds(120)); + client->SetRequestJitter(TimeDelta::FromMinutes(1)); + requesters.push_back(client); + simulation.AddActor(client); + } + simulation.AddActor(server); + + simulation.RunSimulation(TimeDelta::FromMinutes(6), + TimeDelta::FromSeconds(1)); +} + +TEST(URLRequestThrottlerSimulation, HelpsInAttack) { + base::MessageLoopForIO message_loop; + Server unprotected_server(30, 1.0); + RequesterResults unprotected_attacker_results; + RequesterResults unprotected_client_results; + Server protected_server(30, 1.0); + RequesterResults protected_attacker_results; + RequesterResults protected_client_results; + SimulateAttack(&unprotected_server, &unprotected_attacker_results, + &unprotected_client_results, false); + SimulateAttack(&protected_server, &protected_attacker_results, + &protected_client_results, true); + + // These assert that the DDoS protection actually benefits the + // server. Manual inspection of the traffic graphs will show this + // even more clearly. + EXPECT_GT(unprotected_server.num_overloaded_ticks(), + protected_server.num_overloaded_ticks()); + EXPECT_GT(unprotected_server.max_experienced_queries_per_tick(), + protected_server.max_experienced_queries_per_tick()); + + // These assert that the DDoS protection actually benefits non-malicious + // (and non-degenerate/accidentally DDoSing) users. + EXPECT_LT(protected_client_results.GetBlockedRatio(), + protected_attacker_results.GetBlockedRatio()); + EXPECT_GT(protected_client_results.GetSuccessRatio(), + unprotected_client_results.GetSuccessRatio()); + + // The rest is just for optional manual evaluation of the results; + // in particular the traffic pattern is interesting. + + VerboseOut("\nUnprotected server's results:\n\n"); + VerboseOut(unprotected_server.VisualizeASCII(132).c_str()); + VerboseOut("\n\n"); + VerboseOut("Protected server's results:\n\n"); + VerboseOut(protected_server.VisualizeASCII(132).c_str()); + VerboseOut("\n\n"); + + unprotected_attacker_results.PrintResults( + "attackers attacking unprotected server."); + unprotected_client_results.PrintResults( + "normal clients making requests to unprotected server."); + protected_attacker_results.PrintResults( + "attackers attacking protected server."); + protected_client_results.PrintResults( + "normal clients making requests to protected server."); +} + +// Returns the downtime perceived by the client, as a ratio of the +// actual downtime. +double SimulateDowntime(const TimeDelta& duration, + const TimeDelta& average_client_interval, + bool enable_throttling) { + TimeDelta time_between_ticks = duration / 200; + TimeTicks start_downtime = TimeTicks() + (duration / 2); + + // A server that never rejects requests, but will go down for maintenance. + Server server(std::numeric_limits<int>::max(), 1.0); + server.SetDowntime(start_downtime, duration); + + ExtensionThrottleManager manager; + scoped_refptr<MockExtensionThrottleEntry> throttler_entry( + new MockExtensionThrottleEntry(&manager)); + if (!enable_throttling) + throttler_entry->DisableBackoffThrottling(); + + Requester requester(throttler_entry.get(), average_client_interval, &server, + NULL); + requester.SetStartupJitter(duration / 3); + requester.SetRequestJitter(average_client_interval); + + DiscreteTimeSimulation simulation; + simulation.AddActor(&requester); + simulation.AddActor(&server); + + simulation.RunSimulation(duration * 2, time_between_ticks); + + return static_cast<double>( + requester.last_downtime_duration().InMilliseconds()) / + static_cast<double>(duration.InMilliseconds()); +} + +TEST(URLRequestThrottlerSimulation, PerceivedDowntimeRatio) { + base::MessageLoopForIO message_loop; + struct Stats { + // Expected interval that we expect the ratio of downtime when anti-DDoS + // is enabled and downtime when anti-DDoS is not enabled to fall within. + // + // The expected interval depends on two things: The exponential back-off + // policy encoded in ExtensionThrottleEntry, and the test or set of + // tests that the Stats object is tracking (e.g. a test where the client + // retries very rapidly on a very long downtime will tend to increase the + // number). + // + // To determine an appropriate new interval when parameters have changed, + // run the test a few times (you may have to Ctrl-C out of it after a few + // seconds) and choose an interval that the test converges quickly and + // reliably to. Then set the new interval, and run the test e.g. 20 times + // in succession to make sure it never takes an obscenely long time to + // converge to this interval. + double expected_min_increase; + double expected_max_increase; + + size_t num_runs; + double total_ratio_unprotected; + double total_ratio_protected; + + bool DidConverge(double* increase_ratio_out) { + double unprotected_ratio = total_ratio_unprotected / num_runs; + double protected_ratio = total_ratio_protected / num_runs; + double increase_ratio = protected_ratio / unprotected_ratio; + if (increase_ratio_out) + *increase_ratio_out = increase_ratio; + return expected_min_increase <= increase_ratio && + increase_ratio <= expected_max_increase; + } + + void ReportTrialResult(double increase_ratio) { + VerboseOut( + " Perceived downtime with throttling is %.4f times without.\n", + increase_ratio); + VerboseOut(" Test result after %d trials.\n", num_runs); + } + }; + + Stats global_stats = {1.08, 1.15}; + + struct Trial { + TimeDelta duration; + TimeDelta average_client_interval; + Stats stats; + + void PrintTrialDescription() { + double duration_minutes = + static_cast<double>(duration.InSeconds()) / 60.0; + double interval_minutes = + static_cast<double>(average_client_interval.InSeconds()) / 60.0; + VerboseOut("Trial with %.2f min downtime, avg. interval %.2f min.\n", + duration_minutes, interval_minutes); + } + }; + + // We don't set or check expected ratio intervals on individual + // experiments as this might make the test too fragile, but we + // print them out at the end for manual evaluation (we want to be + // able to make claims about the expected ratios depending on the + // type of behavior of the client and the downtime, e.g. the difference + // in behavior between a client making requests every few minutes vs. + // one that makes a request every 15 seconds). + Trial trials[] = { + {TimeDelta::FromSeconds(10), TimeDelta::FromSeconds(3)}, + {TimeDelta::FromSeconds(30), TimeDelta::FromSeconds(7)}, + {TimeDelta::FromMinutes(5), TimeDelta::FromSeconds(30)}, + {TimeDelta::FromMinutes(10), TimeDelta::FromSeconds(20)}, + {TimeDelta::FromMinutes(20), TimeDelta::FromSeconds(15)}, + {TimeDelta::FromMinutes(20), TimeDelta::FromSeconds(50)}, + {TimeDelta::FromMinutes(30), TimeDelta::FromMinutes(2)}, + {TimeDelta::FromMinutes(30), TimeDelta::FromMinutes(5)}, + {TimeDelta::FromMinutes(40), TimeDelta::FromMinutes(7)}, + {TimeDelta::FromMinutes(40), TimeDelta::FromMinutes(2)}, + {TimeDelta::FromMinutes(40), TimeDelta::FromSeconds(15)}, + {TimeDelta::FromMinutes(60), TimeDelta::FromMinutes(7)}, + {TimeDelta::FromMinutes(60), TimeDelta::FromMinutes(2)}, + {TimeDelta::FromMinutes(60), TimeDelta::FromSeconds(15)}, + {TimeDelta::FromMinutes(80), TimeDelta::FromMinutes(20)}, + {TimeDelta::FromMinutes(80), TimeDelta::FromMinutes(3)}, + {TimeDelta::FromMinutes(80), TimeDelta::FromSeconds(15)}, + + // Most brutal? + {TimeDelta::FromMinutes(45), TimeDelta::FromMilliseconds(500)}, + }; + + // If things don't converge by the time we've done 100K trials, then + // clearly one or more of the expected intervals are wrong. + while (global_stats.num_runs < 100000) { + for (size_t i = 0; i < arraysize(trials); ++i) { + ++global_stats.num_runs; + ++trials[i].stats.num_runs; + double ratio_unprotected = SimulateDowntime( + trials[i].duration, trials[i].average_client_interval, false); + double ratio_protected = SimulateDowntime( + trials[i].duration, trials[i].average_client_interval, true); + global_stats.total_ratio_unprotected += ratio_unprotected; + global_stats.total_ratio_protected += ratio_protected; + trials[i].stats.total_ratio_unprotected += ratio_unprotected; + trials[i].stats.total_ratio_protected += ratio_protected; + } + + double increase_ratio; + if (global_stats.DidConverge(&increase_ratio)) + break; + + if (global_stats.num_runs > 200) { + VerboseOut("Test has not yet converged on expected interval.\n"); + global_stats.ReportTrialResult(increase_ratio); + } + } + + double average_increase_ratio; + EXPECT_TRUE(global_stats.DidConverge(&average_increase_ratio)); + + // Print individual trial results for optional manual evaluation. + double max_increase_ratio = 0.0; + for (size_t i = 0; i < arraysize(trials); ++i) { + double increase_ratio; + trials[i].stats.DidConverge(&increase_ratio); + max_increase_ratio = std::max(max_increase_ratio, increase_ratio); + trials[i].PrintTrialDescription(); + trials[i].stats.ReportTrialResult(increase_ratio); + } + + VerboseOut("Average increase ratio was %.4f\n", average_increase_ratio); + VerboseOut("Maximum increase ratio was %.4f\n", max_increase_ratio); +} + +} // namespace +} // namespace extensions diff --git a/extensions/browser/extension_throttle_test_support.cc b/extensions/browser/extension_throttle_test_support.cc new file mode 100644 index 0000000..d5a5228 --- /dev/null +++ b/extensions/browser/extension_throttle_test_support.cc @@ -0,0 +1,22 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "extensions/browser/extension_throttle_test_support.h" + +namespace extensions { + +TestTickClock::TestTickClock() { +} + +TestTickClock::TestTickClock(base::TimeTicks now) : now_ticks_(now) { +} + +TestTickClock::~TestTickClock() { +} + +base::TimeTicks TestTickClock::NowTicks() { + return now_ticks_; +} + +} // namespace extensions diff --git a/extensions/browser/extension_throttle_test_support.h b/extensions/browser/extension_throttle_test_support.h new file mode 100644 index 0000000..a6fde03 --- /dev/null +++ b/extensions/browser/extension_throttle_test_support.h @@ -0,0 +1,33 @@ +// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef EXTENSIONS_BROWSER_EXTENSION_THROTTLE_TEST_SUPPORT_H_ +#define EXTENSIONS_BROWSER_EXTENSION_THROTTLE_TEST_SUPPORT_H_ + +#include <string> + +#include "base/macros.h" +#include "base/time/tick_clock.h" +#include "base/time/time.h" +#include "net/base/backoff_entry.h" + +namespace extensions { + +class TestTickClock : public base::TickClock { + public: + TestTickClock(); + explicit TestTickClock(base::TimeTicks now); + ~TestTickClock() override; + + base::TimeTicks NowTicks() override; + void set_now(base::TimeTicks now) { now_ticks_ = now; } + + private: + base::TimeTicks now_ticks_; + DISALLOW_COPY_AND_ASSIGN(TestTickClock); +}; + +} // namespace extensions + +#endif // EXTENSIONS_BROWSER_EXTENSION_THROTTLE_TEST_SUPPORT_H_ diff --git a/extensions/browser/extension_throttle_unittest.cc b/extensions/browser/extension_throttle_unittest.cc new file mode 100644 index 0000000..480700d --- /dev/null +++ b/extensions/browser/extension_throttle_unittest.cc @@ -0,0 +1,469 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/memory/scoped_ptr.h" +#include "base/message_loop/message_loop.h" +#include "base/pickle.h" +#include "base/stl_util.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/stringprintf.h" +#include "base/time/time.h" +#include "extensions/browser/extension_throttle_entry.h" +#include "extensions/browser/extension_throttle_manager.h" +#include "extensions/browser/extension_throttle_test_support.h" +#include "net/base/load_flags.h" +#include "net/base/request_priority.h" +#include "net/base/test_completion_callback.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context.h" +#include "net/url_request/url_request_test_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +using base::TimeDelta; +using base::TimeTicks; +using net::BackoffEntry; +using net::NetworkChangeNotifier; +using net::TestNetworkDelegate; +using net::TestURLRequestContext; +using net::URLRequest; +using net::URLRequestContext; + +namespace extensions { + +namespace { + +class MockExtensionThrottleEntry : public ExtensionThrottleEntry { + public: + explicit MockExtensionThrottleEntry(ExtensionThrottleManager* manager) + : ExtensionThrottleEntry(manager, std::string()), + backoff_entry_(&backoff_policy_, &fake_clock_) { + InitPolicy(); + } + MockExtensionThrottleEntry(ExtensionThrottleManager* manager, + const TimeTicks& exponential_backoff_release_time, + const TimeTicks& sliding_window_release_time, + const TimeTicks& fake_now) + : ExtensionThrottleEntry(manager, std::string()), + fake_clock_(fake_now), + backoff_entry_(&backoff_policy_, &fake_clock_) { + InitPolicy(); + + set_exponential_backoff_release_time(exponential_backoff_release_time); + set_sliding_window_release_time(sliding_window_release_time); + } + + void InitPolicy() { + // Some tests become flaky if we have jitter. + backoff_policy_.jitter_factor = 0.0; + + // This lets us avoid having to make multiple failures initially (this + // logic is already tested in the BackoffEntry unit tests). + backoff_policy_.num_errors_to_ignore = 0; + } + + const BackoffEntry* GetBackoffEntry() const override { + return &backoff_entry_; + } + + BackoffEntry* GetBackoffEntry() override { return &backoff_entry_; } + + static bool ExplicitUserRequest(int load_flags) { + return ExtensionThrottleEntry::ExplicitUserRequest(load_flags); + } + + void ResetToBlank(const TimeTicks& time_now) { + fake_clock_.set_now(time_now); + + GetBackoffEntry()->Reset(); + set_sliding_window_release_time(time_now); + } + + // Overridden for tests. + TimeTicks ImplGetTimeNow() const override { return fake_clock_.NowTicks(); } + + void set_fake_now(const TimeTicks& now) { fake_clock_.set_now(now); } + + void set_exponential_backoff_release_time(const TimeTicks& release_time) { + GetBackoffEntry()->SetCustomReleaseTime(release_time); + } + + TimeTicks sliding_window_release_time() const { + return ExtensionThrottleEntry::sliding_window_release_time(); + } + + void set_sliding_window_release_time(const TimeTicks& release_time) { + ExtensionThrottleEntry::set_sliding_window_release_time(release_time); + } + + protected: + ~MockExtensionThrottleEntry() override {} + + private: + mutable TestTickClock fake_clock_; + BackoffEntry backoff_entry_; +}; + +class MockExtensionThrottleManager : public ExtensionThrottleManager { + public: + MockExtensionThrottleManager() : create_entry_index_(0) {} + + // Method to process the URL using ExtensionThrottleManager protected + // method. + std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); } + + // Method to use the garbage collecting method of ExtensionThrottleManager. + void DoGarbageCollectEntries() { GarbageCollectEntries(); } + + // Returns the number of entries in the map. + int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); } + + void CreateEntry(bool is_outdated) { + TimeTicks time = TimeTicks::Now(); + if (is_outdated) { + time -= TimeDelta::FromMilliseconds( + MockExtensionThrottleEntry::kDefaultEntryLifetimeMs + 1000); + } + std::string fake_url_string("http://www.fakeurl.com/"); + fake_url_string.append(base::IntToString(create_entry_index_++)); + GURL fake_url(fake_url_string); + OverrideEntryForTests( + fake_url, new MockExtensionThrottleEntry(this, time, TimeTicks::Now(), + TimeTicks::Now())); + } + + private: + int create_entry_index_; +}; + +struct TimeAndBool { + TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) { + time = time_value; + result = expected; + line = line_num; + } + TimeTicks time; + bool result; + int line; +}; + +struct GurlAndString { + GurlAndString(const GURL& url_value, + const std::string& expected, + int line_num) { + url = url_value; + result = expected; + line = line_num; + } + GURL url; + std::string result; + int line; +}; + +} // namespace + +class ExtensionThrottleEntryTest : public testing::Test { + protected: + ExtensionThrottleEntryTest() + : request_(context_.CreateRequest(GURL(), net::DEFAULT_PRIORITY, NULL)) {} + + void SetUp() override; + + TimeTicks now_; + MockExtensionThrottleManager manager_; // Dummy object, not used. + scoped_refptr<MockExtensionThrottleEntry> entry_; + base::MessageLoopForIO message_loop_; + + TestURLRequestContext context_; + scoped_ptr<URLRequest> request_; +}; + +void ExtensionThrottleEntryTest::SetUp() { + request_->SetLoadFlags(0); + + now_ = TimeTicks::Now(); + entry_ = new MockExtensionThrottleEntry(&manager_); + entry_->ResetToBlank(now_); +} + +std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) { + return out << time.ToInternalValue(); +} + +TEST_F(ExtensionThrottleEntryTest, CanThrottleRequest) { + entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow() + + TimeDelta::FromMilliseconds(1)); + + EXPECT_TRUE(entry_->ShouldRejectRequest(*request_)); +} + +TEST_F(ExtensionThrottleEntryTest, InterfaceDuringExponentialBackoff) { + entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow() + + TimeDelta::FromMilliseconds(1)); + EXPECT_TRUE(entry_->ShouldRejectRequest(*request_)); + + // Also end-to-end test the load flags exceptions. + request_->SetLoadFlags(net::LOAD_MAYBE_USER_GESTURE); + EXPECT_FALSE(entry_->ShouldRejectRequest(*request_)); +} + +TEST_F(ExtensionThrottleEntryTest, InterfaceNotDuringExponentialBackoff) { + entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow()); + EXPECT_FALSE(entry_->ShouldRejectRequest(*request_)); + entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow() - + TimeDelta::FromMilliseconds(1)); + EXPECT_FALSE(entry_->ShouldRejectRequest(*request_)); +} + +TEST_F(ExtensionThrottleEntryTest, InterfaceUpdateFailure) { + entry_->UpdateWithResponse(503); + EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), + entry_->ImplGetTimeNow()) + << "A failure should increase the release_time"; +} + +TEST_F(ExtensionThrottleEntryTest, InterfaceUpdateSuccess) { + entry_->UpdateWithResponse(200); + EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), + entry_->ImplGetTimeNow()) + << "A success should not add any delay"; +} + +TEST_F(ExtensionThrottleEntryTest, InterfaceUpdateSuccessThenFailure) { + entry_->UpdateWithResponse(200); + entry_->UpdateWithResponse(503); + EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), + entry_->ImplGetTimeNow()) + << "This scenario should add delay"; + entry_->UpdateWithResponse(200); +} + +TEST_F(ExtensionThrottleEntryTest, IsEntryReallyOutdated) { + TimeDelta lifetime = TimeDelta::FromMilliseconds( + MockExtensionThrottleEntry::kDefaultEntryLifetimeMs); + const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5); + + TimeAndBool test_values[] = { + TimeAndBool(now_, false, __LINE__), + TimeAndBool(now_ - kFiveMs, false, __LINE__), + TimeAndBool(now_ + kFiveMs, false, __LINE__), + TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__), + TimeAndBool(now_ - lifetime, true, __LINE__), + TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)}; + + for (unsigned int i = 0; i < arraysize(test_values); ++i) { + entry_->set_exponential_backoff_release_time(test_values[i].time); + EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) + << "Test case #" << i << " line " << test_values[i].line << " failed"; + } +} + +TEST_F(ExtensionThrottleEntryTest, MaxAllowedBackoff) { + for (int i = 0; i < 30; ++i) { + entry_->UpdateWithResponse(503); + } + + TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_; + EXPECT_EQ(delay.InMilliseconds(), + MockExtensionThrottleEntry::kDefaultMaximumBackoffMs); +} + +TEST_F(ExtensionThrottleEntryTest, MalformedContent) { + for (int i = 0; i < 5; ++i) + entry_->UpdateWithResponse(503); + + TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime(); + + // Inform the entry that a response body was malformed, which is supposed to + // increase the back-off time. Note that we also submit a successful + // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that + // is what happens in practice (if a body is received, then a non-500 + // response must also have been received). + entry_->ReceivedContentWasMalformed(200); + entry_->UpdateWithResponse(200); + EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures); +} + +TEST_F(ExtensionThrottleEntryTest, SlidingWindow) { + int max_send = ExtensionThrottleEntry::kDefaultMaxSendThreshold; + int sliding_window = ExtensionThrottleEntry::kDefaultSlidingWindowPeriodMs; + + TimeTicks time_1 = entry_->ImplGetTimeNow() + + TimeDelta::FromMilliseconds(sliding_window / 3); + TimeTicks time_2 = entry_->ImplGetTimeNow() + + TimeDelta::FromMilliseconds(2 * sliding_window / 3); + TimeTicks time_3 = + entry_->ImplGetTimeNow() + TimeDelta::FromMilliseconds(sliding_window); + TimeTicks time_4 = + entry_->ImplGetTimeNow() + + TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3); + + entry_->set_exponential_backoff_release_time(time_1); + + for (int i = 0; i < max_send / 2; ++i) { + EXPECT_EQ(2 * sliding_window / 3, + entry_->ReserveSendingTimeForNextRequest(time_2)); + } + EXPECT_EQ(time_2, entry_->sliding_window_release_time()); + + entry_->set_fake_now(time_3); + + for (int i = 0; i < (max_send + 1) / 2; ++i) + EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks())); + + EXPECT_EQ(time_4, entry_->sliding_window_release_time()); +} + +TEST_F(ExtensionThrottleEntryTest, ExplicitUserRequest) { + ASSERT_FALSE(MockExtensionThrottleEntry::ExplicitUserRequest(0)); + ASSERT_TRUE(MockExtensionThrottleEntry::ExplicitUserRequest( + net::LOAD_MAYBE_USER_GESTURE)); + ASSERT_FALSE(MockExtensionThrottleEntry::ExplicitUserRequest( + ~net::LOAD_MAYBE_USER_GESTURE)); +} + +class ExtensionThrottleManagerTest : public testing::Test { + protected: + ExtensionThrottleManagerTest() + : request_(context_.CreateRequest(GURL(), net::DEFAULT_PRIORITY, NULL)) {} + + void SetUp() override { request_->SetLoadFlags(0); } + + void ExpectEntryAllowsAllOnErrorIfOptedOut( + ExtensionThrottleEntryInterface* entry, + bool opted_out, + const URLRequest& request) { + EXPECT_FALSE(entry->ShouldRejectRequest(request)); + for (int i = 0; i < 10; ++i) { + entry->UpdateWithResponse(503); + } + EXPECT_NE(opted_out, entry->ShouldRejectRequest(request)); + + if (opted_out) { + // We're not mocking out GetTimeNow() in this scenario + // so add a 100 ms buffer to avoid flakiness (that should always + // give enough time to get from the TimeTicks::Now() call here + // to the TimeTicks::Now() call in the entry class). + EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100), + entry->GetExponentialBackoffReleaseTime()); + } else { + // As above, add 100 ms. + EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100), + entry->GetExponentialBackoffReleaseTime()); + } + } + + base::MessageLoopForIO message_loop_; + // context_ must be declared before request_. + TestURLRequestContext context_; + scoped_ptr<URLRequest> request_; +}; + +TEST_F(ExtensionThrottleManagerTest, IsUrlStandardised) { + MockExtensionThrottleManager manager; + GurlAndString test_values[] = { + GurlAndString(GURL("http://www.example.com"), + std::string("http://www.example.com/"), __LINE__), + GurlAndString(GURL("http://www.Example.com"), + std::string("http://www.example.com/"), __LINE__), + GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"), + std::string("http://www.ex4mple.com/pr4c71c41"), __LINE__), + GurlAndString(GURL("http://www.example.com/0/token/false"), + std::string("http://www.example.com/0/token/false"), + __LINE__), + GurlAndString(GURL("http://www.example.com/index.php?code=javascript"), + std::string("http://www.example.com/index.php"), __LINE__), + GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"), + std::string("http://www.example.com/index.php"), __LINE__), + GurlAndString(GURL("http://www.example.com/index.php#superEntry"), + std::string("http://www.example.com/index.php"), __LINE__), + GurlAndString(GURL("http://www.example.com:1234/"), + std::string("http://www.example.com:1234/"), __LINE__)}; + + for (unsigned int i = 0; i < arraysize(test_values); ++i) { + std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url); + EXPECT_EQ(temp, test_values[i].result) << "Test case #" << i << " line " + << test_values[i].line << " failed"; + } +} + +TEST_F(ExtensionThrottleManagerTest, AreEntriesBeingCollected) { + MockExtensionThrottleManager manager; + + manager.CreateEntry(true); // true = Entry is outdated. + manager.CreateEntry(true); + manager.CreateEntry(true); + manager.DoGarbageCollectEntries(); + EXPECT_EQ(0, manager.GetNumberOfEntries()); + + manager.CreateEntry(false); + manager.CreateEntry(false); + manager.CreateEntry(false); + manager.CreateEntry(true); + manager.DoGarbageCollectEntries(); + EXPECT_EQ(3, manager.GetNumberOfEntries()); +} + +TEST_F(ExtensionThrottleManagerTest, IsHostBeingRegistered) { + MockExtensionThrottleManager manager; + + manager.RegisterRequestUrl(GURL("http://www.example.com/")); + manager.RegisterRequestUrl(GURL("http://www.google.com/")); + manager.RegisterRequestUrl(GURL("http://www.google.com/index/0")); + manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1")); + manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure")); + + EXPECT_EQ(3, manager.GetNumberOfEntries()); +} + +TEST_F(ExtensionThrottleManagerTest, LocalHostOptedOut) { + MockExtensionThrottleManager manager; + // A localhost entry should always be opted out. + scoped_refptr<ExtensionThrottleEntryInterface> localhost_entry = + manager.RegisterRequestUrl(GURL("http://localhost/hello")); + EXPECT_FALSE(localhost_entry->ShouldRejectRequest((*request_))); + for (int i = 0; i < 10; ++i) { + localhost_entry->UpdateWithResponse(503); + } + EXPECT_FALSE(localhost_entry->ShouldRejectRequest((*request_))); + + // We're not mocking out GetTimeNow() in this scenario + // so add a 100 ms buffer to avoid flakiness (that should always + // give enough time to get from the TimeTicks::Now() call here + // to the TimeTicks::Now() call in the entry class). + EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100), + localhost_entry->GetExponentialBackoffReleaseTime()); +} + +TEST_F(ExtensionThrottleManagerTest, ClearOnNetworkChange) { + for (int i = 0; i < 3; ++i) { + MockExtensionThrottleManager manager; + scoped_refptr<ExtensionThrottleEntryInterface> entry_before = + manager.RegisterRequestUrl(GURL("http://www.example.com/")); + for (int j = 0; j < 10; ++j) { + entry_before->UpdateWithResponse(503); + } + EXPECT_TRUE(entry_before->ShouldRejectRequest(*request_)); + + switch (i) { + case 0: + manager.OnIPAddressChanged(); + break; + case 1: + manager.OnConnectionTypeChanged( + NetworkChangeNotifier::CONNECTION_UNKNOWN); + break; + case 2: + manager.OnConnectionTypeChanged(NetworkChangeNotifier::CONNECTION_NONE); + break; + default: + FAIL(); + } + + scoped_refptr<ExtensionThrottleEntryInterface> entry_after = + manager.RegisterRequestUrl(GURL("http://www.example.com/")); + EXPECT_FALSE(entry_after->ShouldRejectRequest(*request_)); + } +} + +} // namespace extensions diff --git a/extensions/extensions.gypi b/extensions/extensions.gypi index e53a4b3..bd0d294 100644 --- a/extensions/extensions.gypi +++ b/extensions/extensions.gypi @@ -613,6 +613,8 @@ 'browser/extension_registry_factory.cc', 'browser/extension_registry_factory.h', 'browser/extension_registry_observer.h', + 'browser/extension_request_limiting_throttle.cc', + 'browser/extension_request_limiting_throttle.h', 'browser/extension_scoped_prefs.h', 'browser/extension_system.cc', 'browser/extension_system.h', @@ -768,6 +770,11 @@ 'browser/updater/update_service_factory.h', 'browser/url_request_util.cc', 'browser/url_request_util.h', + 'browser/extension_throttle_entry.cc', + 'browser/extension_throttle_entry.h', + 'browser/extension_throttle_entry_interface.h', + 'browser/extension_throttle_manager.cc', + 'browser/extension_throttle_manager.h', 'browser/user_script_loader.cc', 'browser/user_script_loader.h', 'browser/value_store/leveldb_value_store.cc', diff --git a/extensions/extensions_tests.gypi b/extensions/extensions_tests.gypi index 5ef1e27..afec595 100644 --- a/extensions/extensions_tests.gypi +++ b/extensions/extensions_tests.gypi @@ -93,6 +93,10 @@ 'browser/quota_service_unittest.cc', 'browser/runtime_data_unittest.cc', 'browser/sandboxed_unpacker_unittest.cc', + 'browser/extension_throttle_simulation_unittest.cc', + 'browser/extension_throttle_test_support.cc', + 'browser/extension_throttle_test_support.h', + 'browser/extension_throttle_unittest.cc', 'browser/value_store/leveldb_value_store_unittest.cc', 'browser/value_store/testing_value_store_unittest.cc', 'browser/value_store/value_store_change_unittest.cc', |