From 6a394af194527fdc2cdc5b8af5e367dfdb16828e Mon Sep 17 00:00:00 2001 From: "willchan@chromium.org" Date: Tue, 13 Sep 2011 01:05:48 +0000 Subject: Move URLFetcher from content/common to content/common/net. Add a OWNERS for content/common/net BUG=none TEST=none Review URL: http://codereview.chromium.org/7875006 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@100828 0039d316-1c4b-4281-b951-d872f2087c98 --- chrome/browser/alternate_nav_url_fetcher.h | 2 +- chrome/browser/autocomplete/search_provider.h | 2 +- chrome/browser/autofill/autofill_download.h | 3 +- chrome/browser/browser_process_impl.cc | 2 +- chrome/browser/bug_report_util.cc | 2 +- .../component_updater/component_updater_service.cc | 2 +- .../component_updater_service_unittest.cc | 2 +- chrome/browser/extensions/apps_promo.h | 2 +- chrome/browser/extensions/extension_updater.h | 2 +- .../browser/extensions/webstore_inline_installer.h | 2 +- .../browser/extensions/webstore_install_helper.h | 2 +- chrome/browser/google/google_url_tracker.h | 2 +- .../browser/google/google_url_tracker_unittest.cc | 2 +- chrome/browser/importer/toolbar_importer.h | 2 +- chrome/browser/intranet_redirect_detector.h | 2 +- chrome/browser/io_thread.cc | 2 +- chrome/browser/metrics/metrics_service.h | 2 +- chrome/browser/net/gaia/gaia_oauth_fetcher.cc | 1 - chrome/browser/net/gaia/gaia_oauth_fetcher.h | 2 +- .../net/gaia/gaia_oauth_fetcher_unittest.cc | 2 +- chrome/browser/net/sdch_dictionary_fetcher.h | 2 +- chrome/browser/plugin_download_helper.h | 4 +- .../policy/cloud_policy_subsystem_unittest.cc | 2 +- chrome/browser/policy/device_management_service.h | 2 +- .../policy/testing_policy_url_fetcher_factory.h | 2 +- .../safe_browsing/client_side_detection_service.cc | 1 - .../safe_browsing/client_side_detection_service.h | 2 +- .../client_side_detection_service_unittest.cc | 2 +- .../browser/safe_browsing/malware_details_cache.h | 2 +- chrome/browser/safe_browsing/protocol_manager.h | 2 +- .../browser/search_engines/template_url_fetcher.cc | 2 +- chrome/browser/spellchecker/spellcheck_host_impl.h | 2 +- chrome/browser/sync/glue/http_bridge.h | 2 +- .../sync/glue/sync_backend_host_unittest.cc | 2 +- chrome/browser/sync/test/integration/sync_test.cc | 2 +- .../browser/tab_contents/spelling_menu_observer.h | 2 +- chrome/browser/translate/translate_manager.h | 2 +- .../browser/web_resource/web_resource_service.cc | 2 +- chrome/common/net/gaia/gaia_auth_fetcher.h | 2 +- .../common/net/gaia/gaia_auth_fetcher_unittest.cc | 2 +- .../common/net/gaia/gaia_auth_fetcher_unittest.h | 2 +- .../common/net/gaia/gaia_oauth_client_unittest.cc | 2 +- .../service/cloud_print/cloud_print_url_fetcher.h | 2 +- chrome/service/gaia/service_gaia_authenticator.h | 3 +- chrome/service/service_process.cc | 2 +- .../browser/geolocation/network_location_request.h | 2 +- .../browser/speech/speech_recognition_request.h | 2 +- content/common/net/OWNERS | 2 + content/common/net/url_fetcher.cc | 1123 ++++++++++++++++++++ content/common/net/url_fetcher.h | 323 ++++++ content/common/net/url_fetcher_unittest.cc | 907 ++++++++++++++++ content/common/url_fetcher.cc | 1123 -------------------- content/common/url_fetcher.h | 323 ------ content/common/url_fetcher_unittest.cc | 906 ---------------- content/content_common.gypi | 4 +- content/content_tests.gypi | 2 +- content/test/test_url_fetcher_factory.h | 2 +- 57 files changed, 2404 insertions(+), 2407 deletions(-) create mode 100644 content/common/net/OWNERS create mode 100644 content/common/net/url_fetcher.cc create mode 100644 content/common/net/url_fetcher.h create mode 100644 content/common/net/url_fetcher_unittest.cc delete mode 100644 content/common/url_fetcher.cc delete mode 100644 content/common/url_fetcher.h delete mode 100644 content/common/url_fetcher_unittest.cc diff --git a/chrome/browser/alternate_nav_url_fetcher.h b/chrome/browser/alternate_nav_url_fetcher.h index 07fdf33..36a9adb 100644 --- a/chrome/browser/alternate_nav_url_fetcher.h +++ b/chrome/browser/alternate_nav_url_fetcher.h @@ -12,7 +12,7 @@ #include "chrome/browser/tab_contents/link_infobar_delegate.h" #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" class NavigationController; diff --git a/chrome/browser/autocomplete/search_provider.h b/chrome/browser/autocomplete/search_provider.h index ebced4e..acdc719 100644 --- a/chrome/browser/autocomplete/search_provider.h +++ b/chrome/browser/autocomplete/search_provider.h @@ -26,7 +26,7 @@ #include "chrome/browser/history/history_types.h" #include "chrome/browser/search_engines/template_url.h" #include "chrome/browser/search_engines/template_url_id.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" class Profile; diff --git a/chrome/browser/autofill/autofill_download.h b/chrome/browser/autofill/autofill_download.h index fdb18b5..647ba21 100644 --- a/chrome/browser/autofill/autofill_download.h +++ b/chrome/browser/autofill/autofill_download.h @@ -17,7 +17,7 @@ #include "base/memory/scoped_vector.h" #include "base/time.h" #include "chrome/browser/autofill/autofill_type.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" class AutofillMetrics; class FormStructure; @@ -170,4 +170,3 @@ class AutofillDownloadManager : public URLFetcher::Delegate { }; #endif // CHROME_BROWSER_AUTOFILL_AUTOFILL_DOWNLOAD_H_ - diff --git a/chrome/browser/browser_process_impl.cc b/chrome/browser/browser_process_impl.cc index e2a5544..d605292 100644 --- a/chrome/browser/browser_process_impl.cc +++ b/chrome/browser/browser_process_impl.cc @@ -79,8 +79,8 @@ #include "content/browser/plugin_service.h" #include "content/browser/renderer_host/render_process_host.h" #include "content/browser/renderer_host/resource_dispatcher_host.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_service.h" -#include "content/common/url_fetcher.h" #include "ipc/ipc_logging.h" #include "net/socket/client_socket_pool_manager.h" #include "net/url_request/url_request_context_getter.h" diff --git a/chrome/browser/bug_report_util.cc b/chrome/browser/bug_report_util.cc index c79b81c..995d26c 100644 --- a/chrome/browser/bug_report_util.cc +++ b/chrome/browser/bug_report_util.cc @@ -22,7 +22,7 @@ #include "chrome/common/chrome_switches.h" #include "chrome/common/chrome_version_info.h" #include "content/browser/tab_contents/tab_contents.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" #include "grit/generated_resources.h" #include "grit/locale_settings.h" diff --git a/chrome/browser/component_updater/component_updater_service.cc b/chrome/browser/component_updater/component_updater_service.cc index d92516a..8706ac0 100644 --- a/chrome/browser/component_updater/component_updater_service.cc +++ b/chrome/browser/component_updater/component_updater_service.cc @@ -24,8 +24,8 @@ #include "chrome/common/chrome_version_info.h" #include "chrome/common/extensions/extension.h" #include "content/browser/utility_process_host.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_service.h" -#include "content/common/url_fetcher.h" #include "googleurl/src/gurl.h" #include "net/base/escape.h" #include "net/base/load_flags.h" diff --git a/chrome/browser/component_updater/component_updater_service_unittest.cc b/chrome/browser/component_updater/component_updater_service_unittest.cc index c5cd516..a8e2af5 100644 --- a/chrome/browser/component_updater/component_updater_service_unittest.cc +++ b/chrome/browser/component_updater/component_updater_service_unittest.cc @@ -15,9 +15,9 @@ #include "chrome/test/base/test_notification_tracker.h" #include "chrome/test/base/test_url_request_context_getter.h" #include "content/browser/browser_thread.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_observer.h" #include "content/common/notification_service.h" -#include "content/common/url_fetcher.h" #include "googleurl/src/gurl.h" #include "libxml/globals.h" diff --git a/chrome/browser/extensions/apps_promo.h b/chrome/browser/extensions/apps_promo.h index 28c5b03..248865a 100644 --- a/chrome/browser/extensions/apps_promo.h +++ b/chrome/browser/extensions/apps_promo.h @@ -11,7 +11,7 @@ #include "base/gtest_prod_util.h" #include "chrome/common/extensions/extension.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" class PrefService; class Profile; diff --git a/chrome/browser/extensions/extension_updater.h b/chrome/browser/extensions/extension_updater.h index 169756e..bcba5e8 100644 --- a/chrome/browser/extensions/extension_updater.h +++ b/chrome/browser/extensions/extension_updater.h @@ -24,7 +24,7 @@ #include "base/timer.h" #include "chrome/browser/extensions/extension_service.h" #include "chrome/common/extensions/update_manifest.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" class Extension; diff --git a/chrome/browser/extensions/webstore_inline_installer.h b/chrome/browser/extensions/webstore_inline_installer.h index 72ea056..7f11aa8 100644 --- a/chrome/browser/extensions/webstore_inline_installer.h +++ b/chrome/browser/extensions/webstore_inline_installer.h @@ -14,7 +14,7 @@ #include "chrome/browser/extensions/extension_install_ui.h" #include "chrome/browser/extensions/webstore_install_helper.h" #include "content/browser/tab_contents/tab_contents_observer.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" #include "third_party/skia/include/core/SkBitmap.h" diff --git a/chrome/browser/extensions/webstore_install_helper.h b/chrome/browser/extensions/webstore_install_helper.h index dd5a931..70d122b 100644 --- a/chrome/browser/extensions/webstore_install_helper.h +++ b/chrome/browser/extensions/webstore_install_helper.h @@ -7,7 +7,7 @@ #pragma once #include "content/browser/utility_process_host.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" #include "third_party/skia/include/core/SkBitmap.h" diff --git a/chrome/browser/google/google_url_tracker.h b/chrome/browser/google/google_url_tracker.h index 9e370ff..d770065 100644 --- a/chrome/browser/google/google_url_tracker.h +++ b/chrome/browser/google/google_url_tracker.h @@ -11,9 +11,9 @@ #include "base/gtest_prod_util.h" #include "base/memory/scoped_ptr.h" #include "chrome/browser/tab_contents/confirm_infobar_delegate.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" -#include "content/common/url_fetcher.h" #include "googleurl/src/gurl.h" #include "net/base/network_change_notifier.h" diff --git a/chrome/browser/google/google_url_tracker_unittest.cc b/chrome/browser/google/google_url_tracker_unittest.cc index 5ef262a..0aa1be6 100644 --- a/chrome/browser/google/google_url_tracker_unittest.cc +++ b/chrome/browser/google/google_url_tracker_unittest.cc @@ -14,8 +14,8 @@ #include "chrome/test/base/testing_browser_process.h" #include "chrome/test/base/testing_pref_service.h" #include "content/browser/browser_thread.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_service.h" -#include "content/common/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "net/url_request/url_request.h" #include "net/url_request/url_request_context_getter.h" diff --git a/chrome/browser/importer/toolbar_importer.h b/chrome/browser/importer/toolbar_importer.h index 0752325..fd1fcf0 100644 --- a/chrome/browser/importer/toolbar_importer.h +++ b/chrome/browser/importer/toolbar_importer.h @@ -18,7 +18,7 @@ #include "base/string16.h" #include "chrome/browser/importer/importer.h" #include "chrome/browser/importer/profile_writer.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" class ImporterBridge; class XmlReader; diff --git a/chrome/browser/intranet_redirect_detector.h b/chrome/browser/intranet_redirect_detector.h index 17080c4..7111eb5 100644 --- a/chrome/browser/intranet_redirect_detector.h +++ b/chrome/browser/intranet_redirect_detector.h @@ -12,7 +12,7 @@ #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" #include "net/base/host_resolver_proc.h" #include "net/base/network_change_notifier.h" diff --git a/chrome/browser/io_thread.cc b/chrome/browser/io_thread.cc index 38474c6..d0f3171 100644 --- a/chrome/browser/io_thread.cc +++ b/chrome/browser/io_thread.cc @@ -32,7 +32,7 @@ #include "content/browser/browser_thread.h" #include "content/browser/gpu/gpu_process_host.h" #include "content/browser/in_process_webkit/indexed_db_key_utility_client.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "net/base/cert_verifier.h" #include "net/base/cookie_monster.h" #include "net/base/default_origin_bound_cert_store.h" diff --git a/chrome/browser/metrics/metrics_service.h b/chrome/browser/metrics/metrics_service.h index 5d3eb2a..46867ed 100644 --- a/chrome/browser/metrics/metrics_service.h +++ b/chrome/browser/metrics/metrics_service.h @@ -20,7 +20,7 @@ #include "chrome/common/metrics_helpers.h" #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #if defined(OS_CHROMEOS) #include "chrome/browser/chromeos/external_metrics.h" diff --git a/chrome/browser/net/gaia/gaia_oauth_fetcher.cc b/chrome/browser/net/gaia/gaia_oauth_fetcher.cc index 1aabeef..10c59fb 100644 --- a/chrome/browser/net/gaia/gaia_oauth_fetcher.cc +++ b/chrome/browser/net/gaia/gaia_oauth_fetcher.cc @@ -24,7 +24,6 @@ #include "chrome/common/net/http_return.h" #include "content/common/notification_details.h" #include "content/common/notification_source.h" -#include "content/common/url_fetcher.h" #include "grit/chromium_strings.h" #include "net/base/load_flags.h" #include "net/url_request/url_request_context_getter.h" diff --git a/chrome/browser/net/gaia/gaia_oauth_fetcher.h b/chrome/browser/net/gaia/gaia_oauth_fetcher.h index d1df1c3..ecc6a4d 100644 --- a/chrome/browser/net/gaia/gaia_oauth_fetcher.h +++ b/chrome/browser/net/gaia/gaia_oauth_fetcher.h @@ -11,9 +11,9 @@ #include "base/memory/scoped_ptr.h" #include "chrome/browser/net/chrome_cookie_notification_details.h" #include "chrome/browser/net/gaia/gaia_oauth_consumer.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" -#include "content/common/url_fetcher.h" #include "googleurl/src/gurl.h" struct ChromeCookieDetails; diff --git a/chrome/browser/net/gaia/gaia_oauth_fetcher_unittest.cc b/chrome/browser/net/gaia/gaia_oauth_fetcher_unittest.cc index d92f41b..5f26ea5 100644 --- a/chrome/browser/net/gaia/gaia_oauth_fetcher_unittest.cc +++ b/chrome/browser/net/gaia/gaia_oauth_fetcher_unittest.cc @@ -17,8 +17,8 @@ #include "chrome/common/net/gaia/google_service_auth_error.h" #include "chrome/common/net/http_return.h" #include "chrome/test/base/testing_profile.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_service.h" -#include "content/common/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "googleurl/src/gurl.h" #include "net/base/net_errors.h" diff --git a/chrome/browser/net/sdch_dictionary_fetcher.h b/chrome/browser/net/sdch_dictionary_fetcher.h index 8ae55c0..1e9643d 100644 --- a/chrome/browser/net/sdch_dictionary_fetcher.h +++ b/chrome/browser/net/sdch_dictionary_fetcher.h @@ -16,7 +16,7 @@ #include "base/memory/scoped_ptr.h" #include "base/task.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "net/base/sdch_manager.h" class SdchDictionaryFetcher : public URLFetcher::Delegate, diff --git a/chrome/browser/plugin_download_helper.h b/chrome/browser/plugin_download_helper.h index b640c3b..dcddaa7 100644 --- a/chrome/browser/plugin_download_helper.h +++ b/chrome/browser/plugin_download_helper.h @@ -12,7 +12,7 @@ #if defined(OS_WIN) #include "base/file_path.h" #include "base/message_loop_proxy.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "net/base/file_stream.h" #include "net/url_request/url_request.h" #include "ui/gfx/native_widget_types.h" @@ -69,5 +69,3 @@ class PluginDownloadUrlHelper : public URLFetcher::Delegate { #endif // OS_WIN #endif // CHROME_BROWSER_PLUGIN_DOWNLOAD_HELPER_H_ - - diff --git a/chrome/browser/policy/cloud_policy_subsystem_unittest.cc b/chrome/browser/policy/cloud_policy_subsystem_unittest.cc index cd11107..a201baf 100644 --- a/chrome/browser/policy/cloud_policy_subsystem_unittest.cc +++ b/chrome/browser/policy/cloud_policy_subsystem_unittest.cc @@ -20,7 +20,7 @@ #include "chrome/test/base/testing_browser_process.h" #include "chrome/test/base/testing_pref_service.h" #include "content/browser/browser_thread.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "policy/policy_constants.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chrome/browser/policy/device_management_service.h b/chrome/browser/policy/device_management_service.h index 9029ace..5df5223 100644 --- a/chrome/browser/policy/device_management_service.h +++ b/chrome/browser/policy/device_management_service.h @@ -12,7 +12,7 @@ #include "base/basictypes.h" #include "chrome/browser/policy/device_management_backend.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" class Profile; diff --git a/chrome/browser/policy/testing_policy_url_fetcher_factory.h b/chrome/browser/policy/testing_policy_url_fetcher_factory.h index 2c27145..d109f58 100644 --- a/chrome/browser/policy/testing_policy_url_fetcher_factory.h +++ b/chrome/browser/policy/testing_policy_url_fetcher_factory.h @@ -11,7 +11,7 @@ #include "base/memory/scoped_ptr.h" #include "base/memory/weak_ptr.h" #include "chrome/browser/policy/logging_work_scheduler.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "googleurl/src/gurl.h" #include "testing/gmock/include/gmock/gmock.h" diff --git a/chrome/browser/safe_browsing/client_side_detection_service.cc b/chrome/browser/safe_browsing/client_side_detection_service.cc index 6843912..1a7b8f7 100644 --- a/chrome/browser/safe_browsing/client_side_detection_service.cc +++ b/chrome/browser/safe_browsing/client_side_detection_service.cc @@ -25,7 +25,6 @@ #include "content/browser/browser_thread.h" #include "content/browser/renderer_host/render_process_host.h" #include "content/common/notification_service.h" -#include "content/common/url_fetcher.h" #include "crypto/sha2.h" #include "googleurl/src/gurl.h" #include "net/base/load_flags.h" diff --git a/chrome/browser/safe_browsing/client_side_detection_service.h b/chrome/browser/safe_browsing/client_side_detection_service.h index 7afe1cd..0a25f8b 100644 --- a/chrome/browser/safe_browsing/client_side_detection_service.h +++ b/chrome/browser/safe_browsing/client_side_detection_service.h @@ -30,9 +30,9 @@ #include "base/memory/scoped_ptr.h" #include "base/task.h" #include "base/time.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" -#include "content/common/url_fetcher.h" #include "googleurl/src/gurl.h" #include "net/base/net_util.h" diff --git a/chrome/browser/safe_browsing/client_side_detection_service_unittest.cc b/chrome/browser/safe_browsing/client_side_detection_service_unittest.cc index 55c0f0e..09a3716 100644 --- a/chrome/browser/safe_browsing/client_side_detection_service_unittest.cc +++ b/chrome/browser/safe_browsing/client_side_detection_service_unittest.cc @@ -19,7 +19,7 @@ #include "chrome/common/safe_browsing/csd.pb.h" #include "chrome/renderer/safe_browsing/features.h" #include "content/browser/browser_thread.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "crypto/sha2.h" #include "googleurl/src/gurl.h" diff --git a/chrome/browser/safe_browsing/malware_details_cache.h b/chrome/browser/safe_browsing/malware_details_cache.h index 115eb1a..2b1a5ba 100644 --- a/chrome/browser/safe_browsing/malware_details_cache.h +++ b/chrome/browser/safe_browsing/malware_details_cache.h @@ -16,7 +16,7 @@ #include "base/memory/linked_ptr.h" #include "base/memory/ref_counted.h" #include "chrome/browser/safe_browsing/report.pb.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "net/base/completion_callback.h" namespace net { diff --git a/chrome/browser/safe_browsing/protocol_manager.h b/chrome/browser/safe_browsing/protocol_manager.h index 98b2b1e..d0ff19b 100644 --- a/chrome/browser/safe_browsing/protocol_manager.h +++ b/chrome/browser/safe_browsing/protocol_manager.h @@ -25,7 +25,7 @@ #include "chrome/browser/safe_browsing/protocol_parser.h" #include "chrome/browser/safe_browsing/safe_browsing_service.h" #include "chrome/browser/safe_browsing/safe_browsing_util.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" namespace net { class URLRequestStatus; diff --git a/chrome/browser/search_engines/template_url_fetcher.cc b/chrome/browser/search_engines/template_url_fetcher.cc index 718049b..aa86952 100644 --- a/chrome/browser/search_engines/template_url_fetcher.cc +++ b/chrome/browser/search_engines/template_url_fetcher.cc @@ -15,10 +15,10 @@ #include "chrome/browser/search_engines/template_url_service_factory.h" #include "chrome/browser/search_engines/template_url_parser.h" #include "chrome/common/chrome_notification_types.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" #include "content/common/notification_source.h" -#include "content/common/url_fetcher.h" #include "net/url_request/url_request_status.h" // RequestDelegate ------------------------------------------------------------ diff --git a/chrome/browser/spellchecker/spellcheck_host_impl.h b/chrome/browser/spellchecker/spellcheck_host_impl.h index cb99838..6ec5ec4 100644 --- a/chrome/browser/spellchecker/spellcheck_host_impl.h +++ b/chrome/browser/spellchecker/spellcheck_host_impl.h @@ -13,9 +13,9 @@ #include "base/memory/scoped_ptr.h" #include "chrome/browser/spellchecker/spellcheck_host.h" #include "chrome/browser/spellchecker/spellcheck_profile_provider.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" -#include "content/common/url_fetcher.h" // This class implements the SpellCheckHost interface to provide the // functionalities listed below: diff --git a/chrome/browser/sync/glue/http_bridge.h b/chrome/browser/sync/glue/http_bridge.h index b5d8236..7f8fdb2 100644 --- a/chrome/browser/sync/glue/http_bridge.h +++ b/chrome/browser/sync/glue/http_bridge.h @@ -15,7 +15,7 @@ #include "base/synchronization/waitable_event.h" #include "chrome/browser/sync/internal_api/http_post_provider_factory.h" #include "chrome/browser/sync/internal_api/http_post_provider_interface.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" #include "net/url_request/url_request_context_getter.h" #include "net/url_request/url_request_context.h" diff --git a/chrome/browser/sync/glue/sync_backend_host_unittest.cc b/chrome/browser/sync/glue/sync_backend_host_unittest.cc index a274909..f3d928a 100644 --- a/chrome/browser/sync/glue/sync_backend_host_unittest.cc +++ b/chrome/browser/sync/glue/sync_backend_host_unittest.cc @@ -14,7 +14,7 @@ #include "chrome/test/base/testing_profile.h" #include "chrome/test/base/test_url_request_context_getter.h" #include "content/browser/browser_thread.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "googleurl/src/gurl.h" #include "testing/gmock/include/gmock/gmock.h" diff --git a/chrome/browser/sync/test/integration/sync_test.cc b/chrome/browser/sync/test/integration/sync_test.cc index b3797d7..de40e1b 100644 --- a/chrome/browser/sync/test/integration/sync_test.cc +++ b/chrome/browser/sync/test/integration/sync_test.cc @@ -32,7 +32,7 @@ #include "chrome/test/base/ui_test_utils.h" #include "content/browser/browser_thread.h" #include "content/browser/tab_contents/tab_contents.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "googleurl/src/gurl.h" #include "net/base/escape.h" diff --git a/chrome/browser/tab_contents/spelling_menu_observer.h b/chrome/browser/tab_contents/spelling_menu_observer.h index 60e7a94..4ea7a54 100644 --- a/chrome/browser/tab_contents/spelling_menu_observer.h +++ b/chrome/browser/tab_contents/spelling_menu_observer.h @@ -12,7 +12,7 @@ #include "base/string16.h" #include "base/timer.h" #include "chrome/browser/tab_contents/render_view_context_menu_observer.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" class GURL; class RenderViewContextMenuProxy; diff --git a/chrome/browser/translate/translate_manager.h b/chrome/browser/translate/translate_manager.h index 7fd73e2..22c6638 100644 --- a/chrome/browser/translate/translate_manager.h +++ b/chrome/browser/translate/translate_manager.h @@ -16,9 +16,9 @@ #include "base/task.h" #include "chrome/browser/prefs/pref_change_registrar.h" #include "chrome/common/translate_errors.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_observer.h" #include "content/common/notification_registrar.h" -#include "content/common/url_fetcher.h" template struct DefaultSingletonTraits; class GURL; diff --git a/chrome/browser/web_resource/web_resource_service.cc b/chrome/browser/web_resource/web_resource_service.cc index 371a520..860cf72 100644 --- a/chrome/browser/web_resource/web_resource_service.cc +++ b/chrome/browser/web_resource/web_resource_service.cc @@ -21,8 +21,8 @@ #include "chrome/common/extensions/extension.h" #include "chrome/common/web_resource/web_resource_unpacker.h" #include "content/browser/browser_thread.h" +#include "content/common/net/url_fetcher.h" #include "content/common/notification_service.h" -#include "content/common/url_fetcher.h" #include "googleurl/src/gurl.h" #include "net/base/load_flags.h" #include "net/url_request/url_request_status.h" diff --git a/chrome/common/net/gaia/gaia_auth_fetcher.h b/chrome/common/net/gaia/gaia_auth_fetcher.h index 1fb471a..e9cbdd3 100644 --- a/chrome/common/net/gaia/gaia_auth_fetcher.h +++ b/chrome/common/net/gaia/gaia_auth_fetcher.h @@ -11,7 +11,7 @@ #include "base/gtest_prod_util.h" #include "base/memory/scoped_ptr.h" #include "chrome/common/net/gaia/gaia_auth_consumer.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" // Authenticate a user against the Google Accounts ClientLogin API diff --git a/chrome/common/net/gaia/gaia_auth_fetcher_unittest.cc b/chrome/common/net/gaia/gaia_auth_fetcher_unittest.cc index 42dbe32..8f49ebb 100644 --- a/chrome/common/net/gaia/gaia_auth_fetcher_unittest.cc +++ b/chrome/common/net/gaia/gaia_auth_fetcher_unittest.cc @@ -17,7 +17,7 @@ #include "chrome/common/net/gaia/google_service_auth_error.h" #include "chrome/common/net/http_return.h" #include "chrome/test/base/testing_profile.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "googleurl/src/gurl.h" #include "net/base/load_flags.h" diff --git a/chrome/common/net/gaia/gaia_auth_fetcher_unittest.h b/chrome/common/net/gaia/gaia_auth_fetcher_unittest.h index 6cea31c..0534c52 100644 --- a/chrome/common/net/gaia/gaia_auth_fetcher_unittest.h +++ b/chrome/common/net/gaia/gaia_auth_fetcher_unittest.h @@ -13,7 +13,7 @@ #include "chrome/common/net/gaia/gaia_auth_fetcher.h" #include "chrome/common/net/http_return.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "net/url_request/url_request_status.h" diff --git a/chrome/common/net/gaia/gaia_oauth_client_unittest.cc b/chrome/common/net/gaia/gaia_oauth_client_unittest.cc index d204a79..97a859b 100644 --- a/chrome/common/net/gaia/gaia_oauth_client_unittest.cc +++ b/chrome/common/net/gaia/gaia_oauth_client_unittest.cc @@ -12,7 +12,7 @@ #include "chrome/common/net/gaia/gaia_oauth_client.h" #include "chrome/common/net/http_return.h" #include "chrome/test/base/testing_profile.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "content/test/test_url_fetcher_factory.h" #include "googleurl/src/gurl.h" #include "net/base/net_errors.h" diff --git a/chrome/service/cloud_print/cloud_print_url_fetcher.h b/chrome/service/cloud_print/cloud_print_url_fetcher.h index f6f3680..ab49923 100644 --- a/chrome/service/cloud_print/cloud_print_url_fetcher.h +++ b/chrome/service/cloud_print/cloud_print_url_fetcher.h @@ -9,7 +9,7 @@ #include #include "base/memory/scoped_ptr.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" class GURL; diff --git a/chrome/service/gaia/service_gaia_authenticator.h b/chrome/service/gaia/service_gaia_authenticator.h index 5076372..fded2d2 100644 --- a/chrome/service/gaia/service_gaia_authenticator.h +++ b/chrome/service/gaia/service_gaia_authenticator.h @@ -11,7 +11,7 @@ #include "base/memory/ref_counted.h" #include "base/synchronization/waitable_event.h" #include "chrome/common/net/gaia/gaia_authenticator.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" namespace base { class MessageLoopProxy; @@ -56,4 +56,3 @@ class ServiceGaiaAuthenticator }; #endif // CHROME_SERVICE_GAIA_SERVICE_GAIA_AUTHENTICATOR_H_ - diff --git a/chrome/service/service_process.cc b/chrome/service/service_process.cc index 49292ef..e763948 100644 --- a/chrome/service/service_process.cc +++ b/chrome/service/service_process.cc @@ -25,7 +25,7 @@ #include "chrome/service/net/service_url_request_context.h" #include "chrome/service/service_ipc_server.h" #include "chrome/service/service_process_prefs.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "grit/chromium_strings.h" #include "grit/generated_resources.h" #include "net/base/network_change_notifier.h" diff --git a/content/browser/geolocation/network_location_request.h b/content/browser/geolocation/network_location_request.h index 0694052..8df175f 100644 --- a/content/browser/geolocation/network_location_request.h +++ b/content/browser/geolocation/network_location_request.h @@ -12,7 +12,7 @@ #include "base/memory/ref_counted.h" #include "base/memory/scoped_ptr.h" #include "content/browser/geolocation/device_data_provider.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" class URLFetcher; diff --git a/content/browser/speech/speech_recognition_request.h b/content/browser/speech/speech_recognition_request.h index b6701a1..0c79ed6 100644 --- a/content/browser/speech/speech_recognition_request.h +++ b/content/browser/speech/speech_recognition_request.h @@ -11,8 +11,8 @@ #include "base/basictypes.h" #include "base/memory/ref_counted.h" #include "base/memory/scoped_ptr.h" +#include "content/common/net/url_fetcher.h" #include "content/common/speech_input_result.h" -#include "content/common/url_fetcher.h" #include "googleurl/src/gurl.h" class URLFetcher; diff --git a/content/common/net/OWNERS b/content/common/net/OWNERS new file mode 100644 index 0000000..2f937b5 --- /dev/null +++ b/content/common/net/OWNERS @@ -0,0 +1,2 @@ +eroman@chromium.org +willchan@chromium.org diff --git a/content/common/net/url_fetcher.cc b/content/common/net/url_fetcher.cc new file mode 100644 index 0000000..be4d448 --- /dev/null +++ b/content/common/net/url_fetcher.cc @@ -0,0 +1,1123 @@ +// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/common/net/url_fetcher.h" + +#include + +#include "base/compiler_specific.h" +#include "base/file_path.h" +#include "base/file_util_proxy.h" +#include "base/lazy_instance.h" +#include "base/memory/scoped_callback_factory.h" +#include "base/memory/scoped_ptr.h" +#include "base/message_loop_proxy.h" +#include "base/platform_file.h" +#include "base/stl_util.h" +#include "base/string_util.h" +#include "base/threading/thread.h" +#include "googleurl/src/gurl.h" +#include "net/base/host_port_pair.h" +#include "net/base/io_buffer.h" +#include "net/base/load_flags.h" +#include "net/base/net_errors.h" +#include "net/http/http_request_headers.h" +#include "net/http/http_response_headers.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context.h" +#include "net/url_request/url_request_context_getter.h" +#include "net/url_request/url_request_throttler_manager.h" + +static const int kBufferSize = 4096; +const int URLFetcher::kInvalidHttpResponseCode = -1; + +class URLFetcher::Core + : public base::RefCountedThreadSafe, + public net::URLRequest::Delegate { + public: + // For POST requests, set |content_type| to the MIME type of the content + // and set |content| to the data to upload. |flags| are flags to apply to + // the load operation--these should be one or more of the LOAD_* flags + // defined in net/base/load_flags.h. + Core(URLFetcher* fetcher, + const GURL& original_url, + RequestType request_type, + URLFetcher::Delegate* d); + + // Starts the load. It's important that this not happen in the constructor + // because it causes the IO thread to begin AddRef()ing and Release()ing + // us. If our caller hasn't had time to fully construct us and take a + // reference, the IO thread could interrupt things, run a task, Release() + // us, and destroy us, leaving the caller with an already-destroyed object + // when construction finishes. + void Start(); + + // Stops any in-progress load and ensures no callback will happen. It is + // safe to call this multiple times. + void Stop(); + + // Reports that the received content was malformed (i.e. failed parsing + // or validation). This makes the throttling logic that does exponential + // back-off when servers are having problems treat the current request as + // a failure. Your call to this method will be ignored if your request is + // already considered a failure based on the HTTP response code or response + // headers. + void ReceivedContentWasMalformed(); + + // Overridden from net::URLRequest::Delegate: + virtual void OnResponseStarted(net::URLRequest* request); + virtual void OnReadCompleted(net::URLRequest* request, int bytes_read); + + URLFetcher::Delegate* delegate() const { return delegate_; } + static void CancelAll(); + + private: + friend class base::RefCountedThreadSafe; + + class Registry { + public: + Registry(); + ~Registry(); + + void AddURLFetcherCore(Core* core); + void RemoveURLFetcherCore(Core* core); + + void CancelAll(); + + int size() const { + return fetchers_.size(); + } + + private: + std::set fetchers_; + + DISALLOW_COPY_AND_ASSIGN(Registry); + }; + + // Class TempFileWriter encapsulates all state involved in writing + // response bytes to a temporary file. It is only used if + // |Core::response_destination_| == TEMP_FILE. Each instance of + // TempFileWriter is owned by a URLFetcher::Core, which manages + // its lifetime and never transfers ownership. While writing to + // a file, all function calls happen on the IO thread. + class TempFileWriter { + public: + TempFileWriter( + URLFetcher::Core* core, + scoped_refptr file_message_loop_proxy); + + ~TempFileWriter(); + void CreateTempFile(); + void DidCreateTempFile(base::PlatformFileError error_code, + base::PassPlatformFile file_handle, + FilePath file_path); + + // Record |num_bytes_| response bytes in |core_->buffer_| to the file. + void WriteBuffer(int num_bytes); + + // Called when a write has been done. Continues writing if there are + // any more bytes to write. Otherwise, initiates a read in core_. + void ContinueWrite(base::PlatformFileError error_code, + int bytes_written); + + // Drop ownership of the file at path |temp_file_|. This class + // will not delete it or write to it again. + void DisownTempFile(); + + // Close the temp file if it is open. + void CloseTempFileAndCompleteRequest(); + + // Remove the temp file if we we created one. + void RemoveTempFile(); + + const FilePath& temp_file() const { return temp_file_; } + int64 total_bytes_written() { return total_bytes_written_; } + base::PlatformFileError error_code() const { return error_code_; } + + private: + // Callback which gets the result of closing the temp file. + void DidCloseTempFile(base::PlatformFileError error); + + // The URLFetcher::Core which instantiated this class. + URLFetcher::Core* core_; + + // The last error encountered on a file operation. base::PLATFORM_FILE_OK + // if no error occurred. + base::PlatformFileError error_code_; + + // Callbacks are created for use with base::FileUtilProxy. + base::ScopedCallbackFactory + callback_factory_; + + // Message loop on which file opperations should happen. + scoped_refptr file_message_loop_proxy_; + + // Path to the temporary file. This path is empty when there + // is no temp file. + FilePath temp_file_; + + // Handle to the temp file. + base::PlatformFile temp_file_handle_; + + // We always append to the file. Track the total number of bytes + // written, so that writes know the offset to give. + int64 total_bytes_written_; + + // How many bytes did the last Write() try to write? Needed so + // that if not all the bytes get written on a Write(), we can + // call Write() again with the rest. + int pending_bytes_; + + // When writing, how many bytes from the buffer have been successfully + // written so far? + int buffer_offset_; + }; + + virtual ~Core(); + + // Wrapper functions that allow us to ensure actions happen on the right + // thread. + void StartOnIOThread(); + void StartURLRequest(); + void StartURLRequestWhenAppropriate(); + void CancelURLRequest(); + void OnCompletedURLRequest(base::TimeDelta backoff_delay); + void InformDelegateFetchIsComplete(); + void NotifyMalformedContent(); + void RetryOrCompleteUrlFetch(); + + // Deletes the request, removes it from the registry, and removes the + // destruction observer. + void ReleaseRequest(); + + // Returns the max value of exponential back-off release time for + // |original_url_| and |url_|. + base::TimeTicks GetBackoffReleaseTime(); + + void CompleteAddingUploadDataChunk(const std::string& data, + bool is_last_chunk); + + // Adds a block of data to be uploaded in a POST body. This can only be + // called after Start(). + void AppendChunkToUpload(const std::string& data, bool is_last_chunk); + + // Store the response bytes in |buffer_| in the container indicated by + // |response_destination_|. Return true if the write has been + // done, and another read can overwrite |buffer_|. If this function + // returns false, it will post a task that will read more bytes once the + // write is complete. + bool WriteBuffer(int num_bytes); + + // Read response bytes from the request. + void ReadResponse(); + + // Drop ownership of any temp file managed by |temp_file_|. + void DisownTempFile(); + + URLFetcher* fetcher_; // Corresponding fetcher object + GURL original_url_; // The URL we were asked to fetch + GURL url_; // The URL we eventually wound up at + RequestType request_type_; // What type of request is this? + net::URLRequestStatus status_; // Status of the request + URLFetcher::Delegate* delegate_; // Object to notify on completion + scoped_refptr delegate_loop_proxy_; + // Message loop proxy of the creating + // thread. + scoped_refptr io_message_loop_proxy_; + // The message loop proxy for the thread + // on which the request IO happens. + scoped_refptr file_message_loop_proxy_; + // The message loop proxy for the thread + // on which file access happens. + scoped_ptr request_; // The actual request this wraps + int load_flags_; // Flags for the load operation + int response_code_; // HTTP status code for the request + std::string data_; // Results of the request, when we are + // storing the response as a string. + scoped_refptr buffer_; + // Read buffer + scoped_refptr request_context_getter_; + // Cookie/cache info for the request + net::ResponseCookies cookies_; // Response cookies + net::HttpRequestHeaders extra_request_headers_; + scoped_refptr response_headers_; + bool was_fetched_via_proxy_; + net::HostPortPair socket_address_; + + std::string upload_content_; // HTTP POST payload + std::string upload_content_type_; // MIME type of POST payload + std::string referrer_; // HTTP Referer header value + bool is_chunked_upload_; // True if using chunked transfer encoding + + // Used to determine how long to wait before making a request or doing a + // retry. + // Both of them can only be accessed on the IO thread. + // We need not only the throttler entry for |original_URL|, but also the one + // for |url|. For example, consider the case that URL A redirects to URL B, + // for which the server returns a 500 response. In this case, the exponential + // back-off release time of URL A won't increase. If we retry without + // considering the back-off constraint of URL B, we may send out too many + // requests for URL A in a short period of time. + scoped_refptr + original_url_throttler_entry_; + scoped_refptr url_throttler_entry_; + + // |num_retries_| indicates how many times we've failed to successfully + // fetch this URL. Once this value exceeds the maximum number of retries + // specified by the owner URLFetcher instance, we'll give up. + int num_retries_; + + // True if the URLFetcher has been cancelled. + bool was_cancelled_; + + // If writing results to a file, |temp_file_writer_| will manage creation, + // writing, and destruction of that file. + scoped_ptr temp_file_writer_; + + // Where should responses be saved? + ResponseDestinationType response_destination_; + + // If |automatically_retry_on_5xx_| is false, 5xx responses will be + // propagated to the observer, if it is true URLFetcher will automatically + // re-execute the request, after the back-off delay has expired. + // true by default. + bool automatically_retry_on_5xx_; + // Maximum retries allowed. + int max_retries_; + + static base::LazyInstance g_registry; + + friend class URLFetcher; + DISALLOW_COPY_AND_ASSIGN(Core); +}; + +URLFetcher::Core::Registry::Registry() {} +URLFetcher::Core::Registry::~Registry() {} + +void URLFetcher::Core::Registry::AddURLFetcherCore(Core* core) { + DCHECK(!ContainsKey(fetchers_, core)); + fetchers_.insert(core); +} + +void URLFetcher::Core::Registry::RemoveURLFetcherCore(Core* core) { + DCHECK(ContainsKey(fetchers_, core)); + fetchers_.erase(core); +} + +void URLFetcher::Core::Registry::CancelAll() { + while (!fetchers_.empty()) + (*fetchers_.begin())->CancelURLRequest(); +} + +// static +base::LazyInstance + URLFetcher::Core::g_registry(base::LINKER_INITIALIZED); + +URLFetcher::Core::TempFileWriter::TempFileWriter( + URLFetcher::Core* core, + scoped_refptr file_message_loop_proxy) + : core_(core), + error_code_(base::PLATFORM_FILE_OK), + callback_factory_(ALLOW_THIS_IN_INITIALIZER_LIST(this)), + file_message_loop_proxy_(file_message_loop_proxy), + temp_file_handle_(base::kInvalidPlatformFileValue) { +} + +URLFetcher::Core::TempFileWriter::~TempFileWriter() { + RemoveTempFile(); +} + +void URLFetcher::Core::TempFileWriter::CreateTempFile() { + DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); + CHECK(file_message_loop_proxy_.get()); + base::FileUtilProxy::CreateTemporary( + file_message_loop_proxy_, + 0, // No additional file flags. + callback_factory_.NewCallback( + &URLFetcher::Core::TempFileWriter::DidCreateTempFile)); +} + +void URLFetcher::Core::TempFileWriter::DidCreateTempFile( + base::PlatformFileError error_code, + base::PassPlatformFile file_handle, + FilePath file_path) { + DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); + + if (base::PLATFORM_FILE_OK != error_code) { + error_code_ = error_code; + RemoveTempFile(); + core_->delegate_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(core_, &Core::InformDelegateFetchIsComplete)); + return; + } + + temp_file_ = file_path; + temp_file_handle_ = file_handle.ReleaseValue(); + total_bytes_written_ = 0; + + core_->io_message_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(core_, &Core::StartURLRequestWhenAppropriate)); +} + +void URLFetcher::Core::TempFileWriter::WriteBuffer(int num_bytes) { + DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); + + // Start writing to the temp file by setting the initial state + // of |pending_bytes_| and |buffer_offset_| to indicate that the + // entire buffer has not yet been written. + pending_bytes_ = num_bytes; + buffer_offset_ = 0; + ContinueWrite(base::PLATFORM_FILE_OK, 0); +} + +void URLFetcher::Core::TempFileWriter::ContinueWrite( + base::PlatformFileError error_code, + int bytes_written) { + DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); + + if (base::PLATFORM_FILE_OK != error_code) { + error_code_ = error_code; + RemoveTempFile(); + core_->delegate_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(core_, &Core::InformDelegateFetchIsComplete)); + return; + } + + total_bytes_written_ += bytes_written; + buffer_offset_ += bytes_written; + pending_bytes_ -= bytes_written; + + if (pending_bytes_ > 0) { + base::FileUtilProxy::Write( + file_message_loop_proxy_, + temp_file_handle_, + total_bytes_written_, // Append to the end + (core_->buffer_->data() + buffer_offset_), + pending_bytes_, + callback_factory_.NewCallback( + &URLFetcher::Core::TempFileWriter::ContinueWrite)); + } else { + // Finished writing core_->buffer_ to the file. Read some more. + core_->ReadResponse(); + } +} + +void URLFetcher::Core::TempFileWriter::DisownTempFile() { + DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); + + // Disowning is done by the delegate's OnURLFetchComplete method. + // The temp file should be closed by the time that method is called. + DCHECK(temp_file_handle_ == base::kInvalidPlatformFileValue); + + // Forget about any temp file by reseting the path. + temp_file_ = FilePath(); +} + +void URLFetcher::Core::TempFileWriter::CloseTempFileAndCompleteRequest() { + DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); + + if (temp_file_handle_ != base::kInvalidPlatformFileValue) { + base::FileUtilProxy::Close( + file_message_loop_proxy_, + temp_file_handle_, + callback_factory_.NewCallback( + &URLFetcher::Core::TempFileWriter::DidCloseTempFile)); + temp_file_handle_ = base::kInvalidPlatformFileValue; + } +} + +void URLFetcher::Core::TempFileWriter::DidCloseTempFile( + base::PlatformFileError error_code) { + DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); + + if (base::PLATFORM_FILE_OK != error_code) { + error_code_ = error_code; + RemoveTempFile(); + core_->delegate_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(core_, &Core::InformDelegateFetchIsComplete)); + return; + } + + // If the file was successfully closed, then the URL request is complete. + core_->RetryOrCompleteUrlFetch(); +} + +void URLFetcher::Core::TempFileWriter::RemoveTempFile() { + DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); + + // Close the temp file if it is open. + if (temp_file_handle_ != base::kInvalidPlatformFileValue) { + base::FileUtilProxy::Close( + file_message_loop_proxy_, + temp_file_handle_, + NULL); // No callback: Ignore errors. + temp_file_handle_ = base::kInvalidPlatformFileValue; + } + + if (!temp_file_.empty()) { + base::FileUtilProxy::Delete( + file_message_loop_proxy_, + temp_file_, + false, // No need to recurse, as the path is to a file. + NULL); // No callback: Ignore errors. + DisownTempFile(); + } +} + +// static +URLFetcher::Factory* URLFetcher::factory_ = NULL; + +void URLFetcher::Delegate::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + NOTREACHED() << "If you don't implemnt this, the no-params version " + << "should also be implemented, in which case this " + << "method won't be called..."; +} + +// TODO(skerner): This default implementation will be removed, and the +// method made pure virtual, once all users of URLFetcher are updated +// to not expect response data as a string argument. Once this is removed, +// the method URLFetcher::GetResponseStringRef() can be removed as well. +// crbug.com/83592 tracks this. +void URLFetcher::Delegate::OnURLFetchComplete(const URLFetcher* source) { + // A delegate that did not override this method is using the old + // parameter list to OnURLFetchComplete(). If a user asked to save + // the response to a file, they must use the new parameter list, + // in which case we can not get here. + // To avoid updating all callers, thunk to the old prototype for now. + OnURLFetchComplete(source, + source->url(), + source->status(), + source->response_code(), + source->cookies(), + source->GetResponseStringRef()); +} + +// static +bool URLFetcher::g_interception_enabled = false; + +URLFetcher::URLFetcher(const GURL& url, + RequestType request_type, + Delegate* d) + : ALLOW_THIS_IN_INITIALIZER_LIST( + core_(new Core(this, url, request_type, d))) { +} + +URLFetcher::~URLFetcher() { + core_->Stop(); +} + +// static +URLFetcher* URLFetcher::Create(int id, const GURL& url, + RequestType request_type, Delegate* d) { + return factory_ ? factory_->CreateURLFetcher(id, url, request_type, d) : + new URLFetcher(url, request_type, d); +} + +URLFetcher::Core::Core(URLFetcher* fetcher, + const GURL& original_url, + RequestType request_type, + URLFetcher::Delegate* d) + : fetcher_(fetcher), + original_url_(original_url), + request_type_(request_type), + delegate_(d), + delegate_loop_proxy_( + base::MessageLoopProxy::current()), + request_(NULL), + load_flags_(net::LOAD_NORMAL), + response_code_(URLFetcher::kInvalidHttpResponseCode), + buffer_(new net::IOBuffer(kBufferSize)), + was_fetched_via_proxy_(false), + is_chunked_upload_(false), + num_retries_(0), + was_cancelled_(false), + response_destination_(STRING), + automatically_retry_on_5xx_(true), + max_retries_(0) { +} + +URLFetcher::Core::~Core() { + // |request_| should be NULL. If not, it's unsafe to delete it here since we + // may not be on the IO thread. + DCHECK(!request_.get()); +} + +void URLFetcher::Core::Start() { + DCHECK(delegate_loop_proxy_); + CHECK(request_context_getter_) << "We need an URLRequestContext!"; + if (io_message_loop_proxy_) { + DCHECK_EQ(io_message_loop_proxy_, + request_context_getter_->GetIOMessageLoopProxy()); + } else { + io_message_loop_proxy_ = request_context_getter_->GetIOMessageLoopProxy(); + } + CHECK(io_message_loop_proxy_.get()) << "We need an IO message loop proxy"; + + io_message_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(this, &Core::StartOnIOThread)); +} + +void URLFetcher::Core::StartOnIOThread() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + switch (response_destination_) { + case STRING: + StartURLRequestWhenAppropriate(); + break; + + case TEMP_FILE: + DCHECK(file_message_loop_proxy_.get()) + << "Need to set the file message loop proxy."; + + temp_file_writer_.reset( + new TempFileWriter(this, file_message_loop_proxy_)); + + // If the temp file is successfully created, + // Core::StartURLRequestWhenAppropriate() will be called. + temp_file_writer_->CreateTempFile(); + break; + + default: + NOTREACHED(); + } +} + +void URLFetcher::Core::Stop() { + DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); + delegate_ = NULL; + fetcher_ = NULL; + if (io_message_loop_proxy_.get()) { + io_message_loop_proxy_->PostTask( + FROM_HERE, NewRunnableMethod(this, &Core::CancelURLRequest)); + } +} + +void URLFetcher::Core::ReceivedContentWasMalformed() { + DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); + if (io_message_loop_proxy_.get()) { + io_message_loop_proxy_->PostTask( + FROM_HERE, NewRunnableMethod(this, &Core::NotifyMalformedContent)); + } +} + +void URLFetcher::Core::CancelAll() { + g_registry.Get().CancelAll(); +} + +void URLFetcher::Core::OnResponseStarted(net::URLRequest* request) { + DCHECK_EQ(request, request_.get()); + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + if (request_->status().is_success()) { + response_code_ = request_->GetResponseCode(); + response_headers_ = request_->response_headers(); + socket_address_ = request_->GetSocketAddress(); + was_fetched_via_proxy_ = request_->was_fetched_via_proxy(); + } + + ReadResponse(); +} + +void URLFetcher::Core::CompleteAddingUploadDataChunk( + const std::string& content, bool is_last_chunk) { + DCHECK(is_chunked_upload_); + DCHECK(request_.get()); + DCHECK(!content.empty()); + request_->AppendChunkToUpload(content.data(), + static_cast(content.length()), + is_last_chunk); +} + +void URLFetcher::Core::AppendChunkToUpload(const std::string& content, + bool is_last_chunk) { + DCHECK(delegate_loop_proxy_); + CHECK(io_message_loop_proxy_.get()); + io_message_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(this, &Core::CompleteAddingUploadDataChunk, content, + is_last_chunk)); +} + +// Return true if the write was done and reading may continue. +// Return false if the write is pending, and the next read will +// be done later. +bool URLFetcher::Core::WriteBuffer(int num_bytes) { + bool write_complete = false; + switch (response_destination_) { + case STRING: + data_.append(buffer_->data(), num_bytes); + write_complete = true; + break; + + case TEMP_FILE: + temp_file_writer_->WriteBuffer(num_bytes); + // WriteBuffer() sends a request the file thread. + // The write is not done yet. + write_complete = false; + break; + + default: + NOTREACHED(); + } + return write_complete; +} + +void URLFetcher::Core::OnReadCompleted(net::URLRequest* request, + int bytes_read) { + DCHECK(request == request_); + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + url_ = request->url(); + url_throttler_entry_ = + net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(url_); + + bool waiting_on_write = false; + do { + if (!request_->status().is_success() || bytes_read <= 0) + break; + + if (!WriteBuffer(bytes_read)) { + // If WriteBuffer() returns false, we have a pending write to + // wait on before reading further. + waiting_on_write = true; + break; + } + } while (request_->Read(buffer_, kBufferSize, &bytes_read)); + + const net::URLRequestStatus status = request_->status(); + + if (status.is_success()) + request_->GetResponseCookies(&cookies_); + + // See comments re: HEAD requests in ReadResponse(). + if ((!status.is_io_pending() && !waiting_on_write) || + (request_type_ == HEAD)) { + status_ = status; + ReleaseRequest(); + + // If a temp file is open, close it. + if (temp_file_writer_.get()) { + // If the file is open, close it. After closing the file, + // RetryOrCompleteUrlFetch() will be called. + temp_file_writer_->CloseTempFileAndCompleteRequest(); + } else { + // Otherwise, complete or retry the URL request directly. + RetryOrCompleteUrlFetch(); + } + } +} + +void URLFetcher::Core::RetryOrCompleteUrlFetch() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + base::TimeDelta backoff_delay; + + // Checks the response from server. + if (response_code_ >= 500 || + status_.error() == net::ERR_TEMPORARILY_THROTTLED) { + // When encountering a server error, we will send the request again + // after backoff time. + ++num_retries_; + + // Note that backoff_delay_ may be 0 because (a) the URLRequestThrottler + // code does not necessarily back off on the first error, and (b) it + // only backs off on some of the 5xx status codes. + base::TimeTicks backoff_release_time = GetBackoffReleaseTime(); + backoff_delay = backoff_release_time - base::TimeTicks::Now(); + if (backoff_delay < base::TimeDelta()) + backoff_delay = base::TimeDelta(); + + if (automatically_retry_on_5xx_ && + num_retries_ <= max_retries_) { + StartOnIOThread(); + return; + } + } else { + backoff_delay = base::TimeDelta(); + } + request_context_getter_ = NULL; + bool posted = delegate_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(this, + &Core::OnCompletedURLRequest, + backoff_delay)); + + // If the delegate message loop does not exist any more, then the delegate + // should be gone too. + DCHECK(posted || !delegate_); +} + +void URLFetcher::Core::ReadResponse() { + // Some servers may treat HEAD requests as GET requests. To free up the + // network connection as soon as possible, signal that the request has + // completed immediately, without trying to read any data back (all we care + // about is the response code and headers, which we already have). + int bytes_read = 0; + if (request_->status().is_success() && (request_type_ != HEAD)) + request_->Read(buffer_, kBufferSize, &bytes_read); + OnReadCompleted(request_.get(), bytes_read); +} + +void URLFetcher::Core::DisownTempFile() { + temp_file_writer_->DisownTempFile(); +} + +void URLFetcher::Core::StartURLRequest() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + if (was_cancelled_) { + // Since StartURLRequest() is posted as a *delayed* task, it may + // run after the URLFetcher was already stopped. + return; + } + + CHECK(request_context_getter_); + DCHECK(!request_.get()); + + g_registry.Get().AddURLFetcherCore(this); + request_.reset(new net::URLRequest(original_url_, this)); + int flags = request_->load_flags() | load_flags_; + if (!g_interception_enabled) { + flags = flags | net::LOAD_DISABLE_INTERCEPT; + } + if (is_chunked_upload_) + request_->EnableChunkedUpload(); + request_->set_load_flags(flags); + request_->set_context(request_context_getter_->GetURLRequestContext()); + request_->set_referrer(referrer_); + + switch (request_type_) { + case GET: + break; + + case POST: + DCHECK(!upload_content_.empty() || is_chunked_upload_); + DCHECK(!upload_content_type_.empty()); + + request_->set_method("POST"); + extra_request_headers_.SetHeader(net::HttpRequestHeaders::kContentType, + upload_content_type_); + if (!upload_content_.empty()) { + request_->AppendBytesToUpload( + upload_content_.data(), static_cast(upload_content_.length())); + } + break; + + case HEAD: + request_->set_method("HEAD"); + break; + + default: + NOTREACHED(); + } + + if (!extra_request_headers_.IsEmpty()) + request_->SetExtraRequestHeaders(extra_request_headers_); + + // There might be data left over from a previous request attempt. + data_.clear(); + + // If we are writing the response to a file, the only caller + // of this function should have created it and not written yet. + CHECK(!temp_file_writer_.get() || + temp_file_writer_->total_bytes_written() == 0); + + request_->Start(); +} + +void URLFetcher::Core::StartURLRequestWhenAppropriate() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + if (was_cancelled_) + return; + + if (original_url_throttler_entry_ == NULL) { + original_url_throttler_entry_ = + net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl( + original_url_); + } + + int64 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest( + GetBackoffReleaseTime()); + if (delay == 0) { + StartURLRequest(); + } else { + MessageLoop::current()->PostDelayedTask( + FROM_HERE, + NewRunnableMethod(this, &Core::StartURLRequest), + delay); + } +} + +void URLFetcher::Core::CancelURLRequest() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + if (request_.get()) { + request_->Cancel(); + ReleaseRequest(); + } + // Release the reference to the request context. There could be multiple + // references to URLFetcher::Core at this point so it may take a while to + // delete the object, but we cannot delay the destruction of the request + // context. + request_context_getter_ = NULL; + was_cancelled_ = true; + temp_file_writer_.reset(); +} + +void URLFetcher::Core::OnCompletedURLRequest( + base::TimeDelta backoff_delay) { + DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); + + // Save the status and backoff_delay so that delegates can read it. + if (delegate_) { + fetcher_->backoff_delay_ = backoff_delay; + InformDelegateFetchIsComplete(); + } +} + +void URLFetcher::Core::InformDelegateFetchIsComplete() { + CHECK(delegate_loop_proxy_->BelongsToCurrentThread()); + if (delegate_) { + delegate_->OnURLFetchComplete(fetcher_); + } +} + +void URLFetcher::Core::NotifyMalformedContent() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + if (url_throttler_entry_ != NULL) { + int status_code = response_code_; + if (status_code == kInvalidHttpResponseCode) { + // The status code will generally be known by the time clients + // call the |ReceivedContentWasMalformed()| function (which ends up + // calling the current function) but if it's not, we need to assume + // the response was successful so that the total failure count + // used to calculate exponential back-off goes up. + status_code = 200; + } + url_throttler_entry_->ReceivedContentWasMalformed(status_code); + } +} + +void URLFetcher::Core::ReleaseRequest() { + request_.reset(); + g_registry.Get().RemoveURLFetcherCore(this); +} + +base::TimeTicks URLFetcher::Core::GetBackoffReleaseTime() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + DCHECK(original_url_throttler_entry_ != NULL); + + base::TimeTicks original_url_backoff = + original_url_throttler_entry_->GetExponentialBackoffReleaseTime(); + base::TimeTicks destination_url_backoff; + if (url_throttler_entry_ != NULL && + original_url_throttler_entry_ != url_throttler_entry_) { + destination_url_backoff = + url_throttler_entry_->GetExponentialBackoffReleaseTime(); + } + + return original_url_backoff > destination_url_backoff ? + original_url_backoff : destination_url_backoff; +} + +void URLFetcher::set_upload_data(const std::string& upload_content_type, + const std::string& upload_content) { + DCHECK(!core_->is_chunked_upload_); + core_->upload_content_type_ = upload_content_type; + core_->upload_content_ = upload_content; +} + +void URLFetcher::set_chunked_upload(const std::string& content_type) { + DCHECK(core_->is_chunked_upload_ || + (core_->upload_content_type_.empty() && + core_->upload_content_.empty())); + core_->upload_content_type_ = content_type; + core_->upload_content_.clear(); + core_->is_chunked_upload_ = true; +} + +void URLFetcher::AppendChunkToUpload(const std::string& data, + bool is_last_chunk) { + DCHECK(data.length()); + core_->AppendChunkToUpload(data, is_last_chunk); +} + +const std::string& URLFetcher::upload_data() const { + return core_->upload_content_; +} + +void URLFetcher::set_referrer(const std::string& referrer) { + core_->referrer_ = referrer; +} + +void URLFetcher::set_load_flags(int load_flags) { + core_->load_flags_ = load_flags; +} + +int URLFetcher::load_flags() const { + return core_->load_flags_; +} + +void URLFetcher::set_extra_request_headers( + const std::string& extra_request_headers) { + core_->extra_request_headers_.Clear(); + core_->extra_request_headers_.AddHeadersFromString(extra_request_headers); +} + +void URLFetcher::GetExtraRequestHeaders(net::HttpRequestHeaders* headers) { + headers->CopyFrom(core_->extra_request_headers_); +} + +void URLFetcher::set_request_context( + net::URLRequestContextGetter* request_context_getter) { + DCHECK(!core_->request_context_getter_); + core_->request_context_getter_ = request_context_getter; +} + +void URLFetcher::set_automatically_retry_on_5xx(bool retry) { + core_->automatically_retry_on_5xx_ = retry; +} + +int URLFetcher::max_retries() const { + return core_->max_retries_; +} + +void URLFetcher::set_max_retries(int max_retries) { + core_->max_retries_ = max_retries; +} + +void URLFetcher::SaveResponseToTemporaryFile( + scoped_refptr file_message_loop_proxy) { + core_->file_message_loop_proxy_ = file_message_loop_proxy; + core_->response_destination_ = TEMP_FILE; +} + +net::HttpResponseHeaders* URLFetcher::response_headers() const { + return core_->response_headers_; +} + +// TODO(panayiotis): socket_address_ is written in the IO thread, +// if this is accessed in the UI thread, this could result in a race. +// Same for response_headers_ above and was_fetched_via_proxy_ below. +net::HostPortPair URLFetcher::socket_address() const { + return core_->socket_address_; +} + +bool URLFetcher::was_fetched_via_proxy() const { + return core_->was_fetched_via_proxy_; +} + +void URLFetcher::Start() { + core_->Start(); +} + +void URLFetcher::StartWithRequestContextGetter( + net::URLRequestContextGetter* request_context_getter) { + set_request_context(request_context_getter); + core_->Start(); +} + +const GURL& URLFetcher::original_url() const { + return core_->original_url_; +} + +const GURL& URLFetcher::url() const { + return core_->url_; +} + +const net::URLRequestStatus& URLFetcher::status() const { + return core_->status_; +} + +int URLFetcher::response_code() const { + return core_->response_code_; +} + +const net::ResponseCookies& URLFetcher::cookies() const { + return core_->cookies_; +} + +bool URLFetcher::FileErrorOccurred( + base::PlatformFileError* out_error_code) const { + + // Can't have a file error if no file is being created or written to. + if (!core_->temp_file_writer_.get()) { + return false; + } + + base::PlatformFileError error_code = core_->temp_file_writer_->error_code(); + if (error_code == base::PLATFORM_FILE_OK) + return false; + + *out_error_code = error_code; + return true; +} + +void URLFetcher::ReceivedContentWasMalformed() { + core_->ReceivedContentWasMalformed(); +} + +bool URLFetcher::GetResponseAsString(std::string* out_response_string) const { + if (core_->response_destination_ != STRING) + return false; + + *out_response_string = core_->data_; + return true; +} + +const std::string& URLFetcher::GetResponseStringRef() const { + CHECK(core_->response_destination_ == STRING); + return core_->data_; +} + +void URLFetcher::SetResponseDestinationForTesting( + ResponseDestinationType value) { + core_->response_destination_ = value; +} + +URLFetcher::ResponseDestinationType +URLFetcher::GetResponseDestinationForTesting() const { + return core_->response_destination_; +} + +bool URLFetcher::GetResponseAsFilePath(bool take_ownership, + FilePath* out_response_path) const { + DCHECK(core_->delegate_loop_proxy_->BelongsToCurrentThread()); + if (core_->response_destination_ != TEMP_FILE || + !core_->temp_file_writer_.get()) + return false; + + *out_response_path = core_->temp_file_writer_->temp_file(); + + if (take_ownership) { + core_->io_message_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(core_.get(), &Core::DisownTempFile)); + } + return true; +} + +// static +void URLFetcher::CancelAll() { + Core::CancelAll(); +} + +// static +int URLFetcher::GetNumFetcherCores() { + return Core::g_registry.Get().size(); +} + +URLFetcher::Delegate* URLFetcher::delegate() const { + return core_->delegate(); +} diff --git a/content/common/net/url_fetcher.h b/content/common/net/url_fetcher.h new file mode 100644 index 0000000..00ff56f --- /dev/null +++ b/content/common/net/url_fetcher.h @@ -0,0 +1,323 @@ +// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file contains URLFetcher, a wrapper around net::URLRequest that handles +// low-level details like thread safety, ref counting, and incremental buffer +// reading. This is useful for callers who simply want to get the data from a +// URL and don't care about all the nitty-gritty details. +// +// NOTE(willchan): Only one "IO" thread is supported for URLFetcher. This is a +// temporary situation. We will work on allowing support for multiple "io" +// threads per process. + +#ifndef CONTENT_COMMON_NET_URL_FETCHER_H_ +#define CONTENT_COMMON_NET_URL_FETCHER_H_ +#pragma once + +#include +#include + +#include "base/memory/ref_counted.h" +#include "base/message_loop.h" +#include "base/platform_file.h" +#include "base/time.h" + +class FilePath; +class GURL; + +namespace base { +class MessageLoopProxy; +} // namespace base + +namespace net { +class HostPortPair; +class HttpResponseHeaders; +class HttpRequestHeaders; +class URLRequestContextGetter; +class URLRequestStatus; +typedef std::vector ResponseCookies; +} // namespace net + +// To use this class, create an instance with the desired URL and a pointer to +// the object to be notified when the URL has been loaded: +// URLFetcher* fetcher = new URLFetcher("http://www.google.com", +// URLFetcher::GET, this); +// +// Then, optionally set properties on this object, like the request context or +// extra headers: +// fetcher->SetExtraRequestHeaders("X-Foo: bar"); +// +// Finally, start the request: +// fetcher->Start(); +// +// +// The object you supply as a delegate must inherit from URLFetcher::Delegate; +// when the fetch is completed, OnURLFetchComplete() will be called with a +// pointer to the URLFetcher. From that point until the original URLFetcher +// instance is destroyed, you may use accessor methods to see the result of +// the fetch. You should copy these objects if you need them to live longer +// than the URLFetcher instance. If the URLFetcher instance is destroyed +// before the callback happens, the fetch will be canceled and no callback +// will occur. +// +// You may create the URLFetcher instance on any thread; OnURLFetchComplete() +// will be called back on the same thread you use to create the instance. +// +// +// NOTE: By default URLFetcher requests are NOT intercepted, except when +// interception is explicitly enabled in tests. + +class URLFetcher { + public: + enum RequestType { + GET, + POST, + HEAD, + }; + + // Imposible http response code. Used to signal that no http response code + // was received. + static const int kInvalidHttpResponseCode; + + class Delegate { + public: + // TODO(skerner): This will be removed in favor of the |source|-only + // version below. Leaving this for now to make the initial code review + // easy to read. + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + + // This will be called when the URL has been fetched, successfully or not. + // Use accessor methods on |source| to get the results. + virtual void OnURLFetchComplete(const URLFetcher* source); + + protected: + virtual ~Delegate() {} + }; + + // URLFetcher::Create uses the currently registered Factory to create the + // URLFetcher. Factory is intended for testing. + class Factory { + public: + virtual URLFetcher* CreateURLFetcher(int id, + const GURL& url, + RequestType request_type, + Delegate* d) = 0; + + protected: + virtual ~Factory() {} + }; + + // |url| is the URL to send the request to. + // |request_type| is the type of request to make. + // |d| the object that will receive the callback on fetch completion. + URLFetcher(const GURL& url, RequestType request_type, Delegate* d); + virtual ~URLFetcher(); + + // Normally interception is disabled for URLFetcher, but you can use this + // to enable it for tests. Also see ScopedURLFetcherFactory for another way + // of testing code that uses an URLFetcher. + static void enable_interception_for_tests(bool enabled) { + g_interception_enabled = enabled; + } + + // Creates a URLFetcher, ownership returns to the caller. If there is no + // Factory (the default) this creates and returns a new URLFetcher. See the + // constructor for a description of the args. |id| may be used during testing + // to identify who is creating the URLFetcher. + static URLFetcher* Create(int id, const GURL& url, RequestType request_type, + Delegate* d); + + // Sets data only needed by POSTs. All callers making POST requests should + // call this before the request is started. |upload_content_type| is the MIME + // type of the content, while |upload_content| is the data to be sent (the + // Content-Length header value will be set to the length of this data). + void set_upload_data(const std::string& upload_content_type, + const std::string& upload_content); + + // Indicates that the POST data is sent via chunked transfer encoding. + // This may only be called before calling Start(). + // Use AppendChunkToUpload() to give the data chunks after calling Start(). + void set_chunked_upload(const std::string& upload_content_type); + + // Adds the given bytes to a request's POST data transmitted using chunked + // transfer encoding. + // This method should be called ONLY after calling Start(). + virtual void AppendChunkToUpload(const std::string& data, bool is_last_chunk); + + // Set one or more load flags as defined in net/base/load_flags.h. Must be + // called before the request is started. + void set_load_flags(int load_flags); + + // Returns the current load flags. + int load_flags() const; + + // The referrer URL for the request. Must be called before the request is + // started. + void set_referrer(const std::string& referrer); + + // Set extra headers on the request. Must be called before the request + // is started. + void set_extra_request_headers(const std::string& extra_request_headers); + + void GetExtraRequestHeaders(net::HttpRequestHeaders* headers); + + // Set the net::URLRequestContext on the request. Must be called before the + // request is started. + void set_request_context( + net::URLRequestContextGetter* request_context_getter); + + // If |retry| is false, 5xx responses will be propagated to the observer, + // if it is true URLFetcher will automatically re-execute the request, + // after backoff_delay() elapses. URLFetcher has it set to true by default. + void set_automatically_retry_on_5xx(bool retry); + + int max_retries() const; + + void set_max_retries(int max_retries); + + // Returns the back-off delay before the request will be retried, + // when a 5xx response was received. + base::TimeDelta backoff_delay() const { return backoff_delay_; } + + // Sets the back-off delay, allowing to mock 5xx requests in unit-tests. +#if defined(UNIT_TEST) + void set_backoff_delay(base::TimeDelta backoff_delay) { + backoff_delay_ = backoff_delay; + } +#endif // defined(UNIT_TEST) + + // By default, the response is saved in a string. Call this method to save the + // response to a temporary file instead. Must be called before Start(). + // |file_message_loop_proxy| will be used for all file operations. + void SaveResponseToTemporaryFile( + scoped_refptr file_message_loop_proxy); + + // Retrieve the response headers from the request. Must only be called after + // the OnURLFetchComplete callback has run. + virtual net::HttpResponseHeaders* response_headers() const; + + // Retrieve the remote socket address from the request. Must only + // be called after the OnURLFetchComplete callback has run and if + // the request has not failed. + net::HostPortPair socket_address() const; + + // Returns true if the request was delivered through a proxy. Must only + // be called after the OnURLFetchComplete callback has run and the request + // has not failed. + bool was_fetched_via_proxy() const; + + // Start the request. After this is called, you may not change any other + // settings. + virtual void Start(); + + // Restarts the URLFetcher with a new URLRequestContextGetter. + void StartWithRequestContextGetter( + net::URLRequestContextGetter* request_context_getter); + + // Return the URL that we were asked to fetch. + virtual const GURL& original_url() const; + + // Return the URL that this fetcher is processing. + virtual const GURL& url() const; + + // The status of the URL fetch. + virtual const net::URLRequestStatus& status() const; + + // The http response code received. Will return + // URLFetcher::kInvalidHttpResponseCode if an error prevented any response + // from being received. + virtual int response_code() const; + + // Cookies recieved. + virtual const net::ResponseCookies& cookies() const; + + // Return true if any file system operation failed. If so, set |error_code| + // to the error code. File system errors are only possible if user called + // SaveResponseToTemporaryFile(). + virtual bool FileErrorOccurred(base::PlatformFileError* out_error_code) const; + + // Reports that the received content was malformed. + void ReceivedContentWasMalformed(); + + // Get the response as a string. Return false if the fetcher was not + // set to store the response as a string. + virtual bool GetResponseAsString(std::string* out_response_string) const; + + // Get the path to the file containing the response body. Returns false + // if the response body was not saved to a file. If take_ownership is + // true, caller takes responsibility for the temp file, and it will not + // be removed once the URLFetcher is destroyed. User should not take + // ownership more than once, or call this method after taking ownership. + virtual bool GetResponseAsFilePath(bool take_ownership, + FilePath* out_response_path) const; + + // Cancels all existing URLFetchers. Will notify the URLFetcher::Delegates. + // Note that any new URLFetchers created while this is running will not be + // cancelled. Typically, one would call this in the CleanUp() method of an IO + // thread, so that no new URLRequests would be able to start on the IO thread + // anyway. This doesn't prevent new URLFetchers from trying to post to the IO + // thread though, even though the task won't ever run. + static void CancelAll(); + + protected: + // How should the response be stored? + enum ResponseDestinationType { + STRING, // Default: In a std::string + TEMP_FILE // Write to a temp file + }; + + // Returns the delegate. + Delegate* delegate() const; + + // Used by tests. + const std::string& upload_data() const; + + // Return a reference to the string data fetched. Response type must + // be STRING, or this will CHECK. This method exists to support the + // old signiture to OnURLFetchComplete(), and will be removed as part + // of crbug.com/83592 . + virtual const std::string& GetResponseStringRef() const; + + virtual void SetResponseDestinationForTesting(ResponseDestinationType); + virtual ResponseDestinationType GetResponseDestinationForTesting() const; + + private: + friend class ScopedURLFetcherFactory; + friend class TestURLFetcher; + friend class URLFetcherTest; + + // Only used by URLFetcherTest, returns the number of URLFetcher::Core objects + // actively running. + static int GetNumFetcherCores(); + + static Factory* factory() { return factory_; } + + // Sets the factory used by the static method Create to create a URLFetcher. + // URLFetcher does not take ownership of |factory|. A value of NULL results + // in a URLFetcher being created directly. + // + // NOTE: for safety, this should only be used through ScopedURLFetcherFactory! + static void set_factory(Factory* factory) { + factory_ = factory; + } + + class Core; + scoped_refptr core_; + + static Factory* factory_; + + // Back-off time delay. 0 by default. + base::TimeDelta backoff_delay_; + + static bool g_interception_enabled; + + DISALLOW_COPY_AND_ASSIGN(URLFetcher); +}; + +#endif // CONTENT_COMMON_NET_URL_FETCHER_H_ diff --git a/content/common/net/url_fetcher_unittest.cc b/content/common/net/url_fetcher_unittest.cc new file mode 100644 index 0000000..86411d1 --- /dev/null +++ b/content/common/net/url_fetcher_unittest.cc @@ -0,0 +1,907 @@ +// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/common/net/url_fetcher.h" + +#include "base/message_loop_proxy.h" +#include "base/synchronization/waitable_event.h" +#include "base/threading/thread.h" +#include "build/build_config.h" +#include "crypto/nss_util.h" +#include "net/http/http_response_headers.h" +#include "net/test/test_server.h" +#include "net/url_request/url_request_context_getter.h" +#include "net/url_request/url_request_test_util.h" +#include "net/url_request/url_request_throttler_manager.h" +#include "testing/gtest/include/gtest/gtest.h" + +#if defined(USE_NSS) +#include "net/ocsp/nss_ocsp.h" +#endif + +using base::Time; +using base::TimeDelta; + +// TODO(eroman): Add a regression test for http://crbug.com/40505. + +namespace { + +const FilePath::CharType kDocRoot[] = FILE_PATH_LITERAL("chrome/test/data"); +const std::string kTestServerFilePrefix = "files/"; + +class CurriedTask : public Task { + public: + CurriedTask(Task* task, MessageLoop* target_loop) + : task_(task), + target_loop_(target_loop) {} + + virtual void Run() { + target_loop_->PostTask(FROM_HERE, task_); + } + + private: + Task* const task_; + MessageLoop* const target_loop_; + + DISALLOW_COPY_AND_ASSIGN(CurriedTask); +}; + +class TestURLRequestContextGetter : public net::URLRequestContextGetter { + public: + explicit TestURLRequestContextGetter( + base::MessageLoopProxy* io_message_loop_proxy) + : io_message_loop_proxy_(io_message_loop_proxy) { + } + virtual net::URLRequestContext* GetURLRequestContext() { + if (!context_) + context_ = new TestURLRequestContext(); + return context_; + } + virtual scoped_refptr GetIOMessageLoopProxy() const { + return io_message_loop_proxy_; + } + + protected: + scoped_refptr io_message_loop_proxy_; + + private: + virtual ~TestURLRequestContextGetter() {} + + scoped_refptr context_; +}; + +} // namespace + +class URLFetcherTest : public testing::Test, public URLFetcher::Delegate { + public: + URLFetcherTest() : fetcher_(NULL) { } + + // Creates a URLFetcher, using the program's main thread to do IO. + virtual void CreateFetcher(const GURL& url); + + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + + scoped_refptr io_message_loop_proxy() { + return io_message_loop_proxy_; + } + + protected: + virtual void SetUp() { + testing::Test::SetUp(); + + io_message_loop_proxy_ = base::MessageLoopProxy::current(); + +#if defined(USE_NSS) + crypto::EnsureNSSInit(); + net::EnsureOCSPInit(); +#endif + } + + virtual void TearDown() { +#if defined(USE_NSS) + net::ShutdownOCSP(); +#endif + } + + int GetNumFetcherCores() const { + return URLFetcher::GetNumFetcherCores(); + } + + // URLFetcher is designed to run on the main UI thread, but in our tests + // we assume that the current thread is the IO thread where the URLFetcher + // dispatches its requests to. When we wish to simulate being used from + // a UI thread, we dispatch a worker thread to do so. + MessageLoopForIO io_loop_; + scoped_refptr io_message_loop_proxy_; + + URLFetcher* fetcher_; +}; + +void URLFetcherTest::CreateFetcher(const GURL& url) { + fetcher_ = new URLFetcher(url, URLFetcher::GET, this); + fetcher_->set_request_context(new TestURLRequestContextGetter( + io_message_loop_proxy())); + fetcher_->Start(); +} + +void URLFetcherTest::OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + EXPECT_TRUE(status.is_success()); + EXPECT_EQ(200, response_code); // HTTP OK + EXPECT_FALSE(data.empty()); + + delete fetcher_; // Have to delete this here and not in the destructor, + // because the destructor won't necessarily run on the + // same thread that CreateFetcher() did. + + io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); + // If the current message loop is not the IO loop, it will be shut down when + // the main loop returns and this thread subsequently goes out of scope. +} + +namespace { + +// Version of URLFetcherTest that does a POST instead +class URLFetcherPostTest : public URLFetcherTest { + public: + virtual void CreateFetcher(const GURL& url); + + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); +}; + +// Version of URLFetcherTest that tests headers. +class URLFetcherHeadersTest : public URLFetcherTest { + public: + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); +}; + +// Version of URLFetcherTest that tests SocketAddress. +class URLFetcherSocketAddressTest : public URLFetcherTest { + public: + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + protected: + std::string expected_host_; + uint16 expected_port_; +}; + +// Version of URLFetcherTest that tests overload protection. +class URLFetcherProtectTest : public URLFetcherTest { + public: + virtual void CreateFetcher(const GURL& url); + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + private: + Time start_time_; +}; + +// Version of URLFetcherTest that tests overload protection, when responses +// passed through. +class URLFetcherProtectTestPassedThrough : public URLFetcherTest { + public: + virtual void CreateFetcher(const GURL& url); + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + private: + Time start_time_; +}; + +// Version of URLFetcherTest that tests bad HTTPS requests. +class URLFetcherBadHTTPSTest : public URLFetcherTest { + public: + URLFetcherBadHTTPSTest(); + + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + + private: + FilePath cert_dir_; +}; + +// Version of URLFetcherTest that tests request cancellation on shutdown. +class URLFetcherCancelTest : public URLFetcherTest { + public: + virtual void CreateFetcher(const GURL& url); + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + + void CancelRequest(); +}; + +// Version of TestURLRequestContext that posts a Quit task to the IO +// thread once it is deleted. +class CancelTestURLRequestContext : public TestURLRequestContext { + virtual ~CancelTestURLRequestContext() { + // The d'tor should execute in the IO thread. Post the quit task to the + // current thread. + MessageLoop::current()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); + } +}; + +class CancelTestURLRequestContextGetter : public net::URLRequestContextGetter { + public: + explicit CancelTestURLRequestContextGetter( + base::MessageLoopProxy* io_message_loop_proxy) + : io_message_loop_proxy_(io_message_loop_proxy), + context_created_(false, false) { + } + virtual net::URLRequestContext* GetURLRequestContext() { + if (!context_) { + context_ = new CancelTestURLRequestContext(); + context_created_.Signal(); + } + return context_; + } + virtual scoped_refptr GetIOMessageLoopProxy() const { + return io_message_loop_proxy_; + } + void WaitForContextCreation() { + context_created_.Wait(); + } + + private: + ~CancelTestURLRequestContextGetter() {} + + scoped_refptr io_message_loop_proxy_; + base::WaitableEvent context_created_; + scoped_refptr context_; +}; + +// Version of URLFetcherTest that tests retying the same request twice. +class URLFetcherMultipleAttemptTest : public URLFetcherTest { + public: + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + private: + std::string data_; +}; + +class URLFetcherTempFileTest : public URLFetcherTest { + public: + URLFetcherTempFileTest() + : take_ownership_of_temp_file_(false) { + } + + // URLFetcher::Delegate + virtual void OnURLFetchComplete(const URLFetcher* source); + + // This obsolete signature should not be used, but must be present + // to make clang happy. + virtual void OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data); + + virtual void CreateFetcher(const GURL& url); + + protected: + FilePath expected_file_; + FilePath temp_file_; + + // Set by the test. Used in OnURLFetchComplete() to decide if + // the URLFetcher should own the temp file, so that we can test + // disowning prevents the file from being deleted. + bool take_ownership_of_temp_file_; +}; + +void URLFetcherTempFileTest::CreateFetcher(const GURL& url) { + fetcher_ = new URLFetcher(url, URLFetcher::GET, this); + fetcher_->set_request_context(new TestURLRequestContextGetter( + io_message_loop_proxy())); + + // Use the IO message loop to do the file operations in this test. + fetcher_->SaveResponseToTemporaryFile(io_message_loop_proxy()); + fetcher_->Start(); +} + +TEST_F(URLFetcherTempFileTest, SmallGet) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + // Get a small file. + const char* kFileToFetch = "simple.html"; + expected_file_ = test_server.document_root().AppendASCII(kFileToFetch); + CreateFetcher(test_server.GetURL(kTestServerFilePrefix + kFileToFetch)); + + MessageLoop::current()->Run(); // OnURLFetchComplete() will Quit(). + + ASSERT_FALSE(file_util::PathExists(temp_file_)) + << temp_file_.value() << " not removed."; +} + +TEST_F(URLFetcherTempFileTest, LargeGet) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + // Get a file large enough to require more than one read into + // URLFetcher::Core's IOBuffer. + const char* kFileToFetch = "animate1.gif"; + expected_file_ = test_server.document_root().AppendASCII(kFileToFetch); + CreateFetcher(test_server.GetURL(kTestServerFilePrefix + kFileToFetch)); + + MessageLoop::current()->Run(); // OnURLFetchComplete() will Quit(). +} + +TEST_F(URLFetcherTempFileTest, CanTakeOwnershipOfFile) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + // Get a small file. + const char* kFileToFetch = "simple.html"; + expected_file_ = test_server.document_root().AppendASCII(kFileToFetch); + CreateFetcher(test_server.GetURL(kTestServerFilePrefix + kFileToFetch)); + + MessageLoop::current()->Run(); // OnURLFetchComplete() will Quit(). + + MessageLoop::current()->RunAllPending(); + ASSERT_FALSE(file_util::PathExists(temp_file_)) + << temp_file_.value() << " not removed."; +} + +// Wrapper that lets us call CreateFetcher() on a thread of our choice. We +// could make URLFetcherTest refcounted and use PostTask(FROM_HERE.. ) to call +// CreateFetcher() directly, but the ownership of the URLFetcherTest is a bit +// confusing in that case because GTest doesn't know about the refcounting. +// It's less confusing to just do it this way. +class FetcherWrapperTask : public Task { + public: + FetcherWrapperTask(URLFetcherTest* test, const GURL& url) + : test_(test), url_(url) { } + virtual void Run() { + test_->CreateFetcher(url_); + } + + private: + URLFetcherTest* test_; + GURL url_; +}; + +void URLFetcherPostTest::CreateFetcher(const GURL& url) { + fetcher_ = new URLFetcher(url, URLFetcher::POST, this); + fetcher_->set_request_context(new TestURLRequestContextGetter( + io_message_loop_proxy())); + fetcher_->set_upload_data("application/x-www-form-urlencoded", + "bobsyeruncle"); + fetcher_->Start(); +} + +void URLFetcherPostTest::OnURLFetchComplete(const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + EXPECT_EQ(std::string("bobsyeruncle"), data); + URLFetcherTest::OnURLFetchComplete(source, url, status, response_code, + cookies, data); +} + +void URLFetcherHeadersTest::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + std::string header; + EXPECT_TRUE(source->response_headers()->GetNormalizedHeader("cache-control", + &header)); + EXPECT_EQ("private", header); + URLFetcherTest::OnURLFetchComplete(source, url, status, response_code, + cookies, data); +} + +void URLFetcherSocketAddressTest::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + EXPECT_EQ("127.0.0.1", source->socket_address().host()); + EXPECT_EQ(expected_port_, source->socket_address().port()); + URLFetcherTest::OnURLFetchComplete(source, url, status, response_code, + cookies, data); +} + +void URLFetcherProtectTest::CreateFetcher(const GURL& url) { + fetcher_ = new URLFetcher(url, URLFetcher::GET, this); + fetcher_->set_request_context(new TestURLRequestContextGetter( + io_message_loop_proxy())); + start_time_ = Time::Now(); + fetcher_->set_max_retries(11); + fetcher_->Start(); +} + +void URLFetcherProtectTest::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + const TimeDelta one_second = TimeDelta::FromMilliseconds(1000); + if (response_code >= 500) { + // Now running ServerUnavailable test. + // It takes more than 1 second to finish all 11 requests. + EXPECT_TRUE(Time::Now() - start_time_ >= one_second); + EXPECT_TRUE(status.is_success()); + EXPECT_FALSE(data.empty()); + delete fetcher_; + io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); + } else { + // Now running Overload test. + static int count = 0; + count++; + if (count < 20) { + fetcher_->StartWithRequestContextGetter(new TestURLRequestContextGetter( + io_message_loop_proxy())); + } else { + // We have already sent 20 requests continuously. And we expect that + // it takes more than 1 second due to the overload protection settings. + EXPECT_TRUE(Time::Now() - start_time_ >= one_second); + URLFetcherTest::OnURLFetchComplete(source, url, status, response_code, + cookies, data); + } + } +} + +void URLFetcherProtectTestPassedThrough::CreateFetcher(const GURL& url) { + fetcher_ = new URLFetcher(url, URLFetcher::GET, this); + fetcher_->set_request_context(new TestURLRequestContextGetter( + io_message_loop_proxy())); + fetcher_->set_automatically_retry_on_5xx(false); + start_time_ = Time::Now(); + fetcher_->set_max_retries(11); + fetcher_->Start(); +} + +void URLFetcherProtectTestPassedThrough::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + const TimeDelta one_minute = TimeDelta::FromMilliseconds(60000); + if (response_code >= 500) { + // Now running ServerUnavailable test. + // It should get here on the first attempt, so almost immediately and + // *not* to attempt to execute all 11 requests (2.5 minutes). + EXPECT_TRUE(Time::Now() - start_time_ < one_minute); + EXPECT_TRUE(status.is_success()); + // Check that suggested back off time is bigger than 0. + EXPECT_GT(fetcher_->backoff_delay().InMicroseconds(), 0); + EXPECT_FALSE(data.empty()); + } else { + // We should not get here! + ADD_FAILURE(); + } + + delete fetcher_; + io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); +} + + +URLFetcherBadHTTPSTest::URLFetcherBadHTTPSTest() { + PathService::Get(base::DIR_SOURCE_ROOT, &cert_dir_); + cert_dir_ = cert_dir_.AppendASCII("chrome"); + cert_dir_ = cert_dir_.AppendASCII("test"); + cert_dir_ = cert_dir_.AppendASCII("data"); + cert_dir_ = cert_dir_.AppendASCII("ssl"); + cert_dir_ = cert_dir_.AppendASCII("certificates"); +} + +// The "server certificate expired" error should result in automatic +// cancellation of the request by +// net::URLRequest::Delegate::OnSSLCertificateError. +void URLFetcherBadHTTPSTest::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + // This part is different from URLFetcherTest::OnURLFetchComplete + // because this test expects the request to be cancelled. + EXPECT_EQ(net::URLRequestStatus::CANCELED, status.status()); + EXPECT_EQ(net::ERR_ABORTED, status.error()); + EXPECT_EQ(-1, response_code); + EXPECT_TRUE(cookies.empty()); + EXPECT_TRUE(data.empty()); + + // The rest is the same as URLFetcherTest::OnURLFetchComplete. + delete fetcher_; + io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); +} + +void URLFetcherCancelTest::CreateFetcher(const GURL& url) { + fetcher_ = new URLFetcher(url, URLFetcher::GET, this); + CancelTestURLRequestContextGetter* context_getter = + new CancelTestURLRequestContextGetter(io_message_loop_proxy()); + fetcher_->set_request_context(context_getter); + fetcher_->set_max_retries(2); + fetcher_->Start(); + // We need to wait for the creation of the net::URLRequestContext, since we + // rely on it being destroyed as a signal to end the test. + context_getter->WaitForContextCreation(); + CancelRequest(); +} + +void URLFetcherCancelTest::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + // We should have cancelled the request before completion. + ADD_FAILURE(); + delete fetcher_; + io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); +} + +void URLFetcherCancelTest::CancelRequest() { + delete fetcher_; + // The URLFetcher's test context will post a Quit task once it is + // deleted. So if this test simply hangs, it means cancellation + // did not work. +} + +void URLFetcherMultipleAttemptTest::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + EXPECT_TRUE(status.is_success()); + EXPECT_EQ(200, response_code); // HTTP OK + EXPECT_FALSE(data.empty()); + if (!data.empty() && data_.empty()) { + data_ = data; + fetcher_->StartWithRequestContextGetter( + new TestURLRequestContextGetter(io_message_loop_proxy())); + } else { + EXPECT_EQ(data, data_); + delete fetcher_; // Have to delete this here and not in the destructor, + // because the destructor won't necessarily run on the + // same thread that CreateFetcher() did. + + io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); + // If the current message loop is not the IO loop, it will be shut down when + // the main loop returns and this thread subsequently goes out of scope. + } +} + +void URLFetcherTempFileTest::OnURLFetchComplete(const URLFetcher* source) { + EXPECT_TRUE(source->status().is_success()); + EXPECT_EQ(source->response_code(), 200); + + EXPECT_TRUE(source->GetResponseAsFilePath( + take_ownership_of_temp_file_, &temp_file_)); + + EXPECT_TRUE(file_util::ContentsEqual(expected_file_, temp_file_)); + + delete fetcher_; + + io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); +} + +void URLFetcherTempFileTest::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + NOTREACHED(); +} + + +TEST_F(URLFetcherTest, SameThreadsTest) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + // Create the fetcher on the main thread. Since IO will happen on the main + // thread, this will test URLFetcher's ability to do everything on one + // thread. + CreateFetcher(test_server.GetURL("defaultresponse")); + + MessageLoop::current()->Run(); +} + +#if defined(OS_MACOSX) +// SIGSEGV on Mac: http://crbug.com/60426 +TEST_F(URLFetcherTest, DISABLED_DifferentThreadsTest) { +#else +TEST_F(URLFetcherTest, DifferentThreadsTest) { +#endif + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + // Create a separate thread that will create the URLFetcher. The current + // (main) thread will do the IO, and when the fetch is complete it will + // terminate the main thread's message loop; then the other thread's + // message loop will be shut down automatically as the thread goes out of + // scope. + base::Thread t("URLFetcher test thread"); + ASSERT_TRUE(t.Start()); + t.message_loop()->PostTask(FROM_HERE, new FetcherWrapperTask(this, + test_server.GetURL("defaultresponse"))); + + MessageLoop::current()->Run(); +} + +#if defined(OS_MACOSX) +// SIGSEGV on Mac: http://crbug.com/60426 +TEST_F(URLFetcherPostTest, DISABLED_Basic) { +#else +TEST_F(URLFetcherPostTest, Basic) { +#endif + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + CreateFetcher(test_server.GetURL("echo")); + MessageLoop::current()->Run(); +} + +TEST_F(URLFetcherHeadersTest, Headers) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, + FilePath(FILE_PATH_LITERAL("net/data/url_request_unittest"))); + ASSERT_TRUE(test_server.Start()); + + CreateFetcher(test_server.GetURL("files/with-headers.html")); + MessageLoop::current()->Run(); + // The actual tests are in the URLFetcherHeadersTest fixture. +} + +TEST_F(URLFetcherSocketAddressTest, SocketAddress) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, + FilePath(FILE_PATH_LITERAL("net/data/url_request_unittest"))); + ASSERT_TRUE(test_server.Start()); + expected_port_ = test_server.host_port_pair().port(); + + // Reusing "with-headers.html" but doesn't really matter. + CreateFetcher(test_server.GetURL("files/with-headers.html")); + MessageLoop::current()->Run(); + // The actual tests are in the URLFetcherSocketAddressTest fixture. +} + +TEST_F(URLFetcherProtectTest, Overload) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + GURL url(test_server.GetURL("defaultresponse")); + + // Registers an entry for test url. It only allows 3 requests to be sent + // in 200 milliseconds. + net::URLRequestThrottlerManager* manager = + net::URLRequestThrottlerManager::GetInstance(); + scoped_refptr entry( + new net::URLRequestThrottlerEntry(manager, "", 200, 3, 1, 2.0, 0.0, 256)); + manager->OverrideEntryForTests(url, entry); + + CreateFetcher(url); + + MessageLoop::current()->Run(); + + net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); +} + +TEST_F(URLFetcherProtectTest, ServerUnavailable) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + GURL url(test_server.GetURL("files/server-unavailable.html")); + + // Registers an entry for test url. The backoff time is calculated by: + // new_backoff = 2.0 * old_backoff + 0 + // and maximum backoff time is 256 milliseconds. + // Maximum retries allowed is set to 11. + net::URLRequestThrottlerManager* manager = + net::URLRequestThrottlerManager::GetInstance(); + scoped_refptr entry( + new net::URLRequestThrottlerEntry(manager, "", 200, 3, 1, 2.0, 0.0, 256)); + manager->OverrideEntryForTests(url, entry); + + CreateFetcher(url); + + MessageLoop::current()->Run(); + + net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); +} + +TEST_F(URLFetcherProtectTestPassedThrough, ServerUnavailablePropagateResponse) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + GURL url(test_server.GetURL("files/server-unavailable.html")); + + // Registers an entry for test url. The backoff time is calculated by: + // new_backoff = 2.0 * old_backoff + 0 + // and maximum backoff time is 150000 milliseconds. + // Maximum retries allowed is set to 11. + net::URLRequestThrottlerManager* manager = + net::URLRequestThrottlerManager::GetInstance(); + scoped_refptr entry( + new net::URLRequestThrottlerEntry( + manager, "", 200, 3, 100, 2.0, 0.0, 150000)); + // Total time if *not* for not doing automatic backoff would be 150s. + // In reality it should be "as soon as server responds". + manager->OverrideEntryForTests(url, entry); + + CreateFetcher(url); + + MessageLoop::current()->Run(); + + net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); +} + +#if defined(OS_MACOSX) +// SIGSEGV on Mac: http://crbug.com/60426 +TEST_F(URLFetcherBadHTTPSTest, DISABLED_BadHTTPSTest) { +#else +TEST_F(URLFetcherBadHTTPSTest, BadHTTPSTest) { +#endif + net::TestServer::HTTPSOptions https_options( + net::TestServer::HTTPSOptions::CERT_EXPIRED); + net::TestServer test_server(https_options, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + CreateFetcher(test_server.GetURL("defaultresponse")); + MessageLoop::current()->Run(); +} + +#if defined(OS_MACOSX) +// SIGSEGV on Mac: http://crbug.com/60426 +TEST_F(URLFetcherCancelTest, DISABLED_ReleasesContext) { +#else +TEST_F(URLFetcherCancelTest, ReleasesContext) { +#endif + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + GURL url(test_server.GetURL("files/server-unavailable.html")); + + // Registers an entry for test url. The backoff time is calculated by: + // new_backoff = 2.0 * old_backoff +0 + // The initial backoff is 2 seconds and maximum backoff is 4 seconds. + // Maximum retries allowed is set to 2. + net::URLRequestThrottlerManager* manager = + net::URLRequestThrottlerManager::GetInstance(); + scoped_refptr entry( + new net::URLRequestThrottlerEntry( + manager, "", 200, 3, 2000, 2.0, 0.0, 4000)); + manager->OverrideEntryForTests(url, entry); + + // Create a separate thread that will create the URLFetcher. The current + // (main) thread will do the IO, and when the fetch is complete it will + // terminate the main thread's message loop; then the other thread's + // message loop will be shut down automatically as the thread goes out of + // scope. + base::Thread t("URLFetcher test thread"); + ASSERT_TRUE(t.Start()); + t.message_loop()->PostTask(FROM_HERE, new FetcherWrapperTask(this, url)); + + MessageLoop::current()->Run(); + + net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); +} + +TEST_F(URLFetcherCancelTest, CancelWhileDelayedStartTaskPending) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + GURL url(test_server.GetURL("files/server-unavailable.html")); + + // Register an entry for test url. + // Using a sliding window of 4 seconds, and max of 1 request, under a fast + // run we expect to have a 4 second delay when posting the Start task. + net::URLRequestThrottlerManager* manager = + net::URLRequestThrottlerManager::GetInstance(); + scoped_refptr entry( + new net::URLRequestThrottlerEntry( + manager, "", 4000, 1, 2000, 2.0, 0.0, 4000)); + manager->OverrideEntryForTests(url, entry); + // Fake that a request has just started. + entry->ReserveSendingTimeForNextRequest(base::TimeTicks()); + + // The next request we try to send will be delayed by ~4 seconds. + // The slower the test runs, the less the delay will be (since it takes the + // time difference from now). + + base::Thread t("URLFetcher test thread"); + ASSERT_TRUE(t.Start()); + t.message_loop()->PostTask(FROM_HERE, new FetcherWrapperTask(this, url)); + + MessageLoop::current()->Run(); + + net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); +} + +TEST_F(URLFetcherMultipleAttemptTest, SameData) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + + // Create the fetcher on the main thread. Since IO will happen on the main + // thread, this will test URLFetcher's ability to do everything on one + // thread. + CreateFetcher(test_server.GetURL("defaultresponse")); + + MessageLoop::current()->Run(); +} + +// Tests to make sure CancelAll() will successfully cancel existing URLFetchers. +TEST_F(URLFetcherTest, CancelAll) { + net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); + ASSERT_TRUE(test_server.Start()); + EXPECT_EQ(0, GetNumFetcherCores()); + + CreateFetcher(test_server.GetURL("defaultresponse")); + io_message_loop_proxy()->PostTask( + FROM_HERE, + new CurriedTask(new MessageLoop::QuitTask(), MessageLoop::current())); + MessageLoop::current()->Run(); + EXPECT_EQ(1, GetNumFetcherCores()); + URLFetcher::CancelAll(); + EXPECT_EQ(0, GetNumFetcherCores()); + delete fetcher_; +} + +} // namespace. diff --git a/content/common/url_fetcher.cc b/content/common/url_fetcher.cc deleted file mode 100644 index 4493c07..0000000 --- a/content/common/url_fetcher.cc +++ /dev/null @@ -1,1123 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "content/common/url_fetcher.h" - -#include - -#include "base/compiler_specific.h" -#include "base/file_path.h" -#include "base/file_util_proxy.h" -#include "base/lazy_instance.h" -#include "base/memory/scoped_callback_factory.h" -#include "base/memory/scoped_ptr.h" -#include "base/message_loop_proxy.h" -#include "base/platform_file.h" -#include "base/stl_util.h" -#include "base/string_util.h" -#include "base/threading/thread.h" -#include "googleurl/src/gurl.h" -#include "net/base/host_port_pair.h" -#include "net/base/io_buffer.h" -#include "net/base/load_flags.h" -#include "net/base/net_errors.h" -#include "net/http/http_request_headers.h" -#include "net/http/http_response_headers.h" -#include "net/url_request/url_request.h" -#include "net/url_request/url_request_context.h" -#include "net/url_request/url_request_context_getter.h" -#include "net/url_request/url_request_throttler_manager.h" - -static const int kBufferSize = 4096; -const int URLFetcher::kInvalidHttpResponseCode = -1; - -class URLFetcher::Core - : public base::RefCountedThreadSafe, - public net::URLRequest::Delegate { - public: - // For POST requests, set |content_type| to the MIME type of the content - // and set |content| to the data to upload. |flags| are flags to apply to - // the load operation--these should be one or more of the LOAD_* flags - // defined in net/base/load_flags.h. - Core(URLFetcher* fetcher, - const GURL& original_url, - RequestType request_type, - URLFetcher::Delegate* d); - - // Starts the load. It's important that this not happen in the constructor - // because it causes the IO thread to begin AddRef()ing and Release()ing - // us. If our caller hasn't had time to fully construct us and take a - // reference, the IO thread could interrupt things, run a task, Release() - // us, and destroy us, leaving the caller with an already-destroyed object - // when construction finishes. - void Start(); - - // Stops any in-progress load and ensures no callback will happen. It is - // safe to call this multiple times. - void Stop(); - - // Reports that the received content was malformed (i.e. failed parsing - // or validation). This makes the throttling logic that does exponential - // back-off when servers are having problems treat the current request as - // a failure. Your call to this method will be ignored if your request is - // already considered a failure based on the HTTP response code or response - // headers. - void ReceivedContentWasMalformed(); - - // Overridden from net::URLRequest::Delegate: - virtual void OnResponseStarted(net::URLRequest* request); - virtual void OnReadCompleted(net::URLRequest* request, int bytes_read); - - URLFetcher::Delegate* delegate() const { return delegate_; } - static void CancelAll(); - - private: - friend class base::RefCountedThreadSafe; - - class Registry { - public: - Registry(); - ~Registry(); - - void AddURLFetcherCore(Core* core); - void RemoveURLFetcherCore(Core* core); - - void CancelAll(); - - int size() const { - return fetchers_.size(); - } - - private: - std::set fetchers_; - - DISALLOW_COPY_AND_ASSIGN(Registry); - }; - - // Class TempFileWriter encapsulates all state involved in writing - // response bytes to a temporary file. It is only used if - // |Core::response_destination_| == TEMP_FILE. Each instance of - // TempFileWriter is owned by a URLFetcher::Core, which manages - // its lifetime and never transfers ownership. While writing to - // a file, all function calls happen on the IO thread. - class TempFileWriter { - public: - TempFileWriter( - URLFetcher::Core* core, - scoped_refptr file_message_loop_proxy); - - ~TempFileWriter(); - void CreateTempFile(); - void DidCreateTempFile(base::PlatformFileError error_code, - base::PassPlatformFile file_handle, - FilePath file_path); - - // Record |num_bytes_| response bytes in |core_->buffer_| to the file. - void WriteBuffer(int num_bytes); - - // Called when a write has been done. Continues writing if there are - // any more bytes to write. Otherwise, initiates a read in core_. - void ContinueWrite(base::PlatformFileError error_code, - int bytes_written); - - // Drop ownership of the file at path |temp_file_|. This class - // will not delete it or write to it again. - void DisownTempFile(); - - // Close the temp file if it is open. - void CloseTempFileAndCompleteRequest(); - - // Remove the temp file if we we created one. - void RemoveTempFile(); - - const FilePath& temp_file() const { return temp_file_; } - int64 total_bytes_written() { return total_bytes_written_; } - base::PlatformFileError error_code() const { return error_code_; } - - private: - // Callback which gets the result of closing the temp file. - void DidCloseTempFile(base::PlatformFileError error); - - // The URLFetcher::Core which instantiated this class. - URLFetcher::Core* core_; - - // The last error encountered on a file operation. base::PLATFORM_FILE_OK - // if no error occurred. - base::PlatformFileError error_code_; - - // Callbacks are created for use with base::FileUtilProxy. - base::ScopedCallbackFactory - callback_factory_; - - // Message loop on which file opperations should happen. - scoped_refptr file_message_loop_proxy_; - - // Path to the temporary file. This path is empty when there - // is no temp file. - FilePath temp_file_; - - // Handle to the temp file. - base::PlatformFile temp_file_handle_; - - // We always append to the file. Track the total number of bytes - // written, so that writes know the offset to give. - int64 total_bytes_written_; - - // How many bytes did the last Write() try to write? Needed so - // that if not all the bytes get written on a Write(), we can - // call Write() again with the rest. - int pending_bytes_; - - // When writing, how many bytes from the buffer have been successfully - // written so far? - int buffer_offset_; - }; - - virtual ~Core(); - - // Wrapper functions that allow us to ensure actions happen on the right - // thread. - void StartOnIOThread(); - void StartURLRequest(); - void StartURLRequestWhenAppropriate(); - void CancelURLRequest(); - void OnCompletedURLRequest(base::TimeDelta backoff_delay); - void InformDelegateFetchIsComplete(); - void NotifyMalformedContent(); - void RetryOrCompleteUrlFetch(); - - // Deletes the request, removes it from the registry, and removes the - // destruction observer. - void ReleaseRequest(); - - // Returns the max value of exponential back-off release time for - // |original_url_| and |url_|. - base::TimeTicks GetBackoffReleaseTime(); - - void CompleteAddingUploadDataChunk(const std::string& data, - bool is_last_chunk); - - // Adds a block of data to be uploaded in a POST body. This can only be - // called after Start(). - void AppendChunkToUpload(const std::string& data, bool is_last_chunk); - - // Store the response bytes in |buffer_| in the container indicated by - // |response_destination_|. Return true if the write has been - // done, and another read can overwrite |buffer_|. If this function - // returns false, it will post a task that will read more bytes once the - // write is complete. - bool WriteBuffer(int num_bytes); - - // Read response bytes from the request. - void ReadResponse(); - - // Drop ownership of any temp file managed by |temp_file_|. - void DisownTempFile(); - - URLFetcher* fetcher_; // Corresponding fetcher object - GURL original_url_; // The URL we were asked to fetch - GURL url_; // The URL we eventually wound up at - RequestType request_type_; // What type of request is this? - net::URLRequestStatus status_; // Status of the request - URLFetcher::Delegate* delegate_; // Object to notify on completion - scoped_refptr delegate_loop_proxy_; - // Message loop proxy of the creating - // thread. - scoped_refptr io_message_loop_proxy_; - // The message loop proxy for the thread - // on which the request IO happens. - scoped_refptr file_message_loop_proxy_; - // The message loop proxy for the thread - // on which file access happens. - scoped_ptr request_; // The actual request this wraps - int load_flags_; // Flags for the load operation - int response_code_; // HTTP status code for the request - std::string data_; // Results of the request, when we are - // storing the response as a string. - scoped_refptr buffer_; - // Read buffer - scoped_refptr request_context_getter_; - // Cookie/cache info for the request - net::ResponseCookies cookies_; // Response cookies - net::HttpRequestHeaders extra_request_headers_; - scoped_refptr response_headers_; - bool was_fetched_via_proxy_; - net::HostPortPair socket_address_; - - std::string upload_content_; // HTTP POST payload - std::string upload_content_type_; // MIME type of POST payload - std::string referrer_; // HTTP Referer header value - bool is_chunked_upload_; // True if using chunked transfer encoding - - // Used to determine how long to wait before making a request or doing a - // retry. - // Both of them can only be accessed on the IO thread. - // We need not only the throttler entry for |original_URL|, but also the one - // for |url|. For example, consider the case that URL A redirects to URL B, - // for which the server returns a 500 response. In this case, the exponential - // back-off release time of URL A won't increase. If we retry without - // considering the back-off constraint of URL B, we may send out too many - // requests for URL A in a short period of time. - scoped_refptr - original_url_throttler_entry_; - scoped_refptr url_throttler_entry_; - - // |num_retries_| indicates how many times we've failed to successfully - // fetch this URL. Once this value exceeds the maximum number of retries - // specified by the owner URLFetcher instance, we'll give up. - int num_retries_; - - // True if the URLFetcher has been cancelled. - bool was_cancelled_; - - // If writing results to a file, |temp_file_writer_| will manage creation, - // writing, and destruction of that file. - scoped_ptr temp_file_writer_; - - // Where should responses be saved? - ResponseDestinationType response_destination_; - - // If |automatically_retry_on_5xx_| is false, 5xx responses will be - // propagated to the observer, if it is true URLFetcher will automatically - // re-execute the request, after the back-off delay has expired. - // true by default. - bool automatically_retry_on_5xx_; - // Maximum retries allowed. - int max_retries_; - - static base::LazyInstance g_registry; - - friend class URLFetcher; - DISALLOW_COPY_AND_ASSIGN(Core); -}; - -URLFetcher::Core::Registry::Registry() {} -URLFetcher::Core::Registry::~Registry() {} - -void URLFetcher::Core::Registry::AddURLFetcherCore(Core* core) { - DCHECK(!ContainsKey(fetchers_, core)); - fetchers_.insert(core); -} - -void URLFetcher::Core::Registry::RemoveURLFetcherCore(Core* core) { - DCHECK(ContainsKey(fetchers_, core)); - fetchers_.erase(core); -} - -void URLFetcher::Core::Registry::CancelAll() { - while (!fetchers_.empty()) - (*fetchers_.begin())->CancelURLRequest(); -} - -// static -base::LazyInstance - URLFetcher::Core::g_registry(base::LINKER_INITIALIZED); - -URLFetcher::Core::TempFileWriter::TempFileWriter( - URLFetcher::Core* core, - scoped_refptr file_message_loop_proxy) - : core_(core), - error_code_(base::PLATFORM_FILE_OK), - callback_factory_(ALLOW_THIS_IN_INITIALIZER_LIST(this)), - file_message_loop_proxy_(file_message_loop_proxy), - temp_file_handle_(base::kInvalidPlatformFileValue) { -} - -URLFetcher::Core::TempFileWriter::~TempFileWriter() { - RemoveTempFile(); -} - -void URLFetcher::Core::TempFileWriter::CreateTempFile() { - DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); - CHECK(file_message_loop_proxy_.get()); - base::FileUtilProxy::CreateTemporary( - file_message_loop_proxy_, - 0, // No additional file flags. - callback_factory_.NewCallback( - &URLFetcher::Core::TempFileWriter::DidCreateTempFile)); -} - -void URLFetcher::Core::TempFileWriter::DidCreateTempFile( - base::PlatformFileError error_code, - base::PassPlatformFile file_handle, - FilePath file_path) { - DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); - - if (base::PLATFORM_FILE_OK != error_code) { - error_code_ = error_code; - RemoveTempFile(); - core_->delegate_loop_proxy_->PostTask( - FROM_HERE, - NewRunnableMethod(core_, &Core::InformDelegateFetchIsComplete)); - return; - } - - temp_file_ = file_path; - temp_file_handle_ = file_handle.ReleaseValue(); - total_bytes_written_ = 0; - - core_->io_message_loop_proxy_->PostTask( - FROM_HERE, - NewRunnableMethod(core_, &Core::StartURLRequestWhenAppropriate)); -} - -void URLFetcher::Core::TempFileWriter::WriteBuffer(int num_bytes) { - DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); - - // Start writing to the temp file by setting the initial state - // of |pending_bytes_| and |buffer_offset_| to indicate that the - // entire buffer has not yet been written. - pending_bytes_ = num_bytes; - buffer_offset_ = 0; - ContinueWrite(base::PLATFORM_FILE_OK, 0); -} - -void URLFetcher::Core::TempFileWriter::ContinueWrite( - base::PlatformFileError error_code, - int bytes_written) { - DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); - - if (base::PLATFORM_FILE_OK != error_code) { - error_code_ = error_code; - RemoveTempFile(); - core_->delegate_loop_proxy_->PostTask( - FROM_HERE, - NewRunnableMethod(core_, &Core::InformDelegateFetchIsComplete)); - return; - } - - total_bytes_written_ += bytes_written; - buffer_offset_ += bytes_written; - pending_bytes_ -= bytes_written; - - if (pending_bytes_ > 0) { - base::FileUtilProxy::Write( - file_message_loop_proxy_, - temp_file_handle_, - total_bytes_written_, // Append to the end - (core_->buffer_->data() + buffer_offset_), - pending_bytes_, - callback_factory_.NewCallback( - &URLFetcher::Core::TempFileWriter::ContinueWrite)); - } else { - // Finished writing core_->buffer_ to the file. Read some more. - core_->ReadResponse(); - } -} - -void URLFetcher::Core::TempFileWriter::DisownTempFile() { - DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); - - // Disowning is done by the delegate's OnURLFetchComplete method. - // The temp file should be closed by the time that method is called. - DCHECK(temp_file_handle_ == base::kInvalidPlatformFileValue); - - // Forget about any temp file by reseting the path. - temp_file_ = FilePath(); -} - -void URLFetcher::Core::TempFileWriter::CloseTempFileAndCompleteRequest() { - DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); - - if (temp_file_handle_ != base::kInvalidPlatformFileValue) { - base::FileUtilProxy::Close( - file_message_loop_proxy_, - temp_file_handle_, - callback_factory_.NewCallback( - &URLFetcher::Core::TempFileWriter::DidCloseTempFile)); - temp_file_handle_ = base::kInvalidPlatformFileValue; - } -} - -void URLFetcher::Core::TempFileWriter::DidCloseTempFile( - base::PlatformFileError error_code) { - DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); - - if (base::PLATFORM_FILE_OK != error_code) { - error_code_ = error_code; - RemoveTempFile(); - core_->delegate_loop_proxy_->PostTask( - FROM_HERE, - NewRunnableMethod(core_, &Core::InformDelegateFetchIsComplete)); - return; - } - - // If the file was successfully closed, then the URL request is complete. - core_->RetryOrCompleteUrlFetch(); -} - -void URLFetcher::Core::TempFileWriter::RemoveTempFile() { - DCHECK(core_->io_message_loop_proxy_->BelongsToCurrentThread()); - - // Close the temp file if it is open. - if (temp_file_handle_ != base::kInvalidPlatformFileValue) { - base::FileUtilProxy::Close( - file_message_loop_proxy_, - temp_file_handle_, - NULL); // No callback: Ignore errors. - temp_file_handle_ = base::kInvalidPlatformFileValue; - } - - if (!temp_file_.empty()) { - base::FileUtilProxy::Delete( - file_message_loop_proxy_, - temp_file_, - false, // No need to recurse, as the path is to a file. - NULL); // No callback: Ignore errors. - DisownTempFile(); - } -} - -// static -URLFetcher::Factory* URLFetcher::factory_ = NULL; - -void URLFetcher::Delegate::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - NOTREACHED() << "If you don't implemnt this, the no-params version " - << "should also be implemented, in which case this " - << "method won't be called..."; -} - -// TODO(skerner): This default implementation will be removed, and the -// method made pure virtual, once all users of URLFetcher are updated -// to not expect response data as a string argument. Once this is removed, -// the method URLFetcher::GetResponseStringRef() can be removed as well. -// crbug.com/83592 tracks this. -void URLFetcher::Delegate::OnURLFetchComplete(const URLFetcher* source) { - // A delegate that did not override this method is using the old - // parameter list to OnURLFetchComplete(). If a user asked to save - // the response to a file, they must use the new parameter list, - // in which case we can not get here. - // To avoid updating all callers, thunk to the old prototype for now. - OnURLFetchComplete(source, - source->url(), - source->status(), - source->response_code(), - source->cookies(), - source->GetResponseStringRef()); -} - -// static -bool URLFetcher::g_interception_enabled = false; - -URLFetcher::URLFetcher(const GURL& url, - RequestType request_type, - Delegate* d) - : ALLOW_THIS_IN_INITIALIZER_LIST( - core_(new Core(this, url, request_type, d))) { -} - -URLFetcher::~URLFetcher() { - core_->Stop(); -} - -// static -URLFetcher* URLFetcher::Create(int id, const GURL& url, - RequestType request_type, Delegate* d) { - return factory_ ? factory_->CreateURLFetcher(id, url, request_type, d) : - new URLFetcher(url, request_type, d); -} - -URLFetcher::Core::Core(URLFetcher* fetcher, - const GURL& original_url, - RequestType request_type, - URLFetcher::Delegate* d) - : fetcher_(fetcher), - original_url_(original_url), - request_type_(request_type), - delegate_(d), - delegate_loop_proxy_( - base::MessageLoopProxy::current()), - request_(NULL), - load_flags_(net::LOAD_NORMAL), - response_code_(URLFetcher::kInvalidHttpResponseCode), - buffer_(new net::IOBuffer(kBufferSize)), - was_fetched_via_proxy_(false), - is_chunked_upload_(false), - num_retries_(0), - was_cancelled_(false), - response_destination_(STRING), - automatically_retry_on_5xx_(true), - max_retries_(0) { -} - -URLFetcher::Core::~Core() { - // |request_| should be NULL. If not, it's unsafe to delete it here since we - // may not be on the IO thread. - DCHECK(!request_.get()); -} - -void URLFetcher::Core::Start() { - DCHECK(delegate_loop_proxy_); - CHECK(request_context_getter_) << "We need an URLRequestContext!"; - if (io_message_loop_proxy_) { - DCHECK_EQ(io_message_loop_proxy_, - request_context_getter_->GetIOMessageLoopProxy()); - } else { - io_message_loop_proxy_ = request_context_getter_->GetIOMessageLoopProxy(); - } - CHECK(io_message_loop_proxy_.get()) << "We need an IO message loop proxy"; - - io_message_loop_proxy_->PostTask( - FROM_HERE, - NewRunnableMethod(this, &Core::StartOnIOThread)); -} - -void URLFetcher::Core::StartOnIOThread() { - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - - switch (response_destination_) { - case STRING: - StartURLRequestWhenAppropriate(); - break; - - case TEMP_FILE: - DCHECK(file_message_loop_proxy_.get()) - << "Need to set the file message loop proxy."; - - temp_file_writer_.reset( - new TempFileWriter(this, file_message_loop_proxy_)); - - // If the temp file is successfully created, - // Core::StartURLRequestWhenAppropriate() will be called. - temp_file_writer_->CreateTempFile(); - break; - - default: - NOTREACHED(); - } -} - -void URLFetcher::Core::Stop() { - DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); - delegate_ = NULL; - fetcher_ = NULL; - if (io_message_loop_proxy_.get()) { - io_message_loop_proxy_->PostTask( - FROM_HERE, NewRunnableMethod(this, &Core::CancelURLRequest)); - } -} - -void URLFetcher::Core::ReceivedContentWasMalformed() { - DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); - if (io_message_loop_proxy_.get()) { - io_message_loop_proxy_->PostTask( - FROM_HERE, NewRunnableMethod(this, &Core::NotifyMalformedContent)); - } -} - -void URLFetcher::Core::CancelAll() { - g_registry.Get().CancelAll(); -} - -void URLFetcher::Core::OnResponseStarted(net::URLRequest* request) { - DCHECK_EQ(request, request_.get()); - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - if (request_->status().is_success()) { - response_code_ = request_->GetResponseCode(); - response_headers_ = request_->response_headers(); - socket_address_ = request_->GetSocketAddress(); - was_fetched_via_proxy_ = request_->was_fetched_via_proxy(); - } - - ReadResponse(); -} - -void URLFetcher::Core::CompleteAddingUploadDataChunk( - const std::string& content, bool is_last_chunk) { - DCHECK(is_chunked_upload_); - DCHECK(request_.get()); - DCHECK(!content.empty()); - request_->AppendChunkToUpload(content.data(), - static_cast(content.length()), - is_last_chunk); -} - -void URLFetcher::Core::AppendChunkToUpload(const std::string& content, - bool is_last_chunk) { - DCHECK(delegate_loop_proxy_); - CHECK(io_message_loop_proxy_.get()); - io_message_loop_proxy_->PostTask( - FROM_HERE, - NewRunnableMethod(this, &Core::CompleteAddingUploadDataChunk, content, - is_last_chunk)); -} - -// Return true if the write was done and reading may continue. -// Return false if the write is pending, and the next read will -// be done later. -bool URLFetcher::Core::WriteBuffer(int num_bytes) { - bool write_complete = false; - switch (response_destination_) { - case STRING: - data_.append(buffer_->data(), num_bytes); - write_complete = true; - break; - - case TEMP_FILE: - temp_file_writer_->WriteBuffer(num_bytes); - // WriteBuffer() sends a request the file thread. - // The write is not done yet. - write_complete = false; - break; - - default: - NOTREACHED(); - } - return write_complete; -} - -void URLFetcher::Core::OnReadCompleted(net::URLRequest* request, - int bytes_read) { - DCHECK(request == request_); - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - - url_ = request->url(); - url_throttler_entry_ = - net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(url_); - - bool waiting_on_write = false; - do { - if (!request_->status().is_success() || bytes_read <= 0) - break; - - if (!WriteBuffer(bytes_read)) { - // If WriteBuffer() returns false, we have a pending write to - // wait on before reading further. - waiting_on_write = true; - break; - } - } while (request_->Read(buffer_, kBufferSize, &bytes_read)); - - const net::URLRequestStatus status = request_->status(); - - if (status.is_success()) - request_->GetResponseCookies(&cookies_); - - // See comments re: HEAD requests in ReadResponse(). - if ((!status.is_io_pending() && !waiting_on_write) || - (request_type_ == HEAD)) { - status_ = status; - ReleaseRequest(); - - // If a temp file is open, close it. - if (temp_file_writer_.get()) { - // If the file is open, close it. After closing the file, - // RetryOrCompleteUrlFetch() will be called. - temp_file_writer_->CloseTempFileAndCompleteRequest(); - } else { - // Otherwise, complete or retry the URL request directly. - RetryOrCompleteUrlFetch(); - } - } -} - -void URLFetcher::Core::RetryOrCompleteUrlFetch() { - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - base::TimeDelta backoff_delay; - - // Checks the response from server. - if (response_code_ >= 500 || - status_.error() == net::ERR_TEMPORARILY_THROTTLED) { - // When encountering a server error, we will send the request again - // after backoff time. - ++num_retries_; - - // Note that backoff_delay_ may be 0 because (a) the URLRequestThrottler - // code does not necessarily back off on the first error, and (b) it - // only backs off on some of the 5xx status codes. - base::TimeTicks backoff_release_time = GetBackoffReleaseTime(); - backoff_delay = backoff_release_time - base::TimeTicks::Now(); - if (backoff_delay < base::TimeDelta()) - backoff_delay = base::TimeDelta(); - - if (automatically_retry_on_5xx_ && - num_retries_ <= max_retries_) { - StartOnIOThread(); - return; - } - } else { - backoff_delay = base::TimeDelta(); - } - request_context_getter_ = NULL; - bool posted = delegate_loop_proxy_->PostTask( - FROM_HERE, - NewRunnableMethod(this, - &Core::OnCompletedURLRequest, - backoff_delay)); - - // If the delegate message loop does not exist any more, then the delegate - // should be gone too. - DCHECK(posted || !delegate_); -} - -void URLFetcher::Core::ReadResponse() { - // Some servers may treat HEAD requests as GET requests. To free up the - // network connection as soon as possible, signal that the request has - // completed immediately, without trying to read any data back (all we care - // about is the response code and headers, which we already have). - int bytes_read = 0; - if (request_->status().is_success() && (request_type_ != HEAD)) - request_->Read(buffer_, kBufferSize, &bytes_read); - OnReadCompleted(request_.get(), bytes_read); -} - -void URLFetcher::Core::DisownTempFile() { - temp_file_writer_->DisownTempFile(); -} - -void URLFetcher::Core::StartURLRequest() { - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - - if (was_cancelled_) { - // Since StartURLRequest() is posted as a *delayed* task, it may - // run after the URLFetcher was already stopped. - return; - } - - CHECK(request_context_getter_); - DCHECK(!request_.get()); - - g_registry.Get().AddURLFetcherCore(this); - request_.reset(new net::URLRequest(original_url_, this)); - int flags = request_->load_flags() | load_flags_; - if (!g_interception_enabled) { - flags = flags | net::LOAD_DISABLE_INTERCEPT; - } - if (is_chunked_upload_) - request_->EnableChunkedUpload(); - request_->set_load_flags(flags); - request_->set_context(request_context_getter_->GetURLRequestContext()); - request_->set_referrer(referrer_); - - switch (request_type_) { - case GET: - break; - - case POST: - DCHECK(!upload_content_.empty() || is_chunked_upload_); - DCHECK(!upload_content_type_.empty()); - - request_->set_method("POST"); - extra_request_headers_.SetHeader(net::HttpRequestHeaders::kContentType, - upload_content_type_); - if (!upload_content_.empty()) { - request_->AppendBytesToUpload( - upload_content_.data(), static_cast(upload_content_.length())); - } - break; - - case HEAD: - request_->set_method("HEAD"); - break; - - default: - NOTREACHED(); - } - - if (!extra_request_headers_.IsEmpty()) - request_->SetExtraRequestHeaders(extra_request_headers_); - - // There might be data left over from a previous request attempt. - data_.clear(); - - // If we are writing the response to a file, the only caller - // of this function should have created it and not written yet. - CHECK(!temp_file_writer_.get() || - temp_file_writer_->total_bytes_written() == 0); - - request_->Start(); -} - -void URLFetcher::Core::StartURLRequestWhenAppropriate() { - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - - if (was_cancelled_) - return; - - if (original_url_throttler_entry_ == NULL) { - original_url_throttler_entry_ = - net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl( - original_url_); - } - - int64 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest( - GetBackoffReleaseTime()); - if (delay == 0) { - StartURLRequest(); - } else { - MessageLoop::current()->PostDelayedTask( - FROM_HERE, - NewRunnableMethod(this, &Core::StartURLRequest), - delay); - } -} - -void URLFetcher::Core::CancelURLRequest() { - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - - if (request_.get()) { - request_->Cancel(); - ReleaseRequest(); - } - // Release the reference to the request context. There could be multiple - // references to URLFetcher::Core at this point so it may take a while to - // delete the object, but we cannot delay the destruction of the request - // context. - request_context_getter_ = NULL; - was_cancelled_ = true; - temp_file_writer_.reset(); -} - -void URLFetcher::Core::OnCompletedURLRequest( - base::TimeDelta backoff_delay) { - DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); - - // Save the status and backoff_delay so that delegates can read it. - if (delegate_) { - fetcher_->backoff_delay_ = backoff_delay; - InformDelegateFetchIsComplete(); - } -} - -void URLFetcher::Core::InformDelegateFetchIsComplete() { - CHECK(delegate_loop_proxy_->BelongsToCurrentThread()); - if (delegate_) { - delegate_->OnURLFetchComplete(fetcher_); - } -} - -void URLFetcher::Core::NotifyMalformedContent() { - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - if (url_throttler_entry_ != NULL) { - int status_code = response_code_; - if (status_code == kInvalidHttpResponseCode) { - // The status code will generally be known by the time clients - // call the |ReceivedContentWasMalformed()| function (which ends up - // calling the current function) but if it's not, we need to assume - // the response was successful so that the total failure count - // used to calculate exponential back-off goes up. - status_code = 200; - } - url_throttler_entry_->ReceivedContentWasMalformed(status_code); - } -} - -void URLFetcher::Core::ReleaseRequest() { - request_.reset(); - g_registry.Get().RemoveURLFetcherCore(this); -} - -base::TimeTicks URLFetcher::Core::GetBackoffReleaseTime() { - DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); - DCHECK(original_url_throttler_entry_ != NULL); - - base::TimeTicks original_url_backoff = - original_url_throttler_entry_->GetExponentialBackoffReleaseTime(); - base::TimeTicks destination_url_backoff; - if (url_throttler_entry_ != NULL && - original_url_throttler_entry_ != url_throttler_entry_) { - destination_url_backoff = - url_throttler_entry_->GetExponentialBackoffReleaseTime(); - } - - return original_url_backoff > destination_url_backoff ? - original_url_backoff : destination_url_backoff; -} - -void URLFetcher::set_upload_data(const std::string& upload_content_type, - const std::string& upload_content) { - DCHECK(!core_->is_chunked_upload_); - core_->upload_content_type_ = upload_content_type; - core_->upload_content_ = upload_content; -} - -void URLFetcher::set_chunked_upload(const std::string& content_type) { - DCHECK(core_->is_chunked_upload_ || - (core_->upload_content_type_.empty() && - core_->upload_content_.empty())); - core_->upload_content_type_ = content_type; - core_->upload_content_.clear(); - core_->is_chunked_upload_ = true; -} - -void URLFetcher::AppendChunkToUpload(const std::string& data, - bool is_last_chunk) { - DCHECK(data.length()); - core_->AppendChunkToUpload(data, is_last_chunk); -} - -const std::string& URLFetcher::upload_data() const { - return core_->upload_content_; -} - -void URLFetcher::set_referrer(const std::string& referrer) { - core_->referrer_ = referrer; -} - -void URLFetcher::set_load_flags(int load_flags) { - core_->load_flags_ = load_flags; -} - -int URLFetcher::load_flags() const { - return core_->load_flags_; -} - -void URLFetcher::set_extra_request_headers( - const std::string& extra_request_headers) { - core_->extra_request_headers_.Clear(); - core_->extra_request_headers_.AddHeadersFromString(extra_request_headers); -} - -void URLFetcher::GetExtraRequestHeaders(net::HttpRequestHeaders* headers) { - headers->CopyFrom(core_->extra_request_headers_); -} - -void URLFetcher::set_request_context( - net::URLRequestContextGetter* request_context_getter) { - DCHECK(!core_->request_context_getter_); - core_->request_context_getter_ = request_context_getter; -} - -void URLFetcher::set_automatically_retry_on_5xx(bool retry) { - core_->automatically_retry_on_5xx_ = retry; -} - -int URLFetcher::max_retries() const { - return core_->max_retries_; -} - -void URLFetcher::set_max_retries(int max_retries) { - core_->max_retries_ = max_retries; -} - -void URLFetcher::SaveResponseToTemporaryFile( - scoped_refptr file_message_loop_proxy) { - core_->file_message_loop_proxy_ = file_message_loop_proxy; - core_->response_destination_ = TEMP_FILE; -} - -net::HttpResponseHeaders* URLFetcher::response_headers() const { - return core_->response_headers_; -} - -// TODO(panayiotis): socket_address_ is written in the IO thread, -// if this is accessed in the UI thread, this could result in a race. -// Same for response_headers_ above and was_fetched_via_proxy_ below. -net::HostPortPair URLFetcher::socket_address() const { - return core_->socket_address_; -} - -bool URLFetcher::was_fetched_via_proxy() const { - return core_->was_fetched_via_proxy_; -} - -void URLFetcher::Start() { - core_->Start(); -} - -void URLFetcher::StartWithRequestContextGetter( - net::URLRequestContextGetter* request_context_getter) { - set_request_context(request_context_getter); - core_->Start(); -} - -const GURL& URLFetcher::original_url() const { - return core_->original_url_; -} - -const GURL& URLFetcher::url() const { - return core_->url_; -} - -const net::URLRequestStatus& URLFetcher::status() const { - return core_->status_; -} - -int URLFetcher::response_code() const { - return core_->response_code_; -} - -const net::ResponseCookies& URLFetcher::cookies() const { - return core_->cookies_; -} - -bool URLFetcher::FileErrorOccurred( - base::PlatformFileError* out_error_code) const { - - // Can't have a file error if no file is being created or written to. - if (!core_->temp_file_writer_.get()) { - return false; - } - - base::PlatformFileError error_code = core_->temp_file_writer_->error_code(); - if (error_code == base::PLATFORM_FILE_OK) - return false; - - *out_error_code = error_code; - return true; -} - -void URLFetcher::ReceivedContentWasMalformed() { - core_->ReceivedContentWasMalformed(); -} - -bool URLFetcher::GetResponseAsString(std::string* out_response_string) const { - if (core_->response_destination_ != STRING) - return false; - - *out_response_string = core_->data_; - return true; -} - -const std::string& URLFetcher::GetResponseStringRef() const { - CHECK(core_->response_destination_ == STRING); - return core_->data_; -} - -void URLFetcher::SetResponseDestinationForTesting( - ResponseDestinationType value) { - core_->response_destination_ = value; -} - -URLFetcher::ResponseDestinationType -URLFetcher::GetResponseDestinationForTesting() const { - return core_->response_destination_; -} - -bool URLFetcher::GetResponseAsFilePath(bool take_ownership, - FilePath* out_response_path) const { - DCHECK(core_->delegate_loop_proxy_->BelongsToCurrentThread()); - if (core_->response_destination_ != TEMP_FILE || - !core_->temp_file_writer_.get()) - return false; - - *out_response_path = core_->temp_file_writer_->temp_file(); - - if (take_ownership) { - core_->io_message_loop_proxy_->PostTask( - FROM_HERE, - NewRunnableMethod(core_.get(), &Core::DisownTempFile)); - } - return true; -} - -// static -void URLFetcher::CancelAll() { - Core::CancelAll(); -} - -// static -int URLFetcher::GetNumFetcherCores() { - return Core::g_registry.Get().size(); -} - -URLFetcher::Delegate* URLFetcher::delegate() const { - return core_->delegate(); -} diff --git a/content/common/url_fetcher.h b/content/common/url_fetcher.h deleted file mode 100644 index b3cee3b..0000000 --- a/content/common/url_fetcher.h +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// This file contains URLFetcher, a wrapper around net::URLRequest that handles -// low-level details like thread safety, ref counting, and incremental buffer -// reading. This is useful for callers who simply want to get the data from a -// URL and don't care about all the nitty-gritty details. -// -// NOTE(willchan): Only one "IO" thread is supported for URLFetcher. This is a -// temporary situation. We will work on allowing support for multiple "io" -// threads per process. - -#ifndef CONTENT_COMMON_URL_FETCHER_H_ -#define CONTENT_COMMON_URL_FETCHER_H_ -#pragma once - -#include -#include - -#include "base/memory/ref_counted.h" -#include "base/message_loop.h" -#include "base/platform_file.h" -#include "base/time.h" - -class FilePath; -class GURL; - -namespace base { -class MessageLoopProxy; -} // namespace base - -namespace net { -class HostPortPair; -class HttpResponseHeaders; -class HttpRequestHeaders; -class URLRequestContextGetter; -class URLRequestStatus; -typedef std::vector ResponseCookies; -} // namespace net - -// To use this class, create an instance with the desired URL and a pointer to -// the object to be notified when the URL has been loaded: -// URLFetcher* fetcher = new URLFetcher("http://www.google.com", -// URLFetcher::GET, this); -// -// Then, optionally set properties on this object, like the request context or -// extra headers: -// fetcher->SetExtraRequestHeaders("X-Foo: bar"); -// -// Finally, start the request: -// fetcher->Start(); -// -// -// The object you supply as a delegate must inherit from URLFetcher::Delegate; -// when the fetch is completed, OnURLFetchComplete() will be called with a -// pointer to the URLFetcher. From that point until the original URLFetcher -// instance is destroyed, you may use accessor methods to see the result of -// the fetch. You should copy these objects if you need them to live longer -// than the URLFetcher instance. If the URLFetcher instance is destroyed -// before the callback happens, the fetch will be canceled and no callback -// will occur. -// -// You may create the URLFetcher instance on any thread; OnURLFetchComplete() -// will be called back on the same thread you use to create the instance. -// -// -// NOTE: By default URLFetcher requests are NOT intercepted, except when -// interception is explicitly enabled in tests. - -class URLFetcher { - public: - enum RequestType { - GET, - POST, - HEAD, - }; - - // Imposible http response code. Used to signal that no http response code - // was received. - static const int kInvalidHttpResponseCode; - - class Delegate { - public: - // TODO(skerner): This will be removed in favor of the |source|-only - // version below. Leaving this for now to make the initial code review - // easy to read. - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - - // This will be called when the URL has been fetched, successfully or not. - // Use accessor methods on |source| to get the results. - virtual void OnURLFetchComplete(const URLFetcher* source); - - protected: - virtual ~Delegate() {} - }; - - // URLFetcher::Create uses the currently registered Factory to create the - // URLFetcher. Factory is intended for testing. - class Factory { - public: - virtual URLFetcher* CreateURLFetcher(int id, - const GURL& url, - RequestType request_type, - Delegate* d) = 0; - - protected: - virtual ~Factory() {} - }; - - // |url| is the URL to send the request to. - // |request_type| is the type of request to make. - // |d| the object that will receive the callback on fetch completion. - URLFetcher(const GURL& url, RequestType request_type, Delegate* d); - virtual ~URLFetcher(); - - // Normally interception is disabled for URLFetcher, but you can use this - // to enable it for tests. Also see ScopedURLFetcherFactory for another way - // of testing code that uses an URLFetcher. - static void enable_interception_for_tests(bool enabled) { - g_interception_enabled = enabled; - } - - // Creates a URLFetcher, ownership returns to the caller. If there is no - // Factory (the default) this creates and returns a new URLFetcher. See the - // constructor for a description of the args. |id| may be used during testing - // to identify who is creating the URLFetcher. - static URLFetcher* Create(int id, const GURL& url, RequestType request_type, - Delegate* d); - - // Sets data only needed by POSTs. All callers making POST requests should - // call this before the request is started. |upload_content_type| is the MIME - // type of the content, while |upload_content| is the data to be sent (the - // Content-Length header value will be set to the length of this data). - void set_upload_data(const std::string& upload_content_type, - const std::string& upload_content); - - // Indicates that the POST data is sent via chunked transfer encoding. - // This may only be called before calling Start(). - // Use AppendChunkToUpload() to give the data chunks after calling Start(). - void set_chunked_upload(const std::string& upload_content_type); - - // Adds the given bytes to a request's POST data transmitted using chunked - // transfer encoding. - // This method should be called ONLY after calling Start(). - virtual void AppendChunkToUpload(const std::string& data, bool is_last_chunk); - - // Set one or more load flags as defined in net/base/load_flags.h. Must be - // called before the request is started. - void set_load_flags(int load_flags); - - // Returns the current load flags. - int load_flags() const; - - // The referrer URL for the request. Must be called before the request is - // started. - void set_referrer(const std::string& referrer); - - // Set extra headers on the request. Must be called before the request - // is started. - void set_extra_request_headers(const std::string& extra_request_headers); - - void GetExtraRequestHeaders(net::HttpRequestHeaders* headers); - - // Set the net::URLRequestContext on the request. Must be called before the - // request is started. - void set_request_context( - net::URLRequestContextGetter* request_context_getter); - - // If |retry| is false, 5xx responses will be propagated to the observer, - // if it is true URLFetcher will automatically re-execute the request, - // after backoff_delay() elapses. URLFetcher has it set to true by default. - void set_automatically_retry_on_5xx(bool retry); - - int max_retries() const; - - void set_max_retries(int max_retries); - - // Returns the back-off delay before the request will be retried, - // when a 5xx response was received. - base::TimeDelta backoff_delay() const { return backoff_delay_; } - - // Sets the back-off delay, allowing to mock 5xx requests in unit-tests. -#if defined(UNIT_TEST) - void set_backoff_delay(base::TimeDelta backoff_delay) { - backoff_delay_ = backoff_delay; - } -#endif // defined(UNIT_TEST) - - // By default, the response is saved in a string. Call this method to save the - // response to a temporary file instead. Must be called before Start(). - // |file_message_loop_proxy| will be used for all file operations. - void SaveResponseToTemporaryFile( - scoped_refptr file_message_loop_proxy); - - // Retrieve the response headers from the request. Must only be called after - // the OnURLFetchComplete callback has run. - virtual net::HttpResponseHeaders* response_headers() const; - - // Retrieve the remote socket address from the request. Must only - // be called after the OnURLFetchComplete callback has run and if - // the request has not failed. - net::HostPortPair socket_address() const; - - // Returns true if the request was delivered through a proxy. Must only - // be called after the OnURLFetchComplete callback has run and the request - // has not failed. - bool was_fetched_via_proxy() const; - - // Start the request. After this is called, you may not change any other - // settings. - virtual void Start(); - - // Restarts the URLFetcher with a new URLRequestContextGetter. - void StartWithRequestContextGetter( - net::URLRequestContextGetter* request_context_getter); - - // Return the URL that we were asked to fetch. - virtual const GURL& original_url() const; - - // Return the URL that this fetcher is processing. - virtual const GURL& url() const; - - // The status of the URL fetch. - virtual const net::URLRequestStatus& status() const; - - // The http response code received. Will return - // URLFetcher::kInvalidHttpResponseCode if an error prevented any response - // from being received. - virtual int response_code() const; - - // Cookies recieved. - virtual const net::ResponseCookies& cookies() const; - - // Return true if any file system operation failed. If so, set |error_code| - // to the error code. File system errors are only possible if user called - // SaveResponseToTemporaryFile(). - virtual bool FileErrorOccurred(base::PlatformFileError* out_error_code) const; - - // Reports that the received content was malformed. - void ReceivedContentWasMalformed(); - - // Get the response as a string. Return false if the fetcher was not - // set to store the response as a string. - virtual bool GetResponseAsString(std::string* out_response_string) const; - - // Get the path to the file containing the response body. Returns false - // if the response body was not saved to a file. If take_ownership is - // true, caller takes responsibility for the temp file, and it will not - // be removed once the URLFetcher is destroyed. User should not take - // ownership more than once, or call this method after taking ownership. - virtual bool GetResponseAsFilePath(bool take_ownership, - FilePath* out_response_path) const; - - // Cancels all existing URLFetchers. Will notify the URLFetcher::Delegates. - // Note that any new URLFetchers created while this is running will not be - // cancelled. Typically, one would call this in the CleanUp() method of an IO - // thread, so that no new URLRequests would be able to start on the IO thread - // anyway. This doesn't prevent new URLFetchers from trying to post to the IO - // thread though, even though the task won't ever run. - static void CancelAll(); - - protected: - // How should the response be stored? - enum ResponseDestinationType { - STRING, // Default: In a std::string - TEMP_FILE // Write to a temp file - }; - - // Returns the delegate. - Delegate* delegate() const; - - // Used by tests. - const std::string& upload_data() const; - - // Return a reference to the string data fetched. Response type must - // be STRING, or this will CHECK. This method exists to support the - // old signiture to OnURLFetchComplete(), and will be removed as part - // of crbug.com/83592 . - virtual const std::string& GetResponseStringRef() const; - - virtual void SetResponseDestinationForTesting(ResponseDestinationType); - virtual ResponseDestinationType GetResponseDestinationForTesting() const; - - private: - friend class ScopedURLFetcherFactory; - friend class TestURLFetcher; - friend class URLFetcherTest; - - // Only used by URLFetcherTest, returns the number of URLFetcher::Core objects - // actively running. - static int GetNumFetcherCores(); - - static Factory* factory() { return factory_; } - - // Sets the factory used by the static method Create to create a URLFetcher. - // URLFetcher does not take ownership of |factory|. A value of NULL results - // in a URLFetcher being created directly. - // - // NOTE: for safety, this should only be used through ScopedURLFetcherFactory! - static void set_factory(Factory* factory) { - factory_ = factory; - } - - class Core; - scoped_refptr core_; - - static Factory* factory_; - - // Back-off time delay. 0 by default. - base::TimeDelta backoff_delay_; - - static bool g_interception_enabled; - - DISALLOW_COPY_AND_ASSIGN(URLFetcher); -}; - -#endif // CONTENT_COMMON_URL_FETCHER_H_ diff --git a/content/common/url_fetcher_unittest.cc b/content/common/url_fetcher_unittest.cc deleted file mode 100644 index e301fe8..0000000 --- a/content/common/url_fetcher_unittest.cc +++ /dev/null @@ -1,906 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "base/message_loop_proxy.h" -#include "base/synchronization/waitable_event.h" -#include "base/threading/thread.h" -#include "build/build_config.h" -#include "content/common/url_fetcher.h" -#include "crypto/nss_util.h" -#include "net/http/http_response_headers.h" -#include "net/test/test_server.h" -#include "net/url_request/url_request_context_getter.h" -#include "net/url_request/url_request_test_util.h" -#include "net/url_request/url_request_throttler_manager.h" -#include "testing/gtest/include/gtest/gtest.h" - -#if defined(USE_NSS) -#include "net/ocsp/nss_ocsp.h" -#endif - -using base::Time; -using base::TimeDelta; - -// TODO(eroman): Add a regression test for http://crbug.com/40505. - -namespace { - -const FilePath::CharType kDocRoot[] = FILE_PATH_LITERAL("chrome/test/data"); -const std::string kTestServerFilePrefix = "files/"; - -class CurriedTask : public Task { - public: - CurriedTask(Task* task, MessageLoop* target_loop) - : task_(task), - target_loop_(target_loop) {} - - virtual void Run() { - target_loop_->PostTask(FROM_HERE, task_); - } - - private: - Task* const task_; - MessageLoop* const target_loop_; - - DISALLOW_COPY_AND_ASSIGN(CurriedTask); -}; - -class TestURLRequestContextGetter : public net::URLRequestContextGetter { - public: - explicit TestURLRequestContextGetter( - base::MessageLoopProxy* io_message_loop_proxy) - : io_message_loop_proxy_(io_message_loop_proxy) { - } - virtual net::URLRequestContext* GetURLRequestContext() { - if (!context_) - context_ = new TestURLRequestContext(); - return context_; - } - virtual scoped_refptr GetIOMessageLoopProxy() const { - return io_message_loop_proxy_; - } - - protected: - scoped_refptr io_message_loop_proxy_; - - private: - virtual ~TestURLRequestContextGetter() {} - - scoped_refptr context_; -}; - -} // namespace - -class URLFetcherTest : public testing::Test, public URLFetcher::Delegate { - public: - URLFetcherTest() : fetcher_(NULL) { } - - // Creates a URLFetcher, using the program's main thread to do IO. - virtual void CreateFetcher(const GURL& url); - - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - - scoped_refptr io_message_loop_proxy() { - return io_message_loop_proxy_; - } - - protected: - virtual void SetUp() { - testing::Test::SetUp(); - - io_message_loop_proxy_ = base::MessageLoopProxy::current(); - -#if defined(USE_NSS) - crypto::EnsureNSSInit(); - net::EnsureOCSPInit(); -#endif - } - - virtual void TearDown() { -#if defined(USE_NSS) - net::ShutdownOCSP(); -#endif - } - - int GetNumFetcherCores() const { - return URLFetcher::GetNumFetcherCores(); - } - - // URLFetcher is designed to run on the main UI thread, but in our tests - // we assume that the current thread is the IO thread where the URLFetcher - // dispatches its requests to. When we wish to simulate being used from - // a UI thread, we dispatch a worker thread to do so. - MessageLoopForIO io_loop_; - scoped_refptr io_message_loop_proxy_; - - URLFetcher* fetcher_; -}; - -void URLFetcherTest::CreateFetcher(const GURL& url) { - fetcher_ = new URLFetcher(url, URLFetcher::GET, this); - fetcher_->set_request_context(new TestURLRequestContextGetter( - io_message_loop_proxy())); - fetcher_->Start(); -} - -void URLFetcherTest::OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - EXPECT_TRUE(status.is_success()); - EXPECT_EQ(200, response_code); // HTTP OK - EXPECT_FALSE(data.empty()); - - delete fetcher_; // Have to delete this here and not in the destructor, - // because the destructor won't necessarily run on the - // same thread that CreateFetcher() did. - - io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); - // If the current message loop is not the IO loop, it will be shut down when - // the main loop returns and this thread subsequently goes out of scope. -} - -namespace { - -// Version of URLFetcherTest that does a POST instead -class URLFetcherPostTest : public URLFetcherTest { - public: - virtual void CreateFetcher(const GURL& url); - - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); -}; - -// Version of URLFetcherTest that tests headers. -class URLFetcherHeadersTest : public URLFetcherTest { - public: - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); -}; - -// Version of URLFetcherTest that tests SocketAddress. -class URLFetcherSocketAddressTest : public URLFetcherTest { - public: - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - protected: - std::string expected_host_; - uint16 expected_port_; -}; - -// Version of URLFetcherTest that tests overload protection. -class URLFetcherProtectTest : public URLFetcherTest { - public: - virtual void CreateFetcher(const GURL& url); - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - private: - Time start_time_; -}; - -// Version of URLFetcherTest that tests overload protection, when responses -// passed through. -class URLFetcherProtectTestPassedThrough : public URLFetcherTest { - public: - virtual void CreateFetcher(const GURL& url); - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - private: - Time start_time_; -}; - -// Version of URLFetcherTest that tests bad HTTPS requests. -class URLFetcherBadHTTPSTest : public URLFetcherTest { - public: - URLFetcherBadHTTPSTest(); - - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - - private: - FilePath cert_dir_; -}; - -// Version of URLFetcherTest that tests request cancellation on shutdown. -class URLFetcherCancelTest : public URLFetcherTest { - public: - virtual void CreateFetcher(const GURL& url); - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - - void CancelRequest(); -}; - -// Version of TestURLRequestContext that posts a Quit task to the IO -// thread once it is deleted. -class CancelTestURLRequestContext : public TestURLRequestContext { - virtual ~CancelTestURLRequestContext() { - // The d'tor should execute in the IO thread. Post the quit task to the - // current thread. - MessageLoop::current()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); - } -}; - -class CancelTestURLRequestContextGetter : public net::URLRequestContextGetter { - public: - explicit CancelTestURLRequestContextGetter( - base::MessageLoopProxy* io_message_loop_proxy) - : io_message_loop_proxy_(io_message_loop_proxy), - context_created_(false, false) { - } - virtual net::URLRequestContext* GetURLRequestContext() { - if (!context_) { - context_ = new CancelTestURLRequestContext(); - context_created_.Signal(); - } - return context_; - } - virtual scoped_refptr GetIOMessageLoopProxy() const { - return io_message_loop_proxy_; - } - void WaitForContextCreation() { - context_created_.Wait(); - } - - private: - ~CancelTestURLRequestContextGetter() {} - - scoped_refptr io_message_loop_proxy_; - base::WaitableEvent context_created_; - scoped_refptr context_; -}; - -// Version of URLFetcherTest that tests retying the same request twice. -class URLFetcherMultipleAttemptTest : public URLFetcherTest { - public: - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - private: - std::string data_; -}; - -class URLFetcherTempFileTest : public URLFetcherTest { - public: - URLFetcherTempFileTest() - : take_ownership_of_temp_file_(false) { - } - - // URLFetcher::Delegate - virtual void OnURLFetchComplete(const URLFetcher* source); - - // This obsolete signature should not be used, but must be present - // to make clang happy. - virtual void OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data); - - virtual void CreateFetcher(const GURL& url); - - protected: - FilePath expected_file_; - FilePath temp_file_; - - // Set by the test. Used in OnURLFetchComplete() to decide if - // the URLFetcher should own the temp file, so that we can test - // disowning prevents the file from being deleted. - bool take_ownership_of_temp_file_; -}; - -void URLFetcherTempFileTest::CreateFetcher(const GURL& url) { - fetcher_ = new URLFetcher(url, URLFetcher::GET, this); - fetcher_->set_request_context(new TestURLRequestContextGetter( - io_message_loop_proxy())); - - // Use the IO message loop to do the file operations in this test. - fetcher_->SaveResponseToTemporaryFile(io_message_loop_proxy()); - fetcher_->Start(); -} - -TEST_F(URLFetcherTempFileTest, SmallGet) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - // Get a small file. - const char* kFileToFetch = "simple.html"; - expected_file_ = test_server.document_root().AppendASCII(kFileToFetch); - CreateFetcher(test_server.GetURL(kTestServerFilePrefix + kFileToFetch)); - - MessageLoop::current()->Run(); // OnURLFetchComplete() will Quit(). - - ASSERT_FALSE(file_util::PathExists(temp_file_)) - << temp_file_.value() << " not removed."; -} - -TEST_F(URLFetcherTempFileTest, LargeGet) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - // Get a file large enough to require more than one read into - // URLFetcher::Core's IOBuffer. - const char* kFileToFetch = "animate1.gif"; - expected_file_ = test_server.document_root().AppendASCII(kFileToFetch); - CreateFetcher(test_server.GetURL(kTestServerFilePrefix + kFileToFetch)); - - MessageLoop::current()->Run(); // OnURLFetchComplete() will Quit(). -} - -TEST_F(URLFetcherTempFileTest, CanTakeOwnershipOfFile) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - // Get a small file. - const char* kFileToFetch = "simple.html"; - expected_file_ = test_server.document_root().AppendASCII(kFileToFetch); - CreateFetcher(test_server.GetURL(kTestServerFilePrefix + kFileToFetch)); - - MessageLoop::current()->Run(); // OnURLFetchComplete() will Quit(). - - MessageLoop::current()->RunAllPending(); - ASSERT_FALSE(file_util::PathExists(temp_file_)) - << temp_file_.value() << " not removed."; -} - -// Wrapper that lets us call CreateFetcher() on a thread of our choice. We -// could make URLFetcherTest refcounted and use PostTask(FROM_HERE.. ) to call -// CreateFetcher() directly, but the ownership of the URLFetcherTest is a bit -// confusing in that case because GTest doesn't know about the refcounting. -// It's less confusing to just do it this way. -class FetcherWrapperTask : public Task { - public: - FetcherWrapperTask(URLFetcherTest* test, const GURL& url) - : test_(test), url_(url) { } - virtual void Run() { - test_->CreateFetcher(url_); - } - - private: - URLFetcherTest* test_; - GURL url_; -}; - -void URLFetcherPostTest::CreateFetcher(const GURL& url) { - fetcher_ = new URLFetcher(url, URLFetcher::POST, this); - fetcher_->set_request_context(new TestURLRequestContextGetter( - io_message_loop_proxy())); - fetcher_->set_upload_data("application/x-www-form-urlencoded", - "bobsyeruncle"); - fetcher_->Start(); -} - -void URLFetcherPostTest::OnURLFetchComplete(const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - EXPECT_EQ(std::string("bobsyeruncle"), data); - URLFetcherTest::OnURLFetchComplete(source, url, status, response_code, - cookies, data); -} - -void URLFetcherHeadersTest::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - std::string header; - EXPECT_TRUE(source->response_headers()->GetNormalizedHeader("cache-control", - &header)); - EXPECT_EQ("private", header); - URLFetcherTest::OnURLFetchComplete(source, url, status, response_code, - cookies, data); -} - -void URLFetcherSocketAddressTest::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - EXPECT_EQ("127.0.0.1", source->socket_address().host()); - EXPECT_EQ(expected_port_, source->socket_address().port()); - URLFetcherTest::OnURLFetchComplete(source, url, status, response_code, - cookies, data); -} - -void URLFetcherProtectTest::CreateFetcher(const GURL& url) { - fetcher_ = new URLFetcher(url, URLFetcher::GET, this); - fetcher_->set_request_context(new TestURLRequestContextGetter( - io_message_loop_proxy())); - start_time_ = Time::Now(); - fetcher_->set_max_retries(11); - fetcher_->Start(); -} - -void URLFetcherProtectTest::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - const TimeDelta one_second = TimeDelta::FromMilliseconds(1000); - if (response_code >= 500) { - // Now running ServerUnavailable test. - // It takes more than 1 second to finish all 11 requests. - EXPECT_TRUE(Time::Now() - start_time_ >= one_second); - EXPECT_TRUE(status.is_success()); - EXPECT_FALSE(data.empty()); - delete fetcher_; - io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); - } else { - // Now running Overload test. - static int count = 0; - count++; - if (count < 20) { - fetcher_->StartWithRequestContextGetter(new TestURLRequestContextGetter( - io_message_loop_proxy())); - } else { - // We have already sent 20 requests continuously. And we expect that - // it takes more than 1 second due to the overload protection settings. - EXPECT_TRUE(Time::Now() - start_time_ >= one_second); - URLFetcherTest::OnURLFetchComplete(source, url, status, response_code, - cookies, data); - } - } -} - -void URLFetcherProtectTestPassedThrough::CreateFetcher(const GURL& url) { - fetcher_ = new URLFetcher(url, URLFetcher::GET, this); - fetcher_->set_request_context(new TestURLRequestContextGetter( - io_message_loop_proxy())); - fetcher_->set_automatically_retry_on_5xx(false); - start_time_ = Time::Now(); - fetcher_->set_max_retries(11); - fetcher_->Start(); -} - -void URLFetcherProtectTestPassedThrough::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - const TimeDelta one_minute = TimeDelta::FromMilliseconds(60000); - if (response_code >= 500) { - // Now running ServerUnavailable test. - // It should get here on the first attempt, so almost immediately and - // *not* to attempt to execute all 11 requests (2.5 minutes). - EXPECT_TRUE(Time::Now() - start_time_ < one_minute); - EXPECT_TRUE(status.is_success()); - // Check that suggested back off time is bigger than 0. - EXPECT_GT(fetcher_->backoff_delay().InMicroseconds(), 0); - EXPECT_FALSE(data.empty()); - } else { - // We should not get here! - ADD_FAILURE(); - } - - delete fetcher_; - io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); -} - - -URLFetcherBadHTTPSTest::URLFetcherBadHTTPSTest() { - PathService::Get(base::DIR_SOURCE_ROOT, &cert_dir_); - cert_dir_ = cert_dir_.AppendASCII("chrome"); - cert_dir_ = cert_dir_.AppendASCII("test"); - cert_dir_ = cert_dir_.AppendASCII("data"); - cert_dir_ = cert_dir_.AppendASCII("ssl"); - cert_dir_ = cert_dir_.AppendASCII("certificates"); -} - -// The "server certificate expired" error should result in automatic -// cancellation of the request by -// net::URLRequest::Delegate::OnSSLCertificateError. -void URLFetcherBadHTTPSTest::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - // This part is different from URLFetcherTest::OnURLFetchComplete - // because this test expects the request to be cancelled. - EXPECT_EQ(net::URLRequestStatus::CANCELED, status.status()); - EXPECT_EQ(net::ERR_ABORTED, status.error()); - EXPECT_EQ(-1, response_code); - EXPECT_TRUE(cookies.empty()); - EXPECT_TRUE(data.empty()); - - // The rest is the same as URLFetcherTest::OnURLFetchComplete. - delete fetcher_; - io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); -} - -void URLFetcherCancelTest::CreateFetcher(const GURL& url) { - fetcher_ = new URLFetcher(url, URLFetcher::GET, this); - CancelTestURLRequestContextGetter* context_getter = - new CancelTestURLRequestContextGetter(io_message_loop_proxy()); - fetcher_->set_request_context(context_getter); - fetcher_->set_max_retries(2); - fetcher_->Start(); - // We need to wait for the creation of the net::URLRequestContext, since we - // rely on it being destroyed as a signal to end the test. - context_getter->WaitForContextCreation(); - CancelRequest(); -} - -void URLFetcherCancelTest::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - // We should have cancelled the request before completion. - ADD_FAILURE(); - delete fetcher_; - io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); -} - -void URLFetcherCancelTest::CancelRequest() { - delete fetcher_; - // The URLFetcher's test context will post a Quit task once it is - // deleted. So if this test simply hangs, it means cancellation - // did not work. -} - -void URLFetcherMultipleAttemptTest::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - EXPECT_TRUE(status.is_success()); - EXPECT_EQ(200, response_code); // HTTP OK - EXPECT_FALSE(data.empty()); - if (!data.empty() && data_.empty()) { - data_ = data; - fetcher_->StartWithRequestContextGetter( - new TestURLRequestContextGetter(io_message_loop_proxy())); - } else { - EXPECT_EQ(data, data_); - delete fetcher_; // Have to delete this here and not in the destructor, - // because the destructor won't necessarily run on the - // same thread that CreateFetcher() did. - - io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); - // If the current message loop is not the IO loop, it will be shut down when - // the main loop returns and this thread subsequently goes out of scope. - } -} - -void URLFetcherTempFileTest::OnURLFetchComplete(const URLFetcher* source) { - EXPECT_TRUE(source->status().is_success()); - EXPECT_EQ(source->response_code(), 200); - - EXPECT_TRUE(source->GetResponseAsFilePath( - take_ownership_of_temp_file_, &temp_file_)); - - EXPECT_TRUE(file_util::ContentsEqual(expected_file_, temp_file_)); - - delete fetcher_; - - io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask()); -} - -void URLFetcherTempFileTest::OnURLFetchComplete( - const URLFetcher* source, - const GURL& url, - const net::URLRequestStatus& status, - int response_code, - const net::ResponseCookies& cookies, - const std::string& data) { - NOTREACHED(); -} - - -TEST_F(URLFetcherTest, SameThreadsTest) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - // Create the fetcher on the main thread. Since IO will happen on the main - // thread, this will test URLFetcher's ability to do everything on one - // thread. - CreateFetcher(test_server.GetURL("defaultresponse")); - - MessageLoop::current()->Run(); -} - -#if defined(OS_MACOSX) -// SIGSEGV on Mac: http://crbug.com/60426 -TEST_F(URLFetcherTest, DISABLED_DifferentThreadsTest) { -#else -TEST_F(URLFetcherTest, DifferentThreadsTest) { -#endif - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - // Create a separate thread that will create the URLFetcher. The current - // (main) thread will do the IO, and when the fetch is complete it will - // terminate the main thread's message loop; then the other thread's - // message loop will be shut down automatically as the thread goes out of - // scope. - base::Thread t("URLFetcher test thread"); - ASSERT_TRUE(t.Start()); - t.message_loop()->PostTask(FROM_HERE, new FetcherWrapperTask(this, - test_server.GetURL("defaultresponse"))); - - MessageLoop::current()->Run(); -} - -#if defined(OS_MACOSX) -// SIGSEGV on Mac: http://crbug.com/60426 -TEST_F(URLFetcherPostTest, DISABLED_Basic) { -#else -TEST_F(URLFetcherPostTest, Basic) { -#endif - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - CreateFetcher(test_server.GetURL("echo")); - MessageLoop::current()->Run(); -} - -TEST_F(URLFetcherHeadersTest, Headers) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, - FilePath(FILE_PATH_LITERAL("net/data/url_request_unittest"))); - ASSERT_TRUE(test_server.Start()); - - CreateFetcher(test_server.GetURL("files/with-headers.html")); - MessageLoop::current()->Run(); - // The actual tests are in the URLFetcherHeadersTest fixture. -} - -TEST_F(URLFetcherSocketAddressTest, SocketAddress) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, - FilePath(FILE_PATH_LITERAL("net/data/url_request_unittest"))); - ASSERT_TRUE(test_server.Start()); - expected_port_ = test_server.host_port_pair().port(); - - // Reusing "with-headers.html" but doesn't really matter. - CreateFetcher(test_server.GetURL("files/with-headers.html")); - MessageLoop::current()->Run(); - // The actual tests are in the URLFetcherSocketAddressTest fixture. -} - -TEST_F(URLFetcherProtectTest, Overload) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - GURL url(test_server.GetURL("defaultresponse")); - - // Registers an entry for test url. It only allows 3 requests to be sent - // in 200 milliseconds. - net::URLRequestThrottlerManager* manager = - net::URLRequestThrottlerManager::GetInstance(); - scoped_refptr entry( - new net::URLRequestThrottlerEntry(manager, "", 200, 3, 1, 2.0, 0.0, 256)); - manager->OverrideEntryForTests(url, entry); - - CreateFetcher(url); - - MessageLoop::current()->Run(); - - net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); -} - -TEST_F(URLFetcherProtectTest, ServerUnavailable) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - GURL url(test_server.GetURL("files/server-unavailable.html")); - - // Registers an entry for test url. The backoff time is calculated by: - // new_backoff = 2.0 * old_backoff + 0 - // and maximum backoff time is 256 milliseconds. - // Maximum retries allowed is set to 11. - net::URLRequestThrottlerManager* manager = - net::URLRequestThrottlerManager::GetInstance(); - scoped_refptr entry( - new net::URLRequestThrottlerEntry(manager, "", 200, 3, 1, 2.0, 0.0, 256)); - manager->OverrideEntryForTests(url, entry); - - CreateFetcher(url); - - MessageLoop::current()->Run(); - - net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); -} - -TEST_F(URLFetcherProtectTestPassedThrough, ServerUnavailablePropagateResponse) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - GURL url(test_server.GetURL("files/server-unavailable.html")); - - // Registers an entry for test url. The backoff time is calculated by: - // new_backoff = 2.0 * old_backoff + 0 - // and maximum backoff time is 150000 milliseconds. - // Maximum retries allowed is set to 11. - net::URLRequestThrottlerManager* manager = - net::URLRequestThrottlerManager::GetInstance(); - scoped_refptr entry( - new net::URLRequestThrottlerEntry( - manager, "", 200, 3, 100, 2.0, 0.0, 150000)); - // Total time if *not* for not doing automatic backoff would be 150s. - // In reality it should be "as soon as server responds". - manager->OverrideEntryForTests(url, entry); - - CreateFetcher(url); - - MessageLoop::current()->Run(); - - net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); -} - -#if defined(OS_MACOSX) -// SIGSEGV on Mac: http://crbug.com/60426 -TEST_F(URLFetcherBadHTTPSTest, DISABLED_BadHTTPSTest) { -#else -TEST_F(URLFetcherBadHTTPSTest, BadHTTPSTest) { -#endif - net::TestServer::HTTPSOptions https_options( - net::TestServer::HTTPSOptions::CERT_EXPIRED); - net::TestServer test_server(https_options, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - CreateFetcher(test_server.GetURL("defaultresponse")); - MessageLoop::current()->Run(); -} - -#if defined(OS_MACOSX) -// SIGSEGV on Mac: http://crbug.com/60426 -TEST_F(URLFetcherCancelTest, DISABLED_ReleasesContext) { -#else -TEST_F(URLFetcherCancelTest, ReleasesContext) { -#endif - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - GURL url(test_server.GetURL("files/server-unavailable.html")); - - // Registers an entry for test url. The backoff time is calculated by: - // new_backoff = 2.0 * old_backoff +0 - // The initial backoff is 2 seconds and maximum backoff is 4 seconds. - // Maximum retries allowed is set to 2. - net::URLRequestThrottlerManager* manager = - net::URLRequestThrottlerManager::GetInstance(); - scoped_refptr entry( - new net::URLRequestThrottlerEntry( - manager, "", 200, 3, 2000, 2.0, 0.0, 4000)); - manager->OverrideEntryForTests(url, entry); - - // Create a separate thread that will create the URLFetcher. The current - // (main) thread will do the IO, and when the fetch is complete it will - // terminate the main thread's message loop; then the other thread's - // message loop will be shut down automatically as the thread goes out of - // scope. - base::Thread t("URLFetcher test thread"); - ASSERT_TRUE(t.Start()); - t.message_loop()->PostTask(FROM_HERE, new FetcherWrapperTask(this, url)); - - MessageLoop::current()->Run(); - - net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); -} - -TEST_F(URLFetcherCancelTest, CancelWhileDelayedStartTaskPending) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - GURL url(test_server.GetURL("files/server-unavailable.html")); - - // Register an entry for test url. - // Using a sliding window of 4 seconds, and max of 1 request, under a fast - // run we expect to have a 4 second delay when posting the Start task. - net::URLRequestThrottlerManager* manager = - net::URLRequestThrottlerManager::GetInstance(); - scoped_refptr entry( - new net::URLRequestThrottlerEntry( - manager, "", 4000, 1, 2000, 2.0, 0.0, 4000)); - manager->OverrideEntryForTests(url, entry); - // Fake that a request has just started. - entry->ReserveSendingTimeForNextRequest(base::TimeTicks()); - - // The next request we try to send will be delayed by ~4 seconds. - // The slower the test runs, the less the delay will be (since it takes the - // time difference from now). - - base::Thread t("URLFetcher test thread"); - ASSERT_TRUE(t.Start()); - t.message_loop()->PostTask(FROM_HERE, new FetcherWrapperTask(this, url)); - - MessageLoop::current()->Run(); - - net::URLRequestThrottlerManager::GetInstance()->EraseEntryForTests(url); -} - -TEST_F(URLFetcherMultipleAttemptTest, SameData) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - - // Create the fetcher on the main thread. Since IO will happen on the main - // thread, this will test URLFetcher's ability to do everything on one - // thread. - CreateFetcher(test_server.GetURL("defaultresponse")); - - MessageLoop::current()->Run(); -} - -// Tests to make sure CancelAll() will successfully cancel existing URLFetchers. -TEST_F(URLFetcherTest, CancelAll) { - net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot)); - ASSERT_TRUE(test_server.Start()); - EXPECT_EQ(0, GetNumFetcherCores()); - - CreateFetcher(test_server.GetURL("defaultresponse")); - io_message_loop_proxy()->PostTask( - FROM_HERE, - new CurriedTask(new MessageLoop::QuitTask(), MessageLoop::current())); - MessageLoop::current()->Run(); - EXPECT_EQ(1, GetNumFetcherCores()); - URLFetcher::CancelAll(); - EXPECT_EQ(0, GetNumFetcherCores()); - delete fetcher_; -} - -} // namespace. diff --git a/content/content_common.gypi b/content/content_common.gypi index 5623da2..2aa7cb7 100644 --- a/content/content_common.gypi +++ b/content/content_common.gypi @@ -159,6 +159,8 @@ 'common/native_web_keyboard_event_win.cc', 'common/navigation_gesture.h', 'common/navigation_types.h', + 'common/net/url_fetcher.cc', + 'common/net/url_fetcher.h', 'common/notification_details.cc', 'common/notification_details.h', 'common/notification_observer.h', @@ -236,8 +238,6 @@ 'common/unix_domain_socket_posix.h', 'common/url_constants.cc', 'common/url_constants.h', - 'common/url_fetcher.cc', - 'common/url_fetcher.h', 'common/utility_messages.h', 'common/view_messages.h', 'common/view_types.cc', diff --git a/content/content_tests.gypi b/content/content_tests.gypi index 29e8809..b05062f 100644 --- a/content/content_tests.gypi +++ b/content/content_tests.gypi @@ -126,11 +126,11 @@ 'common/gpu/gpu_feature_flags_unittest.cc', 'common/gpu/gpu_info_unittest.cc', 'common/hi_res_timer_manager_unittest.cc', + 'common/net/url_fetcher_unittest.cc', 'common/notification_service_unittest.cc', 'common/process_watcher_unittest.cc', 'common/property_bag_unittest.cc', 'common/resource_dispatcher_unittest.cc', - 'common/url_fetcher_unittest.cc', 'gpu/gpu_info_collector_unittest.cc', 'gpu/gpu_info_collector_unittest_win.cc', 'renderer/active_notification_tracker_unittest.cc', diff --git a/content/test/test_url_fetcher_factory.h b/content/test/test_url_fetcher_factory.h index a286bad..c003106 100644 --- a/content/test/test_url_fetcher_factory.h +++ b/content/test/test_url_fetcher_factory.h @@ -12,7 +12,7 @@ #include #include "base/threading/non_thread_safe.h" -#include "content/common/url_fetcher.h" +#include "content/common/net/url_fetcher.h" #include "googleurl/src/gurl.h" #include "net/url_request/url_request_status.h" -- cgit v1.1