diff options
Diffstat (limited to 'content/common/url_fetcher.cc')
| -rw-r--r-- | content/common/url_fetcher.cc | 1041 |
1 files changed, 1041 insertions, 0 deletions
diff --git a/content/common/url_fetcher.cc b/content/common/url_fetcher.cc new file mode 100644 index 0000000..9706a9d --- /dev/null +++ b/content/common/url_fetcher.cc @@ -0,0 +1,1041 @@ +// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/common/url_fetcher.h" + +#include <set> + +#include "base/compiler_specific.h" +#include "base/file_path.h" +#include "base/file_util_proxy.h" +#include "base/lazy_instance.h" +#include "base/memory/scoped_callback_factory.h" +#include "base/memory/scoped_ptr.h" +#include "base/message_loop_proxy.h" +#include "base/platform_file.h" +#include "base/stl_util-inl.h" +#include "base/string_util.h" +#include "base/threading/thread.h" +#include "googleurl/src/gurl.h" +#include "net/base/load_flags.h" +#include "net/base/io_buffer.h" +#include "net/base/net_errors.h" +#include "net/base/host_port_pair.h" +#include "net/http/http_request_headers.h" +#include "net/http/http_response_headers.h" +#include "net/url_request/url_request.h" +#include "net/url_request/url_request_context.h" +#include "net/url_request/url_request_context_getter.h" +#include "net/url_request/url_request_throttler_manager.h" + +static const int kBufferSize = 4096; +const int URLFetcher::kInvalidHttpResponseCode = -1; + +class URLFetcher::Core + : public base::RefCountedThreadSafe<URLFetcher::Core>, + public net::URLRequest::Delegate { + public: + // For POST requests, set |content_type| to the MIME type of the content + // and set |content| to the data to upload. |flags| are flags to apply to + // the load operation--these should be one or more of the LOAD_* flags + // defined in net/base/load_flags.h. + Core(URLFetcher* fetcher, + const GURL& original_url, + RequestType request_type, + URLFetcher::Delegate* d); + + // Starts the load. It's important that this not happen in the constructor + // because it causes the IO thread to begin AddRef()ing and Release()ing + // us. If our caller hasn't had time to fully construct us and take a + // reference, the IO thread could interrupt things, run a task, Release() + // us, and destroy us, leaving the caller with an already-destroyed object + // when construction finishes. + void Start(); + + // Stops any in-progress load and ensures no callback will happen. It is + // safe to call this multiple times. + void Stop(); + + // Reports that the received content was malformed (i.e. failed parsing + // or validation). This makes the throttling logic that does exponential + // back-off when servers are having problems treat the current request as + // a failure. Your call to this method will be ignored if your request is + // already considered a failure based on the HTTP response code or response + // headers. + void ReceivedContentWasMalformed(); + + // Overridden from net::URLRequest::Delegate: + virtual void OnResponseStarted(net::URLRequest* request); + virtual void OnReadCompleted(net::URLRequest* request, int bytes_read); + + URLFetcher::Delegate* delegate() const { return delegate_; } + static void CancelAll(); + + private: + friend class base::RefCountedThreadSafe<URLFetcher::Core>; + + class Registry { + public: + Registry(); + ~Registry(); + + void AddURLFetcherCore(Core* core); + void RemoveURLFetcherCore(Core* core); + + void CancelAll(); + + int size() const { + return fetchers_.size(); + } + + private: + std::set<Core*> fetchers_; + + DISALLOW_COPY_AND_ASSIGN(Registry); + }; + + // Class TempFileWriter encapsulates all state involved in writing + // response bytes to a temporary file. It is only used if + // |Core::response_destination_| == TEMP_FILE. + class TempFileWriter { + public: + TempFileWriter( + URLFetcher::Core* core, + scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy); + + ~TempFileWriter(); + void CreateTempFile(); + void DidCreateTempFile(base::PlatformFileError error_code, + base::PassPlatformFile file_handle, + FilePath file_path); + void DidCloseTempFile(base::PlatformFileError error_code); + void DidReopenTempFile(base::PlatformFileError error_code, + base::PassPlatformFile file_handle, + bool created); + + // Record |num_bytes_| response bytes in |core_->buffer_| to the file. + void WriteBuffer(int num_bytes); + + // Called when a write has been done. Continues writing if there are + // any more bytes to write. Otherwise, initiates a read in core_. + void ContinueWrite(base::PlatformFileError error_code, + int bytes_written); + + // Drop ownership of the file at path |temp_file_|. This class + // will not delete it or write to it again. + void DisownTempFile(); + + // Remove any file created. + void Destroy(); + + const FilePath& temp_file() const { return temp_file_; } + int64 total_bytes_written() { return total_bytes_written_; } + base::PlatformFileError error_code() const { return error_code_; } + + private: + // The URLFetcher::Core which instantiated this class. + URLFetcher::Core* core_; + + // The last error encountered on a file operation. base::PLATFORM_FILE_OK + // if no error occurred. + base::PlatformFileError error_code_; + + // Callbacks are created for use with base::FileUtilProxy. + base::ScopedCallbackFactory<URLFetcher::Core::TempFileWriter> + callback_factory_; + + // Message loop on which file opperations should happen. + scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy_; + + // Path to the temporary file. This path is empty when there + // is no temp file. + FilePath temp_file_; + + // Handle to the temp file. + base::PlatformFile temp_file_handle_; + + // We always append to the file. Track the total number of bytes + // written, so that writes know the offset to give. + int64 total_bytes_written_; + + // How many bytes did the last Write() try to write? Needed so + // that if not all the bytes get written on a Write(), we can + // call Write() again with the rest. + int pending_bytes_; + + // When writing, how many bytes from the buffer have been successfully + // written so far? + int buffer_offset_; + }; + + virtual ~Core(); + + // Wrapper functions that allow us to ensure actions happen on the right + // thread. + void StartURLRequest(); + void StartURLRequestWhenAppropriate(); + void CancelURLRequest(); + void OnCompletedURLRequest(const net::URLRequestStatus& status); + void InformDelegateFetchIsComplete(); + void NotifyMalformedContent(); + + // Deletes the request, removes it from the registry, and removes the + // destruction observer. + void ReleaseRequest(); + + // Returns the max value of exponential back-off release time for + // |original_url_| and |url_|. + base::TimeTicks GetBackoffReleaseTime(); + + void CompleteAddingUploadDataChunk(const std::string& data, + bool is_last_chunk); + + // Adds a block of data to be uploaded in a POST body. This can only be + // called after Start(). + void AppendChunkToUpload(const std::string& data, bool is_last_chunk); + + // Store the response bytes in |buffer_| in the container indicated by + // |response_destination_|. Return true if the write has been + // done, and another read can overwrite |buffer_|. If this function + // returns false, it will post a task that will read more bytes once the + // write is complete. + bool WriteBuffer(int num_bytes); + + // Read response bytes from the request. + void ReadResponse(); + + URLFetcher* fetcher_; // Corresponding fetcher object + GURL original_url_; // The URL we were asked to fetch + GURL url_; // The URL we eventually wound up at + RequestType request_type_; // What type of request is this? + net::URLRequestStatus status_; // Status of the request + URLFetcher::Delegate* delegate_; // Object to notify on completion + scoped_refptr<base::MessageLoopProxy> delegate_loop_proxy_; + // Message loop proxy of the creating + // thread. + scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_; + // The message loop proxy for the thread + // on which the request IO happens. + scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy_; + // The message loop proxy for the thread + // on which file access happens. + scoped_ptr<net::URLRequest> request_; // The actual request this wraps + int load_flags_; // Flags for the load operation + int response_code_; // HTTP status code for the request + std::string data_; // Results of the request, when we are + // storing the response as a string. + scoped_refptr<net::IOBuffer> buffer_; + // Read buffer + scoped_refptr<net::URLRequestContextGetter> request_context_getter_; + // Cookie/cache info for the request + net::ResponseCookies cookies_; // Response cookies + net::HttpRequestHeaders extra_request_headers_; + scoped_refptr<net::HttpResponseHeaders> response_headers_; + bool was_fetched_via_proxy_; + net::HostPortPair socket_address_; + + std::string upload_content_; // HTTP POST payload + std::string upload_content_type_; // MIME type of POST payload + std::string referrer_; // HTTP Referer header value + bool is_chunked_upload_; // True if using chunked transfer encoding + + // Used to determine how long to wait before making a request or doing a + // retry. + // Both of them can only be accessed on the IO thread. + // We need not only the throttler entry for |original_URL|, but also the one + // for |url|. For example, consider the case that URL A redirects to URL B, + // for which the server returns a 500 response. In this case, the exponential + // back-off release time of URL A won't increase. If we retry without + // considering the back-off constraint of URL B, we may send out too many + // requests for URL A in a short period of time. + scoped_refptr<net::URLRequestThrottlerEntryInterface> + original_url_throttler_entry_; + scoped_refptr<net::URLRequestThrottlerEntryInterface> url_throttler_entry_; + + // |num_retries_| indicates how many times we've failed to successfully + // fetch this URL. Once this value exceeds the maximum number of retries + // specified by the owner URLFetcher instance, we'll give up. + int num_retries_; + + // True if the URLFetcher has been cancelled. + bool was_cancelled_; + + // Since GetBackoffReleaseTime() can only be called on the IO thread, we cache + // its value to be used by OnCompletedURLRequest on the creating thread. + base::TimeTicks backoff_release_time_; + + // If writing results to a file, |temp_file_writer_| will manage creation, + // writing, and destruction of that file. + scoped_ptr<TempFileWriter> temp_file_writer_; + + // Where should responses be saved? + ResponseDestinationType response_destination_; + + static base::LazyInstance<Registry> g_registry; + + friend class URLFetcher; + DISALLOW_COPY_AND_ASSIGN(Core); +}; + +URLFetcher::Core::Registry::Registry() {} +URLFetcher::Core::Registry::~Registry() {} + +void URLFetcher::Core::Registry::AddURLFetcherCore(Core* core) { + DCHECK(!ContainsKey(fetchers_, core)); + fetchers_.insert(core); +} + +void URLFetcher::Core::Registry::RemoveURLFetcherCore(Core* core) { + DCHECK(ContainsKey(fetchers_, core)); + fetchers_.erase(core); +} + +void URLFetcher::Core::Registry::CancelAll() { + while (!fetchers_.empty()) + (*fetchers_.begin())->CancelURLRequest(); +} + +// static +base::LazyInstance<URLFetcher::Core::Registry> + URLFetcher::Core::g_registry(base::LINKER_INITIALIZED); + +URLFetcher::Core::TempFileWriter::TempFileWriter( + URLFetcher::Core* core, + scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy) + : core_(core), + error_code_(base::PLATFORM_FILE_OK), + callback_factory_(ALLOW_THIS_IN_INITIALIZER_LIST(this)), + file_message_loop_proxy_(file_message_loop_proxy) { +} + +URLFetcher::Core::TempFileWriter::~TempFileWriter() { + Destroy(); +} + +void URLFetcher::Core::TempFileWriter::CreateTempFile() { + CHECK(file_message_loop_proxy_.get()); + base::FileUtilProxy::CreateTemporary( + file_message_loop_proxy_, + callback_factory_.NewCallback( + &URLFetcher::Core::TempFileWriter::DidCreateTempFile)); +} + +void URLFetcher::Core::TempFileWriter::DidCreateTempFile( + base::PlatformFileError error_code, + base::PassPlatformFile file_handle, + FilePath file_path) { + if (base::PLATFORM_FILE_OK != error_code) { + error_code_ = error_code; + core_->InformDelegateFetchIsComplete(); + return; + } + + temp_file_ = file_path; + + // The file was opened with async writes enabled. FileUtilProxy::Write() + // treats a write that returns IO_PENDING as an error, and does not inform + // the caller. We need to close and reopen the file with asyncronus writes + // disabled. + // TODO(skerner): Make FileUtilProxy::Write() play nice with async IO. + base::FileUtilProxy::Close( + file_message_loop_proxy_, + file_handle.ReleaseValue(), + callback_factory_.NewCallback( + &URLFetcher::Core::TempFileWriter::DidCloseTempFile)); +} + +void URLFetcher::Core::TempFileWriter::DidCloseTempFile( + base::PlatformFileError error_code) { + if (base::PLATFORM_FILE_OK != error_code) { + error_code_ = error_code; + core_->InformDelegateFetchIsComplete(); + return; + } + + int file_flags = + base::PLATFORM_FILE_CREATE_ALWAYS | + base::PLATFORM_FILE_WRITE | + base::PLATFORM_FILE_TEMPORARY; + + base::FileUtilProxy::CreateOrOpen( + file_message_loop_proxy_, + temp_file_, + file_flags, + callback_factory_.NewCallback( + &URLFetcher::Core::TempFileWriter::DidReopenTempFile)); +} + +void URLFetcher::Core::TempFileWriter::DidReopenTempFile( + base::PlatformFileError error_code, + base::PassPlatformFile file_handle, + bool created) { + if (base::PLATFORM_FILE_OK != error_code) { + error_code_ = error_code; + core_->InformDelegateFetchIsComplete(); + return; + } + + temp_file_handle_ = file_handle.ReleaseValue(); + total_bytes_written_ = 0; + + core_->io_message_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(core_, &Core::StartURLRequestWhenAppropriate)); +} + +void URLFetcher::Core::TempFileWriter::WriteBuffer(int num_bytes) { + // Start writing to the temp file by setting the initial state + // of |pending_bytes_| and |buffer_offset_| to indicate that the + // entire buffer has not yet been written. + pending_bytes_ = num_bytes; + buffer_offset_ = 0; + ContinueWrite(base::PLATFORM_FILE_OK, 0); +} + +void URLFetcher::Core::TempFileWriter::ContinueWrite( + base::PlatformFileError error_code, + int bytes_written) { + if (base::PLATFORM_FILE_OK != error_code) { + error_code_ = error_code; + core_->InformDelegateFetchIsComplete(); + return; + } + + total_bytes_written_ += bytes_written; + buffer_offset_ += bytes_written; + pending_bytes_ -= bytes_written; + + if (pending_bytes_ > 0) { + base::FileUtilProxy::Write( + file_message_loop_proxy_, + temp_file_handle_, + total_bytes_written_, // Append to the end + (core_->buffer_->data() + buffer_offset_), + pending_bytes_, + callback_factory_.NewCallback( + &URLFetcher::Core::TempFileWriter::ContinueWrite)); + } else { + // Finished writing core_->buffer_ to the file. Read some more. + core_->ReadResponse(); + } +} + +void URLFetcher::Core::TempFileWriter::DisownTempFile() { + // Forget about any temp file by reseting the path. + if (!temp_file_.empty()) { + base::FileUtilProxy::Close( + file_message_loop_proxy_, + temp_file_handle_, + NULL); + temp_file_ = FilePath(); + } +} + +void URLFetcher::Core::TempFileWriter::Destroy() { + if (!temp_file_.empty()) { + base::FileUtilProxy::Close( + file_message_loop_proxy_, + temp_file_handle_, + NULL); + + base::FileUtilProxy::Delete( + file_message_loop_proxy_, + temp_file_, + false, // No need to recurse, as the path is to a file. + NULL); // No callback. + } + temp_file_ = FilePath(); +} + +// static +URLFetcher::Factory* URLFetcher::factory_ = NULL; + +void URLFetcher::Delegate::OnURLFetchComplete( + const URLFetcher* source, + const GURL& url, + const net::URLRequestStatus& status, + int response_code, + const net::ResponseCookies& cookies, + const std::string& data) { + NOTREACHED() << "If you don't implemnt this, the no-params version " + << "should also be implemented, in which case this " + << "method won't be called..."; +} + +// TODO(skerner): This default implementation will be removed, and the +// method made pure virtual, once all users of URLFetcher are updated +// to not expect response data as a string argument. Once this is removed, +// the method URLFetcher::GetResponseStringRef() can be removed as well. +// crbug.com/83592 tracks this. +void URLFetcher::Delegate::OnURLFetchComplete(const URLFetcher* source) { + // A delegate that did not override this method is using the old + // parameter list to OnURLFetchComplete(). If a user asked to save + // the response to a file, they must use the new parameter list, + // in which case we can not get here. + // To avoid updating all callers, thunk to the old prototype for now. + OnURLFetchComplete(source, + source->url(), + source->status(), + source->response_code(), + source->cookies(), + source->GetResponseStringRef()); +} + +// static +bool URLFetcher::g_interception_enabled = false; + +URLFetcher::URLFetcher(const GURL& url, + RequestType request_type, + Delegate* d) + : ALLOW_THIS_IN_INITIALIZER_LIST( + core_(new Core(this, url, request_type, d))), + automatically_retry_on_5xx_(true), + max_retries_(0) { +} + +URLFetcher::~URLFetcher() { + core_->Stop(); +} + +// static +URLFetcher* URLFetcher::Create(int id, const GURL& url, + RequestType request_type, Delegate* d) { + return factory_ ? factory_->CreateURLFetcher(id, url, request_type, d) : + new URLFetcher(url, request_type, d); +} + +URLFetcher::Core::Core(URLFetcher* fetcher, + const GURL& original_url, + RequestType request_type, + URLFetcher::Delegate* d) + : fetcher_(fetcher), + original_url_(original_url), + request_type_(request_type), + delegate_(d), + delegate_loop_proxy_( + base::MessageLoopProxy::CreateForCurrentThread()), + request_(NULL), + load_flags_(net::LOAD_NORMAL), + response_code_(URLFetcher::kInvalidHttpResponseCode), + buffer_(new net::IOBuffer(kBufferSize)), + is_chunked_upload_(false), + num_retries_(0), + was_cancelled_(false), + response_destination_(STRING) { +} + +URLFetcher::Core::~Core() { + // |request_| should be NULL. If not, it's unsafe to delete it here since we + // may not be on the IO thread. + DCHECK(!request_.get()); +} + +void URLFetcher::Core::Start() { + DCHECK(delegate_loop_proxy_); + CHECK(request_context_getter_) << "We need an URLRequestContext!"; + io_message_loop_proxy_ = request_context_getter_->GetIOMessageLoopProxy(); + CHECK(io_message_loop_proxy_.get()) << "We need an IO message loop proxy"; + + switch (response_destination_) { + case STRING: + io_message_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(this, &Core::StartURLRequestWhenAppropriate)); + break; + + case TEMP_FILE: + CHECK(file_message_loop_proxy_.get()) + << "Need to set the file message loop proxy."; + temp_file_writer_.reset( + new TempFileWriter(this, file_message_loop_proxy_)); + // CreateTempFile() will invoke Core::StartURLRequestWhenAppropriate + // once the file is created. + temp_file_writer_->CreateTempFile(); + break; + + default: + NOTREACHED(); + } +} + +void URLFetcher::Core::Stop() { + DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); + delegate_ = NULL; + fetcher_ = NULL; + if (io_message_loop_proxy_.get()) { + io_message_loop_proxy_->PostTask( + FROM_HERE, NewRunnableMethod(this, &Core::CancelURLRequest)); + } +} + +void URLFetcher::Core::ReceivedContentWasMalformed() { + DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); + if (io_message_loop_proxy_.get()) { + io_message_loop_proxy_->PostTask( + FROM_HERE, NewRunnableMethod(this, &Core::NotifyMalformedContent)); + } +} + +void URLFetcher::Core::CancelAll() { + g_registry.Get().CancelAll(); +} + +void URLFetcher::Core::OnResponseStarted(net::URLRequest* request) { + DCHECK_EQ(request, request_.get()); + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + if (request_->status().is_success()) { + response_code_ = request_->GetResponseCode(); + response_headers_ = request_->response_headers(); + socket_address_ = request_->GetSocketAddress(); + was_fetched_via_proxy_ = request_->was_fetched_via_proxy(); + } + + ReadResponse(); +} + +void URLFetcher::Core::CompleteAddingUploadDataChunk( + const std::string& content, bool is_last_chunk) { + DCHECK(is_chunked_upload_); + DCHECK(request_.get()); + DCHECK(!content.empty()); + request_->AppendChunkToUpload(content.data(), + static_cast<int>(content.length()), + is_last_chunk); +} + +void URLFetcher::Core::AppendChunkToUpload(const std::string& content, + bool is_last_chunk) { + DCHECK(delegate_loop_proxy_); + CHECK(io_message_loop_proxy_.get()); + io_message_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(this, &Core::CompleteAddingUploadDataChunk, content, + is_last_chunk)); +} + +// Return true if the write was done and reading may continue. +// Return false if the write is pending, and the next read will +// be done later. +bool URLFetcher::Core::WriteBuffer(int num_bytes) { + bool write_complete = false; + switch (response_destination_) { + case STRING: + data_.append(buffer_->data(), num_bytes); + write_complete = true; + break; + + case TEMP_FILE: + temp_file_writer_->WriteBuffer(num_bytes); + // WriteBuffer() sends a request the file thread. + // The write is not done yet. + write_complete = false; + break; + + default: + NOTREACHED(); + } + return write_complete; +} + +void URLFetcher::Core::OnReadCompleted(net::URLRequest* request, + int bytes_read) { + DCHECK(request == request_); + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + url_ = request->url(); + url_throttler_entry_ = + net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(url_); + + bool waiting_on_write = false; + do { + if (!request_->status().is_success() || bytes_read <= 0) + break; + + if (!WriteBuffer(bytes_read)) { + // If WriteBuffer() returns false, we have a pending write to + // wait on before reading further. + waiting_on_write = true; + break; + } + } while (request_->Read(buffer_, kBufferSize, &bytes_read)); + + if (request_->status().is_success()) + request_->GetResponseCookies(&cookies_); + + // See comments re: HEAD requests in ReadResponse(). + if ((!request_->status().is_io_pending() && !waiting_on_write) || + (request_type_ == HEAD)) { + backoff_release_time_ = GetBackoffReleaseTime(); + + bool posted = delegate_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(this, + &Core::OnCompletedURLRequest, + request_->status())); + // If the delegate message loop does not exist any more, then the delegate + // should be gone too. + DCHECK(posted || !delegate_); + ReleaseRequest(); + } +} + +void URLFetcher::Core::ReadResponse() { + // Some servers may treat HEAD requests as GET requests. To free up the + // network connection as soon as possible, signal that the request has + // completed immediately, without trying to read any data back (all we care + // about is the response code and headers, which we already have). + int bytes_read = 0; + if (request_->status().is_success() && (request_type_ != HEAD)) + request_->Read(buffer_, kBufferSize, &bytes_read); + OnReadCompleted(request_.get(), bytes_read); +} + +void URLFetcher::Core::StartURLRequest() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + if (was_cancelled_) { + // Since StartURLRequest() is posted as a *delayed* task, it may + // run after the URLFetcher was already stopped. + return; + } + + CHECK(request_context_getter_); + DCHECK(!request_.get()); + + g_registry.Get().AddURLFetcherCore(this); + request_.reset(new net::URLRequest(original_url_, this)); + int flags = request_->load_flags() | load_flags_; + if (!g_interception_enabled) { + flags = flags | net::LOAD_DISABLE_INTERCEPT; + } + if (is_chunked_upload_) + request_->EnableChunkedUpload(); + request_->set_load_flags(flags); + request_->set_context(request_context_getter_->GetURLRequestContext()); + request_->set_referrer(referrer_); + + switch (request_type_) { + case GET: + break; + + case POST: + DCHECK(!upload_content_.empty() || is_chunked_upload_); + DCHECK(!upload_content_type_.empty()); + + request_->set_method("POST"); + extra_request_headers_.SetHeader(net::HttpRequestHeaders::kContentType, + upload_content_type_); + if (!upload_content_.empty()) { + request_->AppendBytesToUpload( + upload_content_.data(), static_cast<int>(upload_content_.length())); + } + break; + + case HEAD: + request_->set_method("HEAD"); + break; + + default: + NOTREACHED(); + } + + if (!extra_request_headers_.IsEmpty()) + request_->SetExtraRequestHeaders(extra_request_headers_); + + // There might be data left over from a previous request attempt. + data_.clear(); + + // If we are writing the response to a file, the only caller + // of this function should have created it and not written yet. + CHECK(!temp_file_writer_.get() || + temp_file_writer_->total_bytes_written() == 0); + + request_->Start(); +} + +void URLFetcher::Core::StartURLRequestWhenAppropriate() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + if (was_cancelled_) + return; + + if (original_url_throttler_entry_ == NULL) { + original_url_throttler_entry_ = + net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl( + original_url_); + } + + int64 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest( + GetBackoffReleaseTime()); + if (delay == 0) { + StartURLRequest(); + } else { + MessageLoop::current()->PostDelayedTask( + FROM_HERE, + NewRunnableMethod(this, &Core::StartURLRequest), + delay); + } +} + +void URLFetcher::Core::CancelURLRequest() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + + if (request_.get()) { + request_->Cancel(); + ReleaseRequest(); + } + // Release the reference to the request context. There could be multiple + // references to URLFetcher::Core at this point so it may take a while to + // delete the object, but we cannot delay the destruction of the request + // context. + request_context_getter_ = NULL; + was_cancelled_ = true; + temp_file_writer_.reset(); +} + +void URLFetcher::Core::OnCompletedURLRequest( + const net::URLRequestStatus& status) { + DCHECK(delegate_loop_proxy_->BelongsToCurrentThread()); + + // Save the status so that delegates can read it. + status_ = status; + + // Checks the response from server. + if (response_code_ >= 500 || + status.os_error() == net::ERR_TEMPORARILY_THROTTLED) { + // When encountering a server error, we will send the request again + // after backoff time. + ++num_retries_; + // Restarts the request if we still need to notify the delegate. + if (delegate_) { + // Note that backoff_delay_ may be 0 because (a) the URLRequestThrottler + // code does not necessarily back off on the first error, and (b) it + // only backs off on some of the 5xx status codes. + fetcher_->backoff_delay_ = backoff_release_time_ - base::TimeTicks::Now(); + if (fetcher_->backoff_delay_ < base::TimeDelta()) + fetcher_->backoff_delay_ = base::TimeDelta(); + + if (fetcher_->automatically_retry_on_5xx_ && + num_retries_ <= fetcher_->max_retries()) { + io_message_loop_proxy_->PostTask( + FROM_HERE, + NewRunnableMethod(this, &Core::StartURLRequestWhenAppropriate)); + } else { + InformDelegateFetchIsComplete(); + } + } + } else { + if (delegate_) { + fetcher_->backoff_delay_ = base::TimeDelta(); + InformDelegateFetchIsComplete(); + } + } +} + +void URLFetcher::Core::InformDelegateFetchIsComplete() { + delegate_->OnURLFetchComplete(fetcher_); +} + +void URLFetcher::Core::NotifyMalformedContent() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + if (url_throttler_entry_ != NULL) { + int status_code = response_code_; + if (status_code == kInvalidHttpResponseCode) { + // The status code will generally be known by the time clients + // call the |ReceivedContentWasMalformed()| function (which ends up + // calling the current function) but if it's not, we need to assume + // the response was successful so that the total failure count + // used to calculate exponential back-off goes up. + status_code = 200; + } + url_throttler_entry_->ReceivedContentWasMalformed(status_code); + } +} + +void URLFetcher::Core::ReleaseRequest() { + request_.reset(); + g_registry.Get().RemoveURLFetcherCore(this); +} + +base::TimeTicks URLFetcher::Core::GetBackoffReleaseTime() { + DCHECK(io_message_loop_proxy_->BelongsToCurrentThread()); + DCHECK(original_url_throttler_entry_ != NULL); + + base::TimeTicks original_url_backoff = + original_url_throttler_entry_->GetExponentialBackoffReleaseTime(); + base::TimeTicks destination_url_backoff; + if (url_throttler_entry_ != NULL && + original_url_throttler_entry_ != url_throttler_entry_) { + destination_url_backoff = + url_throttler_entry_->GetExponentialBackoffReleaseTime(); + } + + return original_url_backoff > destination_url_backoff ? + original_url_backoff : destination_url_backoff; +} + +void URLFetcher::set_upload_data(const std::string& upload_content_type, + const std::string& upload_content) { + DCHECK(!core_->is_chunked_upload_); + core_->upload_content_type_ = upload_content_type; + core_->upload_content_ = upload_content; +} + +void URLFetcher::set_chunked_upload(const std::string& content_type) { + DCHECK(core_->is_chunked_upload_ || + (core_->upload_content_type_.empty() && + core_->upload_content_.empty())); + core_->upload_content_type_ = content_type; + core_->upload_content_.clear(); + core_->is_chunked_upload_ = true; +} + +void URLFetcher::AppendChunkToUpload(const std::string& data, + bool is_last_chunk) { + DCHECK(data.length()); + core_->AppendChunkToUpload(data, is_last_chunk); +} + +const std::string& URLFetcher::upload_data() const { + return core_->upload_content_; +} + +void URLFetcher::set_referrer(const std::string& referrer) { + core_->referrer_ = referrer; +} + +void URLFetcher::set_load_flags(int load_flags) { + core_->load_flags_ = load_flags; +} + +int URLFetcher::load_flags() const { + return core_->load_flags_; +} + +void URLFetcher::set_extra_request_headers( + const std::string& extra_request_headers) { + core_->extra_request_headers_.Clear(); + core_->extra_request_headers_.AddHeadersFromString(extra_request_headers); +} + +void URLFetcher::set_request_context( + net::URLRequestContextGetter* request_context_getter) { + core_->request_context_getter_ = request_context_getter; +} + +void URLFetcher::set_automatically_retry_on_5xx(bool retry) { + automatically_retry_on_5xx_ = retry; +} + +void URLFetcher::SaveResponseToTemporaryFile( + scoped_refptr<base::MessageLoopProxy> file_message_loop_proxy) { + core_->file_message_loop_proxy_ = file_message_loop_proxy; + core_->response_destination_ = TEMP_FILE; +} + +net::HttpResponseHeaders* URLFetcher::response_headers() const { + return core_->response_headers_; +} + +// TODO(panayiotis): socket_address_ is written in the IO thread, +// if this is accessed in the UI thread, this could result in a race. +// Same for response_headers_ above and was_fetched_via_proxy_ below. +net::HostPortPair URLFetcher::socket_address() const { + return core_->socket_address_; +} + +bool URLFetcher::was_fetched_via_proxy() const { + return core_->was_fetched_via_proxy_; +} + +void URLFetcher::Start() { + core_->Start(); +} + +const GURL& URLFetcher::url() const { + return core_->url_; +} + +const net::URLRequestStatus& URLFetcher::status() const { + return core_->status_; +} + +int URLFetcher::response_code() const { + return core_->response_code_; +} + +const net::ResponseCookies& URLFetcher::cookies() const { + return core_->cookies_; +} + +bool URLFetcher::FileErrorOccurred( + base::PlatformFileError* out_error_code) const { + + // Can't have a file error if no file is being created or written to. + if (!core_->temp_file_writer_.get()) { + return false; + } + + base::PlatformFileError error_code = core_->temp_file_writer_->error_code(); + if (error_code == base::PLATFORM_FILE_OK) + return false; + + *out_error_code = error_code; + return true; +} + +void URLFetcher::ReceivedContentWasMalformed() { + core_->ReceivedContentWasMalformed(); +} + +bool URLFetcher::GetResponseAsString(std::string* out_response_string) const { + if (core_->response_destination_ != STRING) + return false; + + *out_response_string = core_->data_; + return true; +} + +const std::string& URLFetcher::GetResponseStringRef() const { + CHECK(core_->response_destination_ == STRING); + return core_->data_; +} + +void URLFetcher::SetResponseDestinationForTesting( + ResponseDestinationType value) { + core_->response_destination_ = value; +} + +URLFetcher::ResponseDestinationType +URLFetcher::GetResponseDestinationForTesting() const { + return core_->response_destination_; +} + +bool URLFetcher::GetResponseAsFilePath(bool take_ownership, + FilePath* out_response_path) const { + if (core_->response_destination_ != TEMP_FILE || + !core_->temp_file_writer_.get()) + return false; + + *out_response_path = core_->temp_file_writer_->temp_file(); + + if (take_ownership) + core_->temp_file_writer_->DisownTempFile(); + + return true; +} + +// static +void URLFetcher::CancelAll() { + Core::CancelAll(); +} + +// static +int URLFetcher::GetNumFetcherCores() { + return Core::g_registry.Get().size(); +} + +URLFetcher::Delegate* URLFetcher::delegate() const { + return core_->delegate(); +} |