// Copyright (c) 2011 The Chromium Authors. All rights reserved. // Copyright (c) 2012, Code Aurora Forum. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "net/http/http_stream_parser.h" #include "base/compiler_specific.h" #include "base/metrics/histogram.h" #include "base/string_util.h" #include "net/base/address_list.h" #include "net/base/auth.h" #include "net/base/io_buffer.h" #include "net/base/ssl_cert_request_info.h" #include "net/http/http_net_log_params.h" #include "net/http/http_request_headers.h" #include "net/http/http_request_info.h" #include "net/http/http_response_headers.h" #include "net/http/http_util.h" #include "net/socket/ssl_client_socket.h" #include "net/socket/client_socket_handle.h" #include "net/http/http_getzip_factory.h" namespace net { HttpStreamParser::HttpStreamParser(ClientSocketHandle* connection, const HttpRequestInfo* request, GrowableIOBuffer* read_buffer, const BoundNetLog& net_log, bool using_proxy) : io_state_(STATE_NONE), request_(request), request_headers_(NULL), request_body_(NULL), read_buf_(read_buffer), read_buf_unused_offset_(0), response_header_start_offset_(-1), response_body_length_(-1), response_body_read_(0), chunked_decoder_(NULL), user_read_buf_(NULL), user_read_buf_len_(0), user_callback_(NULL), connection_(connection), net_log_(net_log), ALLOW_THIS_IN_INITIALIZER_LIST( io_callback_(this, &HttpStreamParser::OnIOComplete)), chunk_length_(0), chunk_length_without_encoding_(0), sent_last_chunk_(false), using_proxy_(using_proxy), has_to_retry_(false){ DCHECK_EQ(0, read_buffer->offset()); } HttpStreamParser::~HttpStreamParser() { if (request_body_ != NULL && request_body_->is_chunked()) request_body_->set_chunk_callback(NULL); } int HttpStreamParser::SendRequest(const std::string& request_line, const HttpRequestHeaders& headers, UploadDataStream* request_body, HttpResponseInfo* response, CompletionCallback* callback) { DCHECK_EQ(STATE_NONE, io_state_); DCHECK(!user_callback_); DCHECK(callback); DCHECK(response); has_to_retry_ = false; if (net_log_.IsLoggingAllEvents()) { net_log_.AddEvent( NetLog::TYPE_HTTP_TRANSACTION_SEND_REQUEST_HEADERS, make_scoped_refptr(new NetLogHttpRequestParameter( request_line, headers))); } response_ = response; // Put the peer's IP address and port into the response. AddressList address; int result = connection_->socket()->GetPeerAddress(&address); if (result != OK) return result; response_->socket_address = HostPortPair::FromAddrInfo(address.head()); //Shutr only for GET/HEAD requests if((!(using_proxy_)) && ((request_line.find(HttpRequestHeaders::kGetMethod) == 0) || (request_line.find(HttpRequestHeaders::kHeadMethod) == 0))) { HttpGetZipFactory::GetGETZipManager()->CompressRequestHeaders( const_cast(headers), connection_->socket()); } std::string request = request_line + headers.ToString(); scoped_refptr headers_io_buf(new StringIOBuffer(request)); request_headers_ = new DrainableIOBuffer(headers_io_buf, headers_io_buf->size()); request_body_.reset(request_body); if (request_body_ != NULL && request_body_->is_chunked()) { request_body_->set_chunk_callback(this); const int kChunkHeaderFooterSize = 12; // 2 CRLFs + max of 8 hex chars. chunk_buf_ = new IOBuffer(request_body_->GetMaxBufferSize() + kChunkHeaderFooterSize); } io_state_ = STATE_SENDING_HEADERS; result = DoLoop(OK); if (result == ERR_IO_PENDING) user_callback_ = callback; return result > 0 ? OK : result; } int HttpStreamParser::ReadResponseHeaders(CompletionCallback* callback) { DCHECK(io_state_ == STATE_REQUEST_SENT || io_state_ == STATE_DONE); DCHECK(!user_callback_); DCHECK(callback); // This function can be called with io_state_ == STATE_DONE if the // connection is closed after seeing just a 1xx response code. if (io_state_ == STATE_DONE) return ERR_CONNECTION_CLOSED; int result = OK; io_state_ = STATE_READ_HEADERS; if (read_buf_->offset() > 0) { // Simulate the state where the data was just read from the socket. result = read_buf_->offset() - read_buf_unused_offset_; read_buf_->set_offset(read_buf_unused_offset_); } if (result > 0) io_state_ = STATE_READ_HEADERS_COMPLETE; result = DoLoop(result); if (result == ERR_IO_PENDING) user_callback_ = callback; return result > 0 ? OK : result; } void HttpStreamParser::Close(bool not_reusable) { if (not_reusable && connection_->socket()) connection_->socket()->Disconnect(); connection_->Reset(); } int HttpStreamParser::ReadResponseBody(IOBuffer* buf, int buf_len, CompletionCallback* callback) { DCHECK(io_state_ == STATE_BODY_PENDING || io_state_ == STATE_DONE); DCHECK(!user_callback_); DCHECK(callback); DCHECK_LE(buf_len, kMaxBufSize); if (io_state_ == STATE_DONE) return OK; user_read_buf_ = buf; user_read_buf_len_ = buf_len; io_state_ = STATE_READ_BODY; int result = DoLoop(OK); if (result == ERR_IO_PENDING) user_callback_ = callback; return result; } void HttpStreamParser::OnIOComplete(int result) { result = DoLoop(result); // The client callback can do anything, including destroying this class, // so any pending callback must be issued after everything else is done. if (result != ERR_IO_PENDING && user_callback_) { CompletionCallback* c = user_callback_; user_callback_ = NULL; if(has_to_retry_ ) { result = ERR_GETZIP; has_to_retry_ = false; } c->Run(result); } } void HttpStreamParser::OnChunkAvailable() { // This method may get called while sending the headers or body, so check // before processing the new data. If we were still initializing or sending // headers, we will automatically start reading the chunks once we get into // STATE_SENDING_BODY so nothing to do here. DCHECK(io_state_ == STATE_SENDING_HEADERS || io_state_ == STATE_SENDING_BODY); if (io_state_ == STATE_SENDING_BODY) OnIOComplete(0); } int HttpStreamParser::DoLoop(int result) { bool can_do_more = true; do { switch (io_state_) { case STATE_SENDING_HEADERS: if (result < 0) can_do_more = false; else result = DoSendHeaders(result); break; case STATE_SENDING_BODY: if (result < 0) can_do_more = false; else result = DoSendBody(result); break; case STATE_REQUEST_SENT: DCHECK(result != ERR_IO_PENDING); can_do_more = false; break; case STATE_READ_HEADERS: net_log_.BeginEvent(NetLog::TYPE_HTTP_STREAM_PARSER_READ_HEADERS, NULL); result = DoReadHeaders(); break; case STATE_READ_HEADERS_COMPLETE: result = DoReadHeadersComplete(result); if(!using_proxy_) { GETZipDecompressionStatus st = HttpGetZipFactory::GetGETZipManager()->DecompressResponseHeaders( response_->headers.get(), connection_->socket()); if( st == REQUEST_RETRY_NEEDED ) { has_to_retry_ = true; } } net_log_.EndEventWithNetErrorCode( NetLog::TYPE_HTTP_STREAM_PARSER_READ_HEADERS, result); break; case STATE_BODY_PENDING: DCHECK(result != ERR_IO_PENDING); can_do_more = false; break; case STATE_READ_BODY: result = DoReadBody(); // DoReadBodyComplete handles error conditions. break; case STATE_READ_BODY_COMPLETE: result = DoReadBodyComplete(result); break; case STATE_DONE: DCHECK(result != ERR_IO_PENDING); can_do_more = false; break; default: NOTREACHED(); can_do_more = false; break; } } while (result != ERR_IO_PENDING && can_do_more); return result; } int HttpStreamParser::DoSendHeaders(int result) { request_headers_->DidConsume(result); int bytes_remaining = request_headers_->BytesRemaining(); if (bytes_remaining > 0) { // Record our best estimate of the 'request time' as the time when we send // out the first bytes of the request headers. if (bytes_remaining == request_headers_->size()) { response_->request_time = base::Time::Now(); // We'll record the count of uncoalesced packets IFF coalescing will help, // and otherwise we'll use an enum to tell why it won't help. enum COALESCE_POTENTIAL { // Coalescing won't reduce packet count. NO_ADVANTAGE = 0, // There is only a header packet or we have a request body but the // request body isn't available yet (can't coalesce). HEADER_ONLY = 1, // Various cases of coalasced savings. COALESCE_POTENTIAL_MAX = 30 }; size_t coalesce = HEADER_ONLY; if (request_body_ != NULL && !request_body_->is_chunked()) { const size_t kBytesPerPacket = 1430; uint64 body_packets = (request_body_->size() + kBytesPerPacket - 1) / kBytesPerPacket; uint64 header_packets = (bytes_remaining + kBytesPerPacket - 1) / kBytesPerPacket; uint64 coalesced_packets = (request_body_->size() + bytes_remaining + kBytesPerPacket - 1) / kBytesPerPacket; if (coalesced_packets < header_packets + body_packets) { if (coalesced_packets > COALESCE_POTENTIAL_MAX) coalesce = COALESCE_POTENTIAL_MAX; else coalesce = static_cast(header_packets + body_packets); } else { coalesce = NO_ADVANTAGE; } } UMA_HISTOGRAM_ENUMERATION("Net.CoalescePotential", coalesce, COALESCE_POTENTIAL_MAX); } result = connection_->socket()->Write(request_headers_, bytes_remaining, &io_callback_); } else if (request_body_ != NULL && (request_body_->is_chunked() || request_body_->size())) { io_state_ = STATE_SENDING_BODY; result = OK; } else { io_state_ = STATE_REQUEST_SENT; } return result; } int HttpStreamParser::DoSendBody(int result) { if (request_body_->is_chunked()) { chunk_length_ -= result; if (chunk_length_) { memmove(chunk_buf_->data(), chunk_buf_->data() + result, chunk_length_); return connection_->socket()->Write(chunk_buf_, chunk_length_, &io_callback_); } if (sent_last_chunk_) { io_state_ = STATE_REQUEST_SENT; return OK; } request_body_->MarkConsumedAndFillBuffer(chunk_length_without_encoding_); chunk_length_without_encoding_ = 0; chunk_length_ = 0; int buf_len = static_cast(request_body_->buf_len()); if (request_body_->eof()) { static const char kLastChunk[] = "0\r\n\r\n"; chunk_length_ = strlen(kLastChunk); memcpy(chunk_buf_->data(), kLastChunk, chunk_length_); sent_last_chunk_ = true; } else if (buf_len) { // Encode and send the buffer as 1 chunk. std::string chunk_header = StringPrintf("%X\r\n", buf_len); char* chunk_ptr = chunk_buf_->data(); memcpy(chunk_ptr, chunk_header.data(), chunk_header.length()); chunk_ptr += chunk_header.length(); memcpy(chunk_ptr, request_body_->buf()->data(), buf_len); chunk_ptr += buf_len; memcpy(chunk_ptr, "\r\n", 2); chunk_length_without_encoding_ = buf_len; chunk_length_ = chunk_header.length() + buf_len + 2; } if (!chunk_length_) // More POST data is yet to come? return ERR_IO_PENDING; return connection_->socket()->Write(chunk_buf_, chunk_length_, &io_callback_); } // Non-chunked request body. request_body_->MarkConsumedAndFillBuffer(result); if (!request_body_->eof()) { int buf_len = static_cast(request_body_->buf_len()); result = connection_->socket()->Write(request_body_->buf(), buf_len, &io_callback_); } else { io_state_ = STATE_REQUEST_SENT; } return result; } int HttpStreamParser::DoReadHeaders() { io_state_ = STATE_READ_HEADERS_COMPLETE; // Grow the read buffer if necessary. if (read_buf_->RemainingCapacity() == 0) read_buf_->SetCapacity(read_buf_->capacity() + kHeaderBufInitialSize); // http://crbug.com/16371: We're seeing |user_buf_->data()| return NULL. // See if the user is passing in an IOBuffer with a NULL |data_|. CHECK(read_buf_->data()); return connection_->socket()->Read(read_buf_, read_buf_->RemainingCapacity(), &io_callback_); } int HttpStreamParser::DoReadHeadersComplete(int result) { if (result == 0) result = ERR_CONNECTION_CLOSED; if (result < 0 && result != ERR_CONNECTION_CLOSED) { io_state_ = STATE_DONE; return result; } // If we've used the connection before, then we know it is not a HTTP/0.9 // response and return ERR_CONNECTION_CLOSED. if (result == ERR_CONNECTION_CLOSED && read_buf_->offset() == 0 && connection_->is_reused()) { io_state_ = STATE_DONE; return result; } // Record our best estimate of the 'response time' as the time when we read // the first bytes of the response headers. if (read_buf_->offset() == 0 && result != ERR_CONNECTION_CLOSED) response_->response_time = base::Time::Now(); if (result == ERR_CONNECTION_CLOSED) { // The connection closed before we detected the end of the headers. // parse things as well as we can and let the caller decide what to do. if (read_buf_->offset() == 0) { // The connection was closed before any data was sent. Likely an error // rather than empty HTTP/0.9 response. io_state_ = STATE_DONE; return ERR_EMPTY_RESPONSE; } else { int end_offset; if (response_header_start_offset_ >= 0) { io_state_ = STATE_READ_BODY_COMPLETE; end_offset = read_buf_->offset(); } else { io_state_ = STATE_BODY_PENDING; end_offset = 0; } int rv = DoParseResponseHeaders(end_offset); if (rv < 0) return rv; return result; } } read_buf_->set_offset(read_buf_->offset() + result); DCHECK_LE(read_buf_->offset(), read_buf_->capacity()); DCHECK_GE(result, 0); int end_of_header_offset = ParseResponseHeaders(); // Note: -1 is special, it indicates we haven't found the end of headers. // Anything less than -1 is a net::Error, so we bail out. if (end_of_header_offset < -1) return end_of_header_offset; if (end_of_header_offset == -1) { io_state_ = STATE_READ_HEADERS; // Prevent growing the headers buffer indefinitely. if (read_buf_->offset() - read_buf_unused_offset_ >= kMaxHeaderBufSize) { io_state_ = STATE_DONE; return ERR_RESPONSE_HEADERS_TOO_BIG; } } else { // Note where the headers stop. read_buf_unused_offset_ = end_of_header_offset; if (response_->headers->response_code() / 100 == 1) { // After processing a 1xx response, the caller will ask for the next // header, so reset state to support that. We don't just skip these // completely because 1xx codes aren't acceptable when establishing a // tunnel. io_state_ = STATE_REQUEST_SENT; response_header_start_offset_ = -1; } else { io_state_ = STATE_BODY_PENDING; CalculateResponseBodySize(); // If the body is 0, the caller may not call ReadResponseBody, which // is where any extra data is copied to read_buf_, so we move the // data here and transition to DONE. if (response_body_length_ == 0) { io_state_ = STATE_DONE; int extra_bytes = read_buf_->offset() - read_buf_unused_offset_; if (extra_bytes) { CHECK_GT(extra_bytes, 0); memmove(read_buf_->StartOfBuffer(), read_buf_->StartOfBuffer() + read_buf_unused_offset_, extra_bytes); } read_buf_->SetCapacity(extra_bytes); read_buf_unused_offset_ = 0; return OK; } } } return result; } int HttpStreamParser::DoReadBody() { io_state_ = STATE_READ_BODY_COMPLETE; // There may be some data left over from reading the response headers. if (read_buf_->offset()) { int available = read_buf_->offset() - read_buf_unused_offset_; if (available) { CHECK_GT(available, 0); int bytes_from_buffer = std::min(available, user_read_buf_len_); memcpy(user_read_buf_->data(), read_buf_->StartOfBuffer() + read_buf_unused_offset_, bytes_from_buffer); read_buf_unused_offset_ += bytes_from_buffer; if (bytes_from_buffer == available) { read_buf_->SetCapacity(0); read_buf_unused_offset_ = 0; } return bytes_from_buffer; } else { read_buf_->SetCapacity(0); read_buf_unused_offset_ = 0; } } // Check to see if we're done reading. if (IsResponseBodyComplete()) return 0; DCHECK_EQ(0, read_buf_->offset()); return connection_->socket()->Read(user_read_buf_, user_read_buf_len_, &io_callback_); } int HttpStreamParser::DoReadBodyComplete(int result) { // If we didn't get a content-length and aren't using a chunked encoding, // the only way to signal the end of a stream is to close the connection, // so we don't treat that as an error, though in some cases we may not // have completely received the resource. if (result == 0 && !IsResponseBodyComplete() && CanFindEndOfResponse()) result = ERR_CONNECTION_CLOSED; // Filter incoming data if appropriate. FilterBuf may return an error. if (result > 0 && chunked_decoder_.get()) { result = chunked_decoder_->FilterBuf(user_read_buf_->data(), result); if (result == 0 && !chunked_decoder_->reached_eof()) { // Don't signal completion of the Read call yet or else it'll look like // we received end-of-file. Wait for more data. io_state_ = STATE_READ_BODY; return OK; } } if (result > 0) response_body_read_ += result; if (result <= 0 || IsResponseBodyComplete()) { io_state_ = STATE_DONE; // Save the overflow data, which can be in two places. There may be // some left over in |user_read_buf_|, plus there may be more // in |read_buf_|. But the part left over in |user_read_buf_| must have // come from the |read_buf_|, so there's room to put it back at the // start first. int additional_save_amount = read_buf_->offset() - read_buf_unused_offset_; int save_amount = 0; if (chunked_decoder_.get()) { save_amount = chunked_decoder_->bytes_after_eof(); } else if (response_body_length_ >= 0) { int64 extra_data_read = response_body_read_ - response_body_length_; if (extra_data_read > 0) { save_amount = static_cast(extra_data_read); if (result > 0) result -= save_amount; } } CHECK_LE(save_amount + additional_save_amount, kMaxBufSize); if (read_buf_->capacity() < save_amount + additional_save_amount) { read_buf_->SetCapacity(save_amount + additional_save_amount); } if (save_amount) { memcpy(read_buf_->StartOfBuffer(), user_read_buf_->data() + result, save_amount); } read_buf_->set_offset(save_amount); if (additional_save_amount) { memmove(read_buf_->data(), read_buf_->StartOfBuffer() + read_buf_unused_offset_, additional_save_amount); read_buf_->set_offset(save_amount + additional_save_amount); } read_buf_unused_offset_ = 0; } else { io_state_ = STATE_BODY_PENDING; user_read_buf_ = NULL; user_read_buf_len_ = 0; } return result; } int HttpStreamParser::ParseResponseHeaders() { int end_offset = -1; // Look for the start of the status line, if it hasn't been found yet. if (response_header_start_offset_ < 0) { response_header_start_offset_ = HttpUtil::LocateStartOfStatusLine( read_buf_->StartOfBuffer() + read_buf_unused_offset_, read_buf_->offset() - read_buf_unused_offset_); } if (response_header_start_offset_ >= 0) { end_offset = HttpUtil::LocateEndOfHeaders( read_buf_->StartOfBuffer() + read_buf_unused_offset_, read_buf_->offset() - read_buf_unused_offset_, response_header_start_offset_); } else if (read_buf_->offset() - read_buf_unused_offset_ >= 8) { // Enough data to decide that this is an HTTP/0.9 response. // 8 bytes = (4 bytes of junk) + "http".length() end_offset = 0; } if (end_offset == -1) return -1; int rv = DoParseResponseHeaders(end_offset); if (rv < 0) return rv; return end_offset + read_buf_unused_offset_; } int HttpStreamParser::DoParseResponseHeaders(int end_offset) { scoped_refptr headers; if (response_header_start_offset_ >= 0) { headers = new HttpResponseHeaders(HttpUtil::AssembleRawHeaders( read_buf_->StartOfBuffer() + read_buf_unused_offset_, end_offset)); } else { // Enough data was read -- there is no status line. headers = new HttpResponseHeaders(std::string("HTTP/0.9 200 OK")); } // Check for multiple Content-Length headers with a Transfer-Encoding header. // If they exist, it's a potential response smuggling attack. void* it = NULL; const std::string content_length_header("Content-Length"); std::string content_length_value; if (!headers->HasHeader("Transfer-Encoding") && headers->EnumerateHeader( &it, content_length_header, &content_length_value)) { // Ok, there's no Transfer-Encoding header and there's at least one // Content-Length header. Check if there are any more Content-Length // headers, and if so, make sure they have the same value. Otherwise, it's // a possible response smuggling attack. std::string content_length_value2; while (headers->EnumerateHeader( &it, content_length_header, &content_length_value2)) { if (content_length_value != content_length_value2) return ERR_RESPONSE_HEADERS_MULTIPLE_CONTENT_LENGTH; } } response_->headers = headers; response_->vary_data.Init(*request_, *response_->headers); return OK; } void HttpStreamParser::CalculateResponseBodySize() { // Figure how to determine EOF: // For certain responses, we know the content length is always 0. From // RFC 2616 Section 4.3 Message Body: // // For response messages, whether or not a message-body is included with // a message is dependent on both the request method and the response // status code (section 6.1.1). All responses to the HEAD request method // MUST NOT include a message-body, even though the presence of entity- // header fields might lead one to believe they do. All 1xx // (informational), 204 (no content), and 304 (not modified) responses // MUST NOT include a message-body. All other responses do include a // message-body, although it MAY be of zero length. switch (response_->headers->response_code()) { // Note that 1xx was already handled earlier. case 204: // No Content case 205: // Reset Content case 304: // Not Modified response_body_length_ = 0; break; } if (request_->method == "HEAD") response_body_length_ = 0; if (response_body_length_ == -1) { // Ignore spurious chunked responses from HTTP/1.0 servers and // proxies. Otherwise "Transfer-Encoding: chunked" trumps // "Content-Length: N" if (response_->headers->GetHttpVersion() >= HttpVersion(1, 1) && response_->headers->HasHeaderValue("Transfer-Encoding", "chunked")) { chunked_decoder_.reset(new HttpChunkedDecoder()); } else { response_body_length_ = response_->headers->GetContentLength(); // If response_body_length_ is still -1, then we have to wait // for the server to close the connection. } } } uint64 HttpStreamParser::GetUploadProgress() const { if (!request_body_.get()) return 0; return request_body_->position(); } HttpResponseInfo* HttpStreamParser::GetResponseInfo() { return response_; } bool HttpStreamParser::IsResponseBodyComplete() const { if (chunked_decoder_.get()) return chunked_decoder_->reached_eof(); if (response_body_length_ != -1) return response_body_read_ >= response_body_length_; return false; // Must read to EOF. } bool HttpStreamParser::CanFindEndOfResponse() const { return chunked_decoder_.get() || response_body_length_ >= 0; } bool HttpStreamParser::IsMoreDataBuffered() const { return read_buf_->offset() > read_buf_unused_offset_; } bool HttpStreamParser::IsConnectionReused() const { ClientSocketHandle::SocketReuseType reuse_type = connection_->reuse_type(); return connection_->is_reused() || reuse_type == ClientSocketHandle::UNUSED_IDLE; } void HttpStreamParser::SetConnectionReused() { connection_->set_is_reused(true); } bool HttpStreamParser::IsConnectionReusable() const { return connection_->socket() && connection_->socket()->IsConnectedAndIdle(); } void HttpStreamParser::GetSSLInfo(SSLInfo* ssl_info) { if (request_->url.SchemeIs("https") && connection_->socket()) { SSLClientSocket* ssl_socket = static_cast(connection_->socket()); ssl_socket->GetSSLInfo(ssl_info); } } void HttpStreamParser::GetSSLCertRequestInfo( SSLCertRequestInfo* cert_request_info) { if (request_->url.SchemeIs("https") && connection_->socket()) { SSLClientSocket* ssl_socket = static_cast(connection_->socket()); ssl_socket->GetSSLCertRequestInfo(cert_request_info); } } } // namespace net