1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
|
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/url_request/url_fetcher_core.h"
#include <stdint.h>
#include "base/bind.h"
#include "base/logging.h"
#include "base/metrics/histogram_macros.h"
#include "base/profiler/scoped_tracker.h"
#include "base/sequenced_task_runner.h"
#include "base/single_thread_task_runner.h"
#include "base/stl_util.h"
#include "base/thread_task_runner_handle.h"
#include "base/tracked_objects.h"
#include "net/base/elements_upload_data_stream.h"
#include "net/base/io_buffer.h"
#include "net/base/load_flags.h"
#include "net/base/net_errors.h"
#include "net/base/request_priority.h"
#include "net/base/upload_bytes_element_reader.h"
#include "net/base/upload_data_stream.h"
#include "net/base/upload_file_element_reader.h"
#include "net/http/http_response_headers.h"
#include "net/url_request/redirect_info.h"
#include "net/url_request/url_fetcher_delegate.h"
#include "net/url_request/url_fetcher_response_writer.h"
#include "net/url_request/url_request_context.h"
#include "net/url_request/url_request_context_getter.h"
#include "net/url_request/url_request_throttler_manager.h"
namespace {
const int kBufferSize = 4096;
const int kUploadProgressTimerInterval = 100;
bool g_ignore_certificate_requests = false;
void EmptyCompletionCallback(int result) {}
} // namespace
namespace net {
// URLFetcherCore::Registry ---------------------------------------------------
URLFetcherCore::Registry::Registry() {}
URLFetcherCore::Registry::~Registry() {}
void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
DCHECK(!ContainsKey(fetchers_, core));
fetchers_.insert(core);
}
void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
DCHECK(ContainsKey(fetchers_, core));
fetchers_.erase(core);
}
void URLFetcherCore::Registry::CancelAll() {
while (!fetchers_.empty())
(*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
}
// URLFetcherCore -------------------------------------------------------------
// static
base::LazyInstance<URLFetcherCore::Registry>
URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
const GURL& original_url,
URLFetcher::RequestType request_type,
URLFetcherDelegate* d)
: fetcher_(fetcher),
original_url_(original_url),
request_type_(request_type),
delegate_(d),
delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
load_flags_(LOAD_NORMAL),
response_code_(URLFetcher::RESPONSE_CODE_INVALID),
buffer_(new IOBuffer(kBufferSize)),
url_request_data_key_(NULL),
was_fetched_via_proxy_(false),
was_cached_(false),
received_response_content_length_(0),
total_received_bytes_(0),
upload_content_set_(false),
upload_range_offset_(0),
upload_range_length_(0),
referrer_policy_(
URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
is_chunked_upload_(false),
was_cancelled_(false),
stop_on_redirect_(false),
stopped_on_redirect_(false),
automatically_retry_on_5xx_(true),
num_retries_on_5xx_(0),
max_retries_on_5xx_(0),
num_retries_on_network_changes_(0),
max_retries_on_network_changes_(0),
current_upload_bytes_(-1),
current_response_bytes_(0),
total_response_bytes_(-1) {
CHECK(original_url_.is_valid());
}
void URLFetcherCore::Start() {
DCHECK(delegate_task_runner_.get());
DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
if (network_task_runner_.get()) {
DCHECK_EQ(network_task_runner_,
request_context_getter_->GetNetworkTaskRunner());
} else {
network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
}
DCHECK(network_task_runner_.get()) << "We need an IO task runner";
network_task_runner_->PostTask(
FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
}
void URLFetcherCore::Stop() {
if (delegate_task_runner_.get()) // May be NULL in tests.
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
delegate_ = NULL;
fetcher_ = NULL;
if (!network_task_runner_.get())
return;
if (network_task_runner_->RunsTasksOnCurrentThread()) {
CancelURLRequest(ERR_ABORTED);
} else {
network_task_runner_->PostTask(
FROM_HERE,
base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
}
}
void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
const std::string& upload_content) {
AssertHasNoUploadData();
DCHECK(!is_chunked_upload_);
DCHECK(upload_content_type_.empty());
// Empty |upload_content_type| is allowed iff the |upload_content| is empty.
DCHECK(upload_content.empty() || !upload_content_type.empty());
upload_content_type_ = upload_content_type;
upload_content_ = upload_content;
upload_content_set_ = true;
}
void URLFetcherCore::SetUploadFilePath(
const std::string& upload_content_type,
const base::FilePath& file_path,
uint64_t range_offset,
uint64_t range_length,
scoped_refptr<base::TaskRunner> file_task_runner) {
AssertHasNoUploadData();
DCHECK(!is_chunked_upload_);
DCHECK_EQ(upload_range_offset_, 0ULL);
DCHECK_EQ(upload_range_length_, 0ULL);
DCHECK(upload_content_type_.empty());
DCHECK(!upload_content_type.empty());
upload_content_type_ = upload_content_type;
upload_file_path_ = file_path;
upload_range_offset_ = range_offset;
upload_range_length_ = range_length;
upload_file_task_runner_ = file_task_runner;
upload_content_set_ = true;
}
void URLFetcherCore::SetUploadStreamFactory(
const std::string& upload_content_type,
const URLFetcher::CreateUploadStreamCallback& factory) {
AssertHasNoUploadData();
DCHECK(!is_chunked_upload_);
DCHECK(upload_content_type_.empty());
upload_content_type_ = upload_content_type;
upload_stream_factory_ = factory;
upload_content_set_ = true;
}
void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
if (!is_chunked_upload_) {
AssertHasNoUploadData();
DCHECK(upload_content_type_.empty());
}
// Empty |content_type| is not allowed here, because it is impossible
// to ensure non-empty upload content as it is not yet supplied.
DCHECK(!content_type.empty());
upload_content_type_ = content_type;
upload_content_.clear();
is_chunked_upload_ = true;
}
void URLFetcherCore::AppendChunkToUpload(const std::string& content,
bool is_last_chunk) {
DCHECK(delegate_task_runner_.get());
DCHECK(network_task_runner_.get());
network_task_runner_->PostTask(
FROM_HERE,
base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
is_last_chunk));
}
void URLFetcherCore::SetLoadFlags(int load_flags) {
load_flags_ = load_flags;
}
int URLFetcherCore::GetLoadFlags() const {
return load_flags_;
}
void URLFetcherCore::SetReferrer(const std::string& referrer) {
referrer_ = referrer;
}
void URLFetcherCore::SetReferrerPolicy(
URLRequest::ReferrerPolicy referrer_policy) {
referrer_policy_ = referrer_policy;
}
void URLFetcherCore::SetExtraRequestHeaders(
const std::string& extra_request_headers) {
extra_request_headers_.Clear();
extra_request_headers_.AddHeadersFromString(extra_request_headers);
}
void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
extra_request_headers_.AddHeaderFromString(header_line);
}
void URLFetcherCore::SetRequestContext(
URLRequestContextGetter* request_context_getter) {
DCHECK(!request_context_getter_.get());
DCHECK(request_context_getter);
request_context_getter_ = request_context_getter;
}
void URLFetcherCore::SetFirstPartyForCookies(
const GURL& first_party_for_cookies) {
DCHECK(first_party_for_cookies_.is_empty());
first_party_for_cookies_ = first_party_for_cookies;
}
void URLFetcherCore::SetURLRequestUserData(
const void* key,
const URLFetcher::CreateDataCallback& create_data_callback) {
DCHECK(key);
DCHECK(!create_data_callback.is_null());
url_request_data_key_ = key;
url_request_create_data_callback_ = create_data_callback;
}
void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
stop_on_redirect_ = stop_on_redirect;
}
void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
automatically_retry_on_5xx_ = retry;
}
void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
max_retries_on_5xx_ = max_retries;
}
int URLFetcherCore::GetMaxRetriesOn5xx() const {
return max_retries_on_5xx_;
}
base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
return backoff_delay_;
}
void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
max_retries_on_network_changes_ = max_retries;
}
void URLFetcherCore::SaveResponseToFileAtPath(
const base::FilePath& file_path,
scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
new URLFetcherFileWriter(file_task_runner, file_path)));
}
void URLFetcherCore::SaveResponseToTemporaryFile(
scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
new URLFetcherFileWriter(file_task_runner, base::FilePath())));
}
void URLFetcherCore::SaveResponseWithWriter(
scoped_ptr<URLFetcherResponseWriter> response_writer) {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
response_writer_ = response_writer.Pass();
}
HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
return response_headers_.get();
}
// TODO(panayiotis): socket_address_ is written in the IO thread,
// if this is accessed in the UI thread, this could result in a race.
// Same for response_headers_ above and was_fetched_via_proxy_ below.
HostPortPair URLFetcherCore::GetSocketAddress() const {
return socket_address_;
}
bool URLFetcherCore::WasFetchedViaProxy() const {
return was_fetched_via_proxy_;
}
bool URLFetcherCore::WasCached() const {
return was_cached_;
}
int64_t URLFetcherCore::GetReceivedResponseContentLength() const {
return received_response_content_length_;
}
int64_t URLFetcherCore::GetTotalReceivedBytes() const {
return total_received_bytes_;
}
const GURL& URLFetcherCore::GetOriginalURL() const {
return original_url_;
}
const GURL& URLFetcherCore::GetURL() const {
return url_;
}
const URLRequestStatus& URLFetcherCore::GetStatus() const {
return status_;
}
int URLFetcherCore::GetResponseCode() const {
return response_code_;
}
const ResponseCookies& URLFetcherCore::GetCookies() const {
return cookies_;
}
void URLFetcherCore::ReceivedContentWasMalformed() {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
if (network_task_runner_.get()) {
network_task_runner_->PostTask(
FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
}
}
bool URLFetcherCore::GetResponseAsString(
std::string* out_response_string) const {
URLFetcherStringWriter* string_writer =
response_writer_ ? response_writer_->AsStringWriter() : NULL;
if (!string_writer)
return false;
*out_response_string = string_writer->data();
UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
(string_writer->data().length() / 1024));
return true;
}
bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
base::FilePath* out_response_path) {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
URLFetcherFileWriter* file_writer =
response_writer_ ? response_writer_->AsFileWriter() : NULL;
if (!file_writer)
return false;
*out_response_path = file_writer->file_path();
if (take_ownership) {
// Intentionally calling a file_writer_ method directly without posting
// the task to network_task_runner_.
//
// This is for correctly handling the case when file_writer_->DisownFile()
// is soon followed by URLFetcherCore::Stop(). We have to make sure that
// DisownFile takes effect before Stop deletes file_writer_.
//
// This direct call should be thread-safe, since DisownFile itself does no
// file operation. It just flips the state to be referred in destruction.
file_writer->DisownFile();
}
return true;
}
void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
const RedirectInfo& redirect_info,
bool* defer_redirect) {
DCHECK_EQ(request, request_.get());
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (stop_on_redirect_) {
stopped_on_redirect_ = true;
url_ = redirect_info.new_url;
response_code_ = request_->GetResponseCode();
was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
was_cached_ = request_->was_cached();
total_received_bytes_ += request_->GetTotalReceivedBytes();
request->Cancel();
OnReadCompleted(request, 0);
}
}
void URLFetcherCore::OnResponseStarted(URLRequest* request) {
DCHECK_EQ(request, request_.get());
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (request_->status().is_success()) {
response_code_ = request_->GetResponseCode();
response_headers_ = request_->response_headers();
socket_address_ = request_->GetSocketAddress();
was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
was_cached_ = request_->was_cached();
total_response_bytes_ = request_->GetExpectedContentSize();
}
ReadResponse();
}
void URLFetcherCore::OnCertificateRequested(
URLRequest* request,
SSLCertRequestInfo* cert_request_info) {
DCHECK_EQ(request, request_.get());
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (g_ignore_certificate_requests) {
request->ContinueWithCertificate(nullptr, nullptr);
} else {
request->Cancel();
}
}
void URLFetcherCore::OnReadCompleted(URLRequest* request,
int bytes_read) {
DCHECK_EQ(request, request_.get());
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (!stopped_on_redirect_)
url_ = request->url();
URLRequestThrottlerManager* throttler_manager =
request->context()->throttler_manager();
if (throttler_manager)
url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
do {
if (!request_->status().is_success() || bytes_read <= 0)
break;
current_response_bytes_ += bytes_read;
InformDelegateDownloadProgress();
const int result =
WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
if (result < 0) {
// Write failed or waiting for write completion.
return;
}
} while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
const URLRequestStatus status = request_->status();
if (status.is_success())
request_->GetResponseCookies(&cookies_);
// See comments re: HEAD requests in ReadResponse().
if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
status_ = status;
received_response_content_length_ =
request_->received_response_content_length();
total_received_bytes_ += request_->GetTotalReceivedBytes();
ReleaseRequest();
// No more data to write.
const int result = response_writer_->Finish(
base::Bind(&URLFetcherCore::DidFinishWriting, this));
if (result != ERR_IO_PENDING)
DidFinishWriting(result);
}
}
void URLFetcherCore::OnContextShuttingDown() {
DCHECK(request_);
CancelRequestAndInformDelegate(ERR_CONTEXT_SHUT_DOWN);
}
void URLFetcherCore::CancelAll() {
g_registry.Get().CancelAll();
}
int URLFetcherCore::GetNumFetcherCores() {
return g_registry.Get().size();
}
void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
g_ignore_certificate_requests = ignored;
}
URLFetcherCore::~URLFetcherCore() {
// |request_| should be NULL. If not, it's unsafe to delete it here since we
// may not be on the IO thread.
DCHECK(!request_.get());
}
void URLFetcherCore::StartOnIOThread() {
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (!response_writer_)
response_writer_.reset(new URLFetcherStringWriter);
const int result = response_writer_->Initialize(
base::Bind(&URLFetcherCore::DidInitializeWriter, this));
if (result != ERR_IO_PENDING)
DidInitializeWriter(result);
}
void URLFetcherCore::StartURLRequest() {
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (was_cancelled_) {
// Since StartURLRequest() is posted as a *delayed* task, it may
// run after the URLFetcher was already stopped.
return;
}
if (!request_context_getter_->GetURLRequestContext()) {
CancelRequestAndInformDelegate(ERR_CONTEXT_SHUT_DOWN);
return;
}
DCHECK(request_context_getter_.get());
DCHECK(!request_.get());
g_registry.Get().AddURLFetcherCore(this);
current_response_bytes_ = 0;
request_context_getter_->AddObserver(this);
request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
original_url_, DEFAULT_PRIORITY, this);
request_->set_stack_trace(stack_trace_);
int flags = request_->load_flags() | load_flags_;
if (is_chunked_upload_)
request_->EnableChunkedUpload();
request_->SetLoadFlags(flags);
request_->SetReferrer(referrer_);
request_->set_referrer_policy(referrer_policy_);
request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
original_url_ : first_party_for_cookies_);
if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
request_->SetUserData(url_request_data_key_,
url_request_create_data_callback_.Run());
}
switch (request_type_) {
case URLFetcher::GET:
break;
case URLFetcher::POST:
case URLFetcher::PUT:
case URLFetcher::PATCH: {
// Upload content must be set.
DCHECK(is_chunked_upload_ || upload_content_set_);
request_->set_method(
request_type_ == URLFetcher::POST ? "POST" :
request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
if (!upload_content_type_.empty()) {
extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
upload_content_type_);
}
if (!upload_content_.empty()) {
scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
upload_content_.data(), upload_content_.size()));
request_->set_upload(
ElementsUploadDataStream::CreateWithReader(reader.Pass(), 0));
} else if (!upload_file_path_.empty()) {
scoped_ptr<UploadElementReader> reader(
new UploadFileElementReader(upload_file_task_runner_.get(),
upload_file_path_,
upload_range_offset_,
upload_range_length_,
base::Time()));
request_->set_upload(
ElementsUploadDataStream::CreateWithReader(reader.Pass(), 0));
} else if (!upload_stream_factory_.is_null()) {
scoped_ptr<UploadDataStream> stream = upload_stream_factory_.Run();
DCHECK(stream);
request_->set_upload(stream.Pass());
}
current_upload_bytes_ = -1;
// TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
// layer and avoid using timer here.
upload_progress_checker_timer_.reset(new base::RepeatingTimer());
upload_progress_checker_timer_->Start(
FROM_HERE,
base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
this,
&URLFetcherCore::InformDelegateUploadProgress);
break;
}
case URLFetcher::HEAD:
request_->set_method("HEAD");
break;
case URLFetcher::DELETE_REQUEST:
request_->set_method("DELETE");
break;
default:
NOTREACHED();
}
if (!extra_request_headers_.IsEmpty())
request_->SetExtraRequestHeaders(extra_request_headers_);
request_->Start();
}
void URLFetcherCore::DidInitializeWriter(int result) {
if (result != OK) {
CancelRequestAndInformDelegate(result);
return;
}
StartURLRequestWhenAppropriate();
}
void URLFetcherCore::StartURLRequestWhenAppropriate() {
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (was_cancelled_)
return;
DCHECK(request_context_getter_.get());
// Check if the request should be delayed, and if so, post a task to start it
// after the delay has expired. Otherwise, start it now.
URLRequestContext* context = request_context_getter_->GetURLRequestContext();
// If the context has been shut down, or there's no ThrottlerManager, just
// start the request. In the former case, StartURLRequest() will just inform
// the URLFetcher::Delegate the request has been canceled.
if (context && context->throttler_manager()) {
if (!original_url_throttler_entry_.get()) {
original_url_throttler_entry_ =
context->throttler_manager()->RegisterRequestUrl(original_url_);
}
if (original_url_throttler_entry_.get()) {
int64_t delay =
original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
GetBackoffReleaseTime());
if (delay != 0) {
base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
base::TimeDelta::FromMilliseconds(delay));
return;
}
}
}
StartURLRequest();
}
void URLFetcherCore::CancelURLRequest(int error) {
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (request_.get()) {
request_->CancelWithError(error);
ReleaseRequest();
}
// Set the error manually.
// Normally, calling URLRequest::CancelWithError() results in calling
// OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
// URLRequestJob::NotifyDone(). But, because the request was released
// immediately after being canceled, the request could not call
// OnReadCompleted() which overwrites |status_| with the error status.
status_ = URLRequestStatus(URLRequestStatus::CANCELED, error);
// Release the reference to the request context. There could be multiple
// references to URLFetcher::Core at this point so it may take a while to
// delete the object, but we cannot delay the destruction of the request
// context.
request_context_getter_ = NULL;
first_party_for_cookies_ = GURL();
url_request_data_key_ = NULL;
url_request_create_data_callback_.Reset();
was_cancelled_ = true;
}
void URLFetcherCore::OnCompletedURLRequest(
base::TimeDelta backoff_delay) {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
// Save the status and backoff_delay so that delegates can read it.
if (delegate_) {
backoff_delay_ = backoff_delay;
InformDelegateFetchIsComplete();
}
}
void URLFetcherCore::InformDelegateFetchIsComplete() {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
if (delegate_)
delegate_->OnURLFetchComplete(fetcher_);
}
void URLFetcherCore::NotifyMalformedContent() {
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (url_throttler_entry_.get()) {
int status_code = response_code_;
if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
// The status code will generally be known by the time clients
// call the |ReceivedContentWasMalformed()| function (which ends up
// calling the current function) but if it's not, we need to assume
// the response was successful so that the total failure count
// used to calculate exponential back-off goes up.
status_code = 200;
}
url_throttler_entry_->ReceivedContentWasMalformed(status_code);
}
}
void URLFetcherCore::DidFinishWriting(int result) {
if (result != OK) {
CancelRequestAndInformDelegate(result);
return;
}
// If the file was successfully closed, then the URL request is complete.
RetryOrCompleteUrlFetch();
}
void URLFetcherCore::RetryOrCompleteUrlFetch() {
DCHECK(network_task_runner_->BelongsToCurrentThread());
base::TimeDelta backoff_delay;
// Checks the response from server.
if (response_code_ >= 500 ||
status_.error() == ERR_TEMPORARILY_THROTTLED) {
// When encountering a server error, we will send the request again
// after backoff time.
++num_retries_on_5xx_;
// Note that backoff_delay may be 0 because (a) the
// URLRequestThrottlerManager and related code does not
// necessarily back off on the first error, (b) it only backs off
// on some of the 5xx status codes, (c) not all URLRequestContexts
// have a throttler manager.
base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
backoff_delay = backoff_release_time - base::TimeTicks::Now();
if (backoff_delay < base::TimeDelta())
backoff_delay = base::TimeDelta();
if (automatically_retry_on_5xx_ &&
num_retries_on_5xx_ <= max_retries_on_5xx_) {
StartOnIOThread();
return;
}
} else {
backoff_delay = base::TimeDelta();
}
// Retry if the request failed due to network changes.
if (status_.error() == ERR_NETWORK_CHANGED &&
num_retries_on_network_changes_ < max_retries_on_network_changes_) {
++num_retries_on_network_changes_;
// Retry soon, after flushing all the current tasks which may include
// further network change observers.
network_task_runner_->PostTask(
FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
return;
}
request_context_getter_ = NULL;
first_party_for_cookies_ = GURL();
url_request_data_key_ = NULL;
url_request_create_data_callback_.Reset();
bool posted = delegate_task_runner_->PostTask(
FROM_HERE,
base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
// If the delegate message loop does not exist any more, then the delegate
// should be gone too.
DCHECK(posted || !delegate_);
}
void URLFetcherCore::CancelRequestAndInformDelegate(int result) {
CancelURLRequest(result);
delegate_task_runner_->PostTask(
FROM_HERE,
base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
}
void URLFetcherCore::ReleaseRequest() {
request_context_getter_->RemoveObserver(this);
upload_progress_checker_timer_.reset();
request_.reset();
g_registry.Get().RemoveURLFetcherCore(this);
}
base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (!original_url_throttler_entry_.get())
return base::TimeTicks();
base::TimeTicks original_url_backoff =
original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
base::TimeTicks destination_url_backoff;
if (url_throttler_entry_.get() &&
original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
destination_url_backoff =
url_throttler_entry_->GetExponentialBackoffReleaseTime();
}
return original_url_backoff > destination_url_backoff ?
original_url_backoff : destination_url_backoff;
}
void URLFetcherCore::CompleteAddingUploadDataChunk(
const std::string& content, bool is_last_chunk) {
if (was_cancelled_) {
// Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
// may run after the URLFetcher was already stopped.
return;
}
DCHECK(is_chunked_upload_);
DCHECK(request_.get());
DCHECK(!content.empty());
request_->AppendChunkToUpload(content.data(),
static_cast<int>(content.length()),
is_last_chunk);
}
int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
while (data->BytesRemaining() > 0) {
const int result = response_writer_->Write(
data.get(),
data->BytesRemaining(),
base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
if (result < 0) {
if (result != ERR_IO_PENDING)
DidWriteBuffer(data, result);
return result;
}
data->DidConsume(result);
}
return OK;
}
void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
int result) {
if (result < 0) { // Handle errors.
response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
CancelRequestAndInformDelegate(result);
return;
}
// Continue writing.
data->DidConsume(result);
if (WriteBuffer(data) < 0)
return;
// Finished writing buffer_. Read some more, unless the request has been
// cancelled and deleted.
DCHECK_EQ(0, data->BytesRemaining());
if (request_.get())
ReadResponse();
}
void URLFetcherCore::ReadResponse() {
// Some servers may treat HEAD requests as GET requests. To free up the
// network connection as soon as possible, signal that the request has
// completed immediately, without trying to read any data back (all we care
// about is the response code and headers, which we already have).
int bytes_read = 0;
if (request_->status().is_success() &&
(request_type_ != URLFetcher::HEAD)) {
if (!request_->Read(buffer_.get(), kBufferSize, &bytes_read))
bytes_read = -1; // Match OnReadCompleted() interface contract.
}
OnReadCompleted(request_.get(), bytes_read);
}
void URLFetcherCore::InformDelegateUploadProgress() {
DCHECK(network_task_runner_->BelongsToCurrentThread());
if (request_.get()) {
int64_t current = request_->GetUploadProgress().position();
if (current_upload_bytes_ != current) {
current_upload_bytes_ = current;
int64_t total = -1;
if (!is_chunked_upload_) {
total = static_cast<int64_t>(request_->GetUploadProgress().size());
// Total may be zero if the UploadDataStream::Init has not been called
// yet. Don't send the upload progress until the size is initialized.
if (!total)
return;
}
delegate_task_runner_->PostTask(
FROM_HERE,
base::Bind(
&URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
this, current, total));
}
}
}
void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
int64_t current,
int64_t total) {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
if (delegate_)
delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
}
void URLFetcherCore::InformDelegateDownloadProgress() {
DCHECK(network_task_runner_->BelongsToCurrentThread());
// TODO(pkasting): Remove ScopedTracker below once crbug.com/455952 is fixed.
tracked_objects::ScopedTracker tracking_profile2(
FROM_HERE_WITH_EXPLICIT_FUNCTION(
"455952 delegate_task_runner_->PostTask()"));
delegate_task_runner_->PostTask(
FROM_HERE,
base::Bind(
&URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
this, current_response_bytes_, total_response_bytes_));
}
void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
int64_t current,
int64_t total) {
DCHECK(delegate_task_runner_->BelongsToCurrentThread());
if (delegate_)
delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
}
void URLFetcherCore::AssertHasNoUploadData() const {
DCHECK(!upload_content_set_);
DCHECK(upload_content_.empty());
DCHECK(upload_file_path_.empty());
DCHECK(upload_stream_factory_.is_null());
}
} // namespace net
|