Create blobs from Disk Cache entries.

This CL allows blobs to be backed by disk cache entries, and also
migrates the Cache Storage Cache implementation to use this
functionality. This is very useful in the ServiceWorker case, as the
existing implementation was copying entire disk cache entries to RAM
before serving them to consumers.

BUG=None

Review URL: https://codereview.chromium.org/1108083002

Cr-Commit-Position: refs/heads/master@{#335185}

11 files changed, 183 insertions, 39 deletions

diff --git a/storage/browser/blob/blob_data_builder.cc b/storage/browser/blob/blob_data_builder.cc
index 7c305a2..94fb9e5 100644
--- a/storage/browser/blob/blob_data_builder.cc
+++ b/storage/browser/blob/blob_data_builder.cc
@@ -5,6 +5,7 @@
 #include "storage/browser/blob/blob_data_builder.h"
 
 #include "base/time/time.h"
+#include "net/disk_cache/disk_cache.h"
 #include "storage/browser/blob/shareable_file_reference.h"
 
 namespace storage {
@@ -60,4 +61,16 @@ void BlobDataBuilder::AppendFileSystemFile(
   items_.push_back(new BlobDataItem(element.Pass()));
 }
 
+void BlobDataBuilder::AppendDiskCacheEntry(
+    const scoped_refptr<DataHandle>& data_handle,
+    disk_cache::Entry* disk_cache_entry,
+    int disk_cache_stream_index) {
+  scoped_ptr<DataElement> element(new DataElement());
+  element->SetToDiskCacheEntryRange(
+      0U, disk_cache_entry->GetDataSize(disk_cache_stream_index));
+  items_.push_back(
+      new BlobDataItem(element.Pass(), data_handle, disk_cache_entry,
+                       disk_cache_stream_index));
+}
+
 }  // namespace storage
diff --git a/storage/browser/blob/blob_data_builder.h b/storage/browser/blob/blob_data_builder.h
index 8119538..115d1f4 100644
--- a/storage/browser/blob/blob_data_builder.h
+++ b/storage/browser/blob/blob_data_builder.h
@@ -16,11 +16,17 @@
 #include "storage/browser/blob/blob_data_snapshot.h"
 #include "storage/browser/storage_browser_export.h"
 
+namespace disk_cache {
+class Entry;
+}
+
 namespace storage {
 class BlobStorageContext;
 
 class STORAGE_EXPORT BlobDataBuilder {
  public:
+  using DataHandle = BlobDataItem::DataHandle;
+
   explicit BlobDataBuilder(const std::string& uuid);
   ~BlobDataBuilder();
 
@@ -49,6 +55,10 @@ class STORAGE_EXPORT BlobDataBuilder {
                             uint64_t length,
                             const base::Time& expected_modification_time);
 
+  void AppendDiskCacheEntry(const scoped_refptr<DataHandle>& data_handle,
+                            disk_cache::Entry* disk_cache_entry,
+                            int disk_cache_stream_index);
+
   void set_content_type(const std::string& content_type) {
     content_type_ = content_type;
   }
diff --git a/storage/browser/blob/blob_data_item.cc b/storage/browser/blob/blob_data_item.cc
index 504a1c3..c1ec3b3 100644
--- a/storage/browser/blob/blob_data_item.cc
+++ b/storage/browser/blob/blob_data_item.cc
@@ -6,12 +6,33 @@
 
 namespace storage {
 
+BlobDataItem::DataHandle::~DataHandle() {
+}
+
+BlobDataItem::BlobDataItem(scoped_ptr<DataElement> item)
+    : item_(item.Pass()),
+      disk_cache_entry_(nullptr),
+      disk_cache_stream_index_(-1) {
+}
+
 BlobDataItem::BlobDataItem(scoped_ptr<DataElement> item,
-                           scoped_refptr<ShareableFileReference> file_handle)
-    : item_(item.Pass()), file_handle_(file_handle) {
+                           const scoped_refptr<DataHandle>& data_handle)
+    : item_(item.Pass()),
+      data_handle_(data_handle),
+      disk_cache_entry_(nullptr),
+      disk_cache_stream_index_(-1) {
 }
-BlobDataItem::BlobDataItem(scoped_ptr<DataElement> item) : item_(item.Pass()) {
+
+BlobDataItem::BlobDataItem(scoped_ptr<DataElement> item,
+                           const scoped_refptr<DataHandle>& data_handle,
+                           disk_cache::Entry* entry,
+                           int disk_cache_stream_index)
+    : item_(item.Pass()),
+      data_handle_(data_handle),
+      disk_cache_entry_(entry),
+      disk_cache_stream_index_(disk_cache_stream_index) {
 }
+
 BlobDataItem::~BlobDataItem() {
 }
 
diff --git a/storage/browser/blob/blob_data_item.h b/storage/browser/blob/blob_data_item.h
index 1953a1a..5b25db4 100644
--- a/storage/browser/blob/blob_data_item.h
+++ b/storage/browser/blob/blob_data_item.h
@@ -7,21 +7,37 @@
 
 #include "base/basictypes.h"
 #include "base/memory/ref_counted.h"
-#include "storage/browser/blob/shareable_file_reference.h"
 #include "storage/browser/storage_browser_export.h"
 #include "storage/common/data_element.h"
 
+namespace disk_cache {
+class Entry;
+}
+
 namespace storage {
 class BlobDataBuilder;
 class BlobStorageContext;
 
-// Ref counted blob item.  This class owns the backing data of the blob item.
-// The backing data is immutable, and cannot change after creation.
-// The purpose of this class is to allow the resource to stick around in the
-// snapshot even after the resource was swapped in the blob (either to disk or
-// to memory) by the BlobStorageContext.
+// Ref counted blob item. This class owns the backing data of the blob item. The
+// backing data is immutable, and cannot change after creation. The purpose of
+// this class is to allow the resource to stick around in the snapshot even
+// after the resource was swapped in the blob (either to disk or to memory) by
+// the BlobStorageContext.
 class STORAGE_EXPORT BlobDataItem : public base::RefCounted<BlobDataItem> {
  public:
+  // The DataHandle class is used to persist resources that are needed for
+  // reading this BlobDataItem. This object will stay around while any reads are
+  // pending. If all blobs with this item are deleted or the item is swapped for
+  // a different backend version (mem-to-disk or the reverse), then the item
+  // will be destructed after all pending reads are complete.
+  class STORAGE_EXPORT DataHandle : public base::RefCounted<DataHandle> {
+   protected:
+    virtual ~DataHandle() = 0;
+
+   private:
+    friend class base::RefCounted<DataHandle>;
+  };
+
   DataElement::Type type() const { return item_->type(); }
   const char* bytes() const { return item_->bytes(); }
   const base::FilePath& path() const { return item_->path(); }
@@ -35,6 +51,9 @@ class STORAGE_EXPORT BlobDataItem : public base::RefCounted<BlobDataItem> {
   const DataElement& data_element() const { return *item_; }
   const DataElement* data_element_ptr() const { return item_.get(); }
 
+  disk_cache::Entry* disk_cache_entry() const { return disk_cache_entry_; }
+  int disk_cache_stream_index() const { return disk_cache_stream_index_; }
+
  private:
   friend class BlobDataBuilder;
   friend class BlobStorageContext;
@@ -42,16 +61,27 @@ class STORAGE_EXPORT BlobDataItem : public base::RefCounted<BlobDataItem> {
 
   BlobDataItem(scoped_ptr<DataElement> item);
   BlobDataItem(scoped_ptr<DataElement> item,
-               scoped_refptr<ShareableFileReference> file_handle);
+               const scoped_refptr<DataHandle>& data_handle);
+  BlobDataItem(scoped_ptr<DataElement> item,
+               const scoped_refptr<DataHandle>& data_handle,
+               disk_cache::Entry* entry,
+               int disk_cache_stream_index_);
   virtual ~BlobDataItem();
 
   scoped_ptr<DataElement> item_;
-  scoped_refptr<ShareableFileReference> file_handle_;
+  scoped_refptr<DataHandle> data_handle_;
+
+  // This naked pointer is safe because the scope is protected by the DataHandle
+  // instance for disk cache entries during the lifetime of this BlobDataItem.
+  disk_cache::Entry* disk_cache_entry_;
+  int disk_cache_stream_index_;  // For TYPE_DISK_CACHE_ENTRY.
 };
 
 #if defined(UNIT_TEST)
 inline bool operator==(const BlobDataItem& a, const BlobDataItem& b) {
-  return a.data_element() == b.data_element();
+  return a.disk_cache_entry() == b.disk_cache_entry() &&
+         a.disk_cache_stream_index() == b.disk_cache_stream_index() &&
+         a.data_element() == b.data_element();
 }
 
 inline bool operator!=(const BlobDataItem& a, const BlobDataItem& b) {
diff --git a/storage/browser/blob/blob_storage_context.cc b/storage/browser/blob/blob_storage_context.cc
index 66f67ea..d08ccb3 100644
--- a/storage/browser/blob/blob_storage_context.cc
+++ b/storage/browser/blob/blob_storage_context.cc
@@ -16,6 +16,7 @@
 #include "base/thread_task_runner_handle.h"
 #include "base/trace_event/trace_event.h"
 #include "storage/browser/blob/blob_data_builder.h"
+#include "storage/browser/blob/shareable_file_reference.h"
 #include "url/gurl.h"
 
 namespace storage {
@@ -283,6 +284,9 @@ scoped_refptr<BlobDataItem> BlobStorageContext::AllocateBlobItem(
                               ipc_data.length());
       blob_item = new BlobDataItem(element.Pass());
       break;
+    case DataElement::TYPE_DISK_CACHE_ENTRY:  // This type can't be sent by IPC.
+      NOTREACHED();
+      break;
     default:
       NOTREACHED();
       break;
@@ -365,6 +369,13 @@ bool BlobStorageContext::AppendAllocatedBlobItem(
       }
       break;
     }
+    case DataElement::TYPE_DISK_CACHE_ENTRY: {
+      UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.CacheEntry",
+                           (length - offset) / 1024);
+      target_blob_builder->AppendSharedBlobItem(
+          new ShareableBlobDataItem(target_blob_uuid, blob_item));
+      break;
+    }
     default:
       NOTREACHED();
       break;
@@ -440,7 +451,7 @@ bool BlobStorageContext::AppendBlob(
                                     item.expected_modification_time());
         target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
             target_blob_uuid,
-            new BlobDataItem(element.Pass(), item.file_handle_)));
+            new BlobDataItem(element.Pass(), item.data_handle_)));
       } break;
       case DataElement::TYPE_FILE_FILESYSTEM: {
         UMA_HISTOGRAM_COUNTS("Storage.BlobItemSize.BlobSlice.FileSystem",
@@ -452,6 +463,16 @@ bool BlobStorageContext::AppendBlob(
         target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
             target_blob_uuid, new BlobDataItem(element.Pass())));
       } break;
+      case DataElement::TYPE_DISK_CACHE_ENTRY: {
+        scoped_ptr<DataElement> element(new DataElement());
+        element->SetToDiskCacheEntryRange(item.offset() + offset,
+                                          new_length);
+        target_blob_builder->AppendSharedBlobItem(new ShareableBlobDataItem(
+            target_blob_uuid,
+            new BlobDataItem(element.Pass(), item.data_handle_,
+                             item.disk_cache_entry(),
+                             item.disk_cache_stream_index())));
+      } break;
       default:
         CHECK(false) << "Illegal blob item type: " << item.type();
     }
diff --git a/storage/browser/blob/blob_url_request_job.cc b/storage/browser/blob/blob_url_request_job.cc
index 26607cd..f429020 100644
--- a/storage/browser/blob/blob_url_request_job.cc
+++ b/storage/browser/blob/blob_url_request_job.cc
@@ -22,6 +22,7 @@
 #include "base/trace_event/trace_event.h"
 #include "net/base/io_buffer.h"
 #include "net/base/net_errors.h"
+#include "net/disk_cache/disk_cache.h"
 #include "net/http/http_request_headers.h"
 #include "net/http/http_response_headers.h"
 #include "net/http/http_response_info.h"
@@ -356,6 +357,8 @@ bool BlobURLRequestJob::ReadItem() {
   const BlobDataItem& item = *items.at(current_item_index_);
   if (item.type() == DataElement::TYPE_BYTES)
     return ReadBytesItem(item, bytes_to_read);
+  if (item.type() == DataElement::TYPE_DISK_CACHE_ENTRY)
+    return ReadDiskCacheEntryItem(item, bytes_to_read);
   if (!IsFileType(item.type())) {
     NOTREACHED();
     return false;
@@ -412,6 +415,8 @@ bool BlobURLRequestJob::ReadBytesItem(const BlobDataItem& item,
 
 bool BlobURLRequestJob::ReadFileItem(FileStreamReader* reader,
                                      int bytes_to_read) {
+  DCHECK(!GetStatus().is_io_pending())
+      << "Can't begin IO while another IO operation is pending.";
   DCHECK_GE(read_buf_->BytesRemaining(), bytes_to_read);
   DCHECK(reader);
   int chunk_number = current_file_chunk_number_++;
@@ -420,13 +425,9 @@ bool BlobURLRequestJob::ReadFileItem(FileStreamReader* reader,
   const int result =
       reader->Read(read_buf_.get(), bytes_to_read,
                    base::Bind(&BlobURLRequestJob::DidReadFile,
-                              base::Unretained(this), chunk_number));
+                              weak_factory_.GetWeakPtr(), chunk_number));
   if (result >= 0) {
-    // Data is immediately available.
-    if (GetStatus().is_io_pending())
-      DidReadFile(chunk_number, result);
-    else
-      AdvanceBytesRead(result);
+    AdvanceBytesRead(result);
     return true;
   }
   if (result == net::ERR_IO_PENDING)
@@ -437,23 +438,18 @@ bool BlobURLRequestJob::ReadFileItem(FileStreamReader* reader,
 }
 
 void BlobURLRequestJob::DidReadFile(int chunk_number, int result) {
+  DCHECK(GetStatus().is_io_pending())
+      << "Asynchronous IO completed while IO wasn't pending?";
   TRACE_EVENT_ASYNC_END1("Blob", "BlobRequest::ReadFileItem", this, "uuid",
                          blob_data_->uuid());
   if (result <= 0) {
-    NotifyFailure(net::ERR_FAILED);
+    NotifyFailure(result);
     return;
   }
   SetStatus(net::URLRequestStatus());  // Clear the IO_PENDING status
 
   AdvanceBytesRead(result);
 
-  // If the read buffer is completely filled, we're done.
-  if (!read_buf_->BytesRemaining()) {
-    int bytes_read = BytesReadCompleted();
-    NotifyReadComplete(bytes_read);
-    return;
-  }
-
   // Otherwise, continue the reading.
   int bytes_read = 0;
   if (ReadLoop(&bytes_read))
@@ -468,6 +464,43 @@ void BlobURLRequestJob::DeleteCurrentFileReader() {
   }
 }
 
+bool BlobURLRequestJob::ReadDiskCacheEntryItem(const BlobDataItem& item,
+                                               int bytes_to_read) {
+  DCHECK(!GetStatus().is_io_pending())
+      << "Can't begin IO while another IO operation is pending.";
+  DCHECK_GE(read_buf_->BytesRemaining(), bytes_to_read);
+
+  const int result = item.disk_cache_entry()->ReadData(
+      item.disk_cache_stream_index(), current_item_offset_, read_buf_.get(),
+      bytes_to_read, base::Bind(&BlobURLRequestJob::DidReadDiskCacheEntry,
+                                weak_factory_.GetWeakPtr()));
+  if (result >= 0) {
+    AdvanceBytesRead(result);
+    return true;
+  }
+  if (result == net::ERR_IO_PENDING)
+    SetStatus(net::URLRequestStatus(net::URLRequestStatus::IO_PENDING, 0));
+  else
+    NotifyFailure(result);
+  return false;
+}
+
+void BlobURLRequestJob::DidReadDiskCacheEntry(int result) {
+  DCHECK(GetStatus().is_io_pending())
+      << "Asynchronous IO completed while IO wasn't pending?";
+  if (result <= 0) {
+    NotifyFailure(result);
+    return;
+  }
+  SetStatus(net::URLRequestStatus());
+
+  AdvanceBytesRead(result);
+
+  int bytes_read = 0;
+  if (ReadLoop(&bytes_read))
+    NotifyReadComplete(bytes_read);
+}
+
 int BlobURLRequestJob::BytesReadCompleted() {
   int bytes_read = read_buf_->BytesConsumed();
   read_buf_ = NULL;
diff --git a/storage/browser/blob/blob_url_request_job.h b/storage/browser/blob/blob_url_request_job.h
index a4e10ed..74d07ad 100644
--- a/storage/browser/blob/blob_url_request_job.h
+++ b/storage/browser/blob/blob_url_request_job.h
@@ -6,6 +6,7 @@
 #define STORAGE_BROWSER_BLOB_BLOB_URL_REQUEST_JOB_H_
 
 #include <map>
+#include <vector>
 
 #include "base/memory/scoped_ptr.h"
 #include "base/memory/weak_ptr.h"
@@ -19,10 +20,6 @@ namespace base {
 class SingleThreadTaskRunner;
 }
 
-namespace storage {
-class FileSystemContext;
-}
-
 namespace net {
 class DrainableIOBuffer;
 class IOBuffer;
@@ -31,6 +28,7 @@ class IOBuffer;
 namespace storage {
 
 class FileStreamReader;
+class FileSystemContext;
 
 // A request job that handles reading blob URLs.
 class STORAGE_EXPORT BlobURLRequestJob
@@ -71,11 +69,14 @@ class STORAGE_EXPORT BlobURLRequestJob
   void AdvanceItem();
   void AdvanceBytesRead(int result);
   bool ReadBytesItem(const BlobDataItem& item, int bytes_to_read);
-  bool ReadFileItem(FileStreamReader* reader, int bytes_to_read);
 
+  bool ReadFileItem(FileStreamReader* reader, int bytes_to_read);
   void DidReadFile(int chunk_number, int result);
   void DeleteCurrentFileReader();
 
+  bool ReadDiskCacheEntryItem(const BlobDataItem& item, int bytes_to_read);
+  void DidReadDiskCacheEntry(int result);
+
   int ComputeBytesToRead() const;
   int BytesReadCompleted();
 
diff --git a/storage/browser/blob/shareable_file_reference.h b/storage/browser/blob/shareable_file_reference.h
index e2739dc..10f986f 100644
--- a/storage/browser/blob/shareable_file_reference.h
+++ b/storage/browser/blob/shareable_file_reference.h
@@ -5,8 +5,7 @@
 #ifndef STORAGE_BROWSER_BLOB_SHAREABLE_FILE_REFERENCE_H_
 #define STORAGE_BROWSER_BLOB_SHAREABLE_FILE_REFERENCE_H_
 
-#include <vector>
-
+#include "storage/browser/blob/blob_data_item.h"
 #include "storage/browser/blob/scoped_file.h"
 #include "storage/browser/storage_browser_export.h"
 
@@ -16,8 +15,7 @@ namespace storage {
 // same path if it already exists in its internal map.
 // This class is non-thread-safe and all methods must be called on a single
 // thread.
-class STORAGE_EXPORT ShareableFileReference
-    : public base::RefCounted<ShareableFileReference> {
+class STORAGE_EXPORT ShareableFileReference : public BlobDataItem::DataHandle {
  public:
   typedef ScopedFile::ScopeOutCallback FinalReleaseCallback;
 
@@ -60,10 +58,8 @@ class STORAGE_EXPORT ShareableFileReference
   void AddFinalReleaseCallback(const FinalReleaseCallback& callback);
 
  private:
-  friend class base::RefCounted<ShareableFileReference>;
-
   ShareableFileReference(ScopedFile scoped_file);
-  ~ShareableFileReference();
+  ~ShareableFileReference() override;
 
   ScopedFile scoped_file_;
 
diff --git a/storage/browser/blob/view_blob_internals_job.cc b/storage/browser/blob/view_blob_internals_job.cc
index ab1f310..1b5e91e 100644
--- a/storage/browser/blob/view_blob_internals_job.cc
+++ b/storage/browser/blob/view_blob_internals_job.cc
@@ -17,6 +17,7 @@
 #include "base/strings/utf_string_conversions.h"
 #include "net/base/escape.h"
 #include "net/base/net_errors.h"
+#include "net/disk_cache/disk_cache.h"
 #include "net/url_request/url_request.h"
 #include "storage/browser/blob/blob_storage_context.h"
 #include "storage/browser/blob/internal_blob_data.h"
@@ -218,6 +219,10 @@ void ViewBlobInternalsJob::GenerateHTMLForBlobData(
               out);
         }
         break;
+      case DataElement::TYPE_DISK_CACHE_ENTRY:
+        AddHTMLListItem(kType, "disk cache entry", out);
+        AddHTMLListItem(kURL, item.disk_cache_entry()->GetKey(), out);
+        break;
       case DataElement::TYPE_UNKNOWN:
         NOTREACHED();
         break;
diff --git a/storage/common/data_element.cc b/storage/common/data_element.cc
index c6d52f4..f26058b 100644
--- a/storage/common/data_element.cc
+++ b/storage/common/data_element.cc
@@ -46,4 +46,10 @@ void DataElement::SetToFileSystemUrlRange(
   expected_modification_time_ = expected_modification_time;
 }
 
+void DataElement::SetToDiskCacheEntryRange(uint64 offset, uint64 length) {
+  type_ = TYPE_DISK_CACHE_ENTRY;
+  offset_ = offset;
+  length_ = length;
+}
+
 }  // namespace storage
diff --git a/storage/common/data_element.h b/storage/common/data_element.h
index 0efe155..2eeca8e 100644
--- a/storage/common/data_element.h
+++ b/storage/common/data_element.h
@@ -27,6 +27,7 @@ class STORAGE_COMMON_EXPORT DataElement {
     TYPE_FILE,
     TYPE_BLOB,
     TYPE_FILE_FILESYSTEM,
+    TYPE_DISK_CACHE_ENTRY,
   };
 
   DataElement();
@@ -102,6 +103,9 @@ class STORAGE_COMMON_EXPORT DataElement {
                                uint64 offset, uint64 length,
                                const base::Time& expected_modification_time);
 
+  // Sets to TYPE_DISK_CACHE_ENTRY with range.
+  void SetToDiskCacheEntryRange(uint64 offset, uint64 length);
+
  private:
   Type type_;
   std::vector<char> buf_;  // For TYPE_BYTES.
@@ -130,6 +134,10 @@ inline bool operator==(const DataElement& a, const DataElement& b) {
       return a.blob_uuid() == b.blob_uuid();
     case DataElement::TYPE_FILE_FILESYSTEM:
       return a.filesystem_url() == b.filesystem_url();
+    case DataElement::TYPE_DISK_CACHE_ENTRY:
+      // We compare only length and offset; we trust the entry itself was
+      // compared at some higher level such as in BlobDataItem.
+      return true;
     case DataElement::TYPE_UNKNOWN:
       NOTREACHED();
       return false;