diff options
author | felipeg@chromium.org <felipeg@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2013-04-16 11:56:51 +0000 |
---|---|---|
committer | felipeg@chromium.org <felipeg@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2013-04-16 11:56:51 +0000 |
commit | c320155a6bb51d631f07a77d54cdb14b6f3276bb (patch) | |
tree | 8ae53498ae861bc2a443cce6ae8b0b10d594bb97 /net | |
parent | 165dad97b056d560f3c98d01dbe2d3ebe643ca2a (diff) | |
download | chromium_src-c320155a6bb51d631f07a77d54cdb14b6f3276bb.zip chromium_src-c320155a6bb51d631f07a77d54cdb14b6f3276bb.tar.gz chromium_src-c320155a6bb51d631f07a77d54cdb14b6f3276bb.tar.bz2 |
Refactor our SimpleIndex file format and serialization to use Pickle instead of the previously buggy way of writing uninitialized structs and ints into the file.
BUG=230772
Review URL: https://codereview.chromium.org/14263005
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@194347 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'net')
-rw-r--r-- | net/disk_cache/backend_unittest.cc | 2 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_disk_format.cc | 120 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_disk_format.h | 128 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_entry_format.cc | 16 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_entry_format.h | 38 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_entry_impl.h | 2 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_index.cc | 299 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_index.h | 86 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_index_file.cc | 164 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_index_file.h | 92 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_index_file_unittest.cc | 92 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_index_unittest.cc | 95 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_synchronous_entry.h | 2 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_util.cc | 43 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_util.h | 17 | ||||
-rw-r--r-- | net/disk_cache/simple/simple_util_unittest.cc | 65 | ||||
-rw-r--r-- | net/net.gyp | 9 |
17 files changed, 794 insertions, 476 deletions
diff --git a/net/disk_cache/backend_unittest.cc b/net/disk_cache/backend_unittest.cc index 013f623..afef14f 100644 --- a/net/disk_cache/backend_unittest.cc +++ b/net/disk_cache/backend_unittest.cc @@ -21,7 +21,7 @@ #include "net/disk_cache/histogram_macros.h" #include "net/disk_cache/mapped_file.h" #include "net/disk_cache/mem_backend_impl.h" -#include "net/disk_cache/simple/simple_disk_format.h" +#include "net/disk_cache/simple/simple_entry_format.h" #include "net/disk_cache/simple/simple_util.h" #include "net/disk_cache/tracing_cache_backend.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/net/disk_cache/simple/simple_disk_format.cc b/net/disk_cache/simple/simple_disk_format.cc deleted file mode 100644 index 4024767..0000000 --- a/net/disk_cache/simple/simple_disk_format.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "net/disk_cache/simple/simple_disk_format.h" - -#include "base/format_macros.h" -#include "base/hash.h" -#include "base/logging.h" -#include "base/sha1.h" -#include "base/stringprintf.h" -#include "base/strings/string_number_conversions.h" -#include "base/time.h" - -namespace disk_cache { - -SimpleFileHeader::SimpleFileHeader() { - // Make hashing repeatable: leave no padding bytes untouched. - memset(this, 0, sizeof(*this)); -} - -std::string ConvertEntryHashKeyToHexString(uint64 hash_key) { - const std::string hash_key_str = base::StringPrintf("%016" PRIx64, hash_key); - DCHECK_EQ(kEntryHashKeyAsHexStringSize, hash_key_str.size()); - return hash_key_str; -} - -std::string GetEntryHashKeyAsHexString(const std::string& key) { - std::string hash_key_str = - ConvertEntryHashKeyToHexString(GetEntryHashKey(key)); - DCHECK_EQ(kEntryHashKeyAsHexStringSize, hash_key_str.size()); - return hash_key_str; -} - -bool GetEntryHashKeyFromHexString(const std::string& hash_key, - uint64* hash_key_out) { - if (hash_key.size() != kEntryHashKeyAsHexStringSize) { - return false; - } - return base::HexStringToUInt64(hash_key, hash_key_out); -} - -uint64 GetEntryHashKey(const std::string& key) { - const std::string sha_hash = base::SHA1HashString(key); - uint64 hash_key = 0; - sha_hash.copy(reinterpret_cast<char*>(&hash_key), sizeof(hash_key)); - return hash_key; -} - -namespace SimpleIndexFile { - -Footer::Footer() { - // Make hashing repeatable: leave no padding bytes untouched. - memset(this, 0, sizeof(*this)); -} - -Header::Header() { - // Make hashing repeatable: leave no padding bytes untouched. - memset(this, 0, sizeof(*this)); -} - -EntryMetadata::EntryMetadata() { - // Make hashing repeatable: leave no padding bytes untouched. - memset(this, 0, sizeof(*this)); -} - -EntryMetadata::EntryMetadata(uint64 hash_key_p, - base::Time last_used_time_p, - uint64 entry_size_p) { - // Make hashing repeatable: leave no padding bytes untouched. - memset(this, 0, sizeof(*this)); - - // Proceed with field initializations. - hash_key = hash_key_p; - entry_size = entry_size_p; - last_used_time = last_used_time_p.ToInternalValue(); -} - -uint64 EntryMetadata::GetHashKey() const { - return hash_key; -} - -base::Time EntryMetadata::GetLastUsedTime() const { - return base::Time::FromInternalValue(last_used_time); -} - -void EntryMetadata::SetLastUsedTime(const base::Time& last_used_time_p) { - last_used_time = last_used_time_p.ToInternalValue(); -} - -// static -void EntryMetadata::Serialize(const EntryMetadata& in_entry_metadata, - std::string* out_buffer) { - DCHECK(out_buffer); - // TODO(felipeg): We may choose to, instead, serialize each struct member - // separately. - out_buffer->append(reinterpret_cast<const char*>(&in_entry_metadata), - kEntryMetadataSize); -} - -// static -void EntryMetadata::DeSerialize(const char* in_buffer, - EntryMetadata* out_entry_metadata) { - DCHECK(in_buffer); - DCHECK(out_entry_metadata); - memcpy(out_entry_metadata, in_buffer, kEntryMetadataSize); -} - -// static -void EntryMetadata::Merge(const EntryMetadata& from, - EntryMetadata* to) { - if (to->last_used_time == 0) - to->last_used_time = from.last_used_time; - if (to->entry_size == 0) - to->entry_size = from.entry_size; -} - -} // namespace SimpleIndexFile - -} // namespace disk_cache diff --git a/net/disk_cache/simple/simple_disk_format.h b/net/disk_cache/simple/simple_disk_format.h deleted file mode 100644 index 08fd2e2..0000000 --- a/net/disk_cache/simple/simple_disk_format.h +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef NET_DISK_CACHE_SIMPLE_SIMPLE_DISK_FORMAT_H_ -#define NET_DISK_CACHE_SIMPLE_SIMPLE_DISK_FORMAT_H_ - -#include <string> - -#include "base/basictypes.h" -#include "base/port.h" -#include "net/base/net_export.h" - -namespace base { -class Time; -} - -namespace disk_cache { - -const uint64 kSimpleInitialMagicNumber = GG_UINT64_C(0xfcfb6d1ba7725c30); -const uint64 kSimpleIndexInitialMagicNumber = GG_UINT64_C(0x656e74657220796f); - -// A file in the Simple cache consists of a SimpleFileHeader followed -// by data. - -const uint32 kSimpleVersion = 1; - -static const int kSimpleEntryFileCount = 3; - -struct NET_EXPORT_PRIVATE SimpleFileHeader { - SimpleFileHeader(); - uint64 initial_magic_number; - uint32 version; - uint32 key_length; - uint32 key_hash; -}; - -// Simple Index File sketch: -// This is based on the struct Header and Footer as seem below, and the struct -// alignment is platform dependent. -// The CRC check is a guarantee that we don't read incorrect values. -// ------------------------- -// struct Header; -// ------------------------- -// Repeated |size| times { -// struct EntryMetadata; -// } -// ------------------------- -// struct Footer; -// ------------------------- -namespace SimpleIndexFile { - // Simple Index File metadata is defined here. - struct Header { - Header(); - uint64 initial_magic_number; - uint32 version; - uint64 number_of_entries; - uint64 cache_size; // Total cache storage size in bytes. - }; - - // TODO(felipeg): At some point we should consider using a protobuffer. See - // that we are re-implementing most of protobuffer's functionality such as - // Serialize and Merge. - // We must keep this struct a POD. - struct EntryMetadata { - EntryMetadata(); - EntryMetadata(uint64 hash_key_p, - base::Time last_used_time_p, - uint64 entry_size_p); - - base::Time GetLastUsedTime() const; - uint64 GetHashKey() const; - void SetLastUsedTime(const base::Time& last_used_time_p); - - // Serialize the data from |in_entry_metadata| and appends the bytes in - // |out_buffer|. The serialization is platform dependent since it simply - // writes the whole struct from memory into the given buffer. - static void Serialize(const EntryMetadata& in_entry_metadata, - std::string* out_buffer); - - static void DeSerialize(const char* in_buffer, - EntryMetadata* out_entry_metadata); - - // Merge two EntryMetadata instances. - // The existing valid data in |out_entry_metadata| will prevail. - static void Merge(const EntryMetadata& entry_metadata, - EntryMetadata* out_entry_metadata); - - uint64 hash_key; - - // This is the serialized format from Time::ToInternalValue(). - // If you want to make calculations/comparisons, you should use the - // base::Time() class. Use the GetLastUsedTime() method above. - int64 last_used_time; - - uint64 entry_size; // Storage size in bytes. - }; - - const size_t kEntryMetadataSize = sizeof(EntryMetadata); - - struct Footer { - Footer(); - uint32 crc; - }; - -} // namespace SimpleIndexFile - -// Size of the uint64 hash_key number in Hex format in a string. -const size_t kEntryHashKeyAsHexStringSize = 2 * sizeof(uint64); - -std::string ConvertEntryHashKeyToHexString(uint64 hash_key); - -// |key| is the regular HTTP Cache key, which is a URL. -// Returns the Hex ascii representation of the uint64 hash_key. -std::string GetEntryHashKeyAsHexString(const std::string& key); - -// |key| is the regular HTTP Cache key, which is a URL. -// Returns the hash of the key as uint64. -uint64 GetEntryHashKey(const std::string& key); - -// Parses the |hash_key| string into a uint64 buffer. -// |hash_key| string must be of the form: FFFFFFFFFFFFFFFF . -bool GetEntryHashKeyFromHexString(const std::string& hash_key, - uint64* hash_key_out); - -} // namespace disk_cache - -#endif // NET_DISK_CACHE_SIMPLE_SIMPLE_DISK_FORMAT_H_ diff --git a/net/disk_cache/simple/simple_entry_format.cc b/net/disk_cache/simple/simple_entry_format.cc new file mode 100644 index 0000000..3408893 --- /dev/null +++ b/net/disk_cache/simple/simple_entry_format.cc @@ -0,0 +1,16 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "net/disk_cache/simple/simple_entry_format.h" + +#include <cstring> + +namespace disk_cache { + +SimpleFileHeader::SimpleFileHeader() { + // Make hashing repeatable: leave no padding bytes untouched. + std::memset(this, 0, sizeof(*this)); +} + +} // namespace disk_cache diff --git a/net/disk_cache/simple/simple_entry_format.h b/net/disk_cache/simple/simple_entry_format.h new file mode 100644 index 0000000..0e6d270 --- /dev/null +++ b/net/disk_cache/simple/simple_entry_format.h @@ -0,0 +1,38 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef NET_DISK_CACHE_SIMPLE_SIMPLE_ENTRY_FORMAT_H_ +#define NET_DISK_CACHE_SIMPLE_SIMPLE_ENTRY_FORMAT_H_ + + +#include "base/basictypes.h" +#include "base/port.h" +#include "net/base/net_export.h" + +namespace base { +class Time; +} + +namespace disk_cache { + +const uint64 kSimpleInitialMagicNumber = GG_UINT64_C(0xfcfb6d1ba7725c30); + +// A file in the Simple cache consists of a SimpleFileHeader followed +// by data. + +const uint32 kSimpleVersion = 1; + +static const int kSimpleEntryFileCount = 3; + +struct NET_EXPORT_PRIVATE SimpleFileHeader { + SimpleFileHeader(); + uint64 initial_magic_number; + uint32 version; + uint32 key_length; + uint32 key_hash; +}; + +} // namespace disk_cache + +#endif // NET_DISK_CACHE_SIMPLE_SIMPLE_ENTRY_FORMAT_H_ diff --git a/net/disk_cache/simple/simple_entry_impl.h b/net/disk_cache/simple/simple_entry_impl.h index 616aadd..7b898cf 100644 --- a/net/disk_cache/simple/simple_entry_impl.h +++ b/net/disk_cache/simple/simple_entry_impl.h @@ -11,7 +11,7 @@ #include "base/memory/weak_ptr.h" #include "base/threading/thread_checker.h" #include "net/disk_cache/disk_cache.h" -#include "net/disk_cache/simple/simple_disk_format.h" +#include "net/disk_cache/simple/simple_entry_format.h" #include "net/disk_cache/simple/simple_index.h" diff --git a/net/disk_cache/simple/simple_index.cc b/net/disk_cache/simple/simple_index.cc index 2a915be..c87b3c8 100644 --- a/net/disk_cache/simple/simple_index.cc +++ b/net/disk_cache/simple/simple_index.cc @@ -4,41 +4,70 @@ #include "net/disk_cache/simple/simple_index.h" +#include <utility> + #include "base/bind.h" #include "base/bind_helpers.h" +#include "base/file_util.h" +#include "base/logging.h" #include "base/message_loop.h" +#include "base/pickle.h" #include "base/task_runner.h" #include "base/threading/worker_pool.h" -#include "net/base/io_buffer.h" #include "net/base/net_errors.h" -#include "net/disk_cache/simple/simple_disk_format.h" -#include "third_party/zlib/zlib.h" +#include "net/disk_cache/simple/simple_entry_format.h" +#include "net/disk_cache/simple/simple_index_file.h" +#include "net/disk_cache/simple/simple_util.h" + +namespace disk_cache { + +EntryMetadata::EntryMetadata() : + hash_key_(0), + last_used_time_(0), + entry_size_(0) +{} -namespace { -const uint64 kMaxEntiresInIndex = 100000000; +EntryMetadata::EntryMetadata(uint64 hash_key, + base::Time last_used_time, + uint64 entry_size) : + hash_key_(hash_key), + last_used_time_(last_used_time.ToInternalValue()), + entry_size_(entry_size) +{} -bool CheckHeader(disk_cache::SimpleIndexFile::Header header) { - return header.number_of_entries <= kMaxEntiresInIndex && - header.initial_magic_number == - disk_cache::kSimpleIndexInitialMagicNumber && - header.version == disk_cache::kSimpleVersion; +base::Time EntryMetadata::GetLastUsedTime() const { + return base::Time::FromInternalValue(last_used_time_); } -class FileAutoCloser { - public: - explicit FileAutoCloser(const base::PlatformFile& file) : file_(file) { } - ~FileAutoCloser() { - base::ClosePlatformFile(file_); - } - private: - base::PlatformFile file_; - DISALLOW_COPY_AND_ASSIGN(FileAutoCloser); -}; +void EntryMetadata::SetLastUsedTime(const base::Time& last_used_time) { + last_used_time_ = last_used_time.ToInternalValue(); +} + +void EntryMetadata::Serialize(Pickle* pickle) const { + DCHECK(pickle); + COMPILE_ASSERT(sizeof(EntryMetadata) == + (sizeof(uint64) + sizeof(int64) + sizeof(uint64)), + EntryMetadata_has_three_member_variables); + pickle->WriteUInt64(hash_key_); + pickle->WriteInt64(last_used_time_); + pickle->WriteUInt64(entry_size_); +} -} // namespace +bool EntryMetadata::Deserialize(PickleIterator* it) { + DCHECK(it); + return it->ReadUInt64(&hash_key_) && + it->ReadInt64(&last_used_time_) && + it->ReadUInt64(&entry_size_); +} -namespace disk_cache { +void EntryMetadata::MergeWith(const EntryMetadata& from) { + DCHECK_EQ(hash_key_, from.hash_key_); + if (last_used_time_ == 0) + last_used_time_ = from.last_used_time_; + if (entry_size_ == 0) + entry_size_ = from.entry_size_; +} SimpleIndex::SimpleIndex( const scoped_refptr<base::TaskRunner>& cache_thread, @@ -57,8 +86,8 @@ SimpleIndex::~SimpleIndex() { void SimpleIndex::Initialize() { DCHECK(io_thread_checker_.CalledOnValidThread()); - MergeCallback merge_callback = base::Bind(&SimpleIndex::MergeInitializingSet, - this->AsWeakPtr()); + IndexCompletionCallback merge_callback = + base::Bind(&SimpleIndex::MergeInitializingSet, AsWeakPtr()); base::WorkerPool::PostTask(FROM_HERE, base::Bind(&SimpleIndex::LoadFromDisk, index_filename_, @@ -67,97 +96,14 @@ void SimpleIndex::Initialize() { true); } -// static -void SimpleIndex::LoadFromDisk( - const base::FilePath& index_filename, - const scoped_refptr<base::TaskRunner>& io_thread, - const MergeCallback& merge_callback) { - // Open the index file. - base::PlatformFileError error; - base::PlatformFile index_file = base::CreatePlatformFile( - index_filename, - base::PLATFORM_FILE_OPEN_ALWAYS | - base::PLATFORM_FILE_READ | - base::PLATFORM_FILE_WRITE, - NULL, - &error); - FileAutoCloser auto_close_index_file(index_file); - if (error != base::PLATFORM_FILE_OK) { - LOG(ERROR) << "Error opening file " << index_filename.value(); - return RestoreFromDisk(index_filename, io_thread, merge_callback); - } - - uLong incremental_crc = crc32(0L, Z_NULL, 0); - int64 index_file_offset = 0; - SimpleIndexFile::Header header; - if (base::ReadPlatformFile(index_file, - index_file_offset, - reinterpret_cast<char*>(&header), - sizeof(header)) != sizeof(header)) { - return RestoreFromDisk(index_filename, io_thread, merge_callback); - } - index_file_offset += sizeof(header); - incremental_crc = crc32(incremental_crc, - reinterpret_cast<const Bytef*>(&header), - implicit_cast<uInt>(sizeof(header))); - - if (!CheckHeader(header)) { - LOG(ERROR) << "Invalid header on Simple Cache Index."; - return RestoreFromDisk(index_filename, io_thread, merge_callback); - } - - const int entries_buffer_size = - header.number_of_entries * SimpleIndexFile::kEntryMetadataSize; - - scoped_ptr<char[]> entries_buffer(new char[entries_buffer_size]); - if (base::ReadPlatformFile(index_file, - index_file_offset, - entries_buffer.get(), - entries_buffer_size) != entries_buffer_size) { - return RestoreFromDisk(index_filename, io_thread, merge_callback); - } - index_file_offset += entries_buffer_size; - incremental_crc = crc32(incremental_crc, - reinterpret_cast<const Bytef*>(entries_buffer.get()), - implicit_cast<uInt>(entries_buffer_size)); - - SimpleIndexFile::Footer footer; - if (base::ReadPlatformFile(index_file, - index_file_offset, - reinterpret_cast<char*>(&footer), - sizeof(footer)) != sizeof(footer)) { - return RestoreFromDisk(index_filename, io_thread, merge_callback); - } - const uint32 crc_read = footer.crc; - const uint32 crc_calculated = incremental_crc; - if (crc_read != crc_calculated) - return RestoreFromDisk(index_filename, io_thread, merge_callback); - - scoped_ptr<EntrySet> index_file_entries(new EntrySet()); - int entries_buffer_offset = 0; - while(entries_buffer_offset < entries_buffer_size) { - SimpleIndexFile::EntryMetadata entry_metadata; - SimpleIndexFile::EntryMetadata::DeSerialize( - &entries_buffer.get()[entries_buffer_offset], &entry_metadata); - InsertInternal(index_file_entries.get(), entry_metadata); - entries_buffer_offset += SimpleIndexFile::kEntryMetadataSize; - } - DCHECK_EQ(header.number_of_entries, index_file_entries->size()); - - io_thread->PostTask(FROM_HERE, - base::Bind(merge_callback, - base::Passed(&index_file_entries))); -} - void SimpleIndex::Insert(const std::string& key) { DCHECK(io_thread_checker_.CalledOnValidThread()); // Upon insert we don't know yet the size of the entry. // It will be updated later when the SimpleEntryImpl finishes opening or // creating the new entry, and then UpdateEntrySize will be called. - const uint64 hash_key = GetEntryHashKey(key); - InsertInternal(&entries_set_, SimpleIndexFile::EntryMetadata( - hash_key, - base::Time::Now(), 0)); + const uint64 hash_key = simple_util::GetEntryHashKey(key); + InsertInEntrySet(EntryMetadata(hash_key, base::Time::Now(), 0), + &entries_set_); if (!initialized_) removed_entries_.erase(hash_key); } @@ -165,7 +111,7 @@ void SimpleIndex::Insert(const std::string& key) { void SimpleIndex::Remove(const std::string& key) { DCHECK(io_thread_checker_.CalledOnValidThread()); UpdateEntrySize(key, 0); - const uint64 hash_key = GetEntryHashKey(key); + const uint64 hash_key = simple_util::GetEntryHashKey(key); entries_set_.erase(hash_key); if (!initialized_) @@ -175,14 +121,15 @@ void SimpleIndex::Remove(const std::string& key) { bool SimpleIndex::Has(const std::string& key) const { DCHECK(io_thread_checker_.CalledOnValidThread()); // If not initialized, always return true, forcing it to go to the disk. - return !initialized_ || entries_set_.count(GetEntryHashKey(key)) != 0; + return !initialized_ || + entries_set_.count(simple_util::GetEntryHashKey(key)) != 0; } bool SimpleIndex::UseIfExists(const std::string& key) { DCHECK(io_thread_checker_.CalledOnValidThread()); // Always update the last used time, even if it is during initialization. // It will be merged later. - EntrySet::iterator it = entries_set_.find(GetEntryHashKey(key)); + EntrySet::iterator it = entries_set_.find(simple_util::GetEntryHashKey(key)); if (it == entries_set_.end()) // If not initialized, always return true, forcing it to go to the disk. return !initialized_; @@ -192,33 +139,46 @@ bool SimpleIndex::UseIfExists(const std::string& key) { bool SimpleIndex::UpdateEntrySize(const std::string& key, uint64 entry_size) { DCHECK(io_thread_checker_.CalledOnValidThread()); - EntrySet::iterator it = entries_set_.find(GetEntryHashKey(key)); + EntrySet::iterator it = entries_set_.find(simple_util::GetEntryHashKey(key)); if (it == entries_set_.end()) return false; // Update the total cache size with the new entry size. - cache_size_ -= it->second.entry_size; + cache_size_ -= it->second.GetEntrySize(); cache_size_ += entry_size; - it->second.entry_size = entry_size; + it->second.SetEntrySize(entry_size); return true; } // static -void SimpleIndex::InsertInternal( - EntrySet* entry_set, - const SimpleIndexFile::EntryMetadata& entry_metadata) { - // TODO(felipeg): Use a hash_set instead of a hash_map. +void SimpleIndex::InsertInEntrySet( + const disk_cache::EntryMetadata& entry_metadata, + EntrySet* entry_set) { DCHECK(entry_set); entry_set->insert( std::make_pair(entry_metadata.GetHashKey(), entry_metadata)); } // static -void SimpleIndex::RestoreFromDisk( +void SimpleIndex::LoadFromDisk( const base::FilePath& index_filename, const scoped_refptr<base::TaskRunner>& io_thread, - const MergeCallback& merge_callback) { + const IndexCompletionCallback& completion_callback) { + scoped_ptr<EntrySet> index_file_entries = + SimpleIndexFile::LoadFromDisk(index_filename); + + if (!index_file_entries.get()) + index_file_entries = SimpleIndex::RestoreFromDisk(index_filename); + + io_thread->PostTask(FROM_HERE, + base::Bind(completion_callback, + base::Passed(&index_file_entries))); +} + +// static +scoped_ptr<SimpleIndex::EntrySet> SimpleIndex::RestoreFromDisk( + const base::FilePath& index_filename) { using file_util::FileEnumerator; LOG(INFO) << "Simple Cache Index is being restored from disk."; @@ -228,6 +188,8 @@ void SimpleIndex::RestoreFromDisk( // TODO(felipeg,gavinp): Fix this once we have a one-file per entry format. COMPILE_ASSERT(kSimpleEntryFileCount == 3, file_pattern_must_match_file_count); + + const int kFileSuffixLenght = std::string("_0").size(); const base::FilePath::StringType file_pattern = FILE_PATH_LITERAL("*_[0-2]"); FileEnumerator enumerator(index_filename.DirName(), false /* recursive */, @@ -240,9 +202,10 @@ void SimpleIndex::RestoreFromDisk( // file names. const std::string hash_name(base_name.begin(), base_name.end()); const std::string hash_key_string = - hash_name.substr(0, kEntryHashKeyAsHexStringSize); + hash_name.substr(0, hash_name.size() - kFileSuffixLenght); uint64 hash_key = 0; - if (!GetEntryHashKeyFromHexString(hash_key_string, &hash_key)) { + if (!simple_util::GetEntryHashKeyFromHexString( + hash_key_string, &hash_key)) { LOG(WARNING) << "Invalid Entry Hash Key filename while restoring " << "Simple Index from disk: " << hash_name; // TODO(felipeg): Should we delete the invalid file here ? @@ -263,17 +226,21 @@ void SimpleIndex::RestoreFromDisk( int64 file_size = FileEnumerator::GetFilesize(find_info); EntrySet::iterator it = index_file_entries->find(hash_key); if (it == index_file_entries->end()) { - InsertInternal(index_file_entries.get(), SimpleIndexFile::EntryMetadata( - hash_key, last_used_time, file_size)); + InsertInEntrySet(EntryMetadata(hash_key, last_used_time, file_size), + index_file_entries.get()); } else { // Summing up the total size of the entry through all the *_[0-2] files - it->second.entry_size += file_size; + it->second.SetEntrySize(it->second.GetEntrySize() + file_size); } } + return index_file_entries.Pass(); +} - io_thread->PostTask(FROM_HERE, - base::Bind(merge_callback, - base::Passed(&index_file_entries))); + +// static +void SimpleIndex::WriteToDiskInternal(const base::FilePath& index_filename, + scoped_ptr<Pickle> pickle) { + SimpleIndexFile::WriteToDisk(index_filename, *pickle); } void SimpleIndex::MergeInitializingSet( @@ -296,79 +263,27 @@ void SimpleIndex::MergeInitializingSet( EntrySet::iterator current_entry = entries_set_.find(it->first); if (current_entry != entries_set_.end()) { // When Merging, existing valid data in the |current_entry| will prevail. - SimpleIndexFile::EntryMetadata::Merge( - it->second, &(current_entry->second)); - cache_size_ += current_entry->second.entry_size; + current_entry->second.MergeWith(it->second); + cache_size_ += current_entry->second.GetEntrySize(); } else { - InsertInternal(&entries_set_, it->second); - cache_size_ += it->second.entry_size; + InsertInEntrySet(it->second, &entries_set_); + cache_size_ += it->second.GetEntrySize(); } } initialized_ = true; } -void SimpleIndex::Serialize(std::string* out_buffer) { - DCHECK(io_thread_checker_.CalledOnValidThread()); - DCHECK(out_buffer); - SimpleIndexFile::Header header; - SimpleIndexFile::Footer footer; - - header.initial_magic_number = kSimpleIndexInitialMagicNumber; - header.version = kSimpleVersion; - header.number_of_entries = entries_set_.size(); - - out_buffer->reserve( - sizeof(header) + - sizeof(SimpleIndexFile::EntryMetadata) * entries_set_.size() + - sizeof(footer)); - - // The Header goes first. - out_buffer->append(reinterpret_cast<const char*>(&header), - sizeof(header)); - - // Then all the entries from |entries_set_|. - for (EntrySet::const_iterator it = entries_set_.begin(); - it != entries_set_.end(); ++it) { - SimpleIndexFile::EntryMetadata::Serialize(it->second, out_buffer); - } - - // Then, CRC. - footer.crc = crc32(crc32(0, Z_NULL, 0), - reinterpret_cast<const Bytef*>(out_buffer->data()), - implicit_cast<uInt>(out_buffer->size())); - - out_buffer->append(reinterpret_cast<const char*>(&footer), sizeof(footer)); -} - void SimpleIndex::WriteToDisk() { DCHECK(io_thread_checker_.CalledOnValidThread()); - scoped_ptr<std::string> buffer(new std::string()); - Serialize(buffer.get()); + SimpleIndexFile::IndexMetadata index_metadata(entries_set_.size(), + cache_size_); + scoped_ptr<Pickle> pickle = SimpleIndexFile::Serialize(index_metadata, + entries_set_); cache_thread_->PostTask(FROM_HERE, base::Bind( - &SimpleIndex::UpdateFile, + &SimpleIndex::WriteToDiskInternal, index_filename_, - index_filename_.DirName().AppendASCII("index_temp"), - base::Passed(&buffer))); -} - -// static -void SimpleIndex::UpdateFile(const base::FilePath& index_filename, - const base::FilePath& temp_filename, - scoped_ptr<std::string> buffer) { - int bytes_written = file_util::WriteFile( - temp_filename, buffer->data(), buffer->size()); - DCHECK_EQ(bytes_written, implicit_cast<int>(buffer->size())); - if (bytes_written != static_cast<int>(buffer->size())) { - // TODO(felipeg): Add better error handling. - LOG(ERROR) << "Could not write Simple Cache index to temporary file: " - << temp_filename.value(); - file_util::Delete(temp_filename, /* recursive = */ false); - return; - } - // Swap temp and index_file. - bool result = file_util::ReplaceFile(temp_filename, index_filename); - DCHECK(result); + base::Passed(&pickle))); } } // namespace disk_cache diff --git a/net/disk_cache/simple/simple_index.h b/net/disk_cache/simple/simple_index.h index 71f5bb4..76cf55f 100644 --- a/net/disk_cache/simple/simple_index.h +++ b/net/disk_cache/simple/simple_index.h @@ -9,13 +9,17 @@ #include <string> #include "base/basictypes.h" -#include "base/file_util.h" +#include "base/callback.h" #include "base/files/file_path.h" #include "base/hash_tables.h" +#include "base/memory/ref_counted.h" #include "base/memory/scoped_ptr.h" #include "base/memory/weak_ptr.h" -#include "net/disk_cache/disk_cache.h" -#include "net/disk_cache/simple/simple_disk_format.h" +#include "base/time.h" +#include "net/base/net_export.h" + +class Pickle; +class PickleIterator; namespace base { class TaskRunner; @@ -23,8 +27,46 @@ class TaskRunner; namespace disk_cache { +class NET_EXPORT_PRIVATE EntryMetadata { + public: + EntryMetadata(); + EntryMetadata(uint64 hash_key, + base::Time last_used_time, + uint64 entry_size); + + uint64 GetHashKey() const { return hash_key_; } + base::Time GetLastUsedTime() const; + void SetLastUsedTime(const base::Time& last_used_time); + + uint64 GetEntrySize() const { return entry_size_; } + void SetEntrySize(uint64 entry_size) { entry_size_ = entry_size; } + + // Serialize the data into the provided pickle. + void Serialize(Pickle* pickle) const; + bool Deserialize(PickleIterator* it); + + // Merge two EntryMetadata instances. + // The existing current valid data in |this| will prevail. + void MergeWith(const EntryMetadata& entry_metadata); + + private: + friend class SimpleIndexFileTest; + + // When adding new members here, you should update the Serialize() and + // Deserialize() methods. + uint64 hash_key_; + + // This is the serialized format from Time::ToInternalValue(). + // If you want to make calculations/comparisons, you should use the + // base::Time() class. Use the GetLastUsedTime() method above. + // TODO(felipeg): Use Time() here. + int64 last_used_time_; + + uint64 entry_size_; // Storage size in bytes. +}; + // This class is not Thread-safe. -class SimpleIndex +class NET_EXPORT_PRIVATE SimpleIndex : public base::SupportsWeakPtr<SimpleIndex> { public: SimpleIndex( @@ -52,44 +94,32 @@ class SimpleIndex // entry. bool UpdateEntrySize(const std::string& key, uint64 entry_size); - private: - // TODO(felipeg): This way we are storing the hash_key twice (as the + // TODO(felipeg): This way we are storing the hash_key twice, as the // hash_map::key and as a member of EntryMetadata. We could save space if we // use a hash_set. - typedef base::hash_map<uint64, SimpleIndexFile::EntryMetadata> EntrySet; + typedef base::hash_map<uint64, EntryMetadata> EntrySet; - typedef base::Callback<void(scoped_ptr<EntrySet>)> MergeCallback; + static void InsertInEntrySet(const EntryMetadata& entry_metadata, + EntrySet* entry_set); - static void InsertInternal( - EntrySet* entry_set, - const SimpleIndexFile::EntryMetadata& entry_metadata); + private: + typedef base::Callback<void(scoped_ptr<EntrySet>)> IndexCompletionCallback; - // Load index from disk. If it is corrupted, call RestoreFromDisk(). static void LoadFromDisk( const base::FilePath& index_filename, const scoped_refptr<base::TaskRunner>& io_thread, - const MergeCallback& merge_callback); + const IndexCompletionCallback& completion_callback); // Enumerates all entries' files on disk and regenerates the index. - static void RestoreFromDisk( - const base::FilePath& index_filename, - const scoped_refptr<base::TaskRunner>& io_thread, - const MergeCallback& merge_callback); + static scoped_ptr<SimpleIndex::EntrySet> RestoreFromDisk( + const base::FilePath& index_filename); + + static void WriteToDiskInternal(const base::FilePath& index_filename, + scoped_ptr<Pickle> pickle); // Must run on IO Thread. void MergeInitializingSet(scoped_ptr<EntrySet> index_file_entries); - // |out_buffer| needs to be pre-allocated. The serialized index is stored in - // |out_buffer|. - void Serialize(std::string* out_buffer); - - bool OpenIndexFile(); - bool CloseIndexFile(); - - static void UpdateFile(const base::FilePath& index_filename, - const base::FilePath& temp_filename, - scoped_ptr<std::string> buffer); - EntrySet entries_set_; uint64 cache_size_; // Total cache storage size in bytes. diff --git a/net/disk_cache/simple/simple_index_file.cc b/net/disk_cache/simple/simple_index_file.cc new file mode 100644 index 0000000..a291320 --- /dev/null +++ b/net/disk_cache/simple/simple_index_file.cc @@ -0,0 +1,164 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "net/disk_cache/simple/simple_index_file.h" + +#include "base/file_util.h" +#include "base/hash.h" +#include "base/logging.h" +#include "base/pickle.h" +#include "net/disk_cache/simple/simple_entry_format.h" +#include "net/disk_cache/simple/simple_index.h" +#include "net/disk_cache/simple/simple_util.h" +#include "third_party/zlib/zlib.h" + + +namespace { + +const uint64 kMaxEntiresInIndex = 100000000; + +uint32 CalculatePickleCRC(const Pickle& pickle) { + return crc32(crc32(0, Z_NULL, 0), + reinterpret_cast<const Bytef*>(pickle.payload()), + pickle.payload_size()); +} + +} // namespace + +namespace disk_cache { + +SimpleIndexFile::IndexMetadata::IndexMetadata() : + magic_number_(kSimpleIndexMagicNumber), + version_(kSimpleVersion), + number_of_entries_(0), + cache_size_(0) {} + +SimpleIndexFile::IndexMetadata::IndexMetadata( + uint64 number_of_entries, uint64 cache_size) : + magic_number_(kSimpleIndexMagicNumber), + version_(kSimpleVersion), + number_of_entries_(number_of_entries), + cache_size_(cache_size) {} + +void SimpleIndexFile::IndexMetadata::Serialize(Pickle* pickle) const { + DCHECK(pickle); + pickle->WriteUInt64(magic_number_); + pickle->WriteUInt32(version_); + pickle->WriteUInt64(number_of_entries_); + pickle->WriteUInt64(cache_size_); +} + +bool SimpleIndexFile::IndexMetadata::Deserialize(PickleIterator* it) { + DCHECK(it); + return it->ReadUInt64(&magic_number_) && + it->ReadUInt32(&version_) && + it->ReadUInt64(&number_of_entries_)&& + it->ReadUInt64(&cache_size_); +} + +bool SimpleIndexFile::IndexMetadata::CheckIndexMetadata() { + return number_of_entries_ <= kMaxEntiresInIndex && + magic_number_ == disk_cache::kSimpleIndexMagicNumber && + version_ == disk_cache::kSimpleVersion; +} + +// static +scoped_ptr<SimpleIndex::EntrySet> SimpleIndexFile::LoadFromDisk( + const base::FilePath& index_filename) { + std::string contents; + if(!file_util::ReadFileToString(index_filename, &contents)) { + LOG(WARNING) << "Could not read Simple Index file."; + return scoped_ptr<SimpleIndex::EntrySet>(NULL); + } + + return SimpleIndexFile::Deserialize(contents.data(), contents.size()); +} + +// static +scoped_ptr<SimpleIndex::EntrySet> SimpleIndexFile::Deserialize(const char* data, + int data_len) { + DCHECK(data); + Pickle pickle(data, data_len); + if (!pickle.data()) { + LOG(WARNING) << "Corrupt Simple Index File."; + return scoped_ptr<SimpleIndex::EntrySet>(NULL); + } + + PickleIterator pickle_it(pickle); + + SimpleIndexFile::PickleHeader* header_p = + pickle.headerT<SimpleIndexFile::PickleHeader>(); + const uint32 crc_read = header_p->crc; + const uint32 crc_calculated = CalculatePickleCRC(pickle); + + if (crc_read != crc_calculated) { + LOG(WARNING) << "Invalid CRC in Simple Index file."; + return scoped_ptr<SimpleIndex::EntrySet>(NULL); + } + + SimpleIndexFile::IndexMetadata index_metadata; + if (!index_metadata.Deserialize(&pickle_it)) { + LOG(ERROR) << "Invalid index_metadata on Simple Cache Index."; + return scoped_ptr<SimpleIndex::EntrySet>(NULL); + } + + if (!index_metadata.CheckIndexMetadata()) { + LOG(ERROR) << "Invalid index_metadata on Simple Cache Index."; + return scoped_ptr<SimpleIndex::EntrySet>(NULL); + } + + scoped_ptr<SimpleIndex::EntrySet> index_file_entries( + new SimpleIndex::EntrySet()); + while (index_file_entries->size() < index_metadata.GetNumberOfEntries()) { + EntryMetadata entry_metadata; + if (!entry_metadata.Deserialize(&pickle_it)) { + LOG(WARNING) << "Invalid EntryMetadata in Simple Index file."; + return scoped_ptr<SimpleIndex::EntrySet>(NULL); + } + SimpleIndex::InsertInEntrySet(entry_metadata, index_file_entries.get()); + } + + return index_file_entries.Pass(); +} + +// static +scoped_ptr<Pickle> SimpleIndexFile::Serialize( + const SimpleIndexFile::IndexMetadata& index_metadata, + const SimpleIndex::EntrySet& entries) { + scoped_ptr<Pickle> pickle(new Pickle(sizeof(SimpleIndexFile::PickleHeader))); + + index_metadata.Serialize(pickle.get()); + for (SimpleIndex::EntrySet::const_iterator it = entries.begin(); + it != entries.end(); ++it) { + it->second.Serialize(pickle.get()); + } + SimpleIndexFile::PickleHeader* header_p = + pickle->headerT<SimpleIndexFile::PickleHeader>(); + header_p->crc = CalculatePickleCRC(*pickle); + return pickle.Pass(); +} + +// static +void SimpleIndexFile::WriteToDisk(const base::FilePath& index_filename, + const Pickle& pickle) { + const base::FilePath temp_filename = + index_filename.DirName().AppendASCII("index_temp"); + int bytes_written = file_util::WriteFile( + temp_filename, + reinterpret_cast<const char*>(pickle.data()), + pickle.size()); + DCHECK_EQ(bytes_written, implicit_cast<int>(pickle.size())); + if (bytes_written != static_cast<int>(pickle.size())) { + // TODO(felipeg): Add better error handling. + LOG(ERROR) << "Could not write Simple Cache index to temporary file: " + << temp_filename.value(); + file_util::Delete(temp_filename, /* recursive = */ false); + return; + } + // Swap temp and index_file. + bool result = file_util::ReplaceFile(temp_filename, index_filename); + DCHECK(result); +} + +} // namespace disk_cache diff --git a/net/disk_cache/simple/simple_index_file.h b/net/disk_cache/simple/simple_index_file.h new file mode 100644 index 0000000..b61e7d2 --- /dev/null +++ b/net/disk_cache/simple/simple_index_file.h @@ -0,0 +1,92 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_ +#define NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_ + +#include <string> + +#include "base/basictypes.h" +#include "base/files/file_path.h" +#include "base/gtest_prod_util.h" +#include "base/hash_tables.h" +#include "base/logging.h" +#include "base/memory/scoped_ptr.h" +#include "base/pickle.h" +#include "base/port.h" +#include "net/base/net_export.h" +#include "net/disk_cache/simple/simple_index.h" + +namespace base { +class TaskRunner; +} + +namespace disk_cache { + +const uint64 kSimpleIndexMagicNumber = GG_UINT64_C(0x656e74657220796f); + +// Simple Index File format is a pickle serialized data of IndexMetadata and +// EntryMetadata objects. The file format is as follows: one instance of +// serialized |IndexMetadata| followed serialized |EntryMetadata| entries +// repeated |number_of_entries| amount of times. To know more about the format, +// see SimpleIndexFile::Serialize() and SeeSimpleIndexFile::LoadFromDisk() +// methods. +class NET_EXPORT_PRIVATE SimpleIndexFile { + public: + class NET_EXPORT_PRIVATE IndexMetadata { + public: + IndexMetadata(); + IndexMetadata(uint64 number_of_entries, uint64 cache_size); + + void Serialize(Pickle* pickle) const; + bool Deserialize(PickleIterator* it); + + bool CheckIndexMetadata(); + + uint64 GetNumberOfEntries() { return number_of_entries_; } + + private: + FRIEND_TEST_ALL_PREFIXES(IndexMetadataTest, Basics); + FRIEND_TEST_ALL_PREFIXES(IndexMetadataTest, Serialize); + + uint64 magic_number_; + uint32 version_; + uint64 number_of_entries_; + uint64 cache_size_; // Total cache storage size in bytes. + }; + + // Load the index file from disk, deserializing it and returning the + // corresponding EntrySet in a scoped_ptr<>, if successful. + // Uppon failure, the scoped_ptr<> will contain NULL. + static scoped_ptr<SimpleIndex::EntrySet> LoadFromDisk( + const base::FilePath& index_filename); + + // Returns a scoped_ptr for a newly allocated Pickle containing the serialized + // data to be written to a file. + static scoped_ptr<Pickle> Serialize( + const SimpleIndexFile::IndexMetadata& index_metadata, + const SimpleIndex::EntrySet& entries); + + // Write the serialized data from |pickle| into the index file. + static void WriteToDisk(const base::FilePath& index_filename, + const Pickle& pickle); + + private: + FRIEND_TEST_ALL_PREFIXES(SimpleIndexFileTest, Serialize); + + // Deserialize() is separate from LoadFromDisk() for easier testing. + static scoped_ptr<SimpleIndex::EntrySet> Deserialize(const char* data, + int data_len); + + struct PickleHeader : public Pickle::Header { + uint32 crc; + }; + + DISALLOW_COPY_AND_ASSIGN(SimpleIndexFile); +}; + + +} // namespace disk_cache + +#endif // NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_ diff --git a/net/disk_cache/simple/simple_index_file_unittest.cc b/net/disk_cache/simple/simple_index_file_unittest.cc new file mode 100644 index 0000000..32afc30 --- /dev/null +++ b/net/disk_cache/simple/simple_index_file_unittest.cc @@ -0,0 +1,92 @@ +// Copyright (c) 2011 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/hash.h" +#include "base/logging.h" +#include "base/memory/scoped_ptr.h" +#include "base/pickle.h" +#include "base/stringprintf.h" +#include "base/time.h" +#include "net/disk_cache/simple/simple_entry_format.h" +#include "net/disk_cache/simple/simple_index.h" +#include "net/disk_cache/simple/simple_index_file.h" +#include "testing/gtest/include/gtest/gtest.h" + +using base::Time; +using disk_cache::SimpleIndexFile; +using disk_cache::SimpleIndex; + +namespace disk_cache { + +class IndexMetadataTest : public testing::Test {}; + +TEST_F(IndexMetadataTest, Basics) { + SimpleIndexFile::IndexMetadata index_metadata; + + EXPECT_EQ(disk_cache::kSimpleIndexMagicNumber, index_metadata.magic_number_); + EXPECT_EQ(disk_cache::kSimpleVersion, index_metadata.version_); + EXPECT_EQ(0U, index_metadata.GetNumberOfEntries()); + EXPECT_EQ(0U, index_metadata.cache_size_); + + EXPECT_TRUE(index_metadata.CheckIndexMetadata()); +} + +TEST_F(IndexMetadataTest, Serialize) { + SimpleIndexFile::IndexMetadata index_metadata(123, 456); + Pickle pickle; + index_metadata.Serialize(&pickle); + PickleIterator it(pickle); + SimpleIndexFile::IndexMetadata new_index_metadata; + new_index_metadata.Deserialize(&it); + + EXPECT_EQ(new_index_metadata.magic_number_, index_metadata.magic_number_); + EXPECT_EQ(new_index_metadata.version_, index_metadata.version_); + EXPECT_EQ(new_index_metadata.GetNumberOfEntries(), + index_metadata.GetNumberOfEntries()); + EXPECT_EQ(new_index_metadata.cache_size_, index_metadata.cache_size_); + + EXPECT_TRUE(new_index_metadata.CheckIndexMetadata()); +} + +class SimpleIndexFileTest : public testing::Test { + public: + bool CompareTwoEntryMetadata(const EntryMetadata& a, const EntryMetadata& b) { + return a.hash_key_ == b.hash_key_ && + a.last_used_time_ == b.last_used_time_ && + a.entry_size_ == b.entry_size_; + } + +}; + +TEST_F(SimpleIndexFileTest, Serialize) { + SimpleIndex::EntrySet entries; + EntryMetadata entries_array[] = { + EntryMetadata(11, Time::FromInternalValue(22), 33), + EntryMetadata(22, Time::FromInternalValue(33), 44), + EntryMetadata(33, Time::FromInternalValue(44), 55) + }; + SimpleIndexFile::IndexMetadata index_metadata(arraysize(entries_array), 456); + for (uint32 i = 0; i < arraysize(entries_array); ++i) { + SimpleIndex::InsertInEntrySet(entries_array[i], &entries); + } + + scoped_ptr<Pickle> pickle = SimpleIndexFile::Serialize( + index_metadata, entries); + EXPECT_TRUE(pickle.get() != NULL); + + scoped_ptr<SimpleIndex::EntrySet> new_entries = SimpleIndexFile::Deserialize( + reinterpret_cast<const char*>(pickle->data()), + pickle->size()); + EXPECT_TRUE(new_entries.get() != NULL); + EXPECT_EQ(entries.size(), new_entries->size()); + + for (uint32 i = 0; i < arraysize(entries_array); ++i) { + SimpleIndex::EntrySet::iterator it = + new_entries->find(entries_array[i].GetHashKey()); + EXPECT_TRUE(new_entries->end() != it); + EXPECT_TRUE(CompareTwoEntryMetadata(it->second, entries_array[i])); + } +} + +} diff --git a/net/disk_cache/simple/simple_index_unittest.cc b/net/disk_cache/simple/simple_index_unittest.cc new file mode 100644 index 0000000..264dcae --- /dev/null +++ b/net/disk_cache/simple/simple_index_unittest.cc @@ -0,0 +1,95 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/hash.h" +#include "base/logging.h" +#include "base/memory/scoped_ptr.h" +#include "base/pickle.h" +#include "base/sha1.h" +#include "base/stringprintf.h" +#include "base/time.h" +#include "net/disk_cache/simple/simple_index.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace { + +const uint64 kTestHashKey = 4364515; +const int64 kTestLastUsedTimeInternal = 12345; +const base::Time kTestLastUsedTime = + base::Time::FromInternalValue(kTestLastUsedTimeInternal); +const uint64 kTestEntrySize = 789; + +} // namespace + +using disk_cache::EntryMetadata; + +class EntryMetadataTest : public testing::Test { + public: + EntryMetadata NewEntryMetadataWithValues() { + return EntryMetadata(kTestHashKey, + kTestLastUsedTime, + kTestEntrySize); + } + + void CheckEntryMetadataValues(const EntryMetadata& entry_metadata) { + EXPECT_EQ(kTestLastUsedTime, entry_metadata.GetLastUsedTime()); + EXPECT_EQ(kTestEntrySize, entry_metadata.GetEntrySize()); + } +}; + +TEST_F(EntryMetadataTest, Basics) { + EntryMetadata entry_metadata; + EXPECT_EQ(base::Time::FromInternalValue(0), entry_metadata.GetLastUsedTime()); + EXPECT_EQ(size_t(0), entry_metadata.GetEntrySize()); + + entry_metadata = NewEntryMetadataWithValues(); + CheckEntryMetadataValues(entry_metadata); + EXPECT_EQ(kTestHashKey, entry_metadata.GetHashKey()); + + const base::Time new_time = base::Time::FromInternalValue(5); + entry_metadata.SetLastUsedTime(new_time); + EXPECT_EQ(new_time, entry_metadata.GetLastUsedTime()); +} + +TEST_F(EntryMetadataTest, Serialize) { + EntryMetadata entry_metadata = NewEntryMetadataWithValues(); + + Pickle pickle; + entry_metadata.Serialize(&pickle); + + PickleIterator it(pickle); + EntryMetadata new_entry_metadata; + new_entry_metadata.Deserialize(&it); + CheckEntryMetadataValues(new_entry_metadata); + EXPECT_EQ(kTestHashKey, new_entry_metadata.GetHashKey()); +} + +TEST_F(EntryMetadataTest, Merge) { + EntryMetadata entry_metadata_a = NewEntryMetadataWithValues(); + // MergeWith assumes the hash_key of both entries is the same, so we + // initialize it to be that way. + base::Time dummy_time = base::Time::FromInternalValue(0); + EntryMetadata entry_metadata_b(entry_metadata_a.GetHashKey(), dummy_time, 0); + entry_metadata_b.MergeWith(entry_metadata_a); + CheckEntryMetadataValues(entry_metadata_b); + + EntryMetadata entry_metadata_c(entry_metadata_a.GetHashKey(), dummy_time, 0); + entry_metadata_a.MergeWith(entry_metadata_c); + CheckEntryMetadataValues(entry_metadata_a); + + EntryMetadata entry_metadata_d(entry_metadata_a.GetHashKey(), dummy_time, 0); + const base::Time new_time = base::Time::FromInternalValue(5); + entry_metadata_d.SetLastUsedTime(new_time); + entry_metadata_d.MergeWith(entry_metadata_a); + EXPECT_EQ(entry_metadata_a.GetEntrySize(), entry_metadata_d.GetEntrySize()); + EXPECT_EQ(new_time, entry_metadata_d.GetLastUsedTime()); + + EntryMetadata entry_metadata_e(entry_metadata_a.GetHashKey(), dummy_time, 0); + const uint64 entry_size = 9999999; + entry_metadata_e.SetEntrySize(entry_size); + entry_metadata_e.MergeWith(entry_metadata_a); + EXPECT_EQ(entry_size, entry_metadata_e.GetEntrySize()); + EXPECT_EQ(entry_metadata_a.GetLastUsedTime(), + entry_metadata_e.GetLastUsedTime()); +} diff --git a/net/disk_cache/simple/simple_synchronous_entry.h b/net/disk_cache/simple/simple_synchronous_entry.h index 2f06f5b..6d7eac0 100644 --- a/net/disk_cache/simple/simple_synchronous_entry.h +++ b/net/disk_cache/simple/simple_synchronous_entry.h @@ -14,7 +14,7 @@ #include "base/task_runner.h" #include "base/time.h" #include "net/base/completion_callback.h" -#include "net/disk_cache/simple/simple_disk_format.h" +#include "net/disk_cache/simple/simple_entry_format.h" namespace base { class SingleThreadTaskRunner; diff --git a/net/disk_cache/simple/simple_util.cc b/net/disk_cache/simple/simple_util.cc index 6dfe792..e716e16 100644 --- a/net/disk_cache/simple/simple_util.cc +++ b/net/disk_cache/simple/simple_util.cc @@ -6,17 +6,54 @@ #include <limits> +#include "base/format_macros.h" #include "base/logging.h" +#include "base/sha1.h" #include "base/stringprintf.h" -#include "net/disk_cache/simple/simple_disk_format.h" +#include "base/strings/string_number_conversions.h" +#include "net/disk_cache/simple/simple_entry_format.h" + +namespace { + +// Size of the uint64 hash_key number in Hex format in a string. +const size_t kEntryHashKeyAsHexStringSize = 2 * sizeof(uint64); + +} // namespace namespace disk_cache { namespace simple_util { +std::string ConvertEntryHashKeyToHexString(uint64 hash_key) { + const std::string hash_key_str = base::StringPrintf("%016" PRIx64, hash_key); + DCHECK_EQ(kEntryHashKeyAsHexStringSize, hash_key_str.size()); + return hash_key_str; +} + +std::string GetEntryHashKeyAsHexString(const std::string& key) { + std::string hash_key_str = + ConvertEntryHashKeyToHexString(GetEntryHashKey(key)); + DCHECK_EQ(kEntryHashKeyAsHexStringSize, hash_key_str.size()); + return hash_key_str; +} + +bool GetEntryHashKeyFromHexString(const std::string& hash_key, + uint64* hash_key_out) { + if (hash_key.size() != kEntryHashKeyAsHexStringSize) { + return false; + } + return base::HexStringToUInt64(hash_key, hash_key_out); +} + +uint64 GetEntryHashKey(const std::string& key) { + const std::string sha_hash = base::SHA1HashString(key); + uint64 hash_key = 0; + sha_hash.copy(reinterpret_cast<char*>(&hash_key), sizeof(hash_key)); + return hash_key; +} + std::string GetFilenameFromKeyAndIndex(const std::string& key, int index) { - return disk_cache::GetEntryHashKeyAsHexString(key) + - base::StringPrintf("_%1d", index); + return GetEntryHashKeyAsHexString(key) + base::StringPrintf("_%1d", index); } int32 GetDataSizeFromKeyAndFileSize(const std::string& key, int64 file_size) { diff --git a/net/disk_cache/simple/simple_util.h b/net/disk_cache/simple/simple_util.h index 0b84660..5bbd46cb 100644 --- a/net/disk_cache/simple/simple_util.h +++ b/net/disk_cache/simple/simple_util.h @@ -14,6 +14,23 @@ namespace disk_cache { namespace simple_util { +NET_EXPORT_PRIVATE std::string ConvertEntryHashKeyToHexString(uint64 hash_key); + +// |key| is the regular cache key, such as an URL. +// Returns the Hex ascii representation of the uint64 hash_key. +NET_EXPORT_PRIVATE std::string GetEntryHashKeyAsHexString( + const std::string& key); + +// |key| is the regular HTTP Cache key, which is a URL. +// Returns the hash of the key as uint64. +NET_EXPORT_PRIVATE uint64 GetEntryHashKey(const std::string& key); + +// Parses the |hash_key| string into a uint64 buffer. +// |hash_key| string must be of the form: FFFFFFFFFFFFFFFF . +NET_EXPORT_PRIVATE bool GetEntryHashKeyFromHexString( + const std::string& hash_key, + uint64* hash_key_out); + // Given a |key| for a (potential) entry in the simple backend and the |index| // of a stream on that entry, returns the filename in which that stream would be // stored. diff --git a/net/disk_cache/simple/simple_util_unittest.cc b/net/disk_cache/simple/simple_util_unittest.cc new file mode 100644 index 0000000..907a9e0 --- /dev/null +++ b/net/disk_cache/simple/simple_util_unittest.cc @@ -0,0 +1,65 @@ +// Copyright (c) 2013 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/logging.h" +#include "net/disk_cache/simple/simple_util.h" +#include "testing/gtest/include/gtest/gtest.h" + +using disk_cache::simple_util::ConvertEntryHashKeyToHexString; +using disk_cache::simple_util::GetEntryHashKeyAsHexString; +using disk_cache::simple_util::GetEntryHashKeyFromHexString; +using disk_cache::simple_util::GetEntryHashKey; + +class SimpleUtilTest : public testing::Test {}; + +TEST_F(SimpleUtilTest, ConvertEntryHashKeyToHexString) { + EXPECT_EQ("0000000005f5e0ff", + ConvertEntryHashKeyToHexString(99999999)); + EXPECT_EQ("7fffffffffffffff", + ConvertEntryHashKeyToHexString(9223372036854775807UL)); + EXPECT_EQ("8000000000000000", + ConvertEntryHashKeyToHexString(9223372036854775808UL)); + EXPECT_EQ("ffffffffffffffff", + ConvertEntryHashKeyToHexString(18446744073709551615UL)); +} + +TEST_F(SimpleUtilTest, GetEntryHashKey) { + EXPECT_EQ("7ac408c1dff9c84b", + GetEntryHashKeyAsHexString("http://www.amazon.com/")); + EXPECT_EQ(0x7ac408c1dff9c84bUL, GetEntryHashKey("http://www.amazon.com/")); + + EXPECT_EQ("9fe947998c2ccf47", + GetEntryHashKeyAsHexString("www.amazon.com")); + EXPECT_EQ(0x9fe947998c2ccf47UL, GetEntryHashKey("www.amazon.com")); + + EXPECT_EQ("0d4b6b5eeea339da", GetEntryHashKeyAsHexString("")); + EXPECT_EQ(0x0d4b6b5eeea339daUL, GetEntryHashKey("")); + + EXPECT_EQ("a68ac2ecc87dfd04", GetEntryHashKeyAsHexString("http://www.domain.com/uoQ76Kb2QL5hzaVOSAKWeX0W9LfDLqphmRXpsfHN8tgF5lCsfTxlOVWY8vFwzhsRzoNYKhUIOTc5TnUlT0vpdQflPyk2nh7vurXOj60cDnkG3nsrXMhFCsPjhcZAic2jKpF9F9TYRYQwJo81IMi6gY01RK3ZcNl8WGfqcvoZ702UIdetvR7kiaqo1czwSJCMjRFdG6EgMzgXrwE8DYMz4fWqoa1F1c1qwTCBk3yOcmGTbxsPSJK5QRyNea9IFLrBTjfE7ZlN2vZiI7adcDYJef.htm")); + + EXPECT_EQ(0xa68ac2ecc87dfd04UL, GetEntryHashKey("http://www.domain.com/uoQ76Kb2QL5hzaVOSAKWeX0W9LfDLqphmRXpsfHN8tgF5lCsfTxlOVWY8vFwzhsRzoNYKhUIOTc5TnUlT0vpdQflPyk2nh7vurXOj60cDnkG3nsrXMhFCsPjhcZAic2jKpF9F9TYRYQwJo81IMi6gY01RK3ZcNl8WGfqcvoZ702UIdetvR7kiaqo1czwSJCMjRFdG6EgMzgXrwE8DYMz4fWqoa1F1c1qwTCBk3yOcmGTbxsPSJK5QRyNea9IFLrBTjfE7ZlN2vZiI7adcDYJef.htm")); +} + +TEST_F(SimpleUtilTest, GetEntryHashKeyFromHexString) { + uint64 hash_key = 0; + EXPECT_TRUE(GetEntryHashKeyFromHexString("0000000005f5e0ff", &hash_key)); + EXPECT_EQ(99999999UL, hash_key); + + EXPECT_TRUE(GetEntryHashKeyFromHexString("7ffffffffffffffF", &hash_key)); + EXPECT_EQ(9223372036854775807UL, hash_key); + + EXPECT_TRUE(GetEntryHashKeyFromHexString("8000000000000000", &hash_key)); + EXPECT_EQ(9223372036854775808UL, hash_key); + + EXPECT_TRUE(GetEntryHashKeyFromHexString("FFFFFFFFFFFFFFFF", &hash_key)); + EXPECT_EQ(18446744073709551615UL, hash_key); + + // Wrong hash string size. + EXPECT_FALSE(GetEntryHashKeyFromHexString("FFFFFFFFFFFFFFF", &hash_key)); + + // Wrong hash string size. + EXPECT_FALSE(GetEntryHashKeyFromHexString("FFFFFFFFFFFFFFFFF", &hash_key)); + + EXPECT_FALSE(GetEntryHashKeyFromHexString("iwr8wglhg8*(&1231((", &hash_key)); +} diff --git a/net/net.gyp b/net/net.gyp index 1b7b0e8..73daffb 100644 --- a/net/net.gyp +++ b/net/net.gyp @@ -378,12 +378,14 @@ 'disk_cache/tracing_cache_backend.h', 'disk_cache/simple/simple_backend_impl.cc', 'disk_cache/simple/simple_backend_impl.h', - 'disk_cache/simple/simple_disk_format.cc', - 'disk_cache/simple/simple_disk_format.h', + 'disk_cache/simple/simple_entry_format.cc', + 'disk_cache/simple/simple_entry_format.h', 'disk_cache/simple/simple_entry_impl.cc', 'disk_cache/simple/simple_entry_impl.h', 'disk_cache/simple/simple_index.cc', 'disk_cache/simple/simple_index.h', + 'disk_cache/simple/simple_index_file.cc', + 'disk_cache/simple/simple_index_file.h', 'disk_cache/simple/simple_synchronous_entry.cc', 'disk_cache/simple/simple_synchronous_entry.h', 'disk_cache/simple/simple_util.cc', @@ -1469,6 +1471,9 @@ 'disk_cache/cache_util_unittest.cc', 'disk_cache/entry_unittest.cc', 'disk_cache/mapped_file_unittest.cc', + 'disk_cache/simple/simple_index_file_unittest.cc', + 'disk_cache/simple/simple_index_unittest.cc', + 'disk_cache/simple/simple_util_unittest.cc', 'disk_cache/storage_block_unittest.cc', 'disk_cache/flash/flash_entry_unittest.cc', 'disk_cache/flash/log_store_entry_unittest.cc', |