Updated *.pak file format to support both UTF8 and UTF16

Inserted a new field in the header that specifies which encoding is to be used for the text resources.

I also upped file format to version 4.

BUG=76281
TEST=unit_tests

Committed: http://src.chromium.org/viewvc/chrome?view=rev&revision=100973

Committed: http://src.chromium.org/viewvc/chrome?view=rev&revision=101213

Review URL: http://codereview.chromium.org/7744017

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@101846 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 90 insertions, 21 deletions

diff --git a/ui/base/resource/data_pack.cc b/ui/base/resource/data_pack.cc
index c4b1594..5a0dcf1 100644
--- a/ui/base/resource/data_pack.cc
+++ b/ui/base/resource/data_pack.cc
@@ -17,9 +17,9 @@
 
 namespace {
 
-static const uint32 kFileFormatVersion = 3;
-// Length of file header: version and entry count.
-static const size_t kHeaderLength = 2 * sizeof(uint32);
+static const uint32 kFileFormatVersion = 4;
+// Length of file header: version, entry count and text encoding type.
+static const size_t kHeaderLength = 2 * sizeof(uint32) + sizeof(uint8);
 
 #pragma pack(push,2)
 struct DataPackEntry {
@@ -60,7 +60,7 @@ enum LoadErrors {
 namespace ui {
 
 // In .cc for MemoryMappedFile dtor.
-DataPack::DataPack() : resource_count_(0) {
+DataPack::DataPack() : resource_count_(0), text_encoding_type_(BINARY) {
 }
 DataPack::~DataPack() {
 }
@@ -83,7 +83,7 @@ bool DataPack::Load(const FilePath& path) {
   }
 
   // Parse the header of the file.
-  // First uint32: version; second: resource count.
+  // First uint32: version; second: resource count;
   const uint32* ptr = reinterpret_cast<const uint32*>(mmap_->data());
   uint32 version = ptr[0];
   if (version != kFileFormatVersion) {
@@ -96,6 +96,17 @@ bool DataPack::Load(const FilePath& path) {
   }
   resource_count_ = ptr[1];
 
+  // third: text encoding.
+  const uint8* ptr_encoding = reinterpret_cast<const uint8*>(ptr + 2);
+  text_encoding_type_ = static_cast<TextEncodingType>(*ptr_encoding);
+  if (text_encoding_type_ != UTF8 && text_encoding_type_ != UTF16 &&
+      text_encoding_type_ != BINARY) {
+    LOG(ERROR) << "Bad data pack text encoding: got " << text_encoding_type_
+               << ", expected between " << BINARY << " and " << UTF16;
+    mmap_.reset();
+    return false;
+  }
+
   // Sanity check the file.
   // 1) Check we have enough entries.
   if (kHeaderLength + resource_count_ * sizeof(DataPackEntry) >
@@ -163,7 +174,8 @@ RefCountedStaticMemory* DataPack::GetStaticMemory(uint16 resource_id) const {
 
 // static
 bool DataPack::WritePack(const FilePath& path,
-                         const std::map<uint16, base::StringPiece>& resources) {
+                         const std::map<uint16, base::StringPiece>& resources,
+                         TextEncodingType textEncodingType) {
   FILE* file = file_util::OpenFile(path, "wb");
   if (!file)
     return false;
@@ -183,6 +195,21 @@ bool DataPack::WritePack(const FilePath& path,
     return false;
   }
 
+  if (textEncodingType != UTF8 && textEncodingType != UTF16 &&
+      textEncodingType != BINARY) {
+    LOG(ERROR) << "Invalid text encoding type, got " << textEncodingType
+               << ", expected between " << BINARY << " and " << UTF16;
+    file_util::CloseFile(file);
+    return false;
+  }
+
+  uint8 write_buffer = textEncodingType;
+  if (fwrite(&write_buffer, sizeof(uint8), 1, file) != 1) {
+    LOG(ERROR) << "Failed to write file text resources encoding";
+    file_util::CloseFile(file);
+    return false;
+  }
+
   // Each entry is a uint16 + a uint32. We have an extra entry after the last
   // item so we can compute the size of the list item.
   uint32 index_length = (entry_count + 1) * sizeof(DataPackEntry);
diff --git a/ui/base/resource/data_pack.h b/ui/base/resource/data_pack.h
index f6ee78d..0cc3b31 100644
--- a/ui/base/resource/data_pack.h
+++ b/ui/base/resource/data_pack.h
@@ -31,6 +31,13 @@ namespace ui {
 
 class UI_EXPORT DataPack {
  public:
+  // What type of encoding the text resources use.
+  enum TextEncodingType {
+    BINARY,
+    UTF8,
+    UTF16
+  };
+
   DataPack();
   ~DataPack();
 
@@ -47,9 +54,16 @@ class UI_EXPORT DataPack {
   // for localization strings.
   RefCountedStaticMemory* GetStaticMemory(uint16 resource_id) const;
 
-  // Writes a pack file containing |resources| to |path|.
+  // Writes a pack file containing |resources| to |path|. If there are any
+  // text resources to be written, their encoding must already agree to the
+  // |textEncodingType| specified. If no text resources are present, please
+  // indicate BINARY.
   static bool WritePack(const FilePath& path,
-                        const std::map<uint16, base::StringPiece>& resources);
+                        const std::map<uint16, base::StringPiece>& resources,
+                        TextEncodingType textEncodingType);
+
+  // Get the encoding type of text resources.
+  TextEncodingType GetTextEncodingType() const { return text_encoding_type_; }
 
  private:
   // The memory-mapped data.
@@ -58,6 +72,9 @@ class UI_EXPORT DataPack {
   // Number of resources in the data.
   size_t resource_count_;
 
+  // Type of encoding for text resources.
+  TextEncodingType text_encoding_type_;
+
   DISALLOW_COPY_AND_ASSIGN(DataPack);
 };
 
diff --git a/ui/base/resource/data_pack_literal.cc b/ui/base/resource/data_pack_literal.cc
index 4a32fa9..510d07a 100644
--- a/ui/base/resource/data_pack_literal.cc
+++ b/ui/base/resource/data_pack_literal.cc
@@ -7,13 +7,14 @@
 namespace ui {
 
 extern const char kSamplePakContents[] = {
-    0x03, 0x00, 0x00, 0x00,               // header(version
-    0x04, 0x00, 0x00, 0x00,               //        no. entries)
-    0x01, 0x00, 0x26, 0x00, 0x00, 0x00,   // index entry 1
-    0x04, 0x00, 0x26, 0x00, 0x00, 0x00,   // index entry 4
-    0x06, 0x00, 0x32, 0x00, 0x00, 0x00,   // index entry 6
-    0x0a, 0x00, 0x3e, 0x00, 0x00, 0x00,   // index entry 10
-    0x00, 0x00, 0x3e, 0x00, 0x00, 0x00,   // extra entry for the size of last
+    0x04, 0x00, 0x00, 0x00,               // header(version
+    0x04, 0x00, 0x00, 0x00,               //        no. entries
+    0x01,                                 //        encoding)
+    0x01, 0x00, 0x27, 0x00, 0x00, 0x00,   // index entry 1
+    0x04, 0x00, 0x27, 0x00, 0x00, 0x00,   // index entry 4
+    0x06, 0x00, 0x33, 0x00, 0x00, 0x00,   // index entry 6
+    0x0a, 0x00, 0x3f, 0x00, 0x00, 0x00,   // index entry 10
+    0x00, 0x00, 0x3f, 0x00, 0x00, 0x00,   // extra entry for the size of last
     't', 'h', 'i', 's', ' ', 'i', 's', ' ', 'i', 'd', ' ', '4',
     't', 'h', 'i', 's', ' ', 'i', 's', ' ', 'i', 'd', ' ', '6'
 };
diff --git a/ui/base/resource/data_pack_unittest.cc b/ui/base/resource/data_pack_unittest.cc
index fe3613c..1c0bd7a 100644
--- a/ui/base/resource/data_pack_unittest.cc
+++ b/ui/base/resource/data_pack_unittest.cc
@@ -12,6 +12,12 @@
 
 namespace ui {
 
+class DataPackTest
+    : public testing::TestWithParam<DataPack::TextEncodingType> {
+ public:
+  DataPackTest() {}
+};
+
 extern const char kSamplePakContents[];
 extern const size_t kSamplePakSize;
 
@@ -44,6 +50,13 @@ TEST(DataPackTest, Load) {
   ASSERT_FALSE(pack.GetStringPiece(140, &data));
 }
 
+INSTANTIATE_TEST_CASE_P(WriteBINARY, DataPackTest, ::testing::Values(
+    DataPack::BINARY));
+INSTANTIATE_TEST_CASE_P(WriteUTF8, DataPackTest, ::testing::Values(
+    DataPack::UTF8));
+INSTANTIATE_TEST_CASE_P(WriteUTF16, DataPackTest, ::testing::Values(
+    DataPack::UTF16));
+
 TEST(DataPackTest, LoadFileWithTruncatedHeader) {
   FilePath data_path;
   PathService::Get(base::DIR_SOURCE_ROOT, &data_path);
@@ -54,7 +67,7 @@ TEST(DataPackTest, LoadFileWithTruncatedHeader) {
   ASSERT_FALSE(pack.Load(data_path));
 }
 
-TEST(DataPackTest, Write) {
+TEST_P(DataPackTest, Write) {
   ScopedTempDir dir;
   ASSERT_TRUE(dir.CreateUniqueTempDir());
   FilePath file = dir.path().Append(FILE_PATH_LITERAL("data.pak"));
@@ -71,11 +84,12 @@ TEST(DataPackTest, Write) {
   resources.insert(std::make_pair(15, base::StringPiece(fifteen)));
   resources.insert(std::make_pair(3, base::StringPiece(three)));
   resources.insert(std::make_pair(4, base::StringPiece(four)));
-  ASSERT_TRUE(DataPack::WritePack(file, resources));
+  ASSERT_TRUE(DataPack::WritePack(file, resources, GetParam()));
 
   // Now try to read the data back in.
   DataPack pack;
   ASSERT_TRUE(pack.Load(file));
+  EXPECT_EQ(pack.GetTextEncodingType(), GetParam());
 
   base::StringPiece data;
   ASSERT_TRUE(pack.GetStringPiece(1, &data));
diff --git a/ui/base/resource/resource_bundle.cc b/ui/base/resource/resource_bundle.cc
index 7ae0f4e..f1722ff 100644
--- a/ui/base/resource/resource_bundle.cc
+++ b/ui/base/resource/resource_bundle.cc
@@ -11,6 +11,7 @@
 #include "base/stl_util.h"
 #include "base/string_piece.h"
 #include "base/synchronization/lock.h"
+#include "base/utf_string_conversions.h"
 #include "build/build_config.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "ui/base/l10n/l10n_util.h"
@@ -187,10 +188,19 @@ string16 ResourceBundle::GetLocalizedString(int message_id) {
     }
   }
 
-  // Data pack encodes strings as UTF16.
-  DCHECK_EQ(data.length() % 2, 0U);
-  string16 msg(reinterpret_cast<const char16*>(data.data()),
-               data.length() / 2);
+  // Strings should not be loaded from a data pack that contains binary data.
+  DCHECK(locale_resources_data_->GetTextEncodingType() == DataPack::UTF16 ||
+         locale_resources_data_->GetTextEncodingType() == DataPack::UTF8)
+      << "requested localized string from binary pack file";
+
+  // Data pack encodes strings as either UTF8 or UTF16.
+  string16 msg;
+  if (locale_resources_data_->GetTextEncodingType() == DataPack::UTF16) {
+    msg = string16(reinterpret_cast<const char16*>(data.data()),
+                   data.length() / 2);
+  } else if (locale_resources_data_->GetTextEncodingType() == DataPack::UTF8) {
+    msg = UTF8ToUTF16(data);
+  }
   return msg;
 }