Merged duplicate functions from tools/data_pack into the grit/format/data_pack class.

Marked tools/data_pack directory for removal when the dependency in third_party file WebKit.gyp has been updated to point to grit.

BUG=none
TEST=builds


Review URL: http://codereview.chromium.org/7795057

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@100504 0039d316-1c4b-4281-b951-d872f2087c98

10 files changed, 114 insertions, 16 deletions

diff --git a/chrome/chrome.gyp b/chrome/chrome.gyp
index f66c7a3..2c45c1c 100644
--- a/chrome/chrome.gyp
+++ b/chrome/chrome.gyp
@@ -1224,7 +1224,7 @@
           'target_name': 'packed_extra_resources',
           'type': 'none',
           'variables': {
-            'repack_path': '../tools/data_pack/repack.py',
+            'repack_path': '../tools/grit/grit/format/repack.py',
           },
           'dependencies': [
             'chrome_extra_resources',
@@ -1588,7 +1588,7 @@
         'target_name': 'packed_resources',
         'type': 'none',
         'variables': {
-          'repack_path': '../tools/data_pack/repack.py',
+          'repack_path': '../tools/grit/grit/format/repack.py',
         },
         'dependencies': [
           # MSVS needs the dependencies explictly named, Make is able to
diff --git a/chrome/chrome_dll.gypi b/chrome/chrome_dll.gypi
index bfdd6fb..cd33233 100644
--- a/chrome/chrome_dll.gypi
+++ b/chrome/chrome_dll.gypi
@@ -338,7 +338,7 @@
                     'theme_dir_name': 'chromium',
                   }],
                 ],
-                'repack_path': '../tools/data_pack/repack.py',
+                'repack_path': '../tools/grit/grit/format/repack.py',
               },
               'actions': [
                 # TODO(mark): These actions are duplicated for Linux and
diff --git a/chrome/tools/build/repack_locales.py b/chrome/tools/build/repack_locales.py
index 0d51e6d..a1db528 100755
--- a/chrome/tools/build/repack_locales.py
+++ b/chrome/tools/build/repack_locales.py
@@ -15,8 +15,8 @@ import os
 import sys
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..',
-                             'tools', 'data_pack'))
-import repack
+                             'tools', 'grit'))
+from grit.format import data_pack
 
 # The gyp "branding" variable.
 BRANDING = None
@@ -119,7 +119,7 @@ def repack_locales(locales):
     inputs = []
     inputs += calc_inputs(locale)
     output = calc_output(locale)
-    repack.RePack(output, inputs)
+    data_pack.DataPack.RePack(output, inputs)
 
 
 def DoMain(argv):
diff --git a/tools/data_pack/data_pack.py b/tools/data_pack/data_pack.py
index ffd2cb2..93a6cfd 100755
--- a/tools/data_pack/data_pack.py
+++ b/tools/data_pack/data_pack.py
@@ -5,6 +5,9 @@
 
 """A simple utility function to produce data pack files.
 See base/pack_file* for details.
+
+TOOD(adriansc): Remove this file once the dependency has been updated in WebKit
+to point to grit scripts.
 """
 
 import struct
diff --git a/tools/data_pack/repack.py b/tools/data_pack/repack.py
index 21f568d..2729b10 100755
--- a/tools/data_pack/repack.py
+++ b/tools/data_pack/repack.py
@@ -1,10 +1,13 @@
 #!/usr/bin/python
-# Copyright (c) 2008 The Chromium Authors. All rights reserved.
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A simple utility function to merge data pack files into a single data pack.
 See base/pack_file* for details about the file format.
+
+TODO(adriansc): Remove this file once the dependency has been updated in WebKit
+to point to grit scripts.
 """
 
 import exceptions
diff --git a/tools/grit/grit/format/data_pack.py b/tools/grit/grit/format/data_pack.py
index 993ecf6..01c0c9e 100755
--- a/tools/grit/grit/format/data_pack.py
+++ b/tools/grit/grit/format/data_pack.py
@@ -7,16 +7,23 @@
 files.
 '''
 
+import exceptions
+import os
 import struct
+import sys
 
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
 from grit.format import interface
 from grit.node import include
 from grit.node import message
 from grit.node import misc
 
 
-PACK_FILE_VERSION = 3
+FILE_FORMAT_VERSION = 3
+HEADER_LENGTH = 2 * 4  # Two uint32s. (file version and number of entries)
 
+class WrongFileVersion(Exception):
+  pass
 
 class DataPack(interface.ItemFormatter):
   '''Writes out the data pack file format (platform agnostic resource file).'''
@@ -31,7 +38,7 @@ class DataPack(interface.ItemFormatter):
     for node in nodes:
       id, value = node.GetDataPackPair(lang)
       data[id] = value
-    return DataPack.WriteDataPack(data)
+    return DataPack.WriteDataPackToString(data)
 
   @staticmethod
   def GetDataNodes(item):
@@ -50,14 +57,40 @@ class DataPack(interface.ItemFormatter):
     return nodes
 
   @staticmethod
-  def WriteDataPack(resources):
+  def ReadDataPack(input_file):
+    """Reads a data pack file and returns a dictionary."""
+    data = open(input_file, "rb").read()
+    original_data = data
+
+    # Read the header.
+    version, num_entries = struct.unpack("<II", data[:HEADER_LENGTH])
+    if version != FILE_FORMAT_VERSION:
+      raise WrongFileVersion
+
+    resources = {}
+    if num_entries == 0:
+      return resources
+
+    # Read the index and data.
+    data = data[HEADER_LENGTH:]
+    kIndexEntrySize = 2 + 4  # Each entry is a uint16 and a uint32.
+    for _ in range(num_entries):
+      id, offset = struct.unpack("<HI", data[:kIndexEntrySize])
+      data = data[kIndexEntrySize:]
+      next_id, next_offset = struct.unpack("<HI", data[:kIndexEntrySize])
+      resources[id] = original_data[offset:next_offset]
+
+    return resources
+
+  @staticmethod
+  def WriteDataPackToString(resources):
     """Write a map of id=>data into a string in the data pack format and return
     it."""
     ids = sorted(resources.keys())
     ret = []
 
     # Write file header.
-    ret.append(struct.pack("<II", PACK_FILE_VERSION, len(ids)))
+    ret.append(struct.pack("<II", FILE_FORMAT_VERSION, len(ids)))
     HEADER_LENGTH = 2 * 4             # Two uint32s.
 
     # Each entry is a uint16 + a uint32s. We have one extra entry for the last
@@ -76,3 +109,39 @@ class DataPack(interface.ItemFormatter):
     for id in ids:
       ret.append(resources[id])
     return ''.join(ret)
+
+  @staticmethod
+  def WriteDataPack(resources, output_file):
+    """Write a map of id=>data into output_file as a data pack."""
+    file = open(output_file, "wb")
+    content = DataPack.WriteDataPackToString(resources)
+    file.write(content)
+
+  @staticmethod
+  def RePack(output_file, input_files):
+    """Write a new data pack to |output_file| based on a list of filenames
+    (|input_files|)"""
+    resources = {}
+    for filename in input_files:
+      new_resources = DataPack.ReadDataPack(filename)
+
+      # Make sure we have no duplicates.
+      duplicate_keys = set(new_resources.keys()) & set(resources.keys())
+      if len(duplicate_keys) != 0:
+        raise exceptions.KeyError("Duplicate keys: " +
+                                  str(list(duplicate_keys)))
+
+      resources.update(new_resources)
+
+    DataPack.WriteDataPack(resources, output_file)
+
+def main():
+  # Just write a simple file.
+  data = { 1: "", 4: "this is id 4", 6: "this is id 6", 10: "" }
+  WriteDataPack(data, "datapack1.pak")
+  data2 = { 1000: "test", 5: "five" }
+  WriteDataPack(data2, "datapack2.pak")
+  print "wrote datapack1 and datapack2 to current directory."
+
+if __name__ == '__main__':
+  main()
diff --git a/tools/grit/grit/format/data_pack_unittest.py b/tools/grit/grit/format/data_pack_unittest.py
index aa26a2f..35966639 100644
--- a/tools/grit/grit/format/data_pack_unittest.py
+++ b/tools/grit/grit/format/data_pack_unittest.py
@@ -24,10 +24,9 @@ class FormatDataPackUnittest(unittest.TestCase):
         '\x00\x00\x3e\x00\x00\x00'          # extra entry for the size of last
         'this is id 4this is id 6')         # data
     input = { 1: "", 4: "this is id 4", 6: "this is id 6", 10: "" }
-    output = data_pack.DataPack.WriteDataPack(input)
+    output = data_pack.DataPack.WriteDataPackToString(input)
     self.failUnless(output == expected)
 
 
 if __name__ == '__main__':
   unittest.main()
-
diff --git a/tools/grit/grit/format/repack.py b/tools/grit/grit/format/repack.py
new file mode 100644
index 0000000..3f312a4
--- /dev/null
+++ b/tools/grit/grit/format/repack.py
@@ -0,0 +1,24 @@
+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+A simple utility function to merge data pack files into a single data pack. See
+http://dev.chromium.org/developers/design-documents/linuxresourcesandlocalizedstrings
+for details about the file format.
+"""
+
+import sys
+
+import data_pack
+
+def main(argv):
+  if len(argv) < 3:
+    print ("Usage:\n  %s <output_filename> <input_file1> [input_file2] ... " %
+           argv[0])
+    sys.exit(-1)
+  data_pack.DataPack.RePack(argv[1], argv[2:])
+
+if '__main__' == __name__:
+  main(sys.argv)
diff --git a/ui/base/strings/ui_strings.gyp b/ui/base/strings/ui_strings.gyp
index 9af39cb..31bccce 100644
--- a/ui/base/strings/ui_strings.gyp
+++ b/ui/base/strings/ui_strings.gyp
@@ -44,7 +44,7 @@
         'target_name': 'ui_unittest_strings',
         'type': 'none',
         'variables': {
-          'repack_path': '<(DEPTH)/tools/data_pack/repack.py',
+          'repack_path': '<(DEPTH)/tools/grit/grit/format/repack.py',
         },
         'actions': [
           {
diff --git a/webkit/tools/test_shell/test_shell.gypi b/webkit/tools/test_shell/test_shell.gypi
index e241f44..01506fc 100644
--- a/webkit/tools/test_shell/test_shell.gypi
+++ b/webkit/tools/test_shell/test_shell.gypi
@@ -156,7 +156,7 @@
       'target_name': 'test_shell_pak',
       'type': 'none',
       'variables': {
-        'repack_path': '../../../tools/data_pack/repack.py',
+        'repack_path': '../../../tools/grit/grit/format/repack.py',
         'pak_path': '<(INTERMEDIATE_DIR)/repack/test_shell.pak',
       },
       'conditions': [
@@ -284,7 +284,7 @@
             '<(DEPTH)/third_party/mesa/mesa.gyp:osmesa',
           ],
           'variables': {
-            'repack_path': '../../../tools/data_pack/repack.py',
+            'repack_path': '../../../tools/grit/grit/format/repack.py',
           },
           'actions': [
             {