Add data pack file format to the grit output type and enable it for webkit_resources.grd.

This only adds support for <include> nodes.  <message> nodes are forthcoming.

The design doc describing the format is here:
http://dev.chromium.org/developers/design-documents/linuxresourcesandlocalizedstrings

I hooked into the <release> tag because we need to generate an index at the beginning of the file.  The index contains the position of all the resources, so we walk the file at that point.

This currently only handles <include> tags, but will be extended to handle <message> as well.

Review URL: http://codereview.chromium.org/18215


git-svn-id: svn://svn.chromium.org/chrome/trunk/src@8147 0039d316-1c4b-4281-b951-d872f2087c98

7 files changed, 121 insertions, 3 deletions

diff --git a/tools/grit/grit/format/data_pack.py b/tools/grit/grit/format/data_pack.py
new file mode 100644
index 0000000..1f11a9b
--- /dev/null
+++ b/tools/grit/grit/format/data_pack.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python2.4
+# Copyright (c) 2009 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+'''Support for formatting a data pack file used for platform agnostic resource
+files.
+'''
+
+import struct
+
+from grit.format import interface
+from grit.node import misc
+from grit.node import include
+
+
+PACK_FILE_VERSION = 1
+
+
+class DataPack(interface.ItemFormatter):
+  '''Writes out the data pack file format (platform agnostic resource file).'''
+  def Format(self, item, lang='en', begin_item=True, output_dir='.'):
+    if not begin_item:
+      return ''
+    assert isinstance(item, misc.ReleaseNode)
+
+    nodes = DataPack.GetDataNodes(item)
+    data = {}
+    for node in nodes:
+      id, value = node.GetDataPackPair()
+      data[id] = value
+    return DataPack.WriteDataPack(data)
+
+  @staticmethod
+  def GetDataNodes(item):
+    '''Returns a list of nodes that can be packed into the data pack file.'''
+    nodes = []
+    if isinstance(item, include.IncludeNode):
+      return [item]
+    # TODO(tc): Handle message nodes.
+    for child in item.children:
+      nodes.extend(DataPack.GetDataNodes(child))
+    return nodes
+
+  @staticmethod
+  def WriteDataPack(resources):
+    """Write a map of id=>data into a string in the data pack format and return
+    it."""
+    ids = sorted(resources.keys())
+    ret = []
+
+    # Write file header.
+    ret.append(struct.pack("<II", PACK_FILE_VERSION, len(ids)))
+    HEADER_LENGTH = 2 * 4             # Two uint32s.
+
+    index_length = len(ids) * 3 * 4   # Each entry is 3 uint32s.
+
+    # Write index.
+    data_offset = HEADER_LENGTH + index_length
+    for id in ids:
+      ret.append(struct.pack("<III", id, data_offset, len(resources[id])))
+      data_offset += len(resources[id])
+
+    # Write data.
+    for id in ids:
+      ret.append(resources[id])
+    return ''.join(ret)
diff --git a/tools/grit/grit/format/data_pack_unittest.py b/tools/grit/grit/format/data_pack_unittest.py
new file mode 100644
index 0000000..353f8c1
--- /dev/null
+++ b/tools/grit/grit/format/data_pack_unittest.py
@@ -0,0 +1,29 @@
+#!/usr/bin/python2.4
+# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+'''Unit tests for grit.format.data_pack'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+import unittest
+
+from grit.format import data_pack
+
+class FormatDataPackUnittest(unittest.TestCase):
+  def testWriteDataPack(self):
+    expected = ('\x01\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x008\x00\x00'
+        '\x00\x00\x00\x00\x00\x04\x00\x00\x008\x00\x00\x00\x0c\x00\x00\x00'
+        '\x06\x00\x00\x00D\x00\x00\x00\x0c\x00\x00\x00\n\x00\x00\x00P\x00'
+        '\x00\x00\x00\x00\x00\x00this is id 4this is id 6')
+    input = { 1: "", 4: "this is id 4", 6: "this is id 6", 10: "" }
+    output = data_pack.DataPack.WriteDataPack(input)
+    self.failUnless(output == expected)
+
+
+if __name__ == '__main__':
+  unittest.main()
+
diff --git a/tools/grit/grit/node/include.py b/tools/grit/grit/node/include.py
index 67a05ac..d50e7a6 100644
--- a/tools/grit/grit/node/include.py
+++ b/tools/grit/grit/node/include.py
@@ -46,6 +46,18 @@ class IncludeNode(base.Node):
     '''
     return self.FilenameToOpen()
 
+  def GetDataPackPair(self):
+    '''Returns a (id, string) pair that represents the resource id and raw
+    bytes of the data.  This is used to generate the data pack data file.
+    '''
+    from grit.format import rc_header
+    id_map = rc_header.Item.tids_
+    id = id_map[self.GetTextualIds()[0]]
+    file = open(self.FilenameToOpen(), 'rb')
+    data = file.read()
+    file.close()
+    return id, data
+
   # static method
   def Construct(parent, name, type, file, translateable=True, 
       filenameonly=False, relativepath=False):
diff --git a/tools/grit/grit/node/misc.py b/tools/grit/grit/node/misc.py
index 418633f..27f8b66 100644
--- a/tools/grit/grit/node/misc.py
+++ b/tools/grit/grit/node/misc.py
@@ -72,6 +72,12 @@ class ReleaseNode(base.Node):
     '''Returns the sequence number of this release.'''
     return self.attribs['seq']
 
+  def ItemFormatter(self, t):
+    if t == 'data_package':
+      from grit.format import data_pack
+      return data_pack.DataPack()
+    else:
+      return super(type(self), self).ItemFormatter(t)
 
 class GritNode(base.Node):
   '''The <grit> root element.'''
diff --git a/tools/grit/grit/test_suite_all.py b/tools/grit/grit/test_suite_all.py
index 943ac07..a94b160 100644
--- a/tools/grit/grit/test_suite_all.py
+++ b/tools/grit/grit/test_suite_all.py
@@ -33,6 +33,7 @@ class TestSuiteAll(unittest.TestSuite):
     from grit.node import message_unittest
     from grit import tclib_unittest
     import grit.format.rc_unittest
+    import grit.format.data_pack_unittest
     from grit.tool import rc2grd_unittest
     from grit.tool import transl2tc_unittest
     from grit.gather import txt_unittest
@@ -58,6 +59,7 @@ class TestSuiteAll(unittest.TestSuite):
       message_unittest.MessageUnittest,
       tclib_unittest.TclibUnittest,
       grit.format.rc_unittest.FormatRcUnittest,
+      grit.format.data_pack_unittest.FormatDataPackUnittest,
       rc2grd_unittest.Rc2GrdUnittest,
       transl2tc_unittest.TranslationToTcUnittest,
       txt_unittest.TxtUnittest,
diff --git a/tools/grit/grit/tool/build.py b/tools/grit/grit/tool/build.py
index 810c80b..30d2fb3 100644
--- a/tools/grit/grit/tool/build.py
+++ b/tools/grit/grit/tool/build.py
@@ -171,9 +171,10 @@ are exported to translation interchange files (e.g. XMB files), etc.
           oldname = None
       else:
         encoding = 'utf_16'
-      outfile = util.WrapOutputStream(
-        self.fo_create(output.GetOutputFilename(), 'wb'),
-        encoding)
+      outfile = self.fo_create(output.GetOutputFilename(), 'wb')
+
+      if output.GetType() != 'data_package':
+        outfile = util.WrapOutputStream(outfile, encoding)
       
       # Set the context, for conditional inclusion of resources
       self.res.SetOutputContext(output.GetLanguage(), self.defines)
diff --git a/webkit/glue/webkit_resources.grd b/webkit/glue/webkit_resources.grd
index 83a506c..da3d833 100644
--- a/webkit/glue/webkit_resources.grd
+++ b/webkit/glue/webkit_resources.grd
@@ -5,6 +5,7 @@
       <emit emit_type='prepend'></emit>
     </output>
     <output filename="webkit_resources.rc" type="rc_all" />
+    <output filename="webkit_resources.pak" type="data_package" />
   </outputs>
   <release seq="1">
     <includes>