From 6499b9261cc94f7078e5eb4ab91cd4a3a04dfcfd Mon Sep 17 00:00:00 2001 From: "levin@chromium.org" Date: Thu, 15 Sep 2011 02:15:07 +0000 Subject: This broke lots of layout tests on OSX. See this link for the failure list: http://test-results.appspot.com/dashboards/flakiness_dashboard.html#group=@ToT - chromium.org&tests=fast/dom/HTMLTextAreaElement/reset-textarea.html,fast/dom/isindex-001.html,fast/dom/isindex-002.html,fast/forms/blankbuttons.html,fast/forms/box-shadow-override.html,fast/forms/button-default-title.html,fast/forms/file-input-direction.html,fast/forms/hidden-input-file.html,fast/forms/input-appearance-height.html,fast/forms/input-file-re-render.html,fast/forms/input-first-letter.html,fast/forms/isindex-placeholder.html,fast/forms/text-style-color.html,fast/forms/validation-message-appearance.html,fast/html/details-no-summary1.html,fast/html/details-no-summary2.html,fast/html/details-no-summary3.html,fast/html/details-no-summary4.html,fast/html/details-remove-summary-1-and-click.html,fast/html/details-remove-summary-1.html,fast/html/details-remove-summary-4-and-click.html,fast/html/details-remove-summary-4.html,fast/html/keygen.html,tables/mozilla/bugs/bug39209.html,tables/mozilla/bugs/bug4429.html,fast/forms/005.html,fast/forms/input-file-label.html,fast/invalid/residual-style.html,fast/parser/fragment-parser.html,fast/replaced/table-percent-height-text-controls.html,html5lib/runner.html,http/tests/misc/isindex-with-no-form-base-href.html,http/tests/misc/isindex-with-no-form.html Revert 101213 - Updated *.pak file format to support both UTF8 and UTF16 Inserted a new field in the header that specifies which encoding is to be used for the text resources. I also upped file format to version 4. BUG=76281 TEST=unit_tests Committed: http://src.chromium.org/viewvc/chrome?view=rev&revision=100973 Review URL: http://codereview.chromium.org/7744017 TBR=adriansc@chromium.org Review URL: http://codereview.chromium.org/7890060 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@101224 0039d316-1c4b-4281-b951-d872f2087c98 --- tools/grit/grit/format/data_pack.py | 61 +++++++++------------------- tools/grit/grit/format/data_pack_unittest.py | 16 ++++---- tools/grit/grit/node/include.py | 5 +-- tools/grit/grit/node/message.py | 18 +++----- 4 files changed, 35 insertions(+), 65 deletions(-) (limited to 'tools/grit') diff --git a/tools/grit/grit/format/data_pack.py b/tools/grit/grit/format/data_pack.py index 87db064..01c0c9e 100755 --- a/tools/grit/grit/format/data_pack.py +++ b/tools/grit/grit/format/data_pack.py @@ -19,19 +19,12 @@ from grit.node import message from grit.node import misc -PACK_FILE_VERSION = 4 -HEADER_LENGTH = 2 * 4 + 1 # Two uint32s. (file version, number of entries) and - # one uint8 (encoding of text resources) -BINARY, UTF8, UTF16 = range(3) +FILE_FORMAT_VERSION = 3 +HEADER_LENGTH = 2 * 4 # Two uint32s. (file version and number of entries) class WrongFileVersion(Exception): pass -class DataPackContents: - def __init__(self, resources, encoding): - self.resources = resources - self.encoding = encoding - class DataPack(interface.ItemFormatter): '''Writes out the data pack file format (platform agnostic resource file).''' def Format(self, item, lang='en', begin_item=True, output_dir='.'): @@ -43,9 +36,9 @@ class DataPack(interface.ItemFormatter): nodes = DataPack.GetDataNodes(item) data = {} for node in nodes: - id, value = node.GetDataPackPair(lang, UTF8) + id, value = node.GetDataPackPair(lang) data[id] = value - return DataPack.WriteDataPackToString(data, UTF8) + return DataPack.WriteDataPackToString(data) @staticmethod def GetDataNodes(item): @@ -70,15 +63,13 @@ class DataPack(interface.ItemFormatter): original_data = data # Read the header. - version, num_entries, encoding = struct.unpack("data into a string in the data pack format and return it.""" ids = sorted(resources.keys()) ret = [] # Write file header. - ret.append(struct.pack("data into output_file as a data pack.""" file = open(output_file, "wb") - content = DataPack.WriteDataPackToString(resources, encoding) + content = DataPack.WriteDataPackToString(resources) file.write(content) @staticmethod @@ -131,37 +122,25 @@ class DataPack(interface.ItemFormatter): """Write a new data pack to |output_file| based on a list of filenames (|input_files|)""" resources = {} - encoding = None for filename in input_files: - new_content = DataPack.ReadDataPack(filename) + new_resources = DataPack.ReadDataPack(filename) - # Make sure we have no dups. - duplicate_keys = set(new_content.resources.keys()) & set(resources.keys()) + # Make sure we have no duplicates. + duplicate_keys = set(new_resources.keys()) & set(resources.keys()) if len(duplicate_keys) != 0: raise exceptions.KeyError("Duplicate keys: " + str(list(duplicate_keys))) - # Make sure encoding is consistent. - if encoding in (None, BINARY): - encoding = new_content.encoding - elif new_content.encoding not in (BINARY, encoding): - raise exceptions.KeyError("Inconsistent encodings: " + - str(encoding) + " vs " + - str(new_content.encoding)) - - resources.update(new_content.resources) + resources.update(new_resources) - # Encoding is 0 for BINARY, 1 for UTF8 and 2 for UTF16 - if encoding is None: - encoding = BINARY - DataPack.WriteDataPack(resources, output_file, encoding) + DataPack.WriteDataPack(resources, output_file) def main(): # Just write a simple file. data = { 1: "", 4: "this is id 4", 6: "this is id 6", 10: "" } - DataPack.WriteDataPack(data, "datapack1.pak", UTF8) + WriteDataPack(data, "datapack1.pak") data2 = { 1000: "test", 5: "five" } - DataPack.WriteDataPack(data2, "datapack2.pak", UTF8) + WriteDataPack(data2, "datapack2.pak") print "wrote datapack1 and datapack2 to current directory." if __name__ == '__main__': diff --git a/tools/grit/grit/format/data_pack_unittest.py b/tools/grit/grit/format/data_pack_unittest.py index 8de54ef..35966639 100644 --- a/tools/grit/grit/format/data_pack_unittest.py +++ b/tools/grit/grit/format/data_pack_unittest.py @@ -16,17 +16,15 @@ from grit.format import data_pack class FormatDataPackUnittest(unittest.TestCase): def testWriteDataPack(self): expected = ( - '\x04\x00\x00\x00' # header(version - '\x04\x00\x00\x00' # no. entries, - '\x01' # encoding) - '\x01\x00\x27\x00\x00\x00' # index entry 1 - '\x04\x00\x27\x00\x00\x00' # index entry 4 - '\x06\x00\x33\x00\x00\x00' # index entry 6 - '\x0a\x00\x3f\x00\x00\x00' # index entry 10 - '\x00\x00\x3f\x00\x00\x00' # extra entry for the size of last + '\x03\x00\x00\x00\x04\x00\x00\x00' # header (version, no. entries) + '\x01\x00\x26\x00\x00\x00' # index entry 1 + '\x04\x00\x26\x00\x00\x00' # index entry 4 + '\x06\x00\x32\x00\x00\x00' # index entry 6 + '\x0a\x00\x3e\x00\x00\x00' # index entry 10 + '\x00\x00\x3e\x00\x00\x00' # extra entry for the size of last 'this is id 4this is id 6') # data input = { 1: "", 4: "this is id 4", 6: "this is id 6", 10: "" } - output = data_pack.DataPack.WriteDataPackToString(input, data_pack.UTF8) + output = data_pack.DataPack.WriteDataPackToString(input) self.failUnless(output == expected) diff --git a/tools/grit/grit/node/include.py b/tools/grit/grit/node/include.py index dc29315..b073bc7 100644 --- a/tools/grit/grit/node/include.py +++ b/tools/grit/grit/node/include.py @@ -74,7 +74,7 @@ class IncludeNode(base.Node): ''' return self.FilenameToOpen() - def GetDataPackPair(self, lang, encoding): + def GetDataPackPair(self, lang): '''Returns a (id, string) pair that represents the resource id and raw bytes of the data. This is used to generate the data pack data file. ''' @@ -90,8 +90,6 @@ class IncludeNode(base.Node): data = infile.read() infile.close() - # Include does not care about the encoding, because it only returns binary - # data. return id, data def Flatten(self, output_dir): @@ -138,3 +136,4 @@ class IncludeNode(base.Node): node.EndParsing() return node Construct = staticmethod(Construct) + diff --git a/tools/grit/grit/node/message.py b/tools/grit/grit/node/message.py index 3c5ac64..a48a645 100644 --- a/tools/grit/grit/node/message.py +++ b/tools/grit/grit/node/message.py @@ -1,5 +1,5 @@ #!/usr/bin/python2.4 -# Copyright (c) 2011 The Chromium Authors. All rights reserved. +# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -19,7 +19,6 @@ from grit import exception from grit import tclib from grit import util -BINARY, UTF8, UTF16 = range(3) # Finds whitespace at the start and end of a string which can be multiline. _WHITESPACE = re.compile('(?P\s*)(?P.+?)(?P\s*)\Z', @@ -187,7 +186,7 @@ class MessageNode(base.ContentNode): else: return self.attrs['offset'] - def GetDataPackPair(self, lang, encoding): + def GetDataPackPair(self, lang): '''Returns a (id, string) pair that represents the string id and the string in utf8. This is used to generate the data pack data file. ''' @@ -200,15 +199,10 @@ class MessageNode(base.ContentNode): # Windows automatically translates \n to a new line, but GTK+ doesn't. # Manually do the conversion here rather than at run time. message = message.replace("\\n", "\n") - # |message| is a python unicode string, so convert to a byte stream that - # has the correct encoding requested for the datapacks. We skip the first - # 2 bytes of text resources because it is the BOM. - if encoding == UTF8: - return id, message.encode('utf8') - if encoding == UTF16: - return id, message.encode('utf16')[2:] - # Default is BINARY - return id, message + # |message| is a python unicode string, so convert to a utf16 byte stream + # because that's the format of datapacks. We skip the first 2 bytes + # because it is the BOM. + return id, message.encode('utf16')[2:] # static method def Construct(parent, message, name, desc='', meaning='', translateable=True): -- cgit v1.1