diff options
| author | initial.commit <initial.commit@0039d316-1c4b-4281-b951-d872f2087c98> | 2008-07-27 00:12:16 +0000 |
|---|---|---|
| committer | initial.commit <initial.commit@0039d316-1c4b-4281-b951-d872f2087c98> | 2008-07-27 00:12:16 +0000 |
| commit | 920c091ac3ee15079194c82ae8a7a18215f3f23c (patch) | |
| tree | d28515d1e7732e2b6d077df1b4855ace3f4ac84f /tools/grit | |
| parent | ae2c20f398933a9e86c387dcc465ec0f71065ffc (diff) | |
| download | chromium_src-920c091ac3ee15079194c82ae8a7a18215f3f23c.zip chromium_src-920c091ac3ee15079194c82ae8a7a18215f3f23c.tar.gz chromium_src-920c091ac3ee15079194c82ae8a7a18215f3f23c.tar.bz2 | |
Add tools to the repository.
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@17 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools/grit')
86 files changed, 13634 insertions, 0 deletions
diff --git a/tools/grit/README b/tools/grit/README new file mode 100644 index 0000000..8fcdafe --- /dev/null +++ b/tools/grit/README @@ -0,0 +1,2 @@ +GRIT (Google Resource and Internationalization Tool) is a tool for Windows +projects to manage resources and simplify the localization workflow. diff --git a/tools/grit/build/grit_resource_file.bat b/tools/grit/build/grit_resource_file.bat new file mode 100644 index 0000000..3be0231 --- /dev/null +++ b/tools/grit/build/grit_resource_file.bat @@ -0,0 +1,15 @@ +:: Batch file run as build command for .grd files +:: The custom build rule is set to expect (inputfile).h and (inputfile).rc +:: our grd files must generate files with the same basename. +@echo off + +setlocal + +set InFile=%~1 +set SolutionDir=%~2 +set InputDir=%~3 + +:: Use GNU tools +call %SolutionDir%\..\third_party\gnu\setup_env.bat + +%SolutionDir%\..\third_party\python_24\python.exe %SolutionDir%\..\tools\grit\grit.py -i %InFile% build -o %InputDir% diff --git a/tools/grit/build/grit_resource_file.rules b/tools/grit/build/grit_resource_file.rules new file mode 100644 index 0000000..5ab70f5 --- /dev/null +++ b/tools/grit/build/grit_resource_file.rules @@ -0,0 +1,103 @@ +<?xml version="1.0" encoding="utf-8"?> +<VisualStudioToolFile + Name="GRIT resource data file" + Version="8.00" + > + <Rules> + <CustomBuildRule + Name="GRIT Generated Resources" + DisplayName="GRIT Generated Resources" + CommandLine="$(SolutionDir)..\tools\grit\build\grit_resource_file.bat [inputs] "$(SolutionDir)" "$(IntDir)"" + Outputs="$(IntDir)\$(InputName).h; + $(IntDir)\$(InputName)_ar.rc; + $(IntDir)\$(InputName)_bg.rc; + $(IntDir)\$(InputName)_ca.rc; + $(IntDir)\$(InputName)_cs.rc; + $(IntDir)\$(InputName)_da.rc; + $(IntDir)\$(InputName)_de.rc; + $(IntDir)\$(InputName)_el.rc; + $(IntDir)\$(InputName)_en-GB.rc; + $(IntDir)\$(InputName)_en-US.rc; + $(IntDir)\$(InputName)_es.rc; + $(IntDir)\$(InputName)_es-419.rc; + $(IntDir)\$(InputName)_et.rc; + $(IntDir)\$(InputName)_fi.rc; + $(IntDir)\$(InputName)_fil.rc; + $(IntDir)\$(InputName)_fr.rc; + $(IntDir)\$(InputName)_he.rc; + $(IntDir)\$(InputName)_hi.rc; + $(IntDir)\$(InputName)_hr.rc; + $(IntDir)\$(InputName)_hu.rc; + $(IntDir)\$(InputName)_id.rc; + $(IntDir)\$(InputName)_it.rc; + $(IntDir)\$(InputName)_ja.rc; + $(IntDir)\$(InputName)_ko.rc; + $(IntDir)\$(InputName)_lt.rc; + $(IntDir)\$(InputName)_lv.rc; + $(IntDir)\$(InputName)_nl.rc; + $(IntDir)\$(InputName)_nb.rc; + $(IntDir)\$(InputName)_pl.rc; + $(IntDir)\$(InputName)_pt-BR.rc; + $(IntDir)\$(InputName)_pt-PT.rc; + $(IntDir)\$(InputName)_ro.rc; + $(IntDir)\$(InputName)_ru.rc; + $(IntDir)\$(InputName)_sk.rc; + $(IntDir)\$(InputName)_sl.rc; + $(IntDir)\$(InputName)_sr.rc; + $(IntDir)\$(InputName)_sv.rc; + $(IntDir)\$(InputName)_th.rc; + $(IntDir)\$(InputName)_tr.rc; + $(IntDir)\$(InputName)_uk.rc; + $(IntDir)\$(InputName)_vi.rc; + $(IntDir)\$(InputName)_zh-CN.rc; + $(IntDir)\$(InputName)_zh-TW.rc;" + AdditionalDependencies="$(SolutionDir)..\tools\grit\build\grit_resource_file.bat;$(SolutionDir)..\tools\grit\grit.py; + resources\$(InputName)_ar.xtb; + resources\$(InputName)_bg.xtb; + resources\$(InputName)_ca.xtb; + resources\$(InputName)_cs.xtb; + resources\$(InputName)_da.xtb; + resources\$(InputName)_de.xtb; + resources\$(InputName)_el.xtb; + resources\$(InputName)_en-GB.xtb; + resources\$(InputName)_es.xtb; + resources\$(InputName)_es-419.xtb; + resources\$(InputName)_et.xtb; + resources\$(InputName)_fi.xtb; + resources\$(InputName)_fil.xtb; + resources\$(InputName)_fr.xtb; + resources\$(InputName)_he.xtb; + resources\$(InputName)_hi.xtb; + resources\$(InputName)_hr.xtb; + resources\$(InputName)_hu.xtb; + resources\$(InputName)_id.xtb; + resources\$(InputName)_it.xtb; + resources\$(InputName)_ja.xtb; + resources\$(InputName)_ko.xtb; + resources\$(InputName)_lt.xtb; + resources\$(InputName)_lv.xtb; + resources\$(InputName)_nl.xtb; + resources\$(InputName)_no.xtb; + resources\$(InputName)_pl.xtb; + resources\$(InputName)_pt-BR.xtb; + resources\$(InputName)_pt-PT.xtb; + resources\$(InputName)_ro.xtb; + resources\$(InputName)_ru.xtb; + resources\$(InputName)_sk.xtb; + resources\$(InputName)_sl.xtb; + resources\$(InputName)_sr.xtb; + resources\$(InputName)_sv.xtb; + resources\$(InputName)_th.xtb; + resources\$(InputName)_tr.xtb; + resources\$(InputName)_uk.xtb; + resources\$(InputName)_vi.xtb; + resources\$(InputName)_zh-CN.xtb; + resources\$(InputName)_zh-TW.xtb;" + FileExtensions="*.grd" + ExecutionDescription="Generating resources..." + > + <Properties> + </Properties> + </CustomBuildRule> + </Rules> +</VisualStudioToolFile> diff --git a/tools/grit/grit.py b/tools/grit/grit.py new file mode 100644 index 0000000..ed74c23 --- /dev/null +++ b/tools/grit/grit.py @@ -0,0 +1,40 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Bootstrapping for GRIT. +''' + +import sys + +import grit.grit_runner + + +if __name__ == '__main__': + grit.grit_runner.Main(sys.argv[1:]) diff --git a/tools/grit/grit/__init__.py b/tools/grit/grit/__init__.py new file mode 100644 index 0000000..3fb788f --- /dev/null +++ b/tools/grit/grit/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Package 'grit' +''' + +pass diff --git a/tools/grit/grit/clique.py b/tools/grit/grit/clique.py new file mode 100644 index 0000000..0b9d84b --- /dev/null +++ b/tools/grit/grit/clique.py @@ -0,0 +1,467 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Collections of messages and their translations, called cliques. Also +collections of cliques (uber-cliques). +''' + +import types + +from grit import constants +from grit import exception +from grit import pseudo +from grit import tclib + + +class UberClique(object): + '''A factory (NOT a singleton factory) for making cliques. It has several + methods for working with the cliques created using the factory. + ''' + + def __init__(self): + # A map from message ID to list of cliques whose source messages have + # that ID. This will contain all cliques created using this factory. + # Different messages can have the same ID because they have the + # same translateable portion and placeholder names, but occur in different + # places in the resource tree. + self.cliques_ = {} + + # A map of clique IDs to list of languages to indicate translations where we + # fell back to English. + self.fallback_translations_ = {} + + # A map of clique IDs to list of languages to indicate missing translations. + self.missing_translations_ = {} + + def _AddMissingTranslation(self, lang, clique, is_error): + tl = self.fallback_translations_ + if is_error: + tl = self.missing_translations_ + id = clique.GetId() + if id not in tl: + tl[id] = {} + if lang not in tl[id]: + tl[id][lang] = 1 + + def HasMissingTranslations(self): + return len(self.missing_translations_) > 0 + + def MissingTranslationsReport(self): + '''Returns a string suitable for printing to report missing + and fallback translations to the user. + ''' + def ReportTranslation(clique, langs): + text = clique.GetMessage().GetPresentableContent() + extract = text[0:40] + ellipsis = '' + if len(text) > 40: + ellipsis = '...' + langs_extract = langs[0:6] + describe_langs = ','.join(langs_extract) + if len(langs) > 6: + describe_langs += " and %d more" % (len(langs) - 6) + return " %s \"%s%s\" %s" % (clique.GetId(), extract, ellipsis, + describe_langs) + lines = [] + if len(self.fallback_translations_): + lines.append( + "WARNING: Fell back to English for the following translations:") + for (id, langs) in self.fallback_translations_.items(): + lines.append(ReportTranslation(self.cliques_[id][0], langs.keys())) + if len(self.missing_translations_): + lines.append("ERROR: The following translations are MISSING:") + for (id, langs) in self.missing_translations_.items(): + lines.append(ReportTranslation(self.cliques_[id][0], langs.keys())) + return '\n'.join(lines) + + def MakeClique(self, message, translateable=True): + '''Create a new clique initialized with a message. + + Args: + message: tclib.Message() + translateable: True | False + ''' + clique = MessageClique(self, message, translateable) + + # Enable others to find this clique by its message ID + if message.GetId() in self.cliques_: + presentable_text = clique.GetMessage().GetPresentableContent() + for c in self.cliques_[message.GetId()]: + assert c.GetMessage().GetPresentableContent() == presentable_text + self.cliques_[message.GetId()].append(clique) + else: + self.cliques_[message.GetId()] = [clique] + + return clique + + def FindCliqueAndAddTranslation(self, translation, language): + '''Adds the specified translation to the clique with the source message + it is a translation of. + + Args: + translation: tclib.Translation() + language: 'en' | 'fr' ... + + Return: + True if the source message was found, otherwise false. + ''' + if translation.GetId() in self.cliques_: + for clique in self.cliques_[translation.GetId()]: + clique.AddTranslation(translation, language) + return True + else: + return False + + def BestClique(self, id): + '''Returns the "best" clique from a list of cliques. All the cliques + must have the same ID. The "best" clique is chosen in the following + order of preference: + - The first clique that has a non-ID-based description + - If no such clique found, one of the cliques with an ID-based description + - Otherwise an arbitrary clique + ''' + clique_list = self.cliques_[id] + clique_to_ret = None + for clique in clique_list: + if not clique_to_ret: + clique_to_ret = clique + + description = clique.GetMessage().GetDescription() + if description and len(description) > 0: + clique_to_ret = clique + if not description.startswith('ID:'): + break # this is the preferred case so we exit right away + return clique_to_ret + + def BestCliquePerId(self): + '''Iterates over the list of all cliques and returns the best clique for + each ID. This will be the first clique with a source message that has a + non-empty description, or an arbitrary clique if none of them has a + description. + ''' + for id in self.cliques_: + yield self.BestClique(id) + + def BestCliqueByOriginalText(self, text, meaning): + '''Finds the "best" (as in BestClique()) clique that has original text + 'text' and meaning 'meaning'. Returns None if there is no such clique. + ''' + # If needed, this can be optimized by maintaining a map of + # fingerprints of original text+meaning to cliques. + for c in self.BestCliquePerId(): + msg = c.GetMessage() + if msg.GetRealContent() == text and msg.GetMeaning() == meaning: + return msg + return None + + def AllMessageIds(self): + '''Returns a list of all defined message IDs. + ''' + return self.cliques_.keys() + + def AllCliques(self): + '''Iterates over all cliques. Note that this can return multiple cliques + with the same ID. + ''' + for cliques in self.cliques_.values(): + for c in cliques: + yield c + + def GenerateXtbParserCallback(self, lang, debug=False): + '''Creates a callback function as required by grit.xtb_reader.Parse(). + This callback will create Translation objects for each message from + the XTB that exists in this uberclique, and add them as translations for + the relevant cliques. The callback will add translations to the language + specified by 'lang' + + Args: + lang: 'fr' + debug: True | False + ''' + def Callback(id, structure): + if id not in self.cliques_: + if debug: print "Ignoring translation #%s" % id + return + + if debug: print "Adding translation #%s" % id + + # We fetch placeholder information from the original message (the XTB file + # only contains placeholder names). + original_msg = self.BestClique(id).GetMessage() + + translation = tclib.Translation(id=id) + for is_ph,text in structure: + if not is_ph: + translation.AppendText(text) + else: + found_placeholder = False + for ph in original_msg.GetPlaceholders(): + if ph.GetPresentation() == text: + translation.AppendPlaceholder(tclib.Placeholder( + ph.GetPresentation(), ph.GetOriginal(), ph.GetExample())) + found_placeholder = True + break + if not found_placeholder: + raise exception.MismatchingPlaceholders( + 'Translation for message ID %s had <ph name="%s%/>, no match\n' + 'in original message' % (id, text)) + self.FindCliqueAndAddTranslation(translation, lang) + return Callback + + +class CustomType(object): + '''A base class you should implement if you wish to specify a custom type + for a message clique (i.e. custom validation and optional modification of + translations).''' + + def Validate(self, message): + '''Returns true if the message (a tclib.Message object) is valid, + otherwise false. + ''' + raise NotImplementedError() + + def ValidateAndModify(self, lang, translation): + '''Returns true if the translation (a tclib.Translation object) is valid, + otherwise false. The language is also passed in. This method may modify + the translation that is passed in, if it so wishes. + ''' + raise NotImplementedError() + + def ModifyTextPart(self, lang, text): + '''If you call ModifyEachTextPart, it will turn around and call this method + for each text part of the translation. You should return the modified + version of the text, or just the original text to not change anything. + ''' + raise NotImplementedError() + + def ModifyEachTextPart(self, lang, translation): + '''Call this to easily modify one or more of the textual parts of a + translation. It will call ModifyTextPart for each part of the + translation. + ''' + contents = translation.GetContent() + for ix in range(len(contents)): + if (isinstance(contents[ix], types.StringTypes)): + contents[ix] = self.ModifyTextPart(lang, contents[ix]) + + +class OneOffCustomType(CustomType): + '''A very simple custom type that performs the validation expressed by + the input expression on all languages including the source language. + The expression can access the variables 'lang', 'msg' and 'text()' where 'lang' + is the language of 'msg', 'msg' is the message or translation being + validated and 'text()' returns the real contents of 'msg' (for shorthand). + ''' + def __init__(self, expression): + self.expr = expression + def Validate(self, message): + return self.ValidateAndModify(MessageClique.source_language, message) + def ValidateAndModify(self, lang, msg): + def text(): + return msg.GetRealContent() + return eval(self.expr, {}, + {'lang' : lang, + 'text' : text, + 'msg' : msg, + }) + + +class MessageClique(object): + '''A message along with all of its translations. Also code to bring + translations together with their original message.''' + + # change this to the language code of Messages you add to cliques_. + # TODO(joi) Actually change this based on the <grit> node's source language + source_language = 'en' + + # A constant translation we use when asked for a translation into the + # special language constants.CONSTANT_LANGUAGE. + CONSTANT_TRANSLATION = tclib.Translation(text='TTTTTT') + + def __init__(self, uber_clique, message, translateable=True, custom_type=None): + '''Create a new clique initialized with just a message. + + Args: + uber_clique: Our uber-clique (collection of cliques) + message: tclib.Message() + translateable: True | False + custom_type: instance of clique.CustomType interface + ''' + # Our parent + self.uber_clique = uber_clique + # If not translateable, we only store the original message. + self.translateable = translateable + # A mapping of language identifiers to tclib.BaseMessage and its + # subclasses (i.e. tclib.Message and tclib.Translation). + self.clique = { MessageClique.source_language : message } + # A list of the "shortcut groups" this clique is + # part of. Within any given shortcut group, no shortcut key (e.g. &J) + # must appear more than once in each language for all cliques that + # belong to the group. + self.shortcut_groups = [] + # An instance of the CustomType interface, or None. If this is set, it will + # be used to validate the original message and translations thereof, and + # will also get a chance to modify translations of the message. + self.SetCustomType(custom_type) + + def GetMessage(self): + '''Retrieves the tclib.Message that is the source for this clique.''' + return self.clique[MessageClique.source_language] + + def GetId(self): + '''Retrieves the message ID of the messages in this clique.''' + return self.GetMessage().GetId() + + def IsTranslateable(self): + return self.translateable + + def AddToShortcutGroup(self, group): + self.shortcut_groups.append(group) + + def SetCustomType(self, custom_type): + '''Makes this clique use custom_type for validating messages and + translations, and optionally modifying translations. + ''' + self.custom_type = custom_type + if custom_type and not custom_type.Validate(self.GetMessage()): + raise exception.InvalidMessage(self.GetMessage().GetRealContent()) + + def MessageForLanguage(self, lang, pseudo_if_no_match=True, fallback_to_english=False): + '''Returns the message/translation for the specified language, providing + a pseudotranslation if there is no available translation and a pseudo- + translation is requested. + + The translation of any message whatsoever in the special language + 'x_constant' is the message "TTTTTT". + + Args: + lang: 'en' + pseudo_if_no_match: True + fallback_to_english: False + + Return: + tclib.BaseMessage + ''' + if not self.translateable: + return self.GetMessage() + + if lang == constants.CONSTANT_LANGUAGE: + return self.CONSTANT_TRANSLATION + + for msglang in self.clique.keys(): + if lang == msglang: + return self.clique[msglang] + + if fallback_to_english: + self.uber_clique._AddMissingTranslation(lang, self, is_error=False) + return self.GetMessage() + + # If we're not supposed to generate pseudotranslations, we add an error + # report to a list of errors, then fail at a higher level, so that we + # get a list of all messages that are missing translations. + if not pseudo_if_no_match: + self.uber_clique._AddMissingTranslation(lang, self, is_error=True) + + return pseudo.PseudoMessage(self.GetMessage()) + + def AllMessagesThatMatch(self, lang_re, include_pseudo = True): + '''Returns a map of all messages that match 'lang', including the pseudo + translation if requested. + + Args: + lang_re: re.compile('fr|en') + include_pseudo: True + + Return: + { 'en' : tclib.Message, + 'fr' : tclib.Translation, + pseudo.PSEUDO_LANG : tclib.Translation } + ''' + if not self.translateable: + return [self.GetMessage()] + + matches = {} + for msglang in self.clique: + if lang_re.match(msglang): + matches[msglang] = self.clique[msglang] + + if include_pseudo: + matches[pseudo.PSEUDO_LANG] = pseudo.PseudoMessage(self.GetMessage()) + + return matches + + def AddTranslation(self, translation, language): + '''Add a translation to this clique. The translation must have the same + ID as the message that is the source for this clique. + + If this clique is not translateable, the function just returns. + + Args: + translation: tclib.Translation() + language: 'en' + + Throws: + grit.exception.InvalidTranslation if the translation you're trying to add + doesn't have the same message ID as the source message of this clique. + ''' + if not self.translateable: + return + if translation.GetId() != self.GetId(): + raise exception.InvalidTranslation( + 'Msg ID %s, transl ID %s' % (self.GetId(), translation.GetId())) + + assert not language in self.clique + + # Because two messages can differ in the original content of their + # placeholders yet share the same ID (because they are otherwise the + # same), the translation we are getting may have different original + # content for placeholders than our message, yet it is still the right + # translation for our message (because it is for the same ID). We must + # therefore fetch the original content of placeholders from our original + # English message. + # + # See grit.clique_unittest.MessageCliqueUnittest.testSemiIdenticalCliques + # for a concrete explanation of why this is necessary. + + original = self.MessageForLanguage(self.source_language, False) + if len(original.GetPlaceholders()) != len(translation.GetPlaceholders()): + print ("ERROR: '%s' translation of message id %s does not match" % + (language, translation.GetId())) + assert False + + transl_msg = tclib.Translation(id=self.GetId(), + text=translation.GetPresentableContent(), + placeholders=original.GetPlaceholders()) + + if self.custom_type and not self.custom_type.ValidateAndModify(language, transl_msg): + print "WARNING: %s translation failed validation: %s" % ( + language, transl_msg.GetId()) + + self.clique[language] = transl_msg diff --git a/tools/grit/grit/clique_unittest.py b/tools/grit/grit/clique_unittest.py new file mode 100644 index 0000000..e36969c --- /dev/null +++ b/tools/grit/grit/clique_unittest.py @@ -0,0 +1,228 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.clique''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import re +import StringIO +import unittest + +from grit import clique +from grit import exception +from grit import pseudo +from grit import tclib +from grit import grd_reader +from grit import util + +class MessageCliqueUnittest(unittest.TestCase): + def testClique(self): + factory = clique.UberClique() + msg = tclib.Message(text='Hello USERNAME, how are you?', + placeholders=[ + tclib.Placeholder('USERNAME', '%s', 'Joi')]) + c = factory.MakeClique(msg) + + self.failUnless(c.GetMessage() == msg) + self.failUnless(c.GetId() == msg.GetId()) + + msg_fr = tclib.Translation(text='Bonjour USERNAME, comment ca va?', + id=msg.GetId(), placeholders=[ + tclib.Placeholder('USERNAME', '%s', 'Joi')]) + msg_de = tclib.Translation(text='Guten tag USERNAME, wie geht es dir?', + id=msg.GetId(), placeholders=[ + tclib.Placeholder('USERNAME', '%s', 'Joi')]) + + c.AddTranslation(msg_fr, 'fr') + factory.FindCliqueAndAddTranslation(msg_de, 'de') + + # sort() sorts lists in-place and does not return them + for lang in ('en', 'fr', 'de'): + self.failUnless(lang in c.clique) + + self.failUnless(c.MessageForLanguage('fr').GetRealContent() == + msg_fr.GetRealContent()) + + try: + c.MessageForLanguage('zh-CN', False) + self.fail('Should have gotten exception') + except: + pass + + self.failUnless(c.MessageForLanguage('zh-CN', True) != None) + + rex = re.compile('fr|de|bingo') + self.failUnless(len(c.AllMessagesThatMatch(rex, False)) == 2) + self.failUnless(c.AllMessagesThatMatch(rex, True)[pseudo.PSEUDO_LANG] != None) + + def testBestClique(self): + factory = clique.UberClique() + factory.MakeClique(tclib.Message(text='Alfur', description='alfaholl')) + factory.MakeClique(tclib.Message(text='Alfur', description='')) + factory.MakeClique(tclib.Message(text='Vaettur', description='')) + factory.MakeClique(tclib.Message(text='Vaettur', description='')) + factory.MakeClique(tclib.Message(text='Troll', description='')) + factory.MakeClique(tclib.Message(text='Gryla', description='ID: IDS_GRYLA')) + factory.MakeClique(tclib.Message(text='Gryla', description='vondakerling')) + factory.MakeClique(tclib.Message(text='Leppaludi', description='ID: IDS_LL')) + factory.MakeClique(tclib.Message(text='Leppaludi', description='')) + + count_best_cliques = 0 + for c in factory.BestCliquePerId(): + count_best_cliques += 1 + msg = c.GetMessage() + text = msg.GetRealContent() + description = msg.GetDescription() + if text == 'Alfur': + self.failUnless(description == 'alfaholl') + elif text == 'Gryla': + self.failUnless(description == 'vondakerling') + elif text == 'Leppaludi': + self.failUnless(description == 'ID: IDS_LL') + self.failUnless(count_best_cliques == 5) + + def testAllInUberClique(self): + resources = grd_reader.Parse(util.WrapInputStream( + StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + </messages> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="grit/test/data/klonk.rc" /> + <structure type="tr_html" name="ID_HTML" file="grit/test/data/simple.html" /> + </structures> + </release> +</grit>''')), util.PathFromRoot('.')) + resources.RunGatherers(True) + content_list = [] + for clique_list in resources.UberClique().cliques_.values(): + for clique in clique_list: + content_list.append(clique.GetMessage().GetRealContent()) + self.failUnless('Hello %s, how are you doing today?' in content_list) + self.failUnless('Jack "Black" Daniels' in content_list) + self.failUnless('Hello!' in content_list) + + def testCorrectExceptionIfWrongEncodingOnResourceFile(self): + '''This doesn't really belong in this unittest file, but what the heck.''' + resources = grd_reader.Parse(util.WrapInputStream( + StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" file="grit/test/data/klonk.rc" /> + </structures> + </release> +</grit>''')), util.PathFromRoot('.')) + self.assertRaises(exception.SectionNotFound, resources.RunGatherers, True) + + def testSemiIdenticalCliques(self): + messages = [ + tclib.Message(text='Hello USERNAME', + placeholders=[tclib.Placeholder('USERNAME', '$1', 'Joi')]), + tclib.Message(text='Hello USERNAME', + placeholders=[tclib.Placeholder('USERNAME', '%s', 'Joi')]), + ] + self.failUnless(messages[0].GetId() == messages[1].GetId()) + + # Both of the above would share a translation. + translation = tclib.Translation(id=messages[0].GetId(), + text='Bonjour USERNAME', + placeholders=[tclib.Placeholder( + 'USERNAME', '$1', 'Joi')]) + + factory = clique.UberClique() + cliques = [factory.MakeClique(msg) for msg in messages] + + for clq in cliques: + clq.AddTranslation(translation, 'fr') + + self.failUnless(cliques[0].MessageForLanguage('fr').GetRealContent() == + 'Bonjour $1') + self.failUnless(cliques[1].MessageForLanguage('fr').GetRealContent() == + 'Bonjour %s') + + def testMissingTranslations(self): + messages = [ tclib.Message(text='Hello'), tclib.Message(text='Goodbye') ] + factory = clique.UberClique() + cliques = [factory.MakeClique(msg) for msg in messages] + + cliques[1].MessageForLanguage('fr', False, True) + + self.failUnless(not factory.HasMissingTranslations()) + + cliques[0].MessageForLanguage('de', False, False) + + self.failUnless(factory.HasMissingTranslations()) + + report = factory.MissingTranslationsReport() + self.failUnless(report.count('WARNING') == 1) + self.failUnless(report.count('8053599568341804890 "Goodbye" fr') == 1) + self.failUnless(report.count('ERROR') == 1) + self.failUnless(report.count('800120468867715734 "Hello" de') == 1) + + def testCustomTypes(self): + factory = clique.UberClique() + message = tclib.Message(text='Bingo bongo') + c = factory.MakeClique(message) + try: + c.SetCustomType(DummyCustomType()) + self.fail() + except: + pass # expected case - 'Bingo bongo' does not start with 'jjj' + + message = tclib.Message(text='jjjBingo bongo') + c = factory.MakeClique(message) + c.SetCustomType(util.NewClassInstance( + 'grit.clique_unittest.DummyCustomType', clique.CustomType)) + translation = tclib.Translation(id=message.GetId(), text='Bilingo bolongo') + c.AddTranslation(translation, 'fr') + self.failUnless(c.MessageForLanguage('fr').GetRealContent().startswith('jjj')) + + +class DummyCustomType(clique.CustomType): + def Validate(self, message): + return message.GetRealContent().startswith('jjj') + def ValidateAndModify(self, lang, translation): + is_ok = self.Validate(translation) + self.ModifyEachTextPart(lang, translation) + def ModifyTextPart(self, lang, text): + return 'jjj%s' % text + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/constants.py b/tools/grit/grit/constants.py new file mode 100644 index 0000000..5f59883 --- /dev/null +++ b/tools/grit/grit/constants.py @@ -0,0 +1,45 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Constant definitions for GRIT. +''' + + +# This is the Icelandic noun meaning "grit" and is used to check that our +# input files are in the correct encoding. The middle character gets encoded +# as two bytes in UTF-8, so this is sufficient to detect incorrect encoding. +ENCODING_CHECK = u'm\u00f6l' + +# A special language, translations into which are always "TTTTTT". +CONSTANT_LANGUAGE = 'x_constant' + +# The Unicode byte-order-marker character (this is the Unicode code point, +# not the encoding of that character into any particular Unicode encoding). +BOM = u"\ufeff" diff --git a/tools/grit/grit/exception.py b/tools/grit/grit/exception.py new file mode 100644 index 0000000..d5a9c16 --- /dev/null +++ b/tools/grit/grit/exception.py @@ -0,0 +1,177 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Exception types for GRIT. +''' + +class Base(Exception): + '''A base exception that uses the class's docstring in addition to any + user-provided message as the body of the Base. + ''' + def __init__(self, msg=''): + if len(msg): + if self.__doc__: + msg = self.__doc__ + ': ' + msg + else: + msg = self.__doc__ + Exception.__init__(self, msg) + + +class Parsing(Base): + '''An error occurred parsing a GRD or XTB file.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class UnknownElement(Parsing): + '''An unknown node type was encountered.''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class MissingElement(Parsing): + '''An expected element was missing.''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class UnexpectedChild(Parsing): + '''An unexpected child element was encountered (on a leaf node).''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class UnexpectedAttribute(Parsing): + '''The attribute was not expected''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class UnexpectedContent(Parsing): + '''This element should not have content''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class MissingMandatoryAttribute(Parsing): + '''This element is missing a mandatory attribute''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class MutuallyExclusiveMandatoryAttribute(Parsing): + '''This element has 2 mutually exclusive mandatory attributes''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class DuplicateKey(Parsing): + '''A duplicate key attribute was found.''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class TooManyExamples(Parsing): + '''Only one <ex> element is allowed for each <ph> element.''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class GotPathExpectedFilenameOnly(Parsing): + '''The 'filename' attribute of an <output> node must not be a path, only + a filename. + ''' + def __init__(self, msg=''): + Parsing.__init__(self, msg) + + +class InvalidMessage(Base): + '''The specified message failed validation.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class InvalidTranslation(Base): + '''Attempt to add an invalid translation to a clique.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class NoSuchTranslation(Base): + '''Requested translation not available''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class NotReady(Base): + '''Attempt to use an object before it is ready, or attempt to translate + an empty document.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class TooManyPlaceholders(Base): + '''Too many placeholders for elements of the same type.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class MismatchingPlaceholders(Base): + '''Placeholders do not match.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class InvalidPlaceholderName(Base): + '''Placeholder name can only contain A-Z, a-z, 0-9 and underscore.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class BlockTagInTranslateableChunk(Base): + '''A block tag was encountered where it wasn't expected.''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class SectionNotFound(Base): + '''The section you requested was not found in the RC file. Make +sure the section ID is correct (matches the section's ID in the RC file). +Also note that you may need to specify the RC file's encoding (using the +encoding="" attribute) if it is not in the default Windows-1252 encoding. + ''' + def __init__(self, msg=''): + Base.__init__(self, msg) + + +class IdRangeOverlap(Base): + '''ID range overlap.''' + def __init__(self, msg=''): + Base.__init__(self, msg) diff --git a/tools/grit/grit/extern/FP.py b/tools/grit/grit/extern/FP.py new file mode 100644 index 0000000..d6704b6 --- /dev/null +++ b/tools/grit/grit/extern/FP.py @@ -0,0 +1,54 @@ +#!/usr/bin/python2.2 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import md5 + +"""64-bit fingerprint support for strings. + +Usage: + from extern import FP + print 'Fingerprint is %ld' % FP.FingerPrint('Hello world!') +""" + + +def UnsignedFingerPrint(str, encoding='utf-8'): + """Generate a 64-bit fingerprint by taking the first half of the md5 + of the string.""" + hex128 = md5.new(str).hexdigest() + int64 = long(hex128[:16], 16) + return int64 + +def FingerPrint(str, encoding='utf-8'): + fp = UnsignedFingerPrint(str, encoding=encoding) + # interpret fingerprint as signed longs + if fp & 0x8000000000000000L: + fp = - ((~fp & 0xFFFFFFFFFFFFFFFFL) + 1) + return fp +
\ No newline at end of file diff --git a/tools/grit/grit/extern/__init__.py b/tools/grit/grit/extern/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tools/grit/grit/extern/__init__.py diff --git a/tools/grit/grit/extern/tclib.py b/tools/grit/grit/extern/tclib.py new file mode 100644 index 0000000..901c792 --- /dev/null +++ b/tools/grit/grit/extern/tclib.py @@ -0,0 +1,527 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The tclib module contains tools for aggregating, verifying, and storing +# messages destined for the Translation Console, as well as for reading +# translations back and outputting them in some desired format. +# +# This has been stripped down to include only the functionality needed by grit +# for creating Windows .rc and .h files. These are the only parts needed by +# the Chrome build process. + +import exceptions + +from grit.extern import FP + +# This module assumes that within a bundle no two messages can have the +# same id unless they're identical. + +# The basic classes defined here for external use are Message and Translation, +# where the former is used for English messages and the latter for +# translations. These classes have a lot of common functionality, as expressed +# by the common parent class BaseMessage. Perhaps the most important +# distinction is that translated text is stored in UTF-8, whereas original text +# is stored in whatever encoding the client uses (presumably Latin-1). + +# -------------------- +# The public interface +# -------------------- + +# Generate message id from message text and meaning string (optional), +# both in utf-8 encoding +# +def GenerateMessageId(message, meaning=''): + fp = FP.FingerPrint(message) + if meaning: + # combine the fingerprints of message and meaning + fp2 = FP.FingerPrint(meaning) + if fp < 0: + fp = fp2 + (fp << 1) + 1 + else: + fp = fp2 + (fp << 1) + # To avoid negative ids we strip the high-order bit + return str(fp & 0x7fffffffffffffffL) + +# ------------------------------------------------------------------------- +# The MessageTranslationError class is used to signal tclib-specific errors. + +class MessageTranslationError(exceptions.Exception): + def __init__(self, args = ''): + self.args = args + + +# ----------------------------------------------------------- +# The Placeholder class represents a placeholder in a message. + +class Placeholder(object): + # String representation + def __str__(self): + return '%s, "%s", "%s"' % \ + (self.__presentation, self.__original, self.__example) + + # Getters + def GetOriginal(self): + return self.__original + + def GetPresentation(self): + return self.__presentation + + def GetExample(self): + return self.__example + + def __eq__(self, other): + return self.EqualTo(other, strict=1, ignore_trailing_spaces=0) + + # Equality test + # + # ignore_trailing_spaces: TC is using varchar to store the + # phrwr fields, as a result of that, the trailing spaces + # are removed by MySQL when the strings are stored into TC:-( + # ignore_trailing_spaces parameter is used to ignore + # trailing spaces during equivalence comparison. + # + def EqualTo(self, other, strict = 1, ignore_trailing_spaces = 1): + if type(other) is not Placeholder: + return 0 + if StringEquals(self.__presentation, other.__presentation, + ignore_trailing_spaces): + if not strict or (StringEquals(self.__original, other.__original, + ignore_trailing_spaces) and + StringEquals(self.__example, other.__example, + ignore_trailing_spaces)): + return 1 + return 0 + + +# ----------------------------------------------------------------- +# BaseMessage is the common parent class of Message and Translation. +# It is not meant for direct use. + +class BaseMessage(object): + # Three types of message construction is supported. If the message text is a + # simple string with no dynamic content, you can pass it to the constructor + # as the "text" parameter. Otherwise, you can omit "text" and assemble the + # message step by step using AppendText() and AppendPlaceholder(). Or, as an + # alternative, you can give the constructor the "presentable" version of the + # message and a list of placeholders; it will then parse the presentation and + # build the message accordingly. For example: + # Message(text = "There are NUM_BUGS bugs in your code", + # placeholders = [Placeholder("NUM_BUGS", "%d", "33")], + # description = "Bla bla bla") + def __eq__(self, other): + # "source encoding" is nonsense, so ignore it + return _ObjectEquals(self, other, ['_BaseMessage__source_encoding']) + + def GetName(self): + return self.__name + + def GetSourceEncoding(self): + return self.__source_encoding + + # Append a placeholder to the message + def AppendPlaceholder(self, placeholder): + if not isinstance(placeholder, Placeholder): + raise MessageTranslationError, ("Invalid message placeholder %s in " + "message %s" % (placeholder, self.GetId())) + # Are there other placeholders with the same presentation? + # If so, they need to be the same. + for other in self.GetPlaceholders(): + if placeholder.GetPresentation() == other.GetPresentation(): + if not placeholder.EqualTo(other): + raise MessageTranslationError, \ + "Conflicting declarations of %s within message" % \ + placeholder.GetPresentation() + # update placeholder list + dup = 0 + for item in self.__content: + if isinstance(item, Placeholder) and placeholder.EqualTo(item): + dup = 1 + break + if not dup: + self.__placeholders.append(placeholder) + + # update content + self.__content.append(placeholder) + + # Strips leading and trailing whitespace, and returns a tuple + # containing the leading and trailing space that was removed. + def Strip(self): + leading = trailing = '' + if len(self.__content) > 0: + s0 = self.__content[0] + if not isinstance(s0, Placeholder): + s = s0.lstrip() + leading = s0[:-len(s)] + self.__content[0] = s + + s0 = self.__content[-1] + if not isinstance(s0, Placeholder): + s = s0.rstrip() + trailing = s0[len(s):] + self.__content[-1] = s + return leading, trailing + + # Return the id of this message + def GetId(self): + if self.__id is None: + return self.GenerateId() + return self.__id + + # Set the id of this message + def SetId(self, id): + if id is None: + self.__id = None + else: + self.__id = str(id) # Treat numerical ids as strings + + # Return content of this message as a list (internal use only) + def GetContent(self): + return self.__content + + # Return a human-readable version of this message + def GetPresentableContent(self): + presentable_content = "" + for item in self.__content: + if isinstance(item, Placeholder): + presentable_content += item.GetPresentation() + else: + presentable_content += item + + return presentable_content + + # Return a fragment of a message in escaped format + def EscapeFragment(self, fragment): + return fragment.replace('%', '%%') + + # Return the "original" version of this message, doing %-escaping + # properly. If source_msg is specified, the placeholder original + # information inside source_msg will be used instead. + def GetOriginalContent(self, source_msg = None): + original_content = "" + for item in self.__content: + if isinstance(item, Placeholder): + if source_msg: + ph = source_msg.GetPlaceholder(item.GetPresentation()) + if not ph: + raise MessageTranslationError, \ + "Placeholder %s doesn't exist in message: %s" % \ + (item.GetPresentation(), source_msg); + original_content += ph.GetOriginal() + else: + original_content += item.GetOriginal() + else: + original_content += self.EscapeFragment(item) + return original_content + + # Return the example of this message + def GetExampleContent(self): + example_content = "" + for item in self.__content: + if isinstance(item, Placeholder): + example_content += item.GetExample() + else: + example_content += item + return example_content + + # Return a list of all unique placeholders in this message + def GetPlaceholders(self): + return self.__placeholders + + # Return a placeholder in this message + def GetPlaceholder(self, presentation): + for item in self.__content: + if (isinstance(item, Placeholder) and + item.GetPresentation() == presentation): + return item + return None + + # Return this message's description + def GetDescription(self): + return self.__description + + # Add a message source + def AddSource(self, source): + self.__sources.append(source) + + # Return this message's sources as a list + def GetSources(self): + return self.__sources + + # Return this message's sources as a string + def GetSourcesAsText(self, delimiter = "; "): + return delimiter.join(self.__sources) + + # Set the obsolete flag for a message (internal use only) + def SetObsolete(self): + self.__obsolete = 1 + + # Get the obsolete flag for a message (internal use only) + def IsObsolete(self): + return self.__obsolete + + # Get the sequence number (0 by default) + def GetSequenceNumber(self): + return self.__sequence_number + + # Set the sequence number + def SetSequenceNumber(self, number): + self.__sequence_number = number + + # Increment instance counter + def AddInstance(self): + self.__num_instances += 1 + + # Return instance count + def GetNumInstances(self): + return self.__num_instances + + def GetErrors(self, from_tc=0): + """ + Returns a description of the problem if the message is not + syntactically valid, or None if everything is fine. + + Args: + from_tc: indicates whether this message came from the TC. We let + the TC get away with some things we normally wouldn't allow for + historical reasons. + """ + # check that placeholders are unambiguous + pos = 0 + phs = {} + for item in self.__content: + if isinstance(item, Placeholder): + phs[pos] = item + pos += len(item.GetPresentation()) + else: + pos += len(item) + presentation = self.GetPresentableContent() + for ph in self.GetPlaceholders(): + for pos in FindOverlapping(presentation, ph.GetPresentation()): + # message contains the same text as a placeholder presentation + other_ph = phs.get(pos) + if ((not other_ph + and not IsSubstringInPlaceholder(pos, len(ph.GetPresentation()), phs)) + or + (other_ph and len(other_ph.GetPresentation()) < len(ph.GetPresentation()))): + return "message contains placeholder name '%s':\n%s" % ( + ph.GetPresentation(), presentation) + return None + + + def __CopyTo(self, other): + """ + Returns a copy of this BaseMessage. + """ + assert isinstance(other, self.__class__) or isinstance(self, other.__class__) + other.__source_encoding = self.__source_encoding + other.__content = self.__content[:] + other.__description = self.__description + other.__id = self.__id + other.__num_instances = self.__num_instances + other.__obsolete = self.__obsolete + other.__name = self.__name + other.__placeholders = self.__placeholders[:] + other.__sequence_number = self.__sequence_number + other.__sources = self.__sources[:] + + return other + + def HasText(self): + """Returns true iff this message has anything other than placeholders.""" + for item in self.__content: + if not isinstance(item, Placeholder): + return True + return False + +# -------------------------------------------------------- +# The Message class represents original (English) messages + +class Message(BaseMessage): + # See BaseMessage constructor + def __init__(self, source_encoding, text=None, id=None, + description=None, meaning="", placeholders=None, + source=None, sequence_number=0, clone_from=None, + time_created=0, name=None, is_hidden = 0): + + if clone_from is not None: + BaseMessage.__init__(self, None, clone_from=clone_from) + self.__meaning = clone_from.__meaning + self.__time_created = clone_from.__time_created + self.__is_hidden = clone_from.__is_hidden + return + + BaseMessage.__init__(self, source_encoding, text, id, description, + placeholders, source, sequence_number, + name=name) + self.__meaning = meaning + self.__time_created = time_created + self.SetIsHidden(is_hidden) + + # String representation + def __str__(self): + s = 'source: %s, id: %s, content: "%s", meaning: "%s", ' \ + 'description: "%s"' % \ + (self.GetSourcesAsText(), self.GetId(), self.GetPresentableContent(), + self.__meaning, self.GetDescription()) + if self.GetName() is not None: + s += ', name: "%s"' % self.GetName() + placeholders = self.GetPlaceholders() + for i in range(len(placeholders)): + s += ", placeholder[%d]: %s" % (i, placeholders[i]) + return s + + # Strips leading and trailing whitespace, and returns a tuple + # containing the leading and trailing space that was removed. + def Strip(self): + leading = trailing = '' + content = self.GetContent() + if len(content) > 0: + s0 = content[0] + if not isinstance(s0, Placeholder): + s = s0.lstrip() + leading = s0[:-len(s)] + content[0] = s + + s0 = content[-1] + if not isinstance(s0, Placeholder): + s = s0.rstrip() + trailing = s0[len(s):] + content[-1] = s + return leading, trailing + + # Generate an id by hashing message content + def GenerateId(self): + self.SetId(GenerateMessageId(self.GetPresentableContent(), + self.__meaning)) + return self.GetId() + + def GetMeaning(self): + return self.__meaning + + def GetTimeCreated(self): + return self.__time_created + + # Equality operator + def EqualTo(self, other, strict = 1): + # Check id, meaning, content + if self.GetId() != other.GetId(): + return 0 + if self.__meaning != other.__meaning: + return 0 + if self.GetPresentableContent() != other.GetPresentableContent(): + return 0 + # Check descriptions if comparison is strict + if (strict and + self.GetDescription() is not None and + other.GetDescription() is not None and + self.GetDescription() != other.GetDescription()): + return 0 + # Check placeholders + ph1 = self.GetPlaceholders() + ph2 = other.GetPlaceholders() + if len(ph1) != len(ph2): + return 0 + for i in range(len(ph1)): + if not ph1[i].EqualTo(ph2[i], strict): + return 0 + + return 1 + + def Copy(self): + """ + Returns a copy of this Message. + """ + assert isinstance(self, Message) + return Message(None, clone_from=self) + + def SetIsHidden(self, is_hidden): + """Sets whether this message should be hidden. + + Args: + is_hidden : 0 or 1 - if the message should be hidden, 0 otherwise + """ + if is_hidden not in [0, 1]: + raise MessageTranslationError, "is_hidden must be 0 or 1, got %s" + self.__is_hidden = is_hidden + + def IsHidden(self): + """Returns 1 if this message is hidden, and 0 otherwise.""" + return self.__is_hidden + +# ---------------------------------------------------- +# The Translation class represents translated messages + +class Translation(BaseMessage): + # See BaseMessage constructor + def __init__(self, source_encoding, text=None, id=None, + description=None, placeholders=None, source=None, + sequence_number=0, clone_from=None, ignore_ph_errors=0, + name=None): + if clone_from is not None: + BaseMessage.__init__(self, None, clone_from=clone_from) + return + + BaseMessage.__init__(self, source_encoding, text, id, description, + placeholders, source, sequence_number, + ignore_ph_errors=ignore_ph_errors, name=name) + + # String representation + def __str__(self): + s = 'source: %s, id: %s, content: "%s", description: "%s"' % \ + (self.GetSourcesAsText(), self.GetId(), self.GetPresentableContent(), + self.GetDescription()); + placeholders = self.GetPlaceholders() + for i in range(len(placeholders)): + s += ", placeholder[%d]: %s" % (i, placeholders[i]) + return s + + # Equality operator + def EqualTo(self, other, strict=1): + # Check id and content + if self.GetId() != other.GetId(): + return 0 + if self.GetPresentableContent() != other.GetPresentableContent(): + return 0 + # Check placeholders + ph1 = self.GetPlaceholders() + ph2 = other.GetPlaceholders() + if len(ph1) != len(ph2): + return 0 + for i in range(len(ph1)): + if not ph1[i].EqualTo(ph2[i], strict): + return 0 + + return 1 + + def Copy(self): + """ + Returns a copy of this Translation. + """ + return Translation(None, clone_from=self) diff --git a/tools/grit/grit/format/__init__.py b/tools/grit/grit/format/__init__.py new file mode 100644 index 0000000..e811820 --- /dev/null +++ b/tools/grit/grit/format/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Module grit.format +''' + +pass diff --git a/tools/grit/grit/format/interface.py b/tools/grit/grit/format/interface.py new file mode 100644 index 0000000..33104dc --- /dev/null +++ b/tools/grit/grit/format/interface.py @@ -0,0 +1,56 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Base classes for item formatters and file formatters. +''' + + +import re + + +class ItemFormatter(object): + '''Base class for a formatter that knows how to format a single item.''' + + def Format(self, item, lang='', begin_item=True, output_dir='.'): + '''Returns a Unicode string representing 'item' in the format known by this + item formatter, for the language 'lang'. May be called once at the + start of the item (begin_item == True) and again at the end + (begin_item == False), or only at the start of the item (begin_item == True) + + Args: + item: anything + lang: 'en' + begin_item: True | False + output_dir: '.' + + Return: + u'hello' + ''' + raise NotImplementedError() diff --git a/tools/grit/grit/format/rc.py b/tools/grit/grit/format/rc.py new file mode 100644 index 0000000..219a9a0 --- /dev/null +++ b/tools/grit/grit/format/rc.py @@ -0,0 +1,456 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Support for formatting an RC file for compilation. +''' + +import os +import types +import re + +from grit import util +from grit.format import interface + + +# Matches all different types of linebreaks. +_LINEBREAKS = re.compile('\r\n|\n|\r') + +''' +This dictionary defines the langauge charset pair lookup table, which is used +for replacing the GRIT expand variables for language info in Product Version +resource. The key is the language ISO country code, and the value +is the language and character-set pair, which is a hexadecimal string +consisting of the concatenation of the language and character-set identifiers. +The first 4 digit of the value is the hex value of LCID, the remaining +4 digits is the hex value of character-set id(code page)of the language. + +We have defined three GRIT expand_variables to be used in the version resource +file to set the language info. Here is an example how they should be used in +the VS_VERSION_INFO section of the resource file to allow GRIT to localize +the language info correctly according to product locale. + +VS_VERSION_INFO VERSIONINFO +... +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "[GRITVERLANGCHARSETHEX]" + BEGIN + ... + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", [GRITVERLANGID], [GRITVERCHARSETID] + END +END + +''' + +_LANGUAGE_CHARSET_PAIR = { + 'ar' : '040104e8', + 'fi' : '040b04e4', + 'ko' : '041203b5', + 'es' : '040a04e4', + 'bg' : '040204e3', + 'fr' : '040c04e4', + 'lv' : '042604e9', + 'sv' : '041d04e4', + 'ca' : '040304e4', + 'de' : '040704e4', + 'lt' : '042704e9', + 'tl' : '0c0004b0', # no lcid for tl(Tagalog), use default custom locale + 'zh-CN' : '080403a8', + 'el' : '040804e5', + 'no' : '041404e4', + 'th' : '041e036a', + 'zh-TW' : '040403b6', + 'iw' : '040d04e7', + 'pl' : '041504e2', + 'tr' : '041f04e6', + 'hr' : '041a04e4', + 'hi' : '043904b0', # no codepage for hindi, use unicode(1200) + 'pt-BR' : '041604e4', + 'uk' : '042204e3', + 'cs' : '040504e2', + 'hu' : '040e04e2', + 'ro' : '041804e2', + 'ur' : '042004b0', # no codepage for urdu, use unicode(1200) + 'da' : '040604e4', + 'is' : '040f04e4', + 'ru' : '041904e3', + 'vi' : '042a04ea', + 'nl' : '041304e4', + 'id' : '042104e4', + 'sr' : '081a04e2', + 'en-GB' : '0809040e', + 'it' : '041004e4', + 'sk' : '041b04e2', + 'et' : '042504e9', + 'ja' : '041103a4', + 'sl' : '042404e2', + 'en' : '040904b0', +} + +_LANGUAGE_DIRECTIVE_PAIR = { + 'ar' : 'LANG_ARABIC, SUBLANG_DEFAULT', + 'fi' : 'LANG_FINNISH, SUBLANG_DEFAULT', + 'ko' : 'LANG_KOREAN, SUBLANG_KOREAN', + 'es' : 'LANG_SPANISH, SUBLANG_SPANISH_MODERN', + 'bg' : 'LANG_BULGARIAN, SUBLANG_DEFAULT', + 'fr' : 'LANG_FRENCH, SUBLANG_FRENCH', + 'lv' : 'LANG_LATVIAN, SUBLANG_DEFAULT', + 'sv' : 'LANG_SWEDISH, SUBLANG_SWEDISH', + 'ca' : 'LANG_CATALAN, SUBLANG_DEFAULT', + 'de' : 'LANG_GERMAN, SUBLANG_GERMAN', + 'lt' : 'LANG_LITHUANIAN, SUBLANG_LITHUANIAN', + 'tl' : 'LANG_NEUTRAL, SUBLANG_DEFAULT', + 'zh-CN' : 'LANG_CHINESE, SUBLANG_CHINESE_SIMPLIFIED', + 'el' : 'LANG_GREEK, SUBLANG_DEFAULT', + 'no' : 'LANG_NORWEGIAN, SUBLANG_DEFAULT', + 'th' : 'LANG_THAI, SUBLANG_DEFAULT', + 'zh-TW' : 'LANG_CHINESE, SUBLANG_CHINESE_TRADITIONAL', + 'iw' : 'LANG_HEBREW, SUBLANG_DEFAULT', + 'pl' : 'LANG_POLISH, SUBLANG_DEFAULT', + 'tr' : 'LANG_TURKISH, SUBLANG_DEFAULT', + 'hr' : 'LANG_CROATIAN, SUBLANG_DEFAULT', + 'hi' : 'LANG_HINDI, SUBLANG_DEFAULT', + 'pt-BR' : 'LANG_PORTUGUESE, SUBLANG_DEFAULT', + 'uk' : 'LANG_UKRAINIAN, SUBLANG_DEFAULT', + 'cs' : 'LANG_CZECH, SUBLANG_DEFAULT', + 'hu' : 'LANG_HUNGARIAN, SUBLANG_DEFAULT', + 'ro' : 'LANG_ROMANIAN, SUBLANG_DEFAULT', + 'ur' : 'LANG_URDU, SUBLANG_DEFAULT', + 'da' : 'LANG_DANISH, SUBLANG_DEFAULT', + 'is' : 'LANG_ICELANDIC, SUBLANG_DEFAULT', + 'ru' : 'LANG_RUSSIAN, SUBLANG_DEFAULT', + 'vi' : 'LANG_VIETNAMESE, SUBLANG_DEFAULT', + 'nl' : 'LANG_DUTCH, SUBLANG_DEFAULT', + 'id' : 'LANG_INDONESIAN, SUBLANG_DEFAULT', + 'sr' : 'LANG_SERBIAN, SUBLANG_SERBIAN_CYRILLIC', + 'en-GB' : 'LANG_ENGLISH, SUBLANG_ENGLISH_UK', + 'it' : 'LANG_ITALIAN, SUBLANG_DEFAULT', + 'sk' : 'LANG_SLOVAK, SUBLANG_DEFAULT', + 'et' : 'LANG_ESTONIAN, SUBLANG_DEFAULT', + 'ja' : 'LANG_JAPANESE, SUBLANG_DEFAULT', + 'sl' : 'LANG_SLOVENIAN, SUBLANG_DEFAULT', + 'en' : 'LANG_ENGLISH, SUBLANG_ENGLISH_US', +} + +def GetLangCharsetPair(language) : + if _LANGUAGE_CHARSET_PAIR.has_key(language) : + return _LANGUAGE_CHARSET_PAIR[language] + else : + print 'Warning:GetLangCharsetPair() found undefined language %s' %(language) + return '' + +def GetLangDirectivePair(language) : + if _LANGUAGE_DIRECTIVE_PAIR.has_key(language) : + return _LANGUAGE_DIRECTIVE_PAIR[language] + else : + print 'Warning:GetLangDirectivePair() found undefined language %s' %(language) + return 'unknown language: see tools/grit/format/rc.py' + +def GetLangIdHex(language) : + if _LANGUAGE_CHARSET_PAIR.has_key(language) : + langcharset = _LANGUAGE_CHARSET_PAIR[language] + lang_id = '0x' + langcharset[0:4] + return lang_id + else : + print 'Warning:GetLangIdHex() found undefined language %s' %(language) + return '' + + +def GetCharsetIdDecimal(language) : + if _LANGUAGE_CHARSET_PAIR.has_key(language) : + langcharset = _LANGUAGE_CHARSET_PAIR[language] + charset_decimal = int(langcharset[4:], 16) + return str(charset_decimal) + else : + print 'Warning:GetCharsetIdDecimal() found undefined language %s' %(language) + return '' + + +def GetUnifiedLangCode(language) : + r = re.compile('([a-z]{1,2})_([a-z]{1,2})') + if r.match(language) : + underscore = language.find('_') + return language[0:underscore] + '-' + language[underscore + 1:].upper() + else : + return language + + +def _MakeRelativePath(base_path, path_to_make_relative): + '''Returns a relative path such from the base_path to + the path_to_make_relative. + + In other words, os.join(base_path, + MakeRelativePath(base_path, path_to_make_relative)) + is the same location as path_to_make_relative. + + Args: + base_path: the root path + path_to_make_relative: an absolute path that is on the same drive + as base_path + ''' + + def _GetPathAfterPrefix(prefix_path, path_with_prefix): + '''Gets the subpath within in prefix_path for the path_with_prefix + with no beginning or trailing path separators. + + Args: + prefix_path: the base path + path_with_prefix: a path that starts with prefix_path + ''' + assert path_with_prefix.startswith(prefix_path) + path_without_prefix = path_with_prefix[len(prefix_path):] + normalized_path = os.path.normpath(path_without_prefix.strip(os.path.sep)) + if normalized_path == '.': + normalized_path = '' + return normalized_path + + def _GetCommonBaseDirectory(*args): + '''Returns the common prefix directory for the given paths + + Args: + The list of paths (at least one of which should be a directory) + ''' + prefix = os.path.commonprefix(args) + # prefix is a character-by-character prefix (i.e. it does not end + # on a directory bound, so this code fixes that) + + # if the prefix ends with the separator, then it is prefect. + if len(prefix) > 0 and prefix[-1] == os.path.sep: + return prefix + + # We need to loop through all paths or else we can get + # tripped up by "c:\a" and "c:\abc". The common prefix + # is "c:\a" which is a directory and looks good with + # respect to the first directory but it is clear that + # isn't a common directory when the second path is + # examined. + for path in args: + assert len(path) >= len(prefix) + # If the prefix the same length as the path, + # then the prefix must be a directory (since one + # of the arguements should be a directory). + if path == prefix: + continue + # if the character after the prefix in the path + # is the separator, then the prefix appears to be a + # valid a directory as well for the given path + if path[len(prefix)] == os.path.sep: + continue + # Otherwise, the prefix is not a directory, so it needs + # to be shortened to be one + index_sep = prefix.rfind(os.path.sep) + # The use "index_sep + 1" because it includes the final sep + # and it handles the case when the index_sep is -1 as well + prefix = prefix[:index_sep + 1] + # At this point we backed up to a directory bound which is + # common to all paths, so we can quit going through all of + # the paths. + break + return prefix + + prefix = _GetCommonBaseDirectory(base_path, path_to_make_relative) + # If the paths had no commonality at all, then return the absolute path + # because it is the best that can be done. If the path had to be relative + # then eventually this absolute path will be discovered (when a build breaks) + # and an appropriate fix can be made, but having this allows for the best + # backward compatibility with the absolute path behavior in the past. + if len(prefix) <= 0: + return path_to_make_relative + # Build a path from the base dir to the common prefix + remaining_base_path = _GetPathAfterPrefix(prefix, base_path) + + # The follow handles two case: "" and "foo\\bar" + path_pieces = remaining_base_path.split(os.path.sep) + base_depth_from_prefix = len([d for d in path_pieces if len(d)]) + base_to_prefix = (".." + os.path.sep) * base_depth_from_prefix + + # Put add in the path from the prefix to the path_to_make_relative + remaining_other_path = _GetPathAfterPrefix(prefix, path_to_make_relative) + return base_to_prefix + remaining_other_path + + +class TopLevel(interface.ItemFormatter): + '''Writes out the required preamble for RC files.''' + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + assert isinstance(lang, types.StringTypes) + if not begin_item: + return '' + else: + # Find the location of the resource header file, so that we can include + # it. + resource_header = 'resource.h' # fall back to this + language_directive = '' + for child in item.GetRoot().children: + if child.name == 'outputs': + for output in child.children: + if output.attrs['type'] == 'rc_header': + resource_header = os.path.abspath(output.GetOutputFilename()) + resource_header = _MakeRelativePath(output_dir, resource_header) + if output.attrs['lang'] != lang: + continue + if output.attrs['language_section'] == '': + # If no language_section is requested, no directive is added + # (Used when the generated rc will be included from another rc + # file that will have the appropriate language directive) + language_directive = '' + elif output.attrs['language_section'] == 'neutral': + # If a neutral language section is requested (default), add a + # neutral language directive + language_directive = 'LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL' + elif output.attrs['language_section'] == 'lang': + language_directive = 'LANGUAGE %s' % GetLangDirectivePair(lang) + resource_header = resource_header.replace('\\', '\\\\') + return '''// Copyright (c) Google Inc. %d +// All rights reserved. +// This file is automatically generated by GRIT. Do not edit. + +#include "%s" +#include <winres.h> +#include <winresrc.h> + +%s + + +''' % (util.GetCurrentYear(), resource_header, language_directive) +# end Format() function + + + +class StringTable(interface.ItemFormatter): + '''Surrounds a collection of string messages with the required begin and + end blocks to declare a string table.''' + + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + assert isinstance(lang, types.StringTypes) + if begin_item: + return 'STRINGTABLE\nBEGIN\n' + else: + return 'END\n\n' + + +class Message(interface.ItemFormatter): + '''Writes out a single message to a string table.''' + + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + from grit.node import message + if not begin_item: + return '' + + assert isinstance(lang, types.StringTypes) + assert isinstance(item, message.MessageNode) + + message = item.ws_at_start + item.Translate(lang) + item.ws_at_end + # Escape quotation marks (RC format uses doubling-up + message = message.replace('"', '""') + # Replace linebreaks with a \n escape + message = _LINEBREAKS.sub(r'\\n', message) + + name_attr = item.GetTextualIds()[0] + + return ' %-15s "%s"\n' % (name_attr, message) + + +class RcSection(interface.ItemFormatter): + '''Writes out an .rc file section.''' + + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + if not begin_item: + return '' + + assert isinstance(lang, types.StringTypes) + from grit.node import structure + assert isinstance(item, structure.StructureNode) + + if item.IsExcludedFromRc(): + return '' + else: + text = item.gatherer.Translate( + lang, skeleton_gatherer=item.GetSkeletonGatherer(), + pseudo_if_not_available=item.PseudoIsAllowed(), + fallback_to_english=item.ShouldFallbackToEnglish()) + '\n\n' + + # Replace the language expand_variables in version rc info. + unified_lang_code = GetUnifiedLangCode(lang) + text = text.replace('[GRITVERLANGCHARSETHEX]', + GetLangCharsetPair(unified_lang_code)) + text = text.replace('[GRITVERLANGID]', GetLangIdHex(unified_lang_code)) + text = text.replace('[GRITVERCHARSETID]', + GetCharsetIdDecimal(unified_lang_code)) + + return text + + +class RcInclude(interface.ItemFormatter): + '''Writes out an item that is included in an .rc file (e.g. an ICON)''' + + def __init__(self, type, filenameWithoutPath = 0, relative_path = 0): + '''Indicates to the instance what the type of the resource include is, + e.g. 'ICON' or 'HTML'. Case must be correct, i.e. if the type is all-caps + the parameter should be all-caps. + + Args: + type: 'ICON' + ''' + self.type_ = type + self.filenameWithoutPath = filenameWithoutPath + self.relative_path_ = relative_path + + def Format(self, item, lang='en', begin_item=True, output_dir='.'): + if not begin_item: + return '' + + assert isinstance(lang, types.StringTypes) + from grit.node import structure + from grit.node import include + assert isinstance(item, (structure.StructureNode, include.IncludeNode)) + assert (isinstance(item, include.IncludeNode) or + item.attrs['type'] in ['tr_html', 'admin_template', 'txt', 'muppet']) + + # By default, we use relative pathnames to included resources so that + # sharing the resulting .rc files is possible. + # + # The FileForLanguage() Function has the side effect of generating the file + # if needed (e.g. if it is an HTML file include). + filename = os.path.abspath(item.FileForLanguage(lang, output_dir)) + if self.filenameWithoutPath: + filename = os.path.basename(filename) + elif self.relative_path_: + filename = _MakeRelativePath(output_dir, filename) + filename = filename.replace('\\', '\\\\') # escape for the RC format + + if isinstance(item, structure.StructureNode) and item.IsExcludedFromRc(): + return '' + else: + return '%-18s %-18s "%s"\n' % (item.attrs['name'], self.type_, filename) diff --git a/tools/grit/grit/format/rc_header.py b/tools/grit/grit/format/rc_header.py new file mode 100644 index 0000000..63010b2 --- /dev/null +++ b/tools/grit/grit/format/rc_header.py @@ -0,0 +1,207 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Item formatters for RC headers. +''' + +import re +import time + +from grit.format import interface +from grit import exception +from grit import util + +from grit.extern import FP + + +class TopLevel(interface.ItemFormatter): + '''Writes the necessary preamble for a resource.h file.''' + + def Format(self, item, lang='', begin_item=True, output_dir='.'): + print 'TopLevel formatter called for item %s, begin %s' % (item, begin_item) + if not begin_item: + return '' + else: + header_string = '''// Copyright (c) Google Inc. %d +// All rights reserved. +// This file is automatically generated by GRIT. Do not edit. +// Built on %s + +#pragma once +''' % (util.GetCurrentYear(), time.asctime()) + # Check for emit nodes under the rc_header. If any emit node + # is present, we assume it means the GRD file wants to override + # the default header, with no includes. + for output_node in item.GetOutputFiles(): + if output_node.GetType() == 'rc_header': + for child in output_node.children: + if child.name == 'emit': + if child.attrs['emit_type'] == 'prepend': + return header_string + # else print out the default header with include + return header_string + ''' +#include <atlres.h> + +''' + + +class EmitAppender(interface.ItemFormatter): + '''Adds the content of the <emit> nodes to the RC header file.''' + + def Format(self, item, lang='', begin_item=True, output_dir='.'): + if not begin_item: + return '' + else: + return '%s\n' % (item.GetCdata()) + +class Item(interface.ItemFormatter): + '''Writes the #define line(s) for a single item in a resource.h file. If + your node has multiple IDs that need to be defined (as is the case e.g. for + dialog resources) it should define a function GetTextIds(self) that returns + a list of textual IDs (strings). Otherwise the formatter will use the + 'name' attribute of the node.''' + + # All IDs allocated so far, mapped to the textual ID they represent. + # Used to detect and resolve collisions. + ids_ = {} + + # All textual IDs allocated so far, mapped to the numerical ID they + # represent. Used when literal IDs are being defined in the 'identifiers' + # section of the GRD file to define other message IDs. + tids_ = {} + + def _VerifyId(self, id, tid, msg_if_error): + if id in self.ids_ and self.ids_[id] != tid: + raise exception.IdRangeOverlap(msg_if_error + + '\nUse the first_id attribute on grouping nodes (<structures>,\n' + '<includes>, <messages> and <ids>) to fix this problem.') + if id < 101: + print ('WARNING: Numeric resource IDs should be greater than 100 to avoid\n' + 'conflicts with system-defined resource IDs.') + + def Format(self, item, lang='', begin_item=True, output_dir='.'): + if not begin_item: + return '' + + # Resources that use the RES protocol don't need + # any numerical ids generated, so we skip them altogether. + # This is accomplished by setting the flag 'generateid' to false + # in the GRD file. + if 'generateid' in item.attrs: + if item.attrs['generateid'] == 'false': + return '' + + text_ids = item.GetTextualIds() + + # We consider the "parent" of the item to be the GroupingNode containing + # the item, as its immediate parent may be an <if> node. + item_parent = item.parent + import grit.node.empty + while item_parent and not isinstance(item_parent, + grit.node.empty.GroupingNode): + item_parent = item_parent.parent + + lines = [] + for tid in text_ids: + if util.SYSTEM_IDENTIFIERS.match(tid): + # Don't emit a new ID for predefined IDs + continue + + # Some identifier nodes can provide their own id, + # and we use that id in the generated header in that case. + if hasattr(item, 'GetId') and item.GetId(): + id = long(item.GetId()) + + elif ('offset' in item.attrs and item_parent and + 'first_id' in item_parent.attrs and item_parent.attrs['first_id'] != ''): + offset_text = item.attrs['offset'] + parent_text = item_parent.attrs['first_id'] + + try: + offset_id = long(offset_text) + except ValueError: + offset_id = self.tids_[offset_text] + + try: + parent_id = long(parent_text) + except ValueError: + parent_id = self.tids_[parent_text] + + id = parent_id + offset_id + + # We try to allocate IDs sequentially for blocks of items that might + # be related, for instance strings in a stringtable (as their IDs might be + # used e.g. as IDs for some radio buttons, in which case the IDs must + # be sequential). + # + # We do this by having the first item in a section store its computed ID + # (computed from a fingerprint) in its parent object. Subsequent children + # of the same parent will then try to get IDs that sequentially follow + # the currently stored ID (on the parent) and increment it. + elif not item_parent or not hasattr(item_parent, '_last_id_'): + # First check if the starting ID is explicitly specified by the parent. + if (item_parent and 'first_id' in item_parent.attrs and + item_parent.attrs['first_id'] != ''): + id = long(item_parent.attrs['first_id']) + self._VerifyId(id, tid, + 'Explicitly specified numeric first_id %d conflicts with one of the\n' + 'ID ranges already used.' % id) + else: + # Automatically generate the ID based on the first clique from the + # first child of the first child node of our parent (i.e. when we + # first get to this location in the code). + + # According to + # http://msdn.microsoft.com/en-us/library/t2zechd4(VS.71).aspx + # the safe usable range for resource IDs in Windows is from decimal + # 101 to 0x7FFF. + + id = FP.UnsignedFingerPrint(tid) + id = id % (0x7FFF - 101) + id += 101 + + self._VerifyId(id, tid, + 'Automatic (fingerprint-based) numeric ID for %s (%d) overlapped\n' + 'with a previously allocated range.' % (tid, id)) + + if item_parent: + item_parent._last_id_ = id + else: + assert hasattr(item_parent, '_last_id_') + id = item_parent._last_id_ = item_parent._last_id_ + 1 + self._VerifyId(id, tid, + 'Wanted to make numeric value for ID %s (%d) follow the numeric value of\n' + 'the previous ID in the .grd file, but it was already used.' % (tid, id)) + + if tid not in self.ids_.values(): + self.ids_[id] = tid + self.tids_[tid] = id + lines.append('#define %s %d\n' % (tid, id)) + return ''.join(lines) diff --git a/tools/grit/grit/format/rc_header_unittest.py b/tools/grit/grit/format/rc_header_unittest.py new file mode 100644 index 0000000..44d7c1b --- /dev/null +++ b/tools/grit/grit/format/rc_header_unittest.py @@ -0,0 +1,129 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for the rc_header formatter''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import StringIO +import unittest + +from grit.format import rc_header +from grit.node import message +from grit.node import structure +from grit.node import include +from grit.node import misc +from grit import grd_reader +from grit import exception + + +class RcHeaderFormatterUnittest(unittest.TestCase): + def setUp(self): + self.formatter = rc_header.Item() + self.formatter.ids_ = {} # need to reset this between tests + + def FormatAll(self, grd): + output = [] + for node in grd: + if isinstance(node, (message.MessageNode, structure.StructureNode, + include.IncludeNode, misc.IdentifierNode)): + output.append(self.formatter.Format(node)) + output = ''.join(output) + return output.replace(' ', '') + + def testFormatter(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir="."> + <release seq="3"> + <includes first_id="300" comment="bingo"> + <include type="gif" name="ID_LOGO" file="images/logo.gif" /> + </includes> + <messages first_id="10000"> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + <message name="IDS_BONGO"> + Bongo! + </message> + </messages> + <structures> + <structure type="dialog" name="IDD_NARROW_DIALOG" file="rc_files/dialogs.rc" /> + <structure type="version" name="VS_VERSION_INFO" file="rc_files/version.rc" /> + </structures> + </release> + </grit>'''), '.') + output = self.FormatAll(grd) + self.failUnless(output.count('IDS_GREETING10000')) + self.failUnless(output.count('ID_LOGO300')) + + def testExplicitFirstIdOverlaps(self): + # second first_id will overlap preexisting range + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir="."> + <release seq="3"> + <includes first_id="300" comment="bingo"> + <include type="gif" name="ID_LOGO" file="images/logo.gif" /> + <include type="gif" name="ID_LOGO2" file="images/logo2.gif" /> + </includes> + <messages first_id="301"> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + <message name="IDS_SMURFGEBURF">Frubegfrums</message> + </messages> + </release> + </grit>'''), '.') + self.assertRaises(exception.IdRangeOverlap, self.FormatAll, grd) + + def testImplicitOverlapsPreexisting(self): + # second message in <messages> will overlap preexisting range + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir="."> + <release seq="3"> + <includes first_id="301" comment="bingo"> + <include type="gif" name="ID_LOGO" file="images/logo.gif" /> + <include type="gif" name="ID_LOGO2" file="images/logo2.gif" /> + </includes> + <messages first_id="300"> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + <message name="IDS_SMURFGEBURF">Frubegfrums</message> + </messages> + </release> + </grit>'''), '.') + self.assertRaises(exception.IdRangeOverlap, self.FormatAll, grd) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/format/rc_unittest.py b/tools/grit/grit/format/rc_unittest.py new file mode 100644 index 0000000..769545d --- /dev/null +++ b/tools/grit/grit/format/rc_unittest.py @@ -0,0 +1,287 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.format.rc''' + +import os +import re +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import tempfile +import unittest +import StringIO + +from grit.format import rc +from grit import grd_reader +from grit import util +from grit.tool import build + +class DummyOutput(object): + def __init__(self, type, language): + self.type = type + self.language = language + def GetType(self): + return self.type + def GetLanguage(self): + return self.language + def GetOutputFilename(self): + return 'hello.gif' + +class FormatRcUnittest(unittest.TestCase): + def testMessages(self): + root = grd_reader.Parse(StringIO.StringIO(''' + <messages> + <message name="IDS_BTN_GO" desc="Button text" meaning="verb">Go!</message> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + <message name="BONGO" desc="Flippo nippo"> + Howdie "Mr. Elephant", how are you doing? \'\'\' + </message> + <message name="IDS_WITH_LINEBREAKS"> +Good day sir, +I am a bee +Sting sting + </message> + </messages> + '''), flexible_root = True) + util.FixRootForUnittest(root) + + buf = StringIO.StringIO() + build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf) + output = buf.getvalue() + self.failUnless(output.strip() == u''' +STRINGTABLE +BEGIN + IDS_BTN_GO "Go!" + IDS_GREETING "Hello %s, how are you doing today?" + BONGO "Howdie ""Mr. Elephant"", how are you doing? " + IDS_WITH_LINEBREAKS "Good day sir,\\nI am a bee\\nSting sting" +END'''.strip()) + + + def testRcSection(self): + root = grd_reader.Parse(StringIO.StringIO(''' + <structures> + <structure type="menu" name="IDC_KLONKMENU" file="grit\\test\data\klonk.rc" encoding="utf-16" /> + <structure type="dialog" name="IDD_ABOUTBOX" file="grit\\test\data\klonk.rc" encoding="utf-16" /> + <structure type="version" name="VS_VERSION_INFO" file="grit\\test\data\klonk.rc" encoding="utf-16" /> + </structures>'''), flexible_root = True) + util.FixRootForUnittest(root) + root.RunGatherers(recursive = True) + + buf = StringIO.StringIO() + build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf) + output = buf.getvalue() + self.failUnless(output.strip() == u''' +IDC_KLONKMENU MENU +BEGIN + POPUP "&File" + BEGIN + MENUITEM "E&xit", IDM_EXIT + MENUITEM "This be ""Klonk"" me like", ID_FILE_THISBE + POPUP "gonk" + BEGIN + MENUITEM "Klonk && is ""good""", ID_GONK_KLONKIS + END + END + POPUP "&Help" + BEGIN + MENUITEM "&About ...", IDM_ABOUT + END +END + +IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "About" +FONT 8, "System", 0, 0, 0x0 +BEGIN + ICON IDI_KLONK,IDC_MYICON,14,9,20,20 + LTEXT "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8, + SS_NOPREFIX + LTEXT "Copyright (C) 2005",IDC_STATIC,49,20,119,8 + DEFPUSHBUTTON "OK",IDOK,195,6,30,11,WS_GROUP + CONTROL "Jack ""Black"" Daniels",IDC_RADIO1,"Button", + BS_AUTORADIOBUTTON,46,51,84,10 +END + +VS_VERSION_INFO VERSIONINFO + FILEVERSION 1,0,0,1 + PRODUCTVERSION 1,0,0,1 + FILEFLAGSMASK 0x17L +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x4L + FILETYPE 0x1L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "FileDescription", "klonk Application" + VALUE "FileVersion", "1, 0, 0, 1" + VALUE "InternalName", "klonk" + VALUE "LegalCopyright", "Copyright (C) 2005" + VALUE "OriginalFilename", "klonk.exe" + VALUE "ProductName", " klonk Application" + VALUE "ProductVersion", "1, 0, 0, 1" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END'''.strip()) + + + def testRcIncludeStructure(self): + root = grd_reader.Parse(StringIO.StringIO(''' + <structures> + <structure type="tr_html" name="IDR_HTML" file="bingo.html"/> + <structure type="tr_html" name="IDR_HTML2" file="bingo2.html"/> + </structures>'''), flexible_root = True) + util.FixRootForUnittest(root, '/temp') + # We do not run gatherers as it is not needed and wouldn't find the file + + buf = StringIO.StringIO() + build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf) + output = buf.getvalue() + expected = (u'IDR_HTML HTML "%s"\n' + u'IDR_HTML2 HTML "%s"' + % (util.normpath('/temp/bingo.html').replace('\\', '\\\\'), + util.normpath('/temp/bingo2.html').replace('\\', '\\\\'))) + # hackety hack to work on win32&lin + output = re.sub('"[c-zC-Z]:', '"', output) + self.failUnless(output.strip() == expected) + + def testRcIncludeFile(self): + root = grd_reader.Parse(StringIO.StringIO(''' + <includes> + <include type="TXT" name="TEXT_ONE" file="bingo.txt"/> + <include type="TXT" name="TEXT_TWO" file="bingo2.txt" filenameonly="true" /> + </includes>'''), flexible_root = True) + util.FixRootForUnittest(root, '/temp') + + buf = StringIO.StringIO() + build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf) + output = buf.getvalue() + expected = (u'TEXT_ONE TXT "%s"\n' + u'TEXT_TWO TXT "%s"' + % (util.normpath('/temp/bingo.txt').replace('\\', '\\\\'), + 'bingo2.txt')) + # hackety hack to work on win32&lin + output = re.sub('"[c-zC-Z]:', '"', output) + self.failUnless(output.strip() == expected) + + + def testStructureNodeOutputfile(self): + input_file = util.PathFromRoot('grit/test/data/simple.html') + root = grd_reader.Parse(StringIO.StringIO( + '<structure type="tr_html" name="IDR_HTML" file="%s" />' %input_file), + flexible_root = True) + util.FixRootForUnittest(root, '.') + # We must run the gatherers since we'll be wanting the translation of the + # file. The file exists in the location pointed to. + root.RunGatherers(recursive=True) + + output_dir = tempfile.gettempdir() + en_file = root.FileForLanguage('en', output_dir) + self.failUnless(en_file == input_file) + fr_file = root.FileForLanguage('fr', output_dir) + self.failUnless(fr_file == os.path.join(output_dir, 'fr_simple.html')) + + fo = file(fr_file) + contents = fo.read() + fo.close() + + self.failUnless(contents.find('<p>') != -1) # should contain the markup + self.failUnless(contents.find('Hello!') == -1) # should be translated + + + def testFallbackToEnglish(self): + root = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="1" allow_pseudo="False"> + <structures fallback_to_english="True"> + <structure type="dialog" name="IDD_ABOUTBOX" file="grit\\test\data\klonk.rc" encoding="utf-16" /> + </structures> + </release> + </grit>'''), util.PathFromRoot('.')) + util.FixRootForUnittest(root) + root.RunGatherers(recursive = True) + + node = root.GetNodeById("IDD_ABOUTBOX") + formatter = node.ItemFormatter('rc_all') + output = formatter.Format(node, 'bingobongo') + self.failUnless(output.strip() == '''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "About" +FONT 8, "System", 0, 0, 0x0 +BEGIN + ICON IDI_KLONK,IDC_MYICON,14,9,20,20 + LTEXT "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8, + SS_NOPREFIX + LTEXT "Copyright (C) 2005",IDC_STATIC,49,20,119,8 + DEFPUSHBUTTON "OK",IDOK,195,6,30,11,WS_GROUP + CONTROL "Jack ""Black"" Daniels",IDC_RADIO1,"Button", + BS_AUTORADIOBUTTON,46,51,84,10 +END''') + + + def testRelativePath(self): + ''' Verify that _MakeRelativePath works in some tricky cases.''' + def TestRelativePathCombinations(base_path, other_path, expected_result): + ''' Verify that the relative path function works for + the given paths regardless of whether or not they end with + a trailing slash.''' + for path1 in [base_path, base_path + os.path.sep]: + for path2 in [other_path, other_path + os.path.sep]: + result = rc._MakeRelativePath(path1, path2) + self.failUnless(result == expected_result) + + # set-up variables + root_dir = 'c:%sa' % os.path.sep + result1 = '..%sabc' % os.path.sep + path1 = root_dir + 'bc' + result2 = 'bc' + path2 = '%s%s%s' % (root_dir, os.path.sep, result2) + # run the tests + TestRelativePathCombinations(root_dir, path1, result1) + TestRelativePathCombinations(root_dir, path2, result2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/__init__.py b/tools/grit/grit/gather/__init__.py new file mode 100644 index 0000000..d60af7c --- /dev/null +++ b/tools/grit/grit/gather/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Module grit.gather +''' + +pass
\ No newline at end of file diff --git a/tools/grit/grit/gather/admin_template.py b/tools/grit/grit/gather/admin_template.py new file mode 100644 index 0000000..b719de1 --- /dev/null +++ b/tools/grit/grit/gather/admin_template.py @@ -0,0 +1,103 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Gatherer for administrative template files. +''' + +import re +import types + +from grit.gather import regexp +from grit import exception +from grit import tclib +from grit import util + + +class MalformedAdminTemplateException(exception.Base): + '''This file doesn't look like a .adm file to me.''' + def __init__(self, msg=''): + exception.Base.__init__(self, msg) + + +class AdmGatherer(regexp.RegexpGatherer): + '''Gatherer for the translateable portions of an admin template. + + This gatherer currently makes the following assumptions: + - there is only one [strings] section and it is always the last section + of the file + - translateable strings do not need to be escaped. + ''' + + # Finds the strings section as the group named 'strings' + _STRINGS_SECTION = re.compile('(?P<first_part>.+^\[strings\])(?P<strings>.+)\Z', + re.MULTILINE | re.DOTALL) + + # Finds the translateable sections from within the [strings] section. + _TRANSLATEABLES = re.compile('^\s*[A-Za-z0-9_]+\s*=\s*"(?P<text>.+)"\s*$', + re.MULTILINE) + + def __init__(self, text): + regexp.RegexpGatherer.__init__(self, text) + + def Escape(self, text): + return text.replace('\n', '\\n') + + def UnEscape(self, text): + return text.replace('\\n', '\n') + + def Parse(self): + if self.have_parsed_: + return + m = self._STRINGS_SECTION.match(self.text_) + if not m: + raise MalformedAdminTemplateException() + # Add the first part, which is all nontranslateable, to the skeleton + self._AddNontranslateableChunk(m.group('first_part')) + # Then parse the rest using the _TRANSLATEABLES regexp. + self._RegExpParse(self._TRANSLATEABLES, m.group('strings')) + + # static method + def FromFile(adm_file, ext_key=None, encoding='cp1252'): + '''Loads the contents of 'adm_file' in encoding 'encoding' and creates + an AdmGatherer instance that gathers from those contents. + + The 'ext_key' parameter is ignored. + + Args: + adm_file: file('bingo.rc') | 'filename.rc' + encoding: 'utf-8' + + Return: + AdmGatherer(contents_of_file) + ''' + if isinstance(adm_file, types.StringTypes): + adm_file = util.WrapInputStream(file(adm_file, 'r'), encoding) + return AdmGatherer(adm_file.read()) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/admin_template_unittest.py b/tools/grit/grit/gather/admin_template_unittest.py new file mode 100644 index 0000000..230b3b5e --- /dev/null +++ b/tools/grit/grit/gather/admin_template_unittest.py @@ -0,0 +1,141 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for the admin template gatherer.''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import StringIO +import tempfile +import unittest + +from grit.gather import admin_template +from grit import util +from grit import grd_reader +from grit import grit_runner +from grit.tool import build + + +class AdmGathererUnittest(unittest.TestCase): + def testParsingAndTranslating(self): + pseudofile = StringIO.StringIO( + 'bingo bongo\n' + 'ding dong\n' + '[strings] \n' + 'whatcha="bingo bongo"\n' + 'gotcha = "bingolabongola "the wise" fingulafongula" \n') + gatherer = admin_template.AdmGatherer.FromFile(pseudofile) + gatherer.Parse() + self.failUnless(len(gatherer.GetCliques()) == 2) + self.failUnless(gatherer.GetCliques()[1].GetMessage().GetRealContent() == + 'bingolabongola "the wise" fingulafongula') + + translation = gatherer.Translate('en') + self.failUnless(translation == gatherer.GetText().strip()) + + def testErrorHandling(self): + pseudofile = StringIO.StringIO( + 'bingo bongo\n' + 'ding dong\n' + 'whatcha="bingo bongo"\n' + 'gotcha = "bingolabongola "the wise" fingulafongula" \n') + gatherer = admin_template.AdmGatherer.FromFile(pseudofile) + self.assertRaises(admin_template.MalformedAdminTemplateException, + gatherer.Parse) + + _TRANSLATABLES_FROM_FILE = ( + 'Google', 'Google Desktop Search', 'Preferences', + 'Controls Google Deskop Search preferences', + 'Indexing and Capture Control', + 'Controls what files, web pages, and other content will be indexed by Google Desktop Search.', + 'Prevent indexing of e-mail', + # there are lots more but we don't check any further + ) + + def VerifyCliquesFromAdmFile(self, cliques): + self.failUnless(len(cliques) > 20) + for ix in range(len(self._TRANSLATABLES_FROM_FILE)): + text = cliques[ix].GetMessage().GetRealContent() + self.failUnless(text == self._TRANSLATABLES_FROM_FILE[ix]) + + def testFromFile(self): + fname = util.PathFromRoot('grit/test/data/GoogleDesktopSearch.adm') + gatherer = admin_template.AdmGatherer.FromFile(fname) + gatherer.Parse() + cliques = gatherer.GetCliques() + self.VerifyCliquesFromAdmFile(cliques) + + def MakeGrd(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3"> + <release seq="3"> + <structures> + <structure type="admin_template" name="IDAT_GOOGLE_DESKTOP_SEARCH" + file="GoogleDesktopSearch.adm" exclude_from_rc="true" /> + <structure type="txt" name="BINGOBONGO" + file="README.txt" exclude_from_rc="true" /> + </structures> + </release> + <outputs> + <output filename="de_res.rc" type="rc_all" lang="de" /> + </outputs> + </grit>'''), util.PathFromRoot('grit/test/data')) + grd.RunGatherers(recursive=True) + return grd + + def testInGrd(self): + grd = self.MakeGrd() + cliques = grd.children[0].children[0].children[0].GetCliques() + self.VerifyCliquesFromAdmFile(cliques) + + def testFileIsOutput(self): + grd = self.MakeGrd() + dirname = tempfile.mkdtemp() + try: + tool = build.RcBuilder() + tool.o = grit_runner.Options() + tool.output_directory = dirname + tool.res = grd + tool.Process() + + self.failUnless(os.path.isfile( + os.path.join(dirname, 'de_GoogleDesktopSearch.adm'))) + self.failUnless(os.path.isfile( + os.path.join(dirname, 'de_README.txt'))) + finally: + for f in os.listdir(dirname): + os.unlink(os.path.join(dirname, f)) + os.rmdir(dirname) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/interface.py b/tools/grit/grit/gather/interface.py new file mode 100644 index 0000000..821b567 --- /dev/null +++ b/tools/grit/grit/gather/interface.py @@ -0,0 +1,132 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Interface for all gatherers. +''' + + +from grit import clique + + +class GathererBase(object): + '''Interface for all gatherer implementations. Subclasses must implement + all methods that raise NotImplemented.''' + + def __init__(self): + # A default uberclique that is local to this object. Users can override + # this with the uberclique they are using. + self.uberclique = clique.UberClique() + # Indicates whether this gatherer is a skeleton gatherer, in which case + # we should not do some types of processing on the translateable bits. + self.is_skeleton = False + + def SetUberClique(self, uberclique): + '''Overrides the default uberclique so that cliques created by this object + become part of the uberclique supplied by the user. + ''' + self.uberclique = uberclique + + def SetSkeleton(self, is_skeleton): + self.is_skeleton = is_skeleton + + def IsSkeleton(self): + return self.is_skeleton + + def Parse(self): + '''Parses the contents of what is being gathered.''' + raise NotImplementedError() + + def GetText(self): + '''Returns the text of what is being gathered.''' + raise NotImplementedError() + + def GetTextualIds(self): + '''Returns the mnemonic IDs that need to be defined for the resource + being gathered to compile correctly.''' + return [] + + def GetCliques(self): + '''Returns the MessageClique objects for all translateable portions.''' + return [] + + def Translate(self, lang, pseudo_if_not_available=True, + skeleton_gatherer=None, fallback_to_english=False): + '''Returns the resource being gathered, with translateable portions filled + with the translation for language 'lang'. + + If pseudo_if_not_available is true, a pseudotranslation will be used for any + message that doesn't have a real translation available. + + If no translation is available and pseudo_if_not_available is false, + fallback_to_english controls the behavior. If it is false, throw an error. + If it is true, use the English version of the message as its own + "translation". + + If skeleton_gatherer is specified, the translation will use the nontranslateable + parts from the gatherer 'skeleton_gatherer', which must be of the same type + as 'self'. + + If fallback_to_english + + Args: + lang: 'en' + pseudo_if_not_available: True | False + skeleton_gatherer: other_gatherer + fallback_to_english: True | False + + Return: + e.g. 'ID_THIS_SECTION TYPE\n...BEGIN\n "Translated message"\n......\nEND' + + Raises: + grit.exception.NotReady() if used before Parse() has been successfully + called. + grit.exception.NoSuchTranslation() if 'pseudo_if_not_available' and + fallback_to_english are both false and there is no translation for the + requested language. + ''' + raise NotImplementedError() + + def FromFile(rc_file, extkey=None, encoding = 'cp1252'): + '''Loads the resource from the file 'rc_file'. Optionally an external key + (which gets passed to the gatherer's constructor) can be specified. + + If 'rc_file' is a filename, it will be opened for reading using 'encoding'. + Otherwise the 'encoding' parameter is ignored. + + Args: + rc_file: file('') | 'filename.rc' + extkey: e.g. 'ID_MY_DIALOG' + encoding: 'utf-8' + + Return: + grit.gather.interface.GathererBase subclass + ''' + raise NotImplementedError() + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/muppet_strings.py b/tools/grit/grit/gather/muppet_strings.py new file mode 100644 index 0000000..7594762 --- /dev/null +++ b/tools/grit/grit/gather/muppet_strings.py @@ -0,0 +1,165 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Support for "strings.xml" format used by Muppet plug-ins in Google Desktop.''' + +import StringIO +import types +import re +import xml.sax +import xml.sax.handler +import xml.sax.saxutils + +from grit.gather import regexp +from grit import util +from grit import tclib + +# Placeholders can be defined in strings.xml files by putting the name of the +# placeholder between [![ and ]!] e.g. <MSG>Hello [![USER]!] how are you<MSG> +PLACEHOLDER_RE = re.compile('(\[!\[|\]!\])') + + +class MuppetStringsContentHandler(xml.sax.handler.ContentHandler): + '''A very dumb parser for splitting the strings.xml file into translateable + and nontranslateable chunks.''' + + def __init__(self, parent): + self.curr_elem = '' + self.curr_text = '' + self.parent = parent + self.description = '' + self.meaning = '' + self.translateable = True + + def startElement(self, name, attrs): + if (name != 'strings'): + self.curr_elem = name + + attr_names = attrs.getQNames() + if 'desc' in attr_names: + self.description = attrs.getValueByQName('desc') + if 'meaning' in attr_names: + self.meaning = attrs.getValueByQName('meaning') + if 'translateable' in attr_names: + value = attrs.getValueByQName('translateable') + if value.lower() not in ['true', 'yes']: + self.translateable = False + + att_text = [] + for attr_name in attr_names: + att_text.append(' ') + att_text.append(attr_name) + att_text.append('=') + att_text.append( + xml.sax.saxutils.quoteattr(attrs.getValueByQName(attr_name))) + + self.parent._AddNontranslateableChunk("<%s%s>" % + (name, ''.join(att_text))) + + def characters(self, content): + if self.curr_elem != '': + self.curr_text += content + + def endElement(self, name): + if name != 'strings': + self.parent.AddMessage(self.curr_text, self.description, + self.meaning, self.translateable) + self.parent._AddNontranslateableChunk("</%s>\n" % name) + self.curr_elem = '' + self.curr_text = '' + self.description = '' + self.meaning = '' + self.translateable = True + + def ignorableWhitespace(self, whitespace): + pass + +class MuppetStrings(regexp.RegexpGatherer): + '''Supports the strings.xml format used by Muppet gadgets.''' + + def __init__(self, text): + if util.IsExtraVerbose(): + print text + regexp.RegexpGatherer.__init__(self, text) + + def AddMessage(self, msgtext, description, meaning, translateable): + if msgtext == '': + return + + msg = tclib.Message(description=description, meaning=meaning) + + unescaped_text = self.UnEscape(msgtext) + parts = PLACEHOLDER_RE.split(unescaped_text) + in_placeholder = False + for part in parts: + if part == '': + continue + elif part == '[![': + in_placeholder = True + elif part == ']!]': + in_placeholder = False + else: + if in_placeholder: + msg.AppendPlaceholder(tclib.Placeholder(part, '[![%s]!]' % part, + '(placeholder)')) + else: + msg.AppendText(part) + + self.skeleton_.append( + self.uberclique.MakeClique(msg, translateable=translateable)) + + # if statement needed because this is supposed to be idempotent (so never + # set back to false) + if translateable: + self.translatable_chunk_ = True + + # Although we use the RegexpGatherer base class, we do not use the + # _RegExpParse method of that class to implement Parse(). Instead, we + # parse using a SAX parser. + def Parse(self): + if (self.have_parsed_): + return + self._AddNontranslateableChunk(u'<strings>\n') + stream = StringIO.StringIO(self.text_) + handler = MuppetStringsContentHandler(self) + xml.sax.parse(stream, handler) + self._AddNontranslateableChunk(u'</strings>\n') + + def Escape(self, text): + return util.EncodeCdata(text) + + def FromFile(filename_or_stream, extkey=None, encoding='cp1252'): + if isinstance(filename_or_stream, types.StringTypes): + if util.IsVerbose(): + print "MuppetStrings reading file %s, encoding %s" % ( + filename_or_stream, encoding) + filename_or_stream = util.WrapInputStream(file(filename_or_stream, 'r'), encoding) + return MuppetStrings(filename_or_stream.read()) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/muppet_strings_unittest.py b/tools/grit/grit/gather/muppet_strings_unittest.py new file mode 100644 index 0000000..b43c1d1 --- /dev/null +++ b/tools/grit/grit/gather/muppet_strings_unittest.py @@ -0,0 +1,90 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.gather.muppet_strings''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest + +from grit.gather import muppet_strings + +class MuppetStringsUnittest(unittest.TestCase): + def testParsing(self): + original = '''<strings><BLA desc="Says hello">hello!</BLA><BINGO>YEEEESSS!!!</BINGO></strings>''' + gatherer = muppet_strings.MuppetStrings(original) + gatherer.Parse() + self.failUnless(len(gatherer.GetCliques()) == 2) + self.failUnless(gatherer.Translate('en').replace('\n', '') == original) + + def testEscapingAndLinebreaks(self): + original = ('''\ +<strings> +<LINEBREAK desc="Howdie">Hello +there +how +are +you?</LINEBREAK> <ESCAPED meaning="bingo">4 < 6</ESCAPED> +</strings>''') + gatherer = muppet_strings.MuppetStrings(original) + gatherer.Parse() + self.failUnless(gatherer.GetCliques()[0].translateable) + self.failUnless(len(gatherer.GetCliques()) == 2) + self.failUnless(gatherer.GetCliques()[0].GetMessage().GetRealContent() == + 'Hello\nthere\nhow\nare\nyou?') + self.failUnless(gatherer.GetCliques()[0].GetMessage().GetDescription() == 'Howdie') + self.failUnless(gatherer.GetCliques()[1].GetMessage().GetRealContent() == + '4 < 6') + self.failUnless(gatherer.GetCliques()[1].GetMessage().GetMeaning() == 'bingo') + + def testPlaceholders(self): + original = "<strings><MESSAGE translateable='True'>Hello [![USER]!] how are you? [![HOUR]!]:[![MINUTE]!]</MESSAGE></strings>" + gatherer = muppet_strings.MuppetStrings(original) + gatherer.Parse() + self.failUnless(gatherer.GetCliques()[0].translateable) + msg = gatherer.GetCliques()[0].GetMessage() + self.failUnless(len(msg.GetPlaceholders()) == 3) + ph = msg.GetPlaceholders()[0] + self.failUnless(ph.GetOriginal() == '[![USER]!]') + self.failUnless(ph.GetPresentation() == 'USER') + + def testTranslateable(self): + original = "<strings><BINGO translateable='false'>Yo yo hi there</BINGO></strings>" + gatherer = muppet_strings.MuppetStrings(original) + gatherer.Parse() + msg = gatherer.GetCliques()[0].GetMessage() + self.failUnless(msg.GetRealContent() == "Yo yo hi there") + self.failUnless(not gatherer.GetCliques()[0].translateable) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/rc.py b/tools/grit/grit/gather/rc.py new file mode 100644 index 0000000..e056427 --- /dev/null +++ b/tools/grit/grit/gather/rc.py @@ -0,0 +1,427 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Support for gathering resources from RC files. +''' + + +import re +import types + +from grit import clique +from grit import exception +from grit import util +from grit import tclib + +from grit.gather import regexp + + +# Find portions that need unescaping in resource strings. We need to be +# careful that a \\n is matched _first_ as a \\ rather than matching as +# a \ followed by a \n. +# TODO(joi) Handle ampersands if we decide to change them into <ph> +# TODO(joi) May need to handle other control characters than \n +_NEED_UNESCAPE = re.compile(r'""|\\\\|\\n|\\t') + +# Find portions that need escaping to encode string as a resource string. +_NEED_ESCAPE = re.compile(r'"|\n|\t|\\|\ \;') + +# How to escape certain characters +_ESCAPE_CHARS = { + '"' : '""', + '\n' : '\\n', + '\t' : '\\t', + '\\' : '\\\\', + ' ' : ' ' +} + +# How to unescape certain strings +_UNESCAPE_CHARS = dict([[value, key] for key, value in _ESCAPE_CHARS.items()]) + + + +class Section(regexp.RegexpGatherer): + '''A section from a resource file.''' + + def __init__(self, section_text): + '''Creates a new object. + + Args: + section_text: 'ID_SECTION_ID SECTIONTYPE\n.....\nBEGIN\n.....\nEND' + ''' + regexp.RegexpGatherer.__init__(self, section_text) + + # static method + def Escape(text): + '''Returns a version of 'text' with characters escaped that need to be + for inclusion in a resource section.''' + def Replace(match): + return _ESCAPE_CHARS[match.group()] + return _NEED_ESCAPE.sub(Replace, text) + Escape = staticmethod(Escape) + + # static method + def UnEscape(text): + '''Returns a version of 'text' with escaped characters unescaped.''' + def Replace(match): + return _UNESCAPE_CHARS[match.group()] + return _NEED_UNESCAPE.sub(Replace, text) + UnEscape = staticmethod(UnEscape) + + def _RegExpParse(self, rexp, text_to_parse): + '''Overrides _RegExpParse to add shortcut group handling. Otherwise + the same. + ''' + regexp.RegexpGatherer._RegExpParse(self, rexp, text_to_parse) + + if not self.IsSkeleton() and len(self.GetTextualIds()) > 0: + group_name = self.GetTextualIds()[0] + for c in self.GetCliques(): + c.AddToShortcutGroup(group_name) + + # Static method + def FromFileImpl(rc_file, extkey, encoding, type): + '''Implementation of FromFile. Need to keep separate so we can have + a FromFile in this class that has its type set to Section by default. + ''' + if isinstance(rc_file, types.StringTypes): + rc_file = util.WrapInputStream(file(rc_file, 'r'), encoding) + + out = '' + begin_count = 0 + for line in rc_file.readlines(): + if len(out) > 0 or (line.strip().startswith(extkey) and + line.strip().split()[0] == extkey): + out += line + + # we stop once we reach the END for the outermost block. + begin_count_was = begin_count + if len(out) > 0 and line.strip() == 'BEGIN': + begin_count += 1 + elif len(out) > 0 and line.strip() == 'END': + begin_count -= 1 + if begin_count_was == 1 and begin_count == 0: + break + + if len(out) == 0: + raise exception.SectionNotFound('%s in file %s' % (extkey, rc_file)) + + return type(out) + FromFileImpl = staticmethod(FromFileImpl) + + # static method + def FromFile(rc_file, extkey, encoding='cp1252'): + '''Retrieves the section of 'rc_file' that has the key 'extkey'. This is + matched against the start of a line, and that line and the rest of that + section in the RC file is returned. + + If 'rc_file' is a filename, it will be opened for reading using 'encoding'. + Otherwise the 'encoding' parameter is ignored. + + This method instantiates an object of type 'type' with the text from the + file. + + Args: + rc_file: file('') | 'filename.rc' + extkey: 'ID_MY_DIALOG' + encoding: 'utf-8' + type: class to instantiate with text of section + + Return: + type(text_of_section) + ''' + return Section.FromFileImpl(rc_file, extkey, encoding, Section) + FromFile = staticmethod(FromFile) + + +class Dialog(Section): + '''A resource section that contains a dialog resource.''' + + # A typical dialog resource section looks like this: + # + # IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 + # STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU + # CAPTION "About" + # FONT 8, "System", 0, 0, 0x0 + # BEGIN + # ICON IDI_KLONK,IDC_MYICON,14,9,20,20 + # LTEXT "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8, + # SS_NOPREFIX + # LTEXT "Copyright (C) 2005",IDC_STATIC,49,20,119,8 + # DEFPUSHBUTTON "OK",IDOK,195,6,30,11,WS_GROUP + # CONTROL "Jack ""Black"" Daniels",IDC_RADIO1,"Button", + # BS_AUTORADIOBUTTON,46,51,84,10 + # END + + # We are using a sorted set of keys, and we assume that the + # group name used for descriptions (type) will come after the "text" + # group in alphabetical order. We also assume that there cannot be + # more than one description per regular expression match. + # If that's not the case some descriptions will be clobbered. + dialog_re_ = re.compile(''' + # The dialog's ID in the first line + (?P<id1>[A-Z0-9_]+)\s+DIALOG(EX)? + | + # The caption of the dialog + (?P<type1>CAPTION)\s+"(?P<text1>.*?([^"]|""))"\s + | + # Lines for controls that have text and an ID + \s+(?P<type2>[A-Z]+)\s+"(?P<text2>.*?([^"]|"")?)"\s*,\s*(?P<id2>[A-Z0-9_]+)\s*, + | + # Lines for controls that have text only + \s+(?P<type3>[A-Z]+)\s+"(?P<text3>.*?([^"]|"")?)"\s*, + | + # Lines for controls that reference other resources + \s+[A-Z]+\s+[A-Z0-9_]+\s*,\s*(?P<id3>[A-Z0-9_]*[A-Z][A-Z0-9_]*) + | + # This matches "NOT SOME_STYLE" so that it gets consumed and doesn't get + # matched by the next option (controls that have only an ID and then just + # numbers) + \s+NOT\s+[A-Z][A-Z0-9_]+ + | + # Lines for controls that have only an ID and then just numbers + \s+[A-Z]+\s+(?P<id4>[A-Z0-9_]*[A-Z][A-Z0-9_]*)\s*, + ''', re.MULTILINE | re.VERBOSE) + + def Parse(self): + '''Knows how to parse dialog resource sections.''' + self._RegExpParse(self.dialog_re_, self.text_) + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + return Section.FromFileImpl(rc_file, extkey, encoding, Dialog) + FromFile = staticmethod(FromFile) + + +class Menu(Section): + '''A resource section that contains a menu resource.''' + + # A typical menu resource section looks something like this: + # + # IDC_KLONK MENU + # BEGIN + # POPUP "&File" + # BEGIN + # MENUITEM "E&xit", IDM_EXIT + # MENUITEM "This be ""Klonk"" me like", ID_FILE_THISBE + # POPUP "gonk" + # BEGIN + # MENUITEM "Klonk && is ""good""", ID_GONK_KLONKIS + # END + # END + # POPUP "&Help" + # BEGIN + # MENUITEM "&About ...", IDM_ABOUT + # END + # END + + # Description used for the messages generated for menus, to explain to + # the translators how to handle them. + MENU_MESSAGE_DESCRIPTION = ( + 'This message represents a menu. Each of the items appears in sequence ' + '(some possibly within sub-menus) in the menu. The XX01XX placeholders ' + 'serve to separate items. Each item contains an & (ampersand) character ' + 'in front of the keystroke that should be used as a shortcut for that item ' + 'in the menu. Please make sure that no two items in the same menu share ' + 'the same shortcut.' + ) + + # A dandy regexp to suck all the IDs and translateables out of a menu + # resource + menu_re_ = re.compile(''' + # Match the MENU ID on the first line + ^(?P<id1>[A-Z0-9_]+)\s+MENU + | + # Match the translateable caption for a popup menu + POPUP\s+"(?P<text1>.*?([^"]|""))"\s + | + # Match the caption & ID of a MENUITEM + MENUITEM\s+"(?P<text2>.*?([^"]|""))"\s*,\s*(?P<id2>[A-Z0-9_]+) + ''', re.MULTILINE | re.VERBOSE) + + def Parse(self): + '''Knows how to parse menu resource sections. Because it is important that + menu shortcuts are unique within the menu, we return each menu as a single + message with placeholders to break up the different menu items, rather than + return a single message per menu item. we also add an automatic description + with instructions for the translators.''' + self.single_message_ = tclib.Message(description=self.MENU_MESSAGE_DESCRIPTION) + self._RegExpParse(self.menu_re_, self.text_) + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + return Section.FromFileImpl(rc_file, extkey, encoding, Menu) + FromFile = staticmethod(FromFile) + + +class Version(Section): + '''A resource section that contains a VERSIONINFO resource.''' + + # A typical version info resource can look like this: + # + # VS_VERSION_INFO VERSIONINFO + # FILEVERSION 1,0,0,1 + # PRODUCTVERSION 1,0,0,1 + # FILEFLAGSMASK 0x3fL + # #ifdef _DEBUG + # FILEFLAGS 0x1L + # #else + # FILEFLAGS 0x0L + # #endif + # FILEOS 0x4L + # FILETYPE 0x2L + # FILESUBTYPE 0x0L + # BEGIN + # BLOCK "StringFileInfo" + # BEGIN + # BLOCK "040904e4" + # BEGIN + # VALUE "CompanyName", "TODO: <Company name>" + # VALUE "FileDescription", "TODO: <File description>" + # VALUE "FileVersion", "1.0.0.1" + # VALUE "LegalCopyright", "TODO: (c) <Company name>. All rights reserved." + # VALUE "InternalName", "res_format_test.dll" + # VALUE "OriginalFilename", "res_format_test.dll" + # VALUE "ProductName", "TODO: <Product name>" + # VALUE "ProductVersion", "1.0.0.1" + # END + # END + # BLOCK "VarFileInfo" + # BEGIN + # VALUE "Translation", 0x409, 1252 + # END + # END + # + # + # In addition to the above fields, VALUE fields named "Comments" and + # "LegalTrademarks" may also be translateable. + + version_re_ = re.compile(''' + # Match the ID on the first line + ^(?P<id1>[A-Z0-9_]+)\s+VERSIONINFO + | + # Match all potentially translateable VALUE sections + \s+VALUE\s+" + ( + CompanyName|FileDescription|LegalCopyright| + ProductName|Comments|LegalTrademarks + )",\s+"(?P<text1>.*?([^"]|""))"\s + ''', re.MULTILINE | re.VERBOSE) + + def Parse(self): + '''Knows how to parse VERSIONINFO resource sections.''' + self._RegExpParse(self.version_re_, self.text_) + + # TODO(joi) May need to override the Translate() method to change the + # "Translation" VALUE block to indicate the correct language code. + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + return Section.FromFileImpl(rc_file, extkey, encoding, Version) + FromFile = staticmethod(FromFile) + +class RCData(Section): + '''A resource section that contains some data .''' + + # A typical rcdataresource section looks like this: + # + # IDR_BLAH RCDATA { 1, 2, 3, 4 } + + dialog_re_ = re.compile(''' + ^(?P<id1>[A-Z0-9_]+)\s+RCDATA\s+(DISCARDABLE)?\s+\{.*?\} + ''', re.MULTILINE | re.VERBOSE | re.DOTALL) + + def Parse(self): + '''Knows how to parse RCDATA resource sections.''' + self._RegExpParse(self.dialog_re_, self.text_) + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + '''Implementation of FromFile for resource types w/braces (not BEGIN/END) + ''' + if isinstance(rc_file, types.StringTypes): + rc_file = util.WrapInputStream(file(rc_file, 'r'), encoding) + + out = '' + begin_count = 0 + openbrace_count = 0 + for line in rc_file.readlines(): + if len(out) > 0 or line.strip().startswith(extkey): + out += line + + # we stop once balance the braces (could happen on one line) + begin_count_was = begin_count + if len(out) > 0: + openbrace_count += line.count('{') + begin_count += line.count('{') + begin_count -= line.count('}') + if ((begin_count_was == 1 and begin_count == 0) or + (openbrace_count > 0 and begin_count == 0)): + break + + if len(out) == 0: + raise exception.SectionNotFound('%s in file %s' % (extkey, rc_file)) + + return RCData(out) + FromFile = staticmethod(FromFile) + + +class Accelerators(Section): + '''An ACCELERATORS table. + ''' + + # A typical ACCELERATORS section looks like this: + # + # IDR_ACCELERATOR1 ACCELERATORS + # BEGIN + # "^C", ID_ACCELERATOR32770, ASCII, NOINVERT + # "^V", ID_ACCELERATOR32771, ASCII, NOINVERT + # VK_INSERT, ID_ACCELERATOR32772, VIRTKEY, CONTROL, NOINVERT + # END + + accelerators_re_ = re.compile(''' + # Match the ID on the first line + ^(?P<id1>[A-Z0-9_]+)\s+ACCELERATORS\s+ + | + # Match accelerators specified as VK_XXX + \s+VK_[A-Z0-9_]+,\s*(?P<id2>[A-Z0-9_]+)\s*, + | + # Match accelerators specified as e.g. "^C" + \s+"[^"]*",\s+(?P<id3>[A-Z0-9_]+)\s*, + ''', re.MULTILINE | re.VERBOSE) + + def Parse(self): + '''Knows how to parse ACCELERATORS resource sections.''' + self._RegExpParse(self.accelerators_re_, self.text_) + + # static method + def FromFile(rc_file, extkey, encoding = 'cp1252'): + return Section.FromFileImpl(rc_file, extkey, encoding, Accelerators) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/rc_unittest.py b/tools/grit/grit/gather/rc_unittest.py new file mode 100644 index 0000000..150703a --- /dev/null +++ b/tools/grit/grit/gather/rc_unittest.py @@ -0,0 +1,390 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.gather.rc''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest +import StringIO + +from grit.gather import rc +from grit import util + + +class RcUnittest(unittest.TestCase): + + part_we_want = '''IDC_KLONKACC ACCELERATORS +BEGIN + "?", IDM_ABOUT, ASCII, ALT + "/", IDM_ABOUT, ASCII, ALT +END''' + + def testSectionFromFile(self): + buf = '''IDC_SOMETHINGELSE BINGO +BEGIN + BLA BLA + BLA BLA +END +%s + +IDC_KLONK BINGOBONGO +BEGIN + HONGO KONGO +END +''' % self.part_we_want + + f = StringIO.StringIO(buf) + + out = rc.Section.FromFile(f, 'IDC_KLONKACC') + self.failUnless(out.GetText() == self.part_we_want) + + out = rc.Section.FromFile(util.PathFromRoot(r'grit/test/data/klonk.rc'), + 'IDC_KLONKACC', + encoding='utf-16') + self.failUnless(out.GetText() == self.part_we_want) + + + def testDialog(self): + dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "About" +FONT 8, "System", 0, 0, 0x0 +BEGIN + ICON IDI_KLONK,IDC_MYICON,14,9,20,20 + LTEXT "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8, + SS_NOPREFIX + LTEXT "Copyright (C) 2005",IDC_STATIC,49,20,119,8 + DEFPUSHBUTTON "OK",IDOK,195,6,30,11,WS_GROUP + CONTROL "Jack ""Black"" Daniels",IDC_RADIO1,"Button", + BS_AUTORADIOBUTTON,46,51,84,10 + // try a line where the ID is on the continuation line + LTEXT "blablablabla blablabla blablablablablablablabla blablabla", + ID_SMURF, whatever... +END +''') + dlg.Parse() + self.failUnless(len(dlg.GetTextualIds()) == 7) + self.failUnless(len(dlg.GetCliques()) == 6) + self.failUnless(dlg.GetCliques()[1].GetMessage().GetRealContent() == + 'klonk Version "yibbee" 1.0') + + transl = dlg.Translate('en') + self.failUnless(transl.strip() == dlg.GetText().strip()) + + def testAlternateSkeleton(self): + dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "About" +FONT 8, "System", 0, 0, 0x0 +BEGIN + LTEXT "Yipee skippy",IDC_STATIC,49,10,119,8, + SS_NOPREFIX +END +''') + dlg.Parse() + + alt_dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 040704, 17, 230, 75 +STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU +CAPTION "XXXXXXXXX" +FONT 8, "System", 0, 0, 0x0 +BEGIN + LTEXT "XXXXXXXXXXXXXXXXX",IDC_STATIC,110978,10,119,8, + SS_NOPREFIX +END +''') + alt_dlg.Parse() + + transl = dlg.Translate('en', skeleton_gatherer=alt_dlg) + self.failUnless(transl.count('040704') and + transl.count('110978')) + self.failUnless(transl.count('Yipee skippy')) + + def testMenu(self): + menu = rc.Menu('''IDC_KLONK MENU +BEGIN + POPUP "&File """ + BEGIN + MENUITEM "E&xit", IDM_EXIT + MENUITEM "This be ""Klonk"" me like", ID_FILE_THISBE + POPUP "gonk" + BEGIN + MENUITEM "Klonk && is ""good""", ID_GONK_KLONKIS + END + MENUITEM "This is a very long menu caption to try to see if we can make the ID go to a continuation line, blablabla blablabla bla blabla blablabla blablabla blablabla blablabla...", + ID_FILE_THISISAVERYLONGMENUCAPTIONTOTRYTOSEEIFWECANMAKETHEIDGOTOACONTINUATIONLINE + END + POPUP "&Help" + BEGIN + MENUITEM "&About ...", IDM_ABOUT + END +END''') + + menu.Parse() + self.failUnless(len(menu.GetTextualIds()) == 6) + self.failUnless(len(menu.GetCliques()) == 1) + self.failUnless(len(menu.GetCliques()[0].GetMessage().GetPlaceholders()) == + 9) + + transl = menu.Translate('en') + self.failUnless(transl.strip() == menu.GetText().strip()) + + def testVersion(self): + version = rc.Version(''' +VS_VERSION_INFO VERSIONINFO + FILEVERSION 1,0,0,1 + PRODUCTVERSION 1,0,0,1 + FILEFLAGSMASK 0x3fL +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x4L + FILETYPE 0x2L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904e4" + BEGIN + VALUE "CompanyName", "TODO: <Company name>" + VALUE "FileDescription", "TODO: <File description>" + VALUE "FileVersion", "1.0.0.1" + VALUE "LegalCopyright", "TODO: (c) <Company name>. All rights reserved." + VALUE "InternalName", "res_format_test.dll" + VALUE "OriginalFilename", "res_format_test.dll" + VALUE "ProductName", "TODO: <Product name>" + VALUE "ProductVersion", "1.0.0.1" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END +'''.strip()) + version.Parse() + self.failUnless(len(version.GetTextualIds()) == 1) + self.failUnless(len(version.GetCliques()) == 4) + + transl = version.Translate('en') + self.failUnless(transl.strip() == version.GetText().strip()) + + + def testRegressionDialogBox(self): + dialog = rc.Dialog(''' +IDD_SIDEBAR_WEATHER_PANEL_PROPPAGE DIALOGEX 0, 0, 205, 157 +STYLE DS_SETFONT | DS_FIXEDSYS | WS_CHILD +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + EDITTEXT IDC_SIDEBAR_WEATHER_NEW_CITY,3,27,112,14,ES_AUTOHSCROLL + DEFPUSHBUTTON "Add Location",IDC_SIDEBAR_WEATHER_ADD,119,27,50,14 + LISTBOX IDC_SIDEBAR_WEATHER_CURR_CITIES,3,48,127,89, + LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_TABSTOP + PUSHBUTTON "Move Up",IDC_SIDEBAR_WEATHER_MOVE_UP,134,104,50,14 + PUSHBUTTON "Move Down",IDC_SIDEBAR_WEATHER_MOVE_DOWN,134,121,50,14 + PUSHBUTTON "Remove",IDC_SIDEBAR_WEATHER_DELETE,134,48,50,14 + LTEXT "To see current weather conditions and forecasts in the USA, enter the zip code (example: 94043) or city and state (example: Mountain View, CA).", + IDC_STATIC,3,0,199,25 + CONTROL "Fahrenheit",IDC_SIDEBAR_WEATHER_FAHRENHEIT,"Button", + BS_AUTORADIOBUTTON | WS_GROUP | WS_TABSTOP,3,144,51,10 + CONTROL "Celsius",IDC_SIDEBAR_WEATHER_CELSIUS,"Button", + BS_AUTORADIOBUTTON,57,144,38,10 +END'''.strip()) + dialog.Parse() + self.failUnless(len(dialog.GetTextualIds()) == 10) + + + def testRegressionDialogBox2(self): + dialog = rc.Dialog(''' +IDD_SIDEBAR_EMAIL_PANEL_PROPPAGE DIALOG DISCARDABLE 0, 0, 264, 220 +STYLE WS_CHILD +FONT 8, "MS Shell Dlg" +BEGIN + GROUPBOX "Email Filters",IDC_STATIC,7,3,250,190 + LTEXT "Click Add Filter to create the email filter.",IDC_STATIC,16,41,130,9 + PUSHBUTTON "Add Filter...",IDC_SIDEBAR_EMAIL_ADD_FILTER,196,38,50,14 + PUSHBUTTON "Remove",IDC_SIDEBAR_EMAIL_REMOVE,196,174,50,14 + PUSHBUTTON "", IDC_SIDEBAR_EMAIL_HIDDEN, 200, 178, 5, 5, NOT WS_VISIBLE + LISTBOX IDC_SIDEBAR_EMAIL_LIST,16,60,230,108, + LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_TABSTOP + LTEXT "You can prevent certain emails from showing up in the sidebar with a filter.", + IDC_STATIC,16,18,234,18 +END'''.strip()) + dialog.Parse() + self.failUnless('IDC_SIDEBAR_EMAIL_HIDDEN' in dialog.GetTextualIds()) + + + def testRegressionMenuId(self): + menu = rc.Menu(''' +IDR_HYPERMENU_FOLDER MENU +BEGIN + POPUP "HyperFolder" + BEGIN + MENUITEM "Open Containing Folder", IDM_OPENFOLDER + END +END'''.strip()) + menu.Parse() + self.failUnless(len(menu.GetTextualIds()) == 2) + + def testRegressionNewlines(self): + menu = rc.Menu(''' +IDR_HYPERMENU_FOLDER MENU +BEGIN + POPUP "Hyper\\nFolder" + BEGIN + MENUITEM "Open Containing Folder", IDM_OPENFOLDER + END +END'''.strip()) + menu.Parse() + transl = menu.Translate('en') + # Shouldn't find \\n (the \n shouldn't be changed to \\n) + self.failUnless(transl.find('\\\\n') == -1) + + def testRegressionTabs(self): + menu = rc.Menu(''' +IDR_HYPERMENU_FOLDER MENU +BEGIN + POPUP "Hyper\\tFolder" + BEGIN + MENUITEM "Open Containing Folder", IDM_OPENFOLDER + END +END'''.strip()) + menu.Parse() + transl = menu.Translate('en') + # Shouldn't find \\t (the \t shouldn't be changed to \\t) + self.failUnless(transl.find('\\\\t') == -1) + + def testEscapeUnescape(self): + original = 'Hello "bingo"\n How\\are\\you\\n?' + escaped = rc.Section.Escape(original) + self.failUnless(escaped == 'Hello ""bingo""\\n How\\\\are\\\\you\\\\n?') + unescaped = rc.Section.UnEscape(escaped) + self.failUnless(unescaped == original) + + def testRegressionPathsWithSlashN(self): + original = '..\\\\..\\\\trs\\\\res\\\\nav_first.gif' + unescaped = rc.Section.UnEscape(original) + self.failUnless(unescaped == '..\\..\\trs\\res\\nav_first.gif') + + def testRegressionDialogItemsTextOnly(self): + dialog = rc.Dialog('''IDD_OPTIONS_SEARCH DIALOGEX 0, 0, 280, 292 +STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | DS_CENTER | WS_POPUP | + WS_DISABLED | WS_CAPTION | WS_SYSMENU +CAPTION "Search" +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + GROUPBOX "Select search buttons and options",-1,7,5,266,262 + CONTROL "",IDC_OPTIONS,"SysTreeView32",TVS_DISABLEDRAGDROP | + WS_BORDER | WS_TABSTOP | 0x800,16,19,248,218 + LTEXT "Use Google site:",-1,26,248,52,8 + COMBOBOX IDC_GOOGLE_HOME,87,245,177,256,CBS_DROPDOWNLIST | + WS_VSCROLL | WS_TABSTOP + PUSHBUTTON "Restore Defaults...",IDC_RESET,187,272,86,14 +END''') + dialog.Parse() + translateables = [c.GetMessage().GetRealContent() + for c in dialog.GetCliques()] + self.failUnless('Select search buttons and options' in translateables) + self.failUnless('Use Google site:' in translateables) + + def testAccelerators(self): + acc = rc.Accelerators('''\ +IDR_ACCELERATOR1 ACCELERATORS +BEGIN + "^C", ID_ACCELERATOR32770, ASCII, NOINVERT + "^V", ID_ACCELERATOR32771, ASCII, NOINVERT + VK_INSERT, ID_ACCELERATOR32772, VIRTKEY, CONTROL, NOINVERT +END +''') + acc.Parse() + self.failUnless(len(acc.GetTextualIds()) == 4) + self.failUnless(len(acc.GetCliques()) == 0) + + transl = acc.Translate('en') + self.failUnless(transl.strip() == acc.GetText().strip()) + + + def testRegressionEmptyString(self): + dlg = rc.Dialog('''\ +IDD_CONFIRM_QUIT_GD_DLG DIALOGEX 0, 0, 267, 108 +STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | DS_CENTER | WS_POPUP | + WS_CAPTION +EXSTYLE WS_EX_TOPMOST +CAPTION "Google Desktop" +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + DEFPUSHBUTTON "&Yes",IDYES,82,87,50,14 + PUSHBUTTON "&No",IDNO,136,87,50,14 + ICON 32514,IDC_STATIC,7,9,21,20 + EDITTEXT IDC_TEXTBOX,34,7,231,60,ES_MULTILINE | ES_READONLY | NOT WS_BORDER + CONTROL "", + IDC_ENABLE_GD_AUTOSTART,"Button",BS_AUTOCHECKBOX | + WS_TABSTOP,33,70,231,10 +END''') + dlg.Parse() + + def Check(): + self.failUnless(transl.count('IDC_ENABLE_GD_AUTOSTART')) + self.failUnless(transl.count('END')) + + transl = dlg.Translate('de', pseudo_if_not_available=True, + fallback_to_english=True) + Check() + transl = dlg.Translate('de', pseudo_if_not_available=True, + fallback_to_english=False) + Check() + transl = dlg.Translate('de', pseudo_if_not_available=False, + fallback_to_english=True) + Check() + transl = dlg.Translate('de', pseudo_if_not_available=False, + fallback_to_english=False) + Check() + transl = dlg.Translate('en', pseudo_if_not_available=True, + fallback_to_english=True) + Check() + transl = dlg.Translate('en', pseudo_if_not_available=True, + fallback_to_english=False) + Check() + transl = dlg.Translate('en', pseudo_if_not_available=False, + fallback_to_english=True) + Check() + transl = dlg.Translate('en', pseudo_if_not_available=False, + fallback_to_english=False) + Check() + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/regexp.py b/tools/grit/grit/gather/regexp.py new file mode 100644 index 0000000..30c5abdb --- /dev/null +++ b/tools/grit/grit/gather/regexp.py @@ -0,0 +1,224 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''A baseclass for simple gatherers based on regular expressions. +''' + +import re +import types + +from grit.gather import interface +from grit import clique +from grit import tclib + + +class RegexpGatherer(interface.GathererBase): + '''Common functionality of gatherers based on parsing using a single + regular expression. + ''' + + DescriptionMapping_ = { + 'CAPTION' : 'This is a caption for a dialog', + 'CHECKBOX' : 'This is a label for a checkbox', + 'CONTROL': 'This is the text on a control', + 'CTEXT': 'This is a label for a control', + 'DEFPUSHBUTTON': 'This is a button definition', + 'GROUPBOX': 'This is a label for a grouping', + 'ICON': 'This is a label for an icon', + 'LTEXT': 'This is the text for a label', + 'PUSHBUTTON': 'This is the text for a button', + } + + def __init__(self, text): + interface.GathererBase.__init__(self) + # Original text of what we're parsing + self.text_ = text.strip() + # List of parts of the document. Translateable parts are clique.MessageClique + # objects, nontranslateable parts are plain strings. Translated messages are + # inserted back into the skeleton using the quoting rules defined by + # self.Escape() + self.skeleton_ = [] + # A list of the names of IDs that need to be defined for this resource + # section to compile correctly. + self.ids_ = [] + # True if Parse() has already been called. + self.have_parsed_ = False + # True if a translatable chunk has been added + self.translatable_chunk_ = False + # If not None, all parts of the document will be put into this single + # message; otherwise the normal skeleton approach is used. + self.single_message_ = None + # Number to use for the next placeholder name. Used only if single_message + # is not None + self.ph_counter_ = 1 + + def GetText(self): + '''Returns the original text of the section''' + return self.text_ + + def Escape(self, text): + '''Subclasses can override. Base impl is identity. + ''' + return text + + def UnEscape(self, text): + '''Subclasses can override. Base impl is identity. + ''' + return text + + def GetTextualIds(self): + '''Returns the list of textual IDs that need to be defined for this + resource section to compile correctly.''' + return self.ids_ + + def GetCliques(self): + '''Returns the message cliques for each translateable message in the + resource section.''' + return filter(lambda x: isinstance(x, clique.MessageClique), self.skeleton_) + + def Translate(self, lang, pseudo_if_not_available=True, + skeleton_gatherer=None, fallback_to_english=False): + if len(self.skeleton_) == 0: + raise exception.NotReady() + if skeleton_gatherer: + assert len(skeleton_gatherer.skeleton_) == len(self.skeleton_) + + out = [] + for ix in range(len(self.skeleton_)): + if isinstance(self.skeleton_[ix], types.StringTypes): + if skeleton_gatherer: + # Make sure the skeleton is like the original + assert(isinstance(skeleton_gatherer.skeleton_[ix], types.StringTypes)) + out.append(skeleton_gatherer.skeleton_[ix]) + else: + out.append(self.skeleton_[ix]) + else: + if skeleton_gatherer: # Make sure the skeleton is like the original + assert(not isinstance(skeleton_gatherer.skeleton_[ix], + types.StringTypes)) + msg = self.skeleton_[ix].MessageForLanguage(lang, + pseudo_if_not_available, + fallback_to_english) + + def MyEscape(text): + return self.Escape(text) + text = msg.GetRealContent(escaping_function=MyEscape) + out.append(text) + return ''.join(out) + + # Contextualization elements. Used for adding additional information + # to the message bundle description string from RC files. + def AddDescriptionElement(self, string): + if self.DescriptionMapping_.has_key(string): + description = self.DescriptionMapping_[string] + else: + description = string + if self.single_message_: + self.single_message_.SetDescription(description) + else: + if (self.translatable_chunk_): + message = self.skeleton_[len(self.skeleton_) - 1].GetMessage() + message.SetDescription(description) + + def Parse(self): + '''Parses the section. Implemented by subclasses. Idempotent.''' + raise NotImplementedError() + + def _AddNontranslateableChunk(self, chunk): + '''Adds a nontranslateable chunk.''' + if self.single_message_: + ph = tclib.Placeholder('XX%02dXX' % self.ph_counter_, chunk, chunk) + self.ph_counter_ += 1 + self.single_message_.AppendPlaceholder(ph) + else: + self.skeleton_.append(chunk) + + def _AddTranslateableChunk(self, chunk): + '''Adds a translateable chunk. It will be unescaped before being added.''' + # We don't want empty messages since they are redundant and the TC + # doesn't allow them. + if chunk == '': + return + + unescaped_text = self.UnEscape(chunk) + if self.single_message_: + self.single_message_.AppendText(unescaped_text) + else: + self.skeleton_.append(self.uberclique.MakeClique( + tclib.Message(text=unescaped_text))) + self.translatable_chunk_ = True + + def _AddTextualId(self, id): + self.ids_.append(id) + + def _RegExpParse(self, regexp, text_to_parse): + '''An implementation of Parse() that can be used for resource sections that + can be parsed using a single multi-line regular expression. + + All translateables must be in named groups that have names starting with + 'text'. All textual IDs must be in named groups that have names starting + with 'id'. All type definitions that can be included in the description + field for contextualization purposes should have a name that starts with + 'type'. + + Args: + regexp: re.compile('...', re.MULTILINE) + text_to_parse: + ''' + if self.have_parsed_: + return + self.have_parsed_ = True + + chunk_start = 0 + for match in regexp.finditer(text_to_parse): + groups = match.groupdict() + keys = groups.keys() + keys.sort() + self.translatable_chunk_ = False + for group in keys: + if group.startswith('id') and groups[group]: + self._AddTextualId(groups[group]) + elif group.startswith('text') and groups[group]: + self._AddNontranslateableChunk( + text_to_parse[chunk_start : match.start(group)]) + chunk_start = match.end(group) # Next chunk will start after the match + self._AddTranslateableChunk(groups[group]) + elif group.startswith('type') and groups[group]: + # Add the description to the skeleton_ list. This works because + # we are using a sort set of keys, and because we assume that the + # group name used for descriptions (type) will come after the "text" + # group in alphabetical order. We also assume that there cannot be + # more than one description per regular expression match. + self.AddDescriptionElement(groups[group]) + + self._AddNontranslateableChunk(text_to_parse[chunk_start:]) + + if self.single_message_: + self.skeleton_.append(self.uberclique.MakeClique(self.single_message_)) diff --git a/tools/grit/grit/gather/tr_html.py b/tools/grit/grit/gather/tr_html.py new file mode 100644 index 0000000..6591f23 --- /dev/null +++ b/tools/grit/grit/gather/tr_html.py @@ -0,0 +1,703 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''A gatherer for the TotalRecall brand of HTML templates with replaceable +portions. We wanted to reuse extern.tclib.api.handlers.html.TCHTMLParser +but this proved impossible due to the fact that the TotalRecall HTML templates +are in general quite far from parseable HTML and the TCHTMLParser derives +from HTMLParser.HTMLParser which requires relatively well-formed HTML. Some +examples of "HTML" from the TotalRecall HTML templates that wouldn't be +parseable include things like: + + <a [PARAMS]>blabla</a> (not parseable because attributes are invalid) + + <table><tr><td>[LOTSOFSTUFF]</tr></table> (not parseable because closing + </td> is in the HTML [LOTSOFSTUFF] + is replaced by) + +The other problem with using general parsers (such as TCHTMLParser) is that +we want to make sure we output the TotalRecall template with as little changes +as possible in terms of whitespace characters, layout etc. With any parser +that generates a parse tree, and generates output by dumping the parse tree, +we would always have little inconsistencies which could cause bugs (the +TotalRecall template stuff is quite brittle and can break if e.g. a tab +character is replaced with spaces). + +The solution, which may be applicable to some other HTML-like template +languages floating around Google, is to create a parser with a simple state +machine that keeps track of what kind of tag it's inside, and whether it's in +a translateable section or not. Translateable sections are: + +a) text (including [BINGO] replaceables) inside of tags that + can contain translateable text (which is all tags except + for a few) + +b) text inside of an 'alt' attribute in an <image> element, or + the 'value' attribute of a <submit>, <button> or <text> + element. + +The parser does not build up a parse tree but rather a "skeleton" which +is a list of nontranslateable strings intermingled with grit.clique.MessageClique +objects. This simplifies the parser considerably compared to a regular HTML +parser. To output a translated document, each item in the skeleton is +printed out, with the relevant Translation from each MessageCliques being used +for the requested language. + +This implementation borrows some code, constants and ideas from +extern.tclib.api.handlers.html.TCHTMLParser. +''' + + +import re +import types + +from grit import clique +from grit import exception +from grit import util +from grit import tclib + +from grit.gather import interface + + +# HTML tags which break (separate) chunks. +_BLOCK_TAGS = ['script', 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'br', + 'body', 'style', 'head', 'title', 'table', 'tr', 'td', 'th', + 'ul', 'ol', 'dl', 'nl', 'li', 'div', 'object', 'center', + 'html', 'link', 'form', 'select', 'textarea', + 'button', 'option', 'map', 'area', 'blockquote', 'pre', + 'meta', 'xmp', 'noscript', 'label', 'tbody', 'thead', + 'script', 'style', 'pre', 'iframe', 'img', 'input', 'nowrap'] + +# HTML tags which may appear within a chunk. +_INLINE_TAGS = ['b', 'i', 'u', 'tt', 'code', 'font', 'a', 'span', 'small', + 'key', 'nobr', 'url', 'em', 's', 'sup', 'strike', + 'strong'] + +# HTML tags within which linebreaks are significant. +_PREFORMATTED_TAGS = ['textarea', 'xmp', 'pre'] + +# An array mapping some of the inline HTML tags to more meaningful +# names for those tags. This will be used when generating placeholders +# representing these tags. +_HTML_PLACEHOLDER_NAMES = { 'a' : 'link', 'br' : 'break', 'b' : 'bold', + 'i' : 'italic', 'li' : 'item', 'ol' : 'ordered_list', 'p' : 'paragraph', + 'ul' : 'unordered_list', 'img' : 'image', 'em' : 'emphasis' } + +# We append each of these characters in sequence to distinguish between +# different placeholders with basically the same name (e.g. BOLD1, BOLD2). +# Keep in mind that a placeholder name must not be a substring of any other +# placeholder name in the same message, so we can't simply count (BOLD_1 +# would be a substring of BOLD_10). +_SUFFIXES = '123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + +# Matches whitespace in an HTML document. Also matches HTML comments, which are +# treated as whitespace. +_WHITESPACE = re.compile(r'(\s| |\\n|\\r|<!--\s*desc\s*=.*?-->)+', + re.DOTALL) + +# Finds a non-whitespace character +_NON_WHITESPACE = re.compile(r'\S') + +# Matches two or more in a row (a single   is not changed into +# placeholders because different languages require different numbers of spaces +# and placeholders must match exactly; more than one is probably a "special" +# whitespace sequence and should be turned into a placeholder). +_NBSP = re.compile(r' ( )+') + +# Matches nontranslateable chunks of the document +_NONTRANSLATEABLES = re.compile(r''' + <\s*script.+?<\s*/\s*script\s*> + | + <\s*style.+?<\s*/\s*style\s*> + | + <!--.+?--> + | + <\?IMPORT\s.+?> # import tag + | + <\s*[a-zA-Z_]+:.+?> # custom tag (open) + | + <\s*/\s*[a-zA-Z_]+:.+?> # custom tag (close) + | + <!\s*[A-Z]+\s*([^>]+|"[^"]+"|'[^']+')*?> + ''', re.MULTILINE | re.DOTALL | re.VERBOSE | re.IGNORECASE) + +# Matches a tag and its attributes +_ELEMENT = re.compile(r''' + # Optional closing /, element name + <\s*(?P<closing>/)?\s*(?P<element>[a-zA-Z0-9]+)\s* + # Attributes and/or replaceables inside the tag, if any + (?P<atts>( + \s*([a-zA-Z_][-:.a-zA-Z_0-9]*) # Attribute name + (\s*=\s*(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))? + | + \s*\[(\$?\~)?([A-Z0-9-_]+?)(\~\$?)?\] + )*) + \s*(?P<empty>/)?\s*> # Optional empty-tag closing /, and tag close + ''', + re.MULTILINE | re.DOTALL | re.VERBOSE) + +# Matches elements that may have translateable attributes. The value of these +# special attributes is given by group 'value1' or 'value2'. Note that this +# regexp demands that the attribute value be quoted; this is necessary because +# the non-tree-building nature of the parser means we don't know when we're +# writing out attributes, so we wouldn't know to escape spaces. +_SPECIAL_ELEMENT = re.compile(r''' + <\s*( + input[^>]+?value\s*=\s*(\'(?P<value3>[^\']*)\'|"(?P<value4>[^"]*)") + [^>]+type\s*=\s*"?'?(button|reset|text|submit)'?"? + | + ( + table[^>]+?title\s*= + | + img[^>]+?alt\s*= + | + input[^>]+?type\s*=\s*"?'?(button|reset|text|submit)'?"?[^>]+?value\s*= + ) + \s*(\'(?P<value1>[^\']*)\'|"(?P<value2>[^"]*)") + )[^>]*?> + ''', re.MULTILINE | re.DOTALL | re.VERBOSE | re.IGNORECASE) + +# Matches stuff that is translateable if it occurs in the right context +# (between tags). This includes all characters and character entities. +# Note that this also matches which needs to be handled as whitespace +# before this regexp is applied. +_CHARACTERS = re.compile(r''' + ( + \w + | + [\!\@\#\$\%\^\*\(\)\-\=\_\+\[\]\{\}\\\|\;\:\'\"\,\.\/\?\`\~] + | + &(\#[0-9]+|\#x[0-9a-fA-F]+|[A-Za-z0-9]+); + )+ + ''', re.MULTILINE | re.DOTALL | re.VERBOSE) + +# Matches Total Recall's "replaceable" tags, which are just any text +# in capitals enclosed by delimiters like [] or [~~] or [$~~$] (e.g. [HELLO], +# [~HELLO~] and [$~HELLO~$]). +_REPLACEABLE = re.compile(r'\[(\$?\~)?(?P<name>[A-Z0-9-_]+?)(\~\$?)?\]', + re.MULTILINE) + + +# Matches the silly [!]-prefixed "header" that is used in some TotalRecall +# templates. +_SILLY_HEADER = re.compile(r'\[!\]\ntitle\t(?P<title>[^\n]+?)\n.+?\n\n', + re.MULTILINE | re.DOTALL) + + +# Matches a comment that provides a description for the message it occurs in. +_DESCRIPTION_COMMENT = re.compile( + r'<!--\s*desc\s*=\s*(?P<description>.+?)\s*-->', re.DOTALL) + + +_DEBUG = 0 +def _DebugPrint(text): + if _DEBUG: + print text.encode('utf-8') + + +class HtmlChunks(object): + '''A parser that knows how to break an HTML-like document into a list of + chunks, where each chunk is either translateable or non-translateable. + The chunks are unmodified sections of the original document, so concatenating + the text of all chunks would result in the original document.''' + + def InTranslateable(self): + return self.last_translateable != -1 + + def Rest(self): + return self.text_[self.current:] + + def StartTranslateable(self): + assert not self.InTranslateable() + if self.current != 0: + # Append a nontranslateable chunk + chunk_text = self.text_[self.chunk_start : self.last_nontranslateable + 1] + # Needed in the case where document starts with a translateable. + if len(chunk_text) > 0: + self.AddChunk(False, chunk_text) + self.chunk_start = self.last_nontranslateable + 1 + self.last_translateable = self.current + self.last_nontranslateable = -1 + + def EndTranslateable(self): + assert self.InTranslateable() + # Append a translateable chunk + self.AddChunk(True, + self.text_[self.chunk_start : self.last_translateable + 1]) + self.chunk_start = self.last_translateable + 1 + self.last_translateable = -1 + self.last_nontranslateable = self.current + + def AdvancePast(self, match): + self.current += match.end() + + def AddChunk(self, translateable, text): + '''Adds a chunk to self, removing linebreaks and duplicate whitespace + if appropriate. + ''' + if translateable and not self.last_element_ in _PREFORMATTED_TAGS: + text = text.replace('\n', ' ') + text = text.replace('\r', ' ') + text = text.replace(' ', ' ') + text = text.replace(' ', ' ') + + m = _DESCRIPTION_COMMENT.search(text) + if m: + self.last_description = m.group('description') + # remove the description from the output text + text = _DESCRIPTION_COMMENT.sub('', text) + + if translateable: + description = self.last_description + self.last_description = '' + else: + description = '' + + if text != '': + self.chunks_.append((translateable, text, description)) + + def Parse(self, text): + '''Parses self.text_ into an intermediate format stored in self.chunks_ + which is translateable and nontranslateable chunks. Also returns + self.chunks_ + + Return: + [chunk1, chunk2, chunk3, ...] (instances of class Chunk) + ''' + # + # Chunker state + # + + self.text_ = text + + # A list of tuples (is_translateable, text) which represents the document + # after chunking. + self.chunks_ = [] + + # Start index of the last chunk, whether translateable or not + self.chunk_start = 0 + + # Index of the last for-sure translateable character if we are parsing + # a translateable chunk, -1 to indicate we are not in a translateable chunk. + # This is needed so that we don't include trailing whitespace in the + # translateable chunk (whitespace is neutral). + self.last_translateable = -1 + + # Index of the last for-sure nontranslateable character if we are parsing + # a nontranslateable chunk, -1 if we are not in a nontranslateable chunk. + # This is needed to make sure we can group e.g. "<b>Hello</b> there" + # together instead of just "Hello</b> there" which would be much worse + # for translation. + self.last_nontranslateable = -1 + + # Index of the character we're currently looking at. + self.current = 0 + + # The name of the last block element parsed. + self.last_element_ = '' + + # The last explicit description we found. + self.last_description = '' + + while self.current < len(self.text_): + _DebugPrint('REST: %s' % self.text_[self.current:self.current+60]) + + # First try to match whitespace + m = _WHITESPACE.match(self.Rest()) + if m: + # Whitespace is neutral, it just advances 'current' and does not switch + # between translateable/nontranslateable. If we are in a + # nontranslateable section that extends to the current point, we extend + # it to include the whitespace. If we are in a translateable section, + # we do not extend it until we find + # more translateable parts, because we never want a translateable chunk + # to end with whitespace. + if (not self.InTranslateable() and + self.last_nontranslateable == self.current - 1): + self.last_nontranslateable = self.current + m.end() - 1 + self.AdvancePast(m) + continue + + # Then we try to match nontranslateables + m = _NONTRANSLATEABLES.match(self.Rest()) + if m: + if self.InTranslateable(): + self.EndTranslateable() + self.last_nontranslateable = self.current + m.end() - 1 + self.AdvancePast(m) + continue + + # Now match all other HTML element tags (opening, closing, or empty, we + # don't care). + m = _ELEMENT.match(self.Rest()) + if m: + element_name = m.group('element').lower() + if element_name in _BLOCK_TAGS: + self.last_element_ = element_name + if self.InTranslateable(): + self.EndTranslateable() + + # Check for "special" elements, i.e. ones that have a translateable + # attribute, and handle them correctly. Note that all of the + # "special" elements are block tags, so no need to check for this + # if the tag is not a block tag. + sm = _SPECIAL_ELEMENT.match(self.Rest()) + if sm: + # Get the appropriate group name + for group in sm.groupdict().keys(): + if sm.groupdict()[group]: + break + + # First make a nontranslateable chunk up to and including the + # quote before the translateable attribute value + self.AddChunk(False, self.text_[ + self.chunk_start : self.current + sm.start(group)]) + # Then a translateable for the translateable bit + self.AddChunk(True, self.Rest()[sm.start(group) : sm.end(group)]) + # Finally correct the data invariant for the parser + self.chunk_start = self.current + sm.end(group) + + self.last_nontranslateable = self.current + m.end() - 1 + elif self.InTranslateable(): + # We're in a translateable and the tag is an inline tag, so we + # need to include it in the translateable. + self.last_translateable = self.current + m.end() - 1 + self.AdvancePast(m) + continue + + # Anything else we find must be translateable, so we advance one character + # at a time until one of the above matches. + if not self.InTranslateable(): + self.StartTranslateable() + else: + self.last_translateable = self.current + self.current += 1 + + # Close the final chunk + if self.InTranslateable(): + self.AddChunk(True, self.text_[self.chunk_start : ]) + else: + self.AddChunk(False, self.text_[self.chunk_start : ]) + + return self.chunks_ + + +def HtmlToMessage(html, include_block_tags=False, description=''): + '''Takes a bit of HTML, which must contain only "inline" HTML elements, + and changes it into a tclib.Message. This involves escaping any entities and + replacing any HTML code with placeholders. + + If include_block_tags is true, no error will be given if block tags (e.g. + <p> or <br>) are included in the HTML. + + Args: + html: 'Hello <b>[USERNAME]</b>, how <i>are</i> you?' + include_block_tags: False + + Return: + tclib.Message('Hello START_BOLD1USERNAMEEND_BOLD, ' + 'howNBSPSTART_ITALICareEND_ITALIC you?', + [ Placeholder('START_BOLD', '<b>', ''), + Placeholder('USERNAME', '[USERNAME]', ''), + Placeholder('END_BOLD', '</b>', ''), + Placeholder('START_ITALIC', '<i>', ''), + Placeholder('END_ITALIC', '</i>', ''), ]) + ''' + # Approach is: + # - first placeholderize, finding <elements>, [REPLACEABLES] and + # - then escape all character entities in text in-between placeholders + + parts = [] # List of strings (for text chunks) and tuples (ID, original) + # for placeholders + + count_names = {} # Map of base names to number of times used + end_names = {} # Map of base names to stack of end tags (for correct nesting) + + def MakeNameClosure(base, type = ''): + '''Returns a closure that can be called once all names have been allocated + to return the final name of the placeholder. This allows us to minimally + number placeholders for non-overlap. + + Also ensures that END_XXX_Y placeholders have the same Y as the + corresponding BEGIN_XXX_Y placeholder when we have nested tags of the same + type. + + Args: + base: 'phname' + type: '' | 'begin' | 'end' + + Return: + Closure() + ''' + name = base + if type != '': + name = ('%s_%s' % (type, base)).upper() + + if name in count_names.keys(): + count_names[name] += 1 + else: + count_names[name] = 1 + + def MakeFinalName(name_ = name, index = count_names[name] - 1): + if (type.lower() == 'end' and + base in end_names.keys() and len(end_names[base])): + return end_names[base].pop(-1) # For correct nesting + if count_names[name_] != 1: + name_ = '%s_%s' % (name_, _SUFFIXES[index]) + # We need to use a stack to ensure that the end-tag suffixes match + # the begin-tag suffixes. Only needed when more than one tag of the + # same type. + if type == 'begin': + end_name = ('END_%s_%s' % (base, _SUFFIXES[index])).upper() + if base in end_names.keys(): + end_names[base].append(end_name) + else: + end_names[base] = [end_name] + + return name_ + + return MakeFinalName + + current = 0 + + while current < len(html): + m = _NBSP.match(html[current:]) + if m: + parts.append((MakeNameClosure('SPACE'), m.group())) + current += m.end() + continue + + m = _REPLACEABLE.match(html[current:]) + if m: + # Replaceables allow - but placeholders don't, so replace - with _ + ph_name = MakeNameClosure('X_%s_X' % m.group('name').replace('-', '_')) + parts.append((ph_name, m.group())) + current += m.end() + continue + + m = _SPECIAL_ELEMENT.match(html[current:]) + if m: + if not include_block_tags: + raise exception.BlockTagInTranslateableChunk(html) + element_name = 'block' # for simplification + # Get the appropriate group name + for group in m.groupdict().keys(): + if m.groupdict()[group]: + break + parts.append((MakeNameClosure(element_name, 'begin'), + html[current : current + m.start(group)])) + parts.append(m.group(group)) + parts.append((MakeNameClosure(element_name, 'end'), + html[current + m.end(group) : current + m.end()])) + current += m.end() + continue + + m = _ELEMENT.match(html[current:]) + if m: + element_name = m.group('element').lower() + if not include_block_tags and not element_name in _INLINE_TAGS: + raise exception.BlockTagInTranslateableChunk(html[current:]) + if element_name in _HTML_PLACEHOLDER_NAMES: # use meaningful names + element_name = _HTML_PLACEHOLDER_NAMES[element_name] + + # Make a name for the placeholder + type = '' + if not m.group('empty'): + if m.group('closing'): + type = 'end' + else: + type = 'begin' + parts.append((MakeNameClosure(element_name, type), m.group())) + current += m.end() + continue + + if len(parts) and isinstance(parts[-1], types.StringTypes): + parts[-1] += html[current] + else: + parts.append(html[current]) + current += 1 + + msg_text = '' + placeholders = [] + for part in parts: + if isinstance(part, types.TupleType): + final_name = part[0]() + original = part[1] + msg_text += final_name + placeholders.append(tclib.Placeholder(final_name, original, '(HTML code)')) + else: + msg_text += part + + msg = tclib.Message(text=msg_text, placeholders=placeholders, + description=description) + content = msg.GetContent() + for ix in range(len(content)): + if isinstance(content[ix], types.StringTypes): + content[ix] = util.UnescapeHtml(content[ix], replace_nbsp=False) + + return msg + + +class TrHtml(interface.GathererBase): + '''Represents a document or message in the template format used by + Total Recall for HTML documents.''' + + def __init__(self, text): + '''Creates a new object that represents 'text'. + Args: + text: '<html>...</html>' + ''' + super(type(self), self).__init__() + + self.text_ = text + self.have_parsed_ = False + self.skeleton_ = [] # list of strings and MessageClique objects + + def GetText(self): + '''Returns the original text of the HTML document''' + return self.text_ + + def GetCliques(self): + '''Returns the message cliques for each translateable message in the + document.''' + return filter(lambda x: isinstance(x, clique.MessageClique), self.skeleton_) + + def Translate(self, lang, pseudo_if_not_available=True, + skeleton_gatherer=None, fallback_to_english=False): + '''Returns this document with translateable messages filled with + the translation for language 'lang'. + + Args: + lang: 'en' + pseudo_if_not_available: True + + Return: + 'ID_THIS_SECTION TYPE\n...BEGIN\n "Translated message"\n......\nEND + + Raises: + grit.exception.NotReady() if used before Parse() has been successfully + called. + grit.exception.NoSuchTranslation() if 'pseudo_if_not_available' is false + and there is no translation for the requested language. + ''' + if len(self.skeleton_) == 0: + raise exception.NotReady() + + # TODO(joi) Implement support for skeleton gatherers here. + + out = [] + for item in self.skeleton_: + if isinstance(item, types.StringTypes): + out.append(item) + else: + msg = item.MessageForLanguage(lang, + pseudo_if_not_available, + fallback_to_english) + for content in msg.GetContent(): + if isinstance(content, tclib.Placeholder): + out.append(content.GetOriginal()) + else: + # We escape " characters to increase the chance that attributes + # will be properly escaped. + out.append(util.EscapeHtml(content, True)) + + return ''.join(out) + + + # Parsing is done in two phases: First, we break the document into + # translateable and nontranslateable chunks. Second, we run through each + # translateable chunk and insert placeholders for any HTML elements, unescape + # escaped characters, etc. + def Parse(self): + if self.have_parsed_: + return + self.have_parsed_ = True + + text = self.text_ + + # First handle the silly little [!]-prefixed header because it's not + # handled by our HTML parsers. + m = _SILLY_HEADER.match(text) + if m: + self.skeleton_.append(text[:m.start('title')]) + self.skeleton_.append(self.uberclique.MakeClique( + tclib.Message(text=text[m.start('title'):m.end('title')]))) + self.skeleton_.append(text[m.end('title') : m.end()]) + text = text[m.end():] + + chunks = HtmlChunks().Parse(text) + + for chunk in chunks: + if chunk[0]: # Chunk is translateable + self.skeleton_.append(self.uberclique.MakeClique( + HtmlToMessage(chunk[1], description=chunk[2]))) + else: + self.skeleton_.append(chunk[1]) + + # Go through the skeleton and change any messages that consist solely of + # placeholders and whitespace into nontranslateable strings. + for ix in range(len(self.skeleton_)): + got_text = False + if isinstance(self.skeleton_[ix], clique.MessageClique): + msg = self.skeleton_[ix].GetMessage() + for item in msg.GetContent(): + if (isinstance(item, types.StringTypes) and _NON_WHITESPACE.search(item) + and item != ' '): + got_text = True + break + if not got_text: + self.skeleton_[ix] = msg.GetRealContent() + + + # Static method + def FromFile(html, extkey=None, encoding = 'utf-8'): + '''Creates a TrHtml object from the contents of 'html' which are decoded + using 'encoding'. Returns a new TrHtml object, upon which Parse() has not + been called. + + Args: + html: file('') | 'filename.html' + extkey: ignored + encoding: 'utf-8' (note that encoding is ignored if 'html' is not a file + name but instead an open file or file-like object) + + Return: + TrHtml(text_of_file) + ''' + if isinstance(html, types.StringTypes): + html = util.WrapInputStream(file(html, 'r'), encoding) + doc = html.read() + + # Ignore the BOM character if the document starts with one. + if len(doc) and doc[0] == u'\ufeff': + doc = doc[1:] + + return TrHtml(doc) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/tr_html_unittest.py b/tools/grit/grit/gather/tr_html_unittest.py new file mode 100644 index 0000000..b53a8e1 --- /dev/null +++ b/tools/grit/grit/gather/tr_html_unittest.py @@ -0,0 +1,437 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.gather.tr_html''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import types +import unittest + +from grit.gather import tr_html +from grit import clique +from grit import util + + +class ParserUnittest(unittest.TestCase): + def testChunking(self): + p = tr_html.HtmlChunks() + chunks = p.Parse('<p>Hello <b>dear</b> how <i>are</i>you?<p>Fine!') + self.failUnless(chunks == [ + (False, '<p>', ''), (True, 'Hello <b>dear</b> how <i>are</i>you?', ''), + (False, '<p>', ''), (True, 'Fine!', '')]) + + chunks = p.Parse('<p> Hello <b>dear</b> how <i>are</i>you? <p>Fine!') + self.failUnless(chunks == [ + (False, '<p> ', ''), (True, 'Hello <b>dear</b> how <i>are</i>you?', ''), + (False, ' <p>', ''), (True, 'Fine!', '')]) + + chunks = p.Parse('<p> Hello <b>dear how <i>are you? <p> Fine!') + self.failUnless(chunks == [ + (False, '<p> ', ''), (True, 'Hello <b>dear how <i>are you?', ''), + (False, ' <p> ', ''), (True, 'Fine!', '')]) + + # Ensure translateable sections that start with inline tags contain + # the starting inline tag. + chunks = p.Parse('<b>Hello!</b> how are you?<p><i>I am fine.</i>') + self.failUnless(chunks == [ + (True, '<b>Hello!</b> how are you?', ''), (False, '<p>', ''), + (True, '<i>I am fine.</i>', '')]) + + # Ensure translateable sections that end with inline tags contain + # the ending inline tag. + chunks = p.Parse("Hello! How are <b>you?</b><p><i>I'm fine!</i>") + self.failUnless(chunks == [ + (True, 'Hello! How are <b>you?</b>', ''), (False, '<p>', ''), + (True, "<i>I'm fine!</i>", '')]) + + # Check capitals and explicit descriptions + chunks = p.Parse('<!-- desc=bingo! --><B>Hello!</B> how are you?<P><I>I am fine.</I>') + self.failUnless(chunks == [ + (True, '<B>Hello!</B> how are you?', 'bingo!'), (False, '<P>', ''), + (True, '<I>I am fine.</I>', '')]) + chunks = p.Parse('<B><!-- desc=bingo! -->Hello!</B> how are you?<P><I>I am fine.</I>') + self.failUnless(chunks == [ + (True, '<B>Hello!</B> how are you?', 'bingo!'), (False, '<P>', ''), + (True, '<I>I am fine.</I>', '')]) + # Linebreaks get changed to spaces just like any other HTML content + chunks = p.Parse('<B>Hello!</B> <!-- desc=bi\nngo\n! -->how are you?<P><I>I am fine.</I>') + self.failUnless(chunks == [ + (True, '<B>Hello!</B> how are you?', 'bi ngo !'), (False, '<P>', ''), + (True, '<I>I am fine.</I>', '')]) + + # In this case, because the explicit description appears after the first + # translateable, it will actually apply to the second translateable. + chunks = p.Parse('<B>Hello!</B> how are you?<!-- desc=bingo! --><P><I>I am fine.</I>') + self.failUnless(chunks == [ + (True, '<B>Hello!</B> how are you?', ''), (False, '<P>', ''), + (True, '<I>I am fine.</I>', 'bingo!')]) + + # Check that replaceables within block tags (where attributes would go) are + # handled correctly. + chunks = p.Parse('<b>Hello!</b> how are you?<p [BINGO] [$~BONGO~$]>' + '<i>I am fine.</i>') + self.failUnless(chunks == [ + (True, '<b>Hello!</b> how are you?', ''), + (False, '<p [BINGO] [$~BONGO~$]>', ''), + (True, '<i>I am fine.</i>', '')]) + + # Check that the contents of preformatted tags preserve line breaks. + chunks = p.Parse('<textarea>Hello\nthere\nhow\nare\nyou?</textarea>') + self.failUnless(chunks == [(False, '<textarea>', ''), + (True, 'Hello\nthere\nhow\nare\nyou?', ''), (False, '</textarea>', '')]) + + # ...and that other tags' line breaks are converted to spaces + chunks = p.Parse('<p>Hello\nthere\nhow\nare\nyou?</p>') + self.failUnless(chunks == [(False, '<p>', ''), + (True, 'Hello there how are you?', ''), (False, '</p>', '')]) + + def testTranslateableAttributes(self): + p = tr_html.HtmlChunks() + + # Check that the translateable attributes in <img>, <submit>, <button> and + # <text> elements buttons are handled correctly. + chunks = p.Parse('<img src=bingo.jpg alt="hello there">' + '<input type=submit value="hello">' + '<input type="button" value="hello">' + '<input type=\'text\' value=\'Howdie\'>') + self.failUnless(chunks == [ + (False, '<img src=bingo.jpg alt="', ''), (True, 'hello there', ''), + (False, '"><input type=submit value="', ''), (True, 'hello', ''), + (False, '"><input type="button" value="', ''), (True, 'hello', ''), + (False, '"><input type=\'text\' value=\'', ''), (True, 'Howdie', ''), + (False, '\'>', '')]) + + + def testTranslateableHtmlToMessage(self): + msg = tr_html.HtmlToMessage( + 'Hello <b>[USERNAME]</b>, <how> <i>are</i> you?') + pres = msg.GetPresentableContent() + self.failUnless(pres == + 'Hello BEGIN_BOLDX_USERNAME_XEND_BOLD, ' + '<how> BEGIN_ITALICareEND_ITALIC you?') + + msg = tr_html.HtmlToMessage('<b>Hello</b><I>Hello</I><b>Hello</b>') + pres = msg.GetPresentableContent() + self.failUnless(pres == + 'BEGIN_BOLD_1HelloEND_BOLD_1BEGIN_ITALICHelloEND_ITALIC' + 'BEGIN_BOLD_2HelloEND_BOLD_2') + + # Check that nesting (of the <font> tags) is handled correctly - i.e. that + # the closing placeholder numbers match the opening placeholders. + msg = tr_html.HtmlToMessage( + '''<font size=-1><font color=#FF0000>Update!</font> ''' + '''<a href='http://desktop.google.com/whatsnew.html?hl=[$~LANG~$]'>''' + '''New Features</a>: Now search PDFs, MP3s, Firefox web history, and ''' + '''more</font>''') + pres = msg.GetPresentableContent() + self.failUnless(pres == + 'BEGIN_FONT_1BEGIN_FONT_2Update!END_FONT_2 BEGIN_LINK' + 'New FeaturesEND_LINK: Now search PDFs, MP3s, Firefox ' + 'web history, and moreEND_FONT_1') + + msg = tr_html.HtmlToMessage('''<a href='[$~URL~$]'><b>[NUM][CAT]</b></a>''') + pres = msg.GetPresentableContent() + self.failUnless(pres == 'BEGIN_LINKBEGIN_BOLDX_NUM_XX_CAT_XEND_BOLDEND_LINK') + + msg = tr_html.HtmlToMessage( + '''<font size=-1><a class=q onClick='return window.qs?qs(this):1' ''' + '''href='http://[WEBSERVER][SEARCH_URI]'>Desktop</a></font> ''' + ''' ''') + pres = msg.GetPresentableContent() + self.failUnless(pres == + '''BEGIN_FONTBEGIN_LINKDesktopEND_LINKEND_FONTSPACE''') + + msg = tr_html.HtmlToMessage( + '''<br><br><center><font size=-2>©2005 Google </font></center>''', 1) + pres = msg.GetPresentableContent() + self.failUnless(pres == + u'BEGIN_BREAK_1BEGIN_BREAK_2BEGIN_CENTERBEGIN_FONT\xa92005' + u' Google END_FONTEND_CENTER') + + msg = tr_html.HtmlToMessage( + ''' - <a class=c href=[$~CACHE~$]>Cached</a>''') + pres = msg.GetPresentableContent() + self.failUnless(pres == + ' - BEGIN_LINKCachedEND_LINK') + + # Check that upper-case tags are handled correctly. + msg = tr_html.HtmlToMessage( + '''You can read the <A HREF='http://desktop.google.com/privacypolicy.''' + '''html?hl=[LANG_CODE]'>Privacy Policy</A> and <A HREF='http://desktop''' + '''.google.com/privacyfaq.html?hl=[LANG_CODE]'>Privacy FAQ</A> online.''') + pres = msg.GetPresentableContent() + self.failUnless(pres == + 'You can read the BEGIN_LINK_1Privacy PolicyEND_LINK_1 and ' + 'BEGIN_LINK_2Privacy FAQEND_LINK_2 online.') + + # Check that tags with linebreaks immediately preceding them are handled + # correctly. + msg = tr_html.HtmlToMessage( + '''You can read the +<A HREF='http://desktop.google.com/privacypolicy.html?hl=[LANG_CODE]'>Privacy Policy</A> +and <A HREF='http://desktop.google.com/privacyfaq.html?hl=[LANG_CODE]'>Privacy FAQ</A> online.''') + pres = msg.GetPresentableContent() + self.failUnless(pres == '''You can read the +BEGIN_LINK_1Privacy PolicyEND_LINK_1 +and BEGIN_LINK_2Privacy FAQEND_LINK_2 online.''') + + + +class TrHtmlUnittest(unittest.TestCase): + def testTable(self): + html = tr_html.TrHtml('''<table class="shaded-header"><tr> +<td class="header-element b expand">Preferences</td> +<td class="header-element s"> +<a href="http://desktop.google.com/preferences.html">Preferences Help</a> +</td> +</tr></table>''') + html.Parse() + self.failUnless(html.skeleton_[3].GetMessage().GetPresentableContent() == + 'BEGIN_LINKPreferences HelpEND_LINK') + + def testSubmitAttribute(self): + html = tr_html.TrHtml('''</td> +<td class="header-element"><input type=submit value="Save Preferences" +name=submit2></td> +</tr></table>''') + html.Parse() + self.failUnless(html.skeleton_[1].GetMessage().GetPresentableContent() == + 'Save Preferences') + + def testWhitespaceAfterInlineTag(self): + '''Test that even if there is whitespace after an inline tag at the start + of a translateable section the inline tag will be included. + ''' + html = tr_html.TrHtml('''<label for=DISPLAYNONE><font size=-1> Hello</font>''') + html.Parse() + self.failUnless(html.skeleton_[1].GetMessage().GetRealContent() == + '<font size=-1> Hello</font>') + + def testSillyHeader(self): + html = tr_html.TrHtml('''[!] +title\tHello +bingo +bongo +bla + +<p>Other stuff</p>''') + html.Parse() + content = html.skeleton_[1].GetMessage().GetRealContent() + self.failUnless(content == 'Hello') + self.failUnless(html.skeleton_[-1] == '</p>') + # Right after the translateable the nontranslateable should start with + # a linebreak (this catches a bug we had). + self.failUnless(html.skeleton_[2][0] == '\n') + + + def testExplicitDescriptions(self): + html = tr_html.TrHtml('Hello [USER]<br/><!-- desc=explicit --><input type="button">Go!</input>') + html.Parse() + msg = html.GetCliques()[1].GetMessage() + self.failUnless(msg.GetDescription() == 'explicit') + self.failUnless(msg.GetRealContent() == 'Go!') + + + def testRegressionInToolbarAbout(self): + html = tr_html.TrHtml.FromFile( + util.PathFromRoot(r'grit/test/data/toolbar_about.html')) + html.Parse() + cliques = html.GetCliques() + for cl in cliques: + content = cl.GetMessage().GetRealContent() + if content.count('De parvis grandis acervus erit'): + self.failIf(content.count('$/translate')) + + + def HtmlFromFileWithManualCheck(self, f): + html = tr_html.TrHtml.FromFile(f) + html.Parse() + + # For manual results inspection only... + list = [] + for item in html.skeleton_: + if isinstance(item, types.StringTypes): + list.append(item) + else: + list.append(item.GetMessage().GetPresentableContent()) + + return html + + + def testPrivacyHtml(self): + html = self.HtmlFromFileWithManualCheck( + util.PathFromRoot(r'grit/test/data/privacy.html')) + + self.failUnless(html.skeleton_[1].GetMessage().GetRealContent() == + 'Privacy and Google Desktop Search') + self.failUnless(html.skeleton_[3].startswith('<')) + self.failUnless(len(html.skeleton_) > 10) + + + def testPreferencesHtml(self): + html = self.HtmlFromFileWithManualCheck( + util.PathFromRoot(r'grit/test/data/preferences.html')) + + # Verify that we don't get '[STATUS-MESSAGE]' as the original content of + # one of the MessageClique objects (it would be a placeholder-only message + # and we're supposed to have stripped those). + + for item in filter(lambda x: isinstance(x, clique.MessageClique), + html.skeleton_): + if (item.GetMessage().GetRealContent() == '[STATUS-MESSAGE]' or + item.GetMessage().GetRealContent() == '[ADDIN-DO] [ADDIN-OPTIONS]'): + self.fail() + + self.failUnless(len(html.skeleton_) > 100) + + def AssertNumberOfTranslateables(self, files, num): + '''Fails if any of the files in files don't have exactly + num translateable sections. + + Args: + files: ['file1', 'file2'] + num: 3 + ''' + for f in files: + f = util.PathFromRoot(r'grit/test/data/%s' % f) + html = self.HtmlFromFileWithManualCheck(f) + self.failUnless(len(html.GetCliques()) == num) + + def testFewTranslateables(self): + self.AssertNumberOfTranslateables(['browser.html', 'email_thread.html', + 'header.html', 'mini.html', + 'oneclick.html', 'script.html', + 'time_related.html', 'versions.html'], 0) + self.AssertNumberOfTranslateables(['footer.html', 'hover.html'], 1) + + def testOtherHtmlFilesForManualInspection(self): + files = [ + 'about.html', 'bad_browser.html', 'cache_prefix.html', + 'cache_prefix_file.html', 'chat_result.html', 'del_footer.html', + 'del_header.html', 'deleted.html', 'details.html', 'email_result.html', + 'error.html', 'explicit_web.html', 'footer.html', + 'homepage.html', 'indexing_speed.html', + 'install_prefs.html', 'install_prefs2.html', + 'oem_enable.html', 'oem_non_admin.html', 'onebox.html', + 'password.html', 'quit_apps.html', 'recrawl.html', + 'searchbox.html', 'sidebar_h.html', 'sidebar_v.html', 'status.html', + ] + for f in files: + self.HtmlFromFileWithManualCheck( + util.PathFromRoot(r'grit/test/data/%s' % f)) + + def testTranslate(self): + # Note that the English translation of documents that use character + # literals (e.g. ©) will not be the same as the original document + # because the character literal will be transformed into the Unicode + # character itself. So for this test we choose some relatively complex + # HTML without character entities (but with because that's handled + # specially). + html = tr_html.TrHtml(''' <script> + <!-- + function checkOffice() { var w = document.getElementById("h7"); + var e = document.getElementById("h8"); var o = document.getElementById("h10"); + if (!(w.checked || e.checked)) { o.checked=0;o.disabled=1;} else {o.disabled=0;} } + // --> + </script> + <input type=checkbox [CHECK-DOC] name=DOC id=h7 onclick='checkOffice()'> + <label for=h7> Word</label><br> + <input type=checkbox [CHECK-XLS] name=XLS id=h8 onclick='checkOffice()'> + <label for=h8> Excel</label><br> + <input type=checkbox [CHECK-PPT] name=PPT id=h9> + <label for=h9> PowerPoint</label><br> + </span></td><td nowrap valign=top><span class="s"> + <input type=checkbox [CHECK-PDF] name=PDF id=hpdf> + <label for=hpdf> PDF</label><br> + <input type=checkbox [CHECK-TXT] name=TXT id=h6> + <label for=h6> Text, media, and other files</label><br> + </tr> + <tr><td nowrap valign=top colspan=3><span class="s"><br /> + <input type=checkbox [CHECK-SECUREOFFICE] name=SECUREOFFICE id=h10> + <label for=h10> Password-protected Office documents (Word, Excel)</label><br /> + <input type=checkbox [DISABLED-HTTPS] [CHECK-HTTPS] name=HTTPS id=h12><label + for=h12> Secure pages (HTTPS) in web history</label></span></td></tr> + </table>''') + html.Parse() + trans = html.Translate('en') + if (html.GetText() != trans): + self.fail() + + + def testHtmlToMessageWithBlockTags(self): + msg = tr_html.HtmlToMessage( + 'Hello<p>Howdie<img alt="bingo" src="image.gif">', True) + result = msg.GetPresentableContent() + self.failUnless( + result == 'HelloBEGIN_PARAGRAPHHowdieBEGIN_BLOCKbingoEND_BLOCK') + + msg = tr_html.HtmlToMessage( + 'Hello<p>Howdie<input type="button" value="bingo">', True) + result = msg.GetPresentableContent() + self.failUnless( + result == 'HelloBEGIN_PARAGRAPHHowdieBEGIN_BLOCKbingoEND_BLOCK') + + + def testHtmlToMessageRegressions(self): + msg = tr_html.HtmlToMessage(' - ', True) + result = msg.GetPresentableContent() + self.failUnless(result == ' - ') + + + def testEscapeUnescaped(self): + text = '© & "<hello>"' + unescaped = util.UnescapeHtml(text) + self.failUnless(unescaped == u'\u00a9\u00a0 & "<hello>"') + escaped_unescaped = util.EscapeHtml(unescaped, True) + self.failUnless(escaped_unescaped == + u'\u00a9\u00a0 & "<hello>"') + + def testRegressionCjkHtmlFile(self): + # TODO(joi) Fix this problem where unquoted attributes that + # have a value that is CJK characters causes the regular expression + # match never to return. (culprit is the _ELEMENT regexp( + if False: + html = self.HtmlFromFileWithManualCheck(util.PathFromRoot( + r'grit/test/data/ko_oem_enable_bug.html')) + self.failUnless(True) + + def testRegressionCpuHang(self): + # If this regression occurs, the unit test will never return + html = tr_html.TrHtml( + '''<input type=text size=12 id=advFileTypeEntry [~SHOW-FILETYPE-BOX~] value="[EXT]" name=ext>''') + html.Parse() + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/gather/txt.py b/tools/grit/grit/gather/txt.py new file mode 100644 index 0000000..9bc304d --- /dev/null +++ b/tools/grit/grit/gather/txt.py @@ -0,0 +1,76 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Supports making amessage from a text file. +''' + +import types + +from grit.gather import interface +from grit import tclib +from grit import util + + +class TxtFile(interface.GathererBase): + '''A text file gatherer. Very simple, all text from the file becomes a + single clique. + ''' + + def __init__(self, contents): + super(type(self), self).__init__() + self.text_ = contents + self.clique_ = None + + def Parse(self): + self.clique_ = self.uberclique.MakeClique(tclib.Message(text=self.text_)) + pass + + def GetText(self): + '''Returns the text of what is being gathered.''' + return self.text_ + + def GetTextualIds(self): + return [] + + def GetCliques(self): + '''Returns the MessageClique objects for all translateable portions.''' + return [self.clique_] + + def Translate(self, lang, pseudo_if_not_available=True, + skeleton_gatherer=None, fallback_to_english=False): + return self.clique_.MessageForLanguage(lang, + pseudo_if_not_available, + fallback_to_english).GetRealContent() + + def FromFile(filename_or_stream, extkey=None, encoding = 'cp1252'): + if isinstance(filename_or_stream, types.StringTypes): + filename_or_stream = util.WrapInputStream(file(filename_or_stream, 'rb'), encoding) + return TxtFile(filename_or_stream.read()) + FromFile = staticmethod(FromFile) diff --git a/tools/grit/grit/gather/txt_unittest.py b/tools/grit/grit/gather/txt_unittest.py new file mode 100644 index 0000000..ea5c0b3 --- /dev/null +++ b/tools/grit/grit/gather/txt_unittest.py @@ -0,0 +1,58 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for TxtFile gatherer''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + + +import StringIO +import unittest + +from grit.gather import txt + + +class TxtUnittest(unittest.TestCase): + def testGather(self): + input = StringIO.StringIO('Hello there\nHow are you?') + gatherer = txt.TxtFile.FromFile(input) + gatherer.Parse() + self.failUnless(gatherer.GetText() == input.getvalue()) + self.failUnless(len(gatherer.GetCliques()) == 1) + self.failUnless(gatherer.GetCliques()[0].GetMessage().GetRealContent() == + input.getvalue()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/grd_reader.py b/tools/grit/grit/grd_reader.py new file mode 100644 index 0000000..a6c7ce9 --- /dev/null +++ b/tools/grit/grit/grd_reader.py @@ -0,0 +1,166 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Class for reading GRD files into memory, without processing them. +''' + +import os.path +import types +import xml.sax +import xml.sax.handler + +from grit import exception +from grit.node import base +from grit.node import mapping +from grit import util + + +class StopParsingException(Exception): + '''An exception used to stop parsing.''' + pass + + +class GrdContentHandler(xml.sax.handler.ContentHandler): + def __init__(self, stop_after=None, debug=False): + # Invariant of data: + # 'root' is the root of the parse tree being created, or None if we haven't + # parsed out any elements. + # 'stack' is the a stack of elements that we push new nodes onto and + # pop from when they finish parsing, or [] if we are not currently parsing. + # 'stack[-1]' is the top of the stack. + self.root = None + self.stack = [] + self.stop_after = stop_after + self.debug = debug + + def startElement(self, name, attrs): + assert not self.root or len(self.stack) > 0 + + if self.debug: + attr_list = [] + for attr in attrs.getNames(): + attr_list.append('%s="%s"' % (attr, attrs.getValue(attr))) + if len(attr_list) == 0: attr_list = ['(none)'] + attr_list = ' '.join(attr_list) + print "Starting parsing of element %s with attributes %r" % (name, attr_list) + + typeattr = None + if 'type' in attrs.getNames(): + typeattr = attrs.getValue('type') + + node = mapping.ElementToClass(name, typeattr)() + + if not self.root: + self.root = node + + if len(self.stack) > 0: + self.stack[-1].AddChild(node) + node.StartParsing(name, self.stack[-1]) + else: + node.StartParsing(name, None) + + # Push + self.stack.append(node) + + for attr in attrs.getNames(): + node.HandleAttribute(attr, attrs.getValue(attr)) + + def endElement(self, name): + if self.debug: + print "End parsing of element %s" % name + # Pop + self.stack[-1].EndParsing() + assert len(self.stack) > 0 + self.stack = self.stack[:-1] + if self.stop_after and name == self.stop_after: + raise StopParsingException() + + def characters(self, content): + if self.stack[-1]: + self.stack[-1].AppendContent(content) + + def ignorableWhitespace(self, whitespace): + # TODO(joi) This is not supported by expat. Should use a different XML parser? + pass + + +def Parse(filename_or_stream, dir = None, flexible_root = False, + stop_after=None, debug=False): + '''Parses a GRD file into a tree of nodes (from grit.node). + + If flexible_root is False, the root node must be a <grit> element. Otherwise + it can be any element. The "own" directory of the file will only be fixed up + if the root node is a <grit> element. + + 'dir' should point to the directory of the input file, or be the full path + to the input file (the filename will be stripped). + + If 'stop_after' is provided, the parsing will stop once the first node + with this name has been fully parsed (including all its contents). + + If 'debug' is true, lots of information about the parsing events will be + printed out during parsing of the file. + + Args: + filename_or_stream: './bla.xml' (must be filename if dir is None) + dir: '.' or None (only if filename_or_stream is a filename) + flexible_root: True | False + stop_after: 'inputs' + debug: False + + Return: + Subclass of grit.node.base.Node + + Throws: + grit.exception.Parsing + ''' + handler = GrdContentHandler(stop_after=stop_after, debug=debug) + try: + xml.sax.parse(filename_or_stream, handler) + except StopParsingException: + assert stop_after + pass + except: + raise + + if not flexible_root or hasattr(handler.root, 'SetOwnDir'): + assert isinstance(filename_or_stream, types.StringType) or dir != None + if not dir: + dir = util.dirname(filename_or_stream) + if len(dir) == 0: + dir = '.' + # Fix up the base_dir so it is relative to the input file. + handler.root.SetOwnDir(dir) + return handler.root + + +if __name__ == '__main__': + util.ChangeStdoutEncoding() + print unicode(Parse(sys.argv[1])) diff --git a/tools/grit/grit/grd_reader_unittest.py b/tools/grit/grit/grd_reader_unittest.py new file mode 100644 index 0000000..7f62585 --- /dev/null +++ b/tools/grit/grit/grd_reader_unittest.py @@ -0,0 +1,127 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grd_reader package''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest +import StringIO + +from grit import grd_reader +from grit import constants +from grit import util + + +class GrdReaderUnittest(unittest.TestCase): + def testParsingAndXmlOutput(self): + input = u'''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <includes> + <include name="ID_LOGO" file="images/logo.gif" type="gif" /> + </includes> + <messages> + <if expr="True"> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message> + </if> + </messages> + <structures> + <structure name="IDD_NARROW_DIALOG" file="rc_files/dialogs.rc" type="dialog"> + <skeleton variant_of_revision="3" expr="lang == 'fr-FR'" file="bla.rc" /> + </structure> + <structure name="VS_VERSION_INFO" file="rc_files/version.rc" type="version" /> + </structures> + </release> + <translations> + <file lang="nl" path="nl_translations.xtb" /> + </translations> + <outputs> + <output type="rc_header" filename="resource.h" /> + <output lang="en-US" type="rc_all" filename="resource.rc" /> + </outputs> +</grit>''' + pseudo_file = StringIO.StringIO(input) + tree = grd_reader.Parse(pseudo_file, '.') + output = unicode(tree) + # All but first two lines are the same (sans enc_check) + self.failUnless('\n'.join(input.split('\n')[2:]) == + '\n'.join(output.split('\n')[2:])) + self.failUnless(tree.GetNodeById('IDS_GREETING')) + + + def testStopAfter(self): + input = u'''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <outputs> + <output filename="resource.h" type="rc_header" /> + <output filename="resource.rc" lang="en-US" type="rc_all" /> + </outputs> + <release seq="3"> + <includes> + <include type="gif" name="ID_LOGO" file="images/logo.gif"/> + </includes> + </release> +</grit>''' + pseudo_file = util.WrapInputStream(StringIO.StringIO(input)) + tree = grd_reader.Parse(pseudo_file, '.', stop_after='outputs') + # only an <outputs> child + self.failUnless(len(tree.children) == 1) + self.failUnless(tree.children[0].name == 'outputs') + + def testLongLinesWithComments(self): + input = u'''<?xml version="1.0" encoding="UTF-8"?> +<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <message name="IDS_GREETING" desc="Printed to greet the currently logged in user"> + This is a very long line with no linebreaks yes yes it stretches on <!-- + -->and on <!-- + -->and on! + </message> + </messages> + </release> +</grit>''' + pseudo_file = StringIO.StringIO(input) + tree = grd_reader.Parse(pseudo_file, '.') + + greeting = tree.GetNodeById('IDS_GREETING') + self.failUnless(greeting.GetCliques()[0].GetMessage().GetRealContent() == + 'This is a very long line with no linebreaks yes yes it ' + 'stretches on and on and on!') + +if __name__ == '__main__': + unittest.main() +
\ No newline at end of file diff --git a/tools/grit/grit/grit-todo.xml b/tools/grit/grit/grit-todo.xml new file mode 100644 index 0000000..b8c20fd --- /dev/null +++ b/tools/grit/grit/grit-todo.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="windows-1252"?> +<TODOLIST FILEFORMAT="6" PROJECTNAME="GRIT" NEXTUNIQUEID="56" FILEVERSION="69" LASTMODIFIED="2005-08-19"> + <TASK STARTDATESTRING="2005-04-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38453.49975694" TITLE="check 'name' attribute is unique" TIMEESTUNITS="H" ID="2" PERCENTDONE="100" STARTDATE="38450.00000000" DONEDATESTRING="2005-04-11" POS="22" DONEDATE="38453.00000000"/> + <TASK STARTDATESTRING="2005-04-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38488.48189815" TITLE="import id-calculating code" TIMEESTUNITS="H" ID="3" PERCENTDONE="100" STARTDATE="38450.00000000" DONEDATESTRING="2005-05-16" POS="13" DONEDATE="38488.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38488.48209491" TITLE="Import tool for existing translations" TIMEESTUNITS="H" ID="6" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="12" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00805556" TITLE="Export XMBs" TIMEESTUNITS="H" ID="8" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-08" POS="20" DONEDATE="38511.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00924769" TITLE="Initial Integration" TIMEESTUNITS="H" ID="10" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-08" POS="10" DONEDATE="38511.00000000"> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38496.54048611" TITLE="parser for %s strings" TIMEESTUNITS="H" ID="4" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-24" POS="2" DONEDATE="38496.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38497.00261574" TITLE="import tool for existing RC files" TIMEESTUNITS="H" ID="5" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-25" POS="4" DONEDATE="38497.00000000"> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38496.92990741" TITLE="handle button value= and img alt= in message HTML text" TIMEESTUNITS="H" ID="22" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-05-24" POS="1" DONEDATE="38496.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38497.00258102" TITLE="&nbsp; bug" TIMEESTUNITS="H" ID="23" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-05-25" POS="2" DONEDATE="38497.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61171296" TITLE="grit build" TIMEESTUNITS="H" ID="7" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="6" DONEDATE="38490.00000000"> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61168981" TITLE="use IDs gathered from gatherers for .h file" TIMEESTUNITS="H" ID="20" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="1" DONEDATE="38490.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.55199074" TITLE="SCons Integration" TIMEESTUNITS="H" ID="9" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-01" POS="1" DONEDATE="38504.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61181713" TITLE="handle includes" TIMEESTUNITS="H" ID="12" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="5" DONEDATE="38490.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38507.98567130" TITLE="output translated HTML templates" TIMEESTUNITS="H" ID="25" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-04" POS="3" DONEDATE="38507.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38507.99394676" TITLE="bug: re-escape too much in RC dialogs etc." TIMEESTUNITS="H" ID="38" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-04" POS="7" DONEDATE="38507.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46444444" TITLE="handle structure variants" TIMEESTUNITS="H" ID="11" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="15" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46456019" TITLE="handle include variants" TIMEESTUNITS="H" ID="13" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="17" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46537037" TITLE="handle translateable text for includes (e.g. image text)" TIMEESTUNITS="H" ID="14" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="14" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46712963" TITLE="ddoc" TIMEESTUNITS="H" ID="15" STARTDATE="38488.00000000" POS="4"> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46718750" TITLE="review comments miket" TIMEESTUNITS="H" ID="16" STARTDATE="38488.00000000" POS="2"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46722222" TITLE="review comments pdoyle" TIMEESTUNITS="H" ID="17" STARTDATE="38488.00000000" POS="1"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46732639" TITLE="remove 'extkey' from structure" TIMEESTUNITS="H" ID="18" STARTDATE="38488.00000000" POS="3"/> + <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.53537037" TITLE="add 'encoding' to structure" TIMEESTUNITS="H" ID="19" STARTDATE="38488.00000000" POS="6"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38503.55304398" TITLE="document limitation: emitter doesn't emit the translated HTML templates" TIMEESTUNITS="H" ID="30" STARTDATE="38503.00000000" POS="4"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.58541667" TITLE="add 'internal_comment' to <message>" TIMEESTUNITS="H" ID="32" STARTDATE="38503.00000000" POS="5"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.73391204" TITLE="<outputs> can not have paths (because of SCons integration - goes to build dir)" TIMEESTUNITS="H" ID="36" STARTDATE="38503.00000000" POS="9"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38506.64265046" TITLE="<identifers> and <identifier> nodes" TIMEESTUNITS="H" ID="37" STARTDATE="38503.00000000" POS="10"/> + <TASK STARTDATESTRING="2005-06-23" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38526.62344907" TITLE="<structure> can have 'exclude_from_rc' attribute (default false)" TIMEESTUNITS="H" ID="47" STARTDATE="38526.00000000" POS="8"/> + <TASK STARTDATESTRING="2005-06-23" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38531.94135417" TITLE="add 'enc_check' to <grit>" TIMEESTUNITS="H" ID="48" STARTDATE="38526.00000000" POS="7"/> + </TASK> + <TASK STARTDATESTRING="2005-05-18" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38492.51549769" TITLE="handle nontranslateable messages (in MessageClique?)" TIMEESTUNITS="H" ID="21" PERCENTDONE="100" STARTDATE="38490.00000000" DONEDATESTRING="2005-06-16" POS="16" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70454861" TITLE="ask cprince about SCons builder in new mk system" TIMEESTUNITS="H" ID="24" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-02" POS="25" DONEDATE="38505.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.57436343" TITLE="fix AOL resource in trunk ("???????")" TIMEESTUNITS="H" ID="26" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-01" POS="19" DONEDATE="38504.00000000"/> + <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38498.53893519" TITLE="rc_all vs. rc_translateable vs. rc_nontranslateable" TIMEESTUNITS="H" ID="27" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-16" POS="6" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38509.45532407" TITLE="make separate .grb "outputs" file (and change SCons integ) (??)" TIMEESTUNITS="H" ID="28" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-06" POS="8" DONEDATE="38509.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00939815" TITLE="fix unit tests so they run from any directory" TIMEESTUNITS="H" ID="33" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-08" POS="18" DONEDATE="38511.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38508.96640046" TITLE="Change R4 tool to CC correct team(s) on GRIT changes" TIMEESTUNITS="H" ID="39" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-05" POS="23" DONEDATE="38508.00000000"/> + <TASK STARTDATESTRING="2005-06-07" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00881944" TITLE="Document why wrapper.rc" TIMEESTUNITS="H" ID="40" PERCENTDONE="100" STARTDATE="38510.00000000" DONEDATESTRING="2005-06-08" POS="21" DONEDATE="38511.00000000"/> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00804398" TITLE="import XTBs" TIMEESTUNITS="H" ID="41" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="11" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00875000" TITLE="Nightly build integration" TIMEESTUNITS="H" ID="42" STARTDATE="38511.00000000" POS="3"/> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00891204" TITLE="BUGS" TIMEESTUNITS="H" ID="43" STARTDATE="38511.00000000" POS="24"> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38513.03375000" TITLE="Should report error if RC-section structure refers to does not exist" TIMEESTUNITS="H" ID="44" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-10" POS="1" DONEDATE="38513.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00981481" TITLE="NEW FEATURES" TIMEESTUNITS="H" ID="45" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="7" DONEDATE="38519.00000000"> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70077546" TITLE="Implement line-continuation feature (\ at end of line?)" TIMEESTUNITS="H" ID="34" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-16" POS="1" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70262731" TITLE="Implement conditional inclusion & reflect the conditionals from R3 RC file" TIMEESTUNITS="H" ID="35" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-16" POS="2" DONEDATE="38519.00000000"/> + </TASK> + <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.01046296" TITLE="TC integration (one-way TO the TC)" TIMEESTUNITS="H" ID="46" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="5" DONEDATE="38519.00000000"/> + <TASK STARTDATESTRING="2005-06-30" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38533.59072917" TITLE="bazaar20 ad for GRIT help" TIMEESTUNITS="H" ID="49" STARTDATE="38533.00000000" POS="2"> + <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72346065" TITLE="bazaar20 ideas" TIMEESTUNITS="H" ID="51" STARTDATE="38583.00000000" POS="1"> + <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72354167" TITLE="GUI for adding/editing messages" TIMEESTUNITS="H" ID="52" STARTDATE="38583.00000000" POS="2"/> + <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72365741" TITLE="XLIFF import/export" TIMEESTUNITS="H" ID="54" STARTDATE="38583.00000000" POS="1"/> + </TASK> + </TASK> + <TASK STARTDATESTRING="2005-06-30" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.73721065" TITLE="internal_comment for all resource nodes (not just <message>)" TIMEESTUNITS="H" ID="50" PERCENTDONE="100" STARTDATE="38533.00000000" DONEDATESTRING="2005-08-19" POS="9" DONEDATE="38583.73721065"/> + <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.73743056" TITLE="Preserve XML comments - this gives us line continuation and more" TIMEESTUNITS="H" ID="55" STARTDATE="38583.72326389" POS="1"/> +</TODOLIST> diff --git a/tools/grit/grit/grit_runner.py b/tools/grit/grit/grit_runner.py new file mode 100644 index 0000000..e8af95a --- /dev/null +++ b/tools/grit/grit/grit_runner.py @@ -0,0 +1,228 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Command processor for GRIT. This is the script you invoke to run the various +GRIT tools. +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import getopt + +from grit import util + +import grit.exception + +import grit.tool.build +import grit.tool.count +import grit.tool.diff_structures +import grit.tool.menu_from_parts +import grit.tool.newgrd +import grit.tool.resize +import grit.tool.rc2grd +import grit.tool.test +import grit.tool.transl2tc +import grit.tool.unit + + +# Copyright notice +_COPYRIGHT = ''' +GRIT - the Google Resource and Internationalization Tool +Copyright (c) Google Inc. %d +''' % util.GetCurrentYear() + +# Keys for the following map +_CLASS = 1 +_REQUIRES_INPUT = 2 +_HIDDEN = 3 # optional key - presence indicates tool is hidden + + +# Maps tool names to the tool's module. Done as a list of (key, value) tuples +# instead of a map to preserve ordering. +_TOOLS = [ + ['build', { _CLASS : grit.tool.build.RcBuilder, _REQUIRES_INPUT : True }], + ['newgrd', { _CLASS : grit.tool.newgrd.NewGrd, _REQUIRES_INPUT : False }], + ['rc2grd', { _CLASS : grit.tool.rc2grd.Rc2Grd, _REQUIRES_INPUT : False }], + ['transl2tc', { _CLASS : grit.tool.transl2tc.TranslationToTc, + _REQUIRES_INPUT : False }], + ['sdiff', { _CLASS : grit.tool.diff_structures.DiffStructures, + _REQUIRES_INPUT : False }], + ['resize', { _CLASS : grit.tool.resize.ResizeDialog, _REQUIRES_INPUT : True }], + ['unit', { _CLASS : grit.tool.unit.UnitTestTool, _REQUIRES_INPUT : False }], + ['count', { _CLASS : grit.tool.count.CountMessage, _REQUIRES_INPUT : True }], + ['test', { _CLASS: grit.tool.test.TestTool, _REQUIRES_INPUT : True, _HIDDEN : True }], + ['menufromparts', { _CLASS: grit.tool.menu_from_parts.MenuTranslationsFromParts, + _REQUIRES_INPUT : True, _HIDDEN : True }], +] + + +def PrintUsage(): + tool_list = '' + for (tool, info) in _TOOLS: + if not _HIDDEN in info.keys(): + tool_list += ' %-12s %s\n' % (tool, info[_CLASS]().ShortDescription()) + + # TODO(joi) Put these back into the usage when appropriate: + # + # -d Work disconnected. This causes GRIT not to attempt connections with + # e.g. Perforce. + # + # -c Use the specified Perforce CLIENT when talking to Perforce. + print '''Usage: grit [GLOBALOPTIONS] TOOL [args to tool] + +Global options: + + -i INPUT Specifies the INPUT file to use (a .grd file). If this is not + specified, GRIT will look for the environment variable GRIT_INPUT. + If it is not present either, GRIT will try to find an input file + named 'resource.grd' in the current working directory. + + -v Print more verbose runtime information. + + -x Print extremely verbose runtime information. Implies -v + + -p FNAME Specifies that GRIT should profile its execution and output the + results to the file FNAME. + +Tools: + + TOOL can be one of the following: +%s + For more information on how to use a particular tool, and the specific + arguments you can send to that tool, execute 'grit help TOOL' +''' % (tool_list) + + +class Options(object): + '''Option storage and parsing.''' + + def __init__(self): + self.disconnected = False + self.client = '' + self.input = None + self.verbose = False + self.extra_verbose = False + self.output_stream = sys.stdout + self.profile_dest = None + + def ReadOptions(self, args): + '''Reads options from the start of args and returns the remainder.''' + (opts, args) = getopt.getopt(args, 'g:dvxc:i:p:') + for (key, val) in opts: + if key == '-d': self.disconnected = True + elif key == '-c': self.client = val + elif key == '-i': self.input = val + elif key == '-v': + self.verbose = True + util.verbose = True + elif key == '-x': + self.verbose = True + util.verbose = True + self.extra_verbose = True + util.extra_verbose = True + elif key == '-p': self.profile_dest = val + + if not self.input: + if 'GRIT_INPUT' in os.environ: + self.input = os.environ['GRIT_INPUT'] + else: + self.input = 'resource.grd' + + return args + + def __repr__(self): + return '(disconnected: %d, verbose: %d, client: %s, input: %s)' % ( + self.disconnected, self.verbose, self.client, self.input) + + +def _GetToolInfo(tool): + '''Returns the info map for the tool named 'tool' or None if there is no + such tool.''' + matches = filter(lambda t: t[0] == tool, _TOOLS) + if not len(matches): + return None + else: + return matches[0][1] + + +def Main(args): + '''Parses arguments and does the appropriate thing.''' + util.ChangeStdoutEncoding() + print _COPYRIGHT + + if not len(args) or len(args) == 1 and args[0] == 'help': + PrintUsage() + return 0 + elif len(args) == 2 and args[0] == 'help': + tool = args[1].lower() + if not _GetToolInfo(tool): + print "No such tool. Try running 'grit help' for a list of tools." + return 2 + + print ("Help for 'grit %s' (for general help, run 'grit help'):\n" + % (tool)) + print _GetToolInfo(tool)[_CLASS].__doc__ + return 0 + else: + options = Options() + args = options.ReadOptions(args) # args may be shorter after this + tool = args[0] + if not _GetToolInfo(tool): + print "No such tool. Try running 'grit help' for a list of tools." + return 2 + + try: + if _GetToolInfo(tool)[_REQUIRES_INPUT]: + os.stat(options.input) + except OSError: + print ('Input file %s not found.\n' + 'To specify a different input file:\n' + ' 1. Use the GRIT_INPUT environment variable.\n' + ' 2. Use the -i command-line option. This overrides ' + 'GRIT_INPUT.\n' + ' 3. Specify neither GRIT_INPUT or -i and GRIT will try to load ' + "'resource.grd'\n" + ' from the current directory.' % options.input) + return 2 + + toolobject = _GetToolInfo(tool)[_CLASS]() + if options.profile_dest: + import hotshot + prof = hotshot.Profile(options.profile_dest) + prof.runcall(toolobject.Run, options, args[1:]) + else: + toolobject.Run(options, args[1:]) + + +if __name__ == '__main__': + sys.exit(Main(sys.argv[1:])) diff --git a/tools/grit/grit/grit_runner_unittest.py b/tools/grit/grit/grit_runner_unittest.py new file mode 100644 index 0000000..0639f0c --- /dev/null +++ b/tools/grit/grit/grit_runner_unittest.py @@ -0,0 +1,65 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.py''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest +import StringIO + +from grit import util +import grit.grit_runner + +class OptionArgsUnittest(unittest.TestCase): + def setUp(self): + self.buf = StringIO.StringIO() + self.old_stdout = sys.stdout + sys.stdout = self.buf + + def tearDown(self): + sys.stdout = self.old_stdout + + def testSimple(self): + grit.grit_runner.Main(['-i', + util.PathFromRoot('grit/test/data/simple-input.xml'), + '-d', 'test', 'bla', 'voff', 'ga']) + output = self.buf.getvalue() + self.failUnless(output.count('disconnected')) + self.failUnless(output.count("'test'") == 0) # tool name doesn't occur + self.failUnless(output.count('bla')) + self.failUnless(output.count('simple-input.xml')) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/node/__init__.py b/tools/grit/grit/node/__init__.py new file mode 100644 index 0000000..4980b3e --- /dev/null +++ b/tools/grit/grit/node/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Package 'grit.node' +''' + +pass
\ No newline at end of file diff --git a/tools/grit/grit/node/base.py b/tools/grit/grit/node/base.py new file mode 100644 index 0000000..38cfd1f --- /dev/null +++ b/tools/grit/grit/node/base.py @@ -0,0 +1,548 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Base types for nodes in a GRIT resource tree. +''' + +import os +import types +from xml.sax import saxutils + +from grit import exception +from grit import util +from grit import clique +import grit.format.interface + + +class Node(grit.format.interface.ItemFormatter): + '''An item in the tree that has children. Also implements the + ItemFormatter interface to allow formatting a node as a GRD document.''' + + # Valid content types that can be returned by _ContentType() + _CONTENT_TYPE_NONE = 0 # No CDATA content but may have children + _CONTENT_TYPE_CDATA = 1 # Only CDATA, no children. + _CONTENT_TYPE_MIXED = 2 # CDATA and children, possibly intermingled + + def __init__(self): + self.children = [] # A list of child elements + self.mixed_content = [] # A list of u'' and/or child elements (this + # duplicates 'children' but + # is needed to preserve markup-type content). + self.name = u'' # The name of this element + self.attrs = {} # The set of attributes (keys to values) + self.parent = None # Our parent unless we are the root element. + self.uberclique = None # Allows overriding uberclique for parts of tree + + def __iter__(self): + '''An in-order iteration through the tree that this node is the + root of.''' + return self.inorder() + + def inorder(self): + '''Generator that generates first this node, then the same generator for + any child nodes.''' + yield self + for child in self.children: + for iterchild in child.inorder(): + yield iterchild + + def GetRoot(self): + '''Returns the root Node in the tree this Node belongs to.''' + curr = self + while curr.parent: + curr = curr.parent + return curr + + # TODO(joi) Use this (currently untested) optimization?: + #if hasattr(self, '_root'): + # return self._root + #curr = self + #while curr.parent and not hasattr(curr, '_root'): + # curr = curr.parent + #if curr.parent: + # self._root = curr._root + #else: + # self._root = curr + #return self._root + + def StartParsing(self, name, parent): + '''Called at the start of parsing. + + Args: + name: u'elementname' + parent: grit.node.base.Node or subclass or None + ''' + assert isinstance(name, types.StringTypes) + assert not parent or isinstance(parent, Node) + self.name = name + self.parent = parent + + def AddChild(self, child): + '''Adds a child to the list of children of this node, if it is a valid + child for the node.''' + assert isinstance(child, Node) + if (not self._IsValidChild(child) or + self._ContentType() == self._CONTENT_TYPE_CDATA): + if child.parent: + explanation = 'child %s of parent %s' % (child.name, child.parent.name) + else: + explanation = 'node %s with no parent' % child.name + raise exception.UnexpectedChild(explanation) + self.children.append(child) + self.mixed_content.append(child) + + def RemoveChild(self, child_id): + '''Removes the first node that has a "name" attribute which + matches "child_id" in the list of immediate children of + this node. + + Args: + child_id: String identifying the child to be removed + ''' + index = 0 + # Safe not to copy since we only remove the first element found + for child in self.children: + name_attr = child.attrs['name'] + if name_attr == child_id: + self.children.pop(index) + self.mixed_content.pop(index) + break + index += 1 + + def AppendContent(self, content): + '''Appends a chunk of text as content of this node. + + Args: + content: u'hello' + + Return: + None + ''' + assert isinstance(content, types.StringTypes) + if self._ContentType() != self._CONTENT_TYPE_NONE: + self.mixed_content.append(content) + elif content.strip() != '': + raise exception.UnexpectedContent() + + def HandleAttribute(self, attrib, value): + '''Informs the node of an attribute that was parsed out of the GRD file + for it. + + Args: + attrib: 'name' + value: 'fooblat' + + Return: + None + ''' + assert isinstance(attrib, types.StringTypes) + assert isinstance(value, types.StringTypes) + if self._IsValidAttribute(attrib, value): + self.attrs[attrib] = value + else: + raise exception.UnexpectedAttribute(attrib) + + def EndParsing(self): + '''Called at the end of parsing.''' + + # TODO(joi) Rewrite this, it's extremely ugly! + if len(self.mixed_content): + if isinstance(self.mixed_content[0], types.StringTypes): + # Remove leading and trailing chunks of pure whitespace. + while (len(self.mixed_content) and + isinstance(self.mixed_content[0], types.StringTypes) and + self.mixed_content[0].strip() == ''): + self.mixed_content = self.mixed_content[1:] + # Strip leading and trailing whitespace from mixed content chunks + # at front and back. + if (len(self.mixed_content) and + isinstance(self.mixed_content[0], types.StringTypes)): + self.mixed_content[0] = self.mixed_content[0].lstrip() + # Remove leading and trailing ''' (used to demarcate whitespace) + if (len(self.mixed_content) and + isinstance(self.mixed_content[0], types.StringTypes)): + if self.mixed_content[0].startswith("'''"): + self.mixed_content[0] = self.mixed_content[0][3:] + if len(self.mixed_content): + if isinstance(self.mixed_content[-1], types.StringTypes): + # Same stuff all over again for the tail end. + while (len(self.mixed_content) and + isinstance(self.mixed_content[-1], types.StringTypes) and + self.mixed_content[-1].strip() == ''): + self.mixed_content = self.mixed_content[:-1] + if (len(self.mixed_content) and + isinstance(self.mixed_content[-1], types.StringTypes)): + self.mixed_content[-1] = self.mixed_content[-1].rstrip() + if (len(self.mixed_content) and + isinstance(self.mixed_content[-1], types.StringTypes)): + if self.mixed_content[-1].endswith("'''"): + self.mixed_content[-1] = self.mixed_content[-1][:-3] + + # Check that all mandatory attributes are there. + for node_mandatt in self.MandatoryAttributes(): + mandatt_list = [] + if node_mandatt.find('|') >= 0: + mandatt_list = node_mandatt.split('|') + else: + mandatt_list.append(node_mandatt) + + mandatt_option_found = False + for mandatt in mandatt_list: + assert mandatt not in self.DefaultAttributes().keys() + if mandatt in self.attrs: + if not mandatt_option_found: + mandatt_option_found = True + else: + raise exception.MutuallyExclusiveMandatoryAttribute(mandatt) + + if not mandatt_option_found: + raise exception.MissingMandatoryAttribute(mandatt) + + # Add default attributes if not specified in input file. + for defattr in self.DefaultAttributes(): + if not defattr in self.attrs: + self.attrs[defattr] = self.DefaultAttributes()[defattr] + + def GetCdata(self): + '''Returns all CDATA of this element, concatenated into a single + string. Note that this ignores any elements embedded in CDATA.''' + return ''.join(filter(lambda c: isinstance(c, types.StringTypes), + self.mixed_content)) + + def __unicode__(self): + '''Returns this node and all nodes below it as an XML document in a Unicode + string.''' + header = u'<?xml version="1.0" encoding="UTF-8"?>\n' + return header + self.FormatXml() + + # Compliance with ItemFormatter interface. + def Format(self, item, lang_re = None, begin_item=True): + if not begin_item: + return '' + else: + return item.FormatXml() + + def FormatXml(self, indent = u'', one_line = False): + '''Returns this node and all nodes below it as an XML + element in a Unicode string. This differs from __unicode__ in that it does + not include the <?xml> stuff at the top of the string. If one_line is true, + children and CDATA are layed out in a way that preserves internal + whitespace. + ''' + assert isinstance(indent, types.StringTypes) + + content_one_line = (one_line or + self._ContentType() == self._CONTENT_TYPE_MIXED) + inside_content = self.ContentsAsXml(indent, content_one_line) + + # Then the attributes for this node. + attribs = u' ' + for (attrib, value) in self.attrs.iteritems(): + # Only print an attribute if it is other than the default value. + if (not self.DefaultAttributes().has_key(attrib) or + value != self.DefaultAttributes()[attrib]): + attribs += u'%s=%s ' % (attrib, saxutils.quoteattr(value)) + attribs = attribs.rstrip() # if no attribs, we end up with '', otherwise + # we end up with a space-prefixed string + + # Finally build the XML for our node and return it + if len(inside_content) > 0: + if one_line: + return u'<%s%s>%s</%s>' % (self.name, attribs, inside_content, self.name) + elif content_one_line: + return u'%s<%s%s>\n%s %s\n%s</%s>' % ( + indent, self.name, attribs, + indent, inside_content, + indent, self.name) + else: + return u'%s<%s%s>\n%s\n%s</%s>' % ( + indent, self.name, attribs, + inside_content, + indent, self.name) + else: + return u'%s<%s%s />' % (indent, self.name, attribs) + + def ContentsAsXml(self, indent, one_line): + '''Returns the contents of this node (CDATA and child elements) in XML + format. If 'one_line' is true, the content will be laid out on one line.''' + assert isinstance(indent, types.StringTypes) + + # Build the contents of the element. + inside_parts = [] + last_item = None + for mixed_item in self.mixed_content: + if isinstance(mixed_item, Node): + inside_parts.append(mixed_item.FormatXml(indent + u' ', one_line)) + if not one_line: + inside_parts.append(u'\n') + else: + message = mixed_item + # If this is the first item and it starts with whitespace, we add + # the ''' delimiter. + if not last_item and message.lstrip() != message: + message = u"'''" + message + inside_parts.append(util.EncodeCdata(message)) + last_item = mixed_item + + # If there are only child nodes and no cdata, there will be a spurious + # trailing \n + if len(inside_parts) and inside_parts[-1] == '\n': + inside_parts = inside_parts[:-1] + + # If the last item is a string (not a node) and ends with whitespace, + # we need to add the ''' delimiter. + if (isinstance(last_item, types.StringTypes) and + last_item.rstrip() != last_item): + inside_parts[-1] = inside_parts[-1] + u"'''" + + return u''.join(inside_parts) + + def RunGatherers(self, recursive=0, debug=False): + '''Runs all gatherers on this object, which may add to the data stored + by the object. If 'recursive' is true, will call RunGatherers() recursively + on all child nodes first. If 'debug' is True, will print out information + as it is running each nodes' gatherers. + + Gatherers for <translations> child nodes will always be run after all other + child nodes have been gathered. + ''' + if recursive: + process_last = [] + for child in self.children: + if child.name == 'translations': + process_last.append(child) + else: + child.RunGatherers(recursive=recursive, debug=debug) + for child in process_last: + child.RunGatherers(recursive=recursive, debug=debug) + + def ItemFormatter(self, type): + '''Returns an instance of the item formatter for this object of the + specified type, or None if not supported. + + Args: + type: 'rc-header' + + Return: + (object RcHeaderItemFormatter) + ''' + if type == 'xml': + return self + else: + return None + + def SatisfiesOutputCondition(self): + '''Returns true if this node is either not a child of an <if> element + or if it is a child of an <if> element and the conditions for it being + output are satisfied. + + Used to determine whether to return item formatters for formats that + obey conditional output of resources (e.g. the RC formatters). + ''' + from grit.node import misc + if not self.parent or not isinstance(self.parent, misc.IfNode): + return True + else: + return self.parent.IsConditionSatisfied() + + def _IsValidChild(self, child): + '''Returns true if 'child' is a valid child of this node. + Overridden by subclasses.''' + return False + + def _IsValidAttribute(self, name, value): + '''Returns true if 'name' is the name of a valid attribute of this element + and 'value' is a valid value for that attribute. Overriden by + subclasses unless they have only mandatory attributes.''' + return (name in self.MandatoryAttributes() or + name in self.DefaultAttributes()) + + def _ContentType(self): + '''Returns the type of content this element can have. Overridden by + subclasses. The content type can be one of the _CONTENT_TYPE_XXX constants + above.''' + return self._CONTENT_TYPE_NONE + + def MandatoryAttributes(self): + '''Returns a list of attribute names that are mandatory (non-optional) + on the current element. One can specify a list of + "mutually exclusive mandatory" attributes by specifying them as one + element in the list, separated by a "|" character. + ''' + return [] + + def DefaultAttributes(self): + '''Returns a dictionary of attribute names that have defaults, mapped to + the default value. Overridden by subclasses.''' + return {} + + def GetCliques(self): + '''Returns all MessageClique objects belonging to this node. Overridden + by subclasses. + + Return: + [clique1, clique2] or [] + ''' + return [] + + def ToRealPath(self, path_from_basedir): + '''Returns a real path (which can be absolute or relative to the current + working directory), given a path that is relative to the base directory + set for the GRIT input file. + + Args: + path_from_basedir: '..' + + Return: + 'resource' + ''' + return util.normpath(os.path.join(self.GetRoot().GetBaseDir(), + path_from_basedir)) + + def FilenameToOpen(self): + '''Returns a path, either absolute or relative to the current working + directory, that points to the file the node refers to. This is only valid + for nodes that have a 'file' or 'path' attribute. Note that the attribute + is a path to the file relative to the 'base-dir' of the .grd file, whereas + this function returns a path that can be used to open the file.''' + file_attribute = 'file' + if not file_attribute in self.attrs: + file_attribute = 'path' + return self.ToRealPath(self.attrs[file_attribute]) + + def UberClique(self): + '''Returns the uberclique that should be used for messages originating in + a given node. If the node itself has its uberclique set, that is what we + use, otherwise we search upwards until we find one. If we do not find one + even at the root node, we set the root node's uberclique to a new + uberclique instance. + ''' + node = self + while not node.uberclique and node.parent: + node = node.parent + if not node.uberclique: + node.uberclique = clique.UberClique() + return node.uberclique + + def IsTranslateable(self): + '''Returns false if the node has contents that should not be translated, + otherwise returns false (even if the node has no contents). + ''' + if not 'translateable' in self.attrs: + return True + else: + return self.attrs['translateable'] == 'true' + + def GetNodeById(self, id): + '''Returns the node in the subtree parented by this node that has a 'name' + attribute matching 'id'. Returns None if no such node is found. + ''' + for node in self: + if 'name' in node.attrs and node.attrs['name'] == id: + return node + return None + + def GetTextualIds(self): + '''Returns the textual ids of this node, if it has some. + Otherwise it just returns None. + ''' + if 'name' in self.attrs: + return [self.attrs['name']] + return None + + def EvaluateCondition(self, expr): + '''Returns true if and only if the Python expression 'expr' evaluates + to true. + + The expression is given a few local variables: + - 'lang' is the language currently being output + - 'defs' is a map of C preprocessor-style define names to their values + - 'pp_ifdef(define)' which behaves just like the C preprocessors #ifdef, + i.e. it is shorthand for "define in defs" + - 'pp_if(define)' which behaves just like the C preprocessor's #if, i.e. + it is shorthand for "define in defs and defs[define]". + ''' + root = self.GetRoot() + lang = '' + defs = {} + def pp_ifdef(define): + return define in defs + def pp_if(define): + return define in defs and defs[define] + if hasattr(root, 'output_language'): + lang = root.output_language + if hasattr(root, 'defines'): + defs = root.defines + return eval(expr, {}, + {'lang' : lang, + 'defs' : defs, + 'pp_ifdef' : pp_ifdef, + 'pp_if' : pp_if}) + + def OnlyTheseTranslations(self, languages): + '''Turns off loading of translations for languages not in the provided list. + + Attrs: + languages: ['fr', 'zh_cn'] + ''' + for node in self: + if (hasattr(node, 'IsTranslation') and + node.IsTranslation() and + node.GetLang() not in languages): + node.DisableLoading() + + def PseudoIsAllowed(self): + '''Returns true if this node is allowed to use pseudo-translations. This + is true by default, unless this node is within a <release> node that has + the allow_pseudo attribute set to false. + ''' + p = self.parent + while p: + if 'allow_pseudo' in p.attrs: + return (p.attrs['allow_pseudo'].lower() == 'true') + p = p.parent + return True + + def ShouldFallbackToEnglish(self): + '''Returns true iff this node should fall back to English when + pseudotranslations are disabled and no translation is available for a + given message. + ''' + p = self.parent + while p: + if 'fallback_to_english' in p.attrs: + return (p.attrs['fallback_to_english'].lower() == 'true') + p = p.parent + return False + +class ContentNode(Node): + '''Convenience baseclass for nodes that can have content.''' + def _ContentType(self): + return self._CONTENT_TYPE_MIXED diff --git a/tools/grit/grit/node/base_unittest.py b/tools/grit/grit/node/base_unittest.py new file mode 100644 index 0000000..8e13d0f --- /dev/null +++ b/tools/grit/grit/node/base_unittest.py @@ -0,0 +1,193 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for base.Node functionality (as used in various subclasses)''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest + +from grit.node import base +from grit.node import message +from grit.node import structure +from grit.node import variant + +def MakePlaceholder(phname='BINGO'): + ph = message.PhNode() + ph.StartParsing(u'ph', None) + ph.HandleAttribute(u'name', phname) + ph.AppendContent(u'bongo') + ph.EndParsing() + return ph + + +class NodeUnittest(unittest.TestCase): + def testWhitespaceHandling(self): + # We test using the Message node type. + node = message.MessageNode() + node.StartParsing(u'hello', None) + node.HandleAttribute(u'name', u'bla') + node.AppendContent(u" ''' two spaces ") + node.EndParsing() + self.failUnless(node.GetCdata() == u' two spaces') + + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'bla') + node.AppendContent(u" two spaces ''' ") + node.EndParsing() + self.failUnless(node.GetCdata() == u'two spaces ') + + def testWhitespaceHandlingWithChildren(self): + # We test using the Message node type. + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'bla') + node.AppendContent(u" ''' two spaces ") + node.AddChild(MakePlaceholder()) + node.AppendContent(u' space before and after ') + node.AddChild(MakePlaceholder('BONGO')) + node.AppendContent(u" space before two after '''") + node.EndParsing() + self.failUnless(node.mixed_content[0] == u' two spaces ') + self.failUnless(node.mixed_content[2] == u' space before and after ') + self.failUnless(node.mixed_content[-1] == u' space before two after ') + + def testXmlFormatMixedContent(self): + # Again test using the Message node type, because it is the only mixed + # content node. + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'name') + node.AppendContent(u'Hello <young> ') + + ph = message.PhNode() + ph.StartParsing(u'ph', None) + ph.HandleAttribute(u'name', u'USERNAME') + ph.AppendContent(u'$1') + ex = message.ExNode() + ex.StartParsing(u'ex', None) + ex.AppendContent(u'Joi') + ex.EndParsing() + ph.AddChild(ex) + ph.EndParsing() + + node.AddChild(ph) + node.EndParsing() + + non_indented_xml = node.Format(node) + self.failUnless(non_indented_xml == u'<message name="name">\n Hello ' + u'<young> <ph name="USERNAME">$1<ex>Joi</ex></ph>' + u'\n</message>') + + indented_xml = node.FormatXml(u' ') + self.failUnless(indented_xml == u' <message name="name">\n Hello ' + u'<young> <ph name="USERNAME">$1<ex>Joi</ex></ph>' + u'\n </message>') + + def testXmlFormatMixedContentWithLeadingWhitespace(self): + # Again test using the Message node type, because it is the only mixed + # content node. + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'name') + node.AppendContent(u"''' Hello <young> ") + + ph = message.PhNode() + ph.StartParsing(u'ph', None) + ph.HandleAttribute(u'name', u'USERNAME') + ph.AppendContent(u'$1') + ex = message.ExNode() + ex.StartParsing(u'ex', None) + ex.AppendContent(u'Joi') + ex.EndParsing() + ph.AddChild(ex) + ph.EndParsing() + + node.AddChild(ph) + node.AppendContent(u" yessiree '''") + node.EndParsing() + + non_indented_xml = node.Format(node) + self.failUnless(non_indented_xml == + u"<message name=\"name\">\n ''' Hello" + u' <young> <ph name="USERNAME">$1<ex>Joi</ex></ph>' + u" yessiree '''\n</message>") + + indented_xml = node.FormatXml(u' ') + self.failUnless(indented_xml == + u" <message name=\"name\">\n ''' Hello" + u' <young> <ph name="USERNAME">$1<ex>Joi</ex></ph>' + u" yessiree '''\n </message>") + + self.failUnless(node.GetNodeById('name')) + + def testXmlFormatContentWithEntities(self): + '''Tests a bug where would not be escaped correctly.''' + from grit import tclib + msg_node = message.MessageNode.Construct(None, tclib.Message( + text = 'BEGIN_BOLDHelloWHITESPACEthere!END_BOLD Bingo!', + placeholders = [ + tclib.Placeholder('BEGIN_BOLD', '<b>', 'bla'), + tclib.Placeholder('WHITESPACE', ' ', 'bla'), + tclib.Placeholder('END_BOLD', '</b>', 'bla')]), + 'BINGOBONGO') + xml = msg_node.FormatXml() + self.failUnless(xml.find(' ') == -1, 'should have no entities') + + def testIter(self): + # First build a little tree of message and ph nodes. + node = message.MessageNode() + node.StartParsing(u'message', None) + node.HandleAttribute(u'name', u'bla') + node.AppendContent(u" ''' two spaces ") + node.AppendContent(u' space before and after ') + ph = message.PhNode() + ph.StartParsing(u'ph', None) + ph.AddChild(message.ExNode()) + ph.HandleAttribute(u'name', u'BINGO') + ph.AppendContent(u'bongo') + node.AddChild(ph) + node.AddChild(message.PhNode()) + node.AppendContent(u" space before two after '''") + + order = [message.MessageNode, message.PhNode, message.ExNode, message.PhNode] + for n in node: + self.failUnless(type(n) == order[0]) + order = order[1:] + self.failUnless(len(order) == 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/node/custom/__init__.py b/tools/grit/grit/node/custom/__init__.py new file mode 100644 index 0000000..0a30448 --- /dev/null +++ b/tools/grit/grit/node/custom/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/python2.4 +# Copyright 2004 Google Inc. +# All Rights Reserved. +# Author: Joi Sigurdsson <joi@google.com> + +'''Package 'grit.node.custom' +''' + +pass
\ No newline at end of file diff --git a/tools/grit/grit/node/custom/filename.py b/tools/grit/grit/node/custom/filename.py new file mode 100644 index 0000000..1f89b9c --- /dev/null +++ b/tools/grit/grit/node/custom/filename.py @@ -0,0 +1,54 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''A CustomType for filenames.''' + +import re + +from grit import clique + + +class WindowsFilename(clique.CustomType): + '''Validates that messages can be used as Windows filenames, and strips + illegal characters out of translations. + ''' + + BANNED = re.compile('\+|:|\/|\\\\|\*|\?|\"|\<|\>|\|') + + def Validate(self, message): + return not self.BANNED.search(message.GetPresentableContent()) + + def ValidateAndModify(self, lang, translation): + is_ok = self.Validate(translation) + self.ModifyEachTextPart(lang, translation) + return is_ok + + def ModifyTextPart(self, lang, text): + return self.BANNED.sub(' ', text)
\ No newline at end of file diff --git a/tools/grit/grit/node/custom/filename_unittest.py b/tools/grit/grit/node/custom/filename_unittest.py new file mode 100644 index 0000000..5c7cba9 --- /dev/null +++ b/tools/grit/grit/node/custom/filename_unittest.py @@ -0,0 +1,58 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.node.custom.filename''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) + +import unittest +from grit.node.custom import filename +from grit import clique +from grit import tclib + + +class WindowsFilenameUnittest(unittest.TestCase): + + def testValidate(self): + factory = clique.UberClique() + msg = tclib.Message(text='Bingo bongo') + c = factory.MakeClique(msg) + c.SetCustomType(filename.WindowsFilename()) + translation = tclib.Translation(id=msg.GetId(), text='Bilingo bolongo:') + c.AddTranslation(translation, 'fr') + self.failUnless(c.MessageForLanguage('fr').GetRealContent() == 'Bilingo bolongo ') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/node/empty.py b/tools/grit/grit/node/empty.py new file mode 100644 index 0000000..a2aee54 --- /dev/null +++ b/tools/grit/grit/node/empty.py @@ -0,0 +1,94 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Container nodes that don't have any logic. +''' + + +from grit.node import base +from grit.node import include +from grit.node import structure +from grit.node import message +from grit.node import io +from grit.node import misc + + +class GroupingNode(base.Node): + '''Base class for all the grouping elements (<structures>, <includes>, + <messages> and <identifiers>).''' + def DefaultAttributes(self): + return { + 'first_id' : '', + 'comment' : '', + 'fallback_to_english' : 'false', + } + + +class IncludesNode(GroupingNode): + '''The <includes> element.''' + def _IsValidChild(self, child): + return isinstance(child, (include.IncludeNode, misc.IfNode)) + + +class MessagesNode(GroupingNode): + '''The <messages> element.''' + def _IsValidChild(self, child): + return isinstance(child, (message.MessageNode, misc.IfNode)) + + def ItemFormatter(self, t): + '''Return the stringtable itemformatter if an RC is being formatted.''' + if t in ['rc_all', 'rc_translateable', 'rc_nontranslateable']: + from grit.format import rc # avoid circular dep by importing here + return rc.StringTable() + + +class StructuresNode(GroupingNode): + '''The <structures> element.''' + def _IsValidChild(self, child): + return isinstance(child, (structure.StructureNode, misc.IfNode)) + + +class TranslationsNode(base.Node): + '''The <translations> element.''' + def _IsValidChild(self, child): + return isinstance(child, io.FileNode) + + +class OutputsNode(base.Node): + '''The <outputs> element.''' + def _IsValidChild(self, child): + return isinstance(child, io.OutputNode) + + +class IdentifiersNode(GroupingNode): + '''The <identifiers> element.''' + def _IsValidChild(self, child): + from grit.node import misc + return isinstance(child, misc.IdentifierNode) diff --git a/tools/grit/grit/node/include.py b/tools/grit/grit/node/include.py new file mode 100644 index 0000000..0e74865 --- /dev/null +++ b/tools/grit/grit/node/include.py @@ -0,0 +1,95 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Handling of the <include> element. +''' + + +import grit.format.rc_header +import grit.format.rc + +from grit.node import base +from grit import util + +class IncludeNode(base.Node): + '''An <include> element.''' + + def _IsValidChild(self, child): + return False + + def MandatoryAttributes(self): + return ['name', 'type', 'file'] + + def DefaultAttributes(self): + return {'translateable' : 'true', + 'generateid': 'true', + 'filenameonly': 'false', + 'relativepath': 'false', + } + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.Item() + elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and + self.SatisfiesOutputCondition()): + return grit.format.rc.RcInclude(self.attrs['type'].upper(), + self.attrs['filenameonly'] == 'true', + self.attrs['relativepath'] == 'true') + else: + return super(type(self), self).ItemFormatter(t) + + def FileForLanguage(self, lang, output_dir): + '''Returns the file for the specified language. This allows us to return + different files for different language variants of the include file. + ''' + return self.FilenameToOpen() + + # static method + def Construct(parent, name, type, file, translateable=True, + filenameonly=False, relativepath=False): + '''Creates a new node which is a child of 'parent', with attributes set + by parameters of the same name. + ''' + # Convert types to appropriate strings + translateable = util.BoolToString(translateable) + filenameonly = util.BoolToString(filenameonly) + relativepath = util.BoolToString(relativepath) + + node = IncludeNode() + node.StartParsing('include', parent) + node.HandleAttribute('name', name) + node.HandleAttribute('type', type) + node.HandleAttribute('file', file) + node.HandleAttribute('translateable', translateable) + node.HandleAttribute('filenameonly', filenameonly) + node.HandleAttribute('relativepath', relativepath) + node.EndParsing() + return node + Construct = staticmethod(Construct) diff --git a/tools/grit/grit/node/io.py b/tools/grit/grit/node/io.py new file mode 100644 index 0000000..b038185 --- /dev/null +++ b/tools/grit/grit/node/io.py @@ -0,0 +1,130 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The <output> and <file> elements. +''' + +import os +import re +import grit.format.rc_header + +from grit.node import base +from grit import exception +from grit import util +from grit import xtb_reader + + +class FileNode(base.Node): + '''A <file> element.''' + + def __init__(self): + super(type(self), self).__init__() + self.re = None + self.should_load_ = True + + def IsTranslation(self): + return True + + def GetLang(self): + return self.attrs['lang'] + + def DisableLoading(self): + self.should_load_ = False + + def MandatoryAttributes(self): + return ['path', 'lang'] + + def RunGatherers(self, recursive=False, debug=False): + if not self.should_load_: + return + + xtb_file = file(self.GetFilePath()) + try: + lang = xtb_reader.Parse(xtb_file, + self.UberClique().GenerateXtbParserCallback( + self.attrs['lang'], debug=debug)) + except: + print "Exception during parsing of %s" % self.GetFilePath() + raise + assert (lang == self.attrs['lang'], 'The XTB file you ' + 'reference must contain messages in the language specified\n' + 'by the \'lang\' attribute.') + + def GetFilePath(self): + return self.ToRealPath(os.path.expandvars(self.attrs['path'])) + + +class OutputNode(base.Node): + '''An <output> element.''' + + def MandatoryAttributes(self): + return ['filename', 'type'] + + def DefaultAttributes(self): + return { 'lang' : '', # empty lang indicates all languages + 'language_section' : 'neutral' # defines a language neutral section + } + + def GetType(self): + return self.attrs['type'] + + def GetLanguage(self): + '''Returns the language ID, default 'en'.''' + return self.attrs['lang'] + + def GetFilename(self): + return self.attrs['filename'] + + def GetOutputFilename(self): + if hasattr(self, 'output_filename'): + return self.output_filename + else: + return self.attrs['filename'] + + def _IsValidChild(self, child): + return isinstance(child, EmitNode) + +class EmitNode(base.ContentNode): + ''' An <emit> element.''' + + def DefaultAttributes(self): + return { 'emit_type' : 'prepend'} + + def GetEmitType(self): + '''Returns the emit_type for this node. Default is 'append'.''' + return self.attrs['emit_type'] + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.EmitAppender() + else: + return super(type(self), self).ItemFormatter(t) + + diff --git a/tools/grit/grit/node/io_unittest.py b/tools/grit/grit/node/io_unittest.py new file mode 100644 index 0000000..0122adf --- /dev/null +++ b/tools/grit/grit/node/io_unittest.py @@ -0,0 +1,87 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for io.FileNode''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import os +import StringIO +import unittest + +from grit.node import misc +from grit.node import io +from grit.node import empty +from grit import grd_reader +from grit import util + + +class FileNodeUnittest(unittest.TestCase): + def testGetPath(self): + root = misc.GritNode() + root.StartParsing(u'grit', None) + root.HandleAttribute(u'latest_public_release', u'0') + root.HandleAttribute(u'current_release', u'1') + root.HandleAttribute(u'base_dir', ur'..\resource') + translations = empty.TranslationsNode() + translations.StartParsing(u'translations', root) + root.AddChild(translations) + file_node = io.FileNode() + file_node.StartParsing(u'file', translations) + file_node.HandleAttribute(u'path', ur'flugel\kugel.pdf') + translations.AddChild(file_node) + root.EndParsing() + + self.failUnless(file_node.GetFilePath() == + util.normpath( + os.path.join(ur'../resource', ur'flugel/kugel.pdf'))) + + def testLoadTranslations(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <translations> + <file path="fr.xtb" lang="fr" /> + </translations> + <release seq="3"> + <messages> + <message name="ID_HELLO">Hello!</message> + <message name="ID_HELLO_USER">Hello <ph name="USERNAME">%s<ex>Joi</ex></ph></message> + </messages> + </release> + </grit>'''), util.PathFromRoot('grit/test/data')) + grd.RunGatherers(recursive=True) + self.failUnless(True) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/node/mapping.py b/tools/grit/grit/node/mapping.py new file mode 100644 index 0000000..c866b9a --- /dev/null +++ b/tools/grit/grit/node/mapping.py @@ -0,0 +1,82 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Maps each node type to an implementation class. +When adding a new node type, you add to this mapping. +''' + + +from grit import exception + +from grit.node import empty +from grit.node import message +from grit.node import misc +from grit.node import variant +from grit.node import structure +from grit.node import include +from grit.node import io + + +_ELEMENT_TO_CLASS = { + 'includes' : empty.IncludesNode, + 'messages' : empty.MessagesNode, + 'structures' : empty.StructuresNode, + 'translations' : empty.TranslationsNode, + 'outputs' : empty.OutputsNode, + 'message' : message.MessageNode, + 'ph' : message.PhNode, + 'ex' : message.ExNode, + 'grit' : misc.GritNode, + 'include' : include.IncludeNode, + 'structure' : structure.StructureNode, + 'skeleton' : variant.SkeletonNode, + 'release' : misc.ReleaseNode, + 'file' : io.FileNode, + 'output' : io.OutputNode, + 'emit' : io.EmitNode, + 'identifiers' : empty.IdentifiersNode, + 'identifier' : misc.IdentifierNode, + 'if' : misc.IfNode, +} + + +def ElementToClass(name, typeattr): + '''Maps an element to a class that handles the element. + + Args: + name: 'element' (the name of the element) + typeattr: 'type' (the value of the type attribute, if present, else None) + + Return: + type + ''' + if not _ELEMENT_TO_CLASS.has_key(name): + raise exception.UnknownElement() + return _ELEMENT_TO_CLASS[name] diff --git a/tools/grit/grit/node/message.py b/tools/grit/grit/node/message.py new file mode 100644 index 0000000..0e5373d --- /dev/null +++ b/tools/grit/grit/node/message.py @@ -0,0 +1,271 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Handling of the <message> element. +''' + +import re +import types + +from grit.node import base + +import grit.format.rc_header +import grit.format.rc + +from grit import clique +from grit import exception +from grit import tclib +from grit import util + + +# Finds whitespace at the start and end of a string which can be multiline. +_WHITESPACE = re.compile('(?P<start>\s*)(?P<body>.+?)(?P<end>\s*)\Z', + re.DOTALL | re.MULTILINE) + + +class MessageNode(base.ContentNode): + '''A <message> element.''' + + # For splitting a list of things that can be separated by commas or + # whitespace + _SPLIT_RE = re.compile('\s*,\s*|\s+') + + def __init__(self): + super(type(self), self).__init__() + # Valid after EndParsing, this is the MessageClique that contains the + # source message and any translations of it that have been loaded. + self.clique = None + + # We don't send leading and trailing whitespace into the translation + # console, but rather tack it onto the source message and any + # translations when formatting them into RC files or what have you. + self.ws_at_start = '' # Any whitespace characters at the start of the text + self.ws_at_end = '' # --"-- at the end of the text + + # A list of "shortcut groups" this message is in. We check to make sure + # that shortcut keys (e.g. &J) within each shortcut group are unique. + self.shortcut_groups_ = [] + + def _IsValidChild(self, child): + return isinstance(child, (PhNode)) + + def _IsValidAttribute(self, name, value): + if name not in ['name', 'offset', 'translateable', 'desc', 'meaning', + 'internal_comment', 'shortcut_groups', 'custom_type', + 'validation_expr']: + return False + if name == 'translateable' and value not in ['true', 'false']: + return False + return True + + def MandatoryAttributes(self): + return ['name|offset'] + + def DefaultAttributes(self): + return { + 'translateable' : 'true', + 'desc' : '', + 'meaning' : '', + 'internal_comment' : '', + 'shortcut_groups' : '', + 'custom_type' : '', + 'validation_expr' : '', + } + + def GetTextualIds(self): + ''' + Returns the concatenation of the parent's node first_id and + this node's offset if it has one, otherwise just call the + superclass' implementation + ''' + if 'offset' in self.attrs: + # we search for the first grouping node in the parents' list + # to take care of the case where the first parent is an <if> node + grouping_parent = self.parent + import grit.node.empty + while grouping_parent and not isinstance(grouping_parent, + grit.node.empty.GroupingNode): + grouping_parent = grouping_parent.parent + + assert 'first_id' in grouping_parent.attrs + return [grouping_parent.attrs['first_id'] + '_' + self.attrs['offset']] + else: + return super(type(self), self).GetTextualIds() + + def IsTranslateable(self): + return self.attrs['translateable'] == 'true' + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.Item() + elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and + self.SatisfiesOutputCondition()): + return grit.format.rc.Message() + else: + return super(type(self), self).ItemFormatter(t) + + def EndParsing(self): + super(type(self), self).EndParsing() + + # Make the text (including placeholder references) and list of placeholders, + # then strip and store leading and trailing whitespace and create the + # tclib.Message() and a clique to contain it. + + text = '' + placeholders = [] + for item in self.mixed_content: + if isinstance(item, types.StringTypes): + text += item + else: + presentation = item.attrs['name'].upper() + text += presentation + ex = ' ' + if len(item.children): + ex = item.children[0].GetCdata() + original = item.GetCdata() + placeholders.append(tclib.Placeholder(presentation, original, ex)) + + m = _WHITESPACE.match(text) + if m: + self.ws_at_start = m.group('start') + self.ws_at_end = m.group('end') + text = m.group('body') + + self.shortcut_groups_ = self._SPLIT_RE.split(self.attrs['shortcut_groups']) + self.shortcut_groups_ = [i for i in self.shortcut_groups_ if i != ''] + + description_or_id = self.attrs['desc'] + if description_or_id == '' and 'name' in self.attrs: + description_or_id = 'ID: %s' % self.attrs['name'] + + message = tclib.Message(text=text, placeholders=placeholders, + description=description_or_id, + meaning=self.attrs['meaning']) + self.clique = self.UberClique().MakeClique(message, self.IsTranslateable()) + for group in self.shortcut_groups_: + self.clique.AddToShortcutGroup(group) + if self.attrs['custom_type'] != '': + self.clique.SetCustomType(util.NewClassInstance(self.attrs['custom_type'], + clique.CustomType)) + elif self.attrs['validation_expr'] != '': + self.clique.SetCustomType( + clique.OneOffCustomType(self.attrs['validation_expr'])) + + def GetCliques(self): + if self.clique: + return [self.clique] + else: + return [] + + def Translate(self, lang): + '''Returns a translated version of this message. + ''' + assert self.clique + return self.clique.MessageForLanguage(lang, + self.PseudoIsAllowed(), + self.ShouldFallbackToEnglish() + ).GetRealContent() + + def NameOrOffset(self): + if 'name' in self.attrs: + return self.attrs['name'] + else: + return self.attrs['offset'] + + # static method + def Construct(parent, message, name, desc='', meaning='', translateable=True): + '''Constructs a new message node that is a child of 'parent', with the + name, desc, meaning and translateable attributes set using the same-named + parameters and the text of the message and any placeholders taken from + 'message', which must be a tclib.Message() object.''' + # Convert type to appropriate string + if translateable: + translateable = 'true' + else: + translateable = 'false' + + node = MessageNode() + node.StartParsing('message', parent) + node.HandleAttribute('name', name) + node.HandleAttribute('desc', desc) + node.HandleAttribute('meaning', meaning) + node.HandleAttribute('translateable', translateable) + + items = message.GetContent() + for ix in range(len(items)): + if isinstance(items[ix], types.StringTypes): + text = items[ix] + + # Ensure whitespace at front and back of message is correctly handled. + if ix == 0: + text = "'''" + text + if ix == len(items) - 1: + text = text + "'''" + + node.AppendContent(text) + else: + phnode = PhNode() + phnode.StartParsing('ph', node) + phnode.HandleAttribute('name', items[ix].GetPresentation()) + phnode.AppendContent(items[ix].GetOriginal()) + + if len(items[ix].GetExample()) and items[ix].GetExample() != ' ': + exnode = ExNode() + exnode.StartParsing('ex', phnode) + exnode.AppendContent(items[ix].GetExample()) + exnode.EndParsing() + phnode.AddChild(exnode) + + phnode.EndParsing() + node.AddChild(phnode) + + node.EndParsing() + return node + Construct = staticmethod(Construct) + +class PhNode(base.ContentNode): + '''A <ph> element.''' + + def _IsValidChild(self, child): + return isinstance(child, ExNode) + + def MandatoryAttributes(self): + return ['name'] + + def EndParsing(self): + super(type(self), self).EndParsing() + # We only allow a single example for each placeholder + if len(self.children) > 1: + raise exception.TooManyExamples() + + +class ExNode(base.ContentNode): + '''An <ex> element.''' + pass diff --git a/tools/grit/grit/node/message_unittest.py b/tools/grit/grit/node/message_unittest.py new file mode 100644 index 0000000..e9e0939 --- /dev/null +++ b/tools/grit/grit/node/message_unittest.py @@ -0,0 +1,87 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.node.message''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest +import StringIO + +from grit.node import message +from grit import grd_reader +from grit import tclib + +class MessageUnittest(unittest.TestCase): + def testMessage(self): + buf = StringIO.StringIO('''<message name="IDS_GREETING" + desc="Printed to greet the currently logged in user"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today? + </message>''') + res = grd_reader.Parse(buf, flexible_root = True) + cliques = res.GetCliques() + content = cliques[0].GetMessage().GetPresentableContent() + self.failUnless(content == 'Hello USERNAME, how are you doing today?') + + def testMessageWithWhitespace(self): + buf = StringIO.StringIO('<message name="IDS_BLA" desc="">' + '\'\'\' Hello there <ph name="USERNAME">%s</ph> \'\'\'' + '</message>') + res = grd_reader.Parse(buf, flexible_root = True) + content = res.GetCliques()[0].GetMessage().GetPresentableContent() + self.failUnless(content == 'Hello there USERNAME') + self.failUnless(res.ws_at_start == ' ') + self.failUnless(res.ws_at_end == ' ') + + def testConstruct(self): + msg = tclib.Message(text=" Hello USERNAME, how are you? BINGO\t\t", + placeholders=[tclib.Placeholder('USERNAME', '%s', 'Joi'), + tclib.Placeholder('BINGO', '%d', '11')]) + msg_node = message.MessageNode.Construct(None, msg, 'BINGOBONGO') + self.failUnless(msg_node.children[0].name == 'ph') + self.failUnless(msg_node.children[0].children[0].name == 'ex') + self.failUnless(msg_node.children[0].children[0].GetCdata() == 'Joi') + self.failUnless(msg_node.children[1].children[0].GetCdata() == '11') + self.failUnless(msg_node.ws_at_start == ' ') + self.failUnless(msg_node.ws_at_end == '\t\t') + + def testUnicodeConstruct(self): + text = u'Howdie \u00fe' + msg = tclib.Message(text=text) + msg_node = message.MessageNode.Construct(None, msg, 'BINGOBONGO') + msg_from_node = msg_node.GetCdata() + self.failUnless(msg_from_node == text) + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/node/misc.py b/tools/grit/grit/node/misc.py new file mode 100644 index 0000000..749cfed --- /dev/null +++ b/tools/grit/grit/node/misc.py @@ -0,0 +1,284 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Miscellaneous node types. +''' + +import os.path + +from grit.node import base +from grit.node import message + +from grit import exception +from grit import constants +from grit import util + +import grit.format.rc_header + + + +class IfNode(base.Node): + '''A node for conditional inclusion of resources. + ''' + + def _IsValidChild(self, child): + from grit.node import empty + assert self.parent, '<if> node should never be root.' + if isinstance(self.parent, empty.IncludesNode): + from grit.node import include + return isinstance(child, include.IncludeNode) + elif isinstance(self.parent, empty.MessagesNode): + from grit.node import message + return isinstance(child, message.MessageNode) + elif isinstance(self.parent, empty.StructuresNode): + from grit.node import structure + return isinstance(child, structure.StructureNode) + else: + return False + + def MandatoryAttributes(self): + return ['expr'] + + def IsConditionSatisfied(self): + '''Returns true if and only if the Python expression stored in attribute + 'expr' evaluates to true. + ''' + return self.EvaluateCondition(self.attrs['expr']) + + +class ReleaseNode(base.Node): + '''The <release> element.''' + + def _IsValidChild(self, child): + from grit.node import empty + return isinstance(child, (empty.IncludesNode, empty.MessagesNode, + empty.StructuresNode, empty.IdentifiersNode)) + + def _IsValidAttribute(self, name, value): + return ( + (name == 'seq' and int(value) <= self.GetRoot().GetCurrentRelease()) or + name == 'allow_pseudo' + ) + + def MandatoryAttributes(self): + return ['seq'] + + def DefaultAttributes(self): + return { 'allow_pseudo' : 'true' } + + def GetReleaseNumber(): + '''Returns the sequence number of this release.''' + return self.attribs['seq'] + + +class GritNode(base.Node): + '''The <grit> root element.''' + + def __init__(self): + base.Node.__init__(self) + self.output_language = '' + self.defines = {} + + def _IsValidChild(self, child): + from grit.node import empty + return isinstance(child, (ReleaseNode, empty.TranslationsNode, + empty.OutputsNode)) + + def _IsValidAttribute(self, name, value): + if name not in ['base_dir', 'source_lang_id', + 'latest_public_release', 'current_release', + 'enc_check', 'tc_project']: + return False + if name in ['latest_public_release', 'current_release'] and value.strip( + '0123456789') != '': + return False + return True + + def MandatoryAttributes(self): + return ['latest_public_release', 'current_release'] + + def DefaultAttributes(self): + return { + 'base_dir' : '.', + 'source_lang_id' : 'en', + 'enc_check' : constants.ENCODING_CHECK, + 'tc_project' : 'NEED_TO_SET_tc_project_ATTRIBUTE', + } + + def EndParsing(self): + base.Node.EndParsing(self) + if (int(self.attrs['latest_public_release']) + > int(self.attrs['current_release'])): + raise exception.Parsing('latest_public_release cannot have a greater ' + 'value than current_release') + + self.ValidateUniqueIds() + + # Add the encoding check if it's not present (should ensure that it's always + # present in all .grd files generated by GRIT). If it's present, assert if + # it's not correct. + if 'enc_check' not in self.attrs or self.attrs['enc_check'] == '': + self.attrs['enc_check'] = constants.ENCODING_CHECK + else: + assert self.attrs['enc_check'] == constants.ENCODING_CHECK, ( + 'Are you sure your .grd file is in the correct encoding (UTF-8)?') + + def ValidateUniqueIds(self): + '''Validate that 'name' attribute is unique in all nodes in this tree + except for nodes that are children of <if> nodes. + ''' + unique_names = {} + duplicate_names = [] + for node in self: + if isinstance(node, message.PhNode): + continue # PhNode objects have a 'name' attribute which is not an ID + + node_ids = node.GetTextualIds() + if node_ids: + for node_id in node_ids: + if util.SYSTEM_IDENTIFIERS.match(node_id): + continue # predefined IDs are sometimes used more than once + + # Don't complain about duplicate IDs if they occur in a node that is + # inside an <if> node. + if (node_id in unique_names and node_id not in duplicate_names and + (not node.parent or not isinstance(node.parent, IfNode))): + duplicate_names.append(node_id) + unique_names[node_id] = 1 + + if len(duplicate_names): + raise exception.DuplicateKey(', '.join(duplicate_names)) + + + def GetCurrentRelease(self): + '''Returns the current release number.''' + return int(self.attrs['current_release']) + + def GetLatestPublicRelease(self): + '''Returns the latest public release number.''' + return int(self.attrs['latest_public_release']) + + def GetSourceLanguage(self): + '''Returns the language code of the source language.''' + return self.attrs['source_lang_id'] + + def GetTcProject(self): + '''Returns the name of this project in the TranslationConsole, or + 'NEED_TO_SET_tc_project_ATTRIBUTE' if it is not defined.''' + return self.attrs['tc_project'] + + def SetOwnDir(self, dir): + '''Informs the 'grit' element of the directory the file it is in resides. + This allows it to calculate relative paths from the input file, which is + what we desire (rather than from the current path). + + Args: + dir: r'c:\bla' + + Return: + None + ''' + assert dir + self.base_dir = os.path.normpath(os.path.join(dir, self.attrs['base_dir'])) + + def GetBaseDir(self): + '''Returns the base directory, relative to the working directory. To get + the base directory as set in the .grd file, use GetOriginalBaseDir() + ''' + if hasattr(self, 'base_dir'): + return self.base_dir + else: + return self.GetOriginalBaseDir() + + def GetOriginalBaseDir(self): + '''Returns the base directory, as set in the .grd file. + ''' + return self.attrs['base_dir'] + + def GetOutputFiles(self): + '''Returns the list of <file> nodes that are children of this node's + <outputs> child.''' + for child in self.children: + if child.name == 'outputs': + return child.children + raise exception.MissingElement() + + def ItemFormatter(self, t): + if t == 'rc_header': + from grit.format import rc_header # import here to avoid circular dep + return rc_header.TopLevel() + elif t in ['rc_all', 'rc_translateable', 'rc_nontranslateable']: + from grit.format import rc # avoid circular dep + return rc.TopLevel() + else: + return super(type(self), self).ItemFormatter(t) + + def SetOutputContext(self, output_language, defines): + self.output_language = output_language + self.defines = defines + + +class IdentifierNode(base.Node): + '''A node for specifying identifiers that should appear in the resource + header file, and be unique amongst all other resource identifiers, but don't + have any other attributes or reference any resources. + ''' + + def MandatoryAttributes(self): + return ['name'] + + def DefaultAttributes(self): + return { 'comment' : '', 'id' : '' } + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.Item() + + def GetId(self): + '''Returns the id of this identifier if it has one, None otherwise + ''' + if 'id' in self.attrs: + return self.attrs['id'] + return None + + # static method + def Construct(parent, name, id, comment): + '''Creates a new node which is a child of 'parent', with attributes set + by parameters of the same name. + ''' + node = IdentifierNode() + node.StartParsing('identifier', parent) + node.HandleAttribute('name', name) + node.HandleAttribute('id', id) + node.HandleAttribute('comment', comment) + node.EndParsing() + return node + Construct = staticmethod(Construct) + diff --git a/tools/grit/grit/node/misc_unittest.py b/tools/grit/grit/node/misc_unittest.py new file mode 100644 index 0000000..791be25 --- /dev/null +++ b/tools/grit/grit/node/misc_unittest.py @@ -0,0 +1,162 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for misc.GritNode''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest +import StringIO + +from grit import grd_reader +import grit.exception +from grit import util +from grit.node import misc + + +class GritNodeUnittest(unittest.TestCase): + def testUniqueNameAttribute(self): + try: + restree = grd_reader.Parse( + util.PathFromRoot('grit/test/data/duplicate-name-input.xml')) + self.fail('Expected parsing exception because of duplicate names.') + except grit.exception.Parsing: + pass # Expected case + + +class IfNodeUnittest(unittest.TestCase): + def testIffyness(self): + grd = grd_reader.Parse(StringIO.StringIO(''' + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <if expr="'bingo' in defs"> + <message name="IDS_BINGO"> + Bingo! + </message> + </if> + <if expr="'hello' in defs"> + <message name="IDS_HELLO"> + Hello! + </message> + </if> + <if expr="lang == 'fr' or 'FORCE_FRENCH' in defs"> + <message name="IDS_HELLO" internal_comment="French version"> + Good morning + </message> + </if> + </messages> + </release> + </grit>'''), dir='.') + + messages_node = grd.children[0].children[0] + bingo_message = messages_node.children[0].children[0] + hello_message = messages_node.children[1].children[0] + french_message = messages_node.children[2].children[0] + assert bingo_message.name == 'message' + assert hello_message.name == 'message' + assert french_message.name == 'message' + + grd.SetOutputContext('fr', {'hello' : '1'}) + self.failUnless(not bingo_message.SatisfiesOutputCondition()) + self.failUnless(hello_message.SatisfiesOutputCondition()) + self.failUnless(french_message.SatisfiesOutputCondition()) + + grd.SetOutputContext('en', {'bingo' : 1}) + self.failUnless(bingo_message.SatisfiesOutputCondition()) + self.failUnless(not hello_message.SatisfiesOutputCondition()) + self.failUnless(not french_message.SatisfiesOutputCondition()) + + grd.SetOutputContext('en', {'FORCE_FRENCH' : '1', 'bingo' : '1'}) + self.failUnless(bingo_message.SatisfiesOutputCondition()) + self.failUnless(not hello_message.SatisfiesOutputCondition()) + self.failUnless(french_message.SatisfiesOutputCondition()) + + +class ReleaseNodeUnittest(unittest.TestCase): + def testPseudoControl(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="1" source_lang_id="en-US" current_release="2" base_dir="."> + <release seq="1" allow_pseudo="false"> + <messages> + <message name="IDS_HELLO"> + Hello + </message> + </messages> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="klonk.rc" /> + </structures> + </release> + <release seq="2"> + <messages> + <message name="IDS_BINGO"> + Bingo + </message> + </messages> + <structures> + <structure type="menu" name="IDC_KLONKMENU" encoding="utf-16" file="klonk.rc" /> + </structures> + </release> + </grit>'''), util.PathFromRoot('grit/test/data')) + grd.RunGatherers(recursive=True) + + hello = grd.GetNodeById('IDS_HELLO') + aboutbox = grd.GetNodeById('IDD_ABOUTBOX') + bingo = grd.GetNodeById('IDS_BINGO') + menu = grd.GetNodeById('IDC_KLONKMENU') + + for node in [hello, aboutbox]: + self.failUnless(not node.PseudoIsAllowed()) + + for node in [bingo, menu]: + self.failUnless(node.PseudoIsAllowed()) + + for node in [hello, aboutbox]: + try: + formatter = node.ItemFormatter('rc_all') + formatter.Format(node, 'xyz-pseudo') + self.fail('Should have failed during Format since pseudo is not allowed') + except: + pass # expected case + + for node in [bingo, menu]: + try: + formatter = node.ItemFormatter('rc_all') + formatter.Format(node, 'xyz-pseudo') + except: + self.fail('Should not have gotten exception since pseudo is allowed') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/node/structure.py b/tools/grit/grit/node/structure.py new file mode 100644 index 0000000..9bf92244 --- /dev/null +++ b/tools/grit/grit/node/structure.py @@ -0,0 +1,284 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The <structure> element. +''' + +import os + +from grit.node import base +from grit.node import variant + +from grit import constants +from grit import exception +from grit import util + +import grit.gather.rc +import grit.gather.tr_html +import grit.gather.admin_template +import grit.gather.txt +import grit.gather.muppet_strings + +import grit.format.rc +import grit.format.rc_header + +# RTL languages +# TODO(jennyz): remove this fixed set of RTL language array +# when generic expand_variable code is added by grit team. +_RTL_LANGS = [ + 'ar', + 'iw', + 'ur', +] + +# Type of the gatherer to use for each type attribute +_GATHERERS = { + 'accelerators' : grit.gather.rc.Accelerators, + 'admin_template' : grit.gather.admin_template.AdmGatherer, + 'dialog' : grit.gather.rc.Dialog, + 'menu' : grit.gather.rc.Menu, + 'muppet' : grit.gather.muppet_strings.MuppetStrings, + 'rcdata' : grit.gather.rc.RCData, + 'tr_html' : grit.gather.tr_html.TrHtml, + 'txt' : grit.gather.txt.TxtFile, + 'version' : grit.gather.rc.Version, +} + + +# Formatter instance to use for each type attribute +# when formatting .rc files. +_RC_FORMATTERS = { + 'accelerators' : grit.format.rc.RcSection(), + 'admin_template' : grit.format.rc.RcInclude('ADM'), + 'dialog' : grit.format.rc.RcSection(), + 'menu' : grit.format.rc.RcSection(), + 'muppet' : grit.format.rc.RcInclude('XML'), + 'rcdata' : grit.format.rc.RcSection(), + 'tr_html' : grit.format.rc.RcInclude('HTML'), + 'txt' : grit.format.rc.RcInclude('TXT'), + 'version' : grit.format.rc.RcSection(), +} + + +# TODO(joi) Print a warning if the 'variant_of_revision' attribute indicates +# that a skeleton variant is older than the original file. + + +class StructureNode(base.Node): + '''A <structure> element.''' + + def __init__(self): + base.Node.__init__(self) + self.gatherer = None + self.skeletons = {} # expressions to skeleton gatherers + + def _IsValidChild(self, child): + return isinstance(child, variant.SkeletonNode) + + def MandatoryAttributes(self): + return ['type', 'name', 'file'] + + def DefaultAttributes(self): + return { 'encoding' : 'cp1252', + 'exclude_from_rc' : 'false', + 'line_end' : 'unix', + 'output_encoding' : 'utf-8', + 'generateid': 'true', + 'expand_variables' : 'false', + 'output_filename' : '', + # TODO(joi) this is a hack - should output all generated files + # as SCons dependencies; however, for now there is a bug I can't + # find where GRIT doesn't build the matching fileset, therefore + # this hack so that only the files you really need are marked as + # dependencies. + 'sconsdep' : 'false', + } + + def IsExcludedFromRc(self): + return self.attrs['exclude_from_rc'] == 'true' + + def GetLineEnd(self): + '''Returns the end-of-line character or characters for files output because + of this node ('\r\n', '\n', or '\r' depending on the 'line_end' attribute). + ''' + if self.attrs['line_end'] == 'unix': + return '\n' + elif self.attrs['line_end'] == 'windows': + return '\r\n' + elif self.attrs['line_end'] == 'mac': + return '\r' + else: + raise exception.UnexpectedAttribute( + "Attribute 'line_end' must be one of 'linux' (default), 'windows' or 'mac'") + + def GetCliques(self): + if self.gatherer: + return self.gatherer.GetCliques() + else: + return [] + + def GetTextualIds(self): + if self.gatherer and self.attrs['type'] not in ['tr_html', 'admin_template', 'txt']: + return self.gatherer.GetTextualIds() + else: + return [self.attrs['name']] + + def ItemFormatter(self, t): + if t == 'rc_header': + return grit.format.rc_header.Item() + elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and + self.SatisfiesOutputCondition()): + return _RC_FORMATTERS[self.attrs['type']] + else: + return super(type(self), self).ItemFormatter(t) + + def RunGatherers(self, recursive=False, debug=False): + if self.gatherer: + return # idempotent + + gathertype = _GATHERERS[self.attrs['type']] + + if debug: + print 'Running gatherer %s for file %s' % (str(gathertype), self.FilenameToOpen()) + + self.gatherer = gathertype.FromFile(self.FilenameToOpen(), + self.attrs['name'], + self.attrs['encoding']) + self.gatherer.SetUberClique(self.UberClique()) + self.gatherer.Parse() + + for child in self.children: + assert isinstance(child, variant.SkeletonNode) + skel = gathertype.FromFile(child.FilenameToOpen(), + self.attrs['name'], + child.GetEncodingToUse()) + skel.SetUberClique(self.UberClique()) + skel.SetSkeleton(True) + skel.Parse() + self.skeletons[child.attrs['expr']] = skel + + def GetSkeletonGatherer(self): + '''Returns the gatherer for the alternate skeleton that should be used, + based on the expressions for selecting skeletons, or None if the skeleton + from the English version of the structure should be used. + ''' + for expr in self.skeletons: + if self.EvaluateCondition(expr): + return self.skeletons[expr] + return None + + def GetFilePath(self): + return self.ToRealPath(self.attrs['file']) + + def HasFileForLanguage(self): + return self.attrs['type'] in ['tr_html', 'admin_template', 'txt', 'muppet'] + + def FileForLanguage(self, lang, output_dir, create_file=True, + return_if_not_generated=True): + '''Returns the filename of the file associated with this structure, + for the specified language. + + Args: + lang: 'fr' + output_dir: 'c:\temp' + create_file: True + ''' + assert self.HasFileForLanguage() + if (lang == self.GetRoot().GetSourceLanguage() and + self.attrs['expand_variables'] != 'true'): + if return_if_not_generated: + return self.GetFilePath() + else: + return None + else: + if self.attrs['output_filename'] != '': + filename = self.attrs['output_filename'] + else: + filename = os.path.basename(self.attrs['file']) + assert len(filename) + filename = '%s_%s' % (lang, filename) + filename = os.path.join(output_dir, filename) + + if create_file: + text = self.gatherer.Translate( + lang, + pseudo_if_not_available=self.PseudoIsAllowed(), + fallback_to_english=self.ShouldFallbackToEnglish(), + skeleton_gatherer=self.GetSkeletonGatherer()) + + file_object = util.WrapOutputStream(file(filename, 'wb'), + self._GetOutputEncoding()) + file_contents = util.FixLineEnd(text, self.GetLineEnd()) + if self.attrs['expand_variables'] == 'true': + file_contents = file_contents.replace('[GRITLANGCODE]', lang) + # TODO(jennyz): remove this hard coded logic for expanding + # [GRITDIR] variable for RTL languages when the generic + # expand_variable code is added by grit team. + if lang in _RTL_LANGS : + file_contents = file_contents.replace('[GRITDIR]', 'dir="RTL"') + else : + file_contents = file_contents.replace('[GRITDIR]', 'dir="LTR"') + if self._ShouldAddBom(): + file_object.write(constants.BOM) + file_object.write(file_contents) + file_object.close() + + return filename + + def _GetOutputEncoding(self): + '''Python doesn't natively support UTF encodings with a BOM signature, + so we add support by allowing you to append '-sig' to the encoding name. + This function returns the specified output encoding minus that part. + ''' + enc = self.attrs['output_encoding'] + if enc.endswith('-sig'): + return enc[0:len(enc) - len('-sig')] + else: + return enc + + def _ShouldAddBom(self): + '''Returns true if output files should have the Unicode BOM prepended. + ''' + return self.attrs['output_encoding'].endswith('-sig') + + # static method + def Construct(parent, name, type, file, encoding='cp1252'): + '''Creates a new node which is a child of 'parent', with attributes set + by parameters of the same name. + ''' + node = StructureNode() + node.StartParsing('structure', parent) + node.HandleAttribute('name', name) + node.HandleAttribute('type', type) + node.HandleAttribute('file', file) + node.HandleAttribute('encoding', encoding) + node.EndParsing() + return node + Construct = staticmethod(Construct) diff --git a/tools/grit/grit/node/structure_unittest.py b/tools/grit/grit/node/structure_unittest.py new file mode 100644 index 0000000..5fcd678 --- /dev/null +++ b/tools/grit/grit/node/structure_unittest.py @@ -0,0 +1,86 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for <structure> nodes. +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import unittest +import StringIO + +from grit.node import structure +from grit import grd_reader +from grit import util + + +class StructureUnittest(unittest.TestCase): + def testSkeleton(self): + grd = grd_reader.Parse(StringIO.StringIO( + '''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" file="klonk.rc" encoding="utf-16-le"> + <skeleton expr="lang == 'fr'" variant_of_revision="1" file="klonk-alternate-skeleton.rc" /> + </structure> + </structures> + </release> + </grit>'''), dir=util.PathFromRoot('grit\\test\\data')) + grd.RunGatherers(recursive=True) + grd.output_language = 'fr' + + node = grd.GetNodeById('IDD_ABOUTBOX') + formatter = node.ItemFormatter('rc_all') + self.failUnless(formatter) + transl = formatter.Format(node, 'fr') + + self.failUnless(transl.count('040704') and transl.count('110978')) + self.failUnless(transl.count('2005",IDC_STATIC')) + + def testOutputEncoding(self): + grd = grd_reader.Parse(StringIO.StringIO( + '''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" file="klonk.rc" encoding="utf-16-le" output_encoding="utf-8-sig" /> + </structures> + </release> + </grit>'''), dir=util.PathFromRoot('grit\\test\\data')) + node = grd.GetNodeById('IDD_ABOUTBOX') + self.failUnless(node._GetOutputEncoding() == 'utf-8') + self.failUnless(node._ShouldAddBom()) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/node/variant.py b/tools/grit/grit/node/variant.py new file mode 100644 index 0000000..e5da2f9 --- /dev/null +++ b/tools/grit/grit/node/variant.py @@ -0,0 +1,66 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The <skeleton> element. +''' + + +from grit.node import base + + +class SkeletonNode(base.Node): + '''A <skeleton> element.''' + + # TODO(joi) Support inline skeleton variants as CDATA instead of requiring + # a 'file' attribute. + + def MandatoryAttributes(self): + return ['expr', 'variant_of_revision', 'file'] + + def DefaultAttributes(self): + '''If not specified, 'encoding' will actually default to the parent node's + encoding. + ''' + return {'encoding' : ''} + + def _ContentType(self): + if self.attrs.has_key('file'): + return self._CONTENT_TYPE_NONE + else: + return self._CONTENT_TYPE_CDATA + + def GetEncodingToUse(self): + if self.attrs['encoding'] == '': + return self.parent.attrs['encoding'] + else: + return self.attrs['encoding'] + + def GetFilePath(self): + return self.ToRealPath(self.attrs['file']) diff --git a/tools/grit/grit/pseudo.py b/tools/grit/grit/pseudo.py new file mode 100644 index 0000000..53ee9fa --- /dev/null +++ b/tools/grit/grit/pseudo.py @@ -0,0 +1,154 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Pseudotranslation support. Our pseudotranslations are based on the +P-language, which is a simple vowel-extending language. Examples of P: + - "hello" becomes "hepellopo" + - "howdie" becomes "hopowdiepie" + - "because" becomes "bepecaupause" (but in our implementation we don't + handle the silent e at the end so it actually would return "bepecaupausepe" + +The P-language has the excellent quality of increasing the length of text +by around 30-50% which is great for pseudotranslations, to stress test any +GUI layouts etc. + +To make the pseudotranslations more obviously "not a translation" and to make +them exercise any code that deals with encodings, we also transform all English +vowels into equivalent vowels with diacriticals on them (rings, acutes, +diaresis, and circumflex), and we write the "p" in the P-language as a Hebrew +character Qof. It looks sort of like a latin character "p" but it is outside +the latin-1 character set which will stress character encoding bugs. +''' + +import re +import types + +from grit import tclib + + +# An RFC language code for the P pseudolanguage. +PSEUDO_LANG = 'x-P-pseudo' + +# Hebrew character Qof. It looks kind of like a 'p' but is outside +# the latin-1 character set which is good for our purposes. +# TODO(joi) For now using P instead of Qof, because of some bugs it used. Find +# a better solution, i.e. one that introduces a non-latin1 character into the +# pseudotranslation. +#_QOF = u'\u05e7' +_QOF = u'P' + +# How we map each vowel. +_VOWELS = { + u'a' : u'\u00e5', # a with ring + u'e' : u'\u00e9', # e acute + u'i' : u'\u00ef', # i diaresis + u'o' : u'\u00f4', # o circumflex + u'u' : u'\u00fc', # u diaresis + u'y' : u'\u00fd', # y acute + u'A' : u'\u00c5', # A with ring + u'E' : u'\u00c9', # E acute + u'I' : u'\u00cf', # I diaresis + u'O' : u'\u00d4', # O circumflex + u'U' : u'\u00dc', # U diaresis + u'Y' : u'\u00dd', # Y acute +} + +# Matches vowels and P +_PSUB_RE = re.compile("(%s)" % '|'.join(_VOWELS.keys() + ['P'])) + + +# Pseudotranslations previously created. This is important for performance +# reasons, especially since we routinely pseudotranslate the whole project +# several or many different times for each build. +_existing_translations = {} + + +def MapVowels(str, also_p = False): + '''Returns a copy of 'str' where characters that exist as keys in _VOWELS + have been replaced with the corresponding value. If also_p is true, this + function will also change capital P characters into a Hebrew character Qof. + ''' + def Repl(match): + if match.group() == 'p': + if also_p: + return _QOF + else: + return 'p' + else: + return _VOWELS[match.group()] + return _PSUB_RE.sub(Repl, str) + + +def PseudoString(str): + '''Returns a pseudotranslation of the provided string, in our enhanced + P-language.''' + if str in _existing_translations: + return _existing_translations[str] + + outstr = u'' + ix = 0 + while ix < len(str): + if str[ix] not in _VOWELS.keys(): + outstr += str[ix] + ix += 1 + else: + # We want to treat consecutive vowels as one composite vowel. This is not + # always accurate e.g. in composite words but good enough. + consecutive_vowels = u'' + while ix < len(str) and str[ix] in _VOWELS.keys(): + consecutive_vowels += str[ix] + ix += 1 + changed_vowels = MapVowels(consecutive_vowels) + outstr += changed_vowels + outstr += _QOF + outstr += changed_vowels + + _existing_translations[str] = outstr + return outstr + + +def PseudoMessage(message): + '''Returns a pseudotranslation of the provided message. + + Args: + message: tclib.Message() + + Return: + tclib.Translation() + ''' + transl = tclib.Translation() + + for part in message.GetContent(): + if isinstance(part, tclib.Placeholder): + transl.AppendPlaceholder(part) + else: + transl.AppendText(PseudoString(part)) + + return transl diff --git a/tools/grit/grit/pseudo_unittest.py b/tools/grit/grit/pseudo_unittest.py new file mode 100644 index 0000000..6ce5f46 --- /dev/null +++ b/tools/grit/grit/pseudo_unittest.py @@ -0,0 +1,78 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.pseudo''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +from grit import pseudo +from grit import tclib + + +class PseudoUnittest(unittest.TestCase): + def testVowelMapping(self): + self.failUnless(pseudo.MapVowels('abebibobuby') == + u'\u00e5b\u00e9b\u00efb\u00f4b\u00fcb\u00fd') + self.failUnless(pseudo.MapVowels('ABEBIBOBUBY') == + u'\u00c5B\u00c9B\u00cfB\u00d4B\u00dcB\u00dd') + + def testPseudoString(self): + out = pseudo.PseudoString('hello') + self.failUnless(out == pseudo.MapVowels(u'hePelloPo', True)) + + def testConsecutiveVowels(self): + out = pseudo.PseudoString("beautiful weather, ain't it?") + self.failUnless(out == pseudo.MapVowels( + u"beauPeautiPifuPul weaPeathePer, aiPain't iPit?", 1)) + + def testCapitals(self): + out = pseudo.PseudoString("HOWDIE DOODIE, DR. JONES") + self.failUnless(out == pseudo.MapVowels( + u"HOPOWDIEPIE DOOPOODIEPIE, DR. JOPONEPES", 1)) + + def testPseudoMessage(self): + msg = tclib.Message(text='Hello USERNAME, how are you?', + placeholders=[ + tclib.Placeholder('USERNAME', '%s', 'Joi')]) + trans = pseudo.PseudoMessage(msg) + # TODO(joi) It would be nicer if 'you' -> 'youPou' instead of + # 'you' -> 'youPyou' and if we handled the silent e in 'are' + self.failUnless(trans.GetPresentableContent() == + pseudo.MapVowels( + u'HePelloPo USERNAME, hoPow aParePe youPyou?', 1)) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/scons.py b/tools/grit/grit/scons.py new file mode 100644 index 0000000..1b960a6 --- /dev/null +++ b/tools/grit/grit/scons.py @@ -0,0 +1,167 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''SCons integration for GRIT. +''' + +# NOTE: DO NOT IMPORT ANY GRIT STUFF HERE - we import lazily so that grit and +# its dependencies aren't imported until actually needed. + +import os +import types + +def _IsDebugEnabled(): + return 'GRIT_DEBUG' in os.environ and os.environ['GRIT_DEBUG'] == '1' + +def _SourceToFile(source): + '''Return the path to the source file, given the 'source' argument as provided + by SCons to the _Builder or _Emitter functions. + ''' + # Get the filename of the source. The 'source' parameter can be a string, + # a "node", or a list of strings or nodes. + if isinstance(source, types.ListType): + source = str(source[0]) + else: + source = str(source) + return source + + +def _Builder(target, source, env): + from grit import grit_runner + from grit.tool import build + options = grit_runner.Options() + # This sets options to default values TODO(joi) Remove verbose + options.ReadOptions(['-v']) + options.input = _SourceToFile(source) + + # TODO(joi) Check if we can get the 'verbose' option from the environment. + + builder = build.RcBuilder() + + # Get the CPP defines from the environment. + for flag in env['RCFLAGS']: + if flag.startswith('/D'): + flag = flag[2:] + name, val = build.ParseDefine(flag) + # Only apply to first instance of a given define + if name not in builder.defines: + builder.defines[name] = val + + # To ensure that our output files match what we promised SCons, we + # use the list of targets provided by SCons and update the file paths in + # our .grd input file with the targets. + builder.scons_targets = [str(t) for t in target] + builder.Run(options, []) + return None # success + + +def _Emitter(target, source, env): + '''A SCons emitter for .grd files, which modifies the list of targes to + include all files in the <outputs> section of the .grd file as well as + any other files output by 'grit build' for the .grd file. + ''' + from grit import util + from grit import grd_reader + + base_dir = util.dirname(str(target[0])) + + grd = grd_reader.Parse(_SourceToFile(source), debug=_IsDebugEnabled()) + + target = [] + lang_folders = {} + # Add all explicitly-specified output files + for output in grd.GetOutputFiles(): + path = os.path.join(base_dir, output.GetFilename()) + target.append(path) + if _IsDebugEnabled(): + print "GRIT: Added target %s" % path + if output.attrs['lang'] != '': + lang_folders[output.attrs['lang']] = os.path.dirname(path) + + # Add all generated files, once for each output language. + for node in grd: + if node.name == 'structure': + # TODO(joi) Should remove the "if sconsdep is true" thing as it is a + # hack - see grit/node/structure.py + if node.HasFileForLanguage() and node.attrs['sconsdep'] == 'true': + for lang in lang_folders: + path = node.FileForLanguage(lang, lang_folders[lang], + create_file=False, + return_if_not_generated=False) + if path: + target.append(path) + if _IsDebugEnabled(): + print "GRIT: Added target %s" % path + + # return target and source lists + return (target, source) + + +def _Scanner(file_node, env, path): + '''A SCons scanner function for .grd files, which outputs the list of files + that changes in could change the output of building the .grd file. + ''' + from grit import grd_reader + + grd = grd_reader.Parse(str(file_node), debug=_IsDebugEnabled()) + files = [] + for node in grd: + if (node.name == 'structure' or node.name == 'skeleton' or + (node.name == 'file' and node.parent and + node.parent.name == 'translations')): + files.append(os.path.abspath(node.GetFilePath())) + return files + + +# Function name is mandated by newer versions of SCons. +def generate(env): + # Importing this module should be possible whenever this function is invoked + # since it should only be invoked by SCons. + import SCons.Builder + import SCons.Action + + # The varlist parameter tells SCons that GRIT needs to be invoked again + # if RCFLAGS has changed since last compilation. + action = SCons.Action.FunctionAction(_Builder, varlist=['RCFLAGS']) + + builder = SCons.Builder.Builder(action=action, + emitter=_Emitter, + src_suffix='.grd') + + scanner = env.Scanner(function=_Scanner, name='GRIT', skeys=['.grd']) + + # add our builder and scanner to the environment + env.Append(BUILDERS = {'GRIT': builder}) + env.Prepend(SCANNERS = scanner) + + +# Function name is mandated by newer versions of SCons. +def exists(env): + return 1 diff --git a/tools/grit/grit/shortcuts.py b/tools/grit/grit/shortcuts.py new file mode 100644 index 0000000..43da40b --- /dev/null +++ b/tools/grit/grit/shortcuts.py @@ -0,0 +1,119 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Stuff to prevent conflicting shortcuts. +''' + +import re + +from grit import util + + +class ShortcutGroup(object): + '''Manages a list of cliques that belong together in a single shortcut + group. Knows how to detect conflicting shortcut keys. + ''' + + # Matches shortcut keys, e.g. &J + SHORTCUT_RE = re.compile('([^&]|^)(&[A-Za-z])') + + def __init__(self, name): + self.name = name + # Map of language codes to shortcut keys used (which is a map of + # shortcut keys to counts). + self.keys_by_lang = {} + # List of cliques in this group + self.cliques = [] + + def AddClique(self, c): + for existing_clique in self.cliques: + if existing_clique.GetId() == c.GetId(): + # This happens e.g. when we have e.g. + # <if expr1><structure 1></if> <if expr2><structure 2></if> + # where only one will really be included in the output. + return + + self.cliques.append(c) + for (lang, msg) in c.clique.items(): + if lang not in self.keys_by_lang: + self.keys_by_lang[lang] = {} + keymap = self.keys_by_lang[lang] + + content = msg.GetRealContent() + keys = [groups[1] for groups in self.SHORTCUT_RE.findall(content)] + for key in keys: + key = key.upper() + if key in keymap: + keymap[key] += 1 + else: + keymap[key] = 1 + + def GenerateWarnings(self, tc_project): + # For any language that has more than one occurrence of any shortcut, + # make a list of the conflicting shortcuts. + problem_langs = {} + for (lang, keys) in self.keys_by_lang.items(): + for (key, count) in keys.items(): + if count > 1: + if lang not in problem_langs: + problem_langs[lang] = [] + problem_langs[lang].append(key) + + warnings = [] + if len(problem_langs): + warnings.append("WARNING - duplicate keys exist in shortcut group %s" % + self.name) + for (lang,keys) in problem_langs.items(): + warnings.append(" %6s duplicates: %s" % (lang, ', '.join(keys))) + return warnings + + +def GenerateDuplicateShortcutsWarnings(uberclique, tc_project): + '''Given an UberClique and a project name, will print out helpful warnings + if there are conflicting shortcuts within shortcut groups in the provided + UberClique. + + Args: + uberclique: clique.UberClique() + tc_project: 'MyProjectNameInTheTranslationConsole' + + Returns: + ['warning line 1', 'warning line 2', ...] + ''' + warnings = [] + groups = {} + for c in uberclique.AllCliques(): + for group in c.shortcut_groups: + if group not in groups: + groups[group] = ShortcutGroup(group) + groups[group].AddClique(c) + for group in groups.values(): + warnings += group.GenerateWarnings(tc_project) + return warnings diff --git a/tools/grit/grit/shortcuts_unittests.py b/tools/grit/grit/shortcuts_unittests.py new file mode 100644 index 0000000..634177d --- /dev/null +++ b/tools/grit/grit/shortcuts_unittests.py @@ -0,0 +1,103 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.shortcuts +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +from grit import shortcuts +from grit import clique +from grit import tclib +from grit.gather import rc + +class ShortcutsUnittest(unittest.TestCase): + + def setUp(self): + self.uq = clique.UberClique() + + def testFunctionality(self): + c = self.uq.MakeClique(tclib.Message(text="Hello &there")) + c.AddToShortcutGroup('group_name') + c = self.uq.MakeClique(tclib.Message(text="Howdie &there partner")) + c.AddToShortcutGroup('group_name') + + warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT') + self.failUnless(warnings) + + def testAmpersandEscaping(self): + c = self.uq.MakeClique(tclib.Message(text="Hello &there")) + c.AddToShortcutGroup('group_name') + c = self.uq.MakeClique(tclib.Message(text="S&&T are the &letters S and T")) + c.AddToShortcutGroup('group_name') + + warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT') + self.failUnless(len(warnings) == 0) + + def testDialog(self): + dlg = rc.Dialog('''\ +IDD_SIDEBAR_RSS_PANEL_PROPPAGE DIALOGEX 0, 0, 239, 221 +STYLE DS_SETFONT | DS_FIXEDSYS | WS_CHILD +FONT 8, "MS Shell Dlg", 400, 0, 0x1 +BEGIN + PUSHBUTTON "Add &URL",IDC_SIDEBAR_RSS_ADD_URL,182,53,57,14 + EDITTEXT IDC_SIDEBAR_RSS_NEW_URL,0,53,178,15,ES_AUTOHSCROLL + PUSHBUTTON "&Remove",IDC_SIDEBAR_RSS_REMOVE,183,200,56,14 + PUSHBUTTON "&Edit",IDC_SIDEBAR_RSS_EDIT,123,200,56,14 + CONTROL "&Automatically add commonly viewed clips", + IDC_SIDEBAR_RSS_AUTO_ADD,"Button",BS_AUTOCHECKBOX | + BS_MULTILINE | WS_TABSTOP,0,200,120,17 + PUSHBUTTON "",IDC_SIDEBAR_RSS_HIDDEN,179,208,6,6,NOT WS_VISIBLE + LTEXT "You can display clips from blogs, news sites, and other online sources.", + IDC_STATIC,0,0,239,10 + LISTBOX IDC_SIDEBAR_DISPLAYED_FEED_LIST,0,69,239,127,LBS_SORT | + LBS_OWNERDRAWFIXED | LBS_HASSTRINGS | + LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_HSCROLL | + WS_TABSTOP + LTEXT "Add a clip from a recently viewed website by clicking Add Recent Clips.", + IDC_STATIC,0,13,141,19 + LTEXT "Or, if you know a site supports RSS or Atom, you can enter the RSS or Atom URL below and add it to your list of Web Clips.", + IDC_STATIC,0,33,239,18 + PUSHBUTTON "Add Recent &Clips (10)...", + IDC_SIDEBAR_RSS_ADD_RECENT_CLIPS,146,14,93,14 +END''') + dlg.SetUberClique(self.uq) + dlg.Parse() + + warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT') + self.failUnless(len(warnings) == 0) + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/tclib.py b/tools/grit/grit/tclib.py new file mode 100644 index 0000000..0623a5a --- /dev/null +++ b/tools/grit/grit/tclib.py @@ -0,0 +1,233 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Adaptation of the extern.tclib classes for our needs. +''' + + +import re +import types + +from grit import exception +import grit.extern.tclib + +def Identity(i): + return i + + +class BaseMessage(object): + '''Base class with methods shared by Message and Translation. + ''' + + def __init__(self, text='', placeholders=[], description='', meaning=''): + self.parts = [] + self.placeholders = [] + self.description = description + self.meaning = meaning + self.dirty = True # True if self.id is (or might be) wrong + self.id = 0 + + if text != '': + if not placeholders or placeholders == []: + self.AppendText(text) + else: + tag_map = {} + for placeholder in placeholders: + tag_map[placeholder.GetPresentation()] = [placeholder, 0] + tag_re = '(' + '|'.join(tag_map.keys()) + ')' + # This creates a regexp like '(TAG1|TAG2|TAG3)' + chunked_text = re.split(tag_re, text) + for chunk in chunked_text: + if chunk: # ignore empty chunk + if tag_map.has_key(chunk): + self.AppendPlaceholder(tag_map[chunk][0]) + tag_map[chunk][1] += 1 # increase placeholder use count + else: + self.AppendText(chunk) + for key in tag_map.keys(): + assert tag_map[key][1] != 0 + + def GetRealContent(self, escaping_function=Identity): + '''Returns the original content, i.e. what your application and users + will see. + + Specify a function to escape each translateable bit, if you like. + ''' + bits = [] + for item in self.parts: + if isinstance(item, types.StringTypes): + bits.append(escaping_function(item)) + else: + bits.append(item.GetOriginal()) + return ''.join(bits) + + def GetPresentableContent(self): + presentable_content = [] + for part in self.parts: + if isinstance(part, Placeholder): + presentable_content.append(part.GetPresentation()) + else: + presentable_content.append(part) + return ''.join(presentable_content) + + def AppendPlaceholder(self, placeholder): + assert isinstance(placeholder, Placeholder) + dup = False + for other in self.GetPlaceholders(): + if (other.presentation.find(placeholder.presentation) != -1 or + placeholder.presentation.find(other.presentation) != -1): + assert(False, "Placeholder names must be unique and must not overlap") + if other.presentation == placeholder.presentation: + assert other.original == placeholder.original + dup = True + + if not dup: + self.placeholders.append(placeholder) + self.parts.append(placeholder) + self.dirty = True + + def AppendText(self, text): + assert isinstance(text, types.StringTypes) + assert text != '' + + self.parts.append(text) + self.dirty = True + + def GetContent(self): + '''Returns the parts of the message. You may modify parts if you wish. + Note that you must not call GetId() on this object until you have finished + modifying the contents. + ''' + self.dirty = True # user might modify content + return self.parts + + def GetDescription(self): + return self.description + + def SetDescription(self, description): + self.description = description + + def GetMeaning(self): + return self.meaning + + def GetId(self): + if self.dirty: + self.id = self.GenerateId() + self.dirty = False + return self.id + + def GenerateId(self): + # Must use a UTF-8 encoded version of the presentable content, along with + # the meaning attribute, to match the TC. + return grit.extern.tclib.GenerateMessageId( + self.GetPresentableContent().encode('utf-8'), self.meaning) + + def GetPlaceholders(self): + return self.placeholders + + def FillTclibBaseMessage(self, msg): + msg.SetDescription(self.description.encode('utf-8')) + + for part in self.parts: + if isinstance(part, Placeholder): + ph = grit.extern.tclib.Placeholder( + part.presentation.encode('utf-8'), + part.original.encode('utf-8'), + part.example.encode('utf-8')) + msg.AppendPlaceholder(ph) + else: + msg.AppendText(part.encode('utf-8')) + + +class Message(BaseMessage): + '''A message.''' + + def __init__(self, text='', placeholders=[], description='', meaning=''): + BaseMessage.__init__(self, text, placeholders, description, meaning) + + def ToTclibMessage(self): + msg = grit.extern.tclib.Message('utf-8', meaning=self.meaning) + self.FillTclibBaseMessage(msg) + return msg + +class Translation(BaseMessage): + '''A translation.''' + + def __init__(self, text='', id='', placeholders=[], description='', meaning=''): + BaseMessage.__init__(self, text, placeholders, description, meaning) + self.id = id + + def GetId(self): + assert id != '', "ID has not been set." + return self.id + + def SetId(self, id): + self.id = id + + def ToTclibMessage(self): + msg = grit.extern.tclib.Message( + 'utf-8', id=self.id, meaning=self.meaning) + self.FillTclibBaseMessage(msg) + return msg + + +class Placeholder(grit.extern.tclib.Placeholder): + '''Modifies constructor to accept a Unicode string + ''' + + # Must match placeholder presentation names + _NAME_RE = re.compile('[A-Za-z0-9_]+') + + def __init__(self, presentation, original, example): + '''Creates a new placeholder. + + Args: + presentation: 'USERNAME' + original: '%s' + example: 'Joi' + ''' + assert presentation != '' + assert original != '' + assert example != '' + if not self._NAME_RE.match(presentation): + raise exception.InvalidPlaceholderName(presentation) + self.presentation = presentation + self.original = original + self.example = example + + def GetPresentation(self): + return self.presentation + + def GetOriginal(self): + return self.original + + def GetExample(self): + return self.example + diff --git a/tools/grit/grit/tclib_unittest.py b/tools/grit/grit/tclib_unittest.py new file mode 100644 index 0000000..c73dd4d --- /dev/null +++ b/tools/grit/grit/tclib_unittest.py @@ -0,0 +1,189 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.tclib''' + + +import sys +import os.path +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import types +import unittest + +from grit import tclib + +import grit.extern.tclib + + +class TclibUnittest(unittest.TestCase): + def testInit(self): + msg = tclib.Message(text=u'Hello Earthlings') + self.failUnless(msg.GetPresentableContent() == 'Hello Earthlings') + self.failUnless(isinstance(msg.GetPresentableContent(), types.StringTypes)) + + def testGetAttr(self): + msg = tclib.Message() + msg.AppendText(u'Hello') # Tests __getattr__ + self.failUnless(msg.GetPresentableContent() == 'Hello') + self.failUnless(isinstance(msg.GetPresentableContent(), types.StringTypes)) + + def testAll(self): + text = u'Howdie USERNAME' + phs = [tclib.Placeholder(u'USERNAME', u'%s', 'Joi')] + msg = tclib.Message(text=text, placeholders=phs) + self.failUnless(msg.GetPresentableContent() == 'Howdie USERNAME') + + trans = tclib.Translation(text=text, placeholders=phs) + self.failUnless(trans.GetPresentableContent() == 'Howdie USERNAME') + self.failUnless(isinstance(trans.GetPresentableContent(), types.StringTypes)) + + def testUnicodeReturn(self): + text = u'\u00fe' + msg = tclib.Message(text=text) + self.failUnless(msg.GetPresentableContent() == text) + from_list = msg.GetContent()[0] + self.failUnless(from_list == text) + + def testRegressionTranslationInherited(self): + '''Regression tests a bug that was caused by grit.tclib.Translation + inheriting from the translation console's Translation object + instead of only owning an instance of it. + ''' + msg = tclib.Message(text=u"BLA1\r\nFrom: BLA2 \u00fe BLA3", + placeholders=[ + tclib.Placeholder('BLA1', '%s', '%s'), + tclib.Placeholder('BLA2', '%s', '%s'), + tclib.Placeholder('BLA3', '%s', '%s')]) + transl = tclib.Translation(text=msg.GetPresentableContent(), + placeholders=msg.GetPlaceholders()) + content = transl.GetContent() + self.failUnless(isinstance(content[3], types.UnicodeType)) + + def testFingerprint(self): + # This has Windows line endings. That is on purpose. + id = grit.extern.tclib.GenerateMessageId( + 'Google Desktop for Enterprise\r\n' + 'Copyright (C) 2006 Google Inc.\r\n' + 'All Rights Reserved\r\n' + '\r\n' + '---------\r\n' + 'Contents\r\n' + '---------\r\n' + 'This distribution contains the following files:\r\n' + '\r\n' + 'GoogleDesktopSetup.msi - Installation and setup program\r\n' + 'GoogleDesktop.adm - Group Policy administrative template file\r\n' + 'AdminGuide.pdf - Google Desktop for Enterprise administrative guide\r\n' + '\r\n' + '\r\n' + '--------------\r\n' + 'Documentation\r\n' + '--------------\r\n' + 'Full documentation and installation instructions are in the \r\n' + 'administrative guide, and also online at \r\n' + 'http://desktop.google.com/enterprise/adminguide.html.\r\n' + '\r\n' + '\r\n' + '------------------------\r\n' + 'IBM Lotus Notes Plug-In\r\n' + '------------------------\r\n' + 'The Lotus Notes plug-in is included in the release of Google \r\n' + 'Desktop for Enterprise. The IBM Lotus Notes Plug-in for Google \r\n' + 'Desktop indexes mail, calendar, task, contact and journal \r\n' + 'documents from Notes. Discussion documents including those from \r\n' + 'the discussion and team room templates can also be indexed by \r\n' + 'selecting an option from the preferences. Once indexed, this data\r\n' + 'will be returned in Google Desktop searches. The corresponding\r\n' + 'document can be opened in Lotus Notes from the Google Desktop \r\n' + 'results page.\r\n' + '\r\n' + 'Install: The plug-in will install automatically during the Google \r\n' + 'Desktop setup process if Lotus Notes is already installed. Lotus \r\n' + 'Notes must not be running in order for the install to occur. \r\n' + '\r\n' + 'Preferences: Preferences and selection of databases to index are\r\n' + 'set in the \'Google Desktop for Notes\' dialog reached through the \r\n' + '\'Actions\' menu.\r\n' + '\r\n' + 'Reindexing: Selecting \'Reindex all databases\' will index all the \r\n' + 'documents in each database again.\r\n' + '\r\n' + '\r\n' + 'Notes Plug-in Known Issues\r\n' + '---------------------------\r\n' + '\r\n' + 'If the \'Google Desktop for Notes\' item is not available from the \r\n' + 'Lotus Notes Actions menu, then installation was not successful. \r\n' + 'Installation consists of writing one file, notesgdsplugin.dll, to \r\n' + 'the Notes application directory and a setting to the notes.ini \r\n' + 'configuration file. The most likely cause of an unsuccessful \r\n' + 'installation is that the installer was not able to locate the \r\n' + 'notes.ini file. Installation will complete if the user closes Notes\r\n' + 'and manually adds the following setting to this file on a new line:\r\n' + 'AddinMenus=notegdsplugin.dll\r\n' + '\r\n' + 'If the notesgdsplugin.dll file is not in the application directory\r\n' + '(e.g., C:\Program Files\Lotus\Notes) after Google Desktop \r\n' + 'installation, it is likely that Notes was not installed correctly. \r\n' + '\r\n' + 'Only local databases can be indexed. If they can be determined, \r\n' + 'the user\'s local mail file and address book will be included in the\r\n' + 'list automatically. Mail archives and other databases must be \r\n' + 'added with the \'Add\' button.\r\n' + '\r\n' + 'Some users may experience performance issues during the initial \r\n' + 'indexing of a database. The \'Perform the initial index of a \r\n' + 'database only when I\'m idle\' option will limit the indexing process\r\n' + 'to times when the user is not using the machine. If this does not \r\n' + 'alleviate the problem or the user would like to continually index \r\n' + 'but just do so more slowly or quickly, the GoogleWaitTime notes.ini\r\n' + 'value can be set. Increasing the GoogleWaitTime value will slow \r\n' + 'down the indexing process, and lowering the value will speed it up.\r\n' + 'A value of zero causes the fastest possible indexing. Removing the\r\n' + 'ini parameter altogether returns it to the default (20).\r\n' + '\r\n' + 'Crashes have been known to occur with certain types of history \r\n' + 'bookmarks. If the Notes client seems to crash randomly, try \r\n' + 'disabling the \'Index note history\' option. If it crashes before,\r\n' + 'you can get to the preferences, add the following line to your \r\n' + 'notes.ini file:\r\n' + 'GDSNoIndexHistory=1\r\n') + self.failUnless(id == '8961534701379422820') + + def testPlaceholderNameChecking(self): + try: + ph = tclib.Placeholder('BINGO BONGO', 'bla', 'bla') + except exception.InvalidPlaceholderName: + pass # Expect exception to be thrown because presentation contained space + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/test_suite_all.py b/tools/grit/grit/test_suite_all.py new file mode 100644 index 0000000..c102102 --- /dev/null +++ b/tools/grit/grit/test_suite_all.py @@ -0,0 +1,107 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit test suite that collects all test cases for GRIT.''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + + +# TODO(joi) Use unittest.defaultTestLoader to automatically load tests +# from modules. Iterating over the directory and importing could then +# automate this all the way, if desired. + + +class TestSuiteAll(unittest.TestSuite): + def __init__(self): + super(type(self), self).__init__() + # Imports placed here to prevent circular imports. + from grit import grd_reader_unittest + from grit import grit_runner_unittest + from grit.node import base_unittest + from grit.node import io_unittest + from grit import clique_unittest + from grit.node import misc_unittest + from grit.gather import rc_unittest + from grit.gather import tr_html_unittest + from grit.node import message_unittest + from grit import tclib_unittest + import grit.format.rc_unittest + from grit.tool import rc2grd_unittest + from grit.tool import transl2tc_unittest + from grit.gather import txt_unittest + from grit.gather import admin_template_unittest + from grit import xtb_reader_unittest + from grit import util_unittest + from grit.tool import preprocess_unittest + from grit.tool import postprocess_unittest + from grit import shortcuts_unittests + from grit.gather import muppet_strings_unittest + from grit.node.custom import filename_unittest + + test_classes = [ + base_unittest.NodeUnittest, + io_unittest.FileNodeUnittest, + grit_runner_unittest.OptionArgsUnittest, + grd_reader_unittest.GrdReaderUnittest, + clique_unittest.MessageCliqueUnittest, + misc_unittest.GritNodeUnittest, + rc_unittest.RcUnittest, + tr_html_unittest.ParserUnittest, + tr_html_unittest.TrHtmlUnittest, + message_unittest.MessageUnittest, + tclib_unittest.TclibUnittest, + grit.format.rc_unittest.FormatRcUnittest, + rc2grd_unittest.Rc2GrdUnittest, + transl2tc_unittest.TranslationToTcUnittest, + txt_unittest.TxtUnittest, + admin_template_unittest.AdmGathererUnittest, + xtb_reader_unittest.XtbReaderUnittest, + misc_unittest.IfNodeUnittest, + util_unittest.UtilUnittest, + preprocess_unittest.PreProcessingUnittest, + postprocess_unittest.PostProcessingUnittest, + misc_unittest.ReleaseNodeUnittest, + shortcuts_unittests.ShortcutsUnittest, + muppet_strings_unittest.MuppetStringsUnittest, + filename_unittest.WindowsFilenameUnittest, + # add test classes here... + ] + + for test_class in test_classes: + self.addTest(unittest.makeSuite(test_class)) + + +if __name__ == '__main__': + unittest.TextTestRunner(verbosity=2).run(TestSuiteAll()) diff --git a/tools/grit/grit/tool/__init__.py b/tools/grit/grit/tool/__init__.py new file mode 100644 index 0000000..378cbaa --- /dev/null +++ b/tools/grit/grit/tool/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Package grit.tool +''' + +pass diff --git a/tools/grit/grit/tool/build.py b/tools/grit/grit/tool/build.py new file mode 100644 index 0000000..9059af4 --- /dev/null +++ b/tools/grit/grit/tool/build.py @@ -0,0 +1,232 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit build' tool along with integration for this tool with the +SCons build system. +''' + +import os +import getopt +import types +import sys + +from grit import grd_reader +from grit import util +from grit.tool import interface +from grit import shortcuts + + +def ParseDefine(define): + '''Parses a define that is either like "NAME" or "NAME=VAL" and + returns its components, using True as the default value. Values of + "1" and "0" are transformed to True and False respectively. + ''' + parts = [part.strip() for part in define.split('=')] + assert len(parts) >= 1 + name = parts[0] + val = True + if len(parts) > 1: + val = parts[1] + if val == "1": val = True + elif val == "0": val = False + return (name, val) + + +class RcBuilder(interface.Tool): + '''A tool that builds RC files and resource header files for compilation. + +Usage: grit build [-o OUTPUTDIR] [-D NAME[=VAL]]* + +All output options for this tool are specified in the input file (see +'grit help' for details on how to specify the input file - it is a global +option). + +Options: + + -o OUTPUTDIR Specify what directory output paths are relative to. + Defaults to the current directory. + + -D NAME[=VAL] Specify a C-preprocessor-like define NAME with optional + value VAL (defaults to 1) which will be used to control + conditional inclusion of resources. + +Conditional inclusion of resources only affects the output of files which +control which resources get linked into a binary, e.g. it affects .rc files +meant for compilation but it does not affect resource header files (that define +IDs). This helps ensure that values of IDs stay the same, that all messages +are exported to translation interchange files (e.g. XMB files), etc. +''' + + def ShortDescription(self): + return 'A tool that builds RC files for compilation.' + + def Run(self, opts, args): + self.output_directory = '.' + (own_opts, args) = getopt.getopt(args, 'o:D:') + for (key, val) in own_opts: + if key == '-o': + self.output_directory = val + elif key == '-D': + name, val = ParseDefine(val) + self.defines[name] = val + if len(args): + print "This tool takes no tool-specific arguments." + return 2 + self.SetOptions(opts) + if self.scons_targets: + self.VerboseOut('Using SCons targets to identify files to output.\n') + else: + self.VerboseOut('Output directory: %s (absolute path: %s)\n' % + (self.output_directory, + os.path.abspath(self.output_directory))) + self.res = grd_reader.Parse(opts.input, debug=opts.extra_verbose) + self.res.RunGatherers(recursive = True) + self.Process() + return 0 + + def __init__(self): + # Default file-creation function is built-in file(). Only done to allow + # overriding by unit test. + self.fo_create = file + + # key/value pairs of C-preprocessor like defines that are used for + # conditional output of resources + self.defines = {} + + # self.res is a fully-populated resource tree if Run() + # has been called, otherwise None. + self.res = None + + # Set to a list of filenames for the output nodes that are relative + # to the current working directory. They are in the same order as the + # output nodes in the file. + self.scons_targets = None + + # static method + def ProcessNode(node, output_node, outfile): + '''Processes a node in-order, calling its formatter before and after + recursing to its children. + + Args: + node: grit.node.base.Node subclass + output_node: grit.node.io.File + outfile: open filehandle + ''' + base_dir = util.dirname(output_node.GetOutputFilename()) + + try: + formatter = node.ItemFormatter(output_node.GetType()) + if formatter: + outfile.write(formatter.Format(node, output_node.GetLanguage(), + begin_item=True, output_dir=base_dir)) + except: + print u"Error processing node %s" % unicode(node) + raise + + for child in node.children: + RcBuilder.ProcessNode(child, output_node, outfile) + + try: + if formatter: + outfile.write(formatter.Format(node, output_node.GetLanguage(), + begin_item=False, output_dir=base_dir)) + except: + print u"Error processing node %s" % unicode(node) + raise + ProcessNode = staticmethod(ProcessNode) + + + def Process(self): + # Update filenames with those provided by SCons if we're being invoked + # from SCons. The list of SCons targets also includes all <structure> + # node outputs, but it starts with our output files, in the order they + # occur in the .grd + if self.scons_targets: + assert len(self.scons_targets) >= len(self.res.GetOutputFiles()) + outfiles = self.res.GetOutputFiles() + for ix in range(len(outfiles)): + outfiles[ix].output_filename = os.path.abspath( + self.scons_targets[ix]) + else: + for output in self.res.GetOutputFiles(): + output.output_filename = os.path.abspath(os.path.join( + self.output_directory, output.GetFilename())) + + for output in self.res.GetOutputFiles(): + self.VerboseOut('Creating %s...' % output.GetFilename()) + # Microsoft's RC compiler can only deal with single-byte or double-byte + # files (no UTF-8), so we make all RC files UTF-16 to support all + # character sets. + if output.GetType() in ['rc_header']: + encoding = 'cp1252' + outname = output.GetOutputFilename() + oldname = outname + '.tmp' + if os.access(oldname, os.F_OK): + os.remove(oldname) + try: + os.rename(outname, oldname) + except OSError: + oldname = None + else: + encoding = 'utf_16' + outfile = util.WrapOutputStream( + self.fo_create(output.GetOutputFilename(), 'wb'), + encoding) + + # Set the context, for conditional inclusion of resources + self.res.SetOutputContext(output.GetLanguage(), self.defines) + + # TODO(joi) Handle this more gracefully + import grit.format.rc_header + grit.format.rc_header.Item.ids_ = {} + + # Iterate in-order through entire resource tree, calling formatters on + # the entry into a node and on exit out of it. + self.ProcessNode(self.res, output, outfile) + + outfile.close() + if output.GetType() in ['rc_header'] and oldname: + if open(oldname).read() != open(outname).read(): + os.remove(oldname) + else: + os.remove(outname) + os.rename(oldname, outname) + self.VerboseOut(' done.\n') + + # Print warnings if there are any duplicate shortcuts. + print '\n'.join(shortcuts.GenerateDuplicateShortcutsWarnings( + self.res.UberClique(), self.res.GetTcProject())) + + # Print out any fallback warnings, and missing translation errors, and + # exit with an error code if there are missing translations in a non-pseudo + # build + print self.res.UberClique().MissingTranslationsReport() + if self.res.UberClique().HasMissingTranslations(): + sys.exit(-1) diff --git a/tools/grit/grit/tool/count.py b/tools/grit/grit/tool/count.py new file mode 100644 index 0000000..c5ede8c --- /dev/null +++ b/tools/grit/grit/tool/count.py @@ -0,0 +1,68 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Count number of occurrences of a given message ID +''' + +import getopt +import os +import types + +from grit.tool import interface +from grit import grd_reader +from grit import util + +from grit.extern import tclib + + +class CountMessage(interface.Tool): + '''Count the number of times a given message ID is used. +''' + + def __init__(self): + pass + + def ShortDescription(self): + return 'Exports all translateable messages into an XMB file.' + + def Run(self, opts, args): + self.SetOptions(opts) + + id = args[0] + res_tree = grd_reader.Parse(opts.input, debug=opts.extra_verbose) + res_tree.OnlyTheseTranslations([]) + res_tree.RunGatherers(True) + + count = 0 + for c in res_tree.UberClique().AllCliques(): + if c.GetId() == id: + count += 1 + + print "There are %d occurrences of message %s." % (count, id) diff --git a/tools/grit/grit/tool/diff_structures.py b/tools/grit/grit/tool/diff_structures.py new file mode 100644 index 0000000..bef5d56 --- /dev/null +++ b/tools/grit/grit/tool/diff_structures.py @@ -0,0 +1,139 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit sdiff' tool. +''' + +import os +import getopt +import tempfile + +from grit.node import structure +from grit.tool import interface + +from grit import constants +from grit import util + +# Builds the description for the tool (used as the __doc__ +# for the DiffStructures class). +_class_doc = """\ +Allows you to view the differences in the structure of two files, +disregarding their translateable content. Translateable portions of +each file are changed to the string "TTTTTT" before invoking the diff program +specified by the P4DIFF environment variable. + +Usage: grit sdiff [-t TYPE] [-s SECTION] [-e ENCODING] LEFT RIGHT + +LEFT and RIGHT are the files you want to diff. SECTION is required +for structure types like 'dialog' to identify the part of the file to look at. +ENCODING indicates the encoding of the left and right files (default 'cp1252'). +TYPE can be one of the following, defaults to 'tr_html': +""" +for gatherer in structure._GATHERERS: + _class_doc += " - %s\n" % gatherer + + +class DiffStructures(interface.Tool): + __doc__ = _class_doc + + def __init__(self): + self.section = None + self.left_encoding = 'cp1252' + self.right_encoding = 'cp1252' + self.structure_type = 'tr_html' + + def ShortDescription(self): + return 'View differences without regard for translateable portions.' + + def Run(self, global_opts, args): + (opts, args) = getopt.getopt(args, 's:e:t:', + ['left_encoding=', 'right_encoding=']) + for key, val in opts: + if key == '-s': + self.section = val + elif key == '-e': + self.left_encoding = val + self.right_encoding = val + elif key == '-t': + self.structure_type = val + elif key == '--left_encoding': + self.left_encoding = val + elif key == '--right_encoding': + self.right_encoding == val + + if len(args) != 2: + print "Incorrect usage - 'grit help sdiff' for usage details." + return 2 + + if 'P4DIFF' not in os.environ: + print "Environment variable P4DIFF not set; defaulting to 'windiff'." + diff_program = 'windiff' + else: + diff_program = os.environ['P4DIFF'] + + left_trans = self.MakeStaticTranslation(args[0], self.left_encoding) + try: + try: + right_trans = self.MakeStaticTranslation(args[1], self.right_encoding) + + os.system('%s %s %s' % (diff_program, left_trans, right_trans)) + finally: + os.unlink(right_trans) + finally: + os.unlink(left_trans) + + def MakeStaticTranslation(self, original_filename, encoding): + """Given the name of the structure type (self.structure_type), the filename + of the file holding the original structure, and optionally the "section" key + identifying the part of the file to look at (self.section), creates a + temporary file holding a "static" translation of the original structure + (i.e. one where all translateable parts have been replaced with "TTTTTT") + and returns the temporary file name. It is the caller's responsibility to + delete the file when finished. + + Args: + original_filename: 'c:\\bingo\\bla.rc' + + Return: + 'c:\\temp\\werlkjsdf334.tmp' + """ + original = structure._GATHERERS[self.structure_type].FromFile( + original_filename, extkey=self.section, encoding=encoding) + original.Parse() + translated = original.Translate(constants.CONSTANT_LANGUAGE, False) + + fname = tempfile.mktemp() + fhandle = file(fname, 'w') + writer = util.WrapOutputStream(fhandle) + writer.write("Original filename: %s\n=============\n\n" % original_filename) + writer.write(translated) # write in UTF-8 + fhandle.close() + + return fname
\ No newline at end of file diff --git a/tools/grit/grit/tool/interface.py b/tools/grit/grit/tool/interface.py new file mode 100644 index 0000000..d56fac4 --- /dev/null +++ b/tools/grit/grit/tool/interface.py @@ -0,0 +1,84 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Base class and interface for tools. +''' + +import sys + +class Tool(object): + '''Base class for all tools. Tools should use their docstring (i.e. the + class-level docstring) for the help they want to have printed when they + are invoked.''' + + # + # Interface (abstract methods) + # + + def ShortDescription(self): + '''Returns a short description of the functionality of the tool.''' + raise NotImplementedError() + + def Run(self, global_options, my_arguments): + '''Runs the tool. + + Args: + global_options: object grit_runner.Options + my_arguments: [arg1 arg2 ...] + + Return: + 0 for success, non-0 for error + ''' + raise NotImplementedError() + + # + # Base class implementation + # + + def __init__(self): + self.o = None + + def SetOptions(self, opts): + self.o = opts + + def Out(self, text): + '''Always writes out 'text'.''' + self.o.output_stream.write(text) + + def VerboseOut(self, text): + '''Writes out 'text' if the verbose option is on.''' + if self.o.verbose: + self.o.output_stream.write(text) + + def ExtraVerboseOut(self, text): + '''Writes out 'text' if the extra-verbose option is on. + ''' + if self.o.extra_verbose: + self.o.output_stream.write(text)
\ No newline at end of file diff --git a/tools/grit/grit/tool/menu_from_parts.py b/tools/grit/grit/tool/menu_from_parts.py new file mode 100644 index 0000000..480335c --- /dev/null +++ b/tools/grit/grit/tool/menu_from_parts.py @@ -0,0 +1,108 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit menufromparts' tool.''' + +import os +import getopt +import types + +from grit.tool import interface +from grit.tool import transl2tc +from grit import grd_reader +from grit import tclib +from grit import util +from grit import xtb_reader + + +import grit.extern.tclib + + +class MenuTranslationsFromParts(interface.Tool): + '''One-off tool to generate translated menu messages (where each menu is kept +in a single message) based on existing translations of the individual menu +items. Was needed when changing menus from being one message per menu item +to being one message for the whole menu.''' + + def ShortDescription(self): + return ('Create translations of whole menus from existing translations of ' + 'menu items.') + + def Run(self, globopt, args): + self.SetOptions(globopt) + assert len(args) == 2, "Need exactly two arguments, the XTB file and the output file" + + xtb_file = args[0] + output_file = args[1] + + grd = grd_reader.Parse(self.o.input, debug=self.o.extra_verbose) + grd.OnlyTheseTranslations([]) # don't load translations + grd.RunGatherers(recursive = True) + + xtb = {} + def Callback(msg_id, parts): + msg = [] + for part in parts: + if part[0]: + msg = [] + break # it had a placeholder so ignore it + else: + msg.append(part[1]) + if len(msg): + xtb[msg_id] = ''.join(msg) + f = file(xtb_file) + xtb_reader.Parse(f, Callback) + f.close() + + translations = [] # list of translations as per transl2tc.WriteTranslations + for node in grd: + if node.name == 'structure' and node.attrs['type'] == 'menu': + assert len(node.GetCliques()) == 1 + message = node.GetCliques()[0].GetMessage() + translation = [] + + contents = message.GetContent() + for part in contents: + if isinstance(part, types.StringTypes): + id = grit.extern.tclib.GenerateMessageId(part) + if id not in xtb: + print "WARNING didn't find all translations for menu %s" % node.attrs['name'] + translation = [] + break + translation.append(xtb[id]) + else: + translation.append(part.GetPresentation()) + + if len(translation): + translations.append([message.GetId(), ''.join(translation)]) + + f = util.WrapOutputStream(file(output_file, 'w')) + transl2tc.TranslationToTc.WriteTranslations(f, translations) + f.close() diff --git a/tools/grit/grit/tool/newgrd.py b/tools/grit/grit/tool/newgrd.py new file mode 100644 index 0000000..29e284c --- /dev/null +++ b/tools/grit/grit/tool/newgrd.py @@ -0,0 +1,96 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Tool to create a new, empty .grd file with all the basic sections. +''' + +from grit.tool import interface +from grit import constants +from grit import util + +# The contents of the new .grd file +_FILE_CONTENTS = '''\ +<?xml version="1.0" encoding="UTF-8"?> +<grit base_dir="." latest_public_release="0" current_release="1" + source_lang_id="en" enc_check="%s"> + <outputs> + <!-- TODO add each of your output files. Modify the three below, and add + your own for your various languages. See the user's guide for more + details. + Note that all output references are relative to the output directory + which is specified at build time. --> + <output filename="resource.h" type="rc_header" /> + <output filename="en_resource.rc" type="rc_all" /> + <output filename="fr_resource.rc" type="rc_all" /> + </outputs> + <translations> + <!-- TODO add references to each of the XTB files (from the Translation + Console) that contain translations of messages in your project. Each + takes a form like <file path="english.xtb" />. Remember that all file + references are relative to this .grd file. --> + </translations> + <release seq="1"> + <includes> + <!-- TODO add a list of your included resources here, e.g. BMP and GIF + resources. --> + </includes> + <structures> + <!-- TODO add a list of all your structured resources here, e.g. HTML + templates, menus, dialogs etc. Note that for menus, dialogs and version + information resources you reference an .rc file containing them.--> + </structures> + <messages> + <!-- TODO add all of your "string table" messages here. Remember to + change nontranslateable parts of the messages into placeholders (using the + <ph> element). You can also use the 'grit add' tool to help you identify + nontranslateable parts and create placeholders for them. --> + </messages> + </release> +</grit>''' % constants.ENCODING_CHECK + + +class NewGrd(interface.Tool): + '''Usage: grit newgrd OUTPUT_FILE + +Creates a new, empty .grd file OUTPUT_FILE with comments about what to put +where in the file.''' + + def ShortDescription(self): + return 'Create a new empty .grd file.' + + def Run(self, global_options, my_arguments): + if not len(my_arguments) == 1: + print 'This tool requires exactly one argument, the name of the output file.' + return 2 + filename = my_arguments[0] + out = util.WrapOutputStream(file(filename, 'w'), 'utf-8') + out.write(_FILE_CONTENTS) + out.close() + print "Wrote file %s" % filename
\ No newline at end of file diff --git a/tools/grit/grit/tool/postprocess_interface.py b/tools/grit/grit/tool/postprocess_interface.py new file mode 100644 index 0000000..afcb12b --- /dev/null +++ b/tools/grit/grit/tool/postprocess_interface.py @@ -0,0 +1,57 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' Base class for postprocessing of RC files. +''' + +import sys + +class PostProcessor(object): + ''' Base class for postprocessing of the RC file data before being + output through the RC2GRD tool. You should implement this class if + you want GRIT to do specific things to the RC files after it has + converted the data into GRD format, i.e. change the content of the + RC file, and put it into a P4 changelist, etc.''' + + + def Process(self, rctext, rcpath, grdnode): + ''' Processes the data in rctext and grdnode. + Args: + rctext: string containing the contents of the RC file being processed. + rcpath: the path used to access the file. + grdtext: the root node of the grd xml data generated by + the rc2grd tool. + + Return: + The root node of the processed GRD tree. + ''' + raise NotImplementedError() + + diff --git a/tools/grit/grit/tool/postprocess_unittest.py b/tools/grit/grit/tool/postprocess_unittest.py new file mode 100644 index 0000000..16b31e6 --- /dev/null +++ b/tools/grit/grit/tool/postprocess_unittest.py @@ -0,0 +1,87 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit test that checks postprocessing of files. + Tests postprocessing by having the postprocessor + modify the grd data tree, changing the message name attributes. +''' + +import os +import re +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +import grit.tool.postprocess_interface +from grit.tool import rc2grd + + +class PostProcessingUnittest(unittest.TestCase): + + def testPostProcessing(self): + rctext = '''STRINGTABLE +BEGIN + DUMMY_STRING_1 "String 1" + // Some random description + DUMMY_STRING_2 "This text was added during preprocessing" +END + ''' + tool = rc2grd.Rc2Grd() + class DummyOpts(object): + verbose = False + extra_verbose = False + tool.o = DummyOpts() + tool.post_process = 'grit.tool.postprocess_unittest.DummyPostProcessor' + result = tool.Process(rctext, '.\resource.rc') + + self.failUnless( + result.children[2].children[2].children[0].attrs['name'] == 'SMART_STRING_1') + self.failUnless( + result.children[2].children[2].children[1].attrs['name'] == 'SMART_STRING_2') + +class DummyPostProcessor(grit.tool.postprocess_interface.PostProcessor): + ''' + Post processing replaces all message name attributes containing "DUMMY" to + "SMART". + ''' + def Process(self, rctext, rcpath, grdnode): + smarter = re.compile(r'(DUMMY)(.*)') + messages = grdnode.children[2].children[2] + for node in messages.children: + name_attr = node.attrs['name'] + m = smarter.search(name_attr) + if m: + node.attrs['name'] = 'SMART' + m.group(2) + return grdnode + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/tool/preprocess_interface.py b/tools/grit/grit/tool/preprocess_interface.py new file mode 100644 index 0000000..a6a7f4b --- /dev/null +++ b/tools/grit/grit/tool/preprocess_interface.py @@ -0,0 +1,53 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' Base class for preprocessing of RC files. +''' + +import sys + +class PreProcessor(object): + ''' Base class for preprocessing of the RC file data before being + output through the RC2GRD tool. You should implement this class if + you have specific constructs in your RC files that GRIT cannot handle.''' + + + def Process(self, rctext, rcpath): + ''' Processes the data in rctext. + Args: + rctext: string containing the contents of the RC file being processed + rcpath: the path used to access the file. + + Return: + The processed text. + ''' + raise NotImplementedError() + + diff --git a/tools/grit/grit/tool/preprocess_unittest.py b/tools/grit/grit/tool/preprocess_unittest.py new file mode 100644 index 0000000..ef5a5f9 --- /dev/null +++ b/tools/grit/grit/tool/preprocess_unittest.py @@ -0,0 +1,73 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit test that checks preprocessing of files. + Tests preprocessing by adding having the preprocessor + provide the actual rctext data. +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +import grit.tool.preprocess_interface +from grit.tool import rc2grd + + +class PreProcessingUnittest(unittest.TestCase): + + def testPreProcessing(self): + tool = rc2grd.Rc2Grd() + class DummyOpts(object): + verbose = False + extra_verbose = False + tool.o = DummyOpts() + tool.pre_process = 'grit.tool.preprocess_unittest.DummyPreProcessor' + result = tool.Process('', '.\resource.rc') + + self.failUnless( + result.children[2].children[2].children[0].attrs['name'] == 'DUMMY_STRING_1') + +class DummyPreProcessor(grit.tool.preprocess_interface.PreProcessor): + def Process(self, rctext, rcpath): + rctext = '''STRINGTABLE +BEGIN + DUMMY_STRING_1 "String 1" + // Some random description + DUMMY_STRING_2 "This text was added during preprocessing" +END + ''' + return rctext + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/tool/rc2grd.py b/tools/grit/grit/tool/rc2grd.py new file mode 100644 index 0000000..24080eb --- /dev/null +++ b/tools/grit/grit/tool/rc2grd.py @@ -0,0 +1,427 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit rc2grd' tool.''' + + +import os.path +import getopt +import re +import StringIO +import types + +import grit.node.empty +from grit.node import include +from grit.node import structure +from grit.node import message + +from grit.gather import rc +from grit.gather import tr_html + +from grit.tool import interface +from grit.tool import postprocess_interface +from grit.tool import preprocess_interface + +from grit import grd_reader +from grit import tclib +from grit import util + + +# Matches files referenced from an .rc file +_FILE_REF = re.compile(''' + ^(?P<id>[A-Z_0-9.]+)[ \t]+ + (?P<type>[A-Z_0-9]+)[ \t]+ + "(?P<file>.*?([^"]|""))"[ \t]*$''', re.VERBOSE | re.MULTILINE) + + +# Matches a dialog section +_DIALOG = re.compile('^(?P<id>[A-Z0-9_]+)\s+DIALOG(EX)?\s.+?^BEGIN\s*$.+?^END\s*$', + re.MULTILINE | re.DOTALL) + + +# Matches a menu section +_MENU = re.compile('^(?P<id>[A-Z0-9_]+)\s+MENU.+?^BEGIN\s*$.+?^END\s*$', + re.MULTILINE | re.DOTALL) + + +# Matches a versioninfo section +_VERSIONINFO = re.compile('^(?P<id>[A-Z0-9_]+)\s+VERSIONINFO\s.+?^BEGIN\s*$.+?^END\s*$', + re.MULTILINE | re.DOTALL) + + +# Matches a stringtable +_STRING_TABLE = re.compile('^STRINGTABLE(\s+(PRELOAD|DISCARDABLE|CHARACTERISTICS.+|LANGUAGE.+|VERSION.+))*\s*\nBEGIN\s*$(?P<body>.+?)^END\s*$', + re.MULTILINE | re.DOTALL) + + +# Matches each message inside a stringtable, breaking it up into comments, +# the ID of the message, and the (RC-escaped) message text. +_MESSAGE = re.compile(''' + (?P<comment>(^\s+//.+?)*) # 0 or more lines of comments preceding the message + ^\s* + (?P<id>[A-Za-z0-9_]+) # id + \s+ + "(?P<text>.*?([^"]|""))"([^"]|$) # The message itself + ''', re.MULTILINE | re.DOTALL | re.VERBOSE) + + +# Matches each line of comment text in a multi-line comment. +_COMMENT_TEXT = re.compile('^\s*//\s*(?P<text>.+?)$', re.MULTILINE) + + +# Matches a string that is empty or all whitespace +_WHITESPACE_ONLY = re.compile('\A\s*\Z', re.MULTILINE) + + +# Finds printf and FormatMessage style format specifiers +# Uses non-capturing groups except for the outermost group, so the output of +# re.split() should include both the normal text and what we intend to +# replace with placeholders. +# TODO(joi) Check documentation for printf (and Windows variants) and FormatMessage +_FORMAT_SPECIFIER = re.compile( + '(%[-# +]?(?:[0-9]*|\*)(?:\.(?:[0-9]+|\*))?(?:h|l|L)?' # printf up to last char + '(?:d|i|o|u|x|X|e|E|f|F|g|G|c|r|s|ls|ws)' # printf last char + '|\$[1-9][0-9]*)') # FormatMessage + + +class Rc2Grd(interface.Tool): + '''A tool for converting .rc files to .grd files. This tool is only for +converting the source (nontranslated) .rc file to a .grd file. For importing +existing translations, use the rc2xtb tool. + +Usage: grit [global options] rc2grd [OPTIONS] RCFILE + +The tool takes a single argument, which is the path to the .rc file to convert. +It outputs a .grd file with the same name in the same directory as the .rc file. +The .grd file may have one or more TODO comments for things that have to be +cleaned up manually. + +OPTIONS may be any of the following: + + -e ENCODING Specify the ENCODING of the .rc file. Default is 'cp1252'. + + -h TYPE Specify the TYPE attribute for HTML structures. + Default is 'tr_html'. + + -u ENCODING Specify the ENCODING of HTML files. Default is 'utf-8'. + + -n MATCH Specify the regular expression to match in comments that will + indicate that the resource the comment belongs to is not + translateable. Default is 'Not locali(s|z)able'. + + -r GRDFILE Specify that GRDFILE should be used as a "role model" for + any placeholders that otherwise would have had TODO names. + This attempts to find an identical message in the GRDFILE + and uses that instead of the automatically placeholderized + message. + + --pre CLASS Specify an optional, fully qualified classname, which + has to be a subclass of grit.tool.PreProcessor, to + run on the text of the RC file before conversion occurs. + This can be used to support constructs in the RC files + that GRIT cannot handle on its own. + + --post CLASS Specify an optional, fully qualified classname, which + has to be a subclass of grit.tool.PostProcessor, to + run on the text of the converted RC file. + This can be used to alter the content of the RC file + based on the conversion that occured. + +For menus, dialogs and version info, the .grd file will refer to the original +.rc file. Once conversion is complete, you can strip the original .rc file +of its string table and all comments as these will be available in the .grd +file. + +Note that this tool WILL NOT obey C preprocessor rules, so even if something +is #if 0-ed out it will still be included in the output of this tool +Therefore, if your .rc file contains sections like this, you should run the +C preprocessor on the .rc file or manually edit it before using this tool. +''' + + def ShortDescription(self): + return 'A tool for converting .rc source files to .grd files.' + + def __init__(self): + self.input_encoding = 'cp1252' + self.html_type = 'tr_html' + self.html_encoding = 'utf-8' + self.not_localizable_re = re.compile('Not locali(s|z)able') + self.role_model = None + self.pre_process = None + self.post_process = None + + def ParseOptions(self, args): + '''Given a list of arguments, set this object's options and return + all non-option arguments. + ''' + (own_opts, args) = getopt.getopt(args, 'e:h:u:n:r', ['pre=', 'post=']) + for (key, val) in own_opts: + if key == '-e': + self.input_encoding = val + elif key == '-h': + self.html_type = val + elif key == '-u': + self.html_encoding = val + elif key == '-n': + self.not_localizable_re = re.compile(val) + elif key == '-r': + self.role_model = grd_reader.Parse(val) + elif key == '--pre': + self.pre_process = val + elif key == '--post': + self.post_process = val + return args + + def Run(self, opts, args): + args = self.ParseOptions(args) + if len(args) != 1: + print ('This tool takes a single tool-specific argument, the path to the\n' + '.rc file to process.') + return 2 + self.SetOptions(opts) + + path = args[0] + out_path = os.path.join(util.dirname(path), + os.path.splitext(os.path.basename(path))[0] + '.grd') + + rcfile = util.WrapInputStream(file(path, 'r'), self.input_encoding) + rctext = rcfile.read() + + grd_text = unicode(self.Process(rctext, path)) + + rcfile.close() + + outfile = util.WrapOutputStream(file(out_path, 'w'), 'utf-8') + outfile.write(grd_text) + outfile.close() + + print 'Wrote output file %s.\nPlease check for TODO items in the file.' % out_path + + + def Process(self, rctext, rc_path): + '''Processes 'rctext' and returns a resource tree corresponding to it. + + Args: + rctext: complete text of the rc file + rc_path: 'resource\resource.rc' + + Return: + grit.node.base.Node subclass + ''' + + if self.pre_process: + preprocess_class = util.NewClassInstance(self.pre_process, + preprocess_interface.PreProcessor) + if preprocess_class: + rctext = preprocess_class.Process(rctext, rc_path) + else: + self.Out( + 'PreProcessing class could not be found. Skipping preprocessing.\n') + + # Start with a basic skeleton for the .grd file + root = grd_reader.Parse(StringIO.StringIO( + '''<?xml version="1.0" encoding="UTF-8"?> + <grit base_dir="." latest_public_release="0" + current_release="1" source_lang_id="en"> + <outputs /> + <translations /> + <release seq="1"> + <includes /> + <structures /> + <messages /> + </release> + </grit>'''), util.dirname(rc_path)) + includes = root.children[2].children[0] + structures = root.children[2].children[1] + messages = root.children[2].children[2] + assert (isinstance(includes, grit.node.empty.IncludesNode) and + isinstance(structures, grit.node.empty.StructuresNode) and + isinstance(messages, grit.node.empty.MessagesNode)) + + self.AddIncludes(rctext, includes) + self.AddStructures(rctext, structures, os.path.basename(rc_path)) + self.AddMessages(rctext, messages) + + self.VerboseOut('Validating that all IDs are unique...\n') + root.ValidateUniqueIds() + self.ExtraVerboseOut('Done validating that all IDs are unique.\n') + + if self.post_process: + postprocess_class = util.NewClassInstance(self.post_process, + postprocess_interface.PostProcessor) + if postprocess_class: + root = postprocess_class.Process(rctext, rc_path, root) + else: + self.Out( + 'PostProcessing class could not be found. Skipping postprocessing.\n') + + return root + + + def AddIncludes(self, rctext, node): + '''Scans 'rctext' for included resources (e.g. BITMAP, ICON) and + adds each included resource as an <include> child node of 'node'.''' + for m in _FILE_REF.finditer(rctext): + id = m.group('id') + type = m.group('type').upper() + fname = rc.Section.UnEscape(m.group('file')) + assert fname.find('\n') == -1 + if type != 'HTML': + self.VerboseOut('Processing %s with ID %s (filename: %s)\n' % (type, id, fname)) + node.AddChild(include.IncludeNode.Construct(node, id, type, fname)) + + + def AddStructures(self, rctext, node, rc_filename): + '''Scans 'rctext' for structured resources (e.g. menus, dialogs, version + information resources and HTML templates) and adds each as a <structure> + child of 'node'.''' + # First add HTML includes + for m in _FILE_REF.finditer(rctext): + id = m.group('id') + type = m.group('type').upper() + fname = rc.Section.UnEscape(m.group('file')) + if type == 'HTML': + node.AddChild(structure.StructureNode.Construct( + node, id, self.html_type, fname, self.html_encoding)) + + # Then add all RC includes + def AddStructure(type, id): + self.VerboseOut('Processing %s with ID %s\n' % (type, id)) + node.AddChild(structure.StructureNode.Construct(node, id, type, + rc_filename, + encoding=self.input_encoding)) + for m in _MENU.finditer(rctext): + AddStructure('menu', m.group('id')) + for m in _DIALOG.finditer(rctext): + AddStructure('dialog', m.group('id')) + for m in _VERSIONINFO.finditer(rctext): + AddStructure('version', m.group('id')) + + + def AddMessages(self, rctext, node): + '''Scans 'rctext' for all messages in string tables, preprocesses them as + much as possible for placeholders (e.g. messages containing $1, $2 or %s, %d + type format specifiers get those specifiers replaced with placeholders, and + HTML-formatted messages get run through the HTML-placeholderizer). Adds + each message as a <message> node child of 'node'.''' + for tm in _STRING_TABLE.finditer(rctext): + table = tm.group('body') + for mm in _MESSAGE.finditer(table): + comment_block = mm.group('comment') + comment_text = [] + for cm in _COMMENT_TEXT.finditer(comment_block): + comment_text.append(cm.group('text')) + comment_text = ' '.join(comment_text) + + id = mm.group('id') + text = rc.Section.UnEscape(mm.group('text')) + + self.VerboseOut('Processing message %s (text: "%s")\n' % (id, text)) + + msg_obj = self.Placeholderize(text) + + # Messages that contain only placeholders do not need translation. + is_translateable = False + for item in msg_obj.GetContent(): + if isinstance(item, types.StringTypes): + if not _WHITESPACE_ONLY.match(item): + is_translateable = True + + if self.not_localizable_re.search(comment_text): + is_translateable = False + + message_meaning = '' + internal_comment = '' + + # If we have a "role model" (existing GRD file) and this node exists + # in the role model, use the description, meaning and translateable + # attributes from the role model. + if self.role_model: + role_node = self.role_model.GetNodeById(id) + if role_node: + is_translateable = role_node.IsTranslateable() + message_meaning = role_node.attrs['meaning'] + comment_text = role_node.attrs['desc'] + internal_comment = role_node.attrs['internal_comment'] + + # For nontranslateable messages, we don't want the complexity of + # placeholderizing everything. + if not is_translateable: + msg_obj = tclib.Message(text=text) + + msg_node = message.MessageNode.Construct(node, msg_obj, id, + desc=comment_text, + translateable=is_translateable, + meaning=message_meaning) + msg_node.attrs['internal_comment'] = internal_comment + + node.AddChild(msg_node) + self.ExtraVerboseOut('Done processing message %s\n' % id) + + + def Placeholderize(self, text): + '''Creates a tclib.Message object from 'text', attempting to recognize + a few different formats of text that can be automatically placeholderized + (HTML code, printf-style format strings, and FormatMessage-style format + strings). + ''' + + try: + # First try HTML placeholderizing. + # TODO(joi) Allow use of non-TotalRecall flavors of HTML placeholderizing + msg = tr_html.HtmlToMessage(text, True) + for item in msg.GetContent(): + if not isinstance(item, types.StringTypes): + return msg # Contained at least one placeholder, so we're done + + # HTML placeholderization didn't do anything, so try to find printf or + # FormatMessage format specifiers and change them into placeholders. + msg = tclib.Message() + parts = _FORMAT_SPECIFIER.split(text) + todo_counter = 1 # We make placeholder IDs 'TODO_0001' etc. + for part in parts: + if _FORMAT_SPECIFIER.match(part): + msg.AppendPlaceholder(tclib.Placeholder( + 'TODO_%04d' % todo_counter, part, 'TODO')) + todo_counter += 1 + elif part != '': + msg.AppendText(part) + + if self.role_model and len(parts) > 1: # there are TODO placeholders + role_model_msg = self.role_model.UberClique().BestCliqueByOriginalText( + msg.GetRealContent(), '') + if role_model_msg: + # replace wholesale to get placeholder names and examples + msg = role_model_msg + + return msg + except: + print 'Exception processing message with text "%s"' % text + raise diff --git a/tools/grit/grit/tool/rc2grd_unittest.py b/tools/grit/grit/tool/rc2grd_unittest.py new file mode 100644 index 0000000..3e9cc3f --- /dev/null +++ b/tools/grit/grit/tool/rc2grd_unittest.py @@ -0,0 +1,162 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.tool.rc2grd''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import re +import StringIO +import unittest + +from grit.node import base +from grit.tool import rc2grd +from grit.gather import rc +from grit import grd_reader + + +class Rc2GrdUnittest(unittest.TestCase): + def testPlaceholderize(self): + tool = rc2grd.Rc2Grd() + original = "Hello %s, how are you? I'm $1 years old!" + msg = tool.Placeholderize(original) + self.failUnless(msg.GetPresentableContent() == "Hello TODO_0001, how are you? I'm TODO_0002 years old!") + self.failUnless(msg.GetRealContent() == original) + + def testHtmlPlaceholderize(self): + tool = rc2grd.Rc2Grd() + original = "Hello <b>[USERNAME]</b>, how are you? I'm [AGE] years old!" + msg = tool.Placeholderize(original) + self.failUnless(msg.GetPresentableContent() == + "Hello BEGIN_BOLDX_USERNAME_XEND_BOLD, how are you? I'm X_AGE_X years old!") + self.failUnless(msg.GetRealContent() == original) + + def testMenuWithoutWhitespaceRegression(self): + # There was a problem in the original regular expression for parsing out + # menu sections, that would parse the following block of text as a single + # menu instead of two. + two_menus = ''' +// Hyper context menus +IDR_HYPERMENU_FOLDER MENU +BEGIN + POPUP "HyperFolder" + BEGIN + MENUITEM "Open Containing Folder", IDM_OPENFOLDER + END +END + +IDR_HYPERMENU_FILE MENU +BEGIN + POPUP "HyperFile" + BEGIN + MENUITEM "Open Folder", IDM_OPENFOLDER + END +END + +''' + self.failUnless(len(rc2grd._MENU.findall(two_menus)) == 2) + + def testRegressionScriptWithTranslateable(self): + tool = rc2grd.Rc2Grd() + + # test rig + class DummyNode(base.Node): + def AddChild(self, item): + self.node = item + verbose = False + extra_verbose = False + tool.not_localizable_re = re.compile('') + tool.o = DummyNode() + + rc_text = '''STRINGTABLE\nBEGIN\nID_BINGO "<SPAN id=hp style='BEHAVIOR: url(#default#homepage)'></SPAN><script>if (!hp.isHomePage('[$~HOMEPAGE~$]')) {document.write(""<a href=\\""[$~SETHOMEPAGEURL~$]\\"" >Set As Homepage</a> - "");}</script>"\nEND\n''' + tool.AddMessages(rc_text, tool.o) + self.failUnless(tool.o.node.GetCdata().find('Set As Homepage') != -1) + + # TODO(joi) Improve the HTML parser to support translateables inside + # <script> blocks? + self.failUnless(tool.o.node.attrs['translateable'] == 'false') + + def testRoleModel(self): + rc_text = ('STRINGTABLE\n' + 'BEGIN\n' + ' // This should not show up\n' + ' IDS_BINGO "Hello %s, how are you?"\n' + ' // The first description\n' + ' IDS_BONGO "Hello %s, my name is %s, and yours?"\n' + ' IDS_PROGRAMS_SHUTDOWN_TEXT "Google Desktop Search needs to close the following programs:\\n\\n$1\\nThe installation will not proceed if you choose to cancel."\n' + 'END\n') + tool = rc2grd.Rc2Grd() + tool.role_model = grd_reader.Parse(StringIO.StringIO( + '''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <message name="IDS_BINGO"> + Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you? + </message> + <message name="IDS_BONGO" desc="The other description"> + Hello <ph name="USERNAME">%s<ex>Jakob</ex></ph>, my name is <ph name="ADMINNAME">%s<ex>Joi</ex></ph>, and yours? + </message> + <message name="IDS_PROGRAMS_SHUTDOWN_TEXT" desc="LIST_OF_PROGRAMS is replaced by a bulleted list of program names."> + Google Desktop Search needs to close the following programs: + +<ph name="LIST_OF_PROGRAMS">$1<ex>Program 1, Program 2</ex></ph> +The installation will not proceed if you choose to cancel. + </message> + </messages> + </release> + </grit>'''), dir='.') + + # test rig + class DummyOpts(object): + verbose = False + extra_verbose = False + tool.o = DummyOpts() + result = tool.Process(rc_text, '.\resource.rc') + self.failUnless( + result.children[2].children[2].children[0].attrs['desc'] == '') + self.failUnless( + result.children[2].children[2].children[0].children[0].attrs['name'] == 'USERNAME') + self.failUnless( + result.children[2].children[2].children[1].attrs['desc'] == 'The other description') + self.failUnless( + result.children[2].children[2].children[1].attrs['meaning'] == '') + self.failUnless( + result.children[2].children[2].children[1].children[0].attrs['name'] == 'USERNAME') + self.failUnless( + result.children[2].children[2].children[1].children[1].attrs['name'] == 'ADMINNAME') + self.failUnless( + result.children[2].children[2].children[2].children[0].attrs['name'] == 'LIST_OF_PROGRAMS') + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/tool/resize.py b/tools/grit/grit/tool/resize.py new file mode 100644 index 0000000..9081bff --- /dev/null +++ b/tools/grit/grit/tool/resize.py @@ -0,0 +1,325 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit resize' tool. +''' + +import getopt +import os +import types + +from grit.tool import interface +from grit.tool import build +from grit import grd_reader +from grit import pseudo +from grit import util + +from grit.node import include +from grit.node import structure +from grit.node import message + +from grit.format import rc_header + + +# Template for the .vcproj file, with a couple of [[REPLACEABLE]] parts. +PROJECT_TEMPLATE = '''\ +<?xml version="1.0" encoding="Windows-1252"?> +<VisualStudioProject + ProjectType="Visual C++" + Version="7.10" + Name="[[DIALOG_NAME]]" + ProjectGUID="[[PROJECT_GUID]]" + Keyword="Win32Proj"> + <Platforms> + <Platform + Name="Win32"/> + </Platforms> + <Configurations> + <Configuration + Name="Debug|Win32" + OutputDirectory="Debug" + IntermediateDirectory="Debug" + ConfigurationType="1" + CharacterSet="2"> + </Configuration> + </Configurations> + <References> + </References> + <Files> + <Filter + Name="Resource Files" + Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx" + UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"> + <File + RelativePath=".\[[DIALOG_NAME]].rc"> + </File> + </Filter> + </Files> + <Globals> + </Globals> +</VisualStudioProject>''' + + +# Template for the .rc file with a couple of [[REPLACEABLE]] parts. +# TODO(joi) Improve this (and the resource.h template) to allow saving and then +# reopening of the RC file in Visual Studio. Currently you can only open it +# once and change it, then after you close it you won't be able to reopen it. +RC_TEMPLATE = '''\ +// Copyright (c) Google Inc. 2005 +// All rights reserved. +// This file is automatically generated by GRIT and intended for editing +// the layout of the dialogs contained in it. Do not edit anything but the +// dialogs. Any changes made to translateable portions of the dialogs will +// be ignored by GRIT. + +#include "resource.h" +#include <winres.h> +#include <winresrc.h> + +LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL + +#pragma code_page([[CODEPAGE_NUM]]) + +[[INCLUDES]] + +[[DIALOGS]] +''' + + +# Template for the resource.h file with a couple of [[REPLACEABLE]] parts. +HEADER_TEMPLATE = '''\ +// Copyright (c) Google Inc. 2005 +// All rights reserved. +// This file is automatically generated by GRIT. Do not edit. + +#pragma once + +// Edit commands +#define ID_EDIT_CLEAR 0xE120 +#define ID_EDIT_CLEAR_ALL 0xE121 +#define ID_EDIT_COPY 0xE122 +#define ID_EDIT_CUT 0xE123 +#define ID_EDIT_FIND 0xE124 +#define ID_EDIT_PASTE 0xE125 +#define ID_EDIT_PASTE_LINK 0xE126 +#define ID_EDIT_PASTE_SPECIAL 0xE127 +#define ID_EDIT_REPEAT 0xE128 +#define ID_EDIT_REPLACE 0xE129 +#define ID_EDIT_SELECT_ALL 0xE12A +#define ID_EDIT_UNDO 0xE12B +#define ID_EDIT_REDO 0xE12C + + +[[DEFINES]] +''' + + +class ResizeDialog(interface.Tool): + '''Generates an RC file, header and Visual Studio project that you can use +with Visual Studio's GUI resource editor to modify the layout of dialogs for +the language of your choice. You then use the RC file, after you resize the +dialog, for the language or languages of your choice, using the <skeleton> child +of the <structure> node for the dialog. The translateable bits of the dialog +will be ignored when you use the <skeleton> node (GRIT will instead use the +translateable bits from the original dialog) but the layout changes you make +will be used. Note that your layout changes must preserve the order of the +translateable elements in the RC file. + +Usage: grit resize [-f BASEFOLDER] [-l LANG] [-e RCENCODING] DIALOGID* + +Arguments: + DIALOGID The 'name' attribute of a dialog to output for resizing. Zero + or more of these parameters can be used. If none are + specified, all dialogs from the input .grd file are output. + +Options: + + -f BASEFOLDER The project will be created in a subfolder of BASEFOLDER. + The name of the subfolder will be the first DIALOGID you + specify. Defaults to '.' + + -l LANG Specifies that the RC file should contain a dialog translated + into the language LANG. The default is a cp1252-representable + pseudotranslation, because Visual Studio's GUI RC editor only + supports single-byte encodings. + + -c CODEPAGE Code page number to indicate to the RC compiler the encoding + of the RC file, default is something reasonable for the + language you selected (but this does not work for every single + language). See details on codepages below. NOTE that you do + not need to specify the codepage unless the tool complains + that it's not sure which codepage to use. See the following + page for codepage numbers supported by Windows: + http://www.microsoft.com/globaldev/reference/wincp.mspx + + -D NAME[=VAL] Specify a C-preprocessor-like define NAME with optional + value VAL (defaults to 1) which will be used to control + conditional inclusion of resources. + + +IMPORTANT NOTE: For now, the tool outputs a UTF-8 encoded file for any language +that can not be represented in cp1252 (i.e. anything other than Western +European languages). You will need to open this file in a text editor and +save it using the codepage indicated in the #pragma code_page(XXXX) command +near the top of the file, before you open it in Visual Studio. + +''' + + # TODO(joi) It would be cool to have this tool note the Perforce revision + # of the original RC file somewhere, such that the <skeleton> node could warn + # if the original RC file gets updated without the skeleton file being updated. + + # TODO(joi) Would be cool to have option to add the files to Perforce + + def __init__(self): + self.lang = pseudo.PSEUDO_LANG + self.defines = {} + self.base_folder = '.' + self.codepage_number = 1252 + self.codepage_number_specified_explicitly = False + + def SetLanguage(self, lang): + '''Sets the language code to output things in. + ''' + self.lang = lang + if not self.codepage_number_specified_explicitly: + self.codepage_number = util.LanguageToCodepage(lang) + + def GetEncoding(self): + if self.codepage_number == 1200: + return 'utf_16' + if self.codepage_number == 65001: + return 'utf_8' + return 'cp%d' % self.codepage_number + + def ShortDescription(self): + return 'Generate a file where you can resize a given dialog.' + + def Run(self, opts, args): + self.SetOptions(opts) + + own_opts, args = getopt.getopt(args, 'l:f:c:D:') + for key, val in own_opts: + if key == '-l': + self.SetLanguage(val) + if key == '-f': + self.base_folder = val + if key == '-c': + self.codepage_number = int(val) + self.codepage_number_specified_explicitly = True + if key == '-D': + name, val = build.ParseDefine(val) + self.defines[name] = val + + res_tree = grd_reader.Parse(opts.input, debug=opts.extra_verbose) + res_tree.OnlyTheseTranslations([self.lang]) + res_tree.RunGatherers(True) + + # Dialog IDs are either explicitly listed, or we output all dialogs from the + # .grd file + dialog_ids = args + if not len(dialog_ids): + for node in res_tree: + if node.name == 'structure' and node.attrs['type'] == 'dialog': + dialog_ids.append(node.attrs['name']) + + self.Process(res_tree, dialog_ids) + + def Process(self, grd, dialog_ids): + '''Outputs an RC file and header file for the dialog 'dialog_id' stored in + resource tree 'grd', to self.base_folder, as discussed in this class's + documentation. + + Arguments: + grd: grd = grd_reader.Parse(...); grd.RunGatherers() + dialog_ids: ['IDD_MYDIALOG', 'IDD_OTHERDIALOG'] + ''' + grd.SetOutputContext(self.lang, self.defines) + + project_name = dialog_ids[0] + + dir_path = os.path.join(self.base_folder, project_name) + if not os.path.isdir(dir_path): + os.mkdir(dir_path) + + # If this fails then we're not on Windows (or you don't have the required + # win32all Python libraries installed), so what are you doing mucking + # about with RC files anyway? :) + import pythoncom + + # Create the .vcproj file + project_text = PROJECT_TEMPLATE.replace( + '[[PROJECT_GUID]]', str(pythoncom.CreateGuid()) + ).replace('[[DIALOG_NAME]]', project_name) + fname = os.path.join(dir_path, '%s.vcproj' % project_name) + self.WriteFile(fname, project_text) + print "Wrote %s" % fname + + # Create the .rc file + # Output all <include> nodes since the dialogs might depend on them (e.g. + # for icons and bitmaps). + include_items = [] + for node in grd: + if isinstance(node, include.IncludeNode): + formatter = node.ItemFormatter('rc_all') + if formatter: + include_items.append(formatter.Format(node, self.lang)) + rc_text = RC_TEMPLATE.replace('[[CODEPAGE_NUM]]', + str(self.codepage_number)) + rc_text = rc_text.replace('[[INCLUDES]]', ''.join(include_items)) + + # Then output the dialogs we have been asked to output. + dialogs = [] + for dialog_id in dialog_ids: + node = grd.GetNodeById(dialog_id) + # TODO(joi) Add exception handling for better error reporting + formatter = node.ItemFormatter('rc_all') + dialogs.append(formatter.Format(node, self.lang)) + rc_text = rc_text.replace('[[DIALOGS]]', ''.join(dialogs)) + + fname = os.path.join(dir_path, '%s.rc' % project_name) + self.WriteFile(fname, rc_text, self.GetEncoding()) + print "Wrote %s" % fname + + # Create the resource.h file + header_defines = [] + for node in grd: + formatter = node.ItemFormatter('rc_header') + if formatter and not isinstance(formatter, rc_header.TopLevel): + header_defines.append(formatter.Format(node, self.lang)) + header_text = HEADER_TEMPLATE.replace('[[DEFINES]]', ''.join(header_defines)) + fname = os.path.join(dir_path, 'resource.h') + self.WriteFile(fname, header_text) + print "Wrote %s" % fname + + def WriteFile(self, filename, contents, encoding='cp1252'): + f = util.WrapOutputStream(file(filename, 'wb'), encoding) + f.write(contents) + f.close()
\ No newline at end of file diff --git a/tools/grit/grit/tool/test.py b/tools/grit/grit/tool/test.py new file mode 100644 index 0000000..35ffca5 --- /dev/null +++ b/tools/grit/grit/tool/test.py @@ -0,0 +1,48 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from grit.tool import interface + +class TestTool(interface.Tool): + '''This tool does nothing except print out the global options and +tool-specific arguments that it receives. It is intended only for testing, +hence the name :) +''' + + def ShortDescription(self): + return 'A do-nothing tool for testing command-line parsing.' + + def Run(self, global_options, my_arguments): + print 'NOTE This tool is only for testing the parsing of global options and' + print 'tool-specific arguments that it receives. You may have intended to' + print 'run "grit unit" which is the unit-test suite for GRIT.' + print 'Options: %s' % repr(global_options) + print 'Arguments: %s' % repr(my_arguments) + return 0 diff --git a/tools/grit/grit/tool/toolbar_postprocess.py b/tools/grit/grit/tool/toolbar_postprocess.py new file mode 100644 index 0000000..d79eb2f --- /dev/null +++ b/tools/grit/grit/tool/toolbar_postprocess.py @@ -0,0 +1,150 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' Toolbar postprocessing class. Modifies the previously processed GRD tree +by creating separate message groups for each of the IDS_COMMAND macros. +Also adds some identifiers nodes to declare specific ids to be included +in the generated grh file. +''' + +import sys +import re +import postprocess_interface +import grit.node.empty +from grit.node import misc + +class ToolbarPostProcessor(postprocess_interface.PostProcessor): + ''' Defines message groups within the grd file for each of the + IDS_COMMAND stuff. + ''' + + _IDS_COMMAND = re.compile(r'IDS_COMMAND_') + _GRAB_PARAMETERS = re.compile(r'(IDS_COMMAND_[a-zA-Z0-9]+)_([a-zA-z0-9]+)') + + def Process(self, rctext, rcpath, grdnode): + ''' Processes the data in rctext and grdnode. + Args: + rctext: string containing the contents of the RC file being processed. + rcpath: the path used to access the file. + grdnode: the root node of the grd xml data generated by + the rc2grd tool. + + Return: + The root node of the processed GRD tree. + ''' + + release = grdnode.children[2] + messages = release.children[2] + + identifiers = grit.node.empty.IdentifiersNode() + identifiers.StartParsing('identifiers', release) + identifiers.EndParsing() + release.AddChild(identifiers) + + + # + # Turn the IDS_COMMAND messages into separate message groups + # with ids that are offsetted to the message group's first id + # + previous_name_attr = '' + previous_prefix = '' + previous_node = '' + new_messages_node = self.ConstructNewMessages(release) + for node in messages.children[:]: + name_attr = node.attrs['name'] + if self._IDS_COMMAND.search(name_attr): + mo = self._GRAB_PARAMETERS.search(name_attr) + mp = self._GRAB_PARAMETERS.search(previous_name_attr) + if mo and mp: + prefix = mo.group(1) + previous_prefix = mp.group(1) + new_message_id = mp.group(2) + if prefix == previous_prefix: + messages.RemoveChild(previous_name_attr) + previous_node.attrs['offset'] = 'PCI_' + new_message_id + del previous_node.attrs['name'] + new_messages_node.AddChild(previous_node) + else: + messages.RemoveChild(previous_name_attr) + previous_node.attrs['offset'] = 'PCI_' + new_message_id + del previous_node.attrs['name'] + new_messages_node.AddChild(previous_node) + new_messages_node.attrs['first_id'] = previous_prefix + new_messages_node = self.ConstructNewMessages(release) + else: + if self._IDS_COMMAND.search(previous_name_attr): + messages.RemoveChild(previous_name_attr) + previous_prefix = mp.group(1) + new_message_id = mp.group(2) + previous_node.attrs['offset'] = 'PCI_' + new_message_id + del previous_node.attrs['name'] + new_messages_node.AddChild(previous_node) + new_messages_node.attrs['first_id'] = previous_prefix + new_messages_node = self.ConstructNewMessages(release) + else: + if self._IDS_COMMAND.search(previous_name_attr): + messages.RemoveChild(previous_name_attr) + mp = self._GRAB_PARAMETERS.search(previous_name_attr) + previous_prefix = mp.group(1) + new_message_id = mp.group(2) + previous_node.attrs['offset'] = 'PCI_' + new_message_id + del previous_node.attrs['name'] + new_messages_node.AddChild(previous_node) + new_messages_node.attrs['first_id'] = previous_prefix + new_messages_node = self.ConstructNewMessages(release) + previous_name_attr = name_attr + previous_node = node + + + self.AddIdentifiers(rctext, identifiers) + return grdnode + + def ConstructNewMessages(self, parent): + new_node = grit.node.empty.MessagesNode() + new_node.StartParsing('messages', parent) + new_node.EndParsing() + parent.AddChild(new_node) + return new_node + + def AddIdentifiers(self, rctext, node): + node.AddChild(misc.IdentifierNode.Construct(node, 'IDS_COMMAND_gcFirst', '12000', '')) + node.AddChild(misc.IdentifierNode.Construct(node, + 'IDS_COMMAND_PCI_SPACE', '16', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_BUTTON', '0', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_MENU', '1', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP', '2', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_OPTIONS_TEXT', '3', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_DISABLED', '4', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_MENU', '5', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_MENU_DISABLED', '6', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_OPTIONS', '7', '')) + node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_OPTIONS_DISABLED', '8', '')) + node.AddChild(misc.IdentifierNode.Construct(node, + 'PCI_TIP_DISABLED_BY_POLICY', '9', '')) diff --git a/tools/grit/grit/tool/toolbar_preprocess.py b/tools/grit/grit/tool/toolbar_preprocess.py new file mode 100644 index 0000000..6523ee8 --- /dev/null +++ b/tools/grit/grit/tool/toolbar_preprocess.py @@ -0,0 +1,86 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' Toolbar preprocessing code. Turns all IDS_COMMAND macros in the RC file +into simpler constructs that can be understood by GRIT. Also deals with +expansion of $lf; placeholders into the correct linefeed character. +''' + +import preprocess_interface + +import re +import sys +import codecs + +class ToolbarPreProcessor(preprocess_interface.PreProcessor): + ''' Toolbar PreProcessing class. + ''' + + _IDS_COMMAND_MACRO = re.compile(r'(.*IDS_COMMAND)\s*\(([a-zA-Z0-9_]*)\s*,\s*([a-zA-Z0-9_]*)\)(.*)') + _LINE_FEED_PH = re.compile(r'\$lf;') + _PH_COMMENT = re.compile(r'PHRWR') + _COMMENT = re.compile(r'^(\s*)//.*') + + + def Process(self, rctext, rcpath): + ''' Processes the data in rctext. + Args: + rctext: string containing the contents of the RC file being processed + rcpath: the path used to access the file. + + Return: + The processed text. + ''' + + ret = '' + rclines = rctext.splitlines() + for line in rclines: + + if self._LINE_FEED_PH.search(line): + # Replace "$lf;" placeholder comments by an empty line. + # this will not be put into the processed result + if self._PH_COMMENT.search(line): + mm = self._COMMENT.search(line) + if mm: + line = '%s//' % mm.group(1) + + else: + # Replace $lf by the right linefeed character + line = self._LINE_FEED_PH.sub(r'\\n', line) + + # Deal with IDS_COMMAND_MACRO stuff + mo = self._IDS_COMMAND_MACRO.search(line) + if mo: + line = '%s_%s_%s%s' % (mo.group(1), mo.group(2), mo.group(3), mo.group(4)) + + ret += (line + '\n') + + return ret + diff --git a/tools/grit/grit/tool/transl2tc.py b/tools/grit/grit/tool/transl2tc.py new file mode 100644 index 0000000..69fe2e5 --- /dev/null +++ b/tools/grit/grit/tool/transl2tc.py @@ -0,0 +1,278 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''The 'grit transl2tc' tool. +''' + + +import getopt + +from grit.tool import interface +from grit.tool import rc2grd +from grit import grd_reader +from grit import util + +from grit.extern import tclib + + +class TranslationToTc(interface.Tool): + '''A tool for importing existing translations in RC format into the +Translation Console. + +Usage: + +grit -i GRD transl2tc [-l LIMITS] [RCOPTS] SOURCE_RC TRANSLATED_RC OUT_FILE + +The tool needs a "source" RC file, i.e. in English, and an RC file that is a +translation of precisely the source RC file (not of an older or newer version). + +The tool also requires you to provide a .grd file (input file) e.g. using the +-i global option or the GRIT_INPUT environment variable. The tool uses +information from your .grd file to correct placeholder names in the +translations and ensure that only translatable items and translations still +being used are output. + +This tool will accept all the same RCOPTS as the 'grit rc2grd' tool. To get +a list of these options, run 'grit help rc2grd'. + +Additionally, you can use the -l option (which must be the first option to the +tool) to specify a file containing a list of message IDs to which output should +be limited. This is only useful if you are limiting the output to your XMB +files using the 'grit xmb' tool's -l option. See 'grit help xmb' for how to +generate a file containing a list of the message IDs in an XMB file. + +The tool will scan through both of the RC files as well as any HTML files they +refer to, and match together the source messages and translated messages. It +will output a file (OUTPUT_FILE) you can import directly into the TC using the +Bulk Translation Upload tool. +''' + + def ShortDescription(self): + return 'Import existing translations in RC format into the TC' + + def Setup(self, globopt, args): + '''Sets the instance up for use. + ''' + self.SetOptions(globopt) + self.rc2grd = rc2grd.Rc2Grd() + self.rc2grd.SetOptions(globopt) + self.limits = None + if len(args) and args[0] == '-l': + limit_file = file(args[1]) + self.limits = limit_file.read().split('\n') + limit_file.close() + args = args[2:] + return self.rc2grd.ParseOptions(args) + + def Run(self, globopt, args): + args = self.Setup(globopt, args) + + if len(args) != 3: + self.Out('This tool takes exactly three arguments:\n' + ' 1. The path to the original RC file\n' + ' 2. The path to the translated RC file\n' + ' 3. The output file path.\n') + return 2 + + grd = grd_reader.Parse(self.o.input, debug=self.o.extra_verbose) + grd.RunGatherers(recursive = True) + + source_rc = util.WrapInputStream(file(args[0], 'r'), self.rc2grd.input_encoding) + transl_rc = util.WrapInputStream(file(args[1], 'r'), self.rc2grd.input_encoding) + translations = self.ExtractTranslations(grd, + source_rc.read(), args[0], + transl_rc.read(), args[1]) + transl_rc.close() + source_rc.close() + + output_file = util.WrapOutputStream(file(args[2], 'w')) + self.WriteTranslations(output_file, translations.items()) + output_file.close() + + self.Out('Wrote output file %s' % args[2]) + + def ExtractTranslations(self, current_grd, source_rc, source_path, transl_rc, transl_path): + '''Extracts translations from the translated RC file, matching them with + translations in the source RC file to calculate their ID, and correcting + placeholders, limiting output to translateables, etc. using the supplied + .grd file which is the current .grd file for your project. + + If this object's 'limits' attribute is not None but a list, the output of + this function will be further limited to include only messages that have + message IDs in the 'limits' list. + + Args: + current_grd: grit.node.base.Node child, that has had RunGatherers(True) run on it + source_rc: Complete text of source RC file + source_path: Path to the source RC file + transl_rc: Complete text of translated RC file + transl_path: Path to the translated RC file + + Return: + { id1 : text1, '12345678' : 'Hello USERNAME, howzit?' } + ''' + source_grd = self.rc2grd.Process(source_rc, source_path) + self.VerboseOut('Read %s into GRIT format, running gatherers.\n' % source_path) + source_grd.RunGatherers(recursive=True, debug=self.o.extra_verbose) + transl_grd = self.rc2grd.Process(transl_rc, transl_path) + self.VerboseOut('Read %s into GRIT format, running gatherers.\n' % transl_path) + transl_grd.RunGatherers(recursive=True, debug=self.o.extra_verbose) + self.VerboseOut('Done running gatherers for %s.\n' % transl_path) + + # Proceed to create a map from ID to translation, getting the ID from the + # source GRD and the translation from the translated GRD. + id2transl = {} + for source_node in source_grd: + source_cliques = source_node.GetCliques() + if not len(source_cliques): + continue + + assert 'name' in source_node.attrs, 'All nodes with cliques should have an ID' + node_id = source_node.attrs['name'] + self.ExtraVerboseOut('Processing node %s\n' % node_id) + transl_node = transl_grd.GetNodeById(node_id) + + if transl_node: + transl_cliques = transl_node.GetCliques() + if not len(transl_cliques) == len(source_cliques): + self.Out( + 'Warning: Translation for %s has wrong # of cliques, skipping.\n' % + node_id) + continue + else: + self.Out('Warning: No translation for %s, skipping.\n' % node_id) + continue + + if source_node.name == 'message': + # Fixup placeholders as well as possible based on information from + # the current .grd file if they are 'TODO_XXXX' placeholders. We need + # to fixup placeholders in the translated message so that it looks right + # and we also need to fixup placeholders in the source message so that + # its calculated ID will match the current message. + current_node = current_grd.GetNodeById(node_id) + if current_node: + assert len(source_cliques) == 1 and len(current_node.GetCliques()) == 1 + + source_msg = source_cliques[0].GetMessage() + current_msg = current_node.GetCliques()[0].GetMessage() + + # Only do this for messages whose source version has not changed. + if (source_msg.GetRealContent() != current_msg.GetRealContent()): + self.VerboseOut('Info: Message %s has changed; skipping\n' % node_id) + else: + transl_msg = transl_cliques[0].GetMessage() + transl_content = transl_msg.GetContent() + current_content = current_msg.GetContent() + source_content = source_msg.GetContent() + + ok_to_fixup = True + if (len(transl_content) != len(current_content)): + # message structure of translation is different, don't try fixup + ok_to_fixup = False + if ok_to_fixup: + for ix in range(len(transl_content)): + if isinstance(transl_content[ix], tclib.Placeholder): + if not isinstance(current_content[ix], tclib.Placeholder): + ok_to_fixup = False # structure has changed + break + if (transl_content[ix].GetOriginal() != + current_content[ix].GetOriginal()): + ok_to_fixup = False # placeholders have likely been reordered + break + else: # translated part is not a placeholder but a string + if isinstance(current_content[ix], tclib.Placeholder): + ok_to_fixup = False # placeholders have likely been reordered + break + + if not ok_to_fixup: + self.VerboseOut( + 'Info: Structure of message %s has changed; skipping.\n' % node_id) + else: + def Fixup(content, ix): + if (isinstance(content[ix], tclib.Placeholder) and + content[ix].GetPresentation().startswith('TODO_')): + assert isinstance(current_content[ix], tclib.Placeholder) + # Get the placeholder ID and example from the current message + content[ix] = current_content[ix] + for ix in range(len(transl_content)): + Fixup(transl_content, ix) + Fixup(source_content, ix) + + # Only put each translation once into the map. Warn if translations + # for the same message are different. + for ix in range(len(transl_cliques)): + source_msg = source_cliques[ix].GetMessage() + source_msg.GenerateId() # needed to refresh ID based on new placeholders + message_id = source_msg.GetId() + translated_content = transl_cliques[ix].GetMessage().GetPresentableContent() + + if message_id in id2transl: + existing_translation = id2transl[message_id] + if existing_translation != translated_content: + original_text = source_cliques[ix].GetMessage().GetPresentableContent() + self.Out('Warning: Two different translations for "%s":\n' + ' Translation 1: "%s"\n' + ' Translation 2: "%s"\n' % + (original_text, existing_translation, translated_content)) + else: + id2transl[message_id] = translated_content + + # Remove translations for messages that do not occur in the current .grd + # or have been marked as not translateable, or do not occur in the 'limits' + # list (if it has been set). + current_message_ids = current_grd.UberClique().AllMessageIds() + for message_id in id2transl.keys(): + if (message_id not in current_message_ids or + not current_grd.UberClique().BestClique(message_id).IsTranslateable() or + (self.limits and message_id not in self.limits)): + del id2transl[message_id] + + return id2transl + + # static method + def WriteTranslations(output_file, translations): + '''Writes the provided list of translations to the provided output file + in the format used by the TC's Bulk Translation Upload tool. The file + must be UTF-8 encoded. + + Args: + output_file: util.WrapOutputStream(file('bingo.out', 'w')) + translations: [ [id1, text1], ['12345678', 'Hello USERNAME, howzit?'] ] + + Return: + None + ''' + for id, text in translations: + text = text.replace('<', '<').replace('>', '>') + output_file.write(id) + output_file.write(' ') + output_file.write(text) + output_file.write('\n') + WriteTranslations = staticmethod(WriteTranslations) diff --git a/tools/grit/grit/tool/transl2tc_unittest.py b/tools/grit/grit/tool/transl2tc_unittest.py new file mode 100644 index 0000000..f0b97d5 --- /dev/null +++ b/tools/grit/grit/tool/transl2tc_unittest.py @@ -0,0 +1,155 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for the 'grit transl2tc' tool.''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..')) + +import StringIO +import unittest + +from grit.tool import transl2tc +from grit import grd_reader +from grit import util + + +def MakeOptions(): + from grit import grit_runner + return grit_runner.Options() + + +class TranslationToTcUnittest(unittest.TestCase): + + def testOutput(self): + buf = StringIO.StringIO() + tool = transl2tc.TranslationToTc() + translations = [ + ['1', 'Hello USERNAME, how are you?'], + ['12', 'Howdie doodie!'], + ['123', 'Hello\n\nthere\n\nhow are you?'], + ['1234', 'Hello is > goodbye but < howdie pardner'], + ] + tool.WriteTranslations(buf, translations) + output = buf.getvalue() + self.failUnless(output.strip() == ''' +1 Hello USERNAME, how are you? +12 Howdie doodie! +123 Hello + +there + +how are you? +1234 Hello is > goodbye but < howdie pardner +'''.strip()) + + def testExtractTranslations(self): + path = util.PathFromRoot('grit/test/data') + current_grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir="."> + <release seq="3"> + <messages> + <message name="IDS_SIMPLE"> + One + </message> + <message name="IDS_PLACEHOLDER"> + <ph name="NUMBIRDS">%s<ex>3</ex></ph> birds + </message> + <message name="IDS_PLACEHOLDERS"> + <ph name="ITEM">%d<ex>1</ex></ph> of <ph name="COUNT">%d<ex>3</ex></ph> + </message> + <message name="IDS_REORDERED_PLACEHOLDERS"> + <ph name="ITEM">$1<ex>1</ex></ph> of <ph name="COUNT">$2<ex>3</ex></ph> + </message> + <message name="IDS_CHANGED"> + This is the new version + </message> + <message name="IDS_TWIN_1">Hello</message> + <message name="IDS_TWIN_2">Hello</message> + <message name="IDS_NOT_TRANSLATEABLE" translateable="false">:</message> + <message name="IDS_LONGER_TRANSLATED"> + Removed document <ph name="FILENAME">$1<ex>c:\temp</ex></ph> + </message> + <message name="IDS_DIFFERENT_TWIN_1">Howdie</message> + <message name="IDS_DIFFERENT_TWIN_2">Howdie</message> + </messages> + <structures> + <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="klonk.rc" /> + <structure type="menu" name="IDC_KLONKMENU" encoding="utf-16" file="klonk.rc" /> + </structures> + </release> + </grit>'''), path) + current_grd.RunGatherers(recursive=True) + + source_rc_path = util.PathFromRoot('grit/test/data/source.rc') + source_rc = file(source_rc_path).read() + transl_rc_path = util.PathFromRoot('grit/test/data/transl.rc') + transl_rc = file(transl_rc_path).read() + + tool = transl2tc.TranslationToTc() + output_buf = StringIO.StringIO() + globopts = MakeOptions() + globopts.verbose = True + globopts.output_stream = output_buf + tool.Setup(globopts, []) + translations = tool.ExtractTranslations(current_grd, + source_rc, source_rc_path, + transl_rc, transl_rc_path) + + values = translations.values() + output = output_buf.getvalue() + + self.failUnless('Ein' in values) + self.failUnless('NUMBIRDS Vogeln' in values) + self.failUnless('ITEM von COUNT' in values) + self.failUnless(values.count('Hallo') == 1) + self.failIf('Dass war die alte Version' in values) + self.failIf(':' in values) + self.failIf('Dokument FILENAME ist entfernt worden' in values) + self.failIf('Nicht verwendet' in values) + self.failUnless(('Howdie' in values or 'Hallo sagt man' in values) and not + ('Howdie' in values and 'Hallo sagt man' in values)) + + self.failUnless('XX01XX&SkraXX02XX&HaettaXX03XXThetta er "Klonk" sem eg fylaXX04XXgonkurinnXX05XXKlonk && er "gott"XX06XX&HjalpXX07XX&Um...XX08XX' in values) + + self.failUnless('I lagi' in values) + + self.failUnless(output.count('Structure of message IDS_REORDERED_PLACEHOLDERS has changed')) + self.failUnless(output.count('Message IDS_CHANGED has changed')) + self.failUnless(output.count('Structure of message IDS_LONGER_TRANSLATED has changed')) + self.failUnless(output.count('Two different translations for "Howdie"')) + self.failUnless(output.count('IDD_DIFFERENT_LENGTH_IN_TRANSL has wrong # of cliques')) + + +if __name__ == '__main__': + unittest.main()
\ No newline at end of file diff --git a/tools/grit/grit/tool/unit.py b/tools/grit/grit/tool/unit.py new file mode 100644 index 0000000..b021293 --- /dev/null +++ b/tools/grit/grit/tool/unit.py @@ -0,0 +1,50 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''GRIT tool that runs the unit test suite for GRIT.''' + + +import unittest + +import grit.test_suite_all +from grit.tool import interface + + +class UnitTestTool(interface.Tool): + '''By using this tool (e.g. 'grit unit') you run all the unit tests for GRIT. +This happens in the environment that is set up by the basic GRIT runner, i.e. +whether to run disconnected has been specified, etc.''' + + def ShortDescription(self): + return 'Use this tool to run all the unit tests for GRIT.' + + def Run(self, opts, args): + return unittest.TextTestRunner(verbosity=2).run( + grit.test_suite_all.TestSuiteAll()) diff --git a/tools/grit/grit/util.py b/tools/grit/grit/util.py new file mode 100644 index 0000000..08b1cb1 --- /dev/null +++ b/tools/grit/grit/util.py @@ -0,0 +1,334 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Utilities used by GRIT. +''' + +import sys +import os.path +import codecs +import htmlentitydefs +import re +import time +from xml.sax import saxutils + +_root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) + + +# Matches all of the resource IDs predefined by Windows. +# The '\b' before and after each word makes sure these match only whole words and +# not the beginning of any word.. eg. ID_FILE_NEW will not match ID_FILE_NEW_PROJECT +# see http://www.amk.ca/python/howto/regex/ (search for "\bclass\b" inside the html page) +SYSTEM_IDENTIFIERS = re.compile( + r'''\bIDOK\b | \bIDCANCEL\b | \bIDC_STATIC\b | \bIDYES\b | \bIDNO\b | + \bID_FILE_NEW\b | \bID_FILE_OPEN\b | \bID_FILE_CLOSE\b | \bID_FILE_SAVE\b | + \bID_FILE_SAVE_AS\b | \bID_FILE_PAGE_SETUP\b | \bID_FILE_PRINT_SETUP\b | + \bID_FILE_PRINT\b | \bID_FILE_PRINT_DIRECT\b | \bID_FILE_PRINT_PREVIEW\b | + \bID_FILE_UPDATE\b | \bID_FILE_SAVE_COPY_AS\b | \bID_FILE_SEND_MAIL\b | + \bID_FILE_MRU_FIRST\b | \bID_FILE_MRU_LAST\b | + \bID_EDIT_CLEAR\b | \bID_EDIT_CLEAR_ALL\b | \bID_EDIT_COPY\b | + \bID_EDIT_CUT\b | \bID_EDIT_FIND\b | \bID_EDIT_PASTE\b | \bID_EDIT_PASTE_LINK\b | + \bID_EDIT_PASTE_SPECIAL\b | \bID_EDIT_REPEAT\b | \bID_EDIT_REPLACE\b | + \bID_EDIT_SELECT_ALL\b | \bID_EDIT_UNDO\b | \bID_EDIT_REDO\b | + \bVS_VERSION_INFO\b | \bIDRETRY''', re.VERBOSE); + + +# Matches character entities, whether specified by name, decimal or hex. +_HTML_ENTITY = re.compile( + '&(#(?P<decimal>[0-9]+)|#x(?P<hex>[a-fA-F0-9]+)|(?P<named>[a-z0-9]+));', + re.IGNORECASE) + +# Matches characters that should be HTML-escaped. This is <, > and &, but only +# if the & is not the start of an HTML character entity. +_HTML_CHARS_TO_ESCAPE = re.compile('"|<|>|&(?!#[0-9]+|#x[0-9a-z]+|[a-z]+;)', + re.IGNORECASE | re.MULTILINE) + + +def WrapInputStream(stream, encoding = 'utf-8'): + '''Returns a stream that wraps the provided stream, making it read characters + using the specified encoding.''' + (e, d, sr, sw) = codecs.lookup(encoding) + return sr(stream) + + +def WrapOutputStream(stream, encoding = 'utf-8'): + '''Returns a stream that wraps the provided stream, making it write + characters using the specified encoding.''' + (e, d, sr, sw) = codecs.lookup(encoding) + return sw(stream) + + +def ChangeStdoutEncoding(encoding = 'utf-8'): + '''Changes STDOUT to print characters using the specified encoding.''' + sys.stdout = WrapOutputStream(sys.stdout, encoding) + + +def EscapeHtml(text, escape_quotes = False): + '''Returns 'text' with <, > and & (and optionally ") escaped to named HTML + entities. Any existing named entity or HTML entity defined by decimal or + hex code will be left untouched. This is appropriate for escaping text for + inclusion in HTML, but not for XML. + ''' + def Replace(match): + if match.group() == '&': return '&' + elif match.group() == '<': return '<' + elif match.group() == '>': return '>' + elif match.group() == '"': + if escape_quotes: return '"' + else: return match.group() + else: assert False + out = _HTML_CHARS_TO_ESCAPE.sub(Replace, text) + return out + + +def UnescapeHtml(text, replace_nbsp=True): + '''Returns 'text' with all HTML character entities (both named character + entities and those specified by decimal or hexadecimal Unicode ordinal) + replaced by their Unicode characters (or latin1 characters if possible). + + The only exception is that will not be escaped if 'replace_nbsp' is + False. + ''' + def Replace(match): + groups = match.groupdict() + if groups['hex']: + return unichr(int(groups['hex'], 16)) + elif groups['decimal']: + return unichr(int(groups['decimal'], 10)) + else: + name = groups['named'] + if name == 'nbsp' and not replace_nbsp: + return match.group() # Don't replace + assert name != None + if name in htmlentitydefs.name2codepoint.keys(): + return unichr(htmlentitydefs.name2codepoint[name]) + else: + return match.group() # Unknown HTML character entity - don't replace + + out = _HTML_ENTITY.sub(Replace, text) + return out + + +def EncodeCdata(cdata): + '''Returns the provided cdata in either escaped format or <![CDATA[xxx]]> + format, depending on which is more appropriate for easy editing. The data + is escaped for inclusion in an XML element's body. + + Args: + cdata: 'If x < y and y < z then x < z' + + Return: + '<![CDATA[If x < y and y < z then x < z]]>' + ''' + if cdata.count('<') > 1 or cdata.count('>') > 1 and cdata.count(']]>') == 0: + return '<![CDATA[%s]]>' % cdata + else: + return saxutils.escape(cdata) + + +def FixupNamedParam(function, param_name, param_value): + '''Returns a closure that is identical to 'function' but ensures that the + named parameter 'param_name' is always set to 'param_value' unless explicitly + set by the caller. + + Args: + function: callable + param_name: 'bingo' + param_value: 'bongo' (any type) + + Return: + callable + ''' + def FixupClosure(*args, **kw): + if not param_name in kw: + kw[param_name] = param_value + return function(*args, **kw) + return FixupClosure + + +def PathFromRoot(path): + '''Takes a path relative to the root directory for GRIT (the one that grit.py + resides in) and returns a path that is either absolute or relative to the + current working directory (i.e .a path you can use to open the file). + + Args: + path: 'rel_dir\file.ext' + + Return: + 'c:\src\tools\rel_dir\file.ext + ''' + return os.path.normpath(os.path.join(_root_dir, path)) + + +def FixRootForUnittest(root_node, dir=PathFromRoot('.')): + '''Adds a GetBaseDir() method to 'root_node', making unittesting easier.''' + def GetBaseDir(): + '''Returns a fake base directory.''' + return dir + def GetSourceLanguage(): + return 'en' + if not hasattr(root_node, 'GetBaseDir'): + setattr(root_node, 'GetBaseDir', GetBaseDir) + setattr(root_node, 'GetSourceLanguage', GetSourceLanguage) + + +def dirname(filename): + '''Version of os.path.dirname() that never returns empty paths (returns + '.' if the result of os.path.dirname() is empty). + ''' + ret = os.path.dirname(filename) + if ret == '': + ret = '.' + return ret + + +def normpath(path): + '''Version of os.path.normpath that also changes backward slashes to + forward slashes when not running on Windows. + ''' + # This is safe to always do because the Windows version of os.path.normpath + # will replace forward slashes with backward slashes. + path = path.replace('\\', '/') + return os.path.normpath(path) + + +_LANGUAGE_SPLIT_RE = re.compile('-|_|/') + + +def CanonicalLanguage(code): + '''Canonicalizes two-part language codes by using a dash and making the + second part upper case. Returns one-part language codes unchanged. + + Args: + code: 'zh_cn' + + Return: + code: 'zh-CN' + ''' + parts = _LANGUAGE_SPLIT_RE.split(code) + code = [ parts[0] ] + for part in parts[1:]: + code.append(part.upper()) + return '-'.join(code) + + +_LANG_TO_CODEPAGE = { + 'en' : 1252, + 'fr' : 1252, + 'it' : 1252, + 'de' : 1252, + 'es' : 1252, + 'nl' : 1252, + 'sv' : 1252, + 'no' : 1252, + 'da' : 1252, + 'fi' : 1252, + 'pt-BR' : 1252, + 'ru' : 1251, + 'ja' : 932, + 'zh-TW' : 950, + 'zh-CN' : 936, + 'ko' : 949, +} + + +def LanguageToCodepage(lang): + '''Returns the codepage _number_ that can be used to represent 'lang', which + may be either in formats such as 'en', 'pt_br', 'pt-BR', etc. + + The codepage returned will be one of the 'cpXXXX' codepage numbers. + + Args: + lang: 'de' + + Return: + 1252 + ''' + lang = CanonicalLanguage(lang) + if lang in _LANG_TO_CODEPAGE: + return _LANG_TO_CODEPAGE[lang] + else: + print "Not sure which codepage to use for %s, assuming cp1252" % lang + return 1252 + +def NewClassInstance(class_name, class_type): + '''Returns an instance of the class specified in classname + + Args: + class_name: the fully qualified, dot separated package + classname, + i.e. "my.package.name.MyClass". Short class names are not supported. + class_type: the class or superclass this object must implement + + Return: + An instance of the class, or None if none was found + ''' + lastdot = class_name.rfind('.') + module_name = '' + if lastdot >= 0: + module_name = class_name[0:lastdot] + if module_name: + class_name = class_name[lastdot+1:] + module = __import__(module_name, globals(), locals(), ['']) + if hasattr(module, class_name): + class_ = getattr(module, class_name) + class_instance = class_() + if isinstance(class_instance, class_type): + return class_instance + return None + + +def FixLineEnd(text, line_end): + # First normalize + text = text.replace('\r\n', '\n') + text = text.replace('\r', '\n') + # Then fix + text = text.replace('\n', line_end) + return text + + +def BoolToString(bool): + if bool: + return 'true' + else: + return 'false' + + +verbose = False +extra_verbose = False + +def IsVerbose(): + return verbose + +def IsExtraVerbose(): + return extra_verbose + +def GetCurrentYear(): + '''Returns the current 4-digit year as an integer.''' + return time.localtime()[0] diff --git a/tools/grit/grit/util_unittest.py b/tools/grit/grit/util_unittest.py new file mode 100644 index 0000000..eb11067 --- /dev/null +++ b/tools/grit/grit/util_unittest.py @@ -0,0 +1,94 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit test that checks some of util functions. +''' + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import unittest + +from grit import util + + +class UtilUnittest(unittest.TestCase): + ''' Tests functions from util + ''' + + def testNewClassInstance(self): + # Test short class name with no fully qualified package name + # Should fail, it is not supported by the function now (as documented) + cls = util.NewClassInstance('grit.util.TestClassToLoad', + TestBaseClassToLoad) + self.failUnless(cls == None) + + # Test non existent class name + cls = util.NewClassInstance('grit.util_unittest.NotExistingClass', + TestBaseClassToLoad) + self.failUnless(cls == None) + + # Test valid class name and valid base class + cls = util.NewClassInstance('grit.util_unittest.TestClassToLoad', + TestBaseClassToLoad) + self.failUnless(isinstance(cls, TestBaseClassToLoad)) + + # Test valid class name with wrong hierarchy + cls = util.NewClassInstance('grit.util_unittest.TestClassNoBase', + TestBaseClassToLoad) + self.failUnless(cls == None) + + def testCanonicalLanguage(self): + self.failUnless(util.CanonicalLanguage('en') == 'en') + self.failUnless(util.CanonicalLanguage('pt_br') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('pt-br') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('pt-BR') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('pt/br') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('pt/BR') == 'pt-BR') + self.failUnless(util.CanonicalLanguage('no_no_bokmal') == 'no-NO-BOKMAL') + + def testUnescapeHtml(self): + self.failUnless(util.UnescapeHtml('ϲ') == unichr(1010)) + self.failUnless(util.UnescapeHtml('ꯍ') == unichr(43981)) + +class TestBaseClassToLoad(object): + pass + +class TestClassToLoad(TestBaseClassToLoad): + pass + +class TestClassNoBase(object): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/grit/grit/xtb_reader.py b/tools/grit/grit/xtb_reader.py new file mode 100644 index 0000000..01db8d9 --- /dev/null +++ b/tools/grit/grit/xtb_reader.py @@ -0,0 +1,123 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Fast and efficient parser for XTB files. +''' + + +import xml.sax +import xml.sax.handler + + +class XtbContentHandler(xml.sax.handler.ContentHandler): + '''A content handler that calls a given callback function for each + translation in the XTB file. + ''' + + def __init__(self, callback, debug=False): + self.callback = callback + self.debug = debug + # 0 if we are not currently parsing a translation, otherwise the message + # ID of that translation. + self.current_id = 0 + # Empty if we are not currently parsing a translation, otherwise the + # parts we have for that translation - a list of tuples + # (is_placeholder, text) + self.current_structure = [] + # Set to the language ID when we see the <translationbundle> node. + self.language = '' + + def startElement(self, name, attrs): + if name == 'translation': + assert (self.current_id == 0 and len(self.current_structure) == 0, + "Didn't expect a <translation> element here.") + self.current_id = attrs.getValue('id') + elif name == 'ph': + assert self.current_id != 0, "Didn't expect a <ph> element here." + self.current_structure.append((True, attrs.getValue('name'))) + elif name == 'translationbundle': + self.language = attrs.getValue('lang') + + def endElement(self, name): + if name == 'translation': + assert self.current_id != 0 + self.callback(self.current_id, self.current_structure) + self.current_id = 0 + self.current_structure = [] + + def characters(self, content): + if self.current_id != 0: + # We are inside a <translation> node so just add the characters to our + # structure. + # + # This naive way of handling characters is OK because in the XTB format, + # <ph> nodes are always empty (always <ph name="XXX"/>) and whitespace + # inside the <translation> node should be preserved. + self.current_structure.append((False, content)) + + +class XtbErrorHandler(xml.sax.handler.ErrorHandler): + def error(self, exception): + pass + + def fatalError(self, exception): + raise exception + + def warning(self, exception): + pass + + +def Parse(xtb_file, callback_function, debug=False): + '''Parse xtb_file, making a call to callback_function for every translation + in the XTB file. + + The callback function must have the signature as described below. The 'parts' + parameter is a list of tuples (is_placeholder, text). The 'text' part is + either the raw text (if is_placeholder is False) or the name of the placeholder + (if is_placeholder is True). + + Args: + xtb_file: file('fr.xtb') + callback_function: def Callback(msg_id, parts): pass + + Return: + The language of the XTB, e.g. 'fr' + ''' + # Start by advancing the file pointer past the DOCTYPE thing, as the TC + # uses a path to the DTD that only works in Unix. + # TODO(joi) Remove this ugly hack by getting the TC gang to change the + # XTB files somehow? + front_of_file = xtb_file.read(1024) + xtb_file.seek(front_of_file.find('<translationbundle')) + + handler = XtbContentHandler(callback=callback_function, debug=debug) + xml.sax.parse(xtb_file, handler) + assert handler.language != '' + return handler.language diff --git a/tools/grit/grit/xtb_reader_unittest.py b/tools/grit/grit/xtb_reader_unittest.py new file mode 100644 index 0000000..c7beb1d --- /dev/null +++ b/tools/grit/grit/xtb_reader_unittest.py @@ -0,0 +1,108 @@ +#!/usr/bin/python2.4 +# Copyright 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +'''Unit tests for grit.xtb_reader''' + + +import os +import sys +if __name__ == '__main__': + sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) + +import StringIO +import unittest + +from grit import xtb_reader +from grit import clique +from grit import grd_reader +from grit import tclib +from grit import util + + +class XtbReaderUnittest(unittest.TestCase): + def testParsing(self): + xtb_file = StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE translationbundle> + <translationbundle lang="fr"> + <translation id="5282608565720904145">Bingo.</translation> + <translation id="2955977306445326147">Bongo longo.</translation> + <translation id="238824332917605038">Hullo</translation> + <translation id="6629135689895381486"><ph name="PROBLEM_REPORT"/> peut <ph name="START_LINK"/>utilisation excessive de majuscules<ph name="END_LINK"/>.</translation> + <translation id="7729135689895381486">Hello +this is another line +and another + +and another after a blank line.</translation> + </translationbundle>''') + + messages = [] + def Callback(id, structure): + messages.append((id, structure)) + xtb_reader.Parse(xtb_file, Callback) + self.failUnless(len(messages[0][1]) == 1) + self.failUnless(messages[3][1][0]) # PROBLEM_REPORT placeholder + self.failUnless(messages[4][0] == '7729135689895381486') + self.failUnless(messages[4][1][7][1] == 'and another after a blank line.') + + def testParsingIntoMessages(self): + grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <messages> + <message name="ID_MEGA">Fantastic!</message> + <message name="ID_HELLO_USER">Hello <ph name="USERNAME">%s<ex>Joi</ex></ph></message> + </messages>'''), dir='.', flexible_root=True) + + clique_mega = grd.children[0].GetCliques()[0] + msg_mega = clique_mega.GetMessage() + clique_hello_user = grd.children[1].GetCliques()[0] + msg_hello_user = clique_hello_user.GetMessage() + + xtb_file = StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE translationbundle> + <translationbundle lang="is"> + <translation id="%s">Meirihattar!</translation> + <translation id="%s">Saelir <ph name="USERNAME"/></translation> + </translationbundle>''' % (msg_mega.GetId(), msg_hello_user.GetId())) + + xtb_reader.Parse(xtb_file, grd.UberClique().GenerateXtbParserCallback('is')) + self.failUnless(clique_mega.MessageForLanguage('is').GetRealContent() == + 'Meirihattar!') + self.failUnless(clique_hello_user.MessageForLanguage('is').GetRealContent() == + 'Saelir %s') + + def testParseLargeFile(self): + def Callback(id, structure): + pass + xtb = file(util.PathFromRoot('grit/test/data/fr.xtb')) + xtb_reader.Parse(xtb, Callback) + xtb.close() + + +if __name__ == '__main__': + unittest.main() |