Generate the lists of localization files instead of hardcoding them.

This CL modifies remoting/tools/localize.py script such that it can:
  - be invoked via 'pymod_do_main' filter.
  - generate a separate output file for each locale/language.
  - produce the list of output files for given set of parameters.

remoting.gyp now generates the lists of .pak and message.json files invoking the script via 'pymod_do_main' filter. 'pymod_do_main' is necessary to pass paths unaltered between GYP and python.

The script was also renamed to remoting/tools/build/remoting_localize.py to avoid potential name clashes since the script is globally visible now.

TBR is for the DEPS changes only.

TBR=cpu@chromium.org
BUG=155204

Review URL: https://chromiumcodereview.appspot.com/18868009

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@211227 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 154 insertions, 210 deletions

diff --git a/remoting/remoting.gyp b/remoting/remoting.gyp
index 97d9980..32f0e1f 100644
--- a/remoting/remoting.gyp
+++ b/remoting/remoting.gyp
@@ -20,7 +20,7 @@
     'remoting_multi_process%': '<(remoting_multi_process)',
     'remoting_rdp_session%': 1,
 
-    'localize_py_path': 'tools/localize.py',
+    'remoting_localize_path': 'tools/build/remoting_localize.py',
 
     # The |major|, |build| and |patch| versions are inherited from Chrome.
     # Since Chrome's |minor| version is always '0', we replace it with a
@@ -100,93 +100,18 @@
         'rdp_desktop_session_clsid': '<!(python tools/uuidgen.py 2)',
       }],
     ],
-    'remoting_languages': [
-      '-l', 'ar',
-      '-l', 'bg',
-      '-l', 'ca',
-      '-l', 'cs',
-      '-l', 'da',
-      '-l', 'de',
-      '-l', 'el',
-      '-l', 'en',
-      '-l', 'en_GB',
-      '-l', 'es',
-      '-l', 'es_419',
-      '-l', 'et',
-      '-l', 'fi',
-      '-l', 'fil',
-      '-l', 'fr',
-      '-l', 'he',
-      '-l', 'hi',
-      '-l', 'hr',
-      '-l', 'hu',
-      '-l', 'id',
-      '-l', 'it',
-      '-l', 'ja',
-      '-l', 'ko',
-      '-l', 'lt',
-      '-l', 'lv',
-      '-l', 'nb',
-      '-l', 'nl',
-      '-l', 'pl',
-      '-l', 'pt_BR',
-      '-l', 'pt_PT',
-      '-l', 'ro',
-      '-l', 'ru',
-      '-l', 'sk',
-      '-l', 'sl',
-      '-l', 'sr',
-      '-l', 'sv',
-      '-l', 'th',
-      '-l', 'tr',
-      '-l', 'uk',
-      '-l', 'vi',
-      '-l', 'zh_CN',
-      '-l', 'zh_TW',
+    'remoting_locales': [
+      'ar', 'bg', 'ca', 'cs', 'da', 'de', 'el', 'en', 'en-GB', 'es',
+      'es-419', 'et', 'fi', 'fil', 'fr', 'he', 'hi', 'hr', 'hu', 'id',
+      'it', 'ja', 'ko', 'lt', 'lv', 'nb', 'nl', 'pl', 'pt-BR', 'pt-PT',
+      'ro', 'ru', 'sk', 'sl', 'sr', 'sv', 'th', 'tr', 'uk', 'vi',
+      'zh-CN', 'zh-TW',
     ],
     'remoting_webapp_locale_files': [
-      '<(webapp_locale_dir)/ar/messages.json',
-      '<(webapp_locale_dir)/bg/messages.json',
-      '<(webapp_locale_dir)/ca/messages.json',
-      '<(webapp_locale_dir)/cs/messages.json',
-      '<(webapp_locale_dir)/da/messages.json',
-      '<(webapp_locale_dir)/de/messages.json',
-      '<(webapp_locale_dir)/el/messages.json',
-      '<(webapp_locale_dir)/en/messages.json',
-      '<(webapp_locale_dir)/en_GB/messages.json',
-      '<(webapp_locale_dir)/es/messages.json',
-      '<(webapp_locale_dir)/es_419/messages.json',
-      '<(webapp_locale_dir)/et/messages.json',
-      '<(webapp_locale_dir)/fi/messages.json',
-      '<(webapp_locale_dir)/fil/messages.json',
-      '<(webapp_locale_dir)/fr/messages.json',
-      '<(webapp_locale_dir)/he/messages.json',
-      '<(webapp_locale_dir)/hi/messages.json',
-      '<(webapp_locale_dir)/hr/messages.json',
-      '<(webapp_locale_dir)/hu/messages.json',
-      '<(webapp_locale_dir)/id/messages.json',
-      '<(webapp_locale_dir)/it/messages.json',
-      '<(webapp_locale_dir)/ja/messages.json',
-      '<(webapp_locale_dir)/ko/messages.json',
-      '<(webapp_locale_dir)/lt/messages.json',
-      '<(webapp_locale_dir)/lv/messages.json',
-      '<(webapp_locale_dir)/nb/messages.json',
-      '<(webapp_locale_dir)/nl/messages.json',
-      '<(webapp_locale_dir)/pl/messages.json',
-      '<(webapp_locale_dir)/pt_BR/messages.json',
-      '<(webapp_locale_dir)/pt_PT/messages.json',
-      '<(webapp_locale_dir)/ro/messages.json',
-      '<(webapp_locale_dir)/ru/messages.json',
-      '<(webapp_locale_dir)/sk/messages.json',
-      '<(webapp_locale_dir)/sl/messages.json',
-      '<(webapp_locale_dir)/sr/messages.json',
-      '<(webapp_locale_dir)/sv/messages.json',
-      '<(webapp_locale_dir)/th/messages.json',
-      '<(webapp_locale_dir)/tr/messages.json',
-      '<(webapp_locale_dir)/uk/messages.json',
-      '<(webapp_locale_dir)/vi/messages.json',
-      '<(webapp_locale_dir)/zh_CN/messages.json',
-      '<(webapp_locale_dir)/zh_TW/messages.json',
+      # Build the list of .json files generated from remoting_strings.grd.
+      '<!@pymod_do_main(remoting_localize --locale_output '
+          '"<(webapp_locale_dir)/${json_suffix}/messages.json" '
+          '--print_only <(remoting_locales))',
     ],
     'remoting_webapp_files': [
       'resources/chromoting16.webp',
@@ -1619,11 +1544,11 @@
               ],
               'action': [
                 'python',
-                '<(localize_py_path)',
-                '<@(remoting_languages)',
-                '--messages_path', '<(webapp_locale_dir)',
-                '<(RULE_INPUT_PATH)',
-                '<@(_outputs)',
+                '<(remoting_localize_path)',
+                '--locale_dir', '<(webapp_locale_dir)',
+                '--template', '<(RULE_INPUT_PATH)',
+                '--output', '<@(_outputs)',
+                '<@(remoting_locales)',
               ],
               'message': 'Localizing the dialogs and strings'
             },
@@ -1778,11 +1703,11 @@
               ],
               'action': [
                 'python',
-                '<(localize_py_path)',
-                '<@(remoting_languages)',
-                '--messages_path', '<(webapp_locale_dir)',
-                '<(RULE_INPUT_PATH)',
-                '<@(_outputs)',
+                '<(remoting_localize_path)',
+                '--locale_dir', '<(webapp_locale_dir)',
+                '--template', '<(RULE_INPUT_PATH)',
+                '--output', '<@(_outputs)',
+                '<@(remoting_locales)',
               ],
               'message': 'Localizing the event log messages'
             },
@@ -1862,16 +1787,16 @@
               ],
               'action': [
                 'python',
-                '<(localize_py_path)',
-                '<@(remoting_languages)',
-                '-i', '<(chrome_version_path)',
+                '<(remoting_localize_path)',
+                '--variables', '<(chrome_version_path)',
                 # |remoting_version_path| must be after |chrome_version_path|
                 # because it can contain overrides for the version numbers.
-                '-i', '<(remoting_version_path)',
-                '-i', '<(lastchange_path)',
-                '--messages_path', '<(webapp_locale_dir)',
-                '<(RULE_INPUT_PATH)',
-                '<@(_outputs)',
+                '--variables', '<(remoting_version_path)',
+                '--variables', '<(lastchange_path)',
+                '--locale_dir', '<(webapp_locale_dir)',
+                '--template', '<(RULE_INPUT_PATH)',
+                '--output', '<@(_outputs)',
+                '<@(remoting_locales)',
               ],
               'message': 'Localizing the version information'
             },
@@ -2368,48 +2293,10 @@
         {
           'destination': '<(PRODUCT_DIR)/remoting_locales',
           'files': [
-            '<(grit_out_dir)/remoting/resources/ar.pak',
-            '<(grit_out_dir)/remoting/resources/bg.pak',
-            '<(grit_out_dir)/remoting/resources/ca.pak',
-            '<(grit_out_dir)/remoting/resources/cs.pak',
-            '<(grit_out_dir)/remoting/resources/da.pak',
-            '<(grit_out_dir)/remoting/resources/de.pak',
-            '<(grit_out_dir)/remoting/resources/el.pak',
-            '<(grit_out_dir)/remoting/resources/en-US.pak',
-            '<(grit_out_dir)/remoting/resources/en-GB.pak',
-            '<(grit_out_dir)/remoting/resources/es.pak',
-            '<(grit_out_dir)/remoting/resources/es-419.pak',
-            '<(grit_out_dir)/remoting/resources/et.pak',
-            '<(grit_out_dir)/remoting/resources/fi.pak',
-            '<(grit_out_dir)/remoting/resources/fil.pak',
-            '<(grit_out_dir)/remoting/resources/fr.pak',
-            '<(grit_out_dir)/remoting/resources/he.pak',
-            '<(grit_out_dir)/remoting/resources/hi.pak',
-            '<(grit_out_dir)/remoting/resources/hr.pak',
-            '<(grit_out_dir)/remoting/resources/hu.pak',
-            '<(grit_out_dir)/remoting/resources/id.pak',
-            '<(grit_out_dir)/remoting/resources/it.pak',
-            '<(grit_out_dir)/remoting/resources/ja.pak',
-            '<(grit_out_dir)/remoting/resources/ko.pak',
-            '<(grit_out_dir)/remoting/resources/lt.pak',
-            '<(grit_out_dir)/remoting/resources/lv.pak',
-            '<(grit_out_dir)/remoting/resources/nb.pak',
-            '<(grit_out_dir)/remoting/resources/nl.pak',
-            '<(grit_out_dir)/remoting/resources/pl.pak',
-            '<(grit_out_dir)/remoting/resources/pt-BR.pak',
-            '<(grit_out_dir)/remoting/resources/pt-PT.pak',
-            '<(grit_out_dir)/remoting/resources/ro.pak',
-            '<(grit_out_dir)/remoting/resources/ru.pak',
-            '<(grit_out_dir)/remoting/resources/sk.pak',
-            '<(grit_out_dir)/remoting/resources/sl.pak',
-            '<(grit_out_dir)/remoting/resources/sr.pak',
-            '<(grit_out_dir)/remoting/resources/sv.pak',
-            '<(grit_out_dir)/remoting/resources/th.pak',
-            '<(grit_out_dir)/remoting/resources/tr.pak',
-            '<(grit_out_dir)/remoting/resources/uk.pak',
-            '<(grit_out_dir)/remoting/resources/vi.pak',
-            '<(grit_out_dir)/remoting/resources/zh-CN.pak',
-            '<(grit_out_dir)/remoting/resources/zh-TW.pak',
+            # Build the list of .pak files generated from remoting_strings.grd.
+            '<!@pymod_do_main(remoting_localize --locale_output '
+                '"<(grit_out_dir)/remoting/resources/${pak_suffix}.pak" '
+                '--print_only <(remoting_locales))',
           ],
         },
         {
diff --git a/remoting/tools/DEPS b/remoting/tools/DEPS
index 324c396..c3c25f4 100644
--- a/remoting/tools/DEPS
+++ b/remoting/tools/DEPS
@@ -1,4 +1,3 @@
 include_rules = [
   "+remoting/host",
-  "+third_party/jinja2",
 ]
diff --git a/remoting/tools/build/DEPS b/remoting/tools/build/DEPS
new file mode 100644
index 0000000..a7c5f8f6
--- /dev/null
+++ b/remoting/tools/build/DEPS
@@ -0,0 +1,3 @@
+include_rules = [
+  "+third_party/jinja2",
+]
diff --git a/remoting/tools/localize.py b/remoting/tools/build/remoting_localize.py
index fa2d732..8dc0538 100755
--- a/remoting/tools/localize.py
+++ b/remoting/tools/build/remoting_localize.py
@@ -10,7 +10,7 @@ variables and localizing strings.
 The script uses Jinja2 template processing library (src/third_party/jinja2).
 Variables available to the templates:
   - |languages| - the list of languages passed on the command line. ('-l').
-  - Each KEY=VALUE define ('-d') can be accesses as |KEY|.
+  - Each NAME=VALUE define ('-d') can be accesses as {{ NAME }}.
   - |official_build| is set to '1' when CHROME_BUILD_TYPE environment variable
     is set to "_official".
 
@@ -24,6 +24,7 @@ Filters:
     ID.
 
 Globals:
+  - IsRtlLanguage(language) - returns True if the language is right-to-left.
   - SelectLanguage(language) - allows to select the language to the used by
     {% trans %}{% endtrans %} statements.
 
@@ -34,6 +35,8 @@ import json
 from optparse import OptionParser
 import os
 import sys
+from string import Template
+
 
 # Win32 primary languages IDs.
 _LANGUAGE_PRIMARY = {
@@ -542,9 +545,20 @@ def NormalizeLanguageCode(language):
   return language.replace('_', '-', 1)
 
 
+def GetDataPackageSuffix(language):
+  lang = NormalizeLanguageCode(language)
+  if lang == 'en':
+    lang = 'en-US'
+  return lang
+
+
+def GetJsonSuffix(language):
+  return language.replace('-', '_', 1)
+
+
 def ReadValuesFromFile(values_dict, file_name):
   """
-  Reads KEYWORD=VALUE settings from the specified file.
+  Reads NAME=VALUE settings from the specified file.
 
   Everything to the left of the first '=' is the keyword,
   everything to the right is the value.  No stripping of
@@ -596,15 +610,15 @@ def WriteIfChanged(file_name, contents, encoding='utf-16'):
 
 class MessageMap:
   """ Provides a dictionary of localized messages for each language."""
-  def __init__(self, languages, messages_path):
+  def __init__(self, languages, locale_dir):
     self.language = None
     self.message_map = {}
 
     # Populate the message map
-    if messages_path:
+    if locale_dir:
       for language in languages:
-        file_name = os.path.join(messages_path,
-                                 language.replace('-', '_', 1),
+        file_name = os.path.join(locale_dir,
+                                 GetJsonSuffix(language),
                                  'messages.json')
         self.message_map[language] = ReadMessagesFromFile(file_name)
 
@@ -625,39 +639,13 @@ class MessageMap:
     return lambda message: self.GetText(message)
 
 
-def Localize(source, target, options):
-  # Load jinja2 library.
-  if options.jinja2:
-    jinja2_path = os.path.normpath(options.jinja2)
-  else:
-    jinja2_path = os.path.normpath(os.path.join(os.path.abspath(__file__),
-                                                '../../../third_party/jinja2'))
-  sys.path.append(os.path.split(jinja2_path)[0])
-  from jinja2 import Environment, FileSystemLoader
-
-  # Create jinja2 environment.
-  (template_path, template_name) = os.path.split(source)
-  env = Environment(loader=FileSystemLoader(template_path),
-                    extensions=['jinja2.ext.do', 'jinja2.ext.i18n'])
-
-  # Register custom filters.
-  env.filters['GetCodepage'] = GetCodepage
-  env.filters['GetCodepageDecimal'] = GetCodepageDecimal
-  env.filters['GetLangId'] = GetLangId
-  env.filters['GetPrimaryLanguage'] = GetPrimaryLanguage
-  env.filters['GetSublanguage'] = GetSublanguage
-
-  # Set the list of languages to use
-  languages = map(NormalizeLanguageCode, options.languages)
+def Localize(source, locales, options):
+  # Set the list of languages to use.
+  languages = map(NormalizeLanguageCode, locales)
   context = { 'languages' : languages }
-  env.globals['IsRtlLanguage'] = IsRtlLanguage
 
-  # Load the localized messages and register the message map with jinja2.i18n
-  # extension.
-  message_map = MessageMap(languages, options.messages_path)
-  env.globals['SelectLanguage'] = message_map.MakeSelectLanguage()
-  env.install_gettext_callables(message_map.MakeGetText(),
-                                message_map.MakeGetText());
+  # Load the localized messages.
+  message_map = MessageMap(languages, options.locale_dir)
 
   # Add OFFICIAL_BUILD variable the same way chrome/tools/build/version.py
   # does.
@@ -671,28 +659,79 @@ def Localize(source, target, options):
     for define in options.define:
       context.update(dict([define.split('=', 1)]));
 
-  # Read KEYWORD=VALUE variables from file.
-  if options.input:
-    for file_name in options.input:
+  # Read NAME=VALUE variables from file.
+  if options.variables:
+    for file_name in options.variables:
       ReadValuesFromFile(context, file_name)
 
-  template = env.get_template(template_name)
-  WriteIfChanged(target, template.render(context), options.encoding);
-  return 0;
+  env = None
+  template = None
+
+  if source:
+    # Load jinja2 library.
+    if options.jinja2:
+      jinja2_path = os.path.normpath(options.jinja2)
+    else:
+      jinja2_path = os.path.normpath(
+          os.path.join(os.path.abspath(__file__),
+                       '../../../../third_party/jinja2'))
+    sys.path.append(os.path.split(jinja2_path)[0])
+    from jinja2 import Environment, FileSystemLoader
+
+    # Create jinja2 environment.
+    (template_path, template_name) = os.path.split(source)
+    env = Environment(loader=FileSystemLoader(template_path),
+                      extensions=['jinja2.ext.do', 'jinja2.ext.i18n'])
+
+    # Register custom filters.
+    env.filters['GetCodepage'] = GetCodepage
+    env.filters['GetCodepageDecimal'] = GetCodepageDecimal
+    env.filters['GetLangId'] = GetLangId
+    env.filters['GetPrimaryLanguage'] = GetPrimaryLanguage
+    env.filters['GetSublanguage'] = GetSublanguage
+
+    # Register the message map with jinja2.i18n extension.
+    env.globals['IsRtlLanguage'] = IsRtlLanguage
+    env.globals['SelectLanguage'] = message_map.MakeSelectLanguage()
+    env.install_gettext_callables(message_map.MakeGetText(),
+                                  message_map.MakeGetText());
+
+    template = env.get_template(template_name)
+
+  # Generate a separate file per each locale if requested.
+  outputs = []
+  if options.locale_output:
+    target = Template(options.locale_output)
+    for lang in languages:
+      context['languages'] = [ lang ]
+      context['language'] = lang
+      context['pak_suffix'] = GetDataPackageSuffix(lang)
+      context['json_suffix'] = GetJsonSuffix(lang)
+
+      template_file_name = target.safe_substitute(context)
+      outputs.append(template_file_name)
+      if not options.print_only:
+        WriteIfChanged(template_file_name, template.render(context),
+                       options.encoding)
+  else:
+    outputs.append(options.output)
+    if not options.print_only:
+      WriteIfChanged(options.output, template.render(context), options.encoding)
+
+  if options.print_only:
+    # Quote each element so filename spaces don't mess up gyp's attempt to parse
+    # it into a list.
+    return " ".join(['"%s"' % x for x in outputs])
 
+  return
 
-def main():
-  usage = "Usage: localize [options] <input> <output>"
+
+def DoMain(argv):
+  usage = "Usage: localize [options] locales"
   parser = OptionParser(usage=usage)
   parser.add_option(
       '-d', '--define', dest='define', action='append', type='string',
-      help='define a variable (VAR=VALUE).')
-  parser.add_option(
-      '-i', '--input', dest='input', action='append', type='string',
-      help='read variables from INPUT.')
-  parser.add_option(
-      '-l', '--language', dest='languages', action='append', type='string',
-      help='add LANGUAGE to the list of languages to use.')
+      help='define a variable (NAME=VALUE).')
   parser.add_option(
       '--encoding', dest='encoding', type='string', default='utf-16',
       help="set the encoding of <output>. 'utf-16' is the default.")
@@ -700,19 +739,35 @@ def main():
       '--jinja2', dest='jinja2', type='string',
       help="specifies path to the jinja2 library.")
   parser.add_option(
-      '--messages_path', dest='messages_path', type='string',
-      help="set path to localized messages.")
+      '--locale_dir', dest='locale_dir', type='string',
+      help="set path to localized message files.")
+  parser.add_option(
+      '--locale_output', dest='locale_output',  type='string',
+      help='specify the per-locale output file name.')
+  parser.add_option(
+      '-o', '--output', dest='output', type='string',
+      help="specify the output file name.")
+  parser.add_option(
+      '--print_only', dest='print_only', action='store_true',
+      default=False, help='print the output file names only.')
+  parser.add_option(
+      '-t', '--template', dest='template', type='string',
+      help="specify the template file name.")
+  parser.add_option(
+      '--variables', dest='variables', action='append', type='string',
+      help='read variables (NAME=VALUE) from file.')
 
-  options, args = parser.parse_args()
-  if len(args) != 2:
-    parser.error('Two positional arguments (<input> and <output>) are expected')
-  if not options.languages:
-    parser.error('At least one language must be specified')
-  if not options.messages_path:
-    parser.error('--messages_path is required')
+  options, locales = parser.parse_args(argv)
+  if not locales:
+    parser.error('At least one locale must be specified')
+  if bool(options.output) == bool(options.locale_output):
+    parser.error(
+        'Either --output or --locale_output must be specified but not both')
+  if not options.template and not options.print_only:
+    parser.error('The template name is required unless --print_only is used')
 
-  return Localize(args[0], args[1], options)
+  return Localize(options.template, locales, options)
 
 if __name__ == '__main__':
-  sys.exit(main())
+  sys.exit(DoMain(sys.argv[1:]))