[mojo] Add typemap and variant support to generators

This introduces two new flags for the mojom bindings generator:

  --typemap can be used to provide typemap (JSON) files
    to the bindings generator.

  --variant specifies the name of the bindings variant to emit.

Both of these new flags only affect C++ generation, and specifying
--variant at all prevents JS or Java bindings from being emitted.

Both of these new flags have GN mojom template variables associated
with them as well.

Part of a series of changes to support custom mojom serialization:

  1. https://codereview.chromium.org/1515423002
  2. This CL
  3. https://codereview.chromium.org/1524693002
  4. https://codereview.chromium.org/1520153002
  5. https://codereview.chromium.org/1524613002
  6. https://codereview.chromium.org/1526533002
  7. https://codereview.chromium.org/1524703002

BUG=569669

Review URL: https://codereview.chromium.org/1517043004

Cr-Commit-Position: refs/heads/master@{#365674}

14 files changed, 232 insertions, 42 deletions

diff --git a/mojo/public/cpp/bindings/tests/BUILD.gn b/mojo/public/cpp/bindings/tests/BUILD.gn
index 9fd466a..d89daba 100644
--- a/mojo/public/cpp/bindings/tests/BUILD.gn
+++ b/mojo/public/cpp/bindings/tests/BUILD.gn
@@ -49,6 +49,8 @@ source_set("tests") {
     "//mojo/public/cpp/utility",
     "//mojo/public/interfaces/bindings/tests:test_associated_interfaces",
     "//mojo/public/interfaces/bindings/tests:test_interfaces",
+    "//mojo/public/interfaces/bindings/tests:test_interfaces_blink",
+    "//mojo/public/interfaces/bindings/tests:test_interfaces_chromium",
     "//mojo/public/interfaces/bindings/tests:test_interfaces_experimental",
     "//testing/gtest",
   ]
diff --git a/mojo/public/interfaces/bindings/tests/BUILD.gn b/mojo/public/interfaces/bindings/tests/BUILD.gn
index 802bd58..d0af970 100644
--- a/mojo/public/interfaces/bindings/tests/BUILD.gn
+++ b/mojo/public/interfaces/bindings/tests/BUILD.gn
@@ -68,3 +68,37 @@ mojom("versioning_test_client_interfaces") {
 
   with_environment = false
 }
+
+mojom("test_interfaces_chromium") {
+  testonly = true
+
+  sources = [
+    "test_native_types.mojom",
+  ]
+
+  variant = "chromium"
+  typemaps = [ "chromium_test.typemap" ]
+
+  with_environment = false
+
+  public_deps = [
+    ":test_interfaces",
+  ]
+}
+
+mojom("test_interfaces_blink") {
+  testonly = true
+
+  sources = [
+    "test_native_types.mojom",
+  ]
+
+  variant = "blink"
+  typemaps = [ "blink_test.typemap" ]
+
+  with_environment = false
+
+  public_deps = [
+    ":test_interfaces",
+  ]
+}
diff --git a/mojo/public/interfaces/bindings/tests/blink_test.typemap b/mojo/public/interfaces/bindings/tests/blink_test.typemap
new file mode 100644
index 0000000..511a549
--- /dev/null
+++ b/mojo/public/interfaces/bindings/tests/blink_test.typemap
@@ -0,0 +1,14 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+{
+  "c++": {
+    "mojo.test.PickledStruct": {
+      "typename": "mojo::test::PickledStructBlink",
+      "headers": [
+        "mojo/public/cpp/bindings/tests/pickled_struct_blink.h"
+      ]
+    }
+  }
+}
diff --git a/mojo/public/interfaces/bindings/tests/chromium_test.typemap b/mojo/public/interfaces/bindings/tests/chromium_test.typemap
new file mode 100644
index 0000000..1028982
--- /dev/null
+++ b/mojo/public/interfaces/bindings/tests/chromium_test.typemap
@@ -0,0 +1,14 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+{
+  "c++": {
+    "mojo.test.PickledStruct": {
+      "typename": "mojo::test::PickledStructChromium",
+      "headers": [
+        "mojo/public/cpp/bindings/tests/pickled_struct_chromium.h"
+      ]
+    }
+  }
+}
diff --git a/mojo/public/tools/bindings/generators/cpp_templates/module-internal.h.tmpl b/mojo/public/tools/bindings/generators/cpp_templates/module-internal.h.tmpl
index 2d748fc..a3be5a4 100644
--- a/mojo/public/tools/bindings/generators/cpp_templates/module-internal.h.tmpl
+++ b/mojo/public/tools/bindings/generators/cpp_templates/module-internal.h.tmpl
@@ -2,8 +2,15 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-{%- set header_guard = "%s_INTERNAL_H_"|
-        format(module.path|upper|replace("/","_")|replace(".","_")) %}
+{%- if variant -%}
+{%-   set variant_path = "%s-%s"|format(module.path, variant) -%}
+{%- else -%}
+{%-   set variant_path = module.path -%}
+{%- endif -%}
+
+{%- set header_guard = "%s_INTERNAL_H_"|format(
+        variant_path|upper|replace("/","_")|replace(".","_")|
+            replace("-", "_")) %}
 
 #ifndef {{header_guard}}
 #define {{header_guard}}
@@ -27,6 +34,9 @@ class BoundsChecker;
 {%- for namespace in namespaces_as_array %}
 namespace {{namespace}} {
 {%- endfor %}
+{%- if variant %}
+namespace {{variant}} {
+{%- endif %}
 
 {#--- Wrapper forward declarations #}
 {%  for struct in structs %}
@@ -65,6 +75,9 @@ class {{union.name}}_Data;
 #pragma pack(pop)
 
 }  // namespace internal
+{%- if variant %}
+}  // namespace {{variant}}
+{%- endif %}
 {%- for namespace in namespaces_as_array|reverse %}
 }  // namespace {{namespace}}
 {%- endfor %}
diff --git a/mojo/public/tools/bindings/generators/cpp_templates/module.cc.tmpl b/mojo/public/tools/bindings/generators/cpp_templates/module.cc.tmpl
index dddb6ab..fd2f610 100644
--- a/mojo/public/tools/bindings/generators/cpp_templates/module.cc.tmpl
+++ b/mojo/public/tools/bindings/generators/cpp_templates/module.cc.tmpl
@@ -2,6 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+{%- if variant -%}
+{%-   set variant_path = "%s-%s"|format(module.path, variant) -%}
+{%- else -%}
+{%-   set variant_path = module.path -%}
+{%- endif %}
+
 #if defined(__clang__)
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wunused-private-field"
@@ -12,7 +18,7 @@
 #pragma warning(disable:4756)
 #endif
 
-#include "{{module.path}}.h"
+#include "{{variant_path}}.h"
 
 #include <math.h>
 
@@ -34,6 +40,9 @@
 {%- for namespace in namespaces_as_array %}
 namespace {{namespace}} {
 {%- endfor %}
+{%- if variant %}
+namespace {{variant}} {
+{%- endif %}
 
 {#--- Constants #}
 {%- for constant in module.constants %}
@@ -121,6 +130,9 @@ MOJO_STATIC_CONST_MEMBER_DEFINITION const {{constant.kind|cpp_pod_type}} {{struc
 {%-   include "union_serialization_definition.tmpl" %}
 {%- endfor %}
 
+{%- if variant %}
+}  // namespace {{variant}}
+{%- endif %}
 {%- for namespace in namespaces_as_array|reverse %}
 }  // namespace {{namespace}}
 {%- endfor %}
diff --git a/mojo/public/tools/bindings/generators/cpp_templates/module.h.tmpl b/mojo/public/tools/bindings/generators/cpp_templates/module.h.tmpl
index 4b1e658..c54668d 100644
--- a/mojo/public/tools/bindings/generators/cpp_templates/module.h.tmpl
+++ b/mojo/public/tools/bindings/generators/cpp_templates/module.h.tmpl
@@ -2,8 +2,15 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-{%- set header_guard = "%s_H_"|
-        format(module.path|upper|replace("/","_")|replace(".","_")) %}
+{%- if variant -%}
+{%-   set variant_path = "%s-%s"|format(module.path, variant) -%}
+{%- else -%}
+{%-   set variant_path = module.path -%}
+{%- endif -%}
+
+{%- set header_guard = "%s_H_"|format(
+        variant_path|upper|replace("/","_")|replace(".","_")|
+            replace("-", "_")) %}
 
 #ifndef {{header_guard}}
 #define {{header_guard}}
@@ -24,7 +31,7 @@
 #include "mojo/public/cpp/bindings/no_interface.h"
 #include "mojo/public/cpp/bindings/string.h"
 #include "mojo/public/cpp/bindings/struct_ptr.h"
-#include "{{module.path}}-internal.h"
+#include "{{variant_path}}-internal.h"
 {%- for import in imports %}
 #include "{{import.module.path}}.h"
 {%- endfor %}
@@ -32,6 +39,9 @@
 {%- for namespace in namespaces_as_array %}
 namespace {{namespace}} {
 {%- endfor %}
+{%- if variant %}
+namespace {{variant}} {
+{%- endif %}
 
 {#--- Enums #}
 {% from "enum_macros.tmpl" import enum_decl -%}
@@ -139,6 +149,9 @@ typedef mojo::StructPtr<{{union.name}}> {{union.name}}Ptr;
 {%-   endfor %}
 {%- endif %}
 
+{%- if variant %}
+}  // namespace {{variant}}
+{%- endif %}
 {%- for namespace in namespaces_as_array|reverse %}
 }  // namespace {{namespace}}
 {%- endfor %}
diff --git a/mojo/public/tools/bindings/generators/mojom_cpp_generator.py b/mojo/public/tools/bindings/generators/mojom_cpp_generator.py
index 8b153a5..4c63270 100644
--- a/mojo/public/tools/bindings/generators/mojom_cpp_generator.py
+++ b/mojo/public/tools/bindings/generators/mojom_cpp_generator.py
@@ -429,6 +429,7 @@ class Generator(generator.Generator):
       "structs": self.GetStructs(),
       "unions": self.GetUnions(),
       "interfaces": self.GetInterfaces(),
+      "variant": self.variant,
     }
 
   @UseJinja("cpp_templates/module.h.tmpl", filters=cpp_filters)
@@ -444,9 +445,10 @@ class Generator(generator.Generator):
     return self.GetJinjaExports()
 
   def GenerateFiles(self, args):
+    suffix = "-%s" % self.variant if self.variant else ""
     self.Write(self.GenerateModuleHeader(),
-        self.MatchMojomFilePath("%s.h" % self.module.name))
+        self.MatchMojomFilePath("%s%s.h" % (self.module.name, suffix)))
     self.Write(self.GenerateModuleInternalHeader(),
-        self.MatchMojomFilePath("%s-internal.h" % self.module.name))
+        self.MatchMojomFilePath("%s%s-internal.h" % (self.module.name, suffix)))
     self.Write(self.GenerateModuleSource(),
-        self.MatchMojomFilePath("%s.cc" % self.module.name))
+        self.MatchMojomFilePath("%s%s.cc" % (self.module.name, suffix)))
diff --git a/mojo/public/tools/bindings/generators/mojom_java_generator.py b/mojo/public/tools/bindings/generators/mojom_java_generator.py
index c091a5f..5edaf94 100644
--- a/mojo/public/tools/bindings/generators/mojom_java_generator.py
+++ b/mojo/public/tools/bindings/generators/mojom_java_generator.py
@@ -503,6 +503,10 @@ class Generator(generator.Generator):
                  '%s.java' % GetConstantsMainEntityName(self.module))
 
   def GenerateFiles(self, unparsed_args):
+    # TODO(rockot): Support variant output for Java.
+    if self.variant:
+      raise Exception("Variants not supported in Java bindings.")
+
     parser = argparse.ArgumentParser()
     parser.add_argument('--java_output_directory', dest='java_output_directory')
     args = parser.parse_args(unparsed_args)
diff --git a/mojo/public/tools/bindings/generators/mojom_js_generator.py b/mojo/public/tools/bindings/generators/mojom_js_generator.py
index 23c203a..638f1cb 100644
--- a/mojo/public/tools/bindings/generators/mojom_js_generator.py
+++ b/mojo/public/tools/bindings/generators/mojom_js_generator.py
@@ -385,6 +385,9 @@ class Generator(generator.Generator):
     return self.GetParameters()
 
   def GenerateFiles(self, args):
+    if self.variant:
+      raise Exception("Variants not supported in JavaScript bindings.")
+
     self.Write(self.GenerateAMDModule(),
         self.MatchMojomFilePath("%s.js" % self.module.name))
 
diff --git a/mojo/public/tools/bindings/mojom.gni b/mojo/public/tools/bindings/mojom.gni
index 174e5b8..61f43e5 100644
--- a/mojo/public/tools/bindings/mojom.gni
+++ b/mojo/public/tools/bindings/mojom.gni
@@ -25,6 +25,13 @@
 #       Mojo environment implementation. Defaults to |true| and in general
 #       should only be overridden by mojom targets within the Mojo EDK.
 #
+#   typemaps (optional)
+#       A list of typemap files to apply during bindings generation.
+#
+#   variant (optional)
+#       A variant name to apply to generated bindings. Variant influences
+#       generated source filenames as wells the symbols they define.
+#
 #   testonly (optional)
 #
 #   visibility (optional)
@@ -102,15 +109,26 @@ template("mojom") {
       "$generator_root/pylib/mojom/parse/parser.py",
       "$generator_root/pylib/mojom/parse/translate.py",
     ]
-    generator_cpp_outputs = [
-      "{{source_gen_dir}}/{{source_name_part}}.mojom.cc",
-      "{{source_gen_dir}}/{{source_name_part}}.mojom.h",
-      "{{source_gen_dir}}/{{source_name_part}}.mojom-internal.h",
-    ]
-    generator_js_outputs =
-        [ "{{source_gen_dir}}/{{source_name_part}}.mojom.js" ]
-    generator_java_outputs =
-        [ "{{source_gen_dir}}/{{source_name_part}}.mojom.srcjar" ]
+    if (defined(invoker.variant)) {
+      variant = invoker.variant
+      generator_cpp_outputs = [
+        "{{source_gen_dir}}/{{source_name_part}}.mojom-${variant}.cc",
+        "{{source_gen_dir}}/{{source_name_part}}.mojom-${variant}.h",
+        "{{source_gen_dir}}/{{source_name_part}}.mojom-${variant}-internal.h",
+      ]
+      generator_js_outputs = []
+      generator_java_outputs = []
+    } else {
+      generator_cpp_outputs = [
+        "{{source_gen_dir}}/{{source_name_part}}.mojom.cc",
+        "{{source_gen_dir}}/{{source_name_part}}.mojom.h",
+        "{{source_gen_dir}}/{{source_name_part}}.mojom-internal.h",
+      ]
+      generator_js_outputs =
+          [ "{{source_gen_dir}}/{{source_name_part}}.mojom.js" ]
+      generator_java_outputs =
+          [ "{{source_gen_dir}}/{{source_name_part}}.mojom.srcjar" ]
+    }
   }
 
   if (defined(invoker.sources)) {
@@ -140,6 +158,29 @@ template("mojom") {
           ]
         }
       }
+
+      if (defined(invoker.variant)) {
+        args += [
+          "--variant",
+          invoker.variant,
+          "-g",
+          "c++",
+        ]
+      } else {
+        args += [
+          "-g",
+          "c++,javascript,java",
+        ]
+      }
+
+      if (defined(invoker.typemaps)) {
+        foreach(typemap, invoker.typemaps) {
+          args += [
+            "--typemap",
+            rebase_path(typemap, root_build_dir),
+          ]
+        }
+      }
     }
   }
 
@@ -150,7 +191,7 @@ template("mojom") {
     if (defined(invoker.testonly)) {
       testonly = invoker.testonly
     }
-    if (defined(invoker.sources)) {
+    if (defined(invoker.sources) && !defined(invoker.variant)) {
       data = process_file_template(invoker.sources, generator_js_outputs)
     }
 
@@ -225,7 +266,7 @@ template("mojom") {
     }
   }
 
-  if (is_android) {
+  if (is_android && !defined(invoker.variant)) {
     import("//build/config/android/rules.gni")
 
     java_srcjar_target_name = target_name + "_java_sources"
diff --git a/mojo/public/tools/bindings/mojom_bindings_generator.py b/mojo/public/tools/bindings/mojom_bindings_generator.py
index 96098e0..83ab590 100755
--- a/mojo/public/tools/bindings/mojom_bindings_generator.py
+++ b/mojo/public/tools/bindings/mojom_bindings_generator.py
@@ -8,8 +8,10 @@
 
 import argparse
 import imp
+import json
 import os
 import pprint
+import re
 import sys
 
 # Disable lint check for finding modules:
@@ -40,32 +42,29 @@ from mojom.parse.parser import Parse
 from mojom.parse.translate import Translate
 
 
+_BUILTIN_GENERATORS = {
+  "c++": "mojom_cpp_generator.py",
+  "javascript": "mojom_js_generator.py",
+  "java": "mojom_java_generator.py",
+}
+
 def LoadGenerators(generators_string):
   if not generators_string:
     return []  # No generators.
 
   script_dir = os.path.dirname(os.path.abspath(__file__))
-  generators = []
+  generators = {}
   for generator_name in [s.strip() for s in generators_string.split(",")]:
-    # "Built-in" generators:
-    if generator_name.lower() == "c++":
-      generator_name = os.path.join(script_dir, "generators",
-                                    "mojom_cpp_generator.py")
-    elif generator_name.lower() == "javascript":
-      generator_name = os.path.join(script_dir, "generators",
-                                    "mojom_js_generator.py")
-    elif generator_name.lower() == "java":
+    language = generator_name.lower()
+    if language in _BUILTIN_GENERATORS:
       generator_name = os.path.join(script_dir, "generators",
-                                    "mojom_java_generator.py")
-    # Specified generator python module:
-    elif generator_name.endswith(".py"):
-      pass
+                                    _BUILTIN_GENERATORS[language])
     else:
       print "Unknown generator name %s" % generator_name
       sys.exit(1)
     generator_module = imp.load_source(os.path.basename(generator_name)[:-3],
                                        generator_name)
-    generators.append(generator_module)
+    generators[language] = generator_module
   return generators
 
 
@@ -89,6 +88,20 @@ class MojomProcessor(object):
     self._should_generate = should_generate
     self._processed_files = {}
     self._parsed_files = {}
+    self._typemap = {}
+
+  def LoadTypemaps(self, typemaps):
+    # Support some very simple single-line comments in typemap JSON.
+    comment_expr = r"^\s*//.*$"
+    def no_comments(line):
+      return not re.match(comment_expr, line)
+    for filename in typemaps:
+      with open(filename) as f:
+        typemaps = json.loads("".join(filter(no_comments, f.readlines())))
+        for language, typemap in typemaps.iteritems():
+          language_map = self._typemap.get(language, {})
+          language_map.update(typemap)
+          self._typemap[language] = language_map
 
   def ProcessFile(self, args, remaining_args, generator_modules, filename):
     self._ParseFileAndImports(filename, args.import_directories, [])
@@ -126,8 +139,10 @@ class MojomProcessor(object):
     module.path = module.path.replace('\\', '/')
 
     if self._should_generate(filename):
-      for generator_module in generator_modules:
-        generator = generator_module.Generator(module, args.output_dir)
+      for language, generator_module in generator_modules.iteritems():
+        generator = generator_module.Generator(
+            module, args.output_dir, typemap=self._typemap.get(language, {}),
+            variant=args.variant)
         filtered_args = []
         if hasattr(generator_module, 'GENERATOR_PREFIX'):
           prefix = '--' + generator_module.GENERATOR_PREFIX + '_'
@@ -195,13 +210,22 @@ def main():
                       help="add a directory to be searched for import files")
   parser.add_argument("--use_bundled_pylibs", action="store_true",
                       help="use Python modules bundled in the SDK")
+  parser.add_argument("--typemap", action="append", metavar="TYPEMAP",
+                      default=[], dest="typemaps",
+                      help="apply TYPEMAP to generated output")
+  parser.add_argument("--variant", dest="variant", default=None,
+                      help="output a named variant of the bindings")
   (args, remaining_args) = parser.parse_known_args()
 
+  if args.variant == "none":
+    args.variant = None
+
   generator_modules = LoadGenerators(args.generators_string)
 
   fileutil.EnsureDirectoryExists(args.output_dir)
 
   processor = MojomProcessor(lambda filename: filename in args.filename)
+  processor.LoadTypemaps(set(args.typemaps))
   for filename in args.filename:
     processor.ProcessFile(args, remaining_args, generator_modules, filename)
 
diff --git a/mojo/public/tools/bindings/mojom_list_outputs.py b/mojo/public/tools/bindings/mojom_list_outputs.py
index ed8bdfa..267bd80 100755
--- a/mojo/public/tools/bindings/mojom_list_outputs.py
+++ b/mojo/public/tools/bindings/mojom_list_outputs.py
@@ -11,20 +11,32 @@ def main():
   parser = argparse.ArgumentParser(
       description="GYP helper script for mapping mojoms => generated outputs.")
   parser.add_argument("--basedir", required=True)
+  parser.add_argument("--variant", required=True)
   parser.add_argument("mojom", nargs="*")
 
   args = parser.parse_args()
 
+  variant = args.variant if args.variant != "none" else None
+
   for mojom in args.mojom:
     full = os.path.join("<(SHARED_INTERMEDIATE_DIR)", args.basedir, mojom)
     base, ext = os.path.splitext(full)
-    assert ext == ".mojom", mojom
+
+    # Ignore non-mojom files.
+    if ext != ".mojom":
+      continue
+
     # Fix filename escaping issues on Windows.
     base = base.replace("\\", "/")
-    print base + ".mojom.cc"
-    print base + ".mojom.h"
-    print base + ".mojom-internal.h"
-    print base + ".mojom.js"
+    if variant:
+      print base + ".mojom-%s.cc" % variant
+      print base + ".mojom-%s.h" % variant
+      print base + ".mojom-%s-internal.h" % variant
+    else:
+      print base + ".mojom.cc"
+      print base + ".mojom.h"
+      print base + ".mojom-internal.h"
+      print base + ".mojom.js"
 
   return 0
 
diff --git a/mojo/public/tools/bindings/pylib/mojom/generate/generator.py b/mojo/public/tools/bindings/pylib/mojom/generate/generator.py
index 666ef43..8c3e642 100644
--- a/mojo/public/tools/bindings/pylib/mojom/generate/generator.py
+++ b/mojo/public/tools/bindings/pylib/mojom/generate/generator.py
@@ -40,9 +40,11 @@ def WriteFile(contents, full_path):
 class Generator(object):
   # Pass |output_dir| to emit files to disk. Omit |output_dir| to echo all
   # files to stdout.
-  def __init__(self, module, output_dir=None):
+  def __init__(self, module, output_dir=None, typemap={}, variant=None):
     self.module = module
     self.output_dir = output_dir
+    self.typemap = typemap
+    self.variant = variant
 
   def GetStructsFromMethods(self):
     result = []