Add idl_c_proto for generating 'C' style prototypes from IDL

Updated idl_log to provide Log/LogTag (which add the log type tag)
Remove stale switches to idl_lexer.py
Add Regex based Replace function to Node to replace $KEY$ with a property on that node
Added quick resolution of typeinfo by adding typeinfo member to IDLNode
Added idl_c_proto.py which defines a set of function which take an IDL Node then
generate the 'C' style equivelent.

BUG= http://code.google.com/p/chromium/issues/detail?id=84272
TEST= python idl_c_proto.py
Review URL: http://codereview.chromium.org/7085014

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@87567 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 470 insertions, 29 deletions

diff --git a/ppapi/generators/idl_c_proto.py b/ppapi/generators/idl_c_proto.py
new file mode 100644
index 0000000..c8299a6
--- /dev/null
+++ b/ppapi/generators/idl_c_proto.py
@@ -0,0 +1,287 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Generator for C style prototypes and definitions """
+
+import glob
+import os
+import sys
+
+from idl_log import ErrOut, InfoOut, WarnOut
+from idl_node import IDLAttribute, IDLAst, IDLNode
+from idl_option import GetOption, Option, ParseOptions
+from idl_outfile import IDLOutFile
+from idl_parser import IDLParser, ParseFiles
+
+
+#
+# 'C' style parameter and return styles
+#
+
+# Normal parameters (int8_t, etc...)
+NormalType = {
+  'in': '$TYPEREF$',
+  'inout': '$TYPEREF$*',
+  'out': '$TYPEREF$*',
+  'store': '$TYPEREF$',
+  'return': '$TYPEREF$'
+}
+
+# Enum uses the enum's name, except when storing.
+EnumType = {
+  'in': '$TYPEREF$',
+  'inout': '$TYPEREF$*',
+  'out': '$TYPEREF$*',
+  'store': 'int',
+  'return': '$TYPEREF$'
+}
+
+# A mem_t is a 'void *' to memory.
+MemPtrType = {
+  'in': 'const void*',
+  'inout': 'void*',
+  'out': 'void*',
+  'store': 'void*',
+  'return': 'void*'
+}
+
+# A str_t is a string pointer 'char *'.
+CharPtrType = {
+  'in': 'const char*',
+  'inout': 'char*',
+  'out': 'char*',
+  'return': 'char*',
+  'store': 'char*'
+}
+
+# A 'struct' is passed by pointer.
+StructType = {
+  'in': 'const $TYPEREF$*',
+  'inout': '$TYPEREF$*',
+  'out': '$TYPEREF$*',
+  'return': '$TYPEREF$*',
+  'store': '$TYPEREF$'
+}
+
+# A 'void' does not have a parameter or storage type.
+VoidType = {
+  'in': None,
+  'inout': None,
+  'out': None,
+  'return': 'void',
+  'store': None
+}
+
+TypeMap = {
+  'enum': EnumType,
+  'mem_t': MemPtrType,
+  'str_t': CharPtrType,
+  'struct': StructType,
+  'void': VoidType
+}
+
+
+# Place the individual 'normal' types in the type map.
+for name in ['int8_t', 'int16_t' ,'int32_t', 'int64_t', 'uint8_t', 'uint16_t',
+             'uint32_t', 'uint64_t', 'double_t', 'float_t', 'handle_t', 'bool']:
+  TypeMap[name] = NormalType
+
+
+
+# Return the name of the node with suffix/prefix if availible.
+def GetName(node, **keyargs):
+  prefix = keyargs.get('prefix','')
+  suffix = keyargs.get('suffix', '')
+  named = keyargs.get('named', True)
+  name = node.name
+  if not named: name = ''
+  return '%s%s%s' % (prefix, name, suffix)
+
+
+# Return the array specification of the object.
+def GetArrayspec(node):
+  assert(node.cls == 'Array')
+  out = ''
+  fixed = node.GetProperty('FIXED')
+  if fixed:
+    out = '[%s]' % fixed
+  else:
+    out = '[]'
+  # Add children
+  for child in node.GetListOf('Array'):
+    out += GetArrayspec(child)
+  return out
+
+
+# Return the <name>[<size>] form of the node.
+def GetNameArray(node, **keyargs):
+  array = ''
+  for child in node.GetListOf('Array'):
+    array += GetArrayspec(child)
+  return '%s%s' % (GetName(node, **keyargs), array)
+
+
+# Get the 'return' type of the function.
+def GetReturnType(node, **keyargs):
+  root = node
+  while root.typeinfo:
+    root = root.typeinfo
+  typeinfo = root.name
+  typedata = TypeMap[typeinfo]
+  return node.Replace(typedata['return'])
+
+
+# Get the passing form of the parameter.
+def GetParamType(node, **keyargs):
+  root = node
+  while root.typeinfo:
+    root = root.typeinfo
+
+  typeinfo = root.name
+  typedata = TypeMap[typeinfo]
+  if node.GetProperty('in'):
+    return node.Replace(typedata['in'])
+  if node.GetProperty('out'):
+    return node.Replace(typedata['out'])
+  return node.Replace(typedata['inout'])
+
+
+# Return a list of arguments.
+def GetArgList(node, **keyargs):
+  assert(node.cls == 'Callspec')
+  out = []
+  for child in node.GetListOf('Param'):
+    out.append(GetSignature(child, **keyargs))
+  return out
+
+
+# Return the signature of this node
+def GetSignature(node, **keyargs):
+  callnode = node.GetOneOf('Callspec')
+
+  typeinfo  = node.GetProperty('TYPEREF')
+  name = GetNameArray(node, **keyargs)
+  if node.cls == 'Param':
+    type = GetParamType(node, **keyargs)
+  else:
+    type = GetReturnType(node, **keyargs)
+
+  if callnode:
+    arglist = GetArgList(callnode, **keyargs)
+    return '%s (*%s)(%s)' % (type, name, ', '.join(arglist))
+  return '%s %s' % (type, name)
+
+
+# Define an Enum.
+def DefineEnum(node, **keyargs):
+  out = "enum %s {\n" % GetName(node, **keyargs)
+
+  enumlist = []
+  for child in node.GetListOf('EnumItem'):
+    value = child.GetProperty('VALUE')
+    enumlist.append('  %s = %s' % (GetName(child), value))
+  return "%s\n%s\n};\n" % (out, ',\n'.join(enumlist))
+
+# Define an member Function.
+def DefineFunction(node, **keyargs):
+  return '%s;' % GetSignature(node, **keyargs)
+
+# Define an Typedef.
+def DefineTypedef(node, **keyargs):
+  return "typedef %s;\n" % GetSignature(node, **keyargs)
+
+# Define a Struct.
+def DefineStruct(node, **keyargs):
+  out = "struct %s {\n" % (GetName(node, **keyargs))
+  for child in node.GetListOf('Function'):
+    out += '  %s' % Define(child)
+  for child in node.GetListOf('Member'):
+    out += '  %s;\n' % GetSignature(child)
+  out += "};"
+  return out
+
+# Define a C "prototype-able" object
+def Define(node, **keyargs):
+  declmap = {
+    'Enum' : DefineEnum,
+    'Function' : DefineFunction,
+    'Interface' : DefineStruct,
+    'Struct' : DefineStruct,
+    'Typedef' : DefineTypedef,
+  }
+
+  func = declmap.get(node.cls)
+  if not func:
+    ErrLog.Log('Failed to define %s named %s' % (node.cls, node.name))
+  out = func(node, **keyargs)
+
+  tab = ''
+  for i in range(keyargs.get('tabs', 0)):
+    tab += '  '
+
+  return ('%s\n' % tab).join(out.split('\n')) + '\n'
+
+
+# Clean a string representing an object definition and return then string
+# as a single space delimited set of tokens.
+def CleanString(instr):
+  instr = instr.strip()
+  instr = instr.split()
+  return ' '.join(instr)
+
+
+# Test a file, by comparing all it's objects, with their comments.
+def TestFile(filenode):
+  errors = 0
+  for node in filenode.Children()[2:]:
+    if GetOption('verbose'):
+      node.Dump(0, comments=True)
+
+    instr = node.GetOneOf('Comment')
+    instr = CleanString(instr.name)[3:-3]
+
+    outstr = Define(node)
+    outstr = CleanString(outstr)
+
+    if instr != outstr:
+      ErrOut.Log('Failed match of\n>>%s<<\n>>%s<<\nto:' % (instr, outstr))
+      node.Dump(1, comments=True)
+      errors += 1
+  return errors
+
+
+
+# Build and resolve the AST and compare each file individual.
+def TestFiles(filenames):
+  if not filenames:
+    idldir = os.path.split(sys.argv[0])[0]
+    idldir = os.path.join(idldir, 'test_cgen', '*.idl')
+    filenames = glob.glob(idldir)
+
+  filenames = sorted(filenames)
+  ast_result = ParseFiles(filenames)
+
+  total_errs = 0
+  for filenode in ast_result.out.GetListOf('File'):
+    errs = TestFile(filenode)
+    if errs:
+      ErrOut.Log("%s test failed with %d error(s)." % (filenode.name, errs))
+      total_errs += errs
+
+  if total_errs:
+    ErrOut.Log("Failed generator test.")
+  else:
+    InfoOut.Log("Passed generator test.")
+  return total_errs
+
+
+def Main(args):
+  filenames = ParseOptions(args)
+  return TestFiles(filenames)
+
+if __name__ == '__main__':
+  sys.exit(Main(sys.argv[1:]))
+
diff --git a/ppapi/generators/idl_lexer.py b/ppapi/generators/idl_lexer.py
index 744d69b..403f99f 100644
--- a/ppapi/generators/idl_lexer.py
+++ b/ppapi/generators/idl_lexer.py
@@ -37,8 +37,6 @@ from idl_option import GetOption, Option, ParseOptions
 
 
 Option('output', 'Generate output.')
-Option('test_same', 'Test that the output matches.')
-Option('test_expect', 'Test that the output matches.')
 
 #
 # IDL Lexer
diff --git a/ppapi/generators/idl_log.py b/ppapi/generators/idl_log.py
index 7aa4510..1c30aa9 100644
--- a/ppapi/generators/idl_log.py
+++ b/ppapi/generators/idl_log.py
@@ -27,6 +27,12 @@ class IDLLog(object):
     self.log = []
 
   def Log(self, msg):
+    line = "%s\n" % (msg)
+    if self.console: self.out.write(line)
+    if self.capture:
+      self.log.append(msg)
+
+  def LogTag(self, msg):
     line = "%s%s\n" % (self.name, msg)
     if self.console: self.out.write(line)
     if self.capture:
diff --git a/ppapi/generators/idl_node.py b/ppapi/generators/idl_node.py
index a386e40..3f78c23 100644
--- a/ppapi/generators/idl_node.py
+++ b/ppapi/generators/idl_node.py
@@ -17,6 +17,7 @@
 # as the source data by the various generators.
 #
 
+import re
 import sys
 import hashlib
 
@@ -46,6 +47,10 @@ NamedSet = set(['Enum', 'EnumItem', 'Function', 'Interface', 'Member', 'Param',
 # a.k.a. ExtendedAttributes.
 #
 class IDLNode(object):
+  # Regular expression to parse property keys in a string such that a string
+  #  "My string #NAME#" will find the key "NAME".
+  regex_var = re.compile("(?P<src>[^\\$]+)|(?P<key>\\$\\w+\\$)")
+
   def __init__(self, cls, name, filename, lineno, pos, children):
     self.cls = cls
     self.name = name
@@ -100,6 +105,13 @@ class IDLNode(object):
   def GetListOf(self, key):
     return self.children.get(key, [])
 
+  def GetOneOf(self, key):
+    children = self.children.get(key, None)
+    if children:
+      assert(len(children) == 1)
+      return children[0]
+    return None
+
   # Get a list of all objects
   def Children(self):
     out = []
@@ -108,7 +120,7 @@ class IDLNode(object):
     return out
 
   # Dump this object and its children
-  def Dump(self, depth, comments = False, out=sys.stdout):
+  def Dump(self, depth=0, comments=False, out=sys.stdout):
     if self.cls == 'Comment' or self.cls == 'Copyright':
       is_comment = True
     else:
@@ -151,7 +163,7 @@ class IDLNode(object):
       if child.cls in NamedSet:
         if name in self.namespace:
           other = self.namespace[name]
-          child.Error('Attempting to add % to namespace of %s when already '
+          child.Error('Attempting to add %s to namespace of %s when already '
                       'declared in %s' % (name, str(self), str(other)))
         self.namespace[name] = child
 
@@ -169,6 +181,9 @@ class IDLNode(object):
         self.Error('Unable to resolve typename %s.' % typename)
         errs += 1
         sys.exit(-1)
+      self.typeinfo = typeinfo
+    else:
+      self.typeinfo = None
 
     for child in self.Children():
       errs += child.Resolve()
@@ -190,12 +205,32 @@ class IDLNode(object):
     self.hash = hash.hexdigest()
     return self.hash
 
+  def GetProperty(self, name):
+    return self.properties.get(name, None)
+
+  # Recursively expands text keys in the form of $KEY$ with the value
+  # of the property of the same name.  Since this is done recursively
+  # one property can be defined in tems of another.
+  def Replace(self, text):
+    itr = IDLNode.regex_var.finditer(text)
+    out = ""
+    for m in itr:
+      (min,max) = m.span()
+      if m.lastgroup == "src":
+        out += text[min:max]
+      if m.lastgroup == "key":
+        key = text[min+1:max-1]
+        val = self.properties.get(key, None)
+        if not val:
+          self.Error("No property '%s'" % key)
+        out += self.Replace(str(val))
+    return out
 #
 # IDL Predefined types
 #
 BuiltIn = set(['int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t', 'uint16_t',
               'uint32_t', 'uint64_t', 'double_t', 'float_t', 'handle_t',
-              'mem_t', 'str_t', 'void', 'enum', 'struct', 'struct_by_value'])
+              'mem_t', 'str_t', 'void', 'enum', 'struct', 'bool'])
 
 #
 # IDLAst
diff --git a/ppapi/generators/idl_parser.py b/ppapi/generators/idl_parser.py
index 615d4dc..4f38d49 100644
--- a/ppapi/generators/idl_parser.py
+++ b/ppapi/generators/idl_parser.py
@@ -394,11 +394,11 @@ class IDLParser(IDLLexer):
   def p_param_list(self, p):
     """param_list : param_item param_cont
                   | """
-    func = self.BuildExtAttribute('FUNCTION', 'True')
     if len(p) > 1:
-      p[0] = ListFromConcat(p[1], p[2], func)
+      args = ListFromConcat(p[1], p[2])
     else:
-      p[0] = [func]
+      args = []
+    p[0] = self.BuildProduction('Callspec', p, -1, args)
     if self.parse_debug: DumpReduction('param_list', p)
 
   def p_param_item(self, p):
@@ -450,6 +450,23 @@ class IDLParser(IDLLexer):
     if self.parse_debug: DumpReduction('arrays', p)
 
 #
+# Typedef
+#
+# A typedef creates a new referencable type.  The tyepdef can specify an array
+# definition as well as a function declaration.
+#
+  def p_typedef_data(self, p):
+    """typedef_def : modifiers TYPEDEF typeref SYMBOL ';' """
+    p[0] = self.BuildProduction('Typedef', p, 4, ListFromConcat(p[1], p[3]))
+    if self.parse_debug: DumpReduction('typedef_data', p)
+
+  def p_typedef_func(self, p):
+    """typedef_def : modifiers TYPEDEF typeref SYMBOL '(' param_list ')' ';'"""
+    children = ListFromConcat(p[1], p[3], p[6])
+    p[0] = self.BuildProduction('Typedef', p, 4, children)
+    if self.parse_debug: DumpReduction('typedef_func', p)
+
+#
 # Enumeration
 #
 # An enumeration is a set of named integer constants.  An enumeration
@@ -457,7 +474,8 @@ class IDLParser(IDLLexer):
 #
   def p_enum_block(self, p):
     """enum_block : modifiers ENUM SYMBOL '{' enum_list '}' ';'"""
-    p[0] = self.BuildProduction('Enum', p, 3, ListFromConcat(p[1], p[5]))
+    Type = self.BuildExtAttribute('TYPEREF', 'enum')
+    p[0] = self.BuildProduction('Enum', p, 3, ListFromConcat(Type, p[1], p[5]))
     if self.parse_debug: DumpReduction('enum_block', p)
 
   def p_enum_list(self, p):
@@ -498,7 +516,8 @@ class IDLParser(IDLLexer):
 
   def p_member_function(self, p):
     """member_function : modifiers typeref SYMBOL '(' param_list ')' ';'"""
-    p[0] = self.BuildProduction('Function', p, 3, ListFromConcat(p[1], p[5]))
+    children = ListFromConcat(p[1], p[2], p[5])
+    p[0] = self.BuildProduction('Function', p, 3, children)
     if self.parse_debug: DumpReduction('member_function', p)
 
   def p_member_error(self, p):
@@ -513,7 +532,9 @@ class IDLParser(IDLLexer):
 #
   def p_struct_block(self, p):
     """struct_block : modifiers STRUCT SYMBOL '{' struct_list '}' ';'"""
-    p[0] = self.BuildProduction('Struct', p, 3, ListFromConcat(p[1], p[5]))
+    Type = self.BuildExtAttribute('TYPEREF', 'struct')
+    children = ListFromConcat(Type, p[1], p[5])
+    p[0] = self.BuildProduction('Struct', p, 3, children)
     if self.parse_debug: DumpReduction('struct_block', p)
 
   def p_struct_list(self, p):
@@ -524,22 +545,7 @@ class IDLParser(IDLLexer):
       p[0] = ListFromConcat(member, p[5])
     if self.parse_debug: DumpReduction('struct_list', p)
 
-#
-# Typedef
-#
-# A typedef creates a new referencable type.  The tyepdef can specify an array
-# definition as well as a function declaration.
-#
-  def p_typedef_data(self, p):
-    """typedef_def : modifiers TYPEDEF typeref SYMBOL ';' """
-    p[0] = self.BuildProduction('Typedef', p, 4, ListFromConcat(p[1], p[3]))
-    if self.parse_debug: DumpReduction('typedef_data', p)
 
-  def p_typedef_func(self, p):
-    """typedef_def : modifiers TYPEDEF typeref SYMBOL '(' param_list ')' ';'"""
-    children = ListFromConcat(p[1], p[3], p[6])
-    p[0] = self.BuildProduction('Typedef', p, 4, children)
-    if self.parse_debug: DumpReduction('typedef_func', p)
 
 
 #
@@ -577,6 +583,9 @@ class IDLParser(IDLLexer):
     ErrOut.LogLine(filename, lineno, pos, msg)
     self.parse_errors += 1
 
+  def Warn(self, node, msg):
+    WarnOut.LogLine(node.filename, node.lineno, node.pos, msg)
+    self.parse_warnings += 1
 
   def __init__(self):
     IDLLexer.__init__(self)
@@ -604,6 +613,22 @@ class IDLParser(IDLLexer):
     return tok
 
 #
+# VerifyProduction
+#
+# Once the node is built, we will check for certain types of issues
+#
+  def VerifyProduction(self, node):
+    comment = node.GetOneOf('Comment')
+    if node.cls in ['Interface', 'Struct', 'Function'] and not comment:
+      self.Warn(node, 'Missing comment.')
+    if node.cls in ['Param']:
+      found = False;
+      for form in ['in', 'inout', 'out']:
+        if node.GetProperty(form): found = True
+      if not found: self.Warn(node, 'Missing argument type: [in|out|inout]')
+
+
+#
 # BuildProduction
 #
 # Production is the set of items sent to a grammar rule resulting in a new
@@ -621,6 +646,7 @@ class IDLParser(IDLLexer):
     if self.build_debug:
       InfoOut.Log("Building %s(%s)" % (cls, name))
     out = IDLNode(cls, name, filename, lineno, pos, childlist)
+    self.VerifyProduction(out)
     return out
 
 #
@@ -641,9 +667,11 @@ class IDLParser(IDLLexer):
 #
 # Attempts to parse the current data loaded in the lexer.
 #
-  def ParseData(self):
+  def ParseData(self, data, filename='<Internal>'):
+    self.SetData(filename, data)
     try:
       self.parse_errors = 0
+      self.parse_warnings = 0
       return self.yaccobj.parse(lexer=self)
 
     except lex.LexError as le:
@@ -657,11 +685,10 @@ class IDLParser(IDLLexer):
 #
   def ParseFile(self, filename):
     data = open(filename).read()
-    self.SetData(filename, data)
     if self.verbose:
       InfoOut.Log("Parsing %s" % filename)
     try:
-      out = self.ParseData()
+      out = self.ParseData(data, filename)
       return StageResult(filename, out, self.parse_errors)
 
     except Exception as e:
diff --git a/ppapi/generators/test_cgen/enum_typedef.idl b/ppapi/generators/test_cgen/enum_typedef.idl
new file mode 100644
index 0000000..cb47402
--- /dev/null
+++ b/ppapi/generators/test_cgen/enum_typedef.idl
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2011 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/* This file will test that the IDL snippet matches the comment */
+
+/* enum et1 { A = 1, B = 2, C = 3 }; */
+enum et1 { A=1, B=2, C=3 };
+
+/* typedef int32_t i; */
+typedef int32_t i;
+
+/* typedef int32_t i2[2]; */
+typedef int32_t[2] i2;
+
+/* typedef int32_t (*i_func)(); */
+typedef int32_t i_func();
+
+/* typedef int32_t (*i_func_i)(int32_t i); */
+typedef int32_t i_func_i([in] int32_t i);
+
+/* typedef et1 et4[4]; */
+typedef et1[4] et4;
+
+/*
+typedef int8_t (*PPB_Audio_Callback)(const void* sample_buffer,
+                                     uint32_t buffer_size_in_bytes,
+                                     const void* user_data);
+*/
+typedef int8_t PPB_Audio_Callback([in] mem_t sample_buffer,
+                                  [in] uint32_t buffer_size_in_bytes,
+                                  [in] mem_t user_data);
+
diff --git a/ppapi/generators/test_cgen/interface.idl b/ppapi/generators/test_cgen/interface.idl
new file mode 100644
index 0000000..cee50d6
--- /dev/null
+++ b/ppapi/generators/test_cgen/interface.idl
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2011 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/* This file will test that the IDL snippet matches the comment */
+
+/* struct ist { void* X; }; */
+struct ist {
+  mem_t X;
+};
+
+/* struct iface1 { 
+  int8_t (*mem1)(int16_t x, int32_t y);
+  int32_t (*mem2)(const ist* a);
+  int32_t (*mem3)(ist* b);
+   
+}; */
+interface iface1 {
+  int8_t mem1([in] int16_t x, [in] int32_t y);
+  int32_t mem2([in] ist a);
+  int32_t mem3([out] ist b);
+};
+
diff --git a/ppapi/generators/test_cgen/structs.idl b/ppapi/generators/test_cgen/structs.idl
new file mode 100644
index 0000000..02906ae
--- /dev/null
+++ b/ppapi/generators/test_cgen/structs.idl
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2011 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/* This file will test that the IDL snippet matches the comment */
+
+/* typedef uint8_t s_array[3]; */
+typedef uint8_t[3] s_array;
+
+/* enum senum { esv1 = 1, esv2 = 2 }; */
+enum senum {
+  esv1=1,
+  esv2=2
+};
+
+/* struct st1 { int32_t i; senum j; }; */
+struct st1 {
+  int32_t i;
+  senum j;
+};
+
+/* struct st2 { s_array pixels[640][480]; }; */
+struct st2 {
+  s_array[640][480] pixels;
+};
+