IDL cleanup, added AST node, namespace, StageResult

Added the ability to track namespace within the tree to look for
references.  Added a BuildTree and Resolve step.  Added a 
StageResult object to return partial trees and error counts.
Built test search into parser so now test files no longer need
to be specified on the command line.

BUG=77551
TEST= python idl_parser.py --test
Review URL: http://codereview.chromium.org/6903097

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@84086 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 264 insertions, 48 deletions

diff --git a/ppapi/generators/idl_node.py b/ppapi/generators/idl_node.py
index d54f1ad..a386e40 100644
--- a/ppapi/generators/idl_node.py
+++ b/ppapi/generators/idl_node.py
@@ -34,6 +34,10 @@ class IDLAttribute(object):
     self.name = name
     self.value = value
 
+# Set of object IDLNode types which have a name and belong in the namespace.
+NamedSet = set(['Enum', 'EnumItem', 'Function', 'Interface', 'Member', 'Param',
+               'Struct', 'Type', 'Typedef'])
+
 #
 # IDLNode
 #
@@ -47,11 +51,21 @@ class IDLNode(object):
     self.name = name
     self.lineno = lineno
     self.pos = pos
+    self.filename = filename
+
+    # Dictionary of type to in order child list
     self.children = {}
+
+    # Dictionary of child name to child object
+    self.namespace = {}
+
+    # Dictionary of properties (ExtAttributes)
     self.properties = { 'NAME' : name }
+
     self.hash = None
     self.typeref = None
     self.parent = None
+
     if children:
       for child in children:
         #
@@ -75,12 +89,12 @@ class IDLNode(object):
   # Log an error for this object
   def Error(self, msg):
     ErrOut.LogLine(self.filename, self.lineno, 0, " %s %s" %
-                   (self.FullName(), msg))
+                   (str(self), msg))
 
   # Log a warning for this object
   def Warning(self, msg):
     WarnOut.LogLine(self.filename, self.lineno, 0, " %s %s" %
-                    (self.FullName(), msg))
+                    (str(self), msg))
 
   # Get a list of objects for this key
   def GetListOf(self, key):
@@ -124,5 +138,84 @@ class IDLNode(object):
 
       out.write("%s  %ss\n" % (tab, cls))
       for c in self.children[cls]:
-        c.Dump(depth + 1, comments = comments, out=out)
+        c.Dump(depth + 1, comments=comments, out=out)
+
+  # Link the parents and add the object to the parent's namespace
+  def BuildTree(self, parent):
+    self.parent = parent
+    for child in self.Children():
+      child.BuildTree(self)
+      name = child.name
+
+      # Add this child to the local namespace if it's a 'named' type
+      if child.cls in NamedSet:
+        if name in self.namespace:
+          other = self.namespace[name]
+          child.Error('Attempting to add % to namespace of %s when already '
+                      'declared in %s' % (name, str(self), str(other)))
+        self.namespace[name] = child
+
+  def Resolve(self):
+    errs = 0
+    typename = self.properties.get('TYPEREF', None)
+
+    if typename:
+      if typename in self.namespace:
+        self.Error('Type collision in local namespace')
+        errs += 1
+
+      typeinfo = self.parent.Find(typename)
+      if not typeinfo:
+        self.Error('Unable to resolve typename %s.' % typename)
+        errs += 1
+        sys.exit(-1)
+
+    for child in self.Children():
+      errs += child.Resolve()
+
+    return errs
+
+  def Find(self, name):
+    if name in self.namespace: return self.namespace[name]
+    if self.parent: return self.parent.Find(name)
+    return None
+
+  def Hash(self):
+    hash = hashlib.sha1()
+    for key, val in self.properties.iteritems():
+      hash.update('%s=%s' % (key, str(val)))
+    for child in self.Children():
+      hash.update(child.Hash())
+    if self.typeref: hash.update(self.typeref.hash)
+    self.hash = hash.hexdigest()
+    return self.hash
+
+#
+# IDL Predefined types
+#
+BuiltIn = set(['int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t', 'uint16_t',
+              'uint32_t', 'uint64_t', 'double_t', 'float_t', 'handle_t',
+              'mem_t', 'str_t', 'void', 'enum', 'struct', 'struct_by_value'])
+
+#
+# IDLAst
+#
+# A specialized version of the IDLNode for containing the whole of the
+# AST.  The specialized BuildTree function pulls the per file namespaces
+# into the global AST namespace and checks for collisions.
+#
+class IDLAst(IDLNode):
+  def __init__(self, children):
+    objs = [IDLNode('Type', name, 'BuiltIn', 1, 0, None) for name in BuiltIn]
+    IDLNode.__init__(self, 'AST', 'PPAPI', 'BuiltIn', 1, 0, objs + children)
+
+  def BuildTree(self, parent):
+    IDLNode.BuildTree(self, parent)
+    for fileobj in self.GetListOf('File'):
+      for name, val in fileobj.namespace.iteritems():
+        if name in self.namespace:
+          other = self.namespace[name]
+          val.Error('Attempting to add %s to namespace of %s when already '
+                    'declared in %s' % (name, str(self), str(other)))
+        self.namespace[name] = val
 
diff --git a/ppapi/generators/idl_parser.py b/ppapi/generators/idl_parser.py
index 476072f..5c33555 100644
--- a/ppapi/generators/idl_parser.py
+++ b/ppapi/generators/idl_parser.py
@@ -24,14 +24,14 @@
 
 
 import getopt
-import hashlib
+import glob
 import os.path
 import re
 import sys
 
 from idl_log import ErrOut, InfoOut, WarnOut
 from idl_lexer import IDLLexer
-from idl_node import IDLNode, IDLAttribute
+from idl_node import IDLAttribute, IDLAst, IDLNode
 
 from ply import lex
 from ply import yacc
@@ -116,6 +116,17 @@ def TokenTypeName(t):
   return 'keyword "%s"' % t.value
 
 
+# StageResult
+#
+# The result object stores the result of parsing stage.
+#
+class StageResult(object):
+  def __init__(self, name, out, errs):
+    self.name = name
+    self.out = out
+    self.errs = errs
+
+
 #
 # IDL Parser
 #
@@ -381,19 +392,27 @@ class IDLParser(IDLLexer):
 # Parameter List
 #
 # A parameter list is a collection of arguments which are passed to a
-# function.  In the case of a PPAPI, it is illegal to have a function
-# which passes no parameters.
+# function.
 #
-# NOTE:-We currently do not support functions which take no arguments in PPAPI.
   def p_param_list(self, p):
-    """param_list : modifiers typeref SYMBOL param_cont"""
+    """param_list : param_item param_cont
+                  | """
+    func = self.BuildExtAttribute('FUNCTION', 'True')
+    if len(p) > 1:
+      p[0] = ListFromConcat(p[1], p[2], func)
+    else:
+      p[0] = [func]
+    if self.parse_debug: DumpReduction('param_list', p)
+
+  def p_param_item(self, p):
+    """param_item : modifiers typeref SYMBOL param_cont"""
     children = ListFromConcat(p[1], p[2])
     param = self.BuildProduction('Param', p, 3, children)
     p[0] = ListFromConcat(param, p[4])
-    if self.parse_debug: DumpReduction('param_list', p)
+    if self.parse_debug: DumpReduction('param_item', p)
 
   def p_param_cont(self, p):
-    """param_cont : ',' param_list
+    """param_cont : ',' param_item param_cont
                   | """
     if len(p) > 1:
       p[0] = p[2]
@@ -632,6 +651,7 @@ class IDLParser(IDLLexer):
 #
   def ParseData(self):
     try:
+      self.parse_errors = 0
       return self.yaccobj.parse(lexer=self)
 
     except lex.LexError as le:
@@ -650,13 +670,13 @@ class IDLParser(IDLLexer):
       InfoOut.Log("Parsing %s" % filename)
     try:
       out = self.ParseData()
-      return out
+      return StageResult(filename, out, self.parse_errors)
 
     except Exception as e:
       ErrOut.LogLine(filename, self.last.lineno, self.last.lexpos,
                      'Internal parsing error - %s.' % str(e))
       raise
-      return []
+      return StageResult(filename, [], self.parse_errors)
 
 
 
@@ -680,7 +700,7 @@ def FlattenTree(node):
   return out
 
 
-def Test(filename, ast):
+def TestErrors(filename, nodelist):
   lexer = IDLLexer()
   data = open(filename).read()
   lexer.SetData(filename, data)
@@ -698,7 +718,7 @@ def Test(filename, ast):
         if args[1] == 'FAIL':
           fail_comments.append((tok.lineno, ' '.join(args[2:-1])))
   obj_list = []
-  for node in ast.Children():
+  for node in nodelist:
     obj_list.extend(FlattenTree(node))
 
   errors = 0
@@ -751,12 +771,82 @@ def Test(filename, ast):
   return errors
 
 
+def TestFile(parser, filename):
+  # Capture errors instead of reporting them so we can compare them
+  # with the expected errors.
+  ErrOut.SetConsole(False)
+  ErrOut.SetCapture(True)
+
+  result = parser.ParseFile(filename)
+
+  # Renable output
+  ErrOut.SetConsole(True)
+  ErrOut.SetCapture(False)
+
+  # Compare captured errors
+  return TestErrors(filename, result.out)
+
+
+def TestErrorFiles(options):
+  idldir = os.path.split(sys.argv[0])[0]
+  idldir = os.path.join(idldir, 'test_parser', '*.idl')
+  filenames = glob.glob(idldir)
+  parser = IDLParser(options)
+  total_errs = 0
+  for filename in filenames:
+    errs = TestFile(parser, filename)
+    if errs:
+      ErrOut.Log("%s test failed with %d error(s)." % (filename, errs))
+      total_errs += errs
+
+  if total_errs:
+    ErrOut.Log("Failed parsing test.")
+  else:
+    InfoOut.Log("Passed parsing test.")
+  return total_errs
+
+def TestNamespaceFiles(options):
+  idldir = os.path.split(sys.argv[0])[0]
+  idldir = os.path.join(idldir, 'test_namespace', '*.idl')
+  filenames = glob.glob(idldir)
+
+  InfoOut.SetConsole(False)
+  result = ParseFiles(filenames, options)
+  InfoOut.SetConsole(True)
+
+  if result.errs:
+    ErrOut.Log("Failed namespace test.")
+  else:
+    InfoOut.Log("Passed namespace test.")
+  return result.errs
+
+
+def ParseFiles(filenames, options):
+  parser = IDLParser(options)
+  filenodes = []
+  errors = 0
+
+  for filename in filenames:
+    result = parser.ParseFile(filename)
+    if result.errs:
+      ErrOut.Log("%d error(s) parsing %s." % (result.errs, filename))
+      errors += result.errs
+    else:
+      InfoOut.Log("Parsed %s." % filename)
+    filenode = IDLNode('File', filename, filename, 1, 0, result.out)
+    filenodes.append(filenode)
+
+  ast = IDLAst(filenodes)
+
+  # Build the links
+  ast.BuildTree(None)
+
+  # Resolve type references
+  errors += ast.Resolve()
+
+  ast.Resolve()
+  return StageResult('Parsing', ast, errors)
 
-def ParseFile(parser, filename, options):
-  InfoOut.Log('Parsing %s.' % filename)
-  children = parser.ParseFile(filename)
-  node = IDLNode('File', filename, filename, 1, 0, children)
-  return node
 
 def Main(args):
   global PARSER_OPTIONS
@@ -773,36 +863,22 @@ def Main(args):
   for opt, val in opts:
     PARSER_OPTIONS[opt[2:]] = True
 
-  parser = IDLParser(PARSER_OPTIONS)
-  total_errs = 0
-  for filename in filenames:
-    if PARSER_OPTIONS['test']:
-      ErrOut.SetConsole(False)
-      ErrOut.SetCapture(True)
-
-      ast = ParseFile(parser, filename, PARSER_OPTIONS)
+  if PARSER_OPTIONS['test']:
+    errs = TestErrorFiles(PARSER_OPTIONS)
+    errs = TestNamespaceFiles(PARSER_OPTIONS)
+    if errs:
+      ErrOut.Log("Parser failed with %d errors." % errs)
+      return  -1
 
-      ErrOut.SetConsole(True)
-      ErrOut.SetCapture(False)
+  result = ParseFiles(filenames, PARSER_OPTIONS)
+  if PARSER_OPTIONS['output']:
+    result.out.Dump(0)
 
-      errs = Test(filename, ast)
-      total_errs += errs
-      if errs:
-        ErrOut.Log("%s test failed with %d error(s)." % (filename, errs))
-    else:
-      ErrOut.SetConsole(True)
-      ast = ParseFile(parser, filename, PARSER_OPTIONS)
-
-    if PARSER_OPTIONS['output']:
-      ast.Dump(0, False)
-
-  if not PARSER_OPTIONS['test']:
-    total_errs = parser.parse_errors
-
-  InfoOut.Log("Parsed %d file(s) with %d error(s)." %
-      (len(filenames), total_errs))
-  return total_errs
+  if result.errs:
+    ErrOut.Log('Found %d error(s).' % result.errors);
 
+  InfoOut.Log("%d files processed." % len(filenames))
+  return result.errs
 
 if __name__ == '__main__':
   sys.exit(Main(sys.argv[1:]))
diff --git a/ppapi/generators/test_namespace/bar.idl b/ppapi/generators/test_namespace/bar.idl
new file mode 100644
index 0000000..ea73402
--- /dev/null
+++ b/ppapi/generators/test_namespace/bar.idl
@@ -0,0 +1,29 @@
+/* Copyright (c) 2011 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/* This file tests the namespace functions in the parser. */
+
+/* PPAPI ID */
+typedef int32_t PP_Instance;
+
+/* PPAPI ID */
+typedef int32_t PP_Resource;
+
+/* Interface test */
+interface PPB_Bar_0_3 {
+  /* Face create */
+  PP_Resource Create(
+      [in] PP_Instance instance,
+      [in] PP_Size size,
+      [in] PP_Bool is_always_opaque);
+
+  /* Returns PP_TRUE if the given resource is a valid Graphics2D, PP_FALSE if it
+   * is an invalid resource or is a resource of another type.
+   */
+  PP_Bool IsGraphics2D(
+      [in] PP_Resource resource);
+};
+
+
diff --git a/ppapi/generators/test_namespace/foo.idl b/ppapi/generators/test_namespace/foo.idl
new file mode 100644
index 0000000..75798a9
--- /dev/null
+++ b/ppapi/generators/test_namespace/foo.idl
@@ -0,0 +1,18 @@
+/* Copyright (c) 2011 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/* PPAPI Structure */
+struct PP_Size {
+  /* This value represents the width of the rectangle. */
+  int32_t width;
+  /* This value represents the height of the rectangle. */
+  int32_t height;
+};
+
+/* PPAPI Enum */
+enum PP_Bool {
+  PP_FALSE = 0,
+  PP_TRUE  = 1
+};
diff --git a/ppapi/generators/test_parser/typedef.idl b/ppapi/generators/test_parser/typedef.idl
index 6b48b58..92b220f 100644
--- a/ppapi/generators/test_parser/typedef.idl
+++ b/ppapi/generators/test_parser/typedef.idl
@@ -16,7 +16,7 @@ typedef int32_t[] T3;
 /* OK Typedef(T4) */
 typedef int32_t[][4] T4;
 
-/* FAIL Empty argument list. */
+/* OK Typedef(T5) */
 typedef int32_t[][4] T5();
 
 /* OK Typedef(T6) */