IDL Cleanup - add logger, and node

Cleanup parser by splitting out logging function and an AST node object.
Cleanup use of 'lineno' by starting the lexer on line 1 instead of line 0.

BUG= 77551
TEST= python idl_parser.py --test test_parser/*.idl
Review URL: http://codereview.chromium.org/6905006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@83049 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 248 insertions, 134 deletions

diff --git a/ppapi/generators/idl_lexer.py b/ppapi/generators/idl_lexer.py
index b8092ec..56594a2 100644
--- a/ppapi/generators/idl_lexer.py
+++ b/ppapi/generators/idl_lexer.py
@@ -155,7 +155,9 @@ class IDLLexer(object):
 
   def SourceLine(self, file, line, pos):
     caret = '\t^'.expandtabs(pos)
-    return "%s\n%s" % (self.lines[line], caret)
+    # We decrement the line number since the array is 0 based while the
+    # line numbers are 1 based.
+    return "%s\n%s" % (self.lines[line - 1], caret)
 
   def ErrorMessage(self, file, line, pos, msg):
     return "\n%s\n%s" % (
@@ -163,8 +165,9 @@ class IDLLexer(object):
         self.SourceLine(file, line, pos))
 
   def SetData(self, filename, data):
+    # Start with line 1, not zero
+    self.lexobj.lineno = 1
     self.lexobj.filename = filename
-    self.lexobj.lineno = 0
     self.lines = data.split('\n')
     self.index = [0]
     self.lexobj.input(data)
diff --git a/ppapi/generators/idl_log.py b/ppapi/generators/idl_log.py
new file mode 100644
index 0000000..7aa4510
--- /dev/null
+++ b/ppapi/generators/idl_log.py
@@ -0,0 +1,55 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Error and information logging for IDL """
+
+#
+# IDL Log
+#
+# And IDLLog object provides a mechanism for capturing logging output
+# and/or sending out via a file handle (usually stdout or stderr).
+#
+import sys
+
+class IDLLog(object):
+  def __init__(self, name, out):
+    if name:
+      self.name = '%s : ' % name
+    else:
+      self.name = ''
+
+    self.out = out
+    self.capture = False
+    self.console = True
+    self.log = []
+
+  def Log(self, msg):
+    line = "%s%s\n" % (self.name, msg)
+    if self.console: self.out.write(line)
+    if self.capture:
+      self.log.append(msg)
+
+  def LogLine(self, filename, lineno, pos, msg):
+    line = "%s(%d) : %s%s\n" % (filename, lineno, self.name, msg)
+    if self.console: self.out.write(line)
+    if self.capture: self.log.append(msg)
+
+  def SetConsole(self, enable, out = None):
+    self.console = enable
+    if out: self.out = out
+
+  def SetCapture(self, enable):
+    self.capture = enable
+
+  def DrainLog(self):
+    out = self.log
+    self.log = []
+    return out
+
+ErrOut  = IDLLog('Error', sys.stderr)
+WarnOut = IDLLog('Warning', sys.stdout)
+InfoOut = IDLLog('', sys.stdout)
+
diff --git a/ppapi/generators/idl_node.py b/ppapi/generators/idl_node.py
new file mode 100644
index 0000000..d54f1ad
--- /dev/null
+++ b/ppapi/generators/idl_node.py
@@ -0,0 +1,128 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Nodes for PPAPI IDL AST """
+
+#
+# IDL Node
+#
+# IDL Node defines the IDLAttribute and IDLNode objects which are constructed
+# by the parser as it processes the various 'productions'.  The IDLAttribute
+# objects are assigned to the IDLNode's property dictionary instead of being
+# applied as children of The IDLNodes, so they do not exist in the final tree.
+# The AST of IDLNodes is the output from the parsing state and will be used
+# as the source data by the various generators.
+#
+
+import sys
+import hashlib
+
+from idl_log import ErrOut, InfoOut, WarnOut
+
+#
+# IDLAttribute
+#
+# A temporary object used by the parsing process to hold an Extended Attribute
+# which will be passed as a child to a standard IDLNode.
+#
+class IDLAttribute(object):
+  def __init__(self, name, value):
+    self.cls = 'ExtAttribute'
+    self.name = name
+    self.value = value
+
+#
+# IDLNode
+#
+# A node in the AST.  The Node is composed of children, implemented as a
+# dictionary of lists of child types and a dictionary of local properties,
+# a.k.a. ExtendedAttributes.
+#
+class IDLNode(object):
+  def __init__(self, cls, name, filename, lineno, pos, children):
+    self.cls = cls
+    self.name = name
+    self.lineno = lineno
+    self.pos = pos
+    self.children = {}
+    self.properties = { 'NAME' : name }
+    self.hash = None
+    self.typeref = None
+    self.parent = None
+    if children:
+      for child in children:
+        #
+        # Copy children into local dictionary such that children of the
+        # same class are added in order to the per key list.  All
+        # ExtAttributes are filtered and applied to a property dictionary
+        # instead of becoming children of IDLNode
+        #
+        if child.cls == 'ExtAttribute':
+          self.properties[child.name] =  child.value
+        else:
+          if child.cls in self.children:
+            self.children[child.cls].append(child)
+          else:
+            self.children[child.cls] = [child]
+
+  # Return a string representation of this node
+  def __str__(self):
+    return "%s(%s)" % (self.cls, self.name)
+
+  # Log an error for this object
+  def Error(self, msg):
+    ErrOut.LogLine(self.filename, self.lineno, 0, " %s %s" %
+                   (self.FullName(), msg))
+
+  # Log a warning for this object
+  def Warning(self, msg):
+    WarnOut.LogLine(self.filename, self.lineno, 0, " %s %s" %
+                    (self.FullName(), msg))
+
+  # Get a list of objects for this key
+  def GetListOf(self, key):
+    return self.children.get(key, [])
+
+  # Get a list of all objects
+  def Children(self):
+    out = []
+    for key in sorted(self.children.keys()):
+      out.extend(self.children[key])
+    return out
+
+  # Dump this object and its children
+  def Dump(self, depth, comments = False, out=sys.stdout):
+    if self.cls == 'Comment' or self.cls == 'Copyright':
+      is_comment = True
+    else:
+      is_comment = False
+
+    # Skip this node if it's a comment, and we are not printing comments
+    if not comments and is_comment: return
+
+    tab = ""
+    for t in range(depth):
+      tab += '    '
+
+    if is_comment:
+      for line in self.name.split('\n'):
+        out.write("%s%s\n" % (tab, line))
+    else:
+      out.write("%s%s\n" % (tab, self))
+
+    if self.properties:
+      out.write("%s  Properties\n" % tab)
+      for p in self.properties:
+        out.write("%s    %s : %s\n" % (tab, p, self.properties[p]))
+
+    for cls in sorted(self.children.keys()):
+      # Skip comments
+      if (cls == 'Comment' or cls == 'Copyright') and not comments: continue
+
+      out.write("%s  %ss\n" % (tab, cls))
+      for c in self.children[cls]:
+        c.Dump(depth + 1, comments = comments, out=out)
+
diff --git a/ppapi/generators/idl_parser.py b/ppapi/generators/idl_parser.py
index 2d681d2..476072f 100644
--- a/ppapi/generators/idl_parser.py
+++ b/ppapi/generators/idl_parser.py
@@ -24,11 +24,15 @@
 
 
 import getopt
+import hashlib
 import os.path
 import re
 import sys
 
+from idl_log import ErrOut, InfoOut, WarnOut
 from idl_lexer import IDLLexer
+from idl_node import IDLNode, IDLAttribute
+
 from ply import lex
 from ply import yacc
 
@@ -63,12 +67,12 @@ ERROR_REMAP = {
 # new item or set it was reduced to.
 def DumpReduction(cls, p):
   if p[0] is None:
-    PrintInfo("OBJ: %s(%d) - None\n" % (cls, len(p)))
+    InfoOut.Log("OBJ: %s(%d) - None\n" % (cls, len(p)))
   else:
     out = ""
     for index in range(len(p) - 1):
       out += " >%s< " % str(p[index + 1])
-    PrintInfo("OBJ: %s(%d) - %s : %s\n"  % (cls, len(p), str(p[0]), out))
+    InfoOut.Log("OBJ: %s(%d) - %s : %s\n"  % (cls, len(p), str(p[0]), out))
 
 
 # CopyToList
@@ -112,92 +116,6 @@ def TokenTypeName(t):
   return 'keyword "%s"' % t.value
 
 
-# Send a string to stdout
-def PrintInfo(text):
-  sys.stdout.write("%s\n" % text)
-
-def PrintError(text):
-  sys.stderr.write("%s\n" % text)
-
-# Send a string to stderr containing a file, line number and error message
-def LogError(filename, lineno, pos, msg):
-  PrintError("%s(%d) : %s\n" % (filename, lineno + 1, msg))
-
-
-#
-# IDLAttribute
-#
-# A temporary object used by the parsing process to hold and Extended Attribute
-# which will be passed as a child to a standard IDLNode.
-#
-class IDLAttribute(object):
-  def __init__(self, name, value):
-    self.cls = 'ExtAttribute'
-    self.name = name
-    self.value = value
-
-#
-# IDLNode
-#
-# A node in the AST.  The Node is composed of children, implemented as a
-# dictionary of lists of child types and a dictionary of local properties.
-# a.k.a. ExtendedAttributes.
-#
-class IDLNode(object):
-  def __init__(self, cls, name, filename, lineno, pos, children):
-    self.cls = cls
-    self.name = name
-    self.lineno = lineno
-    self.pos = pos
-    self.children = {}
-    self.properties = {}
-    if children:
-      for child in children:
-        if child.cls == 'ExtAttribute':
-          self.properties[child.name] =  child.value
-        else:
-          if child.cls in self.children:
-            self.children[child.cls].append(child)
-          else:
-            self.children[child.cls] = [child]
-
-  def __str__(self):
-    return "%s(%s)" % (self.cls, self.name)
-
-  def Dump(self, depth, comments = False, out=sys.stdout):
-    if self.cls == 'Comment' or self.cls == 'Copyright':
-      is_comment = True
-    else:
-      is_comment = False
-
-    # Skip this node if it's a comment, and we are not printing comments
-    if not comments and is_comment: return
-
-    tab = ""
-    for t in range(depth):
-      tab += '    '
-
-    if is_comment:
-      for line in self.name.split('\n'):
-        out.write("%s%s\n" % (tab, line))
-    else:
-      out.write("%s%s\n" % (tab, self))
-
-    if self.properties:
-      out.write("%s  Properties\n" % tab)
-      for p in self.properties:
-        out.write("%s    %s : %s\n" % (tab, p, self.properties[p]))
-
-    for cls in sorted(self.children.keys()):
-      # Skip comments
-      if (cls == 'Comment' or cls == 'Copyright') and not comments: continue
-
-      out.write("%s  %ss\n" % (tab, cls))
-      for c in self.children[cls]:
-        c.Dump(depth + 1, comments = comments, out=out)
-
-
-
 #
 # IDL Parser
 #
@@ -603,8 +521,7 @@ class IDLParser(IDLLexer):
 
   def p_typedef_func(self, p):
     """typedef_def : modifiers TYPEDEF typeref SYMBOL '(' param_list ')' ';'"""
-    params = self.BuildProduction('Callspec', p, 5, p[6])
-    children = ListFromConcat(p[1], p[3], params)
+    children = ListFromConcat(p[1], p[3], p[6])
     p[0] = self.BuildProduction('Typedef', p, 4, children)
     if self.parse_debug: DumpReduction('typedef_func', p)
 
@@ -641,10 +558,11 @@ class IDLParser(IDLLexer):
       msg = ERROR_REMAP[msg]
 
     # Log the error
-    self.Logger(filename, lineno, pos, msg)
+    ErrOut.LogLine(filename, lineno, pos, msg)
+    self.parse_errors += 1
 
 
-  def __init__(self, logger, options = {}):
+  def __init__(self, options = {}):
     global PARSER_OPTIONS
 
     IDLLexer.__init__(self, options)
@@ -658,7 +576,6 @@ class IDLParser(IDLLexer):
     self.parse_debug = PARSER_OPTIONS['parse_debug']
     self.token_debug = PARSER_OPTIONS['token_debug']
     self.verbose = PARSER_OPTIONS['verbose']
-    self.Logger = logger
     self.parse_errors = 0
 
 #
@@ -672,7 +589,7 @@ class IDLParser(IDLLexer):
     if tok:
       self.last = tok
       if self.token_debug:
-        PrintInfo("TOKEN %s(%s)" % (tok.type, tok.value))
+        InfoOut.Log("TOKEN %s(%s)" % (tok.type, tok.value))
     return tok
 
 #
@@ -691,7 +608,7 @@ class IDLParser(IDLLexer):
     lineno = p.lineno(index)
     pos = p.lexpos(index)
     if self.build_debug:
-      PrintInfo("Building %s(%s)" % (cls, name))
+      InfoOut.Log("Building %s(%s)" % (cls, name))
     out = IDLNode(cls, name, filename, lineno, pos, childlist)
     return out
 
@@ -704,7 +621,7 @@ class IDLParser(IDLLexer):
 #
   def BuildExtAttribute(self, name, value):
     if self.build_debug:
-      PrintInfo("Adding ExtAttribute %s = %s" % (name, str(value)))
+      InfoOut.Log("Adding ExtAttribute %s = %s" % (name, str(value)))
     out = IDLAttribute(name, value)
     return out
 
@@ -718,7 +635,7 @@ class IDLParser(IDLLexer):
       return self.yaccobj.parse(lexer=self)
 
     except lex.LexError as le:
-      PrintError(str(le))
+      ErrOut.Log(str(le))
       return []
 
 #
@@ -730,14 +647,14 @@ class IDLParser(IDLLexer):
     data = open(filename).read()
     self.SetData(filename, data)
     if self.verbose:
-      PrintInfo("Parsing %s" % filename)
+      InfoOut.Log("Parsing %s" % filename)
     try:
       out = self.ParseData()
       return out
 
     except Exception as e:
-      LogError(filename, self.last.lineno, self.last.lexpos,
-               'Internal parsing error - %s.' % str(e))
+      ErrOut.LogLine(filename, self.last.lineno, self.last.lexpos,
+                     'Internal parsing error - %s.' % str(e))
       raise
       return []
 
@@ -763,18 +680,7 @@ def FlattenTree(node):
   return out
 
 
-err_list = []
-def TestLog(filename, lineno, pos, msg):
-  global err_list
-  global PARSER_OPTIONS
-
-  err_list.append(msg)
-  if PARSER_OPTIONS['verbose']:
-    PrintInfo("%s(%d) : %s\n" % (filename, lineno + 1, msg))
-
-
-def Test(filename, nodes):
-  global err_list
+def Test(filename, ast):
   lexer = IDLLexer()
   data = open(filename).read()
   lexer.SetData(filename, data)
@@ -792,7 +698,7 @@ def Test(filename, nodes):
         if args[1] == 'FAIL':
           fail_comments.append((tok.lineno, ' '.join(args[2:-1])))
   obj_list = []
-  for node in nodes:
+  for node in ast.Children():
     obj_list.extend(FlattenTree(node))
 
   errors = 0
@@ -803,37 +709,40 @@ def Test(filename, nodes):
   obj_cnt = len(obj_list)
   pass_cnt = len(pass_comments)
   if obj_cnt != pass_cnt:
-    PrintInfo("Mismatched pass (%d) vs. nodes built (%d)."
+    InfoOut.Log("Mismatched pass (%d) vs. nodes built (%d)."
         % (pass_cnt, obj_cnt))
-    PrintInfo("PASS: %s" % [x[1] for x in pass_comments])
-    PrintInfo("OBJS: %s" % obj_list)
+    InfoOut.Log("PASS: %s" % [x[1] for x in pass_comments])
+    InfoOut.Log("OBJS: %s" % obj_list)
     errors += 1
     if pass_cnt > obj_cnt: pass_cnt = obj_cnt
 
   for i in range(pass_cnt):
     line, comment = pass_comments[i]
     if obj_list[i] != comment:
-      PrintError("%s(%d) Error: OBJ %s : EXPECTED %s\n" % (
-        filename, line, obj_list[i], comment))
+      ErrOut.LogLine(filename, line, None, "OBJ %s : EXPECTED %s\n" %
+                     (obj_list[i], comment))
       errors += 1
 
   #
   # Check for expected errors
   #
+  err_list = ErrOut.DrainLog()
   err_cnt = len(err_list)
   fail_cnt = len(fail_comments)
   if err_cnt != fail_cnt:
-    PrintInfo("Mismatched fail (%d) vs. errors seen (%d)."
+    InfoOut.Log("Mismatched fail (%d) vs. errors seen (%d)."
         % (fail_cnt, err_cnt))
-    PrintInfo("FAIL: %s" % [x[1] for x in fail_comments])
-    PrintInfo("ERRS: %s" % err_list)
+    InfoOut.Log("FAIL: %s" % [x[1] for x in fail_comments])
+    InfoOut.Log("ERRS: %s" % err_list)
     errors += 1
     if fail_cnt > err_cnt:  fail_cnt = err_cnt
 
   for i in range(fail_cnt):
     line, comment = fail_comments[i]
+    err = err_list[i].strip()
+
     if err_list[i] != comment:
-      PrintError("%s(%d) Error\n\tERROR : %s\n\tEXPECT: %s" % (
+      ErrOut.Log("%s(%d) Error\n\tERROR : %s\n\tEXPECT: %s" % (
         filename, line, err_list[i], comment))
       errors += 1
 
@@ -842,6 +751,13 @@ def Test(filename, nodes):
   return errors
 
 
+
+def ParseFile(parser, filename, options):
+  InfoOut.Log('Parsing %s.' % filename)
+  children = parser.ParseFile(filename)
+  node = IDLNode('File', filename, filename, 1, 0, children)
+  return node
+
 def Main(args):
   global PARSER_OPTIONS
 
@@ -851,28 +767,40 @@ def Main(args):
     opts, filenames = getopt.getopt(args, '', long_opts)
 
   except getopt.error, e:
-    PrintError('Illegal option: %s\n\t%s' % (str(e) % usage))
+    ErrOut.Log('Illegal option: %s\n\t%s' % (str(e) % usage))
     return 1
 
   for opt, val in opts:
     PARSER_OPTIONS[opt[2:]] = True
 
-  parser = IDLParser(TestLog, PARSER_OPTIONS)
-
+  parser = IDLParser(PARSER_OPTIONS)
   total_errs = 0
   for filename in filenames:
-    tokens = parser.ParseFile(filename)
     if PARSER_OPTIONS['test']:
-      errs = Test(filename, tokens)
+      ErrOut.SetConsole(False)
+      ErrOut.SetCapture(True)
+
+      ast = ParseFile(parser, filename, PARSER_OPTIONS)
+
+      ErrOut.SetConsole(True)
+      ErrOut.SetCapture(False)
+
+      errs = Test(filename, ast)
       total_errs += errs
       if errs:
-        PrintError("%s test failed with %d error(s)." % (filename, errs))
-      else:
-        PrintInfo("%s passed." % filename)
+        ErrOut.Log("%s test failed with %d error(s)." % (filename, errs))
+    else:
+      ErrOut.SetConsole(True)
+      ast = ParseFile(parser, filename, PARSER_OPTIONS)
+
     if PARSER_OPTIONS['output']:
-      for token in tokens:
-         token.Dump(0, False)
+      ast.Dump(0, False)
+
+  if not PARSER_OPTIONS['test']:
+    total_errs = parser.parse_errors
 
+  InfoOut.Log("Parsed %d file(s) with %d error(s)." %
+      (len(filenames), total_errs))
   return total_errs