Add label, inline, and ppapi basic types

This CL adds the basic pepper types (pp_stdint.idl) into the compiler as a primitive type.  Adds idl_ppapi_parser.py.

Moves test_parser/*.idl to test_parser/*_web.idl
(These files are unchanged)

Addes test_parser/*_ppapi.idl to test pepper specific productions.

BUG=224150
R=nfullagar@chromium.org

Review URL: https://codereview.chromium.org/18191014

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@209339 0039d316-1c4b-4281-b951-d872f2087c98

15 files changed, 268 insertions, 23 deletions

diff --git a/tools/idl_parser/idl_lexer.py b/tools/idl_parser/idl_lexer.py
index 85ad7e2..9b92b3ec 100755
--- a/tools/idl_parser/idl_lexer.py
+++ b/tools/idl_parser/idl_lexer.py
@@ -257,6 +257,11 @@ class IDLLexer(object):
       self._AddToken(value)
       self.keywords[key] = value
 
+  def _DelKeywords(self, keywords):
+    for key in keywords:
+      self.tokens.remove(key.upper())
+      del self.keywords[key]
+
   def __init__(self):
     self.index = [0]
     self._lex_errors = 0
diff --git a/tools/idl_parser/idl_parser.py b/tools/idl_parser/idl_parser.py
index 3e57295..22f7262 100755
--- a/tools/idl_parser/idl_parser.py
+++ b/tools/idl_parser/idl_parser.py
@@ -751,7 +751,7 @@ class IDLParser(object):
     """TypeSuffix : '[' integer ']' TypeSuffix
                   | '[' ']' TypeSuffix
                   | '?' TypeSuffixStartingWithArray
-                  |"""
+                  | """
     if len(p) == 5:
       p[0] = self.BuildNamed('Array', p, 2, p[4])
 
@@ -981,7 +981,7 @@ class IDLParser(object):
 
     try:
       self.lexer.Tokenize(data, filename)
-      nodes = self.yaccobj.parse(lexer=self.lexer)
+      nodes = self.yaccobj.parse(lexer=self.lexer) or []
       name = self.BuildAttribute('NAME', filename)
       return IDLNode('File', filename, 0, 0, nodes + [name])
 
@@ -1002,7 +1002,7 @@ def ParseFile(parser, filename):
 
     except Exception as e:
       last = parser.LastToken()
-      sys.stderr.write('%s(%d) : Internal parsing error - %s.' % (
+      sys.stderr.write('%s(%d) : Internal parsing error\n\t%s.\n' % (
                        filename, last.lineno, str(e)))
 
 
@@ -1012,8 +1012,9 @@ def main(argv):
   errors = 0
   for filename in argv:
     filenode = ParseFile(parser, filename)
-    errors += filenode.GetProperty('ERRORS')
-    nodes.append(filenode)
+    if (filenode):
+      errors += filenode.GetProperty('ERRORS')
+      nodes.append(filenode)
 
   ast = IDLNode('AST', '__AST__', 0, 0, nodes)
 
@@ -1021,9 +1022,8 @@ def main(argv):
   if errors:
     print '\nFound %d errors.\n' % errors
 
-
   return errors
 
 
 if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
\ No newline at end of file
+  sys.exit(main(sys.argv[1:]))
diff --git a/tools/idl_parser/idl_parser_test.py b/tools/idl_parser/idl_parser_test.py
index 5b7669e..ad48f23 100755
--- a/tools/idl_parser/idl_parser_test.py
+++ b/tools/idl_parser/idl_parser_test.py
@@ -8,6 +8,8 @@ import unittest
 
 from idl_lexer import IDLLexer
 from idl_parser import IDLParser, ParseFile
+from idl_ppapi_lexer import IDLPPAPILexer
+from idl_ppapi_parser import IDLPPAPIParser
 
 def ParseCommentTest(comment):
   comment = comment.strip()
@@ -18,7 +20,7 @@ def ParseCommentTest(comment):
 class WebIDLParser(unittest.TestCase):
   def setUp(self):
     self.parser = IDLParser(IDLLexer(), mute_error=True)
-    self.filenames = glob.glob('test_parser/*.idl')
+    self.filenames = glob.glob('test_parser/*_web.idl')
 
   def _TestNode(self, node):
     comments = node.GetListOf('Comment')
@@ -56,6 +58,46 @@ class WebIDLParser(unittest.TestCase):
         self._TestNode(node)
 
 
+class PepperIDLParser(unittest.TestCase):
+  def setUp(self):
+    self.parser = IDLPPAPIParser(IDLPPAPILexer(), mute_error=True)
+    self.filenames = glob.glob('test_parser/*_ppapi.idl')
+
+  def _TestNode(self, node):
+    comments = node.GetListOf('Comment')
+    for comment in comments:
+      check, value = ParseCommentTest(comment.GetName())
+      if check == 'BUILD':
+        msg = 'Expecting %s, but found %s.\n' % (value, str(node))
+        self.assertEqual(value, str(node), msg)
+
+      if check == 'ERROR':
+        msg = node.GetLogLine('Expecting\n\t%s\nbut found \n\t%s\n' % (
+                              value, str(node)))
+        self.assertEqual(value, node.GetName(), msg)
+
+      if check == 'PROP':
+        key, expect = value.split('=')
+        actual = str(node.GetProperty(key))
+        msg = 'Mismatched property %s: %s vs %s.\n' % (key, expect, actual)
+        self.assertEqual(expect, actual, msg)
+
+      if check == 'TREE':
+        quick = '\n'.join(node.Tree())
+        lineno = node.GetProperty('LINENO')
+        msg = 'Mismatched tree at line %d:\n%sVS\n%s' % (lineno, value, quick)
+        self.assertEqual(value, quick, msg)
+
+  def testExpectedNodes(self):
+    for filename in self.filenames:
+      filenode = ParseFile(self.parser, filename)
+      children = filenode.GetChildren()
+      self.assertTrue(len(children) > 2, 'Expecting children in %s.' %
+                      filename)
+
+      for node in filenode.GetChildren()[2:]:
+        self._TestNode(node)
+
 if __name__ == '__main__':
   unittest.main(verbosity=2)
 
diff --git a/tools/idl_parser/idl_ppapi_lexer.py b/tools/idl_parser/idl_ppapi_lexer.py
index f615c77..9261399 100644..100755
--- a/tools/idl_parser/idl_ppapi_lexer.py
+++ b/tools/idl_parser/idl_ppapi_lexer.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -40,10 +41,28 @@ class IDLPPAPILexer(IDLLexer):
   # Return a "preprocessor" inline block
   def __init__(self):
     IDLLexer.__init__(self)
-    self._AddTokens(['LSHIFT', 'RSHIFT', 'INLINE'])
-    self._AddKeywords(['label', 'namespace', 'struct'])
+    self._AddTokens(['INLINE', 'LSHIFT', 'RSHIFT'])
+    self._AddKeywords(['label'])
+
+    # Add number types
+    self._AddKeywords(['char', 'int8_t', 'int16_t', 'int32_t', 'int64_t']);
+    self._AddKeywords(['uint8_t', 'uint16_t', 'uint32_t', 'uint64_t']);
+    self._AddKeywords(['double_t', 'float_t']);
+
+    # Add handle types
+    self._AddKeywords(['handle_t', 'PP_FileHandle']);
+
+    # Add pointer types (void*, char*, const char*, const void*)
+    self._AddKeywords(['mem_t', 'str_t', 'cstr_t', 'interface_t']);
+
+    # Remove JS types
+    self._DelKeywords(['boolean', 'byte', 'Date', 'DOMString', 'double',
+                       'float', 'long', 'object', 'octet', 'short', 'unsigned'])
 
 
 # If run by itself, attempt to build the lexer
 if __name__ == '__main__':
   lexer = IDLPPAPILexer()
+  lexer.Tokenize(open('test_parser/inline_ppapi.idl').read())
+  for tok in lexer.GetTokens():
+    print '\n' + str(tok)
\ No newline at end of file
diff --git a/tools/idl_parser/test_lexer/keywords_ppapi.in b/tools/idl_parser/test_lexer/keywords_ppapi.in
index e155753..62567e4 100644
--- a/tools/idl_parser/test_lexer/keywords_ppapi.in
+++ b/tools/idl_parser/test_lexer/keywords_ppapi.in
@@ -1,17 +1,11 @@
 ANY any
 ATTRIBUTE attribute
-BOOLEAN boolean
-BYTE byte
 CALLBACK callback
 CONST const
 CREATOR creator
-DATE Date
 DELETER deleter
 DICTIONARY dictionary
-DOMSTRING DOMString
-DOUBLE double
 FALSE false
-FLOAT float
 EXCEPTION exception
 GETTER getter
 IMPLEMENTS implements
@@ -19,22 +13,32 @@ INFINITY Infinity
 INTERFACE interface
 LABEL label
 LEGACYCALLER legacycaller
-LONG long
-NAMESPACE namespace
 NAN Nan
 NULL null
-OBJECT object
 OPTIONAL optional
 OR or
 PARTIAL partial
 READONLY readonly
 SETTER setter
-SHORT short
 STATIC static
 STRINGIFIER stringifier
-STRUCT struct
 TYPEDEF typedef
 TRUE true
-UNSIGNED unsigned
-UNRESTRICTED unrestricted
 VOID void
+CHAR char
+INT8_T int8_t
+INT16_T int16_t
+INT32_T int32_t
+INT64_T int64_t
+UINT8_T uint8_t
+UINT16_T uint16_t
+UINT32_T uint32_t
+UINT64_T uint64_t
+DOUBLE_T double_t
+FLOAT_T float_t
+MEM_T mem_t
+STR_T str_t
+CSTR_T cstr_t
+INTERFACE_T interface_t
+HANDLE_T handle_t
+PP_FILEHANDLE PP_FileHandle
\ No newline at end of file
diff --git a/tools/idl_parser/test_parser/callback.idl b/tools/idl_parser/test_parser/callback_web.idl
index b16b6b5..b16b6b5 100644
--- a/tools/idl_parser/test_parser/callback.idl
+++ b/tools/idl_parser/test_parser/callback_web.idl
diff --git a/tools/idl_parser/test_parser/dictionary.idl b/tools/idl_parser/test_parser/dictionary_web.idl
index 5030686..5030686 100644
--- a/tools/idl_parser/test_parser/dictionary.idl
+++ b/tools/idl_parser/test_parser/dictionary_web.idl
diff --git a/tools/idl_parser/test_parser/enum.idl b/tools/idl_parser/test_parser/enum_web.idl
index 233410c..233410c 100644
--- a/tools/idl_parser/test_parser/enum.idl
+++ b/tools/idl_parser/test_parser/enum_web.idl
diff --git a/tools/idl_parser/test_parser/exception.idl b/tools/idl_parser/test_parser/exception_web.idl
index 3801a4a..3801a4a 100644
--- a/tools/idl_parser/test_parser/exception.idl
+++ b/tools/idl_parser/test_parser/exception_web.idl
diff --git a/tools/idl_parser/test_parser/implements.idl b/tools/idl_parser/test_parser/implements_web.idl
index 252dd4b..252dd4b 100644
--- a/tools/idl_parser/test_parser/implements.idl
+++ b/tools/idl_parser/test_parser/implements_web.idl
diff --git a/tools/idl_parser/test_parser/inline_ppapi.idl b/tools/idl_parser/test_parser/inline_ppapi.idl
new file mode 100644
index 0000000..134f60d
--- /dev/null
+++ b/tools/idl_parser/test_parser/inline_ppapi.idl
@@ -0,0 +1,46 @@
+/* Copyright (c) 2013 The Chromium Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file. */
+
+/* Test Typedef productions
+
+Run with --test to generate an AST and verify that all comments accurately
+reflect the state of the Nodes.
+
+BUILD Type(Name)
+This comment signals that a node of type <Type> is created with the
+name <Name>.
+
+ERROR Error String
+This comment signals that a error of <Error String> is generated.  The error
+is not assigned to a node, but are expected in order.
+
+PROP Key=Value
+This comment signals that a property has been set on the Node such that
+<Key> = <Value>.
+
+TREE
+Type(Name)
+  Type(Name)
+  Type(Name)
+    Type(Name)
+    ...
+This comment signals that a tree of nodes matching the BUILD comment
+symatics should exist.  This is an exact match.
+*/
+
+/* TREE
+ *Inline(C)
+ */
+
+#inline C
+This is my block of C code
+#endinl
+
+/* TREE
+ *Inline(CC)
+ */
+#inline CC
+This is my block of CC code
+#endinl
+
diff --git a/tools/idl_parser/test_parser/interface.idl b/tools/idl_parser/test_parser/interface_web.idl
index a8a4135..a8a4135 100644
--- a/tools/idl_parser/test_parser/interface.idl
+++ b/tools/idl_parser/test_parser/interface_web.idl
diff --git a/tools/idl_parser/test_parser/label_ppapi.idl b/tools/idl_parser/test_parser/label_ppapi.idl
new file mode 100644
index 0000000..264699d
--- /dev/null
+++ b/tools/idl_parser/test_parser/label_ppapi.idl
@@ -0,0 +1,48 @@
+/* Copyright (c) 2013 The Chromium Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file. */
+
+/* Test Typedef productions
+
+Run with --test to generate an AST and verify that all comments accurately
+reflect the state of the Nodes.
+
+BUILD Type(Name)
+This comment signals that a node of type <Type> is created with the
+name <Name>.
+
+ERROR Error String
+This comment signals that a error of <Error String> is generated.  The error
+is not assigned to a node, but are expected in order.
+
+PROP Key=Value
+This comment signals that a property has been set on the Node such that
+<Key> = <Value>.
+
+TREE
+Type(Name)
+  Type(Name)
+  Type(Name)
+    Type(Name)
+    ...
+This comment signals that a tree of nodes matching the BUILD comment
+symatics should exist.  This is an exact match.
+*/
+
+/* TREE
+ *Label(Chrome1)
+ *  LabelItem(M13)
+ */
+label Chrome1 {
+  M13 = 0.0
+};
+
+/* TREE
+ *Label(Chrome2)
+ *  LabelItem(M12)
+ *  LabelItem(M13)
+ */
+label Chrome2 {
+  M12 = 1.0,
+  M13 = 2.0,
+};
\ No newline at end of file
diff --git a/tools/idl_parser/test_parser/typedef_ppapi.idl b/tools/idl_parser/test_parser/typedef_ppapi.idl
new file mode 100644
index 0000000..72ac110
--- /dev/null
+++ b/tools/idl_parser/test_parser/typedef_ppapi.idl
@@ -0,0 +1,81 @@
+/* Copyright (c) 2013 The Chromium Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file. */
+
+/* Test Typedef productions
+
+Run with --test to generate an AST and verify that all comments accurately
+reflect the state of the Nodes.
+
+BUILD Type(Name)
+This comment signals that a node of type <Type> is created with the
+name <Name>.
+
+ERROR Error String
+This comment signals that a error of <Error String> is generated.  The error
+is not assigned to a node, but are expected in order.
+
+PROP Key=Value
+This comment signals that a property has been set on the Node such that
+<Key> = <Value>.
+
+TREE
+Type(Name)
+  Type(Name)
+  Type(Name)
+    Type(Name)
+    ...
+This comment signals that a tree of nodes matching the BUILD comment
+symatics should exist.  This is an exact match.
+*/
+
+/* TREE
+ *Callback(foo)
+ *  Type()
+ *    PrimitiveType(void)
+ *  Arguments()
+ *    Argument(x)
+ *      Type()
+ *        PrimitiveType(int32_t)
+ */
+callback foo = void (int32_t x);
+
+/* TREE
+ *Typedef(MyLong)
+ *  Type()
+ *    PrimitiveType(int32_t)
+ */
+typedef int32_t MyLong;
+
+/* TREE
+ *Typedef(MyLongArray)
+ *  Type()
+ *    PrimitiveType(str_t)
+ *    Array()
+ */
+typedef str_t[] MyLongArray;
+
+/* TREE
+ *Typedef(MyLongArray5)
+ *  Type()
+ *    PrimitiveType(mem_t)
+ *    Array(5)
+ */
+typedef mem_t[5] MyLongArray5;
+
+/* TREE
+ *Typedef(MyLongArrayN5)
+ *  Type()
+ *    PrimitiveType(handle_t)
+ *    Array()
+ *      Array(5)
+ */
+typedef handle_t[][5] MyLongArrayN5;
+
+
+/* TREE
+ *Typedef(bar)
+ *  Type()
+ *    Typeref(foo)
+ */
+typedef foo bar;
\ No newline at end of file
diff --git a/tools/idl_parser/test_parser/typedef.idl b/tools/idl_parser/test_parser/typedef_web.idl
index ba95db7c..ba95db7c 100644
--- a/tools/idl_parser/test_parser/typedef.idl
+++ b/tools/idl_parser/test_parser/typedef_web.idl