IDL parser: fix lint errors and PPAPI ExtAttr grammar (+ test)

This fixes Pylint errors in the IDL parser.
It also fixes the grammar for PPAPI-specific extended attributes,
and adds tests.

Beyond beautification, this allows us to use Pylint in future
(skipping long line checks).

Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=216379

Review URL: https://chromiumcodereview.appspot.com/22411002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@217432 0039d316-1c4b-4281-b951-d872f2087c98

6 files changed, 154 insertions, 43 deletions

diff --git a/tools/idl_parser/idl_lexer.py b/tools/idl_parser/idl_lexer.py
index c2569e9..d7311df 100755
--- a/tools/idl_parser/idl_lexer.py
+++ b/tools/idl_parser/idl_lexer.py
@@ -14,7 +14,6 @@ PLY can be found at:
    http://www.dabeaz.com/ply/
 """
 
-import optparse
 import os.path
 import sys
 
@@ -26,7 +25,7 @@ try:
   # Disable lint check which fails to find the ply module.
   # pylint: disable=F0401
   from ply import lex
-except:
+except ImportError:
   module_path, module_name = os.path.split(__file__)
   third_party = os.path.join(module_path, '..', '..', 'third_party')
   sys.path.append(third_party)
@@ -37,6 +36,13 @@ except:
 # IDL Lexer
 #
 class IDLLexer(object):
+  # 'literals' is a value expected by lex which specifies a list of valid
+  # literal tokens, meaning the token type and token value are identical.
+  literals = r'"*.(){}[],;:=+-/~|&^?<>'
+
+  # 't_ignore' contains ignored characters (spaces and tabs)
+  t_ignore = ' \t'
+
   # 'tokens' is a value required by lex which specifies the complete list
   # of valid token types.
   tokens = [
@@ -106,10 +112,15 @@ class IDLLexer(object):
   # regular expression where a match will emit a token of type <TYPE>.  In the
   # case of a function, the function is called when a match is made. These
   # definitions come from WebIDL.
+  #
+  # These need to be methods for lexer construction, despite not using self.
+  # pylint: disable=R0201
   def t_ELLIPSIS(self, t):
     r'\.\.\.'
     return t
 
+  # Regex needs to be in the docstring
+  # pylint: disable=C0301
   def t_float(self, t):
     r'-?(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)([Ee][+-]?[0-9]+)?|[0-9]+[Ee][+-]?[0-9]+)'
     return t
@@ -236,12 +247,6 @@ class IDLLexer(object):
       self._lexobj = lex.lex(object=self, lextab=None, optimize=0)
     return self._lexobj
 
-  def _AddConstDefs(self):
-    # 'literals' is a value expected by lex which specifies a list of valid
-    # literal tokens, meaning the token type and token value are identical.
-    self.literals = r'"*.(){}[],;:=+-/~|&^?<>'
-    self.t_ignore = ' \t'
-
   def _AddToken(self, token):
     if token in self.tokens:
       raise RuntimeError('Same token: ' + token)
@@ -269,11 +274,12 @@ class IDLLexer(object):
     self.filename = None
     self.keywords = {}
     self.tokens = []
-    self._AddConstDefs()
     self._AddTokens(IDLLexer.tokens)
     self._AddKeywords(IDLLexer.keywords)
     self._lexobj = None
+    self.last = None
+    self.lines = None
 
 # If run by itself, attempt to build the lexer
 if __name__ == '__main__':
-  lexer = IDLLexer()
+  lexer_object = IDLLexer()
diff --git a/tools/idl_parser/idl_lexer_test.py b/tools/idl_parser/idl_lexer_test.py
index cba4e48..8b20da8 100755
--- a/tools/idl_parser/idl_lexer_test.py
+++ b/tools/idl_parser/idl_lexer_test.py
@@ -3,10 +3,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import json
-import optparse
-import os
-import sys
 import unittest
 
 from idl_lexer import IDLLexer
@@ -100,4 +96,4 @@ class PepperIDLLexer(WebIDLLexer):
 
 
 if __name__ == '__main__':
-  unittest.main()
\ No newline at end of file
+  unittest.main()
diff --git a/tools/idl_parser/idl_parser.py b/tools/idl_parser/idl_parser.py
index 10428de..8a682ea 100755
--- a/tools/idl_parser/idl_parser.py
+++ b/tools/idl_parser/idl_parser.py
@@ -36,8 +36,22 @@ import time
 from idl_lexer import IDLLexer
 from idl_node import IDLAttribute, IDLNode
 
-from ply import lex
-from ply import yacc
+#
+# Try to load the ply module, if not, then assume it is in the third_party
+# directory.
+#
+try:
+  # Disable lint check which fails to find the ply module.
+  # pylint: disable=F0401
+  from ply import lex
+  from ply import yacc
+except ImportError:
+  module_path, module_name = os.path.split(__file__)
+  third_party = os.path.join(module_path, os.par, os.par, 'third_party')
+  sys.path.append(third_party)
+  # pylint: disable=F0401
+  from ply import lex
+  from ply import yacc
 
 #
 # ERROR_REMAP
@@ -862,7 +876,7 @@ class IDLParser(object):
   def __init__(self, lexer, verbose=False, debug=False, mute_error=False):
     self.lexer = lexer
     self.tokens = lexer.KnownTokens()
-    self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=False,
+    self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=debug,
                              optimize=0, write_tables=0)
     self.parse_debug = debug
     self.verbose = verbose
@@ -973,6 +987,8 @@ class IDLParser(object):
     return IDLAttribute(key, Boolean(True))
 
   def GetErrors(self):
+    # Access lexer errors, despite being private
+    # pylint: disable=W0212
     return self._parse_errors + self.lexer._lex_errors
 
 #
diff --git a/tools/idl_parser/idl_ppapi_lexer.py b/tools/idl_parser/idl_ppapi_lexer.py
index 2595f1c..9e83486 100755
--- a/tools/idl_parser/idl_ppapi_lexer.py
+++ b/tools/idl_parser/idl_ppapi_lexer.py
@@ -15,23 +15,25 @@ PLY can be found at:
 """
 
 from idl_lexer import IDLLexer
-import optparse
-import os.path
-import sys
 
 
 #
 # IDL PPAPI Lexer
 #
 class IDLPPAPILexer(IDLLexer):
+  # Token definitions
+  #
+  # These need to be methods for lexer construction, despite not using self.
+  # pylint: disable=R0201
+
   # Special multi-character operators
   def t_LSHIFT(self, t):
     r'<<'
-    return t;
+    return t
 
   def t_RSHIFT(self, t):
     r'>>'
-    return t;
+    return t
 
   def t_INLINE(self, t):
     r'\#inline (.|\n)*?\#endinl.*'
@@ -45,15 +47,15 @@ class IDLPPAPILexer(IDLLexer):
     self._AddKeywords(['label', 'struct'])
 
     # Add number types
-    self._AddKeywords(['char', 'int8_t', 'int16_t', 'int32_t', 'int64_t']);
-    self._AddKeywords(['uint8_t', 'uint16_t', 'uint32_t', 'uint64_t']);
-    self._AddKeywords(['double_t', 'float_t']);
+    self._AddKeywords(['char', 'int8_t', 'int16_t', 'int32_t', 'int64_t'])
+    self._AddKeywords(['uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'])
+    self._AddKeywords(['double_t', 'float_t'])
 
     # Add handle types
-    self._AddKeywords(['handle_t', 'PP_FileHandle']);
+    self._AddKeywords(['handle_t', 'PP_FileHandle'])
 
     # Add pointer types (void*, char*, const char*, const void*)
-    self._AddKeywords(['mem_t', 'str_t', 'cstr_t', 'interface_t']);
+    self._AddKeywords(['mem_t', 'str_t', 'cstr_t', 'interface_t'])
 
     # Remove JS types
     self._DelKeywords(['boolean', 'byte', 'Date', 'DOMString', 'double',
@@ -65,4 +67,4 @@ if __name__ == '__main__':
   lexer = IDLPPAPILexer()
   lexer.Tokenize(open('test_parser/inline_ppapi.idl').read())
   for tok in lexer.GetTokens():
-    print '\n' + str(tok)
\ No newline at end of file
+    print '\n' + str(tok)
diff --git a/tools/idl_parser/idl_ppapi_parser.py b/tools/idl_parser/idl_ppapi_parser.py
index 33eb503..c02e42c 100755
--- a/tools/idl_parser/idl_ppapi_parser.py
+++ b/tools/idl_parser/idl_ppapi_parser.py
@@ -29,13 +29,11 @@
 # pylint: disable=R0201
 # pylint: disable=C0301
 
-import os.path
 import sys
-import time
 
 from idl_ppapi_lexer import IDLPPAPILexer
 from idl_parser import IDLParser, ListFromConcat, ParseFile
-from idl_node import IDLAttribute, IDLNode
+from idl_node import IDLNode
 
 class IDLPPAPIParser(IDLParser):
 #
@@ -104,7 +102,8 @@ class IDLPPAPIParser(IDLParser):
   def p_LabelCont(self, p):
     """LabelCont : ',' LabelList
                  |"""
-    if len(p) > 1: p[0] = p[2]
+    if len(p) > 1:
+      p[0] = p[2]
 
   def p_LabelContError(self, p):
     """LabelCont : error LabelCont"""
@@ -260,20 +259,21 @@ class IDLPPAPIParser(IDLParser):
     arguments = self.BuildProduction('Values', p, 2, p[3])
     p[0] = self.BuildNamed('ExtAttribute', p, 1, arguments)
 
-  # [76]
-  def p_ExtendedAttributeIdentConst(self, p):
-    """ExtendedAttributeIdentConst : identifier '=' ConstValue"""
-    p[0] = self.BuildNamed('ExtAttribute', p, 1, p[3])
+  def p_ValueList(self, p):
+    """ValueList : ConstValue ValueListCont"""
+    p[0] = ListFromConcat(p[1], p[2])
 
   def p_ValueListCont(self, p):
-    """ValueListCont : ConstValue ValueListCont
+    """ValueListCont : ValueList
                      |"""
     if len(p) > 1:
-      p[0] = ListFromConcat(p[1], p[2])
+      p[0] = p[1]
+
+  # [76]
+  def p_ExtendedAttributeIdentConst(self, p):
+    """ExtendedAttributeIdentConst : identifier '=' ConstValue"""
+    p[0] = self.BuildNamed('ExtAttribute', p, 1, p[3])
 
-  def p_ValueList(self, p):
-    """ValueList : ConstValue ValueListCont"""
-    values = self.BuildProduction('Values', p, 2, ListFromConcat(p[1], p[2]))
 
   def __init__(self, lexer, verbose=False, debug=False, mute_error=False):
     IDLParser.__init__(self, lexer, verbose, debug, mute_error)
@@ -301,4 +301,4 @@ def main(argv):
 
 
 if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
\ No newline at end of file
+  sys.exit(main(sys.argv[1:]))
diff --git a/tools/idl_parser/test_parser/extattr_ppapi.idl b/tools/idl_parser/test_parser/extattr_ppapi.idl
new file mode 100644
index 0000000..0206901
--- /dev/null
+++ b/tools/idl_parser/test_parser/extattr_ppapi.idl
@@ -0,0 +1,91 @@
+/* Copyright 2013 The Chromium Authors. All rights reserved.
+   Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file. */
+
+/* Test ExtendedAttribute productions
+
+Run with --test to generate an AST and verify that all comments accurately
+reflect the state of the Nodes.
+
+BUILD Type(Name)
+This comment signals that a node of type <Type> is created with the
+name <Name>.
+
+ERROR Error String
+This comment signals that a error of <Error String> is generated.  The error
+is not assigned to a node, but are expected in order.
+
+PROP Key=Value
+This comment signals that a property has been set on the Node such that
+<Key> = <Value>.
+
+TREE
+Type(Name)
+  Type(Name)
+  Type(Name)
+    Type(Name)
+    ...
+This comment signals that a tree of nodes matching the BUILD comment
+symatics should exist.  This is an exact match.
+*/
+
+/* TREE
+ *Interface(Foo)
+ *  ExtAttributes()
+ *    ExtAttribute(foo)
+ *      Arguments()
+ */
+
+[foo()] interface Foo {};
+
+/* TREE
+ *Interface(Foo)
+ *  ExtAttributes()
+ *    ExtAttribute(foo)
+ *      Values()
+ */
+
+[foo(1)] interface Foo {};
+
+/* TREE
+ *Interface(Foo)
+ *  ExtAttributes()
+ *    ExtAttribute(foo)
+ *      Values()
+ */
+
+[foo(1 true 1.2e-3)] interface Foo {};
+
+/* TREE
+ *Interface(Foo)
+ *  ExtAttributes()
+ *    ExtAttribute(foo)
+ *      Arguments()
+ *        Error(Unexpected ).)
+ */
+
+[foo(null)] interface Foo {};
+
+/* TREE
+ *Interface(Foo)
+ *  ExtAttributes()
+ *    ExtAttribute(foo)
+ */
+
+[foo=1] interface Foo {};
+
+/* TREE
+ *Interface(Foo)
+ *  ExtAttributes()
+ *    ExtAttribute(foo)
+ */
+
+[foo=true] interface Foo {};
+
+/* TREE
+ *Interface(Foo)
+ *  ExtAttributes()
+ *    ExtAttribute(foo)
+ */
+
+[foo=1.2e-3] interface Foo {};