Some additional cleanup:

  Split tests into separate files to make them easier to work on.
  Convert tests to unittests.
  Split lexer into pure WebIDL and Pepper components

Tests now run automatically via PRESUBMIT when any change is made to the generator directory.

NOTRY=true
R=asargent@chromium.org
BUG=224150


Review URL: https://chromiumcodereview.appspot.com/13841013

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@193661 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 303 insertions, 139 deletions

diff --git a/tools/idl_parser/PRESUBMIT.py b/tools/idl_parser/PRESUBMIT.py
new file mode 100644
index 0000000..6b12b33
--- /dev/null
+++ b/tools/idl_parser/PRESUBMIT.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+WHITELIST = [ r'.+_test.py$' ]
+
+def CheckChangeOnUpload(input_api, output_api):
+  return input_api.canned_checks.RunUnitTestsInDirectory(
+      input_api, output_api, '.', whitelist=WHITELIST)
+
+
+def CheckChangeOnCommit(input_api, output_api):
+  return input_api.canned_checks.RunUnitTestsInDirectory(
+      input_api, output_api, '.', whitelist=WHITELIST)
diff --git a/tools/idl_parser/idl_lexer.py b/tools/idl_parser/idl_lexer.py
index 3e42734..a38b97e 100755
--- a/tools/idl_parser/idl_lexer.py
+++ b/tools/idl_parser/idl_lexer.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -45,17 +44,12 @@ class IDLLexer(object):
       'integer',
       'string',
 
-    # Operators
-      'ELLIPSIS',
-      'LSHIFT',
-      'RSHIFT',
-
     # Symbol and keywords types
       'COMMENT',
       'identifier',
 
-    # Pepper Extras
-      'INLINE',
+    # MultiChar operators
+      'ELLIPSIS',
   ]
 
   # 'keywords' is a map of string to token type.  All tokens matching
@@ -83,10 +77,8 @@ class IDLLexer(object):
     'Infinity' : 'INFINITY',
     'inherit' : 'INHERIT',
     'interface' : 'INTERFACE',
-    'label' : 'LABEL',
     'legacycaller' : 'LEGACYCALLER',
     'long' : 'LONG',
-    'namespace' : 'NAMESPACE',
     'Nan' : 'NAN',
     'null' : 'NULL',
     'object' : 'OBJECT',
@@ -100,7 +92,6 @@ class IDLLexer(object):
     'short' : 'SHORT',
     'static' : 'STATIC',
     'stringifier' : 'STRINGIFIER',
-    'struct' : 'STRUCT',
     'typedef' : 'TYPEDEF',
     'true' : 'TRUE',
     'unsigned' : 'UNSIGNED',
@@ -126,16 +117,14 @@ class IDLLexer(object):
   # 't_ignore' is a special match of items to ignore
   t_ignore = ' \t'
 
+  # Ellipsis operator
+  t_ELLIPSIS = r'\.\.\.'
+
   # Constant values
   t_integer = r'-?(0([0-7]*|[Xx][0-9A-Fa-f]+)|[1-9][0-9]*)'
   t_float = r'-?(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)'
   t_float += r'([Ee][+-]?[0-9]+)?|[0-9]+[Ee][+-]?[0-9]+)'
 
-  # Special multi-character operators
-  t_ELLIPSIS = r'\.\.\.'
-  t_LSHIFT = r'<<'
-  t_RSHIFT = r'>>'
-
   # A line ending '\n', we use this to increment the line number
   def t_LINE_END(self, t):
     r'\n+'
@@ -155,12 +144,6 @@ class IDLLexer(object):
     self.AddLines(t.value.count('\n'))
     return t
 
-  # Return a "preprocessor" inline block
-  def t_INLINE(self, t):
-    r'\#inline (.|\n)*?\#endinl.*'
-    self.AddLines(t.value.count('\n'))
-    return t
-
   # A symbol or keyword.
   def t_KEYWORD_OR_SYMBOL(self, t):
     r'_?[A-Za-z][A-Za-z_0-9]*'
@@ -191,7 +174,7 @@ class IDLLexer(object):
     pos = self.lexobj.lexpos - self.index[line]
     out = self.ErrorMessage(line, pos, msg)
     sys.stderr.write(out + '\n')
-    self.lex_errors += 1
+    self._lex_errors += 1
 
 
   def AddLines(self, count):
@@ -212,7 +195,7 @@ class IDLLexer(object):
 
   def SourceLine(self, line, pos):
     # Create a source line marker
-    caret = '\t^'.expandtabs(pos)
+    caret = ' ' * pos + '^'
     # We decrement the line number since the array is 0 based while the
     # line numbers are 1 based.
     return "%s\n%s" % (self.lines[line - 1], caret)
@@ -222,6 +205,19 @@ class IDLLexer(object):
         self.FileLineMsg(line, msg),
         self.SourceLine(line, pos))
 
+#
+# Tokenizer
+#
+# The token function returns the next token provided by IDLLexer for matching
+# against the leaf paterns.
+#
+  def token(self):
+    tok = self.lexobj.token()
+    if tok:
+      self.last = tok
+    return tok
+
+
   def GetTokens(self):
     outlist = []
     while True:
@@ -231,120 +227,15 @@ class IDLLexer(object):
       outlist.append(t)
     return outlist
 
-  def __init__(self, filename, data):
-    self.index = [0]
-    self.lex_errors = 0
-    self.lines = data.split('\n')
-    self.lexobj = lex.lex(object=self, lextab=None, optimize=0)
+  def Tokenize(self, data, filename='__no_file__'):
     self.lexobj.filename = filename
     self.lexobj.input(data)
+    self.lines = data.split('\n')
 
+  def __init__(self):
+    self.index = [0]
+    self._lex_errors = 0
+    self.linex = []
+    self.filename = None
+    self.lexobj = lex.lex(object=self, lextab=None, optimize=0)
 
-#
-# FileToTokens
-#
-# From a source file generate a list of tokens.
-#
-def FileToTokens(filename):
-  with open(filename, 'rb') as srcfile:
-    lexer = IDLLexer(filename, srcfile.read())
-    return lexer.GetTokens()
-
-
-#
-# TextToTokens
-#
-# From a source file generate a list of tokens.
-#
-def TextToTokens(text):
-  lexer = IDLLexer(None, text)
-  return lexer.GetTokens()
-
-
-#
-# TestSameText
-#
-# From a set of tokens, generate a new source text by joining with a
-# single space.  The new source is then tokenized and compared against the
-# old set.
-#
-def TestSameText(filename):
-  tokens1 = FileToTokens(filename)
-  to_text = '\n'.join(['%s' % t.value for t in tokens1])
-  tokens2 = TextToTokens(to_text)
-
-  count1 = len(tokens1)
-  count2 = len(tokens2)
-  if count1 != count2:
-    print "Size mismatch original %d vs %d\n" % (count1, count2)
-    if count1 > count2:
-      count1 = count2
-
-  failed = 0
-  for i in range(count1):
-    if tokens1[i].value != tokens2[i].value:
-      print "%d >>%s<< >>%s<<" % (i, tokens1[i].type, tokens2[i].value)
-      failed = failed + 1
-
-  return failed
-
-
-#
-# TestExpectedText
-#
-# From a set of tokens pairs, verify the type field of the second matches
-# the value of the first, so that:
-# integer 123 float 1.1
-# will generate a passing test, where the first token has both the type and
-# value of the keyword integer and the second has the type of integer and
-# value of 123.
-#
-def TestExpect(filename):
-  tokens = FileToTokens(filename)
-  count = len(tokens)
-  index = 0
-  errors = 0
-  while index < count:
-    expect_type = tokens[index].value
-    actual_type = tokens[index + 1].type
-    index += 2
-
-    if expect_type != actual_type:
-      sys.stderr.write('Mismatch:  Expected %s, but got %s.\n' %
-                       (expect_type, actual_type))
-      errors += 1
-
-  return errors
-
-
-def Main(args):
-  parser = optparse.OptionParser()
-  parser.add_option('--test', help='Run tests.', action='store_true')
-
-  # If no arguments are provided, run tests.
-  if len(args) == 0:
-    args = ['--test', 'test_lexer/values.in', 'test_lexer/keywords.in']
-
-  options, filenames = parser.parse_args(args)
-  if not filenames:
-    parser.error('No files specified.')
-
-  for filename in filenames:
-    try:
-      if options.test:
-        if TestSameText(filename):
-          sys.stderr.write('Failed text match on %s.\n' % filename)
-          return -1
-        if TestExpect(filename):
-          sys.stderr.write('Failed expected type match on %s.\n' % filename)
-          return -1
-        print 'Passed: ' + filename
-
-    except lex.LexError as le:
-      sys.stderr.write('%s\n' % str(le))
-
-  return 0
-
-
-if __name__ == '__main__':
-  sys.exit(Main(sys.argv[1:]))
diff --git a/tools/idl_parser/idl_lexer_test.py b/tools/idl_parser/idl_lexer_test.py
new file mode 100755
index 0000000..cba4e48
--- /dev/null
+++ b/tools/idl_parser/idl_lexer_test.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import optparse
+import os
+import sys
+import unittest
+
+from idl_lexer import IDLLexer
+from idl_ppapi_lexer import IDLPPAPILexer
+
+#
+# FileToTokens
+#
+# From a source file generate a list of tokens.
+#
+def FileToTokens(lexer, filename):
+  with open(filename, 'rb') as srcfile:
+    lexer.Tokenize(srcfile.read(), filename)
+    return lexer.GetTokens()
+
+
+#
+# TextToTokens
+#
+# From a source file generate a list of tokens.
+#
+def TextToTokens(lexer, text):
+  lexer.Tokenize(text)
+  return lexer.GetTokens()
+
+
+class WebIDLLexer(unittest.TestCase):
+  def setUp(self):
+    self.lexer = IDLLexer()
+    self.filenames = [
+        'test_lexer/values.in',
+        'test_lexer/keywords.in'
+    ]
+
+  #
+  # testRebuildText
+  #
+  # From a set of tokens, generate a new source text by joining with a
+  # single space.  The new source is then tokenized and compared against the
+  # old set.
+  #
+  def testRebuildText(self):
+    for filename in self.filenames:
+      tokens1 = FileToTokens(self.lexer, filename)
+      to_text = '\n'.join(['%s' % t.value for t in tokens1])
+      tokens2 = TextToTokens(self.lexer, to_text)
+
+      count1 = len(tokens1)
+      count2 = len(tokens2)
+      self.assertEqual(count1, count2)
+
+      for i in range(count1):
+        msg = 'Value %s does not match original %s on line %d of %s.' % (
+              tokens2[i].value, tokens1[i].value, tokens1[i].lineno, filename)
+        self.assertEqual(tokens1[i].value, tokens2[i].value, msg)
+
+  #
+  # testExpectedType
+  #
+  # From a set of tokens pairs, verify the type field of the second matches
+  # the value of the first, so that:
+  # integer 123 float 1.1 ...
+  # will generate a passing test, when the first token has both the type and
+  # value of the keyword integer and the second has the type of integer and
+  # value of 123 and so on.
+  #
+  def testExpectedType(self):
+    for filename in self.filenames:
+      tokens = FileToTokens(self.lexer, filename)
+      count = len(tokens)
+      self.assertTrue(count > 0)
+      self.assertFalse(count & 1)
+
+      index = 0
+      while index < count:
+        expect_type = tokens[index].value
+        actual_type = tokens[index + 1].type
+        msg = 'Type %s does not match expected %s on line %d of %s.' % (
+              actual_type, expect_type, tokens[index].lineno, filename)
+        index += 2
+        self.assertEqual(expect_type, actual_type, msg)
+
+
+class PepperIDLLexer(WebIDLLexer):
+  def setUp(self):
+    self.lexer = IDLPPAPILexer()
+    self.filenames = [
+        'test_lexer/values_ppapi.in',
+        'test_lexer/keywords_ppapi.in'
+    ]
+
+
+if __name__ == '__main__':
+  unittest.main()
\ No newline at end of file
diff --git a/tools/idl_parser/idl_ppapi_lexer.py b/tools/idl_parser/idl_ppapi_lexer.py
new file mode 100644
index 0000000..3accd2d
--- /dev/null
+++ b/tools/idl_parser/idl_ppapi_lexer.py
@@ -0,0 +1,54 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Lexer for PPAPI IDL
+
+The lexer uses the PLY library to build a tokenizer which understands both
+WebIDL and Pepper tokens.
+
+WebIDL, and WebIDL regular expressions can be found at:
+   http://www.w3.org/TR/2012/CR-WebIDL-20120419/
+PLY can be found at:
+   http://www.dabeaz.com/ply/
+"""
+
+from idl_lexer import IDLLexer
+import optparse
+import os.path
+import sys
+
+
+#
+# IDL PPAPI Lexer
+#
+class IDLPPAPILexer(IDLLexer):
+  # 'tokens' is a value required by lex which specifies the complete list
+  # of valid token types.  To WebIDL we add the following token types
+  IDLLexer.tokens += [
+    # Operators
+      'LSHIFT',
+      'RSHIFT',
+
+    # Pepper Extras
+      'INLINE',
+  ]
+
+  # 'keywords' is a map of string to token type.  All tokens matching
+  # KEYWORD_OR_SYMBOL are matched against keywords dictionary, to determine
+  # if the token is actually a keyword.  Add the new keywords to the
+  # dictionary and set of tokens
+  ppapi_keywords = ['LABEL', 'NAMESPACE', 'STRUCT']
+  for keyword in ppapi_keywords:
+    IDLLexer.keywords[ keyword.lower() ] = keyword
+    IDLLexer.tokens.append(keyword)
+
+  # Special multi-character operators
+  t_LSHIFT = r'<<'
+  t_RSHIFT = r'>>'
+
+  # Return a "preprocessor" inline block
+  def t_INLINE(self, t):
+    r'\#inline (.|\n)*?\#endinl.*'
+    self.AddLines(t.value.count('\n'))
+    return t
diff --git a/tools/idl_parser/run_tests.py b/tools/idl_parser/run_tests.py
new file mode 100755
index 0000000..9bb1356
--- /dev/null
+++ b/tools/idl_parser/run_tests.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import glob
+import sys
+import unittest
+
+if __name__ == '__main__':
+  testlist = glob.glob('*_test.py')
+  for testname in testlist:
+    unittest.main(verbosity=2, module=testname[:-3])
\ No newline at end of file
diff --git a/tools/idl_parser/test_lexer/keywords_ppapi.in b/tools/idl_parser/test_lexer/keywords_ppapi.in
new file mode 100644
index 0000000..e155753
--- /dev/null
+++ b/tools/idl_parser/test_lexer/keywords_ppapi.in
@@ -0,0 +1,40 @@
+ANY any
+ATTRIBUTE attribute
+BOOLEAN boolean
+BYTE byte
+CALLBACK callback
+CONST const
+CREATOR creator
+DATE Date
+DELETER deleter
+DICTIONARY dictionary
+DOMSTRING DOMString
+DOUBLE double
+FALSE false
+FLOAT float
+EXCEPTION exception
+GETTER getter
+IMPLEMENTS implements
+INFINITY Infinity
+INTERFACE interface
+LABEL label
+LEGACYCALLER legacycaller
+LONG long
+NAMESPACE namespace
+NAN Nan
+NULL null
+OBJECT object
+OPTIONAL optional
+OR or
+PARTIAL partial
+READONLY readonly
+SETTER setter
+SHORT short
+STATIC static
+STRINGIFIER stringifier
+STRUCT struct
+TYPEDEF typedef
+TRUE true
+UNSIGNED unsigned
+UNRESTRICTED unrestricted
+VOID void
diff --git a/tools/idl_parser/test_lexer/values.in b/tools/idl_parser/test_lexer/values.in
index 1040724..9ece967 100644
--- a/tools/idl_parser/test_lexer/values.in
+++ b/tools/idl_parser/test_lexer/values.in
@@ -29,8 +29,6 @@ COMMENT /*MULTI LINE*/
 ^ ^
 > >
 < <
-LSHIFT <<
-RSHIFT >>
 
 ELLIPSIS ...
 
diff --git a/tools/idl_parser/test_lexer/values_ppapi.in b/tools/idl_parser/test_lexer/values_ppapi.in
new file mode 100644
index 0000000..33fa577
--- /dev/null
+++ b/tools/idl_parser/test_lexer/values_ppapi.in
@@ -0,0 +1,50 @@
+integer 1 integer 123 integer 12345
+identifier A123 identifier A_A
+
+COMMENT /*XXXX*/
+COMMENT //XXXX
+
+COMMENT /*MULTI LINE*/
+
+[ [
+] ]
+* *
+. .
+( (
+) )
+{ {
+} }
+[ [
+] ]
+, ,
+; ;
+: :
+= =
++ +
+- -
+/ /
+~ ~
+| |
+& &
+^ ^
+> >
+< <
+
+LSHIFT <<
+RSHIFT >>
+ELLIPSIS ...
+
+float 1.1
+float 1e1
+float -1.1
+float -1e1
+float 1e-1
+float -1e-1
+float 1.0e1
+float -1.0e-1
+
+integer 00
+integer 01
+integer 0123
+
+identifier blah