Moving prepopulated search engines to a JSON file.

This CL also includes the python script required to convert the JSON file to a .cc/.h pair. The generated .cc/.h are not generated by the build process and must be committed to the repository.

BUG=159990


Review URL: https://chromiumcodereview.appspot.com/11377049

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@167793 0039d316-1c4b-4281-b951-d872f2087c98

6 files changed, 599 insertions, 0 deletions

diff --git a/tools/json_to_struct/PRESUBMIT.py b/tools/json_to_struct/PRESUBMIT.py
new file mode 100644
index 0000000..dd4d9a4
--- /dev/null
+++ b/tools/json_to_struct/PRESUBMIT.py
@@ -0,0 +1,20 @@
+# Copyright 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Presubmit script for changes affecting tools/json_to_struct/
+
+See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
+for more details about the presubmit API built into gcl.
+"""
+
+WHITELIST = [ r'.+_test.py$' ]
+
+def CheckChangeOnUpload(input_api, output_api):
+  return input_api.canned_checks.RunUnitTestsInDirectory(
+      input_api, output_api, '.', whitelist=WHITELIST)
+
+
+def CheckChangeOnCommit(input_api, output_api):
+  return input_api.canned_checks.RunUnitTestsInDirectory(
+      input_api, output_api, '.', whitelist=WHITELIST)
diff --git a/tools/json_to_struct/element_generator.py b/tools/json_to_struct/element_generator.py
new file mode 100644
index 0000000..3a592d8
--- /dev/null
+++ b/tools/json_to_struct/element_generator.py
@@ -0,0 +1,121 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import struct_generator
+
+def _JSONToCString16(json_string_literal):
+  """Converts a JSON string literal to a C++ UTF-16 string literal. This is
+  done by converting \\u#### to \\x####.
+  """
+  c_string_literal = json_string_literal
+  escape_index = c_string_literal.find('\\')
+  while escape_index > 0:
+    if c_string_literal[escape_index + 1] == 'u':
+      # We close the C string literal after the 4 hex digits and reopen it right
+      # after, otherwise the Windows compiler will sometimes try to get more
+      # than 4 characters in the hex string.
+      c_string_literal = (c_string_literal[0:escape_index + 1] + 'x' +
+          c_string_literal[escape_index + 2:escape_index + 6] + '" L"' +
+          c_string_literal[escape_index + 6:])
+    escape_index = c_string_literal.find('\\', escape_index + 6)
+  return c_string_literal
+
+def _GenerateString(content, lines):
+  """Generates an UTF-8 string to be included in a static structure initializer.
+  If content is not specified, uses NULL.
+  """
+  if content is None:
+    lines.append('  NULL,')
+  else:
+    # json.dumps quotes the string and escape characters as required.
+    lines.append('  %s,' % json.dumps(content))
+
+def _GenerateString16(content, lines):
+  """Generates an UTF-16 string to be included in a static structure
+  initializer. If content is not specified, uses NULL.
+  """
+  if content is None:
+    lines.append('  NULL,')
+  else:
+    # json.dumps quotes the string and escape characters as required.
+    lines.append('  L%s,' % _JSONToCString16(json.dumps(content)))
+
+def _GenerateArray(field_info, content, lines):
+  """Generates an array to be included in a static structure initializer. If
+  content is not specified, uses NULL. The array is assigned to a temporary
+  variable which is initialized before the structure.
+  """
+  if content is None:
+    lines.append('  NULL,')
+    lines.append('  0,')  # Size of the array.
+    return
+
+  # Create a new array variable and use it in the structure initializer.
+  # This prohibits nested arrays. Add a clash detection and renaming mechanism
+  # to solve the problem.
+  var = 'array_%s' % field_info['field'];
+  lines.append('  %s,' % var)
+  lines.append('  %s,' % len(content))  # Size of the array.
+  # Generate the array content.
+  array_lines = []
+  field_info['contents']['field'] = var;
+  array_lines.append(struct_generator.GenerateField(
+                     field_info['contents']) + '[] = {')
+  for subcontent in content:
+    GenerateFieldContent(field_info['contents'], subcontent, array_lines)
+  array_lines.append('};')
+  # Prepend the generated array so it is initialized before the structure.
+  lines.reverse()
+  array_lines.reverse()
+  lines.extend(array_lines)
+  lines.reverse()
+
+def GenerateFieldContent(field_info, content, lines):
+  """Generate the content of a field to be included in the static structure
+  initializer. If the field's content is not specified, uses the default value
+  if one exists.
+  """
+  if content is None:
+    content = field_info.get('default', None)
+  type = field_info['type']
+  if type == 'int' or type == 'enum':
+    lines.append('  %s,' % content)
+  elif type == 'string':
+    _GenerateString(content, lines)
+  elif type == 'string16':
+    _GenerateString16(content, lines)
+  elif type == 'array':
+    _GenerateArray(field_info, content, lines)
+  else:
+    raise RuntimeError('Unknown field type "%s"' % type)
+
+def GenerateElement(type_name, schema, element_name, element):
+  """Generate the static structure initializer for one element.
+  """
+  lines = [];
+  lines.append('const %s %s = {' % (type_name, element_name));
+  for field_info in schema:
+    content = element.get(field_info['field'], None)
+    if (content == None and not field_info.get('optional', False)):
+      raise RuntimeError('Mandatory field "%s" omitted in element "%s".' %
+                         (field_info['field'], element_name))
+    GenerateFieldContent(field_info, content, lines)
+  lines.append('};')
+  return '\n'.join(lines)
+
+def GenerateElements(type_name, schema, description):
+  """Generate the static structure initializer for all the elements in the
+  description['elements'] dictionary, as well as for any variables in
+  description['int_variables'].
+  """
+  result = [];
+  for var_name, value in description.get('int_variables', {}).items():
+    result.append('const int %s = %s;' % (var_name, value))
+  result.append('')
+
+  for element_name, element in description.get('elements', {}).items():
+    result.append(GenerateElement(type_name, schema, element_name, element))
+    result.append('')
+  return '\n'.join(result)
diff --git a/tools/json_to_struct/element_generator_test.py b/tools/json_to_struct/element_generator_test.py
new file mode 100755
index 0000000..484e83f
--- /dev/null
+++ b/tools/json_to_struct/element_generator_test.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from element_generator import GenerateFieldContent
+from element_generator import GenerateElements
+import unittest
+
+class ElementGeneratorTest(unittest.TestCase):
+  def testGenerateIntFieldContent(self):
+    lines = [];
+    GenerateFieldContent({'type': 'int', 'default': 5}, None, lines)
+    self.assertEquals(['  5,'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'int', 'default': 5}, 12, lines)
+    self.assertEquals(['  12,'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'int'}, -3, lines)
+    self.assertEquals(['  -3,'], lines)
+
+  def testGenerateStringFieldContent(self):
+    lines = [];
+    GenerateFieldContent({'type': 'string', 'default': 'foo_bar'}, None, lines)
+    self.assertEquals(['  "foo_bar",'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'string', 'default': 'foo'}, 'bar\n', lines)
+    self.assertEquals(['  "bar\\n",'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'string'}, None, lines)
+    self.assertEquals(['  NULL,'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'string'}, 'foo', lines)
+    self.assertEquals(['  "foo",'], lines)
+
+  def testGenerateString16FieldContent(self):
+    lines = [];
+    GenerateFieldContent({'type': 'string16', 'default': u'f\u00d8\u00d81a'},
+                         None, lines)
+    self.assertEquals(['  L"f\\x00d8" L"\\x00d8" L"1a",'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'string16', 'default': 'foo'}, u'b\uc3a5r',
+                         lines)
+    self.assertEquals(['  L"b\\xc3a5" L"r",'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'string16'}, None, lines)
+    self.assertEquals(['  NULL,'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'string16'}, u'foo\\u1234', lines)
+    self.assertEquals(['  L"foo\\\\u1234",'], lines)
+
+  def testGenerateEnumFieldContent(self):
+    lines = [];
+    GenerateFieldContent({'type': 'enum', 'default': 'RED'}, None, lines)
+    self.assertEquals(['  RED,'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'enum', 'default': 'RED'}, 'BLACK', lines)
+    self.assertEquals(['  BLACK,'], lines)
+    lines = [];
+    GenerateFieldContent({'type': 'enum'}, 'BLUE', lines)
+    self.assertEquals(['  BLUE,'], lines)
+
+  def testGenerateArrayFieldContent(self):
+    lines = ['STRUCT BEGINS'];
+    GenerateFieldContent({'type': 'array', 'contents': {'type': 'int'}},
+                         None, lines)
+    self.assertEquals(['STRUCT BEGINS', '  NULL,', '  0,'], lines)
+    lines = ['STRUCT BEGINS'];
+    GenerateFieldContent({'field': 'my_array', 'type': 'array',
+                          'contents': {'type': 'int'}}, [3, 4], lines)
+    self.assertEquals('const int array_my_array[] = {\n' +
+      '  3,\n' +
+      '  4,\n' +
+      '};\n' +
+      'STRUCT BEGINS\n' +
+      '  array_my_array,\n' +
+      '  2,', '\n'.join(lines))
+
+  def testGenerateElements(self):
+    schema = [
+      {'field': 'f0', 'type': 'int', 'default': 1000, 'optional': True},
+      {'field': 'f1', 'type': 'string'},
+      {'field': 'f2', 'type': 'enum', 'ctype': 'QuasiBool', 'default': 'MAYBE',
+       'optional': True},
+      {'field': 'f3', 'type': 'array', 'contents': {'type': 'string16'},
+       'optional': True}
+    ]
+    description = {
+      'int_variables': {'a': -5, 'b': 5},
+      'elements': {
+        'elem0': {'f0': 5, 'f1': 'foo', 'f2': 'SURE'},
+        'elem1': {'f2': 'NOWAY', 'f0': -2, 'f1': 'bar'},
+        'elem2': {'f1': 'foo_bar', 'f3': [u'bar', u'foo']}
+      }
+    }
+
+    # Build the expected result stream based on the unpredicatble order the
+    # dictionary element are listed in.
+    int_variable_expected = {
+      'a': 'const int a = -5;\n',
+      'b': 'const int b = 5;\n',
+    }
+    elements_expected = {
+      'elem0': 'const MyType elem0 = {\n' +
+               '  5,\n' +
+               '  "foo",\n' +
+               '  SURE,\n' +
+               '  NULL,\n' +
+               '  0,\n'
+               '};\n',
+      'elem1': 'const MyType elem1 = {\n' +
+               '  -2,\n' +
+               '  "bar",\n' +
+               '  NOWAY,\n' +
+               '  NULL,\n' +
+               '  0,\n'
+               '};\n',
+      'elem2': 'const wchar_t* const array_f3[] = {\n' +
+               '  L"bar",\n' +
+               '  L"foo",\n' +
+               '};\n' +
+               'const MyType elem2 = {\n' +
+               '  1000,\n' +
+               '  "foo_bar",\n' +
+               '  MAYBE,\n' +
+               '  array_f3,\n' +
+               '  2,\n'
+               '};\n'
+    }
+    expected = ''
+    for key, value in description['int_variables'].items():
+      expected += int_variable_expected[key]
+    expected += '\n'
+    elements = []
+    for key, value in description['elements'].items():
+      elements.append(elements_expected[key])
+    expected += '\n'.join(elements)
+
+    result = GenerateElements('MyType', schema, description)
+    self.assertEquals(expected, result)
+
+  def testGenerateElementsMissingMandatoryField(self):
+    schema = [
+      {'field': 'f0', 'type': 'int'},
+      {'field': 'f1', 'type': 'string'},
+    ]
+    description = {
+      'int_variables': {'a': -5, 'b': 5},
+      'elements': {
+        'elem0': {'f0': 5},
+      }
+    }
+
+    self.assertRaises(RuntimeError,
+      lambda: GenerateElements('MyType', schema, description))
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/tools/json_to_struct/json_to_struct.py b/tools/json_to_struct/json_to_struct.py
new file mode 100755
index 0000000..8dfda44
--- /dev/null
+++ b/tools/json_to_struct/json_to_struct.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+# Copyright 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Format for the JSON schema file:
+# {
+#   "type_name": "DesiredCStructName",
+#   "headers": [          // Optional list of headers to be included by the .h.
+#     "path/to/header.h"
+#   ],
+#   "schema": [           // Fields of the generated structure.
+#     {
+#       "field": "my_enum_field",
+#       "type": "enum",   // Either: int, string, string16, enum, array.
+#       "default": "RED", // Optional. Cannot be used for array.
+#       "ctype": "Color"  // Only for enum, specify the C type.
+#     },
+#     {
+#       "field": "my_int_array_field",  // my_int_array_field_size will also
+#       "type": "array",                // be generated.
+#       "contents": {
+#         "type": "int"   // Either: int, string, string16, enum, array.
+#       }
+#     },
+#     ...
+#   ]
+# }
+#
+# Format for the JSON description file:
+# {
+#   "int_variables": {    // An optional list of constant int variables.
+#     "kDesiredConstantName": 45
+#   },
+#   "elements": {         // All the elements for which to create static
+#                         // initialization code in the .cc file.
+#     "my_const_variable": {
+#       "my_int_field": 10,
+#       "my_string_field": "foo bar",
+#       "my_enum_field": "BLACK",
+#       "my_int_array_field": [ 1, 2, 3, 5, 7 ],
+#     },
+#     "my_other_const_variable": {
+#       ...
+#     }
+#   }
+# }
+
+import json
+from datetime import datetime
+import os.path
+import sys
+import optparse
+import copy
+_script_path = os.path.realpath(__file__)
+
+sys.path.insert(0, os.path.normpath(_script_path + "/../../"))
+try:
+  import json_comment_eater
+finally:
+  sys.path.pop(0)
+
+import struct_generator
+import element_generator
+
+HEAD = """// Copyright %d The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// GENERATED FROM THE SCHEMA DEFINITION AND DESCRIPTION IN
+//   %s
+//   %s
+// DO NOT EDIT.
+
+"""
+
+def _GenerateHeaderGuard(h_filename):
+  """Generates the string used in #ifndef guarding the header file.
+  """
+  return (('%s_' % h_filename)
+          .upper().replace(os.sep, '_').replace('/', '_').replace('.', '_'))
+
+def _GenerateH(fileroot, head, namespace, schema, description):
+  """Generates the .h file containing the definition of the structure specified
+  by the schema.
+
+  Args:
+    fileroot: The filename and path of the file to create, without an extension.
+    head: The string to output as the header of the .h file.
+    namespace: A string corresponding to the C++ namespace to use.
+    schema: A dict containing the schema. See comment at the top of this file.
+    description: A dict containing the description. See comment at the top of
+        this file.
+  """
+
+  h_filename = fileroot + '.h'
+  with open(h_filename, 'w') as f:
+    f.write(head)
+
+    f.write('#include <cstddef>\n')
+    f.write('\n')
+
+    header_guard = _GenerateHeaderGuard(h_filename)
+    f.write('#ifndef %s\n' % header_guard)
+    f.write('#define %s\n' % header_guard)
+    f.write('\n')
+
+    for header in schema.get('headers', []):
+      f.write('#include "%s"\n' % header)
+    f.write('\n')
+
+    if namespace:
+      f.write('namespace %s {\n' % namespace)
+      f.write('\n')
+
+    f.write(struct_generator.GenerateStruct(
+      schema['type_name'], schema['schema']))
+    f.write('\n')
+
+    for var_name, value in description.get('int_variables', []).items():
+      f.write('extern const int %s;\n' % var_name)
+    f.write('\n')
+
+    for element_name, element in description['elements'].items():
+      f.write('extern const %s %s;\n' % (schema['type_name'], element_name))
+
+    if namespace:
+      f.write('\n')
+      f.write('}  // namespace %s\n' % namespace)
+
+    f.write('\n')
+    f.write( '#endif  // %s\n' % header_guard)
+
+def _GenerateCC(fileroot, head, namespace, schema, description):
+  """Generates the .cc file containing the static initializers for the
+  of the elements specified in the description.
+
+  Args:
+    fileroot: The filename and path of the file to create, without an extension.
+    head: The string to output as the header of the .cc file.
+    namespace: A string corresponding to the C++ namespace to use.
+    schema: A dict containing the schema. See comment at the top of this file.
+    description: A dict containing the description. See comment at the top of
+        this file.
+  """
+  with open(fileroot + '.cc', 'w') as f:
+    f.write(head)
+
+    f.write('#include "%s"\n' % (fileroot + '.h'))
+    f.write('\n')
+
+    if namespace:
+      f.write('namespace %s {\n' % namespace)
+      f.write('\n')
+
+    f.write(element_generator.GenerateElements(schema['type_name'],
+        schema['schema'], description))
+
+    if namespace:
+      f.write('\n')
+      f.write('}  // namespace %s\n' % namespace)
+
+def _Load(filename):
+  """Loads a JSON file int a Python object and return this object.
+  """
+  # TODO(beaudoin): When moving to Python 2.7 use object_pairs_hook=OrderedDict.
+  with open(filename, 'r') as handle:
+    result = json.loads(json_comment_eater.Nom(handle.read()))
+  return result
+
+if __name__ == '__main__':
+  parser = optparse.OptionParser(
+      description='Generates an C++ array of struct from a JSON description.',
+      usage='usage: %prog [option] -s schema description')
+  parser.add_option('-d', '--destdir',
+      help='root directory to output generated files.')
+  parser.add_option('-n', '--namespace',
+      help='C++ namespace for generated files. e.g search_providers.')
+  parser.add_option('-s', '--schema', help='path to the schema file, '
+      'mandatory.')
+  (opts, args) = parser.parse_args()
+
+  if not opts.schema:
+    parser.error('You must specify a --schema.')
+
+  description_filename = os.path.normpath(args[0])
+  root, ext = os.path.splitext(description_filename)
+  shortroot = os.path.split(root)[1]
+  if opts.destdir:
+    output_root = os.path.join(os.path.normpath(opts.destdir), shortroot)
+  else:
+    output_root = shortroot
+
+  schema = _Load(opts.schema)
+  description = _Load(description_filename)
+
+  head = HEAD % (datetime.now().year, opts.schema, description_filename)
+  _GenerateH(output_root, head, opts.namespace, schema, description)
+  _GenerateCC(output_root, head, opts.namespace, schema, description)
diff --git a/tools/json_to_struct/struct_generator.py b/tools/json_to_struct/struct_generator.py
new file mode 100644
index 0000000..501cb57
--- /dev/null
+++ b/tools/json_to_struct/struct_generator.py
@@ -0,0 +1,43 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+def _GenerateArrayField(field_info):
+  """Generate a string defining an array field in a C structure.
+  """
+  contents = field_info['contents']
+  contents['field'] = '* ' + field_info['field']
+  if contents['type'] == 'array':
+    raise RuntimeError('Nested arrays are not supported.')
+  return (GenerateField(contents) + ';\n' +
+          '  const size_t %s_size') % field_info['field'];
+
+def GenerateField(field_info):
+  """Generate a string defining a field of the type specified by
+  field_info['type'] in a C structure.
+  """
+  field = field_info['field']
+  type = field_info['type']
+  if type == 'int':
+    return 'const int %s' % field
+  elif type == 'string':
+    return 'const char* const %s' % field
+  elif type == 'string16':
+    return 'const wchar_t* const %s' % field
+  elif type == 'enum':
+    return 'const %s %s' % (field_info['ctype'], field)
+  elif type == 'array':
+    return _GenerateArrayField(field_info)
+  else:
+    raise RuntimeError('Unknown field type "%s"' % type)
+
+def GenerateStruct(type_name, schema):
+  """Generate a string defining a structure containing the fields specified in
+  the schema list.
+  """
+  lines = [];
+  lines.append('struct %s {' % type_name)
+  for field_info in schema:
+    lines.append('  ' + GenerateField(field_info) + ';')
+  lines.append('};');
+  return '\n'.join(lines) + '\n';
diff --git a/tools/json_to_struct/struct_generator_test.py b/tools/json_to_struct/struct_generator_test.py
new file mode 100755
index 0000000..8566c33
--- /dev/null
+++ b/tools/json_to_struct/struct_generator_test.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from struct_generator import GenerateField
+from struct_generator import GenerateStruct
+import unittest
+
+class StructGeneratorTest(unittest.TestCase):
+  def testGenerateIntField(self):
+    self.assertEquals('const int foo_bar',
+        GenerateField({'type': 'int', 'field': 'foo_bar'}))
+
+  def testGenerateStringField(self):
+    self.assertEquals('const char* const bar_foo',
+        GenerateField({'type': 'string', 'field': 'bar_foo'}))
+
+  def testGenerateString16Field(self):
+    self.assertEquals('const wchar_t* const foo_bar',
+        GenerateField({'type': 'string16', 'field': 'foo_bar'}))
+
+  def testGenerateEnumField(self):
+    self.assertEquals('const MyEnumType foo_foo',
+        GenerateField({'type': 'enum',
+                       'field': 'foo_foo',
+                       'ctype': 'MyEnumType'}))
+
+  def testGenerateArrayField(self):
+    self.assertEquals('const int * bar_bar;\n'
+                      '  const size_t bar_bar_size',
+        GenerateField({'type': 'array',
+                       'field': 'bar_bar',
+                       'contents': {'type': 'int'}}))
+
+  def testGenerateStruct(self):
+    schema = [
+      {'type': 'int', 'field': 'foo_bar'},
+      {'type': 'string', 'field': 'bar_foo', 'default': 'dummy'},
+      {
+        'type': 'array',
+        'field': 'bar_bar',
+        'contents': {
+          'type': 'enum',
+          'ctype': 'MyEnumType'
+        }
+      }
+    ]
+    struct = ('struct MyTypeName {\n'
+        '  const int foo_bar;\n'
+        '  const char* const bar_foo;\n'
+        '  const MyEnumType * bar_bar;\n'
+        '  const size_t bar_bar_size;\n'
+        '};\n')
+    self.assertEquals(struct, GenerateStruct('MyTypeName', schema))
+
+if __name__ == '__main__':
+  unittest.main()