Componentize script to generate UI string overrides mapping.

Change the API of the script to generate both the header and source file
and to take a list of header files as input (to allow overridding strings
from components/components_strings.grd).

The script is now generating a method CreateUIStringOverrider() and hides
the arrays in the implementation file and allow customization of the
namespace in which the function and code is generated.

Introduce a new target "chrome_ui_string_overrider_factory" in the gyp
build to mimic the same target in the gn build (compiles the source file
generated by the script).

Rename the script from to generate_ui_string_overrider.py to reflect the
new role of the script and move it to components/variations/service.

Introduce .gni file to help using the script by different embedders (don't
introduce .gypi files as it is much harder to share code for gyp).

Componentize ui_string_overrider_unittest.cc.

BUG=534257

Review URL: https://codereview.chromium.org/1374773002

Cr-Commit-Position: refs/heads/master@{#352308}

6 files changed, 552 insertions, 1 deletions

diff --git a/components/variations/service/BUILD.gn b/components/variations/service/BUILD.gn
index 385c724..c6cf5b1 100644
--- a/components/variations/service/BUILD.gn
+++ b/components/variations/service/BUILD.gn
@@ -30,6 +30,7 @@ source_set("service") {
 source_set("unit_tests") {
   testonly = true
   sources = [
+    "ui_string_overrider_unittest.cc",
     "variations_service_unittest.cc",
   ]
 
diff --git a/components/variations/service/generate_ui_string_overrider.gni b/components/variations/service/generate_ui_string_overrider.gni
new file mode 100644
index 0000000..11de0a0
--- /dev/null
+++ b/components/variations/service/generate_ui_string_overrider.gni
@@ -0,0 +1,66 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Runs the resources map generation script other the given header files to
+# produce an output file and a source_set to build it.
+#
+# Parameters:
+#   inputs:
+#     List of file name to read. Each file should be an header file generated
+#     by grit with line like "#define IDS_FOO 12345".
+#
+#   namespace (optional):
+#     Namespace in which the generated code should be scoped. If left empty,
+#     the code will be in the global namespace.
+#
+#   header_filename:
+#     Name of the generated header file.
+#
+#   source_filename:
+#     Name of the generated source file.
+#
+#   deps (optional):
+#     List of targets to depend on.
+#
+template("generate_ui_string_overrider") {
+  # Copy "target_name" to allow restrict the visibility of the generation
+  # target to that target (as ":$target_name" will have a different meaning
+  # in the "action" block).
+  source_set_target_name = target_name
+  gen_action_target_name = target_name + "_gen_sources"
+
+  action(gen_action_target_name) {
+    header_filename = "$target_gen_dir/" + invoker.header_filename
+    source_filename = "$target_gen_dir/" + invoker.source_filename
+
+    visibility = [ ":$source_set_target_name" ]
+    script = "//components/variations/service/generate_ui_string_overrider.py"
+    outputs = [
+      header_filename,
+      source_filename,
+    ]
+
+    inputs = invoker.inputs
+    if (defined(invoker.deps)) {
+      deps = invoker.deps
+    }
+
+    args = [
+             "-N" + invoker.namespace,
+             "-o" + rebase_path(root_gen_dir, root_build_dir),
+             "-H" + rebase_path(header_filename, root_gen_dir),
+             "-S" + rebase_path(source_filename, root_gen_dir),
+           ] + rebase_path(inputs, root_build_dir)
+  }
+
+  source_set(target_name) {
+    sources = get_target_outputs(":$gen_action_target_name")
+    deps = [
+      "//components/variations/service",
+      ":$gen_action_target_name",
+    ]
+
+    forward_variables_from(invoker, [ "visibility" ])
+  }
+}
diff --git a/components/variations/service/generate_ui_string_overrider.py b/components/variations/service/generate_ui_string_overrider.py
new file mode 100755
index 0000000..288272c
--- /dev/null
+++ b/components/variations/service/generate_ui_string_overrider.py
@@ -0,0 +1,292 @@
+#!/usr/bin/python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import collections
+import hashlib
+import operator
+import os
+import re
+import sys
+
+SCRIPT_NAME = "generate_ui_string_overrider.py"
+RESOURCE_EXTRACT_REGEX = re.compile('^#define (\S*) (\d*)$', re.MULTILINE)
+
+class Error(Exception):
+  """Base error class for all exceptions in generated_resources_map."""
+
+
+class HashCollisionError(Error):
+  """Multiple resource names hash to the same value."""
+
+
+Resource = collections.namedtuple("Resource", ['hash', 'name', 'index'])
+
+
+def _HashName(name):
+  """Returns the hash id for a name.
+
+  Args:
+    name: The name to hash.
+
+  Returns:
+    An int that is at most 32 bits.
+  """
+  md5hash = hashlib.md5()
+  md5hash.update(name)
+  return int(md5hash.hexdigest()[:8], 16)
+
+
+def _GetNameIndexPairsIter(string_to_scan):
+  """Gets an iterator of the resource name and index pairs of the given string.
+
+  Scans the input string for lines of the form "#define NAME INDEX" and returns
+  an iterator over all matching (NAME, INDEX) pairs.
+
+  Args:
+    string_to_scan: The input string to scan.
+
+  Yields:
+    A tuple of name and index.
+  """
+  for match in RESOURCE_EXTRACT_REGEX.finditer(string_to_scan):
+    yield match.group(1, 2)
+
+
+def _GetResourceListFromString(resources_content):
+  """Produces a list of |Resource| objects from a string.
+
+  The input string contains lines of the form "#define NAME INDEX". The returned
+  list is sorted primarily by hash, then name, and then index.
+
+  Args:
+    resources_content: The input string to process, contains lines of the form
+        "#define NAME INDEX".
+
+  Returns:
+    A sorted list of |Resource| objects.
+  """
+  resources = [Resource(_HashName(name), name, index) for name, index in
+               _GetNameIndexPairsIter(resources_content)]
+
+  # The default |Resource| order makes |resources| sorted by the hash, then
+  # name, then index.
+  resources.sort()
+
+  return resources
+
+
+def _CheckForHashCollisions(sorted_resource_list):
+  """Checks a sorted list of |Resource| objects for hash collisions.
+
+  Args:
+    sorted_resource_list: A sorted list of |Resource| objects.
+
+  Returns:
+    A set of all |Resource| objects with collisions.
+  """
+  collisions = set()
+  for i in xrange(len(sorted_resource_list) - 1):
+    resource = sorted_resource_list[i]
+    next_resource = sorted_resource_list[i+1]
+    if resource.hash == next_resource.hash:
+      collisions.add(resource)
+      collisions.add(next_resource)
+
+  return collisions
+
+
+def _GenDataArray(
+    resources, entry_pattern, array_name, array_type, data_getter):
+  """Generates a C++ statement defining a literal array containing the hashes.
+
+  Args:
+    resources: A sorted list of |Resource| objects.
+    entry_pattern: A pattern to be used to generate each entry in the array. The
+        pattern is expected to have a place for data and one for a comment, in
+        that order.
+    array_name: The name of the array being generated.
+    array_type: The type of the array being generated.
+    data_getter: A function that gets the array data from a |Resource| object.
+
+  Returns:
+    A string containing a C++ statement defining the an array.
+  """
+  lines = [entry_pattern % (data_getter(r), r.name) for r in resources]
+  pattern = """const %(type)s %(name)s[] = {
+%(content)s
+};
+"""
+  return pattern % {'type': array_type,
+                    'name': array_name,
+                    'content': '\n'.join(lines)}
+
+
+def _GenerateNamespacePrefixAndSuffix(namespace):
+  """Generates the namespace prefix and suffix for |namespace|.
+
+  Args:
+    namespace: A string corresponding to the namespace name. May be empty.
+
+  Returns:
+    A tuple of strings corresponding to the namespace prefix and suffix for
+    putting the code in the corresponding namespace in C++. If namespace is
+    the empty string, both returned strings are empty too.
+  """
+  if not namespace:
+    return "", ""
+  return "namespace %s {\n\n" % namespace, "\n}  // namespace %s\n" % namespace
+
+
+def _GenerateSourceFileContent(resources_content, namespace, header_filename):
+  """Generates the .cc content from the given generated grit headers content.
+
+  Args:
+    resources_content: The input string to process, contains lines of the form
+        "#define NAME INDEX".
+
+    namespace: The namespace in which the generated code should be scoped. If
+        not defined, then the code will be in the global namespace.
+
+    header_filename: Path to the corresponding .h.
+
+  Returns:
+    .cc file content implementing the CreateUIStringOverrider() factory.
+  """
+  hashed_tuples = _GetResourceListFromString(resources_content)
+
+  collisions = _CheckForHashCollisions(hashed_tuples)
+  if collisions:
+    error_message = "\n".join(
+        ["hash: %i, name: %s" % (i[0], i[1]) for i in sorted(collisions)])
+    error_message = ("\nThe following names had hash collisions "
+                     "(sorted by the hash value):\n%s\n" %(error_message))
+    raise HashCollisionError(error_message)
+
+  hashes_array = _GenDataArray(
+      hashed_tuples, "    %iU,  // %s", 'kResourceHashes', 'uint32_t',
+      operator.attrgetter('hash'))
+  indices_array = _GenDataArray(
+      hashed_tuples, "    %s,  // %s", 'kResourceIndices', 'int',
+      operator.attrgetter('index'))
+
+  namespace_prefix, namespace_suffix = _GenerateNamespacePrefixAndSuffix(
+      namespace)
+
+  return (
+      "// This file was generated by %(script_name)s. Do not edit.\n"
+      "\n"
+      "#include \"%(header_filename)s\"\n\n"
+      "%(namespace_prefix)s"
+      "namespace {\n\n"
+      "const size_t kNumResources = %(num_resources)i;\n\n"
+      "%(hashes_array)s"
+      "\n"
+      "%(indices_array)s"
+      "\n"
+      "}  // namespace\n"
+      "\n"
+      "variations::UIStringOverrider CreateUIStringOverrider() {\n"
+      "  return variations::UIStringOverrider(\n"
+      "      kResourceHashes, kResourceIndices, kNumResources);\n"
+      "}\n"
+      "%(namespace_suffix)s") % {
+          'script_name': SCRIPT_NAME,
+          'header_filename': header_filename,
+          'namespace_prefix': namespace_prefix,
+          'num_resources': len(hashed_tuples),
+          'hashes_array': hashes_array,
+          'indices_array': indices_array,
+          'namespace_suffix': namespace_suffix,
+      }
+
+
+def _GenerateHeaderFileContent(namespace, header_filename):
+  """Generates the .h for to the .cc generated by _GenerateSourceFileContent.
+
+  Args:
+    namespace: The namespace in which the generated code should be scoped. If
+        not defined, then the code will be in the global namespace.
+
+    header_filename: Path to the corresponding .h. Used to generate the include
+        guards.
+
+  Returns:
+    .cc file content implementing the CreateUIStringOverrider() factory.
+  """
+
+  include_guard = re.sub('[^A-Z]', '_', header_filename.upper()) + '_'
+  namespace_prefix, namespace_suffix = _GenerateNamespacePrefixAndSuffix(
+      namespace)
+
+  return (
+      "// This file was generated by %(script_name)s. Do not edit.\n"
+      "\n"
+      "#ifndef %(include_guard)s\n"
+      "#define %(include_guard)s\n"
+      "\n"
+      "#include \"components/variations/service/ui_string_overrider.h\"\n\n"
+      "%(namespace_prefix)s"
+      "// Returns an initialized UIStringOverrider.\n"
+      "variations::UIStringOverrider CreateUIStringOverrider();\n"
+      "%(namespace_suffix)s"
+      "\n"
+      "#endif  // %(include_guard)s\n"
+      ) % {
+          'script_name': SCRIPT_NAME,
+          'include_guard': include_guard,
+          'namespace_prefix': namespace_prefix,
+          'namespace_suffix': namespace_suffix,
+      }
+
+
+def main():
+  arg_parser = argparse.ArgumentParser(
+      description="Generate UIStringOverrider factory from resources headers "
+                  "generated by grit.")
+  arg_parser.add_argument(
+      "--output_dir", "-o", required=True,
+      help="Base directory to for generated files.")
+  arg_parser.add_argument(
+      "--source_filename", "-S", required=True,
+      help="File name of the generated source file.")
+  arg_parser.add_argument(
+      "--header_filename", "-H", required=True,
+      help="File name of the generated header file.")
+  arg_parser.add_argument(
+      "--namespace", "-N", default="",
+      help="Namespace of the generated factory function (code will be in "
+           "the global namespace if this is omitted).")
+  arg_parser.add_argument(
+      "--test_support", "-t", action="store_true", default=False,
+      help="Make internal variables accessible for testing.")
+  arg_parser.add_argument(
+      "inputs", metavar="FILENAME", nargs="+",
+      help="Path to resources header file generated by grit.")
+  arguments = arg_parser.parse_args()
+
+  generated_resources_h = ""
+  for resources_file in arguments.inputs:
+    with open(resources_file, "r") as resources:
+      generated_resources_h += resources.read()
+
+  if len(generated_resources_h) == 0:
+    raise Error("No content loaded for %s." % (resources_file))
+
+  source_file_content = _GenerateSourceFileContent(
+      generated_resources_h, arguments.namespace, arguments.header_filename)
+  header_file_content = _GenerateHeaderFileContent(
+      arguments.namespace, arguments.header_filename)
+
+  with open(os.path.join(
+      arguments.output_dir, arguments.source_filename), "w") as generated_file:
+    generated_file.write(source_file_content)
+  with open(os.path.join(
+      arguments.output_dir, arguments.header_filename), "w") as generated_file:
+    generated_file.write(header_file_content)
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/components/variations/service/generate_ui_string_overrider_unittest.py b/components/variations/service/generate_ui_string_overrider_unittest.py
new file mode 100755
index 0000000..55d55f3
--- /dev/null
+++ b/components/variations/service/generate_ui_string_overrider_unittest.py
@@ -0,0 +1,132 @@
+#!/usr/bin/python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unittests for generate_ui_string_overrider.py"""
+
+import unittest
+
+import generate_ui_string_overrider
+
+
+class GenerateResourcesMapUnittest(unittest.TestCase):
+  NAMESPACE = "chrome_variations"
+  OUT_HEADER = "components/variations/service/ui_string_overrider_factory.h"
+  TEST_INPUT = """
+// This file is automatically generated by GRIT. Do not edit.
+
+#pragma once
+
+#define IDS_BOOKMARKS_NO_ITEMS 12500
+#define IDS_BOOKMARK_BAR_IMPORT_LINK 12501
+#define IDS_BOOKMARK_GROUP_FROM_IE 12502
+#define IDS_BOOKMARK_GROUP_FROM_FIREFOX 12503
+"""
+
+  def testGetResourceListFromString(self):
+    expected_tuples = [(301430091, "IDS_BOOKMARKS_NO_ITEMS", "12500"),
+                       (2654138887, "IDS_BOOKMARK_BAR_IMPORT_LINK", "12501"),
+                       (2894469061, "IDS_BOOKMARK_GROUP_FROM_IE", "12502"),
+                       (3847176170, "IDS_BOOKMARK_GROUP_FROM_FIREFOX", "12503")]
+    expected = [
+        generate_ui_string_overrider.Resource(*t) for t in expected_tuples]
+
+    actual_tuples = generate_ui_string_overrider._GetResourceListFromString(
+        self.TEST_INPUT)
+
+    self.assertEqual(expected_tuples, actual_tuples)
+
+
+  def testCheckForHashCollisions(self):
+    collisions_tuples = [(123, "IDS_FOO", "12500"),
+                         (456, "IDS_BAR", "12501"),
+                         (456, "IDS_BAZ", "12502"),
+                         (890, "IDS_QUX", "12503"),
+                         (899, "IDS_NO", "12504"),
+                         (899, "IDS_YES", "12505")]
+    list_with_collisions = [generate_ui_string_overrider.Resource(*t)
+                            for t in collisions_tuples]
+
+    expected_collision_tuples = [(456, "IDS_BAR", "12501"),
+                                 (456, "IDS_BAZ", "12502"),
+                                 (899, "IDS_NO", "12504"),
+                                 (899, "IDS_YES", "12505")]
+    expected_collisions = [generate_ui_string_overrider.Resource(*t)
+                          for t in expected_collision_tuples]
+
+    actual_collisions = sorted(
+        generate_ui_string_overrider._CheckForHashCollisions(
+            list_with_collisions))
+    actual_collisions
+
+    self.assertEqual(expected_collisions, actual_collisions)
+
+  def testGenerateSourceFileContent(self):
+    expected = (
+        """\
+// This file was generated by generate_ui_string_overrider.py. Do not edit.
+
+#include "components/variations/service/ui_string_overrider_factory.h"
+
+namespace chrome_variations {
+
+namespace {
+
+const size_t kNumResources = 4;
+
+const uint32_t kResourceHashes[] = {
+    301430091U,  // IDS_BOOKMARKS_NO_ITEMS
+    2654138887U,  // IDS_BOOKMARK_BAR_IMPORT_LINK
+    2894469061U,  // IDS_BOOKMARK_GROUP_FROM_IE
+    3847176170U,  // IDS_BOOKMARK_GROUP_FROM_FIREFOX
+};
+
+const int kResourceIndices[] = {
+    12500,  // IDS_BOOKMARKS_NO_ITEMS
+    12501,  // IDS_BOOKMARK_BAR_IMPORT_LINK
+    12502,  // IDS_BOOKMARK_GROUP_FROM_IE
+    12503,  // IDS_BOOKMARK_GROUP_FROM_FIREFOX
+};
+
+}  // namespace
+
+variations::UIStringOverrider CreateUIStringOverrider() {
+  return variations::UIStringOverrider(
+      kResourceHashes, kResourceIndices, kNumResources);
+}
+
+}  // namespace chrome_variations
+""")
+    actual = generate_ui_string_overrider._GenerateSourceFileContent(
+        self.TEST_INPUT, self.NAMESPACE, self.OUT_HEADER)
+
+    self.assertEqual(expected, actual)
+
+
+  def testGenerateHeaderFileContent(self):
+    expected = (
+        """\
+// This file was generated by generate_ui_string_overrider.py. Do not edit.
+
+#ifndef COMPONENTS_VARIATIONS_SERVICE_UI_STRING_OVERRIDER_FACTORY_H_
+#define COMPONENTS_VARIATIONS_SERVICE_UI_STRING_OVERRIDER_FACTORY_H_
+
+#include "components/variations/service/ui_string_overrider.h"
+
+namespace chrome_variations {
+
+// Returns an initialized UIStringOverrider.
+variations::UIStringOverrider CreateUIStringOverrider();
+
+}  // namespace chrome_variations
+
+#endif  // COMPONENTS_VARIATIONS_SERVICE_UI_STRING_OVERRIDER_FACTORY_H_
+""")
+    actual = generate_ui_string_overrider._GenerateHeaderFileContent(
+        self.NAMESPACE, self.OUT_HEADER)
+
+    self.assertEqual(expected, actual)
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/components/variations/service/ui_string_overrider.h b/components/variations/service/ui_string_overrider.h
index 5908444..8aa4784 100644
--- a/components/variations/service/ui_string_overrider.h
+++ b/components/variations/service/ui_string_overrider.h
@@ -16,7 +16,7 @@ namespace variations {
 // array of resource name hashes, and |resource_indices| an array of resource
 // indices in the same order.
 //
-// The mapping is created by generate_resources_map.py based on generated
+// The mapping is created by generate_ui_string_overrider.py based on generated
 // resources header files. The script ensure that if one header file contains
 // |#define IDS_FOO 12345| then for some index |i|, |resource_hashes[i]| will
 // be equal to |HASH("IDS_FOO")| and |resource_indices[i]| will be equal to
diff --git a/components/variations/service/ui_string_overrider_unittest.cc b/components/variations/service/ui_string_overrider_unittest.cc
new file mode 100644
index 0000000..640d51c
--- /dev/null
+++ b/components/variations/service/ui_string_overrider_unittest.cc
@@ -0,0 +1,60 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/variations/service/ui_string_overrider.h"
+
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace chrome_variations {
+
+namespace {
+
+const size_t kNumResources = 4;
+
+const uint32_t kResourceHashes[] = {
+    301430091U,  // IDS_BOOKMARKS_NO_ITEMS
+    2654138887U,  // IDS_BOOKMARK_BAR_IMPORT_LINK
+    2894469061U,  // IDS_BOOKMARK_GROUP_FROM_IE
+    3847176170U,  // IDS_BOOKMARK_GROUP_FROM_FIREFOX
+};
+
+const int kResourceIndices[] = {
+    12500,  // IDS_BOOKMARKS_NO_ITEMS
+    12501,  // IDS_BOOKMARK_BAR_IMPORT_LINK
+    12502,  // IDS_BOOKMARK_GROUP_FROM_IE
+    12503,  // IDS_BOOKMARK_GROUP_FROM_FIREFOX
+};
+
+}  // namespace
+
+class UIStringOverriderTest : public ::testing::Test {
+ public:
+  UIStringOverriderTest()
+      : provider_(kResourceHashes, kResourceIndices, kNumResources) {}
+
+  int GetResourceIndex(uint32_t hash) {
+    return provider_.GetResourceIndex(hash);
+  }
+
+ private:
+  variations::UIStringOverrider provider_;
+
+  DISALLOW_COPY_AND_ASSIGN(UIStringOverriderTest);
+};
+
+TEST_F(UIStringOverriderTest, LookupNotFound) {
+  EXPECT_EQ(-1, GetResourceIndex(0));
+  EXPECT_EQ(-1, GetResourceIndex(kResourceHashes[kNumResources - 1] + 1));
+
+  // Lookup a hash that shouldn't exist.
+  // 3847176171U is 1 + the hash for IDS_BOOKMARK_GROUP_FROM_FIREFOX.
+  EXPECT_EQ(-1, GetResourceIndex(3847176171U));
+}
+
+TEST_F(UIStringOverriderTest, LookupFound) {
+  for (size_t i = 0; i < kNumResources; ++i)
+    EXPECT_EQ(kResourceIndices[i], GetResourceIndex(kResourceHashes[i]));
+}
+
+}  // namespace chrome_variations