From a213cebe6af1a80367336908d37f30ab6542940d Mon Sep 17 00:00:00 2001 From: sdefresne Date: Mon, 5 Oct 2015 02:23:39 -0700 Subject: Componentize script to generate UI string overrides mapping. Change the API of the script to generate both the header and source file and to take a list of header files as input (to allow overridding strings from components/components_strings.grd). The script is now generating a method CreateUIStringOverrider() and hides the arrays in the implementation file and allow customization of the namespace in which the function and code is generated. Introduce a new target "chrome_ui_string_overrider_factory" in the gyp build to mimic the same target in the gn build (compiles the source file generated by the script). Rename the script from to generate_ui_string_overrider.py to reflect the new role of the script and move it to components/variations/service. Introduce .gni file to help using the script by different embedders (don't introduce .gypi files as it is much harder to share code for gyp). Componentize ui_string_overrider_unittest.cc. BUG=534257 Review URL: https://codereview.chromium.org/1374773002 Cr-Commit-Position: refs/heads/master@{#352308} --- components/variations/service/BUILD.gn | 1 + .../service/generate_ui_string_overrider.gni | 66 +++++ .../service/generate_ui_string_overrider.py | 292 +++++++++++++++++++++ .../generate_ui_string_overrider_unittest.py | 132 ++++++++++ .../variations/service/ui_string_overrider.h | 2 +- .../service/ui_string_overrider_unittest.cc | 60 +++++ 6 files changed, 552 insertions(+), 1 deletion(-) create mode 100644 components/variations/service/generate_ui_string_overrider.gni create mode 100755 components/variations/service/generate_ui_string_overrider.py create mode 100755 components/variations/service/generate_ui_string_overrider_unittest.py create mode 100644 components/variations/service/ui_string_overrider_unittest.cc (limited to 'components/variations') diff --git a/components/variations/service/BUILD.gn b/components/variations/service/BUILD.gn index 385c724..c6cf5b1 100644 --- a/components/variations/service/BUILD.gn +++ b/components/variations/service/BUILD.gn @@ -30,6 +30,7 @@ source_set("service") { source_set("unit_tests") { testonly = true sources = [ + "ui_string_overrider_unittest.cc", "variations_service_unittest.cc", ] diff --git a/components/variations/service/generate_ui_string_overrider.gni b/components/variations/service/generate_ui_string_overrider.gni new file mode 100644 index 0000000..11de0a0 --- /dev/null +++ b/components/variations/service/generate_ui_string_overrider.gni @@ -0,0 +1,66 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Runs the resources map generation script other the given header files to +# produce an output file and a source_set to build it. +# +# Parameters: +# inputs: +# List of file name to read. Each file should be an header file generated +# by grit with line like "#define IDS_FOO 12345". +# +# namespace (optional): +# Namespace in which the generated code should be scoped. If left empty, +# the code will be in the global namespace. +# +# header_filename: +# Name of the generated header file. +# +# source_filename: +# Name of the generated source file. +# +# deps (optional): +# List of targets to depend on. +# +template("generate_ui_string_overrider") { + # Copy "target_name" to allow restrict the visibility of the generation + # target to that target (as ":$target_name" will have a different meaning + # in the "action" block). + source_set_target_name = target_name + gen_action_target_name = target_name + "_gen_sources" + + action(gen_action_target_name) { + header_filename = "$target_gen_dir/" + invoker.header_filename + source_filename = "$target_gen_dir/" + invoker.source_filename + + visibility = [ ":$source_set_target_name" ] + script = "//components/variations/service/generate_ui_string_overrider.py" + outputs = [ + header_filename, + source_filename, + ] + + inputs = invoker.inputs + if (defined(invoker.deps)) { + deps = invoker.deps + } + + args = [ + "-N" + invoker.namespace, + "-o" + rebase_path(root_gen_dir, root_build_dir), + "-H" + rebase_path(header_filename, root_gen_dir), + "-S" + rebase_path(source_filename, root_gen_dir), + ] + rebase_path(inputs, root_build_dir) + } + + source_set(target_name) { + sources = get_target_outputs(":$gen_action_target_name") + deps = [ + "//components/variations/service", + ":$gen_action_target_name", + ] + + forward_variables_from(invoker, [ "visibility" ]) + } +} diff --git a/components/variations/service/generate_ui_string_overrider.py b/components/variations/service/generate_ui_string_overrider.py new file mode 100755 index 0000000..288272c --- /dev/null +++ b/components/variations/service/generate_ui_string_overrider.py @@ -0,0 +1,292 @@ +#!/usr/bin/python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import collections +import hashlib +import operator +import os +import re +import sys + +SCRIPT_NAME = "generate_ui_string_overrider.py" +RESOURCE_EXTRACT_REGEX = re.compile('^#define (\S*) (\d*)$', re.MULTILINE) + +class Error(Exception): + """Base error class for all exceptions in generated_resources_map.""" + + +class HashCollisionError(Error): + """Multiple resource names hash to the same value.""" + + +Resource = collections.namedtuple("Resource", ['hash', 'name', 'index']) + + +def _HashName(name): + """Returns the hash id for a name. + + Args: + name: The name to hash. + + Returns: + An int that is at most 32 bits. + """ + md5hash = hashlib.md5() + md5hash.update(name) + return int(md5hash.hexdigest()[:8], 16) + + +def _GetNameIndexPairsIter(string_to_scan): + """Gets an iterator of the resource name and index pairs of the given string. + + Scans the input string for lines of the form "#define NAME INDEX" and returns + an iterator over all matching (NAME, INDEX) pairs. + + Args: + string_to_scan: The input string to scan. + + Yields: + A tuple of name and index. + """ + for match in RESOURCE_EXTRACT_REGEX.finditer(string_to_scan): + yield match.group(1, 2) + + +def _GetResourceListFromString(resources_content): + """Produces a list of |Resource| objects from a string. + + The input string contains lines of the form "#define NAME INDEX". The returned + list is sorted primarily by hash, then name, and then index. + + Args: + resources_content: The input string to process, contains lines of the form + "#define NAME INDEX". + + Returns: + A sorted list of |Resource| objects. + """ + resources = [Resource(_HashName(name), name, index) for name, index in + _GetNameIndexPairsIter(resources_content)] + + # The default |Resource| order makes |resources| sorted by the hash, then + # name, then index. + resources.sort() + + return resources + + +def _CheckForHashCollisions(sorted_resource_list): + """Checks a sorted list of |Resource| objects for hash collisions. + + Args: + sorted_resource_list: A sorted list of |Resource| objects. + + Returns: + A set of all |Resource| objects with collisions. + """ + collisions = set() + for i in xrange(len(sorted_resource_list) - 1): + resource = sorted_resource_list[i] + next_resource = sorted_resource_list[i+1] + if resource.hash == next_resource.hash: + collisions.add(resource) + collisions.add(next_resource) + + return collisions + + +def _GenDataArray( + resources, entry_pattern, array_name, array_type, data_getter): + """Generates a C++ statement defining a literal array containing the hashes. + + Args: + resources: A sorted list of |Resource| objects. + entry_pattern: A pattern to be used to generate each entry in the array. The + pattern is expected to have a place for data and one for a comment, in + that order. + array_name: The name of the array being generated. + array_type: The type of the array being generated. + data_getter: A function that gets the array data from a |Resource| object. + + Returns: + A string containing a C++ statement defining the an array. + """ + lines = [entry_pattern % (data_getter(r), r.name) for r in resources] + pattern = """const %(type)s %(name)s[] = { +%(content)s +}; +""" + return pattern % {'type': array_type, + 'name': array_name, + 'content': '\n'.join(lines)} + + +def _GenerateNamespacePrefixAndSuffix(namespace): + """Generates the namespace prefix and suffix for |namespace|. + + Args: + namespace: A string corresponding to the namespace name. May be empty. + + Returns: + A tuple of strings corresponding to the namespace prefix and suffix for + putting the code in the corresponding namespace in C++. If namespace is + the empty string, both returned strings are empty too. + """ + if not namespace: + return "", "" + return "namespace %s {\n\n" % namespace, "\n} // namespace %s\n" % namespace + + +def _GenerateSourceFileContent(resources_content, namespace, header_filename): + """Generates the .cc content from the given generated grit headers content. + + Args: + resources_content: The input string to process, contains lines of the form + "#define NAME INDEX". + + namespace: The namespace in which the generated code should be scoped. If + not defined, then the code will be in the global namespace. + + header_filename: Path to the corresponding .h. + + Returns: + .cc file content implementing the CreateUIStringOverrider() factory. + """ + hashed_tuples = _GetResourceListFromString(resources_content) + + collisions = _CheckForHashCollisions(hashed_tuples) + if collisions: + error_message = "\n".join( + ["hash: %i, name: %s" % (i[0], i[1]) for i in sorted(collisions)]) + error_message = ("\nThe following names had hash collisions " + "(sorted by the hash value):\n%s\n" %(error_message)) + raise HashCollisionError(error_message) + + hashes_array = _GenDataArray( + hashed_tuples, " %iU, // %s", 'kResourceHashes', 'uint32_t', + operator.attrgetter('hash')) + indices_array = _GenDataArray( + hashed_tuples, " %s, // %s", 'kResourceIndices', 'int', + operator.attrgetter('index')) + + namespace_prefix, namespace_suffix = _GenerateNamespacePrefixAndSuffix( + namespace) + + return ( + "// This file was generated by %(script_name)s. Do not edit.\n" + "\n" + "#include \"%(header_filename)s\"\n\n" + "%(namespace_prefix)s" + "namespace {\n\n" + "const size_t kNumResources = %(num_resources)i;\n\n" + "%(hashes_array)s" + "\n" + "%(indices_array)s" + "\n" + "} // namespace\n" + "\n" + "variations::UIStringOverrider CreateUIStringOverrider() {\n" + " return variations::UIStringOverrider(\n" + " kResourceHashes, kResourceIndices, kNumResources);\n" + "}\n" + "%(namespace_suffix)s") % { + 'script_name': SCRIPT_NAME, + 'header_filename': header_filename, + 'namespace_prefix': namespace_prefix, + 'num_resources': len(hashed_tuples), + 'hashes_array': hashes_array, + 'indices_array': indices_array, + 'namespace_suffix': namespace_suffix, + } + + +def _GenerateHeaderFileContent(namespace, header_filename): + """Generates the .h for to the .cc generated by _GenerateSourceFileContent. + + Args: + namespace: The namespace in which the generated code should be scoped. If + not defined, then the code will be in the global namespace. + + header_filename: Path to the corresponding .h. Used to generate the include + guards. + + Returns: + .cc file content implementing the CreateUIStringOverrider() factory. + """ + + include_guard = re.sub('[^A-Z]', '_', header_filename.upper()) + '_' + namespace_prefix, namespace_suffix = _GenerateNamespacePrefixAndSuffix( + namespace) + + return ( + "// This file was generated by %(script_name)s. Do not edit.\n" + "\n" + "#ifndef %(include_guard)s\n" + "#define %(include_guard)s\n" + "\n" + "#include \"components/variations/service/ui_string_overrider.h\"\n\n" + "%(namespace_prefix)s" + "// Returns an initialized UIStringOverrider.\n" + "variations::UIStringOverrider CreateUIStringOverrider();\n" + "%(namespace_suffix)s" + "\n" + "#endif // %(include_guard)s\n" + ) % { + 'script_name': SCRIPT_NAME, + 'include_guard': include_guard, + 'namespace_prefix': namespace_prefix, + 'namespace_suffix': namespace_suffix, + } + + +def main(): + arg_parser = argparse.ArgumentParser( + description="Generate UIStringOverrider factory from resources headers " + "generated by grit.") + arg_parser.add_argument( + "--output_dir", "-o", required=True, + help="Base directory to for generated files.") + arg_parser.add_argument( + "--source_filename", "-S", required=True, + help="File name of the generated source file.") + arg_parser.add_argument( + "--header_filename", "-H", required=True, + help="File name of the generated header file.") + arg_parser.add_argument( + "--namespace", "-N", default="", + help="Namespace of the generated factory function (code will be in " + "the global namespace if this is omitted).") + arg_parser.add_argument( + "--test_support", "-t", action="store_true", default=False, + help="Make internal variables accessible for testing.") + arg_parser.add_argument( + "inputs", metavar="FILENAME", nargs="+", + help="Path to resources header file generated by grit.") + arguments = arg_parser.parse_args() + + generated_resources_h = "" + for resources_file in arguments.inputs: + with open(resources_file, "r") as resources: + generated_resources_h += resources.read() + + if len(generated_resources_h) == 0: + raise Error("No content loaded for %s." % (resources_file)) + + source_file_content = _GenerateSourceFileContent( + generated_resources_h, arguments.namespace, arguments.header_filename) + header_file_content = _GenerateHeaderFileContent( + arguments.namespace, arguments.header_filename) + + with open(os.path.join( + arguments.output_dir, arguments.source_filename), "w") as generated_file: + generated_file.write(source_file_content) + with open(os.path.join( + arguments.output_dir, arguments.header_filename), "w") as generated_file: + generated_file.write(header_file_content) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/components/variations/service/generate_ui_string_overrider_unittest.py b/components/variations/service/generate_ui_string_overrider_unittest.py new file mode 100755 index 0000000..55d55f3 --- /dev/null +++ b/components/variations/service/generate_ui_string_overrider_unittest.py @@ -0,0 +1,132 @@ +#!/usr/bin/python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unittests for generate_ui_string_overrider.py""" + +import unittest + +import generate_ui_string_overrider + + +class GenerateResourcesMapUnittest(unittest.TestCase): + NAMESPACE = "chrome_variations" + OUT_HEADER = "components/variations/service/ui_string_overrider_factory.h" + TEST_INPUT = """ +// This file is automatically generated by GRIT. Do not edit. + +#pragma once + +#define IDS_BOOKMARKS_NO_ITEMS 12500 +#define IDS_BOOKMARK_BAR_IMPORT_LINK 12501 +#define IDS_BOOKMARK_GROUP_FROM_IE 12502 +#define IDS_BOOKMARK_GROUP_FROM_FIREFOX 12503 +""" + + def testGetResourceListFromString(self): + expected_tuples = [(301430091, "IDS_BOOKMARKS_NO_ITEMS", "12500"), + (2654138887, "IDS_BOOKMARK_BAR_IMPORT_LINK", "12501"), + (2894469061, "IDS_BOOKMARK_GROUP_FROM_IE", "12502"), + (3847176170, "IDS_BOOKMARK_GROUP_FROM_FIREFOX", "12503")] + expected = [ + generate_ui_string_overrider.Resource(*t) for t in expected_tuples] + + actual_tuples = generate_ui_string_overrider._GetResourceListFromString( + self.TEST_INPUT) + + self.assertEqual(expected_tuples, actual_tuples) + + + def testCheckForHashCollisions(self): + collisions_tuples = [(123, "IDS_FOO", "12500"), + (456, "IDS_BAR", "12501"), + (456, "IDS_BAZ", "12502"), + (890, "IDS_QUX", "12503"), + (899, "IDS_NO", "12504"), + (899, "IDS_YES", "12505")] + list_with_collisions = [generate_ui_string_overrider.Resource(*t) + for t in collisions_tuples] + + expected_collision_tuples = [(456, "IDS_BAR", "12501"), + (456, "IDS_BAZ", "12502"), + (899, "IDS_NO", "12504"), + (899, "IDS_YES", "12505")] + expected_collisions = [generate_ui_string_overrider.Resource(*t) + for t in expected_collision_tuples] + + actual_collisions = sorted( + generate_ui_string_overrider._CheckForHashCollisions( + list_with_collisions)) + actual_collisions + + self.assertEqual(expected_collisions, actual_collisions) + + def testGenerateSourceFileContent(self): + expected = ( + """\ +// This file was generated by generate_ui_string_overrider.py. Do not edit. + +#include "components/variations/service/ui_string_overrider_factory.h" + +namespace chrome_variations { + +namespace { + +const size_t kNumResources = 4; + +const uint32_t kResourceHashes[] = { + 301430091U, // IDS_BOOKMARKS_NO_ITEMS + 2654138887U, // IDS_BOOKMARK_BAR_IMPORT_LINK + 2894469061U, // IDS_BOOKMARK_GROUP_FROM_IE + 3847176170U, // IDS_BOOKMARK_GROUP_FROM_FIREFOX +}; + +const int kResourceIndices[] = { + 12500, // IDS_BOOKMARKS_NO_ITEMS + 12501, // IDS_BOOKMARK_BAR_IMPORT_LINK + 12502, // IDS_BOOKMARK_GROUP_FROM_IE + 12503, // IDS_BOOKMARK_GROUP_FROM_FIREFOX +}; + +} // namespace + +variations::UIStringOverrider CreateUIStringOverrider() { + return variations::UIStringOverrider( + kResourceHashes, kResourceIndices, kNumResources); +} + +} // namespace chrome_variations +""") + actual = generate_ui_string_overrider._GenerateSourceFileContent( + self.TEST_INPUT, self.NAMESPACE, self.OUT_HEADER) + + self.assertEqual(expected, actual) + + + def testGenerateHeaderFileContent(self): + expected = ( + """\ +// This file was generated by generate_ui_string_overrider.py. Do not edit. + +#ifndef COMPONENTS_VARIATIONS_SERVICE_UI_STRING_OVERRIDER_FACTORY_H_ +#define COMPONENTS_VARIATIONS_SERVICE_UI_STRING_OVERRIDER_FACTORY_H_ + +#include "components/variations/service/ui_string_overrider.h" + +namespace chrome_variations { + +// Returns an initialized UIStringOverrider. +variations::UIStringOverrider CreateUIStringOverrider(); + +} // namespace chrome_variations + +#endif // COMPONENTS_VARIATIONS_SERVICE_UI_STRING_OVERRIDER_FACTORY_H_ +""") + actual = generate_ui_string_overrider._GenerateHeaderFileContent( + self.NAMESPACE, self.OUT_HEADER) + + self.assertEqual(expected, actual) + +if __name__ == '__main__': + unittest.main() diff --git a/components/variations/service/ui_string_overrider.h b/components/variations/service/ui_string_overrider.h index 5908444..8aa4784 100644 --- a/components/variations/service/ui_string_overrider.h +++ b/components/variations/service/ui_string_overrider.h @@ -16,7 +16,7 @@ namespace variations { // array of resource name hashes, and |resource_indices| an array of resource // indices in the same order. // -// The mapping is created by generate_resources_map.py based on generated +// The mapping is created by generate_ui_string_overrider.py based on generated // resources header files. The script ensure that if one header file contains // |#define IDS_FOO 12345| then for some index |i|, |resource_hashes[i]| will // be equal to |HASH("IDS_FOO")| and |resource_indices[i]| will be equal to diff --git a/components/variations/service/ui_string_overrider_unittest.cc b/components/variations/service/ui_string_overrider_unittest.cc new file mode 100644 index 0000000..640d51c --- /dev/null +++ b/components/variations/service/ui_string_overrider_unittest.cc @@ -0,0 +1,60 @@ +// Copyright 2014 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/variations/service/ui_string_overrider.h" + +#include "testing/gtest/include/gtest/gtest.h" + +namespace chrome_variations { + +namespace { + +const size_t kNumResources = 4; + +const uint32_t kResourceHashes[] = { + 301430091U, // IDS_BOOKMARKS_NO_ITEMS + 2654138887U, // IDS_BOOKMARK_BAR_IMPORT_LINK + 2894469061U, // IDS_BOOKMARK_GROUP_FROM_IE + 3847176170U, // IDS_BOOKMARK_GROUP_FROM_FIREFOX +}; + +const int kResourceIndices[] = { + 12500, // IDS_BOOKMARKS_NO_ITEMS + 12501, // IDS_BOOKMARK_BAR_IMPORT_LINK + 12502, // IDS_BOOKMARK_GROUP_FROM_IE + 12503, // IDS_BOOKMARK_GROUP_FROM_FIREFOX +}; + +} // namespace + +class UIStringOverriderTest : public ::testing::Test { + public: + UIStringOverriderTest() + : provider_(kResourceHashes, kResourceIndices, kNumResources) {} + + int GetResourceIndex(uint32_t hash) { + return provider_.GetResourceIndex(hash); + } + + private: + variations::UIStringOverrider provider_; + + DISALLOW_COPY_AND_ASSIGN(UIStringOverriderTest); +}; + +TEST_F(UIStringOverriderTest, LookupNotFound) { + EXPECT_EQ(-1, GetResourceIndex(0)); + EXPECT_EQ(-1, GetResourceIndex(kResourceHashes[kNumResources - 1] + 1)); + + // Lookup a hash that shouldn't exist. + // 3847176171U is 1 + the hash for IDS_BOOKMARK_GROUP_FROM_FIREFOX. + EXPECT_EQ(-1, GetResourceIndex(3847176171U)); +} + +TEST_F(UIStringOverriderTest, LookupFound) { + for (size_t i = 0; i < kNumResources; ++i) + EXPECT_EQ(kResourceIndices[i], GetResourceIndex(kResourceHashes[i])); +} + +} // namespace chrome_variations -- cgit v1.1