From 125a48a1d40b94453e9282f71b16127c1b8643af Mon Sep 17 00:00:00 2001 From: "siggi@chromium.org" Date: Mon, 28 Nov 2011 16:47:29 +0000 Subject: Build a Syzygy-reordered mini_installer.exe. This CL adds an All_syzygy target to the build/all.gyp solution, that builds a Syzygy-reordered chrome.dll and mini_installer. If there there is a chrome.dll-order.json file present in the output directory, chrome.dll will be ordered according to that, otherwise it will be randomized. Moved most of mini_installer.gyp to a common .gypi file for reuse. Review URL: http://codereview.chromium.org/8698009 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@111710 0039d316-1c4b-4281-b951-d872f2087c98 --- build/all.gyp | 12 ++ chrome/chrome_syzygy.gyp | 49 +++++ chrome/installer/mini_installer.gypi | 213 +++++++++++++++++++++ chrome/installer/mini_installer_syzygy.gyp | 45 +++++ chrome/tools/build/win/create_installer_archive.py | 29 ++- chrome/tools/build/win/syzygy_reorder.py | 111 +++++++++++ 6 files changed, 452 insertions(+), 7 deletions(-) create mode 100644 chrome/chrome_syzygy.gyp create mode 100644 chrome/installer/mini_installer.gypi create mode 100644 chrome/installer/mini_installer_syzygy.gyp create mode 100644 chrome/tools/build/win/syzygy_reorder.py diff --git a/build/all.gyp b/build/all.gyp index f0110ab..abbf211 100644 --- a/build/all.gyp +++ b/build/all.gyp @@ -150,6 +150,18 @@ ], }, # target_name: All { + 'target_name': 'All_syzygy', + 'type': 'none', + 'conditions': [ + ['OS=="win" and fastbuild==0', { + 'dependencies': [ + '../chrome/installer/mini_installer_syzygy.gyp:*', + ], + }, + ], + ], + }, # target_name: All_syzygy + { 'target_name': 'chromium_builder_tests', 'type': 'none', 'dependencies': [ diff --git a/chrome/chrome_syzygy.gyp b/chrome/chrome_syzygy.gyp new file mode 100644 index 0000000..d174ee4 --- /dev/null +++ b/chrome/chrome_syzygy.gyp @@ -0,0 +1,49 @@ +# Copyright (c) 2011 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +{ + 'conditions': [ + ['OS=="win" and fastbuild==0', { + # Reorder the initial chrome DLL executable, placing the optimized + # output and corresponding PDB file into the "syzygy" subdirectory. + # If there's a matching chrome.dll-ordering.json file present in + # the output directory, chrome.dll will be ordered according to that, + # otherwise it will be randomized. + # This target won't build in fastbuild, since there are no PDBs. + 'targets': [ + { + 'target_name': 'chrome_dll_syzygy', + 'type': 'none', + 'sources' : [], + 'dependencies': [ + '<(DEPTH)/chrome/chrome.gyp:chrome_dll', + ], + 'variables': { + 'dest_dir': '<(PRODUCT_DIR)\\syzygy', + }, + 'actions': [ + { + 'action_name': 'Reorder Chrome with Syzygy', + 'msvs_cygwin_shell': 0, + 'inputs': [ + '<(PRODUCT_DIR)\\chrome.dll', + '<(PRODUCT_DIR)\\chrome_dll.pdb', + ], + 'outputs': [ + '<(dest_dir)\\chrome.dll', + '<(dest_dir)\\chrome_dll.pdb', + ], + 'action': [ + 'python', + '<(DEPTH)/chrome/tools/build/win/syzygy_reorder.py', + '--input_executable', '<(PRODUCT_DIR)\\chrome.dll', + '--input_symbol', '<(PRODUCT_DIR)\\chrome_dll.pdb', + '--destination_dir', '<(dest_dir)', + ], + }, + ], + }, + ], + }], + ], +} diff --git a/chrome/installer/mini_installer.gypi b/chrome/installer/mini_installer.gypi new file mode 100644 index 0000000..4014263 --- /dev/null +++ b/chrome/installer/mini_installer.gypi @@ -0,0 +1,213 @@ +# Copyright (c) 2011 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +{ + 'dependencies': [ + '<(chrome_dll_project)', + '../chrome.gyp:chrome', + '../chrome.gyp:chrome_nacl_win64', + '../chrome.gyp:default_extensions', + '../chrome.gyp:setup', + ], + 'include_dirs': [ + '../..', + '<(INTERMEDIATE_DIR)', + '<(SHARED_INTERMEDIATE_DIR)/chrome', + ], + 'sources': [ + 'mini_installer/appid.h', + 'mini_installer/chrome.release', + 'mini_installer/chrome_appid.cc', + 'mini_installer/configuration.cc', + 'mini_installer/configuration.h', + 'mini_installer/decompress.cc', + 'mini_installer/decompress.h', + 'mini_installer/mini_installer.cc', + 'mini_installer/mini_installer.h', + 'mini_installer/mini_installer.ico', + 'mini_installer/mini_installer.rc', + 'mini_installer/mini_installer_exe_version.rc.version', + 'mini_installer/mini_installer_resource.h', + 'mini_installer/mini_string.cc', + 'mini_installer/mini_string.h', + 'mini_installer/pe_resource.cc', + 'mini_installer/pe_resource.h', + '<(INTERMEDIATE_DIR)/packed_files.rc', + ], + # Disable precompiled headers for this project, to avoid + # linker errors when building with VS 2008. + 'msvs_precompiled_header': '', + 'msvs_precompiled_source': '', + 'msvs_settings': { + 'VCCLCompilerTool': { + 'EnableIntrinsicFunctions': 'true', + 'BufferSecurityCheck': 'false', + 'BasicRuntimeChecks': '0', + 'ExceptionHandling': '0', + }, + 'VCLinkerTool': { + 'OutputFile': '<(output_dir)/mini_installer.exe', + 'RandomizedBaseAddress': '1', + 'DataExecutionPrevention': '0', + 'AdditionalLibraryDirectories': [ + '<(DEPTH)/third_party/platformsdk_win7/files/Lib', + '<(PRODUCT_DIR)/lib' + ], + 'DelayLoadDLLs': [], + 'EntryPointSymbol': 'MainEntryPoint', + 'GenerateMapFile': 'true', + 'IgnoreAllDefaultLibraries': 'true', + 'OptimizeForWindows98': '1', + 'SubSystem': '2', # Set /SUBSYSTEM:WINDOWS + 'AdditionalDependencies': [ + 'shlwapi.lib', + 'setupapi.lib', + ], + 'conditions': [ + ['MSVS_VERSION=="2005e"', { + 'AdditionalDependencies': [ # Must explicitly link in VC2005E + 'advapi32.lib', + 'shell32.lib', + ], + }], + ], + }, + 'VCManifestTool': { + 'AdditionalManifestFiles': [ + '$(ProjectDir)\\mini_installer\\mini_installer.exe.manifest', + ], + }, + }, + 'configurations': { + 'Debug_Base': { + 'msvs_settings': { + 'VCCLCompilerTool': { + 'BasicRuntimeChecks': '0', + 'BufferSecurityCheck': 'false', + 'ExceptionHandling': '0', + }, + 'VCLinkerTool': { + 'SubSystem': '2', # Set /SUBSYSTEM:WINDOWS + 'AdditionalOptions': [ + '/safeseh:no', + '/dynamicbase:no', + '/ignore:4199', + '/ignore:4221', + '/nxcompat', + ], + }, + }, + }, + 'Release_Base': { + 'includes': ['../../build/internal/release_defaults.gypi'], + 'msvs_settings': { + 'VCCLCompilerTool': { + 'EnableIntrinsicFunctions': 'true', + 'BasicRuntimeChecks': '0', + 'BufferSecurityCheck': 'false', + 'ExceptionHandling': '0', + }, + 'VCLinkerTool': { + 'SubSystem': '2', # Set /SUBSYSTEM:WINDOWS + 'Profile': 'false', # Conflicts with /FIXED + 'AdditionalOptions': [ + '/SAFESEH:NO', + '/NXCOMPAT', + '/DYNAMICBASE:NO', + '/FIXED', + ], + }, + }, + }, + }, + 'rules': [ + { + 'rule_name': 'mini_installer_version', + 'extension': 'version', + 'variables': { + 'template_input_path': + 'mini_installer/mini_installer_exe_version.rc.version', + }, + 'inputs': [ + '<(template_input_path)', + '<(version_path)', + '<(lastchange_path)', + '<(branding_dir)/BRANDING', + ], + 'outputs': [ + '<(INTERMEDIATE_DIR)/mini_installer_exe_version.rc', + ], + 'action': [ + 'python', '<(version_py)', + '-f', '<(version_path)', + '-f', '<(lastchange_path)', + '-f', '<(branding_dir)/BRANDING', + '<(template_input_path)', + '<@(_outputs)', + ], + 'process_outputs_as_sources': 1, + 'message': 'Generating version information' + }, + { + 'rule_name': 'installer_archive', + 'extension': 'release', + 'variables': { + 'create_installer_archive_py_path': + '../tools/build/win/create_installer_archive.py', + }, + 'inputs': [ + '<(create_installer_archive_py_path)', + '<(PRODUCT_DIR)/chrome.exe', + '<(chrome_dll_path)', + '<(PRODUCT_DIR)/nacl64.exe', + '<(PRODUCT_DIR)/ppGoogleNaClPluginChrome.dll', + '<(PRODUCT_DIR)/nacl_irt_x86_32.nexe', + '<(PRODUCT_DIR)/nacl_irt_x86_64.nexe', + '<(PRODUCT_DIR)/locales/en-US.pak', + '<(PRODUCT_DIR)/icudt.dll', + ], + 'outputs': [ + 'xxx.out', + '<(output_dir)/<(RULE_INPUT_NAME).7z', + '<(output_dir)/<(RULE_INPUT_NAME).packed.7z', + '<(output_dir)/setup.ex_', + '<(INTERMEDIATE_DIR)/packed_files.rc', + ], + 'action': [ + 'python', + '<(create_installer_archive_py_path)', + '--build_dir=<(PRODUCT_DIR)', + '--output_dir=<(output_dir)', + '--staging_dir=<(INTERMEDIATE_DIR)', + '--input_file=<(RULE_INPUT_PATH)', + '--resource_file_path=<(INTERMEDIATE_DIR)/packed_files.rc', + # TODO(sgk): may just use environment variables + #'--distribution=$(CHROMIUM_BUILD)', + '--distribution=_google_chrome', + # Optional arguments to generate diff installer + #'--last_chrome_installer=C:/Temp/base', + #'--setup_exe_format=DIFF', + #'--diff_algorithm=COURGETTE', + ], + 'message': 'Create installer archive' + }, + ], + # TODO(mark): <(branding_dir) should be defined by the + # global condition block at the bottom of the file, but + # this doesn't work due to the following issue: + # + # http://code.google.com/p/gyp/issues/detail?id=22 + # + # Remove this block once the above issue is fixed. + 'conditions': [ + [ 'branding == "Chrome"', { + 'variables': { + 'branding_dir': '../app/theme/google_chrome', + }, + }, { # else branding!="Chrome" + 'variables': { + 'branding_dir': '../app/theme/chromium', + }, + }], + ], +} \ No newline at end of file diff --git a/chrome/installer/mini_installer_syzygy.gyp b/chrome/installer/mini_installer_syzygy.gyp new file mode 100644 index 0000000..461dd59 --- /dev/null +++ b/chrome/installer/mini_installer_syzygy.gyp @@ -0,0 +1,45 @@ +# Copyright (c) 2011 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +{ + 'variables': { + 'version_py': '<(DEPTH)/chrome/tools/build/version.py', + 'version_path': '<(DEPTH)/chrome/VERSION', + 'lastchange_path': '<(SHARED_INTERMEDIATE_DIR)/build/LASTCHANGE', + # 'branding_dir' is set in the 'conditions' section at the bottom. + 'msvs_use_common_release': 0, + 'msvs_use_common_linker_extras': 0, + }, + 'includes': [ + '../../build/win_precompile.gypi', + ], + 'conditions': [ + # This target won't build in fastbuild, since there are no PDBs. + ['OS=="win" and fastbuild==0', { + 'targets': [ + { + 'target_name': 'mini_installer_syzygy', + 'type': 'executable', + 'product_name': 'mini_installer', + + 'variables': { + 'chrome_dll_project': '../chrome_syzygy.gyp:chrome_dll_syzygy', + 'chrome_dll_path': '<(PRODUCT_DIR)/syzygy/chrome.dll', + 'output_dir': '<(PRODUCT_DIR)/syzygy', + }, + # Bulk of the build configuration comes from here. + 'includes': [ 'mini_installer.gypi', ], + }, + ], + }], + [ 'branding == "Chrome"', { + 'variables': { + 'branding_dir': '../app/theme/google_chrome', + }, + }, { # else branding!="Chrome" + 'variables': { + 'branding_dir': '../app/theme/chromium', + }, + }], + ], +} diff --git a/chrome/tools/build/win/create_installer_archive.py b/chrome/tools/build/win/create_installer_archive.py index 5a4e96c..6c42c1c 100755 --- a/chrome/tools/build/win/create_installer_archive.py +++ b/chrome/tools/build/win/create_installer_archive.py @@ -115,7 +115,9 @@ def CopySectionFilesToStagingDir(config, section, staging_dir, build_dir): if not os.path.exists(dst): os.makedirs(dst) for file in glob.glob(os.path.join(build_dir, option)): - shutil.copy(file, dst) + dst_file = os.path.join(dst, os.path.basename(file)) + if not os.path.exists(dst_file): + shutil.copy(file, dst) def GenerateDiffPatch(options, orig_file, new_file, patch_file): if (options.diff_algorithm == "COURGETTE"): @@ -186,7 +188,7 @@ def CreateArchiveFile(options, staging_dir, current_version, prev_version): """ # First create an uncompressed archive file for the current build (chrome.7z) lzma_exec = GetLZMAExec(options.build_dir) - archive_file = os.path.join(options.build_dir, + archive_file = os.path.join(options.output_dir, options.output_name + ARCHIVE_SUFFIX) cmd = [lzma_exec, 'a', @@ -220,7 +222,7 @@ def CreateArchiveFile(options, staging_dir, current_version, prev_version): compressed_archive_file = options.output_name + COMPRESSED_ARCHIVE_SUFFIX orig_file = archive_file - compressed_archive_file_path = os.path.join(options.build_dir, + compressed_archive_file_path = os.path.join(options.output_dir, compressed_archive_file) CompressUsingLZMA(options.build_dir, compressed_archive_file_path, orig_file) @@ -248,7 +250,7 @@ def PrepareSetupExec(options, staging_dir, current_version, prev_version): cmd = ['makecab.exe', '/D', 'CompressionType=LZX', '/V1', - '/L', options.build_dir, + '/L', options.output_dir, os.path.join(options.build_dir, SETUP_EXEC),] RunSystemCommand(cmd) setup_file = SETUP_EXEC[:-1] + "_" @@ -316,6 +318,14 @@ def main(options): prev_version = GetPrevVersion(options.build_dir, temp_dir, options.last_chrome_installer) + # Preferentially copy the files we can find from the output_dir, as + # this is where we'll find the Syzygy-optimized executables when + # building the optimized mini_installer. + if options.build_dir != options.output_dir: + CopyAllFilesToStagingDir(config, options.distribution, + staging_dir, options.output_dir) + + # Now copy the remainder of the files from the build dir. CopyAllFilesToStagingDir(config, options.distribution, staging_dir, options.build_dir) @@ -341,11 +351,14 @@ def _ParseOptions(): parser = optparse.OptionParser() parser.add_option('-i', '--input_file', help='Input file describing which files to archive.') - parser.add_option('-o', '--build_dir', + parser.add_option('-b', '--build_dir', help='Build directory. The paths in input_file are relative to this.') parser.add_option('--staging_dir', help='Staging directory where intermediate files and directories ' 'will be created'), + parser.add_option('-o', '--output_dir', + help='The output directory where the archives will be written. ' + 'Defaults to the build_dir.') parser.add_option('--resource_file_path', help='The path where the resource file will be output. ' 'Defaults to %s in the build directory.' % @@ -368,20 +381,22 @@ def _ParseOptions(): help='Name used to prefix names of generated archives.') options, args = parser.parse_args() - if not options.build_dir: parser.error('You must provide a build dir.') if not options.staging_dir: parser.error('You must provide a staging dir.') + if not options.output_dir: + options.output_dir = options.build_dir + if not options.resource_file_path: options.options.resource_file_path = os.path.join(options.build_dir, MINI_INSTALLER_INPUT_FILE) - print sys.argv return options if '__main__' == __name__: + print sys.argv sys.exit(main(_ParseOptions())) diff --git a/chrome/tools/build/win/syzygy_reorder.py b/chrome/tools/build/win/syzygy_reorder.py new file mode 100644 index 0000000..a08f8b9 --- /dev/null +++ b/chrome/tools/build/win/syzygy_reorder.py @@ -0,0 +1,111 @@ +#!/usr/bin/python +# Copyright (c) 2011 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A utility script to help building Syzygy-reordered Chrome binaries.""" + +import logging +import optparse +import os +import subprocess +import sys + + +# The default relink executable to use to reorder binaries. +_DEFAULT_RELINKER = os.path.join( + os.path.join(os.path.dirname(__file__), '../../../..'), + 'third_party/syzygy/binaries/exe/relink.exe') + +_LOGGER = logging.getLogger() + +# We use the same seed for all random reorderings to get a deterministic build. +_RANDOM_SEED = 1347344 + + +def _Shell(*cmd, **kw): + """Shells out to "cmd". Returns a tuple of cmd's stdout, stderr.""" + _LOGGER.info('Running command "%s".', cmd) + prog = subprocess.Popen(cmd, **kw) + + stdout, stderr = prog.communicate() + if prog.returncode != 0: + raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode)) + + return stdout, stderr + + +def _ReorderBinary(relink_exe, executable, symbol, destination_dir): + """Reorders the executable found in input_dir, and writes the resultant + reordered executable and symbol files to destination_dir. + + If a file named -order.json exists, imposes that order on the + output binaries, otherwise orders them randomly. + """ + cmd = [relink_exe, + '--verbose', + '--input-dll=%s' % executable, + '--input-pdb=%s' % symbol, + '--output-dll=%s' % os.path.join(destination_dir, + os.path.basename(executable)), + '--output-pdb=%s' % os.path.join(destination_dir, + os.path.basename(symbol)),] + + # Check whether there's an order file available for the executable. + order_file = '%s-order.json' % executable + if os.path.exists(order_file): + # The ordering file exists, let's use that. + _LOGGER.info('Reordering "%s" according to "%s".', + os.path.basename(executable), + os.path.basename(order_file)) + cmd.append('--order-file=%s' % order_file) + else: + # No ordering file, we randomize the output. + _LOGGER.info('Randomly reordering "%s"', executable) + cmd.append('--seed=%d' % _RANDOM_SEED) + + return _Shell(*cmd) + + +def main(options): + logging.basicConfig(level=logging.INFO) + + # Make sure the destination directory exists. + if not os.path.isdir(options.destination_dir): + _LOGGER.info('Creating destination directory "%s".', + options.destination_dir) + os.makedirs(options.destination_dir) + + # Reorder the binaries into the destination directory. + _ReorderBinary(options.relinker, + options.input_executable, + options.input_symbol, + options.destination_dir) + + +def _ParseOptions(): + option_parser = optparse.OptionParser() + option_parser.add_option('--input_executable', + help='The path to the input executable.') + option_parser.add_option('--input_symbol', + help='The path to the input symbol file.') + option_parser.add_option('--relinker', default=_DEFAULT_RELINKER, + help='Relinker exectuable to use, defaults to "%s"' % _DEFAULT_RELINKER) + option_parser.add_option('-d', '--destination_dir', + help='Destination directory for reordered files, defaults to ' + 'the subdirectory "reordered" in the output_dir.') + options, args = option_parser.parse_args() + + if not options.input_executable: + option_parser.error('You must provide an input executable.') + if not options.input_symbol: + option_parser.error('You must provide an input symbol file.') + + if not options.destination_dir: + options.destination_dir = os.path.join(options.output_dir, 'reordered') + + return options + + +if '__main__' == __name__: + sys.exit(main(_ParseOptions())) -- cgit v1.1