#!/usr/bin/env python # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """MB - the Meta-Build wrapper around GYP and GN MB is a wrapper script for GYP and GN that can be used to generate build files for sets of canned configurations and analyze them. """ from __future__ import print_function import argparse import ast import errno import json import os import pipes import pprint import re import shutil import sys import subprocess import tempfile def main(args): mbw = MetaBuildWrapper() mbw.ParseArgs(args) return mbw.args.func() class MetaBuildWrapper(object): def __init__(self): p = os.path d = os.path.dirname self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__))))) self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb', 'mb_config.pyl') self.executable = sys.executable self.platform = sys.platform self.sep = os.sep self.args = argparse.Namespace() self.configs = {} self.masters = {} self.mixins = {} self.private_configs = [] self.common_dev_configs = [] self.unsupported_configs = [] def ParseArgs(self, argv): def AddCommonOptions(subp): subp.add_argument('-b', '--builder', help='builder name to look up config from') subp.add_argument('-m', '--master', help='master name to look up config from') subp.add_argument('-c', '--config', help='configuration to analyze') subp.add_argument('-f', '--config-file', metavar='PATH', default=self.default_config, help='path to config file ' '(default is //tools/mb/mb_config.pyl)') subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'), help='path to goma directory (default is %(default)s).') subp.add_argument('-n', '--dryrun', action='store_true', help='Do a dry run (i.e., do nothing, just print ' 'the commands that will run)') subp.add_argument('-v', '--verbose', action='store_true', help='verbose logging') parser = argparse.ArgumentParser(prog='mb') subps = parser.add_subparsers() subp = subps.add_parser('analyze', help='analyze whether changes to a set of files ' 'will cause a set of binaries to be rebuilt.') AddCommonOptions(subp) subp.add_argument('path', nargs=1, help='path build was generated into.') subp.add_argument('input_path', nargs=1, help='path to a file containing the input arguments ' 'as a JSON object.') subp.add_argument('output_path', nargs=1, help='path to a file containing the output arguments ' 'as a JSON object.') subp.set_defaults(func=self.CmdAnalyze) subp = subps.add_parser('gen', help='generate a new set of build files') AddCommonOptions(subp) subp.add_argument('--swarming-targets-file', help='save runtime dependencies for targets listed ' 'in file.') subp.add_argument('path', nargs=1, help='path to generate build into') subp.set_defaults(func=self.CmdGen) subp = subps.add_parser('isolate', help='generate the .isolate files for a given' 'binary') AddCommonOptions(subp) subp.add_argument('path', nargs=1, help='path build was generated into') subp.add_argument('target', nargs=1, help='ninja target to generate the isolate for') subp.set_defaults(func=self.CmdIsolate) subp = subps.add_parser('lookup', help='look up the command for a given config or ' 'builder') AddCommonOptions(subp) subp.set_defaults(func=self.CmdLookup) subp = subps.add_parser('run', help='build and run the isolated version of a ' 'binary') AddCommonOptions(subp) subp.add_argument('-j', '--jobs', dest='jobs', type=int, help='Number of jobs to pass to ninja') subp.add_argument('--no-build', dest='build', default=True, action='store_false', help='Do not build, just isolate and run') subp.add_argument('path', nargs=1, help='path to generate build into') subp.add_argument('target', nargs=1, help='ninja target to build and run') subp.set_defaults(func=self.CmdRun) subp = subps.add_parser('validate', help='validate the config file') subp.add_argument('-f', '--config-file', metavar='PATH', default=self.default_config, help='path to config file ' '(default is //tools/mb/mb_config.pyl)') subp.set_defaults(func=self.CmdValidate) subp = subps.add_parser('help', help='Get help on a subcommand.') subp.add_argument(nargs='?', action='store', dest='subcommand', help='The command to get help for.') subp.set_defaults(func=self.CmdHelp) self.args = parser.parse_args(argv) def CmdAnalyze(self): vals = self.Lookup() if vals['type'] == 'gn': return self.RunGNAnalyze(vals) else: return self.RunGYPAnalyze(vals) def CmdGen(self): vals = self.Lookup() self.ClobberIfNeeded(vals) if vals['type'] == 'gn': return self.RunGNGen(vals) else: return self.RunGYPGen(vals) def CmdHelp(self): if self.args.subcommand: self.ParseArgs([self.args.subcommand, '--help']) else: self.ParseArgs(['--help']) def CmdIsolate(self): vals = self.GetConfig() if not vals: return 1 if vals['type'] == 'gn': return self.RunGNIsolate(vals) else: return self.Build('%s_run' % self.args.target[0]) def CmdLookup(self): vals = self.Lookup() if vals['type'] == 'gn': cmd = self.GNCmd('gen', '_path_', vals['gn_args']) env = None else: cmd, env = self.GYPCmd('_path_', vals) self.PrintCmd(cmd, env) return 0 def CmdRun(self): vals = self.GetConfig() if not vals: return 1 build_dir = self.args.path[0] target = self.args.target[0] if vals['type'] == 'gn': if self.args.build: ret = self.Build(target) if ret: return ret ret = self.RunGNIsolate(vals) if ret: return ret else: ret = self.Build('%s_run' % target) if ret: return ret ret, _, _ = self.Run([ self.executable, self.PathJoin('tools', 'swarming_client', 'isolate.py'), 'run', '-s', self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target))], force_verbose=False, buffer_output=False) return ret def CmdValidate(self): errs = [] # Read the file to make sure it parses. self.ReadConfigFile() # Figure out the whole list of configs and ensure that no config is # listed in more than one category. all_configs = {} for config in self.common_dev_configs: all_configs[config] = 'common_dev_configs' for config in self.private_configs: if config in all_configs: errs.append('config "%s" listed in "private_configs" also ' 'listed in "%s"' % (config, all_configs['config'])) else: all_configs[config] = 'private_configs' for config in self.unsupported_configs: if config in all_configs: errs.append('config "%s" listed in "unsupported_configs" also ' 'listed in "%s"' % (config, all_configs['config'])) else: all_configs[config] = 'unsupported_configs' for master in self.masters: for builder in self.masters[master]: config = self.masters[master][builder] if config in all_configs and all_configs[config] not in self.masters: errs.append('Config "%s" used by a bot is also listed in "%s".' % (config, all_configs[config])) else: all_configs[config] = master # Check that every referenced config actually exists. for config, loc in all_configs.items(): if not config in self.configs: errs.append('Unknown config "%s" referenced from "%s".' % (config, loc)) # Check that every actual config is actually referenced. for config in self.configs: if not config in all_configs: errs.append('Unused config "%s".' % config) # Figure out the whole list of mixins, and check that every mixin # listed by a config or another mixin actually exists. referenced_mixins = set() for config, mixins in self.configs.items(): for mixin in mixins: if not mixin in self.mixins: errs.append('Unknown mixin "%s" referenced by config "%s".' % (mixin, config)) referenced_mixins.add(mixin) for mixin in self.mixins: for sub_mixin in self.mixins[mixin].get('mixins', []): if not sub_mixin in self.mixins: errs.append('Unknown mixin "%s" referenced by mixin "%s".' % (sub_mixin, mixin)) referenced_mixins.add(sub_mixin) # Check that every mixin defined is actually referenced somewhere. for mixin in self.mixins: if not mixin in referenced_mixins: errs.append('Unreferenced mixin "%s".' % mixin) if errs: raise MBErr(('mb config file %s has problems:' % self.args.config_file) + '\n ' + '\n '.join(errs)) self.Print('mb config file %s looks ok.' % self.args.config_file) return 0 def GetConfig(self): build_dir = self.args.path[0] vals = {} if self.args.builder or self.args.master or self.args.config: vals = self.Lookup() if vals['type'] == 'gn': # Re-run gn gen in order to ensure the config is consistent with the # build dir. self.RunGNGen(vals) return vals # TODO: We can only get the config for GN build dirs, not GYP build dirs. # GN stores the args that were used in args.gn in the build dir, # but GYP doesn't store them anywhere. We should consider modifying # gyp_chromium to record the arguments it runs with in a similar # manner. mb_type_path = self.PathJoin(self.ToAbsPath(build_dir), 'mb_type') if not self.Exists(mb_type_path): gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn') if not self.Exists(gn_args_path): self.Print('Must either specify a path to an existing GN build dir ' 'or pass in a -m/-b pair or a -c flag to specify the ' 'configuration') return {} else: mb_type = 'gn' else: mb_type = self.ReadFile(mb_type_path).strip() if mb_type == 'gn': vals = self.GNValsFromDir(build_dir) else: vals = {} vals['type'] = mb_type return vals def GNValsFromDir(self, build_dir): args_contents = self.ReadFile( self.PathJoin(self.ToAbsPath(build_dir), 'args.gn')) gn_args = [] for l in args_contents.splitlines(): fields = l.split(' ') name = fields[0] val = ' '.join(fields[2:]) gn_args.append('%s=%s' % (name, val)) return { 'gn_args': ' '.join(gn_args), 'type': 'gn', } def Lookup(self): vals = self.ReadBotConfig() if not vals: self.ReadConfigFile() config = self.ConfigFromArgs() if not config in self.configs: raise MBErr('Config "%s" not found in %s' % (config, self.args.config_file)) vals = self.FlattenConfig(config) # Do some basic sanity checking on the config so that we # don't have to do this in every caller. assert 'type' in vals, 'No meta-build type specified in the config' assert vals['type'] in ('gn', 'gyp'), ( 'Unknown meta-build type "%s"' % vals['gn_args']) return vals def ReadBotConfig(self): if not self.args.master or not self.args.builder: return {} path = self.PathJoin(self.chromium_src_dir, 'ios', 'build', 'bots', self.args.master, self.args.builder + '.json') if not self.Exists(path): return {} contents = json.loads(self.ReadFile(path)) gyp_vals = contents.get('GYP_DEFINES', {}) if isinstance(gyp_vals, dict): gyp_defines = ' '.join('%s=%s' % (k, v) for k, v in gyp_vals.items()) else: gyp_defines = ' '.join(gyp_vals) gn_args = ' '.join(contents.get('gn_args', [])) return { 'type': contents.get('mb_type', ''), 'gn_args': gn_args, 'gyp_defines': gyp_defines, 'gyp_crosscompile': False, } def ReadConfigFile(self): if not self.Exists(self.args.config_file): raise MBErr('config file not found at %s' % self.args.config_file) try: contents = ast.literal_eval(self.ReadFile(self.args.config_file)) except SyntaxError as e: raise MBErr('Failed to parse config file "%s": %s' % (self.args.config_file, e)) self.common_dev_configs = contents['common_dev_configs'] self.configs = contents['configs'] self.masters = contents['masters'] self.mixins = contents['mixins'] self.private_configs = contents['private_configs'] self.unsupported_configs = contents['unsupported_configs'] def ConfigFromArgs(self): if self.args.config: if self.args.master or self.args.builder: raise MBErr('Can not specific both -c/--config and -m/--master or ' '-b/--builder') return self.args.config if not self.args.master or not self.args.builder: raise MBErr('Must specify either -c/--config or ' '(-m/--master and -b/--builder)') if not self.args.master in self.masters: raise MBErr('Master name "%s" not found in "%s"' % (self.args.master, self.args.config_file)) if not self.args.builder in self.masters[self.args.master]: raise MBErr('Builder name "%s" not found under masters[%s] in "%s"' % (self.args.builder, self.args.master, self.args.config_file)) return self.masters[self.args.master][self.args.builder] def FlattenConfig(self, config): mixins = self.configs[config] vals = { 'type': None, 'gn_args': [], 'gyp_defines': '', 'gyp_crosscompile': False, } visited = [] self.FlattenMixins(mixins, vals, visited) return vals def FlattenMixins(self, mixins, vals, visited): for m in mixins: if m not in self.mixins: raise MBErr('Unknown mixin "%s"' % m) # TODO: check for cycles in mixins. visited.append(m) mixin_vals = self.mixins[m] if 'type' in mixin_vals: vals['type'] = mixin_vals['type'] if 'gn_args' in mixin_vals: if vals['gn_args']: vals['gn_args'] += ' ' + mixin_vals['gn_args'] else: vals['gn_args'] = mixin_vals['gn_args'] if 'gyp_crosscompile' in mixin_vals: vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile'] if 'gyp_defines' in mixin_vals: if vals['gyp_defines']: vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines'] else: vals['gyp_defines'] = mixin_vals['gyp_defines'] if 'mixins' in mixin_vals: self.FlattenMixins(mixin_vals['mixins'], vals, visited) return vals def ClobberIfNeeded(self, vals): path = self.args.path[0] build_dir = self.ToAbsPath(path) mb_type_path = self.PathJoin(build_dir, 'mb_type') needs_clobber = False new_mb_type = vals['type'] if self.Exists(build_dir): if self.Exists(mb_type_path): old_mb_type = self.ReadFile(mb_type_path) if old_mb_type != new_mb_type: self.Print("Build type mismatch: was %s, will be %s, clobbering %s" % (old_mb_type, new_mb_type, path)) needs_clobber = True else: # There is no 'mb_type' file in the build directory, so this probably # means that the prior build(s) were not done through mb, and we # have no idea if this was a GYP build or a GN build. Clobber it # to be safe. self.Print("%s/mb_type missing, clobbering to be safe" % path) needs_clobber = True if self.args.dryrun: return if needs_clobber: self.RemoveDirectory(build_dir) self.MaybeMakeDirectory(build_dir) self.WriteFile(mb_type_path, new_mb_type) def RunGNGen(self, vals): build_dir = self.args.path[0] cmd = self.GNCmd('gen', build_dir, vals['gn_args'], extra_args=['--check']) swarming_targets = [] if getattr(self.args, 'swarming_targets_file', None): # We need GN to generate the list of runtime dependencies for # the compile targets listed (one per line) in the file so # we can run them via swarming. We use ninja_to_gn.pyl to convert # the compile targets to the matching GN labels. contents = self.ReadFile(self.args.swarming_targets_file) swarming_targets = contents.splitlines() gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin( self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl'))) gn_labels = [] for target in swarming_targets: if not target in gn_isolate_map: raise MBErr('test target "%s" not found in %s' % (target, '//testing/buildbot/gn_isolate_map.pyl')) gn_labels.append(gn_isolate_map[target]['label']) gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps') # Since GN hasn't run yet, the build directory may not even exist. self.MaybeMakeDirectory(self.ToAbsPath(build_dir)) self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n') cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path) ret, _, _ = self.Run(cmd) if ret: # If `gn gen` failed, we should exit early rather than trying to # generate isolates. Run() will have already logged any error output. self.Print('GN gen failed: %d' % ret) return ret for target in swarming_targets: if gn_isolate_map[target]['type'] == 'gpu_browser_test': runtime_deps_target = 'browser_tests' elif gn_isolate_map[target]['type'] == 'script': # For script targets, the build target is usually a group, # for which gn generates the runtime_deps next to the stamp file # for the label, which lives under the obj/ directory. label = gn_isolate_map[target]['label'] runtime_deps_target = 'obj/%s.stamp' % label.replace(':', '/') else: runtime_deps_target = target if self.platform == 'win32': deps_path = self.ToAbsPath(build_dir, runtime_deps_target + '.exe.runtime_deps') else: deps_path = self.ToAbsPath(build_dir, runtime_deps_target + '.runtime_deps') if not self.Exists(deps_path): raise MBErr('did not generate %s' % deps_path) command, extra_files = self.GetIsolateCommand(target, vals, gn_isolate_map) runtime_deps = self.ReadFile(deps_path).splitlines() self.WriteIsolateFiles(build_dir, command, target, runtime_deps, extra_files) return 0 def RunGNIsolate(self, vals): gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin( self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl'))) build_dir = self.args.path[0] target = self.args.target[0] command, extra_files = self.GetIsolateCommand(target, vals, gn_isolate_map) label = gn_isolate_map[target]['label'] ret, out, _ = self.Call(['gn', 'desc', build_dir, label, 'runtime_deps']) if ret: return ret runtime_deps = out.splitlines() self.WriteIsolateFiles(build_dir, command, target, runtime_deps, extra_files) ret, _, _ = self.Run([ self.executable, self.PathJoin('tools', 'swarming_client', 'isolate.py'), 'check', '-i', self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), '-s', self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target))], buffer_output=False) return ret def WriteIsolateFiles(self, build_dir, command, target, runtime_deps, extra_files): isolate_path = self.ToAbsPath(build_dir, target + '.isolate') self.WriteFile(isolate_path, pprint.pformat({ 'variables': { 'command': command, 'files': sorted(runtime_deps + extra_files), } }) + '\n') self.WriteJSON( { 'args': [ '--isolated', self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)), '--isolate', self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), ], 'dir': self.chromium_src_dir, 'version': 1, }, isolate_path + 'd.gen.json', ) def GNCmd(self, subcommand, path, gn_args='', extra_args=None): if self.platform == 'linux2': subdir = 'linux64' elif self.platform == 'darwin': subdir = 'mac' else: subdir = 'win' gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, 'gn') cmd = [gn_path, subcommand, path] gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir) if gn_args: cmd.append('--args=%s' % gn_args) if extra_args: cmd.extend(extra_args) return cmd def RunGYPGen(self, vals): path = self.args.path[0] output_dir = self.ParseGYPConfigPath(path) cmd, env = self.GYPCmd(output_dir, vals) ret, _, _ = self.Run(cmd, env=env) return ret def RunGYPAnalyze(self, vals): output_dir = self.ParseGYPConfigPath(self.args.path[0]) if self.args.verbose: inp = self.ReadInputJSON(['files', 'test_targets', 'additional_compile_targets']) self.Print() self.Print('analyze input:') self.PrintJSON(inp) self.Print() cmd, env = self.GYPCmd(output_dir, vals) cmd.extend(['-f', 'analyzer', '-G', 'config_path=%s' % self.args.input_path[0], '-G', 'analyzer_output_path=%s' % self.args.output_path[0]]) ret, _, _ = self.Run(cmd, env=env) if not ret and self.args.verbose: outp = json.loads(self.ReadFile(self.args.output_path[0])) self.Print() self.Print('analyze output:') self.PrintJSON(outp) self.Print() return ret def GetIsolateCommand(self, target, vals, gn_isolate_map): # This needs to mirror the settings in //build/config/ui.gni: # use_x11 = is_linux && !use_ozone. # TODO(dpranke): Figure out how to keep this in sync better. use_x11 = (self.platform == 'linux2' and not 'target_os="android"' in vals['gn_args'] and not 'use_ozone=true' in vals['gn_args']) asan = 'is_asan=true' in vals['gn_args'] msan = 'is_msan=true' in vals['gn_args'] tsan = 'is_tsan=true' in vals['gn_args'] executable_suffix = '.exe' if self.platform == 'win32' else '' test_type = gn_isolate_map[target]['type'] cmdline = [] extra_files = [] if use_x11 and test_type == 'windowed_test_launcher': extra_files = [ 'xdisplaycheck', '../../testing/test_env.py', '../../testing/xvfb.py', ] cmdline = [ '../../testing/xvfb.py', '.', './' + str(target), '--brave-new-test-launcher', '--test-launcher-bot-mode', '--asan=%d' % asan, '--msan=%d' % msan, '--tsan=%d' % tsan, ] elif test_type in ('windowed_test_launcher', 'console_test_launcher'): extra_files = [ '../../testing/test_env.py' ] cmdline = [ '../../testing/test_env.py', './' + str(target) + executable_suffix, '--brave-new-test-launcher', '--test-launcher-bot-mode', '--asan=%d' % asan, '--msan=%d' % msan, '--tsan=%d' % tsan, ] elif test_type == 'gpu_browser_test': extra_files = [ '../../testing/test_env.py' ] gtest_filter = gn_isolate_map[target]['gtest_filter'] cmdline = [ '../../testing/test_env.py', './browser_tests' + executable_suffix, '--test-launcher-bot-mode', '--enable-gpu', '--test-launcher-jobs=1', '--gtest_filter=%s' % gtest_filter, ] elif test_type == 'script': extra_files = [ '../../testing/test_env.py' ] cmdline = [ '../../testing/test_env.py', '../../' + self.ToSrcRelPath(gn_isolate_map[target]['script']) ] + gn_isolate_map[target].get('args', []) elif test_type in ('raw'): extra_files = [] cmdline = [ './' + str(target) + executable_suffix, ] + gn_isolate_map[target].get('args') else: self.WriteFailureAndRaise('No command line for %s found (test type %s).' % (target, test_type), output_path=None) return cmdline, extra_files def ToAbsPath(self, build_path, *comps): return self.PathJoin(self.chromium_src_dir, self.ToSrcRelPath(build_path), *comps) def ToSrcRelPath(self, path): """Returns a relative path from the top of the repo.""" # TODO: Support normal paths in addition to source-absolute paths. assert(path.startswith('//')) return path[2:].replace('/', self.sep) def ParseGYPConfigPath(self, path): rpath = self.ToSrcRelPath(path) output_dir, _, _ = rpath.rpartition(self.sep) return output_dir def GYPCmd(self, output_dir, vals): gyp_defines = vals['gyp_defines'] goma_dir = self.args.goma_dir # GYP uses shlex.split() to split the gyp defines into separate arguments, # so we can support backslashes and and spaces in arguments by quoting # them, even on Windows, where this normally wouldn't work. if '\\' in goma_dir or ' ' in goma_dir: goma_dir = "'%s'" % goma_dir gyp_defines = gyp_defines.replace("$(goma_dir)", goma_dir) cmd = [ self.executable, self.PathJoin('build', 'gyp_chromium'), '-G', 'output_dir=' + output_dir, ] # Ensure that we have an environment that only contains # the exact values of the GYP variables we need. env = os.environ.copy() env['GYP_GENERATORS'] = 'ninja' if 'GYP_CHROMIUM_NO_ACTION' in env: del env['GYP_CHROMIUM_NO_ACTION'] if 'GYP_CROSSCOMPILE' in env: del env['GYP_CROSSCOMPILE'] env['GYP_DEFINES'] = gyp_defines if vals['gyp_crosscompile']: env['GYP_CROSSCOMPILE'] = '1' return cmd, env def RunGNAnalyze(self, vals): # analyze runs before 'gn gen' now, so we need to run gn gen # in order to ensure that we have a build directory. ret = self.RunGNGen(vals) if ret: return ret inp = self.ReadInputJSON(['files', 'test_targets', 'additional_compile_targets']) if self.args.verbose: self.Print() self.Print('analyze input:') self.PrintJSON(inp) self.Print() # TODO(crbug.com/555273) - currently GN treats targets and # additional_compile_targets identically since we can't tell the # difference between a target that is a group in GN and one that isn't. # We should eventually fix this and treat the two types differently. targets = (set(inp['test_targets']) | set(inp['additional_compile_targets'])) output_path = self.args.output_path[0] # Bail out early if a GN file was modified, since 'gn refs' won't know # what to do about it. Also, bail out early if 'all' was asked for, # since we can't deal with it yet. if (any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']) or 'all' in targets): self.WriteJSON({ 'status': 'Found dependency (all)', 'compile_targets': sorted(targets), 'test_targets': sorted(targets & set(inp['test_targets'])), }, output_path) return 0 # This shouldn't normally happen, but could due to unusual race conditions, # like a try job that gets scheduled before a patch lands but runs after # the patch has landed. if not inp['files']: self.Print('Warning: No files modified in patch, bailing out early.') self.WriteJSON({ 'status': 'No dependency', 'compile_targets': [], 'test_targets': [], }, output_path) return 0 ret = 0 response_file = self.TempFile() response_file.write('\n'.join(inp['files']) + '\n') response_file.close() matching_targets = set() try: cmd = self.GNCmd('refs', self.args.path[0]) + [ '@%s' % response_file.name, '--all', '--as=output'] ret, out, _ = self.Run(cmd, force_verbose=False) if ret and not 'The input matches no targets' in out: self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out), output_path) build_dir = self.ToSrcRelPath(self.args.path[0]) + self.sep for output in out.splitlines(): build_output = output.replace(build_dir, '') if build_output in targets: matching_targets.add(build_output) cmd = self.GNCmd('refs', self.args.path[0]) + [ '@%s' % response_file.name, '--all'] ret, out, _ = self.Run(cmd, force_verbose=False) if ret and not 'The input matches no targets' in out: self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out), output_path) for label in out.splitlines(): build_target = label[2:] # We want to accept 'chrome/android:chrome_public_apk' and # just 'chrome_public_apk'. This may result in too many targets # getting built, but we can adjust that later if need be. for input_target in targets: if (input_target == build_target or build_target.endswith(':' + input_target)): matching_targets.add(input_target) finally: self.RemoveFile(response_file.name) if matching_targets: self.WriteJSON({ 'status': 'Found dependency', 'compile_targets': sorted(matching_targets), 'test_targets': sorted(matching_targets & set(inp['test_targets'])), }, output_path) else: self.WriteJSON({ 'status': 'No dependency', 'compile_targets': [], 'test_targets': [], }, output_path) if self.args.verbose: outp = json.loads(self.ReadFile(output_path)) self.Print() self.Print('analyze output:') self.PrintJSON(outp) self.Print() return 0 def ReadInputJSON(self, required_keys): path = self.args.input_path[0] output_path = self.args.output_path[0] if not self.Exists(path): self.WriteFailureAndRaise('"%s" does not exist' % path, output_path) try: inp = json.loads(self.ReadFile(path)) except Exception as e: self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' % (path, e), output_path) for k in required_keys: if not k in inp: self.WriteFailureAndRaise('input file is missing a "%s" key' % k, output_path) return inp def WriteFailureAndRaise(self, msg, output_path): if output_path: self.WriteJSON({'error': msg}, output_path, force_verbose=True) raise MBErr(msg) def WriteJSON(self, obj, path, force_verbose=False): try: self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n', force_verbose=force_verbose) except Exception as e: raise MBErr('Error %s writing to the output path "%s"' % (e, path)) def PrintCmd(self, cmd, env): if self.platform == 'win32': env_prefix = 'set ' env_quoter = QuoteForSet shell_quoter = QuoteForCmd else: env_prefix = '' env_quoter = pipes.quote shell_quoter = pipes.quote def print_env(var): if env and var in env: self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var]))) print_env('GYP_CROSSCOMPILE') print_env('GYP_DEFINES') if cmd[0] == self.executable: cmd = ['python'] + cmd[1:] self.Print(*[shell_quoter(arg) for arg in cmd]) def PrintJSON(self, obj): self.Print(json.dumps(obj, indent=2, sort_keys=True)) def Print(self, *args, **kwargs): # This function largely exists so it can be overridden for testing. print(*args, **kwargs) def Build(self, target): build_dir = self.ToSrcRelPath(self.args.path[0]) ninja_cmd = ['ninja', '-C', build_dir] if self.args.jobs: ninja_cmd.extend(['-j', '%d' % self.args.jobs]) ninja_cmd.append(target) ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False) return ret def Run(self, cmd, env=None, force_verbose=True, buffer_output=True): # This function largely exists so it can be overridden for testing. if self.args.dryrun or self.args.verbose or force_verbose: self.PrintCmd(cmd, env) if self.args.dryrun: return 0, '', '' ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output) if self.args.verbose or force_verbose: if ret: self.Print(' -> returned %d' % ret) if out: self.Print(out, end='') if err: self.Print(err, end='', file=sys.stderr) return ret, out, err def Call(self, cmd, env=None, buffer_output=True): if buffer_output: p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) out, err = p.communicate() else: p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir, env=env) p.wait() out = err = '' return p.returncode, out, err def ExpandUser(self, path): # This function largely exists so it can be overridden for testing. return os.path.expanduser(path) def Exists(self, path): # This function largely exists so it can be overridden for testing. return os.path.exists(path) def MaybeMakeDirectory(self, path): try: os.makedirs(path) except OSError, e: if e.errno != errno.EEXIST: raise def PathJoin(self, *comps): # This function largely exists so it can be overriden for testing. return os.path.join(*comps) def ReadFile(self, path): # This function largely exists so it can be overriden for testing. with open(path) as fp: return fp.read() def RemoveFile(self, path): # This function largely exists so it can be overriden for testing. os.remove(path) def RemoveDirectory(self, abs_path): if self.platform == 'win32': # In other places in chromium, we often have to retry this command # because we're worried about other processes still holding on to # file handles, but when MB is invoked, it will be early enough in the # build that their should be no other processes to interfere. We # can change this if need be. self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path]) else: shutil.rmtree(abs_path, ignore_errors=True) def TempFile(self, mode='w'): # This function largely exists so it can be overriden for testing. return tempfile.NamedTemporaryFile(mode=mode, delete=False) def WriteFile(self, path, contents, force_verbose=False): # This function largely exists so it can be overriden for testing. if self.args.dryrun or self.args.verbose or force_verbose: self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) with open(path, 'w') as fp: return fp.write(contents) class MBErr(Exception): pass # See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful # details of this next section, which handles escaping command lines # so that they can be copied and pasted into a cmd window. UNSAFE_FOR_SET = set('^<>&|') UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%')) ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"')) def QuoteForSet(arg): if any(a in UNSAFE_FOR_SET for a in arg): arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg) return arg def QuoteForCmd(arg): # First, escape the arg so that CommandLineToArgvW will parse it properly. # From //tools/gyp/pylib/gyp/msvs_emulation.py:23. if arg == '' or ' ' in arg or '"' in arg: quote_re = re.compile(r'(\\*)"') arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) # Then check to see if the arg contains any metacharacters other than # double quotes; if it does, quote everything (including the double # quotes) for safety. if any(a in UNSAFE_FOR_CMD for a in arg): arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg) return arg if __name__ == '__main__': try: sys.exit(main(sys.argv[1:])) except MBErr as e: print(e) sys.exit(1) except KeyboardInterrupt: print("interrupted, exiting", stream=sys.stderr) sys.exit(130)