diff options
| -rw-r--r-- | DEPS | 5 | ||||
| -rwxr-xr-x | build/android/buildbot/buildbot_functions.sh | 47 | ||||
| -rwxr-xr-x | build/gyp_chromium | 36 | ||||
| -rw-r--r-- | build/gyp_helper.py | 50 | ||||
| -rwxr-xr-x | build/landmines.py | 210 |
5 files changed, 298 insertions, 50 deletions
@@ -631,4 +631,9 @@ hooks = [ "pattern": ".", "action": ["python", "src/build/gyp_chromium"], }, + { + # Check for landmines (reasons to clobber the build). + "pattern": ".", + "action": ["python", "src/build/landmines.py"], + }, ] diff --git a/build/android/buildbot/buildbot_functions.sh b/build/android/buildbot/buildbot_functions.sh index 6030e67..9f56a18 100755 --- a/build/android/buildbot/buildbot_functions.sh +++ b/build/android/buildbot/buildbot_functions.sh @@ -47,18 +47,6 @@ function bb_baseline_setup { shift cd $SRC_ROOT - if [[ $BUILDBOT_CLOBBER ]]; then - echo "@@@BUILD_STEP Clobber@@@" - # Sdk key expires, delete android folder. - # crbug.com/145860 - rm -rf ~/.android - rm -rf "${SRC_ROOT}"/out - if [ -e "${SRC_ROOT}"/out ] ; then - echo "Clobber appeared to fail? ${SRC_ROOT}/out still exists." - echo "@@@STEP_WARNINGS@@@" - fi - fi - echo "@@@BUILD_STEP Environment setup@@@" bb_parse_args "$@" @@ -68,17 +56,44 @@ function bb_baseline_setup { fi export GOMA_DIR=/b/build/goma . build/android/envsetup.sh - adb kill-server - adb start-server -} -function bb_compile_setup { local extra_gyp_defines="$(bb_get_json_prop "$FACTORY_PROPERTIES" \ extra_gyp_defines)" export GYP_DEFINES+=" fastbuild=1 $extra_gyp_defines" if echo $extra_gyp_defines | grep -q clang; then unset CXX_target fi + + adb kill-server + adb start-server + + local build_path="${SRC_ROOT}/out/${BUILDTYPE}" + local landmines_triggered_path="$build_path/.landmines_triggered" + python "$SRC_ROOT/build/landmines.py" + + if [[ $BUILDBOT_CLOBBER || -f "$landmines_triggered_path" ]]; then + echo "@@@BUILD_STEP Clobber@@@" + + if [[ -z $BUILDBOT_CLOBBER ]]; then + echo "Clobbering due to triggered landmines: " + cat "$landmines_triggered_path" + else + # Also remove all the files under out/ on an explicit clobber + find "${SRC_ROOT}/out" -maxdepth 1 -type f -exec rm -f {} + + fi + + # Sdk key expires, delete android folder. + # crbug.com/145860 + rm -rf ~/.android + rm -rf "$build_path" + if [[ -e $build_path ]] ; then + echo "Clobber appeared to fail? $build_path still exists." + echo "@@@STEP_WARNINGS@@@" + fi + fi +} + +function bb_compile_setup { bb_setup_goma_internal # Should be called only after envsetup is done. gclient runhooks diff --git a/build/gyp_chromium b/build/gyp_chromium index d134034..499fa5c 100755 --- a/build/gyp_chromium +++ b/build/gyp_chromium @@ -8,6 +8,7 @@ # is invoked by Chromium beyond what can be done in the gclient hooks. import glob +import gyp_helper import os import shlex import subprocess @@ -44,36 +45,6 @@ if sys.platform == 'win32': else: psyco = None -def apply_gyp_environment(file_path=None): - """ - Reads in a *.gyp_env file and applies the valid keys to os.environ. - """ - if not file_path or not os.path.exists(file_path): - return - file_contents = open(file_path).read() - try: - file_data = eval(file_contents, {'__builtins__': None}, None) - except SyntaxError, e: - e.filename = os.path.abspath(file_path) - raise - supported_vars = ( 'CC', - 'CHROMIUM_GYP_FILE', - 'CHROMIUM_GYP_SYNTAX_CHECK', - 'CXX', - 'GYP_DEFINES', - 'GYP_GENERATOR_FLAGS', - 'GYP_GENERATOR_OUTPUT', - 'GYP_GENERATORS', ) - for var in supported_vars: - val = file_data.get(var) - if val: - if var in os.environ: - print 'INFO: Environment value for "%s" overrides value in %s.' % ( - var, os.path.abspath(file_path) - ) - else: - os.environ[var] = val - def additional_include_files(args=[]): """ Returns a list of additional (.gypi) files to include, without @@ -124,10 +95,7 @@ if __name__ == '__main__': p.communicate() sys.exit(p.returncode) - if 'SKIP_CHROMIUM_GYP_ENV' not in os.environ: - # Update the environment based on chromium.gyp_env - gyp_env_path = os.path.join(os.path.dirname(chrome_src), 'chromium.gyp_env') - apply_gyp_environment(gyp_env_path) + gyp_helper.apply_chromium_gyp_env() # This could give false positives since it doesn't actually do real option # parsing. Oh well. diff --git a/build/gyp_helper.py b/build/gyp_helper.py new file mode 100644 index 0000000..69a341d --- /dev/null +++ b/build/gyp_helper.py @@ -0,0 +1,50 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This file helps gyp_chromium and landmines correctly set up the gyp +# environment from chromium.gyp_env on disk + +import os + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +CHROME_SRC = os.path.dirname(SCRIPT_DIR) + + +def apply_gyp_environment_from_file(file_path): + """Reads in a *.gyp_env file and applies the valid keys to os.environ.""" + if not os.path.exists(file_path): + return + with open(file_path) as f: + file_contents = f.read() + try: + file_data = eval(file_contents, {'__builtins__': None}, None) + except SyntaxError, e: + e.filename = os.path.abspath(file_path) + raise + supported_vars = ( + 'CC', + 'CHROMIUM_GYP_FILE', + 'CHROMIUM_GYP_SYNTAX_CHECK', + 'CXX', + 'GYP_DEFINES', + 'GYP_GENERATOR_FLAGS', + 'GYP_GENERATOR_OUTPUT', + 'GYP_GENERATORS', + ) + for var in supported_vars: + file_val = file_data.get(var) + if file_val: + if var in os.environ: + print 'INFO: Environment value for "%s" overrides value in %s.' % ( + var, os.path.abspath(file_path) + ) + else: + os.environ[var] = file_val + + +def apply_chromium_gyp_env(): + if 'SKIP_CHROMIUM_GYP_ENV' not in os.environ: + # Update the environment based on chromium.gyp_env + path = os.path.join(os.path.dirname(CHROME_SRC), 'chromium.gyp_env') + apply_gyp_environment_from_file(path) diff --git a/build/landmines.py b/build/landmines.py new file mode 100755 index 0000000..36253e9 --- /dev/null +++ b/build/landmines.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +This file holds a list of reasons why a particular build needs to be clobbered +(or a list of 'landmines'). + +This script runs every build as a hook. If it detects that the build should +be clobbered, it will touch the file <build_dir>/.landmine_triggered. The +various build scripts will then check for the presence of this file and clobber +accordingly. The script will also emit the reasons for the clobber to stdout. + +A landmine is tripped when a builder checks out a different revision, and the +diff between the new landmines and the old ones is non-null. At this point, the +build is clobbered. +""" + +import difflib +import functools +import gyp_helper +import os +import shlex +import sys +import time + +SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +def memoize(default=None): + """This decorator caches the return value of a parameterless pure function""" + def memoizer(func): + val = [] + @functools.wraps(func) + def inner(): + if not val: + ret = func() + val.append(ret if ret is not None else default) + print '%s -> %r' % (func.__name__, val[0]) + return val[0] + return inner + return memoizer + + +@memoize() +def IsWindows(): + return sys.platform.startswith('win') or sys.platform == 'cygwin' + + +@memoize() +def IsLinux(): + return sys.platform.startswith('linux') + + +@memoize() +def IsMac(): + return sys.platform.startswith('darwin') + + +@memoize() +def gyp_defines(): + """Parses and returns GYP_DEFINES env var as a dictionary.""" + return dict(arg.split('=', 1) + for arg in shlex.split(os.environ.get('GYP_DEFINES', ''))) + + +@memoize() +def distributor(): + """ + Returns a string which is the distributed build engine in use (if any). + Possible values: 'goma', 'ib', '' + """ + if 'goma' in gyp_defines(): + return 'goma' + elif IsWindows(): + if 'CHROME_HEADLESS' in os.environ: + return 'ib' # use (win and !goma and headless) as approximation of ib + + +@memoize() +def platform(): + """ + Returns a string representing the platform this build is targetted for. + Possible values: 'win', 'mac', 'linux', 'ios', 'android' + """ + if 'OS' in gyp_defines(): + if 'android' in gyp_defines()['OS']: + return 'android' + else: + return gyp_defines()['OS'] + elif IsWindows(): + return 'win' + elif IsLinux(): + return 'linux' + else: + return 'mac' + + +@memoize() +def builder(): + """ + Returns a string representing the build engine (not compiler) to use. + Possible values: 'make', 'ninja', 'xcode', 'msvs', 'scons' + """ + if 'GYP_GENERATORS' in os.environ: + # for simplicity, only support the first explicit generator + generator = os.environ['GYP_GENERATORS'].split(',')[0] + if generator.endswith('-android'): + return generator.split('-')[0] + else: + return generator + else: + if platform() == 'android': + # Good enough for now? Do any android bots use make? + return 'ninja' + elif platform() == 'ios': + return 'xcode' + elif IsWindows(): + return 'msvs' + elif IsLinux(): + return 'make' + elif IsMac(): + return 'xcode' + else: + assert False, 'Don\'t know what builder we\'re using!' + + +def get_landmines(target): + """ + ALL LANDMINES ARE DEFINED HERE. + target is 'Release' or 'Debug' + """ + landmines = [] + add = lambda item: landmines.append(item + '\n') + + if (distributor() == 'goma' and platform() == 'win32' and + builder() == 'ninja'): + add('Need to clobber winja goma due to backend cwd cache fix.') + + return landmines + + +def get_target_build_dir(build_tool, target, is_iphone=False): + """ + Returns output directory absolute path dependent on build and targets. + Examples: + r'c:\b\build\slave\win\build\src\out\Release' + '/mnt/data/b/build/slave/linux/build/src/out/Debug' + '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos' + + Keep this function in sync with tools/build/scripts/slave/compile.py + """ + ret = None + if build_tool == 'xcode': + ret = os.path.join(SRC_DIR, 'xcodebuild', + target + ('-iphoneos' if is_iphone else '')) + elif build_tool == 'make': + ret = os.path.join(SRC_DIR, 'out', target) + elif build_tool == 'ninja': + ret = os.path.join(SRC_DIR, 'out', target) + elif build_tool == 'msvs': + ret = os.path.join(SRC_DIR, 'build', target) + elif build_tool == 'scons': + ret = os.path.join(SRC_DIR, 'sconsbuild', target) + else: + raise NotImplementedError() + return os.path.abspath(ret) + + +def main(argv): + if len(argv) > 1: + print('Unknown arguments %s' % argv[1:]) + return 1 + + gyp_helper.apply_chromium_gyp_env() + + for target in ('Debug', 'Release'): + out_dir = get_target_build_dir(builder(), target, + platform() == 'ios') + + landmines_path = os.path.join(out_dir, '.landmines') + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + new_landmines = get_landmines(target) + + if not os.path.exists(landmines_path): + with open(landmines_path, 'w') as f: + f.writelines(new_landmines) + else: + triggered = os.path.join(out_dir, '.landmines_triggered') + with open(landmines_path, 'r') as f: + old_landmines = f.readlines() + if old_landmines != new_landmines: + old_date = time.ctime(os.stat(landmines_path).st_ctime) + diff = difflib.unified_diff(old_landmines, new_landmines, + fromfile='old_landmines', tofile='new_landmines', + fromfiledate=old_date, tofiledate=time.ctime(), n=0) + + with open(triggered, 'w') as f: + f.writelines(diff) + elif os.path.exists(triggered): + # Remove false triggered landmines. + os.remove(triggered) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) |