Selective build clobbering feature (landmines.py and android build scripts).

Adds the ability for devs/troopers/etc. to set 'landmines' in the tree so that
the build will selectively clobber when a builder moves over a revision with such
a change.

This cl has an basis landmines.py, and hooks the clobber mechanism to the android
build scripts.

The relevant cl which implements this for
compile.py is here: https://chromiumcodereview.appspot.com/11234013/

I'm planning to also implement an informational invocation for gclient to let devs know
about any potential landmines so they can decide if they need to clobber.

This previously attempted to land as: https://chromiumcodereview.appspot.com/11175016

R=ilevy@chromium.org,maruel@chromium.org
BUG=121897


Review URL: https://chromiumcodereview.appspot.com/11377141

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@167595 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 298 insertions, 50 deletions

diff --git a/DEPS b/DEPS
index 34c4262..5dace0e 100644
--- a/DEPS
+++ b/DEPS
@@ -631,4 +631,9 @@ hooks = [
     "pattern": ".",
     "action": ["python", "src/build/gyp_chromium"],
   },
+  {
+    # Check for landmines (reasons to clobber the build).
+    "pattern": ".",
+    "action": ["python", "src/build/landmines.py"],
+  },
 ]
diff --git a/build/android/buildbot/buildbot_functions.sh b/build/android/buildbot/buildbot_functions.sh
index 6030e67..9f56a18 100755
--- a/build/android/buildbot/buildbot_functions.sh
+++ b/build/android/buildbot/buildbot_functions.sh
@@ -47,18 +47,6 @@ function bb_baseline_setup {
   shift
   cd $SRC_ROOT
 
-  if [[ $BUILDBOT_CLOBBER ]]; then
-    echo "@@@BUILD_STEP Clobber@@@"
-    # Sdk key expires, delete android folder.
-    # crbug.com/145860
-    rm -rf ~/.android
-    rm -rf "${SRC_ROOT}"/out
-    if [ -e "${SRC_ROOT}"/out ] ; then
-      echo "Clobber appeared to fail?  ${SRC_ROOT}/out still exists."
-      echo "@@@STEP_WARNINGS@@@"
-    fi
-  fi
-
   echo "@@@BUILD_STEP Environment setup@@@"
   bb_parse_args "$@"
 
@@ -68,17 +56,44 @@ function bb_baseline_setup {
   fi
   export GOMA_DIR=/b/build/goma
   . build/android/envsetup.sh
-  adb kill-server
-  adb start-server
-}
 
-function bb_compile_setup {
   local extra_gyp_defines="$(bb_get_json_prop "$FACTORY_PROPERTIES" \
      extra_gyp_defines)"
   export GYP_DEFINES+=" fastbuild=1 $extra_gyp_defines"
   if echo $extra_gyp_defines | grep -q clang; then
     unset CXX_target
   fi
+
+  adb kill-server
+  adb start-server
+
+  local build_path="${SRC_ROOT}/out/${BUILDTYPE}"
+  local landmines_triggered_path="$build_path/.landmines_triggered"
+  python "$SRC_ROOT/build/landmines.py"
+
+  if [[ $BUILDBOT_CLOBBER || -f "$landmines_triggered_path" ]]; then
+    echo "@@@BUILD_STEP Clobber@@@"
+
+    if [[ -z $BUILDBOT_CLOBBER ]]; then
+      echo "Clobbering due to triggered landmines: "
+      cat "$landmines_triggered_path"
+    else
+      # Also remove all the files under out/ on an explicit clobber
+      find "${SRC_ROOT}/out" -maxdepth 1 -type f -exec rm -f {} +
+    fi
+
+    # Sdk key expires, delete android folder.
+    # crbug.com/145860
+    rm -rf ~/.android
+    rm -rf "$build_path"
+    if [[ -e $build_path ]] ; then
+      echo "Clobber appeared to fail?  $build_path still exists."
+      echo "@@@STEP_WARNINGS@@@"
+    fi
+  fi
+}
+
+function bb_compile_setup {
   bb_setup_goma_internal
   # Should be called only after envsetup is done.
   gclient runhooks
diff --git a/build/gyp_chromium b/build/gyp_chromium
index d134034..499fa5c 100755
--- a/build/gyp_chromium
+++ b/build/gyp_chromium
@@ -8,6 +8,7 @@
 # is invoked by Chromium beyond what can be done in the gclient hooks.
 
 import glob
+import gyp_helper
 import os
 import shlex
 import subprocess
@@ -44,36 +45,6 @@ if sys.platform == 'win32':
 else:
   psyco = None
 
-def apply_gyp_environment(file_path=None):
-  """
-  Reads in a *.gyp_env file and applies the valid keys to os.environ.
-  """
-  if not file_path or not os.path.exists(file_path):
-    return
-  file_contents = open(file_path).read()
-  try:
-    file_data = eval(file_contents, {'__builtins__': None}, None)
-  except SyntaxError, e:
-    e.filename = os.path.abspath(file_path)
-    raise
-  supported_vars = ( 'CC',
-                     'CHROMIUM_GYP_FILE',
-                     'CHROMIUM_GYP_SYNTAX_CHECK',
-                     'CXX',
-                     'GYP_DEFINES',
-                     'GYP_GENERATOR_FLAGS',
-                     'GYP_GENERATOR_OUTPUT',
-                     'GYP_GENERATORS', )
-  for var in supported_vars:
-    val = file_data.get(var)
-    if val:
-      if var in os.environ:
-        print 'INFO: Environment value for "%s" overrides value in %s.' % (
-            var, os.path.abspath(file_path)
-        )
-      else:
-        os.environ[var] = val
-
 def additional_include_files(args=[]):
   """
   Returns a list of additional (.gypi) files to include, without
@@ -124,10 +95,7 @@ if __name__ == '__main__':
     p.communicate()
     sys.exit(p.returncode)
 
-  if 'SKIP_CHROMIUM_GYP_ENV' not in os.environ:
-    # Update the environment based on chromium.gyp_env
-    gyp_env_path = os.path.join(os.path.dirname(chrome_src), 'chromium.gyp_env')
-    apply_gyp_environment(gyp_env_path)
+  gyp_helper.apply_chromium_gyp_env()
 
   # This could give false positives since it doesn't actually do real option
   # parsing.  Oh well.
diff --git a/build/gyp_helper.py b/build/gyp_helper.py
new file mode 100644
index 0000000..69a341d
--- /dev/null
+++ b/build/gyp_helper.py
@@ -0,0 +1,50 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file helps gyp_chromium and landmines correctly set up the gyp
+# environment from chromium.gyp_env on disk
+
+import os
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+CHROME_SRC = os.path.dirname(SCRIPT_DIR)
+
+
+def apply_gyp_environment_from_file(file_path):
+  """Reads in a *.gyp_env file and applies the valid keys to os.environ."""
+  if not os.path.exists(file_path):
+    return
+  with open(file_path) as f:
+    file_contents = f.read()
+  try:
+    file_data = eval(file_contents, {'__builtins__': None}, None)
+  except SyntaxError, e:
+    e.filename = os.path.abspath(file_path)
+    raise
+  supported_vars = (
+      'CC',
+      'CHROMIUM_GYP_FILE',
+      'CHROMIUM_GYP_SYNTAX_CHECK',
+      'CXX',
+      'GYP_DEFINES',
+      'GYP_GENERATOR_FLAGS',
+      'GYP_GENERATOR_OUTPUT',
+      'GYP_GENERATORS',
+  )
+  for var in supported_vars:
+    file_val = file_data.get(var)
+    if file_val:
+      if var in os.environ:
+        print 'INFO: Environment value for "%s" overrides value in %s.' % (
+            var, os.path.abspath(file_path)
+        )
+      else:
+        os.environ[var] = file_val
+
+
+def apply_chromium_gyp_env():
+  if 'SKIP_CHROMIUM_GYP_ENV' not in os.environ:
+    # Update the environment based on chromium.gyp_env
+    path = os.path.join(os.path.dirname(CHROME_SRC), 'chromium.gyp_env')
+    apply_gyp_environment_from_file(path)
diff --git a/build/landmines.py b/build/landmines.py
new file mode 100755
index 0000000..36253e9
--- /dev/null
+++ b/build/landmines.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This file holds a list of reasons why a particular build needs to be clobbered
+(or a list of 'landmines').
+
+This script runs every build as a hook. If it detects that the build should
+be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
+various build scripts will then check for the presence of this file and clobber
+accordingly. The script will also emit the reasons for the clobber to stdout.
+
+A landmine is tripped when a builder checks out a different revision, and the
+diff between the new landmines and the old ones is non-null. At this point, the
+build is clobbered.
+"""
+
+import difflib
+import functools
+import gyp_helper
+import os
+import shlex
+import sys
+import time
+
+SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+def memoize(default=None):
+  """This decorator caches the return value of a parameterless pure function"""
+  def memoizer(func):
+    val = []
+    @functools.wraps(func)
+    def inner():
+      if not val:
+        ret = func()
+        val.append(ret if ret is not None else default)
+        print '%s -> %r' % (func.__name__, val[0])
+      return val[0]
+    return inner
+  return memoizer
+
+
+@memoize()
+def IsWindows():
+  return sys.platform.startswith('win') or sys.platform == 'cygwin'
+
+
+@memoize()
+def IsLinux():
+  return sys.platform.startswith('linux')
+
+
+@memoize()
+def IsMac():
+  return sys.platform.startswith('darwin')
+
+
+@memoize()
+def gyp_defines():
+  """Parses and returns GYP_DEFINES env var as a dictionary."""
+  return dict(arg.split('=', 1)
+      for arg in shlex.split(os.environ.get('GYP_DEFINES', '')))
+
+
+@memoize()
+def distributor():
+  """
+  Returns a string which is the distributed build engine in use (if any).
+  Possible values: 'goma', 'ib', ''
+  """
+  if 'goma' in gyp_defines():
+    return 'goma'
+  elif IsWindows():
+    if 'CHROME_HEADLESS' in os.environ:
+      return 'ib' # use (win and !goma and headless) as approximation of ib
+
+
+@memoize()
+def platform():
+  """
+  Returns a string representing the platform this build is targetted for.
+  Possible values: 'win', 'mac', 'linux', 'ios', 'android'
+  """
+  if 'OS' in gyp_defines():
+    if 'android' in gyp_defines()['OS']:
+      return 'android'
+    else:
+      return gyp_defines()['OS']
+  elif IsWindows():
+    return 'win'
+  elif IsLinux():
+    return 'linux'
+  else:
+    return 'mac'
+
+
+@memoize()
+def builder():
+  """
+  Returns a string representing the build engine (not compiler) to use.
+  Possible values: 'make', 'ninja', 'xcode', 'msvs', 'scons'
+  """
+  if 'GYP_GENERATORS' in os.environ:
+    # for simplicity, only support the first explicit generator
+    generator = os.environ['GYP_GENERATORS'].split(',')[0]
+    if generator.endswith('-android'):
+      return generator.split('-')[0]
+    else:
+      return generator
+  else:
+    if platform() == 'android':
+      # Good enough for now? Do any android bots use make?
+      return 'ninja'
+    elif platform() == 'ios':
+      return 'xcode'
+    elif IsWindows():
+      return 'msvs'
+    elif IsLinux():
+      return 'make'
+    elif IsMac():
+      return 'xcode'
+    else:
+      assert False, 'Don\'t know what builder we\'re using!'
+
+
+def get_landmines(target):
+  """
+  ALL LANDMINES ARE DEFINED HERE.
+  target is 'Release' or 'Debug'
+  """
+  landmines = []
+  add = lambda item: landmines.append(item + '\n')
+
+  if (distributor() == 'goma' and platform() == 'win32' and
+      builder() == 'ninja'):
+    add('Need to clobber winja goma due to backend cwd cache fix.')
+
+  return landmines
+
+
+def get_target_build_dir(build_tool, target, is_iphone=False):
+  """
+  Returns output directory absolute path dependent on build and targets.
+  Examples:
+    r'c:\b\build\slave\win\build\src\out\Release'
+    '/mnt/data/b/build/slave/linux/build/src/out/Debug'
+    '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
+
+  Keep this function in sync with tools/build/scripts/slave/compile.py
+  """
+  ret = None
+  if build_tool == 'xcode':
+    ret = os.path.join(SRC_DIR, 'xcodebuild',
+        target + ('-iphoneos' if is_iphone else ''))
+  elif build_tool == 'make':
+    ret = os.path.join(SRC_DIR, 'out', target)
+  elif build_tool == 'ninja':
+    ret = os.path.join(SRC_DIR, 'out', target)
+  elif build_tool == 'msvs':
+    ret = os.path.join(SRC_DIR, 'build', target)
+  elif build_tool == 'scons':
+    ret = os.path.join(SRC_DIR, 'sconsbuild', target)
+  else:
+    raise NotImplementedError()
+  return os.path.abspath(ret)
+
+
+def main(argv):
+  if len(argv) > 1:
+    print('Unknown arguments %s' % argv[1:])
+    return 1
+
+  gyp_helper.apply_chromium_gyp_env()
+
+  for target in ('Debug', 'Release'):
+    out_dir = get_target_build_dir(builder(), target,
+                                   platform() == 'ios')
+
+    landmines_path = os.path.join(out_dir, '.landmines')
+    if not os.path.exists(out_dir):
+      os.makedirs(out_dir)
+
+    new_landmines = get_landmines(target)
+
+    if not os.path.exists(landmines_path):
+      with open(landmines_path, 'w') as f:
+        f.writelines(new_landmines)
+    else:
+      triggered = os.path.join(out_dir, '.landmines_triggered')
+      with open(landmines_path, 'r') as f:
+        old_landmines = f.readlines()
+      if old_landmines != new_landmines:
+        old_date = time.ctime(os.stat(landmines_path).st_ctime)
+        diff = difflib.unified_diff(old_landmines, new_landmines,
+            fromfile='old_landmines', tofile='new_landmines',
+            fromfiledate=old_date, tofiledate=time.ctime(), n=0)
+
+        with open(triggered, 'w') as f:
+          f.writelines(diff)
+      elif os.path.exists(triggered):
+        # Remove false triggered landmines.
+        os.remove(triggered)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))