Selective build clobbering feature (landmines.py and android build scripts).

Adds the ability for devs/troopers/etc. to set 'landmines' in the tree so that
the build will selectively clobber when a builder moves over a revision with such
a change.

This cl has an basis landmines.py, and hooks the clobber mechanism to the android
build scripts. 

The relevant cl which implements this for
compile.py is here: https://chromiumcodereview.appspot.com/11234013/

I'm planning to also implement an informational invocation for gclient to let devs know
about any potential landmines so they can decide if they need to clobber.

R=cmp,maruel@chromium.org
BUG=121897

Review URL: https://chromiumcodereview.appspot.com/11175016

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@166085 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 289 insertions, 50 deletions

diff --git a/build/android/buildbot/buildbot_functions.sh b/build/android/buildbot/buildbot_functions.sh
index 7835b46..117dbee 100755
--- a/build/android/buildbot/buildbot_functions.sh
+++ b/build/android/buildbot/buildbot_functions.sh
@@ -47,18 +47,6 @@ function bb_baseline_setup {
   shift
   cd $SRC_ROOT
 
-  if [[ $BUILDBOT_CLOBBER ]]; then
-    echo "@@@BUILD_STEP Clobber@@@"
-    # Sdk key expires, delete android folder.
-    # crbug.com/145860
-    rm -rf ~/.android
-    rm -rf "${SRC_ROOT}"/out
-    if [ -e "${SRC_ROOT}"/out ] ; then
-      echo "Clobber appeared to fail?  ${SRC_ROOT}/out still exists."
-      echo "@@@STEP_WARNINGS@@@"
-    fi
-  fi
-
   echo "@@@BUILD_STEP Environment setup@@@"
   bb_parse_args "$@"
 
@@ -68,17 +56,44 @@ function bb_baseline_setup {
   fi
   export GOMA_DIR=/b/build/goma
   . build/android/envsetup.sh
-  adb kill-server
-  adb start-server
-}
 
-function bb_compile_setup {
   local extra_gyp_defines="$(bb_get_json_prop "$FACTORY_PROPERTIES" \
      extra_gyp_defines)"
   export GYP_DEFINES+=" fastbuild=1 $extra_gyp_defines"
   if echo $extra_gyp_defines | grep -q clang; then
     unset CXX_target
   fi
+
+  adb kill-server
+  adb start-server
+
+  local build_path="${SRC_ROOT}/out/${BUILDTYPE}"
+  local landmines_triggered_path="$build_path/.landmines_triggered"
+  python "$SRC_ROOT/build/landmines.py"
+
+  if [[ $BUILDBOT_CLOBBER || -f "$landmines_triggered_path" ]]; then
+    echo "@@@BUILD_STEP Clobber@@@"
+
+    if [[ -z $BUILDBOT_CLOBBER ]]; then
+      echo "Clobbering due to triggered landmines: "
+      cat "$landmines_triggered_path"
+    else
+      # Also remove all the files under out/ on an explicit clobber
+      find "${SRC_ROOT}/out" -maxdepth 1 -type f -exec rm -f {} +
+    fi
+
+    # Sdk key expires, delete android folder.
+    # crbug.com/145860
+    rm -rf ~/.android
+    rm -rf "$build_path"
+    if [[ -e $build_path ]] ; then
+      echo "Clobber appeared to fail?  $build_path still exists."
+      echo "@@@STEP_WARNINGS@@@"
+    fi
+  fi
+}
+
+function bb_compile_setup {
   bb_setup_goma_internal
   # Should be called only after envsetup is done.
   gclient runhooks
diff --git a/build/gyp_chromium b/build/gyp_chromium
index d134034..499fa5c 100755
--- a/build/gyp_chromium
+++ b/build/gyp_chromium
@@ -8,6 +8,7 @@
 # is invoked by Chromium beyond what can be done in the gclient hooks.
 
 import glob
+import gyp_helper
 import os
 import shlex
 import subprocess
@@ -44,36 +45,6 @@ if sys.platform == 'win32':
 else:
   psyco = None
 
-def apply_gyp_environment(file_path=None):
-  """
-  Reads in a *.gyp_env file and applies the valid keys to os.environ.
-  """
-  if not file_path or not os.path.exists(file_path):
-    return
-  file_contents = open(file_path).read()
-  try:
-    file_data = eval(file_contents, {'__builtins__': None}, None)
-  except SyntaxError, e:
-    e.filename = os.path.abspath(file_path)
-    raise
-  supported_vars = ( 'CC',
-                     'CHROMIUM_GYP_FILE',
-                     'CHROMIUM_GYP_SYNTAX_CHECK',
-                     'CXX',
-                     'GYP_DEFINES',
-                     'GYP_GENERATOR_FLAGS',
-                     'GYP_GENERATOR_OUTPUT',
-                     'GYP_GENERATORS', )
-  for var in supported_vars:
-    val = file_data.get(var)
-    if val:
-      if var in os.environ:
-        print 'INFO: Environment value for "%s" overrides value in %s.' % (
-            var, os.path.abspath(file_path)
-        )
-      else:
-        os.environ[var] = val
-
 def additional_include_files(args=[]):
   """
   Returns a list of additional (.gypi) files to include, without
@@ -124,10 +95,7 @@ if __name__ == '__main__':
     p.communicate()
     sys.exit(p.returncode)
 
-  if 'SKIP_CHROMIUM_GYP_ENV' not in os.environ:
-    # Update the environment based on chromium.gyp_env
-    gyp_env_path = os.path.join(os.path.dirname(chrome_src), 'chromium.gyp_env')
-    apply_gyp_environment(gyp_env_path)
+  gyp_helper.apply_chromium_gyp_env()
 
   # This could give false positives since it doesn't actually do real option
   # parsing.  Oh well.
diff --git a/build/gyp_helper.py b/build/gyp_helper.py
new file mode 100644
index 0000000..b9b55b0
--- /dev/null
+++ b/build/gyp_helper.py
@@ -0,0 +1,48 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file helps gyp_chromium and landmines correctly set up the gyp
+# environment from chromium.gyp_env on disk
+
+import os
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+CHROME_SRC = os.path.dirname(SCRIPT_DIR)
+
+
+def apply_gyp_environment_from_file(file_path):
+  """Reads in a *.gyp_env file and applies the valid keys to os.environ."""
+  if not os.path.exists(file_path):
+    return
+  with open(file_path) as f:
+    file_contents = f.read()
+  try:
+    file_data = eval(file_contents, {'__builtins__': None}, None)
+  except SyntaxError, e:
+    e.filename = os.path.abspath(file_path)
+    raise
+  supported_vars = ( 'CC',
+                     'CHROMIUM_GYP_FILE',
+                     'CHROMIUM_GYP_SYNTAX_CHECK',
+                     'CXX',
+                     'GYP_DEFINES',
+                     'GYP_GENERATOR_FLAGS',
+                     'GYP_GENERATOR_OUTPUT',
+                     'GYP_GENERATORS', )
+  for var in supported_vars:
+    file_val = file_data.get(var)
+    if file_val:
+      if var in os.environ:
+        print 'INFO: Environment value for "%s" overrides value in %s.' % (
+            var, os.path.abspath(file_path)
+        )
+      else:
+        os.environ[var] = file_val
+
+
+def apply_chromium_gyp_env():
+  if 'SKIP_CHROMIUM_GYP_ENV' not in os.environ:
+    # Update the environment based on chromium.gyp_env
+    path = os.path.join(os.path.dirname(CHROME_SRC), 'chromium.gyp_env')
+    apply_gyp_environment_from_file(path)
diff --git a/build/landmines.py b/build/landmines.py
new file mode 100755
index 0000000..d825380
--- /dev/null
+++ b/build/landmines.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This file holds a list of reasons why a particular build needs to be clobbered
+(or a list of 'landmines').
+
+This script runs every build as a hook. If it detects that the build should
+be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
+various build scripts will then check for the presence of this file and clobber
+accordingly. The script will also emit the reasons for the clobber to stdout.
+
+A landmine is tripped when a builder checks out a different revision, and the
+diff between the new landmines and the old ones is non-null. At this point, the
+build is clobbered.
+"""
+
+import difflib
+import functools
+import gyp_helper
+import os
+import shlex
+import sys
+import time
+
+SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+def memoize(default=None):
+  """This decorator caches the return value of a parameterless pure function"""
+  def memoizer(func):
+    val = []
+    @functools.wraps(func)
+    def inner():
+      if not val:
+        ret = func()
+        val.append(ret if ret is not None else default)
+        print '%s -> %r' % (func.__name__, val[0])
+      return val[0]
+    return inner
+  return memoizer
+
+
+@memoize()
+def IsWindows():
+  return sys.platform.startswith('win') or sys.platform == 'cygwin'
+
+
+@memoize()
+def IsLinux():
+  return sys.platform.startswith('linux')
+
+
+@memoize()
+def IsMac():
+  return sys.platform.startswith('darwin')
+
+
+@memoize()
+def gyp_defines():
+  """Parses and returns GYP_DEFINES env var as a dictionary."""
+  return dict(arg.split('=', 1)
+      for arg in shlex.split(os.environ.get('GYP_DEFINES', '')))
+
+
+@memoize()
+def distributor():
+  """
+  Returns a string which is the distributed build engine in use (if any).
+  Possible values: 'goma', 'ib', ''
+  """
+  if 'goma' in gyp_defines():
+    return 'goma'
+  elif IsWindows():
+    if 'CHROME_HEADLESS' in os.environ:
+      return 'ib' # use (win and !goma and headless) as approximation of ib
+
+
+@memoize()
+def platform():
+  """
+  Returns a string representing the platform this build is targetted for.
+  Possible values: 'win', 'mac', 'linux', 'ios', 'android'
+  """
+  if 'OS' in gyp_defines():
+    if 'android' in gyp_defines()['OS']:
+      return 'android'
+    else:
+      return gyp_defines()['OS']
+  elif IsWindows():
+    return 'win'
+  elif IsLinux():
+    return 'linux'
+  else:
+    return 'mac'
+
+
+@memoize()
+def builder():
+  """
+  Returns a string representing the build engine (not compiler) to use.
+  Possible values: 'make', 'ninja', 'xcode', 'msvs', 'scons'
+  """
+  if 'GYP_GENERATORS' in os.environ:
+    # for simplicity, only support the first explicit generator
+    generator = os.environ['GYP_GENERATORS'].split(',')[0]
+    if generator.endswith('-android'):
+      return generator.split('-')[0]
+    else:
+      return generator
+  else:
+    if platform() == 'android':
+      # Good enough for now? Do any android bots use make?
+      return 'ninja'
+    elif platform() == 'ios':
+      return 'xcode'
+    elif IsWindows():
+      return 'msvs'
+    elif IsLinux():
+      return 'make'
+    elif IsMac():
+      return 'xcode'
+    else:
+      assert False, 'Don\'t know what builder we\'re using!'
+
+
+def get_landmines(target):
+  """
+  ALL LANDMINES ARE DEFINED HERE.
+  target is 'Release' or 'Debug'
+  """
+  landmines = []
+  add = lambda item: landmines.append(item + '\n')
+
+  if (distributor() == 'goma' and platform() == 'win32' and
+      builder() == 'ninja'):
+    add('Need to clobber winja goma due to backend cwd cache fix.')
+
+  return landmines
+
+
+def get_target_build_dir(build_tool, target, is_iphone=False):
+  """
+  Returns output directory absolute path dependent on build and targets.
+  Examples:
+    r'c:\b\build\slave\win\build\src\out\Release'
+    '/mnt/data/b/build/slave/linux/build/src/out/Debug'
+    '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
+  """
+  ret = None
+  if build_tool == 'xcode':
+    ret = os.path.join(SRC_DIR, 'xcodebuild',
+        target + ('-iphoneos' if is_iphone else ''))
+  elif build_tool == 'make':
+    ret = os.path.join(SRC_DIR, 'out', target)
+  elif build_tool == 'ninja':
+    ret = os.path.join(SRC_DIR, 'out', target)
+  elif build_tool == 'msvs':
+    ret = os.path.join(SRC_DIR, 'build', target)
+  elif build_tool == 'scons':
+    ret = os.path.join(SRC_DIR, 'sconsbuild', target)
+  else:
+    raise NotImplementedError()
+  return os.path.abspath(ret)
+
+
+def main(argv):
+  if len(argv) > 1:
+    print('Unknown arguments %s' % argv[1:])
+    return 1
+
+  gyp_helper.apply_chromium_gyp_env()
+
+  for target in ('Debug', 'Release'):
+    out_dir = get_target_build_dir(builder(), target,
+                                   platform() == 'ios')
+
+    landmines_path = os.path.join(out_dir, '.landmines')
+    if not os.path.exists(out_dir):
+      os.makedirs(out_dir)
+
+    new_landmines = get_landmines(target)
+
+    if not os.path.exists(landmines_path):
+      with open(landmines_path, 'w') as f:
+        f.writelines(new_landmines)
+    else:
+      triggered = os.path.join(out_dir, '.landmines_triggered')
+      with open(landmines_path, 'r') as f:
+        old_landmines = f.readlines()
+      if old_landmines != new_landmines:
+        old_date = time.ctime(os.stat(landmines_path).st_ctime)
+        diff = difflib.unified_diff(old_landmines, new_landmines,
+            fromfile='old_landmines', tofile='new_landmines',
+            fromfiledate=old_date, tofiledate=time.ctime(), n=0)
+
+        with open(triggered, 'w') as f:
+          f.writelines(diff)
+      elif os.path.exists(triggered):
+        # Remove false triggered landmines.
+        os.remove(triggered)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))