[android_webview] Deps whitelist and scripts for AOSP builder.

This check in the whitelist that will be used to calculate
the set of DEPS used to checkout Chromium in the AOSP 
builder configuration.
This also adds the script to
generate the local manifest which excludes any previous
Chromium snapshots.
Finally this also adds a simple wrapper used to execute
commands within a lunch-ed environment.

BUG=175394
TEST=None
NOTRY=true

Review URL: https://chromiumcodereview.appspot.com/14669003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@202067 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 296 insertions, 0 deletions

diff --git a/android_webview/buildbot/OWNERS b/android_webview/buildbot/OWNERS
new file mode 100644
index 0000000..d95a0fb
--- /dev/null
+++ b/android_webview/buildbot/OWNERS
@@ -0,0 +1,4 @@
+benm@chromium.org
+joth@chromium.org
+mkosiba@chromium.org
+torne@chromium.org
diff --git a/android_webview/buildbot/deps_whitelist.py b/android_webview/buildbot/deps_whitelist.py
new file mode 100755
index 0000000..6439436
--- /dev/null
+++ b/android_webview/buildbot/deps_whitelist.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+"""Logic to generate lists of DEPS used by various parts of
+the android_webview continuous integration (buildbot) infrastructure.
+
+Note: The root Chromium project (which is not explicitly listed here)
+has a couple of third_party libraries checked in directly into it. This means
+that the list of third parties present in this file is not a comprehensive
+list of third party android_webview dependencies.
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+
+
+class DepsWhitelist(object):
+  def __init__(self):
+    # Dependencies required to build android_webview.
+    self._compile_dependencies = [
+      'googleurl',
+      'sdch/open-vcdiff',
+      'testing/gtest',
+      'third_party/WebKit',
+      'third_party/angle',
+      ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/'
+       'braille'),
+      'third_party/freetype',
+      'third_party/icu',
+      'third_party/leveldatabase/src',
+      'third_party/libjingle/source',
+      'third_party/libphonenumber/src/phonenumbers',
+      'third_party/libphonenumber/src/resources',
+      'third_party/openssl',
+      'third_party/opus/src',
+      'third_party/ots',
+      'third_party/skia/gyp',
+      'third_party/skia/include',
+      'third_party/skia/src',
+      'third_party/smhasher/src',
+      'third_party/v8-i18n',
+      'third_party/yasm/source/patched-yasm',
+      'tools/grit',
+      'tools/gyp',
+      'v8',
+    ]
+
+    # Dependencies that need to be merged into the Android tree.
+    self._snapshot_into_android_dependencies = self._compile_dependencies
+
+    # Dependencies required to run android_webview tests but not required to
+    # compile.
+    self._test_data_dependencies = [
+      'chrome/test/data/perf/third_party/octane',
+    ]
+
+  @staticmethod
+  def _read_deps_file(deps_file_path):
+    class FileImplStub(object):
+      """Stub for the File syntax."""
+      def __init__(self, file_location):
+        pass
+
+      @staticmethod
+      def GetPath():
+        return ''
+
+      @staticmethod
+      def GetFilename():
+        return ''
+
+      @staticmethod
+      def GetRevision():
+        return None
+
+    def from_stub(__, _=None):
+      """Stub for the From syntax."""
+      return ''
+
+    class VarImpl(object):
+      def __init__(self, custom_vars, local_scope):
+        self._custom_vars = custom_vars
+        self._local_scope = local_scope
+
+      def Lookup(self, var_name):
+        """Implements the Var syntax."""
+        if var_name in self._custom_vars:
+          return self._custom_vars[var_name]
+        elif var_name in self._local_scope.get("vars", {}):
+          return self._local_scope["vars"][var_name]
+        raise Exception("Var is not defined: %s" % var_name)
+
+    local_scope = {}
+    var = VarImpl({}, local_scope)
+    global_scope = {
+        'File': FileImplStub,
+        'From': from_stub,
+        'Var': var.Lookup,
+        'deps_os': {},
+    }
+    execfile(deps_file_path, global_scope, local_scope)
+    deps = local_scope.get('deps', {})
+    deps_os = local_scope.get('deps_os', {})
+    for os_specific_deps in deps_os.itervalues():
+      deps.update(os_specific_deps)
+    return deps.keys()
+
+  def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
+    """Calculates the list of deps that need to be excluded from the deps_file
+    so that the only deps left are the one in the whitelist."""
+    all_deps = self._read_deps_file(deps_file_path)
+    # The list of deps read from the DEPS file are prefixed with the source
+    # tree root, which is 'src' for Chromium.
+    def prepend_root(path):
+      return os.path.join('src', path)
+    whitelisted_deps = map(prepend_root, whitelisted_deps)
+    deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
+    return dict(map(lambda(x): (x, None), deps_blacklist))
+
+  def get_deps_for_android_build(self, deps_file_path):
+    """This is used to calculate the custom_deps list for the Android bot.
+    """
+    if not deps_file_path:
+      raise Exception('You need to specify a DEPS file path.')
+    return self._make_gclient_blacklist(deps_file_path,
+                                        self._compile_dependencies)
+
+  def get_deps_for_android_build_and_test(self, deps_file_path):
+    """This is used to calculate the custom_deps list for the Android perf bot.
+    """
+    if not deps_file_path:
+      raise Exception('You need to specify a DEPS file path.')
+    return self._make_gclient_blacklist(deps_file_path,
+                                        self._compile_dependencies +
+                                        self._test_data_dependencies)
+
+  def get_deps_for_android_merge(self, _):
+    """Calculates the list of deps that need to be merged into the Android tree
+    in order to build the C++ and Java android_webview code."""
+    return self._snapshot_into_android_dependencies
+
+  def get_deps_for_license_check(self, _):
+    """Calculates the list of deps that need to be checked for Android license
+    compatibility"""
+    return self._snapshot_into_android_dependencies
+
+  def execute_method(self, method_name, deps_file_path):
+    methods = {
+      'android_build': self.get_deps_for_android_build,
+      'android_build_and_test':
+        self.get_deps_for_android_build_and_test,
+      'android_merge': self.get_deps_for_android_merge,
+      'license_check': self.get_deps_for_license_check
+    }
+    if not method_name in methods:
+      raise Exception('Method name %s is not valid. Valid choices are %s' %
+                      (method_name, methods.keys()))
+    return methods[method_name](deps_file_path)
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--method', help='Method to use to fetch from whitelist.',
+                      required=True)
+  parser.add_argument('--path-to-deps', help='Path to DEPS file.')
+  parser.add_argument('--output-json', help='Name of file to write output to.')
+  parser.add_argument('verbose', action='store_true', default=False)
+  opts = parser.parse_args()
+
+  logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN)
+
+  deps_whitelist = DepsWhitelist()
+  blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps)
+
+  if (opts.output_json):
+    output_dict = {
+        'blacklist' : blacklist
+    }
+    with open(opts.output_json, 'w') as output_json_file:
+      json.dump(output_dict, output_json_file)
+  else:
+    print blacklist
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/android_webview/buildbot/generate_local_manifest.py b/android_webview/buildbot/generate_local_manifest.py
new file mode 100755
index 0000000..59cb2de
--- /dev/null
+++ b/android_webview/buildbot/generate_local_manifest.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Generate local manifest in an Android repository.
+
+This is used to generate a local manifest in an Android repository. The purpose
+of the generated manifest is to remove the set of projects that exist under a
+certain path.
+"""
+
+from optparse import OptionParser
+import os
+import xml.etree.ElementTree as ET
+
+def createLocalManifest(manifest_path, local_manifest_path, path_to_exclude,
+                        pinned_projects=None):
+  manifest_tree = ET.parse(manifest_path)
+  local_manifest_root = ET.Element('manifest')
+
+  def remove_project(project):
+    remove_project = ET.SubElement(local_manifest_root, 'remove-project')
+    remove_project.set('name', project.get('name'))
+
+  def pin_project(project, revision):
+    pin_project = ET.SubElement(local_manifest_root, 'project')
+    pin_project.set('name', project.get('name'))
+    if project.get('path') != None:
+      pin_project.set('path', project.get('path'))
+    pin_project.set('revision', revision)
+
+  for project in manifest_tree.getroot().findall('project'):
+    project_path = project.get('path')
+    project_name = project.get('name')
+    exclude_project = ((project_path != None and
+                        project_path.startswith(path_to_exclude)) or
+                       (project_path == None and
+                        project_name.startswith(path_to_exclude)))
+    if exclude_project:
+      print 'Excluding project name="%s" path="%s"' % (project_name,
+                                                       project_path)
+      remove_project(project)
+      continue
+
+    pinned_projects = pinned_projects or []
+    for pinned in pinned_projects:
+      if pinned['path'] == project_path and pinned['name'] == project_name:
+        remove_project(project)
+        pin_project(project, pinned['revision'])
+        break
+
+  local_manifest_tree = ET.ElementTree(local_manifest_root)
+  local_manifest_dir = os.path.dirname(local_manifest_path)
+  if not os.path.exists(local_manifest_dir):
+    os.makedirs(local_manifest_dir)
+  local_manifest_tree.write(local_manifest_path,
+                            xml_declaration=True,
+                            encoding='UTF-8',
+                            method='xml')
+
+def main():
+  usage = 'usage: %prog [options] <android_build_top> <path_to_exclude>'
+  parser = OptionParser(usage=usage)
+  parser.add_option('--ndk-revision', dest='ndk_revision',
+                    help='pin the ndk project at a particular REVISION',
+                    metavar='REVISION', default=None)
+  parser.add_option('--manifest_filename', dest='manifest_filename',
+                    help='name of the manifest file', default='default.xml')
+  (options, args) = parser.parse_args()
+
+  if len(args) != 2:
+    parser.error('Wrong number of arguments.')
+
+  android_build_top = args[0]
+  path_to_exclude = args[1]
+
+  manifest_filename = options.manifest_filename
+
+  manifest_path = os.path.join(android_build_top, '.repo/manifests',
+                               manifest_filename)
+  local_manifest_path = os.path.join(android_build_top,
+                                     '.repo/local_manifest.xml')
+
+  pinned_projects = []
+  if options.ndk_revision:
+    pinned_projects = [{
+        'path': 'ndk',
+        'name': 'platform/ndk',
+        'revision' : options.ndk_revision,
+    },]
+
+  print 'Path to exclude: %s' % path_to_exclude
+  print 'Path to manifest file: %s' % manifest_path
+  createLocalManifest(manifest_path, local_manifest_path, path_to_exclude,
+                      pinned_projects)
+  print 'Local manifest created in: %s' % local_manifest_path
+
+if __name__ == '__main__':
+  main()