path: root/android_webview/buildbot/deps_whitelist.py

#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
"""Logic to generate lists of DEPS used by various parts of
the android_webview continuous integration (buildbot) infrastructure.

Note: The root Chromium project (which is not explicitly listed here)
has a couple of third_party libraries checked in directly into it. This means
that the list of third parties present in this file is not a comprehensive
list of third party android_webview dependencies.
"""

import argparse
import json
import logging
import os
import sys

# Add android_webview/tools to path to get at known_issues.
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'tools'))
import known_issues


class DepsWhitelist(object):
  def __init__(self):
    # If a new DEPS entry is needed for the AOSP bot to compile please add it
    # here first.
    # This is a staging area for deps that are accepted by the android_webview
    # team and are in the process of having the required branches being created
    # in the Android tree.
    self._compile_but_not_snapshot_dependencies = [
      'third_party/libaddressinput/src/cpp',
    ]

    # Dependencies that need to be merged into the Android tree.
    self._snapshot_into_android_dependencies = [
      'sdch/open-vcdiff',
      'testing/gtest',
      'third_party/WebKit',
      'third_party/angle',
      'third_party/brotli/src',
      ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/'
       'braille'),
      'third_party/freetype',
      'third_party/icu',
      'third_party/leveldatabase/src',
      'third_party/libjingle/source/talk',
      'third_party/libphonenumber/src/phonenumbers',
      'third_party/libphonenumber/src/resources',
      'third_party/libsrtp',
      'third_party/libvpx',
      'third_party/libyuv',
      'third_party/mesa/src',
      'third_party/openmax_dl',
      'third_party/openssl',
      'third_party/opus/src',
      'third_party/ots',
      'third_party/sfntly/cpp/src',
      'third_party/skia',
      'third_party/smhasher/src',
      'third_party/usrsctp/usrsctplib',
      'third_party/webrtc',
      'third_party/yasm/source/patched-yasm',
      'tools/grit',
      'tools/gyp',
      'v8',
    ]

    # We can save some time by not rsyncing code we don't use.
    self._prune_from_rsync_build = [
        'third_party/WebKit/LayoutTests',
    ]

    # Dependencies required to build android_webview.
    self._compile_dependencies = (self._snapshot_into_android_dependencies +
                                  self._compile_but_not_snapshot_dependencies)

    # Dependencies required to run android_webview tests but not required to
    # compile.
    self._test_data_dependencies = [
      'chrome/test/data/perf/third_party/octane',
    ]

  @staticmethod
  def _read_deps_file(deps_file_path):
    class FileImplStub(object):
      """Stub for the File syntax."""
      def __init__(self, file_location):
        pass

      @staticmethod
      def GetPath():
        return ''

      @staticmethod
      def GetFilename():
        return ''

      @staticmethod
      def GetRevision():
        return None

    def from_stub(__, _=None):
      """Stub for the From syntax."""
      return ''

    class VarImpl(object):
      def __init__(self, custom_vars, local_scope):
        self._custom_vars = custom_vars
        self._local_scope = local_scope

      def Lookup(self, var_name):
        """Implements the Var syntax."""
        if var_name in self._custom_vars:
          return self._custom_vars[var_name]
        elif var_name in self._local_scope.get("vars", {}):
          return self._local_scope["vars"][var_name]
        raise Exception("Var is not defined: %s" % var_name)

    local_scope = {}
    var = VarImpl({}, local_scope)
    global_scope = {
        'File': FileImplStub,
        'From': from_stub,
        'Var': var.Lookup,
        'deps_os': {},
    }
    execfile(deps_file_path, global_scope, local_scope)
    deps = local_scope.get('deps', {})
    deps_os = local_scope.get('deps_os', {})
    for os_specific_deps in deps_os.itervalues():
      deps.update(os_specific_deps)
    return deps.keys()

  def _get_known_issues(self):
    issues = []
    for root, paths in known_issues.KNOWN_INCOMPATIBLE.items():
      for path in paths:
        issues.append(os.path.normpath(os.path.join(root, path)))
    return issues

  def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
    """Calculates the list of deps that need to be excluded from the deps_file
    so that the only deps left are the one in the whitelist."""
    all_deps = self._read_deps_file(deps_file_path)
    # The list of deps read from the DEPS file are prefixed with the source
    # tree root, which is 'src' for Chromium.
    def prepend_root(path):
      return os.path.join('src', path)
    whitelisted_deps = map(prepend_root, whitelisted_deps)
    deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
    return dict(map(lambda(x): (x, None), deps_blacklist))

  def _make_blacklist(self, deps_file_path, whitelisted_deps):
    """Calculates the list of paths we should exclude """
    all_deps = self._read_deps_file(deps_file_path)
    def remove_src_prefix(path):
      return path.replace('src/', '', 1)
    all_deps = map(remove_src_prefix, all_deps)
    # Ignore all deps except those whitelisted.
    blacklist = set(all_deps).difference(whitelisted_deps)
    # Ignore the 'known issues'. Typically these are the licence incompatible
    # things checked directly into Chromium.
    blacklist = blacklist.union(self._get_known_issues())
    # Ignore any other non-deps, non-licence paths we don't like.
    blacklist = blacklist.union(self._prune_from_rsync_build)
    return list(blacklist)

  def get_deps_for_android_build(self, deps_file_path):
    """This is used to calculate the custom_deps list for the Android bot.
    """
    if not deps_file_path:
      raise Exception('You need to specify a DEPS file path.')
    return self._make_gclient_blacklist(deps_file_path,
                                        self._compile_dependencies)

  def get_deps_for_android_build_and_test(self, deps_file_path):
    """This is used to calculate the custom_deps list for the Android perf bot.
    """
    if not deps_file_path:
      raise Exception('You need to specify a DEPS file path.')
    return self._make_gclient_blacklist(deps_file_path,
                                        self._compile_dependencies +
                                        self._test_data_dependencies)

  def get_blacklist_for_android_rsync_build(self, deps_file_path):
    """Calculates the list of paths we should exclude when building Android
    either because of license compatibility or because they are large and
    uneeded.
    """
    if not deps_file_path:
      raise Exception('You need to specify a DEPS file path.')
    return self._make_blacklist(deps_file_path, self._compile_dependencies)

  def get_deps_for_android_merge(self, _):
    """Calculates the list of deps that need to be merged into the Android tree
    in order to build the C++ and Java android_webview code."""
    return self._snapshot_into_android_dependencies

  def get_deps_for_license_check(self, _):
    """Calculates the list of deps that need to be checked for Android license
    compatibility"""
    return self._compile_dependencies


  def execute_method(self, method_name, deps_file_path):
    methods = {
      'android_build': self.get_deps_for_android_build,
      'android_build_and_test':
        self.get_deps_for_android_build_and_test,
      'android_merge': self.get_deps_for_android_merge,
      'license_check': self.get_deps_for_license_check,
      'android_rsync_build': self.get_blacklist_for_android_rsync_build,
    }
    if not method_name in methods:
      raise Exception('Method name %s is not valid. Valid choices are %s' %
                      (method_name, methods.keys()))
    return methods[method_name](deps_file_path)

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('--method', help='Method to use to fetch from whitelist.',
                      required=True)
  parser.add_argument('--path-to-deps', help='Path to DEPS file.')
  parser.add_argument('--output-json', help='Name of file to write output to.')
  parser.add_argument('verbose', action='store_true', default=False)
  opts = parser.parse_args()

  logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN)

  deps_whitelist = DepsWhitelist()
  blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps)

  if (opts.output_json):
    output_dict = {
        'blacklist' : blacklist
    }
    with open(opts.output_json, 'w') as output_json_file:
      json.dump(output_dict, output_json_file)
  else:
    print blacklist

  return 0


if __name__ == '__main__':
  sys.exit(main())