Add known_incompatible.py items to the blacklist in deps_whitelist.py

This allows the android_aosp bot to better model the behavior of
the automerger.

We add a 'android_rsync_build' method to 
android_webview/buildbot/deps_whitelist.py
which returns all black listed paths (not just blacklisted deps)
and has a list output format (as opposed to dictionary format of
the other methods).

Then we add the 'all_incompatible_directories' option to
android_webview/tools/webview_licenses.py to get all the
incompatible directories rather that just the unknown
ones. This exposes the same code the automeger uses directly as
a command line argument.

BUG=311868

Review URL: https://codereview.chromium.org/324813006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@276639 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 59 insertions, 11 deletions

diff --git a/android_webview/buildbot/deps_whitelist.py b/android_webview/buildbot/deps_whitelist.py
index 1cccd636..69bb2d7 100755
--- a/android_webview/buildbot/deps_whitelist.py
+++ b/android_webview/buildbot/deps_whitelist.py
@@ -18,6 +18,10 @@ import logging
 import os
 import sys
 
+# Add android_webview/tools to path to get at known_issues.
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'tools'))
+import known_issues
+
 
 class DepsWhitelist(object):
   def __init__(self):
@@ -64,6 +68,11 @@ class DepsWhitelist(object):
       'v8',
     ]
 
+    # We can save some time by not rsyncing code we don't use.
+    self._prune_from_rsync_build = [
+        'third_party/WebKit/LayoutTests',
+    ]
+
     # Dependencies required to build android_webview.
     self._compile_dependencies = (self._snapshot_into_android_dependencies +
                                   self._compile_but_not_snapshot_dependencies)
@@ -125,6 +134,13 @@ class DepsWhitelist(object):
       deps.update(os_specific_deps)
     return deps.keys()
 
+  def _get_known_issues(self):
+    issues = []
+    for root, paths in known_issues.KNOWN_INCOMPATIBLE.items():
+      for path in paths:
+        issues.append(os.path.normpath(os.path.join(root, path)))
+    return issues
+
   def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
     """Calculates the list of deps that need to be excluded from the deps_file
     so that the only deps left are the one in the whitelist."""
@@ -137,6 +153,21 @@ class DepsWhitelist(object):
     deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
     return dict(map(lambda(x): (x, None), deps_blacklist))
 
+  def _make_blacklist(self, deps_file_path, whitelisted_deps):
+    """Calculates the list of paths we should exclude """
+    all_deps = self._read_deps_file(deps_file_path)
+    def remove_src_prefix(path):
+      return path.replace('src/', '', 1)
+    all_deps = map(remove_src_prefix, all_deps)
+    # Ignore all deps except those whitelisted.
+    blacklist = set(all_deps).difference(whitelisted_deps)
+    # Ignore the 'known issues'. Typically these are the licence incompatible
+    # things checked directly into Chromium.
+    blacklist = blacklist.union(self._get_known_issues())
+    # Ignore any other non-deps, non-licence paths we don't like.
+    blacklist = blacklist.union(self._prune_from_rsync_build)
+    return list(blacklist)
+
   def get_deps_for_android_build(self, deps_file_path):
     """This is used to calculate the custom_deps list for the Android bot.
     """
@@ -154,6 +185,15 @@ class DepsWhitelist(object):
                                         self._compile_dependencies +
                                         self._test_data_dependencies)
 
+  def get_blacklist_for_android_rsync_build(self, deps_file_path):
+    """Calculates the list of paths we should exclude when building Android
+    either because of license compatibility or because they are large and
+    uneeded.
+    """
+    if not deps_file_path:
+      raise Exception('You need to specify a DEPS file path.')
+    return self._make_blacklist(deps_file_path, self._compile_dependencies)
+
   def get_deps_for_android_merge(self, _):
     """Calculates the list of deps that need to be merged into the Android tree
     in order to build the C++ and Java android_webview code."""
@@ -164,13 +204,15 @@ class DepsWhitelist(object):
     compatibility"""
     return self._compile_dependencies
 
+
   def execute_method(self, method_name, deps_file_path):
     methods = {
       'android_build': self.get_deps_for_android_build,
       'android_build_and_test':
         self.get_deps_for_android_build_and_test,
       'android_merge': self.get_deps_for_android_merge,
-      'license_check': self.get_deps_for_license_check
+      'license_check': self.get_deps_for_license_check,
+      'android_rsync_build': self.get_blacklist_for_android_rsync_build,
     }
     if not method_name in methods:
       raise Exception('Method name %s is not valid. Valid choices are %s' %
diff --git a/android_webview/tools/webview_licenses.py b/android_webview/tools/webview_licenses.py
index d2775e7..b0c4b9c 100755
--- a/android_webview/tools/webview_licenses.py
+++ b/android_webview/tools/webview_licenses.py
@@ -297,6 +297,13 @@ def GenerateNoticeFile():
   return '\n'.join(content)
 
 
+def _ProcessIncompatibleResult(incompatible_directories):
+  if incompatible_directories:
+    print ("Incompatibly licensed directories found:\n" +
+           "\n".join(sorted(incompatible_directories)))
+    return ScanResult.Errors
+  return ScanResult.Ok
+
 def main():
   class FormatterWithNewLines(optparse.IndentedHelpFormatter):
     def format_description(self, description):
@@ -308,10 +315,13 @@ def main():
                                  usage='%prog [options]')
   parser.description = (__doc__ +
                        '\nCommands:\n' \
-                       '  scan  Check licenses.\n' \
-                       '  notice Generate Android NOTICE file on stdout. \n' \
+                       '  scan Check licenses.\n' \
+                       '  notice Generate Android NOTICE file on stdout.\n' \
                        '  incompatible_directories Scan for incompatibly'
-                       ' licensed directories.')
+                       ' licensed directories.\n'
+                       '  all_incompatible_directories Scan for incompatibly'
+                       ' licensed directories (even those in'
+                       ' known_issues.py).\n')
   (_, args) = parser.parse_args()
   if len(args) != 1:
     parser.print_help()
@@ -326,13 +336,9 @@ def main():
     print GenerateNoticeFile()
     return ScanResult.Ok
   elif args[0] == 'incompatible_directories':
-    incompatible_directories = GetUnknownIncompatibleDirectories()
-    if incompatible_directories:
-      print ("Incompatibly licensed directories found:\n" +
-             "\n".join(sorted(incompatible_directories)))
-      return ScanResult.Errors
-    return ScanResult.Ok
-
+    return _ProcessIncompatibleResult(GetUnknownIncompatibleDirectories())
+  elif args[0] == 'all_incompatible_directories':
+    return _ProcessIncompatibleResult(GetIncompatibleDirectories())
   parser.print_help()
   return ScanResult.Errors