From c3aae52ad4abdfbaea23891b104d5f2b4f26ab7e Mon Sep 17 00:00:00 2001 From: "cjhopman@chromium.org" Date: Thu, 4 Apr 2013 09:35:50 +0000 Subject: Add input content checking to some build scripts Some build steps, particularly javac, have really loose input rules. I.e. javac steps are re-built when any input jar changes. Often, this leads to unnecessary rebuilds of all the following steps. Other build tools (ninja, goma), will check the contents of the inputs to a step, and if those inputs haven't changed that tool doesn't actually re-run the command for creating the output. This change brings that same benefit to some of the Android python build scripts. Particularly those that will save a significant amount of time by adding input content checks. The checking checks both the input files and the command that will be run. It compares this against a stored md5 digest. If it has not changed, then the output does not need to be recreated (though it is still touched to trigger following steps). BUG=158821 Review URL: https://chromiumcodereview.appspot.com/13432002 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@192265 0039d316-1c4b-4281-b951-d872f2087c98 --- build/android/dex.py | 40 ----------------- build/android/gyp/dex.py | 53 ++++++++++++++++++++++ build/android/gyp/jar.py | 67 +++++++++++++++++++++++++++ build/android/gyp/javac.py | 90 +++++++++++++++++++++++++++++++++++++ build/android/gyp/push_libraries.py | 14 +++++- build/android/gyp/util/__init__.py | 4 ++ build/android/gyp/util/md5_check.py | 52 +++++++++++++++++++++ build/android/jar.py | 53 ---------------------- build/android/javac.py | 87 ----------------------------------- build/java.gypi | 14 +++--- build/java_apk.gypi | 15 ++++--- build/java_prebuilt.gypi | 4 +- 12 files changed, 297 insertions(+), 196 deletions(-) delete mode 100755 build/android/dex.py create mode 100755 build/android/gyp/dex.py create mode 100755 build/android/gyp/jar.py create mode 100755 build/android/gyp/javac.py create mode 100644 build/android/gyp/util/__init__.py create mode 100644 build/android/gyp/util/md5_check.py delete mode 100755 build/android/jar.py delete mode 100755 build/android/javac.py (limited to 'build') diff --git a/build/android/dex.py b/build/android/dex.py deleted file mode 100755 index 00c2c52..0000000 --- a/build/android/dex.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import fnmatch -import optparse -import os -import sys - -from pylib import build_utils - - -def DoDex(options, paths): - dx_binary = os.path.join(options.android_sdk_root, 'platform-tools', 'dx') - dex_cmd = [dx_binary, '--dex', '--output', options.dex_path] + paths - build_utils.CheckCallDie(dex_cmd) - - -def main(argv): - parser = optparse.OptionParser() - parser.add_option('--android-sdk-root', help='Android sdk root directory.') - parser.add_option('--dex-path', help='Dex output path.') - parser.add_option('--stamp', help='Path to touch on success.') - - # TODO(newt): remove this once http://crbug.com/177552 is fixed in ninja. - parser.add_option('--ignore', help='Ignored.') - - options, paths = parser.parse_args() - - DoDex(options, paths) - - if options.stamp: - build_utils.Touch(options.stamp) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) - diff --git a/build/android/gyp/dex.py b/build/android/gyp/dex.py new file mode 100755 index 0000000..69d3d96 --- /dev/null +++ b/build/android/gyp/dex.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import fnmatch +import optparse +import os +import sys + +from util import md5_check + +BUILD_ANDROID_DIR = os.path.join(os.path.dirname(__file__), '..') +sys.path.append(BUILD_ANDROID_DIR) + +from pylib import build_utils + + +def DoDex(options, paths): + dx_binary = os.path.join(options.android_sdk_root, 'platform-tools', 'dx') + dex_cmd = [dx_binary, '--dex', '--output', options.dex_path] + paths + + md5_stamp = '%s.md5' % options.dex_path + md5_checker = md5_check.Md5Checker( + stamp=md5_stamp, inputs=paths, command=dex_cmd) + if md5_checker.IsStale(): + build_utils.CheckCallDie(dex_cmd) + else: + build_utils.Touch(options.dex_path) + md5_checker.Write() + + +def main(argv): + parser = optparse.OptionParser() + parser.add_option('--android-sdk-root', help='Android sdk root directory.') + parser.add_option('--dex-path', help='Dex output path.') + parser.add_option('--stamp', help='Path to touch on success.') + + # TODO(newt): remove this once http://crbug.com/177552 is fixed in ninja. + parser.add_option('--ignore', help='Ignored.') + + options, paths = parser.parse_args() + + DoDex(options, paths) + + if options.stamp: + build_utils.Touch(options.stamp) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + diff --git a/build/android/gyp/jar.py b/build/android/gyp/jar.py new file mode 100755 index 0000000..c8931c9 --- /dev/null +++ b/build/android/gyp/jar.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import fnmatch +import optparse +import os +import sys + +from util import md5_check + +BUILD_ANDROID_DIR = os.path.join(os.path.dirname(__file__), '..') +sys.path.append(BUILD_ANDROID_DIR) + +from pylib import build_utils + + +def DoJar(options): + class_files = build_utils.FindInDirectory(options.classes_dir, '*.class') + for exclude in build_utils.ParseGypList(options.excluded_classes): + class_files = filter( + lambda f: not fnmatch.fnmatch(f, exclude), class_files) + + jar_path = os.path.abspath(options.jar_path) + + # The paths of the files in the jar will be the same as they are passed in to + # the command. Because of this, the command should be run in + # options.classes_dir so the .class file paths in the jar are correct. + jar_cwd = options.classes_dir + class_files_rel = [os.path.relpath(f, jar_cwd) for f in class_files] + jar_cmd = ['jar', 'cf0', jar_path] + class_files_rel + + + md5_stamp = '%s.md5' % options.jar_path + md5_checker = md5_check.Md5Checker( + stamp=md5_stamp, inputs=class_files, command=jar_cmd) + if md5_checker.IsStale(): + build_utils.CheckCallDie(jar_cmd, cwd=jar_cwd) + else: + build_utils.Touch(options.jar_path) + md5_checker.Write() + + +def main(argv): + parser = optparse.OptionParser() + parser.add_option('--classes-dir', help='Directory containing .class files.') + parser.add_option('--jar-path', help='Jar output path.') + parser.add_option('--excluded-classes', + help='List of .class file patterns to exclude from the jar.') + parser.add_option('--stamp', help='Path to touch on success.') + + # TODO(newt): remove this once http://crbug.com/177552 is fixed in ninja. + parser.add_option('--ignore', help='Ignored.') + + options, _ = parser.parse_args() + + DoJar(options) + + if options.stamp: + build_utils.Touch(options.stamp) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + diff --git a/build/android/gyp/javac.py b/build/android/gyp/javac.py new file mode 100755 index 0000000..2968017 --- /dev/null +++ b/build/android/gyp/javac.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import fnmatch +import optparse +import os +import sys + +BUILD_ANDROID_DIR = os.path.join(os.path.dirname(__file__), '..') +sys.path.append(BUILD_ANDROID_DIR) + +from pylib import build_utils + +def DoJavac(options): + output_dir = options.output_dir + + src_dirs = build_utils.ParseGypList(options.src_dirs) + java_files = build_utils.FindInDirectories(src_dirs, '*.java') + if options.javac_includes: + javac_includes = build_utils.ParseGypList(options.javac_includes) + filtered_java_files = [] + for f in java_files: + for include in javac_includes: + if fnmatch.fnmatch(f, include): + filtered_java_files.append(f) + break + java_files = filtered_java_files + + # Compiling guava with certain orderings of input files causes a compiler + # crash... Sorted order works, so use that. + # See https://code.google.com/p/guava-libraries/issues/detail?id=950 + java_files.sort() + + classpath = build_utils.ParseGypList(options.classpath) + + # Delete the classes directory. This ensures that all .class files in the + # output are actually from the input .java files. For example, if a .java + # file is deleted or an inner class is removed, the classes directory should + # not contain the corresponding old .class file after running this action. + build_utils.DeleteDirectory(output_dir) + build_utils.MakeDirectory(output_dir) + + cmd = [ + 'javac', + '-g', + '-source', '1.5', + '-target', '1.5', + '-classpath', ':'.join(classpath), + '-d', output_dir] + + # Only output Java warnings for chromium code + if options.chromium_code: + cmd += ['-Xlint:unchecked'] + else: + cmd += [# Suppress "Sun proprietary API" warnings. See: goo.gl/OYxUM + '-XDignore.symbol.file'] + + build_utils.CheckCallDie(cmd + java_files) + +def main(argv): + parser = optparse.OptionParser() + parser.add_option('--src-dirs', help='Directories containing java files.') + parser.add_option('--javac-includes', + help='A list of file patterns. If provided, only java files that match' + + 'one of the patterns will be compiled.') + parser.add_option('--classpath', help='Classpath for javac.') + parser.add_option('--output-dir', help='Directory for javac output.') + parser.add_option('--stamp', help='Path to touch on success.') + parser.add_option('--chromium-code', type='int', help='Whether code being ' + 'compiled should be built with stricter warnings for ' + 'chromium code.') + + # TODO(newt): remove this once http://crbug.com/177552 is fixed in ninja. + parser.add_option('--ignore', help='Ignored.') + + options, _ = parser.parse_args() + + DoJavac(options) + + if options.stamp: + build_utils.Touch(options.stamp) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + + diff --git a/build/android/gyp/push_libraries.py b/build/android/gyp/push_libraries.py index 9c7366e..9a32760 100755 --- a/build/android/gyp/push_libraries.py +++ b/build/android/gyp/push_libraries.py @@ -13,6 +13,8 @@ import optparse import os import sys +from util import md5_check + BUILD_ANDROID_DIR = os.path.join(os.path.dirname(__file__), '..') sys.path.append(BUILD_ANDROID_DIR) @@ -24,11 +26,19 @@ def DoPush(options): libraries = build_utils.ReadJson(options.libraries_json) adb = android_commands.AndroidCommands() - adb.RunShellCommand('mkdir ' + options.device_dir) + needs_directory = True for lib in libraries: device_path = os.path.join(options.device_dir, lib) host_path = os.path.join(options.libraries_dir, lib) - adb.PushIfNeeded(host_path, device_path) + + md5_stamp = '%s.%s.md5' % (host_path, adb.Adb().GetSerialNumber()) + md5_checker = md5_check.Md5Checker(stamp=md5_stamp, inputs=[host_path]) + if md5_checker.IsStale(): + if needs_directory: + adb.RunShellCommand('mkdir ' + options.device_dir) + needs_directory = False + adb.PushIfNeeded(host_path, device_path) + md5_checker.Write() def main(argv): diff --git a/build/android/gyp/util/__init__.py b/build/android/gyp/util/__init__.py new file mode 100644 index 0000000..727e987 --- /dev/null +++ b/build/android/gyp/util/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + diff --git a/build/android/gyp/util/md5_check.py b/build/android/gyp/util/md5_check.py new file mode 100644 index 0000000..47f1ec9 --- /dev/null +++ b/build/android/gyp/util/md5_check.py @@ -0,0 +1,52 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import hashlib +import os + + +def UpdateMd5ForFile(md5, path, block_size=2**16): + with open(path, 'rb') as infile: + while True: + data = infile.read(block_size) + if not data: + break + md5.update(data) + + +def UpdateMd5ForDirectory(md5, dir_path): + for root, _, files in os.walk(dir_path): + for f in files: + UpdateMd5ForFile(md5, os.path.join(root, f)) + + +def UpdateMd5ForPath(md5, path): + if os.path.isdir(path): + UpdateMd5ForDirectory(md5, path) + else: + UpdateMd5ForFile(md5, path) + + +class Md5Checker(object): + def __init__(self, stamp=None, inputs=[], command=[]): + self.stamp = stamp + + md5 = hashlib.md5() + for i in inputs: + UpdateMd5ForPath(md5, i) + for s in command: + md5.update(s) + self.new_digest = md5.hexdigest() + + self.old_digest = '' + if os.path.exists(stamp): + with open(stamp, 'r') as old_stamp: + self.old_digest = old_stamp.read() + + def IsStale(self): + return self.old_digest != self.new_digest + + def Write(self): + with open(self.stamp, 'w') as new_stamp: + new_stamp.write(self.new_digest) diff --git a/build/android/jar.py b/build/android/jar.py deleted file mode 100755 index 9120a79..0000000 --- a/build/android/jar.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import fnmatch -import optparse -import os -import sys - -from pylib import build_utils - - -def DoJar(options): - class_files = build_utils.FindInDirectory(options.classes_dir, '*.class') - for exclude in build_utils.ParseGypList(options.excluded_classes): - class_files = filter( - lambda f: not fnmatch.fnmatch(f, exclude), class_files) - - jar_path = os.path.abspath(options.jar_path) - - # The paths of the files in the jar will be the same as they are passed in to - # the command. Because of this, the command should be run in - # options.classes_dir so the .class file paths in the jar are correct. - jar_cwd = options.classes_dir - class_files = [os.path.relpath(f, jar_cwd) for f in class_files] - jar_cmd = ['jar', 'cf0', jar_path] + class_files - build_utils.CheckCallDie(jar_cmd, cwd=jar_cwd) - - -def main(argv): - parser = optparse.OptionParser() - parser.add_option('--classes-dir', help='Directory containing .class files.') - parser.add_option('--jar-path', help='Jar output path.') - parser.add_option('--excluded-classes', - help='List of .class file patterns to exclude from the jar.') - parser.add_option('--stamp', help='Path to touch on success.') - - # TODO(newt): remove this once http://crbug.com/177552 is fixed in ninja. - parser.add_option('--ignore', help='Ignored.') - - options, _ = parser.parse_args() - - DoJar(options) - - if options.stamp: - build_utils.Touch(options.stamp) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) - diff --git a/build/android/javac.py b/build/android/javac.py deleted file mode 100755 index 2b19185..0000000 --- a/build/android/javac.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import fnmatch -import optparse -import os -import sys - -from pylib import build_utils - -def DoJavac(options): - output_dir = options.output_dir - - src_dirs = build_utils.ParseGypList(options.src_dirs) - java_files = build_utils.FindInDirectories(src_dirs, '*.java') - if options.javac_includes: - javac_includes = build_utils.ParseGypList(options.javac_includes) - filtered_java_files = [] - for f in java_files: - for include in javac_includes: - if fnmatch.fnmatch(f, include): - filtered_java_files.append(f) - break - java_files = filtered_java_files - - # Compiling guava with certain orderings of input files causes a compiler - # crash... Sorted order works, so use that. - # See https://code.google.com/p/guava-libraries/issues/detail?id=950 - java_files.sort() - - classpath = build_utils.ParseGypList(options.classpath) - - # Delete the classes directory. This ensures that all .class files in the - # output are actually from the input .java files. For example, if a .java - # file is deleted or an inner class is removed, the classes directory should - # not contain the corresponding old .class file after running this action. - build_utils.DeleteDirectory(output_dir) - build_utils.MakeDirectory(output_dir) - - cmd = [ - 'javac', - '-g', - '-source', '1.5', - '-target', '1.5', - '-classpath', ':'.join(classpath), - '-d', output_dir] - - # Only output Java warnings for chromium code - if options.chromium_code: - cmd += ['-Xlint:unchecked'] - else: - cmd += [# Suppress "Sun proprietary API" warnings. See: goo.gl/OYxUM - '-XDignore.symbol.file'] - - build_utils.CheckCallDie(cmd + java_files) - -def main(argv): - parser = optparse.OptionParser() - parser.add_option('--src-dirs', help='Directories containing java files.') - parser.add_option('--javac-includes', - help='A list of file patterns. If provided, only java files that match' + - 'one of the patterns will be compiled.') - parser.add_option('--classpath', help='Classpath for javac.') - parser.add_option('--output-dir', help='Directory for javac output.') - parser.add_option('--stamp', help='Path to touch on success.') - parser.add_option('--chromium-code', type='int', help='Whether code being ' - 'compiled should be built with stricter warnings for ' - 'chromium code.') - - # TODO(newt): remove this once http://crbug.com/177552 is fixed in ninja. - parser.add_option('--ignore', help='Ignored.') - - options, _ = parser.parse_args() - - DoJavac(options) - - if options.stamp: - build_utils.Touch(options.stamp) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) - - diff --git a/build/java.gypi b/build/java.gypi index b93a246..68725f5 100644 --- a/build/java.gypi +++ b/build/java.gypi @@ -187,7 +187,7 @@ }, 'inputs': [ '<(DEPTH)/build/android/pylib/build_utils.py', - '<(DEPTH)/build/android/javac.py', + '<(DEPTH)/build/android/gyp/javac.py', '>!@(find >(java_in_dir) >(additional_src_dirs) -name "*.java")', '>@(input_jars_paths)', '>@(additional_input_paths)', @@ -196,7 +196,7 @@ '<(compile_stamp)', ], 'action': [ - 'python', '<(DEPTH)/build/android/javac.py', + 'python', '<(DEPTH)/build/android/gyp/javac.py', '--output-dir=<(classes_dir)', '--classpath=>(input_jars_paths)', '--src-dirs=>(all_src_dirs)', @@ -213,14 +213,15 @@ 'message': 'Creating <(_target_name) jar', 'inputs': [ '<(DEPTH)/build/android/pylib/build_utils.py', - '<(DEPTH)/build/android/jar.py', + '<(DEPTH)/build/android/gyp/util/md5_check.py', + '<(DEPTH)/build/android/gyp/jar.py', '<(compile_stamp)', ], 'outputs': [ '<(jar_path)', ], 'action': [ - 'python', '<(DEPTH)/build/android/jar.py', + 'python', '<(DEPTH)/build/android/gyp/jar.py', '--classes-dir=<(classes_dir)', '--jar-path=<(jar_path)', '--excluded-classes=<(jar_excluded_classes)', @@ -234,14 +235,15 @@ 'message': 'Dexing <(_target_name) jar', 'inputs': [ '<(DEPTH)/build/android/pylib/build_utils.py', - '<(DEPTH)/build/android/dex.py', + '<(DEPTH)/build/android/gyp/util/md5_check.py', + '<(DEPTH)/build/android/gyp/dex.py', '<(jar_path)', ], 'outputs': [ '<(dex_path)', ], 'action': [ - 'python', '<(DEPTH)/build/android/dex.py', + 'python', '<(DEPTH)/build/android/gyp/dex.py', '--dex-path=<(dex_path)', '--android-sdk-root=<(android_sdk_root)', diff --git a/build/java_apk.gypi b/build/java_apk.gypi index ad88d9b..af6046e 100644 --- a/build/java_apk.gypi +++ b/build/java_apk.gypi @@ -231,6 +231,7 @@ 'message': 'Pushing libraries to device for <(_target_name)', 'inputs': [ '<(DEPTH)/build/android/pylib/build_utils.py', + '<(DEPTH)/build/android/gyp/util/md5_check.py', '<(DEPTH)/build/android/gyp/push_libraries.py', '<(strip_stamp)', ], @@ -378,7 +379,7 @@ }, 'inputs': [ '<(DEPTH)/build/android/pylib/build_utils.py', - '<(DEPTH)/build/android/javac.py', + '<(DEPTH)/build/android/gyp/javac.py', # If there is a separate find for additional_src_dirs, it will find the # wrong .java files when additional_src_dirs is empty. '>!@(find >(java_in_dir) >(additional_src_dirs) -name "*.java")', @@ -390,7 +391,7 @@ '<(compile_stamp)', ], 'action': [ - 'python', '<(DEPTH)/build/android/javac.py', + 'python', '<(DEPTH)/build/android/gyp/javac.py', '--output-dir=<(classes_dir)', '--classpath=>(input_jars_paths) <(android_sdk_jar)', '--src-dirs=>(all_src_dirs)', @@ -407,14 +408,15 @@ 'message': 'Creating <(_target_name) jar', 'inputs': [ '<(DEPTH)/build/android/pylib/build_utils.py', - '<(DEPTH)/build/android/jar.py', + '<(DEPTH)/build/android/gyp/util/md5_check.py', + '<(DEPTH)/build/android/gyp/jar.py', '<(compile_stamp)', ], 'outputs': [ '<(jar_stamp)', ], 'action': [ - 'python', '<(DEPTH)/build/android/jar.py', + 'python', '<(DEPTH)/build/android/gyp/jar.py', '--classes-dir=<(classes_dir)', '--jar-path=<(jar_path)', '--excluded-classes=<(jar_excluded_classes)', @@ -486,7 +488,8 @@ }, 'inputs': [ '<(DEPTH)/build/android/pylib/build_utils.py', - '<(DEPTH)/build/android/dex.py', + '<(DEPTH)/build/android/gyp/util/md5_check.py', + '<(DEPTH)/build/android/gyp/dex.py', '<(compile_stamp)', '>@(dex_inputs)', ], @@ -494,7 +497,7 @@ '<(dex_path)', ], 'action': [ - 'python', '<(DEPTH)/build/android/dex.py', + 'python', '<(DEPTH)/build/android/gyp/dex.py', '--dex-path=<(dex_path)', '--android-sdk-root=<(android_sdk_root)', diff --git a/build/java_prebuilt.gypi b/build/java_prebuilt.gypi index 7d12018..7c864d3 100644 --- a/build/java_prebuilt.gypi +++ b/build/java_prebuilt.gypi @@ -37,14 +37,14 @@ 'message': 'Dexing <(_target_name) jar', 'inputs': [ '<(DEPTH)/build/android/pylib/build_utils.py', - '<(DEPTH)/build/android/dex.py', + '<(DEPTH)/build/android/gyp/dex.py', '<(jar_path)', ], 'outputs': [ '<(dex_path)', ], 'action': [ - 'python', '<(DEPTH)/build/android/dex.py', + 'python', '<(DEPTH)/build/android/gyp/dex.py', '--dex-path=<(dex_path)', '--android-sdk-root=<(android_sdk_root)', -- cgit v1.1