Add input content checking to some build scripts

Some build steps, particularly javac, have really loose input rules.
I.e. javac steps are re-built when any input jar changes. Often, this
leads to unnecessary rebuilds of all the following steps.

Other build tools (ninja, goma), will check the contents of the inputs
to a step, and if those inputs haven't changed that tool doesn't
actually re-run the command for creating the output.

This change brings that same benefit to some of the Android python
build scripts. Particularly those that will save a significant amount
of time by adding input content checks.

The checking checks both the input files and the command that will be
run. It compares this against a stored md5 digest. If it has not
changed, then the output does not need to be recreated (though it is
still touched to trigger following steps).

BUG=158821


Review URL: https://chromiumcodereview.appspot.com/13432002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@192265 0039d316-1c4b-4281-b951-d872f2087c98

9 files changed, 121 insertions, 20 deletions

diff --git a/build/android/dex.py b/build/android/gyp/dex.py
index 00c2c52..69d3d96 100755
--- a/build/android/dex.py
+++ b/build/android/gyp/dex.py
@@ -9,13 +9,26 @@ import optparse
 import os
 import sys
 
+from util import md5_check
+
+BUILD_ANDROID_DIR = os.path.join(os.path.dirname(__file__), '..')
+sys.path.append(BUILD_ANDROID_DIR)
+
 from pylib import build_utils
 
 
 def DoDex(options, paths):
   dx_binary = os.path.join(options.android_sdk_root, 'platform-tools', 'dx')
   dex_cmd = [dx_binary, '--dex', '--output', options.dex_path] + paths
-  build_utils.CheckCallDie(dex_cmd)
+
+  md5_stamp = '%s.md5' % options.dex_path
+  md5_checker = md5_check.Md5Checker(
+      stamp=md5_stamp, inputs=paths, command=dex_cmd)
+  if md5_checker.IsStale():
+    build_utils.CheckCallDie(dex_cmd)
+  else:
+    build_utils.Touch(options.dex_path)
+  md5_checker.Write()
 
 
 def main(argv):
diff --git a/build/android/jar.py b/build/android/gyp/jar.py
index 9120a79..c8931c9 100755
--- a/build/android/jar.py
+++ b/build/android/gyp/jar.py
@@ -9,6 +9,11 @@ import optparse
 import os
 import sys
 
+from util import md5_check
+
+BUILD_ANDROID_DIR = os.path.join(os.path.dirname(__file__), '..')
+sys.path.append(BUILD_ANDROID_DIR)
+
 from pylib import build_utils
 
 
@@ -24,9 +29,18 @@ def DoJar(options):
   # the command. Because of this, the command should be run in
   # options.classes_dir so the .class file paths in the jar are correct.
   jar_cwd = options.classes_dir
-  class_files = [os.path.relpath(f, jar_cwd) for f in class_files]
-  jar_cmd = ['jar', 'cf0', jar_path] + class_files
-  build_utils.CheckCallDie(jar_cmd, cwd=jar_cwd)
+  class_files_rel = [os.path.relpath(f, jar_cwd) for f in class_files]
+  jar_cmd = ['jar', 'cf0', jar_path] + class_files_rel
+
+
+  md5_stamp = '%s.md5' % options.jar_path
+  md5_checker = md5_check.Md5Checker(
+      stamp=md5_stamp, inputs=class_files, command=jar_cmd)
+  if md5_checker.IsStale():
+    build_utils.CheckCallDie(jar_cmd, cwd=jar_cwd)
+  else:
+    build_utils.Touch(options.jar_path)
+  md5_checker.Write()
 
 
 def main(argv):
diff --git a/build/android/javac.py b/build/android/gyp/javac.py
index 2b19185..2968017 100755
--- a/build/android/javac.py
+++ b/build/android/gyp/javac.py
@@ -9,6 +9,9 @@ import optparse
 import os
 import sys
 
+BUILD_ANDROID_DIR = os.path.join(os.path.dirname(__file__), '..')
+sys.path.append(BUILD_ANDROID_DIR)
+
 from pylib import build_utils
 
 def DoJavac(options):
diff --git a/build/android/gyp/push_libraries.py b/build/android/gyp/push_libraries.py
index 9c7366e..9a32760 100755
--- a/build/android/gyp/push_libraries.py
+++ b/build/android/gyp/push_libraries.py
@@ -13,6 +13,8 @@ import optparse
 import os
 import sys
 
+from util import md5_check
+
 BUILD_ANDROID_DIR = os.path.join(os.path.dirname(__file__), '..')
 sys.path.append(BUILD_ANDROID_DIR)
 
@@ -24,11 +26,19 @@ def DoPush(options):
   libraries = build_utils.ReadJson(options.libraries_json)
 
   adb = android_commands.AndroidCommands()
-  adb.RunShellCommand('mkdir ' + options.device_dir)
+  needs_directory = True
   for lib in libraries:
     device_path = os.path.join(options.device_dir, lib)
     host_path = os.path.join(options.libraries_dir, lib)
-    adb.PushIfNeeded(host_path, device_path)
+
+    md5_stamp = '%s.%s.md5' % (host_path, adb.Adb().GetSerialNumber())
+    md5_checker = md5_check.Md5Checker(stamp=md5_stamp, inputs=[host_path])
+    if md5_checker.IsStale():
+      if needs_directory:
+        adb.RunShellCommand('mkdir ' + options.device_dir)
+        needs_directory = False
+      adb.PushIfNeeded(host_path, device_path)
+    md5_checker.Write()
 
 
 def main(argv):
diff --git a/build/android/gyp/util/__init__.py b/build/android/gyp/util/__init__.py
new file mode 100644
index 0000000..727e987
--- /dev/null
+++ b/build/android/gyp/util/__init__.py
@@ -0,0 +1,4 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
diff --git a/build/android/gyp/util/md5_check.py b/build/android/gyp/util/md5_check.py
new file mode 100644
index 0000000..47f1ec9
--- /dev/null
+++ b/build/android/gyp/util/md5_check.py
@@ -0,0 +1,52 @@
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import hashlib
+import os
+
+
+def UpdateMd5ForFile(md5, path, block_size=2**16):
+  with open(path, 'rb') as infile:
+    while True:
+      data = infile.read(block_size)
+      if not data:
+        break
+      md5.update(data)
+
+
+def UpdateMd5ForDirectory(md5, dir_path):
+  for root, _, files in os.walk(dir_path):
+    for f in files:
+      UpdateMd5ForFile(md5, os.path.join(root, f))
+
+
+def UpdateMd5ForPath(md5, path):
+  if os.path.isdir(path):
+    UpdateMd5ForDirectory(md5, path)
+  else:
+    UpdateMd5ForFile(md5, path)
+
+
+class Md5Checker(object):
+  def __init__(self, stamp=None, inputs=[], command=[]):
+    self.stamp = stamp
+
+    md5 = hashlib.md5()
+    for i in inputs:
+      UpdateMd5ForPath(md5, i)
+    for s in command:
+      md5.update(s)
+    self.new_digest = md5.hexdigest()
+
+    self.old_digest = ''
+    if os.path.exists(stamp):
+      with open(stamp, 'r') as old_stamp:
+        self.old_digest = old_stamp.read()
+
+  def IsStale(self):
+    return self.old_digest != self.new_digest
+
+  def Write(self):
+    with open(self.stamp, 'w') as new_stamp:
+      new_stamp.write(self.new_digest)
diff --git a/build/java.gypi b/build/java.gypi
index b93a246..68725f5 100644
--- a/build/java.gypi
+++ b/build/java.gypi
@@ -187,7 +187,7 @@
       },
       'inputs': [
         '<(DEPTH)/build/android/pylib/build_utils.py',
-        '<(DEPTH)/build/android/javac.py',
+        '<(DEPTH)/build/android/gyp/javac.py',
         '>!@(find >(java_in_dir) >(additional_src_dirs) -name "*.java")',
         '>@(input_jars_paths)',
         '>@(additional_input_paths)',
@@ -196,7 +196,7 @@
         '<(compile_stamp)',
       ],
       'action': [
-        'python', '<(DEPTH)/build/android/javac.py',
+        'python', '<(DEPTH)/build/android/gyp/javac.py',
         '--output-dir=<(classes_dir)',
         '--classpath=>(input_jars_paths)',
         '--src-dirs=>(all_src_dirs)',
@@ -213,14 +213,15 @@
       'message': 'Creating <(_target_name) jar',
       'inputs': [
         '<(DEPTH)/build/android/pylib/build_utils.py',
-        '<(DEPTH)/build/android/jar.py',
+        '<(DEPTH)/build/android/gyp/util/md5_check.py',
+        '<(DEPTH)/build/android/gyp/jar.py',
         '<(compile_stamp)',
       ],
       'outputs': [
         '<(jar_path)',
       ],
       'action': [
-        'python', '<(DEPTH)/build/android/jar.py',
+        'python', '<(DEPTH)/build/android/gyp/jar.py',
         '--classes-dir=<(classes_dir)',
         '--jar-path=<(jar_path)',
         '--excluded-classes=<(jar_excluded_classes)',
@@ -234,14 +235,15 @@
       'message': 'Dexing <(_target_name) jar',
       'inputs': [
         '<(DEPTH)/build/android/pylib/build_utils.py',
-        '<(DEPTH)/build/android/dex.py',
+        '<(DEPTH)/build/android/gyp/util/md5_check.py',
+        '<(DEPTH)/build/android/gyp/dex.py',
         '<(jar_path)',
       ],
       'outputs': [
         '<(dex_path)',
       ],
       'action': [
-        'python', '<(DEPTH)/build/android/dex.py',
+        'python', '<(DEPTH)/build/android/gyp/dex.py',
         '--dex-path=<(dex_path)',
         '--android-sdk-root=<(android_sdk_root)',
 
diff --git a/build/java_apk.gypi b/build/java_apk.gypi
index ad88d9b..af6046e 100644
--- a/build/java_apk.gypi
+++ b/build/java_apk.gypi
@@ -231,6 +231,7 @@
               'message': 'Pushing libraries to device for <(_target_name)',
               'inputs': [
                 '<(DEPTH)/build/android/pylib/build_utils.py',
+                '<(DEPTH)/build/android/gyp/util/md5_check.py',
                 '<(DEPTH)/build/android/gyp/push_libraries.py',
                 '<(strip_stamp)',
               ],
@@ -378,7 +379,7 @@
       },
       'inputs': [
         '<(DEPTH)/build/android/pylib/build_utils.py',
-        '<(DEPTH)/build/android/javac.py',
+        '<(DEPTH)/build/android/gyp/javac.py',
         # If there is a separate find for additional_src_dirs, it will find the
         # wrong .java files when additional_src_dirs is empty.
         '>!@(find >(java_in_dir) >(additional_src_dirs) -name "*.java")',
@@ -390,7 +391,7 @@
         '<(compile_stamp)',
       ],
       'action': [
-        'python', '<(DEPTH)/build/android/javac.py',
+        'python', '<(DEPTH)/build/android/gyp/javac.py',
         '--output-dir=<(classes_dir)',
         '--classpath=>(input_jars_paths) <(android_sdk_jar)',
         '--src-dirs=>(all_src_dirs)',
@@ -407,14 +408,15 @@
       'message': 'Creating <(_target_name) jar',
       'inputs': [
         '<(DEPTH)/build/android/pylib/build_utils.py',
-        '<(DEPTH)/build/android/jar.py',
+        '<(DEPTH)/build/android/gyp/util/md5_check.py',
+        '<(DEPTH)/build/android/gyp/jar.py',
         '<(compile_stamp)',
       ],
       'outputs': [
         '<(jar_stamp)',
       ],
       'action': [
-        'python', '<(DEPTH)/build/android/jar.py',
+        'python', '<(DEPTH)/build/android/gyp/jar.py',
         '--classes-dir=<(classes_dir)',
         '--jar-path=<(jar_path)',
         '--excluded-classes=<(jar_excluded_classes)',
@@ -486,7 +488,8 @@
       },
       'inputs': [
         '<(DEPTH)/build/android/pylib/build_utils.py',
-        '<(DEPTH)/build/android/dex.py',
+        '<(DEPTH)/build/android/gyp/util/md5_check.py',
+        '<(DEPTH)/build/android/gyp/dex.py',
         '<(compile_stamp)',
         '>@(dex_inputs)',
       ],
@@ -494,7 +497,7 @@
         '<(dex_path)',
       ],
       'action': [
-        'python', '<(DEPTH)/build/android/dex.py',
+        'python', '<(DEPTH)/build/android/gyp/dex.py',
         '--dex-path=<(dex_path)',
         '--android-sdk-root=<(android_sdk_root)',
 
diff --git a/build/java_prebuilt.gypi b/build/java_prebuilt.gypi
index 7d12018..7c864d3 100644
--- a/build/java_prebuilt.gypi
+++ b/build/java_prebuilt.gypi
@@ -37,14 +37,14 @@
       'message': 'Dexing <(_target_name) jar',
       'inputs': [
         '<(DEPTH)/build/android/pylib/build_utils.py',
-        '<(DEPTH)/build/android/dex.py',
+        '<(DEPTH)/build/android/gyp/dex.py',
         '<(jar_path)',
       ],
       'outputs': [
         '<(dex_path)',
       ],
       'action': [
-        'python', '<(DEPTH)/build/android/dex.py',
+        'python', '<(DEPTH)/build/android/gyp/dex.py',
         '--dex-path=<(dex_path)',
         '--android-sdk-root=<(android_sdk_root)',