GN: Add _incremental targets for gunit tests

E.g. instead of "base_unittests", build "base_unittests_incremental".
Then, run "bin/run_base_unittests_incremental"

When changing a java file, recompiling & re-running test,
Non-incremental: 27 seconds
Incremental: 14 seconds

BUG=540857,520082

Review URL: https://codereview.chromium.org/1403073003

Cr-Commit-Position: refs/heads/master@{#354364}

8 files changed, 207 insertions, 126 deletions

diff --git a/build/android/incremental_install/__init__.py b/build/android/incremental_install/__init__.py
new file mode 100644
index 0000000..1aaf0e1
--- /dev/null
+++ b/build/android/incremental_install/__init__.py
@@ -0,0 +1,4 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
diff --git a/build/android/incremental_install/create_install_script.py b/build/android/incremental_install/create_install_script.py
index 614cbb7..5040a56 100755
--- a/build/android/incremental_install/create_install_script.py
+++ b/build/android/incremental_install/create_install_script.py
@@ -28,20 +28,40 @@ import os
 import subprocess
 import sys
 
-def main():
+
+def _ResolvePath(path):
   script_directory = os.path.dirname(__file__)
+  return os.path.abspath(os.path.join(script_directory, path))
+
 
-  def resolve_path(path):
-    return os.path.abspath(os.path.join(script_directory, path))
+# Exported to allow test runner to be able to install incremental apks.
+def GetInstallParameters():
+  apk_path = {apk_path}
+  lib_dir = {lib_dir}
+  dex_files = {dex_files}
+  splits = {splits}
 
-  cmd_path = resolve_path({cmd_path})
-  cmd_args = [cmd_path] + {cmd_args}
-  cmd_path_args = {cmd_path_args}
-  for arg, path in cmd_path_args:
-    if arg:
-      cmd_args.append(arg)
-    cmd_args.append(resolve_path(path))
+  return dict(apk_path=_ResolvePath(apk_path),
+              dex_files=[_ResolvePath(p) for p in dex_files],
+              lib_dir=_ResolvePath(lib_dir),
+              splits=[_ResolvePath(p) for p in splits])
 
+
+def main():
+  output_directory = {output_directory}
+  cmd_path = {cmd_path}
+  params = GetInstallParameters()
+  cmd_args = [
+      _ResolvePath(cmd_path),
+      '--output-directory', _ResolvePath(output_directory),
+  ]
+  if params['lib_dir']:
+    cmd_args.extend(('--lib-dir', params['lib_dir']))
+  for dex_path in params['dex_files']:
+    cmd_args.extend(('--dex-file', dex_path))
+  for split in params['splits']:
+    cmd_args.extend(('--split', split))
+  cmd_args.append(params['apk_path'])
   return subprocess.call(cmd_args + sys.argv[1:])
 
 if __name__ == '__main__':
@@ -87,32 +107,23 @@ def main(args):
   options = _ParseArgs(args)
 
   def relativize(path):
-    return os.path.relpath(path, os.path.dirname(options.script_output_path))
+    script_dir = os.path.dirname(options.script_output_path)
+    return path and os.path.relpath(path, script_dir)
 
   installer_path = os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'android',
                                 'incremental_install', 'installer.py')
-  installer_path = relativize(installer_path)
-
-  path_args = [
-      ('--output-directory', relativize(options.output_directory)),
-      (None, relativize(options.apk_path)),
-  ]
-
-  if options.lib_dir:
-    path_args.append(('--lib-dir', relativize(options.lib_dir)))
-
-  if options.dex_files:
-    for dex_file in options.dex_files:
-      path_args.append(('--dex-file', relativize(dex_file)))
-
-  for split_arg in options.splits:
-    path_args.append(('--split', relativize(split_arg)))
+  pformat = pprint.pformat
+  template_args = {
+      'cmd_path': pformat(relativize(installer_path)),
+      'apk_path': pformat(relativize(options.apk_path)),
+      'output_directory': pformat(relativize(options.output_directory)),
+      'lib_dir': pformat(relativize(options.lib_dir)),
+      'dex_files': pformat([relativize(p) for p in options.dex_files]),
+      'splits': pformat([relativize(p) for p in options.splits]),
+  }
 
   with open(options.script_output_path, 'w') as script:
-    script.write(SCRIPT_TEMPLATE.format(
-        cmd_path=pprint.pformat(installer_path),
-        cmd_args='[]',
-        cmd_path_args=pprint.pformat(path_args)))
+    script.write(SCRIPT_TEMPLATE.format(**template_args))
 
   os.chmod(options.script_output_path, 0750)
 
diff --git a/build/android/incremental_install/installer.py b/build/android/incremental_install/installer.py
index 40d2d7b..b5ed5ce 100755
--- a/build/android/incremental_install/installer.py
+++ b/build/android/incremental_install/installer.py
@@ -24,8 +24,10 @@ from pylib import constants
 from pylib.utils import run_tests_helper
 from pylib.utils import time_profile
 
-sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'gyp'))
+prev_sys_path = list(sys.path)
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, 'gyp'))
 from util import build_utils
+sys.path = prev_sys_path
 
 
 def _TransformDexPaths(paths):
@@ -46,98 +48,48 @@ def _Execute(concurrently, *funcs):
   return timer
 
 
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('apk_path',
-                      help='The path to the APK to install.')
-  parser.add_argument('--split',
-                      action='append',
-                      dest='splits',
-                      help='A glob matching the apk splits. '
-                           'Can be specified multiple times.')
-  parser.add_argument('--lib-dir',
-                      help='Path to native libraries directory.')
-  parser.add_argument('--dex-files',
-                      help='List of dex files to push.',
-                      action='append',
-                      default=[])
-  parser.add_argument('-d', '--device', dest='device',
-                      help='Target device for apk to install on.')
-  parser.add_argument('--uninstall',
-                      action='store_true',
-                      default=False,
-                      help='Remove the app and all side-loaded files.')
-  parser.add_argument('--output-directory',
-                      help='Path to the root build directory.')
-  parser.add_argument('--no-threading',
-                      action='store_false',
-                      default=True,
-                      dest='threading',
-                      help='Do not install and push concurrently')
-  parser.add_argument('--no-cache',
-                      action='store_false',
-                      default=True,
-                      dest='cache',
-                      help='Do not use cached information about what files are '
-                           'currently on the target device.')
-  parser.add_argument('-v',
-                      '--verbose',
-                      dest='verbose_count',
-                      default=0,
-                      action='count',
-                      help='Verbose level (multiple times for more)')
+def _GetDeviceIncrementalDir(package):
+  """Returns the device path to put incremental files for the given package."""
+  return '/data/local/tmp/incremental-app-%s' % package
 
-  args = parser.parse_args()
 
-  run_tests_helper.SetLogLevel(args.verbose_count)
-  constants.SetBuildType('Debug')
-  if args.output_directory:
-    constants.SetOutputDirectory(args.output_directory)
+def Uninstall(device, package):
+  """Uninstalls and removes all incremental files for the given package."""
+  main_timer = time_profile.TimeProfile()
+  device.Uninstall(package)
+  device.RunShellCommand(['rm', '-rf', _GetDeviceIncrementalDir(package)],
+                         check_return=True)
+  logging.info('Uninstall took %s seconds.', main_timer.GetDelta())
+
 
+def Install(device, apk, split_globs=None, lib_dir=None, dex_files=None,
+            enable_device_cache=True, use_concurrency=True):
+  """Installs the given incremental apk and all required supporting files.
+
+  Args:
+    device: A DeviceUtils instance.
+    apk: The path to the apk, or an ApkHelper instance.
+    split_globs: Glob patterns for any required apk splits (optional).
+    lib_dir: Directory containing the app's native libraries (optional).
+    dex_files: List of .dex.jar files that comprise the app's Dalvik code.
+    enable_device_cache: Whether to enable on-device caching of checksums.
+    use_concurrency: Whether to speed things up using multiple threads.
+  """
   main_timer = time_profile.TimeProfile()
   install_timer = time_profile.TimeProfile()
   push_native_timer = time_profile.TimeProfile()
   push_dex_timer = time_profile.TimeProfile()
 
-  if args.device:
-    # Retries are annoying when commands fail for legitimate reasons. Might want
-    # to enable them if this is ever used on bots though.
-    device = device_utils.DeviceUtils(
-        args.device, default_retries=0, enable_device_files_cache=True)
-  else:
-    devices = device_utils.DeviceUtils.HealthyDevices(
-        default_retries=0, enable_device_files_cache=True)
-    if not devices:
-      raise device_errors.NoDevicesError()
-    elif len(devices) == 1:
-      device = devices[0]
-    else:
-      all_devices = device_utils.DeviceUtils.parallel(devices)
-      msg = ('More than one device available.\n'
-             'Use --device=SERIAL to select a device.\n'
-             'Available devices:\n')
-      descriptions = all_devices.pMap(lambda d: d.build_description).pGet(None)
-      for d, desc in zip(devices, descriptions):
-        msg += '  %s (%s)\n' % (d, desc)
-      raise Exception(msg)
-
-  apk = apk_helper.ApkHelper(args.apk_path)
+  apk = apk_helper.ToHelper(apk)
   apk_package = apk.GetPackageName()
-  device_incremental_dir = '/data/local/tmp/incremental-app-%s' % apk_package
-
-  if args.uninstall:
-    device.Uninstall(apk_package)
-    device.RunShellCommand(['rm', '-rf', device_incremental_dir],
-                           check_return=True)
-    logging.info('Uninstall took %s seconds.', main_timer.GetDelta())
-    return
+  device_incremental_dir = _GetDeviceIncrementalDir(apk_package)
 
   # Install .apk(s) if any of them have changed.
   def do_install():
     install_timer.Start()
-    if args.splits:
+    if split_globs:
       splits = []
-      for split_glob in args.splits:
+      for split_glob in split_globs:
         splits.extend((f for f in glob.glob(split_glob)))
       device.InstallSplitApk(apk, splits, reinstall=True,
                              allow_cached_props=True, permissions=())
@@ -147,22 +99,22 @@ def main():
 
   # Push .so and .dex files to the device (if they have changed).
   def do_push_files():
-    if args.lib_dir:
+    if lib_dir:
       push_native_timer.Start()
       device_lib_dir = posixpath.join(device_incremental_dir, 'lib')
-      device.PushChangedFiles([(args.lib_dir, device_lib_dir)],
+      device.PushChangedFiles([(lib_dir, device_lib_dir)],
                               delete_device_stale=True)
       push_native_timer.Stop(log=False)
 
-    if args.dex_files:
+    if dex_files:
       push_dex_timer.Start()
       # Put all .dex files to be pushed into a temporary directory so that we
       # can use delete_device_stale=True.
       with build_utils.TempDir() as temp_dir:
         device_dex_dir = posixpath.join(device_incremental_dir, 'dex')
         # Ensure no two files have the same name.
-        transformed_names = _TransformDexPaths(args.dex_files)
-        for src_path, dest_name in zip(args.dex_files, transformed_names):
+        transformed_names = _TransformDexPaths(dex_files)
+        for src_path, dest_name in zip(dex_files, transformed_names):
           shutil.copyfile(src_path, os.path.join(temp_dir, dest_name))
         device.PushChangedFiles([(temp_dir, device_dex_dir)],
                                 delete_device_stale=True)
@@ -183,7 +135,7 @@ def main():
 
   cache_path = '%s/files-cache.json' % device_incremental_dir
   def restore_cache():
-    if not args.cache:
+    if not enable_device_cache:
       logging.info('Ignoring device cache')
       return
     # Delete the cached file so that any exceptions cause the next attempt
@@ -219,11 +171,11 @@ def main():
   # been designed for multi-threading. Enabling only because this is a
   # developer-only tool.
   setup_timer = _Execute(
-      args.threading, create_lock_files, restore_cache, check_selinux)
+      use_concurrency, create_lock_files, restore_cache, check_selinux)
 
-  _Execute(args.threading, do_install, do_push_files)
+  _Execute(use_concurrency, do_install, do_push_files)
 
-  finalize_timer = _Execute(args.threading, release_installer_lock, save_cache)
+  finalize_timer = _Execute(use_concurrency, release_installer_lock, save_cache)
 
   logging.info(
       'Took %s seconds (setup=%s, install=%s, libs=%s, dex=%s, finalize=%s)',
@@ -232,6 +184,84 @@ def main():
       finalize_timer.GetDelta())
 
 
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('apk_path',
+                      help='The path to the APK to install.')
+  parser.add_argument('--split',
+                      action='append',
+                      dest='splits',
+                      help='A glob matching the apk splits. '
+                           'Can be specified multiple times.')
+  parser.add_argument('--lib-dir',
+                      help='Path to native libraries directory.')
+  parser.add_argument('--dex-files',
+                      help='List of dex files to push.',
+                      action='append',
+                      default=[])
+  parser.add_argument('-d', '--device', dest='device',
+                      help='Target device for apk to install on.')
+  parser.add_argument('--uninstall',
+                      action='store_true',
+                      default=False,
+                      help='Remove the app and all side-loaded files.')
+  parser.add_argument('--output-directory',
+                      help='Path to the root build directory.')
+  parser.add_argument('--no-threading',
+                      action='store_false',
+                      default=True,
+                      dest='threading',
+                      help='Do not install and push concurrently')
+  parser.add_argument('--no-cache',
+                      action='store_false',
+                      default=True,
+                      dest='cache',
+                      help='Do not use cached information about what files are '
+                           'currently on the target device.')
+  parser.add_argument('-v',
+                      '--verbose',
+                      dest='verbose_count',
+                      default=0,
+                      action='count',
+                      help='Verbose level (multiple times for more)')
+
+  args = parser.parse_args()
+
+  run_tests_helper.SetLogLevel(args.verbose_count)
+  constants.SetBuildType('Debug')
+  if args.output_directory:
+    constants.SetOutputDirectory(args.output_directory)
+
+  if args.device:
+    # Retries are annoying when commands fail for legitimate reasons. Might want
+    # to enable them if this is ever used on bots though.
+    device = device_utils.DeviceUtils(
+        args.device, default_retries=0, enable_device_files_cache=True)
+  else:
+    devices = device_utils.DeviceUtils.HealthyDevices(
+        default_retries=0, enable_device_files_cache=True)
+    if not devices:
+      raise device_errors.NoDevicesError()
+    elif len(devices) == 1:
+      device = devices[0]
+    else:
+      all_devices = device_utils.DeviceUtils.parallel(devices)
+      msg = ('More than one device available.\n'
+             'Use --device=SERIAL to select a device.\n'
+             'Available devices:\n')
+      descriptions = all_devices.pMap(lambda d: d.build_description).pGet(None)
+      for d, desc in zip(devices, descriptions):
+        msg += '  %s (%s)\n' % (d, desc)
+      raise Exception(msg)
+
+  apk = apk_helper.ToHelper(args.apk_path)
+  if args.uninstall:
+    Uninstall(device, apk.GetPackageName())
+  else:
+    Install(device, apk, split_globs=args.splits, lib_dir=args.lib_dir,
+            dex_files=args.dex_files, enable_device_cache=args.cache,
+            use_concurrency=args.threading)
+
+
 if __name__ == '__main__':
   sys.exit(main())
-
diff --git a/build/android/pylib/gtest/gtest_test_instance.py b/build/android/pylib/gtest/gtest_test_instance.py
index e61b89c..d276592 100644
--- a/build/android/pylib/gtest/gtest_test_instance.py
+++ b/build/android/pylib/gtest/gtest_test_instance.py
@@ -138,9 +138,10 @@ class GtestTestInstance(test_instance.TestInstance):
 
     self._shard_timeout = args.shard_timeout
 
+    incremental_part = '_incremental' if args.incremental_install else ''
     apk_path = os.path.join(
         constants.GetOutDirectory(), '%s_apk' % self._suite,
-        '%s-debug.apk' % self._suite)
+        '%s-debug%s.apk' % (self._suite, incremental_part))
     self._exe_path = os.path.join(constants.GetOutDirectory(),
                                   self._suite)
     if not os.path.exists(apk_path):
diff --git a/build/android/pylib/local/device/local_device_environment.py b/build/android/pylib/local/device/local_device_environment.py
index 25ef582..b877dbc 100644
--- a/build/android/pylib/local/device/local_device_environment.py
+++ b/build/android/pylib/local/device/local_device_environment.py
@@ -32,6 +32,7 @@ class LocalDeviceEnvironment(environment.Environment):
     self._max_tries = 1 + args.num_retries
     self._tool_name = args.tool
     self._enable_device_cache = args.enable_device_cache
+    self._incremental_install = args.incremental_install
 
   #override
   def SetUp(self):
@@ -64,6 +65,10 @@ class LocalDeviceEnvironment(environment.Environment):
     return self._devices
 
   @property
+  def incremental_install(self):
+    return self._incremental_install
+
+  @property
   def parallel_devices(self):
     return parallelizer.SyncParallelizer(self.devices)
 
diff --git a/build/android/pylib/local/device/local_device_gtest_run.py b/build/android/pylib/local/device/local_device_gtest_run.py
index 81d85fd..55747ac 100644
--- a/build/android/pylib/local/device/local_device_gtest_run.py
+++ b/build/android/pylib/local/device/local_device_gtest_run.py
@@ -2,6 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import imp
 import itertools
 import os
 import posixpath
@@ -9,6 +10,7 @@ import posixpath
 from devil.android import device_errors
 from devil.android import device_temp_file
 from devil.android import ports
+from incremental_install import installer
 from pylib import constants
 from pylib.gtest import gtest_test_instance
 from pylib.local import local_test_server_spawner
@@ -85,12 +87,27 @@ class _ApkDelegate(object):
     self._package = test_instance.package
     self._runner = test_instance.runner
     self._permissions = test_instance.permissions
-
+    self._suite = test_instance.suite
     self._component = '%s/%s' % (self._package, self._runner)
     self._extras = test_instance.extras
 
-  def Install(self, device):
-    device.Install(self._apk_helper, permissions=self._permissions)
+  def Install(self, device, incremental=False):
+    if not incremental:
+      device.Install(self._apk_helper, permissions=self._permissions)
+      return
+
+    installer_script = os.path.join(constants.GetOutDirectory(), 'bin',
+                                    'install_%s_apk_incremental' % self._suite)
+    try:
+      install_wrapper = imp.load_source('install_wrapper', installer_script)
+    except IOError:
+      raise Exception(('Incremental install script not found: %s\n'
+                       'Make sure to first build "%s_incremental"') %
+                      (installer_script, self._suite))
+    params = install_wrapper.GetInstallParameters()
+
+    installer.Install(device, self._apk_helper, split_globs=params['splits'],
+                      lib_dir=params['lib_dir'], dex_files=params['dex_files'])
 
   def Run(self, test, device, flags=None, **kwargs):
     extras = dict(self._extras)
@@ -141,7 +158,8 @@ class _ExeDelegate(object):
       self._deps_host_path = None
     self._test_run = tr
 
-  def Install(self, device):
+  def Install(self, device, incremental=False):
+    assert not incremental
     # TODO(jbudorick): Look into merging this with normal data deps pushing if
     # executables become supported on nonlocal environments.
     host_device_tuples = [(self._exe_host_path, self._exe_device_path)]
@@ -210,7 +228,7 @@ class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun):
     @local_device_test_run.handle_shard_failures
     def individual_device_set_up(dev, host_device_tuples):
       # Install test APK.
-      self._delegate.Install(dev)
+      self._delegate.Install(dev, incremental=self._env.incremental_install)
 
       # Push data dependencies.
       external_storage = dev.GetExternalStoragePath()
diff --git a/build/android/test_runner.py b/build/android/test_runner.py
index a3a3a98..955e09b 100755
--- a/build/android/test_runner.py
+++ b/build/android/test_runner.py
@@ -188,6 +188,8 @@ def AddDeviceOptions(parser):
   group.add_argument('--blacklist-file', help='Device blacklist file.')
   group.add_argument('--enable-device-cache', action='store_true',
                      help='Cache device state to disk between runs')
+  group.add_argument('--incremental-install', action='store_true',
+                     help='Use an _incremental apk.')
 
 
 def AddGTestOptions(parser):
diff --git a/build/config/android/internal_rules.gni b/build/config/android/internal_rules.gni
index 6950cca..a64f14a 100644
--- a/build/config/android/internal_rules.gni
+++ b/build/config/android/internal_rules.gni
@@ -1661,6 +1661,16 @@ template("test_runner_script") {
         rebase_path(invoker.isolate_file, root_build_dir),
       ]
     }
+    if (defined(invoker.incremental_install) && invoker.incremental_install) {
+      test_runner_args += [ "--incremental-install" ]
+
+      # These can still be overridden, but make more better defaults during
+      # development.
+      test_runner_args += [
+        "--enable-device-cache",
+        "--num_retries=0",
+      ]
+    }
 
     generated_script = "$root_build_dir/bin/run_${_test_name}"
     outputs = [