tools/android/loading: Optimize chrome_cache.{Pull,Push}BrowserCache()

Some websites have a lot of resources, slowing down sandwich's
cache pulling and pushing. This CL optimizes these operations
by pulling the cache directory recursively, but also by
introducing the commands queue that are written to a temporary
shell file and then pushed and ran on the device.

BUG=582080

Review URL: https://codereview.chromium.org/1753343002

Cr-Commit-Position: refs/heads/master@{#381200}

2 files changed, 43 insertions, 14 deletions

diff --git a/tools/android/loading/chrome_cache.py b/tools/android/loading/chrome_cache.py
index 40d15bb..bdbe1e4 100644
--- a/tools/android/loading/chrome_cache.py
+++ b/tools/android/loading/chrome_cache.py
@@ -20,6 +20,7 @@ _SRC_DIR = os.path.abspath(os.path.join(
 sys.path.append(os.path.join(_SRC_DIR, 'build', 'android'))
 from pylib import constants
 
+import device_setup
 import options
 
 
@@ -43,21 +44,10 @@ def _RemoteCacheDirectory():
       constants.PACKAGE_INFO[OPTIONS.chrome_package_name].package)
 
 
-def _UpdateTimestampFromAdbStat(filename, stat):
-  os.utime(filename, (stat.st_time, stat.st_time))
-
-
 def _AdbShell(adb, cmd):
   adb.Shell(subprocess.list2cmdline(cmd))
 
 
-def _AdbUtime(adb, filename, timestamp):
-  """Adb equivalent of os.utime(filename, (timestamp, timestamp))
-  """
-  touch_stamp = datetime.fromtimestamp(timestamp).strftime('%Y%m%d.%H%M%S')
-  _AdbShell(adb, ['touch', '-t', touch_stamp, filename])
-
-
 def PullBrowserCache(device):
   """Pulls the browser cache from the device and saves it locally.
 
@@ -71,8 +61,16 @@ def PullBrowserCache(device):
   _REAL_INDEX_FILE_NAME = 'the-real-index'
 
   remote_cache_directory = _RemoteCacheDirectory()
-  print remote_cache_directory
   save_target = tempfile.mkdtemp(suffix='.cache')
+
+  # Pull the cache recursively.
+  device.adb.Pull(remote_cache_directory, save_target)
+
+  # Update the modification time stamp on the local cache copy.
+  def _UpdateTimestampFromAdbStat(filename, stat):
+    assert os.path.exists(filename)
+    os.utime(filename, (stat.st_time, stat.st_time))
+
   for filename, stat in device.adb.Ls(remote_cache_directory):
     if filename == '..':
       continue
@@ -81,7 +79,6 @@ def PullBrowserCache(device):
       continue
     original_file = os.path.join(remote_cache_directory, filename)
     saved_file = os.path.join(save_target, filename)
-    device.adb.Pull(original_file, saved_file)
     _UpdateTimestampFromAdbStat(saved_file, stat)
     if filename == _INDEX_DIRECTORY_NAME:
       # The directory containing the index was pulled recursively, update the
@@ -119,11 +116,16 @@ def PushBrowserCache(device, local_cache_path):
   # Push cache content.
   device.adb.Push(local_cache_path, remote_cache_directory)
 
+  # Command queue to touch all files with correct timestamp.
+  command_queue = []
+
   # Walk through the local cache to update mtime on the device.
   def MirrorMtime(local_path):
     cache_relative_path = os.path.relpath(local_path, start=local_cache_path)
     remote_path = os.path.join(remote_cache_directory, cache_relative_path)
-    _AdbUtime(device.adb, remote_path, os.stat(local_path).st_mtime)
+    timestamp = os.stat(local_path).st_mtime
+    touch_stamp = datetime.fromtimestamp(timestamp).strftime('%Y%m%d.%H%M%S')
+    command_queue.append(['touch', '-t', touch_stamp, remote_path])
 
   for local_directory_path, dirnames, filenames in os.walk(
         local_cache_path, topdown=False):
@@ -133,6 +135,8 @@ def PushBrowserCache(device, local_cache_path):
       MirrorMtime(os.path.join(local_directory_path, dirname))
   MirrorMtime(local_cache_path)
 
+  device_setup.DeviceSubmitShellCommandQueue(device, command_queue)
+
 
 def ZipDirectoryContent(root_directory_path, archive_dest_path):
   """Zip a directory's content recursively with all the directories'
diff --git a/tools/android/loading/device_setup.py b/tools/android/loading/device_setup.py
index bb2b4d7..4ffbb51 100644
--- a/tools/android/loading/device_setup.py
+++ b/tools/android/loading/device_setup.py
@@ -59,6 +59,31 @@ def GetFirstDevice():
   return devices[0]
 
 
+def DeviceSubmitShellCommandQueue(device, command_queue):
+  """Executes on the device a command queue.
+
+  Args:
+    device: The device to execute the shell commands to.
+    command_queue: a list of commands to be executed in that order.
+  """
+  REMOTE_COMMAND_FILE_PATH = '/data/local/tmp/adb_command_file.sh'
+  if not command_queue:
+    return
+  with tempfile.NamedTemporaryFile(prefix='adb_command_file_',
+                                   suffix='.sh') as command_file:
+    command_file.write('#!/bin/sh\n')
+    command_file.write('# Shell file generated by {}\'s {}\n'.format(
+        __file__, DeviceSubmitShellCommandQueue.__name__))
+    command_file.write('set -e\n')
+    for command in command_queue:
+      command_file.write(subprocess.list2cmdline(command) + ' ;\n')
+    command_file.write('exit 0;\n'.format(
+        REMOTE_COMMAND_FILE_PATH))
+    command_file.flush()
+    device.adb.Push(command_file.name, REMOTE_COMMAND_FILE_PATH)
+    device.adb.Shell('sh {p} && rm {p}'.format(p=REMOTE_COMMAND_FILE_PATH))
+
+
 @contextlib.contextmanager
 def FlagReplacer(device, command_line_path, new_flags):
   """Replaces chrome flags in a context, restores them afterwards.