Add tool to use the manifest, fetch and cache dependencies and run the test.

Only keep functions needed on the slave in tree_creator.py. Move the rest to
isolate.py.

R=rogerta@chromium.org
BUG=98834
TEST=Run from the ouput directory:
    GYP_DEFINES="$GYP_DEFINES tests_run=hashtable" ../../build/gyp_chromium; \
    ninja base_unittests_run; \
    ../../tools/isolate/tree_creator.py -m base_unittests.results --remote .

Review URL: https://chromiumcodereview.appspot.com/9834090

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@129455 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 316 insertions, 108 deletions

diff --git a/tools/isolate/isolate.py b/tools/isolate/isolate.py
index 9a4dcdc..47925dc 100755
--- a/tools/isolate/isolate.py
+++ b/tools/isolate/isolate.py
@@ -19,11 +19,13 @@ See more information at
 http://dev.chromium.org/developers/testing/isolated-testing
 """
 
+import hashlib
 import json
 import logging
 import optparse
 import os
 import re
+import stat
 import subprocess
 import sys
 import tempfile
@@ -50,6 +52,98 @@ def to_relative(path, root, relative):
   return path
 
 
+def expand_directories(indir, infiles, blacklist):
+  """Expands the directories, applies the blacklist and verifies files exist."""
+  logging.debug('expand_directories(%s, %s, %s)' % (indir, infiles, blacklist))
+  outfiles = []
+  for relfile in infiles:
+    if os.path.isabs(relfile):
+      raise tree_creator.MappingError('Can\'t map absolute path %s' % relfile)
+    infile = os.path.normpath(os.path.join(indir, relfile))
+    if not infile.startswith(indir):
+      raise tree_creator.MappingError(
+          'Can\'t map file %s outside %s' % (infile, indir))
+
+    if relfile.endswith('/'):
+      if not os.path.isdir(infile):
+        raise tree_creator.MappingError(
+            'Input directory %s must have a trailing slash' % infile)
+      for dirpath, dirnames, filenames in os.walk(infile):
+        # Convert the absolute path to subdir + relative subdirectory.
+        reldirpath = dirpath[len(indir)+1:]
+        outfiles.extend(os.path.join(reldirpath, f) for f in filenames)
+        for index, dirname in enumerate(dirnames):
+          # Do not process blacklisted directories.
+          if blacklist(os.path.join(reldirpath, dirname)):
+            del dirnames[index]
+    else:
+      if not os.path.isfile(infile):
+        raise tree_creator.MappingError('Input file %s doesn\'t exist' % infile)
+      outfiles.append(relfile)
+  return outfiles
+
+
+def process_inputs(indir, infiles, need_hash, read_only):
+  """Returns a dictionary of input files, populated with the files' mode and
+  hash.
+
+  The file mode is manipulated if read_only is True. In practice, we only save
+  one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r).
+  """
+  outdict = {}
+  for infile in infiles:
+    filepath = os.path.join(indir, infile)
+    filemode = stat.S_IMODE(os.stat(filepath).st_mode)
+    # Remove write access for non-owner.
+    filemode &= ~(stat.S_IWGRP | stat.S_IWOTH)
+    if read_only:
+      filemode &= ~stat.S_IWUSR
+    if filemode & stat.S_IXUSR:
+      filemode |= (stat.S_IXGRP | stat.S_IXOTH)
+    else:
+      filemode &= ~(stat.S_IXGRP | stat.S_IXOTH)
+    outdict[infile] = {
+      'mode': filemode,
+    }
+    if need_hash:
+      h = hashlib.sha1()
+      with open(filepath, 'rb') as f:
+        h.update(f.read())
+      outdict[infile]['sha-1'] = h.hexdigest()
+  return outdict
+
+
+def recreate_tree(outdir, indir, infiles, action):
+  """Creates a new tree with only the input files in it.
+
+  Arguments:
+    outdir:    Output directory to create the files in.
+    indir:     Root directory the infiles are based in.
+    infiles:   List of files to map from |indir| to |outdir|.
+    action:    See assert below.
+  """
+  logging.debug(
+      'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, infiles, action))
+  logging.info('Mapping from %s to %s' % (indir, outdir))
+
+  assert action in (
+      tree_creator.HARDLINK, tree_creator.SYMLINK, tree_creator.COPY)
+  outdir = os.path.normpath(outdir)
+  if not os.path.isdir(outdir):
+    logging.info ('Creating %s' % outdir)
+    os.makedirs(outdir)
+  # Do not call abspath until the directory exists.
+  outdir = os.path.abspath(outdir)
+
+  for relfile in infiles:
+    infile = os.path.join(indir, relfile)
+    outfile = os.path.join(outdir, relfile)
+    outsubdir = os.path.dirname(outfile)
+    if not os.path.isdir(outsubdir):
+      os.makedirs(outsubdir)
+    tree_creator.link_file(outfile, infile, action)
+
+
 def separate_inputs_command(args, root, files):
   """Strips off the command line from the inputs.
 
@@ -94,8 +188,8 @@ def isolate(outdir, resultfile, indir, infiles, mode, read_only, cmd, no_save):
   assert mode_fn
   assert os.path.isabs(resultfile)
 
-  infiles = tree_creator.expand_directories(
-        indir, infiles, lambda x: re.match(r'.*\.(svn|pyc)$', x))
+  infiles = expand_directories(
+      indir, infiles, lambda x: re.match(r'.*\.(svn|pyc)$', x))
 
   # Note the relative current directory.
   # In general, this path will be the path containing the gyp file where the
@@ -116,8 +210,7 @@ def isolate(outdir, resultfile, indir, infiles, mode, read_only, cmd, no_save):
     cmd.insert(0, sys.executable)
 
   # Only hashtable mode really needs the sha-1.
-  dictfiles = tree_creator.process_inputs(
-      indir, infiles, mode == 'hashtable', read_only)
+  dictfiles = process_inputs(indir, infiles, mode == 'hashtable', read_only)
 
   result = mode_fn(
       outdir, indir, dictfiles, read_only, cmd, relative_cwd, resultfile)
@@ -164,8 +257,7 @@ def MODEremap(
   if len(os.listdir(outdir)):
     print 'Can\'t remap in a non-empty directory'
     return 1
-  tree_creator.recreate_tree(
-      outdir, indir, dictfiles.keys(), tree_creator.HARDLINK)
+  recreate_tree(outdir, indir, dictfiles.keys(), tree_creator.HARDLINK)
   if read_only:
     tree_creator.make_writable(outdir, True)
   return 0
@@ -176,8 +268,7 @@ def MODErun(
   """Always uses a temporary directory."""
   try:
     outdir = tempfile.mkdtemp(prefix='isolate')
-    tree_creator.recreate_tree(
-        outdir, indir, dictfiles.keys(), tree_creator.HARDLINK)
+    recreate_tree(outdir, indir, dictfiles.keys(), tree_creator.HARDLINK)
     cwd = os.path.join(outdir, relative_cwd)
     if not os.path.isdir(cwd):
       os.makedirs(cwd)
@@ -187,8 +278,6 @@ def MODErun(
     logging.info('Running %s, cwd=%s' % (cmd, cwd))
     return subprocess.call(cmd, cwd=cwd)
   finally:
-    if read_only:
-      tree_creator.make_writable(outdir, False)
     tree_creator.rmtree(outdir)
 
 
diff --git a/tools/isolate/tree_creator.py b/tools/isolate/tree_creator.py
index ad0990c..074f721 100644..100755
--- a/tools/isolate/tree_creator.py
+++ b/tools/isolate/tree_creator.py
@@ -1,21 +1,25 @@
+#!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""File related utility functions.
+"""Reads a manifest, creates a tree of hardlinks and runs the test.
 
-Creates a tree of hardlinks, symlinks or copy the inputs files. Calculate files
-hash.
+Keeps a local cache.
 """
 
 import ctypes
-import hashlib
+import json
 import logging
+import optparse
 import os
+import re
 import shutil
-import stat
+import subprocess
 import sys
+import tempfile
 import time
+import urllib
 
 
 # Types of action accepted by recreate_tree().
@@ -37,77 +41,20 @@ def os_link(source, link_name):
     os.link(source, link_name)
 
 
-def expand_directories(indir, infiles, blacklist):
-  """Expands the directories, applies the blacklist and verifies files exist."""
-  logging.debug('expand_directories(%s, %s, %s)' % (indir, infiles, blacklist))
-  outfiles = []
-  for relfile in infiles:
-    if os.path.isabs(relfile):
-      raise MappingError('Can\'t map absolute path %s' % relfile)
-    infile = os.path.normpath(os.path.join(indir, relfile))
-    if not infile.startswith(indir):
-      raise MappingError('Can\'t map file %s outside %s' % (infile, indir))
-
-    if relfile.endswith('/'):
-      if not os.path.isdir(infile):
-        raise MappingError(
-            'Input directory %s must have a trailing slash' % infile)
-      for dirpath, dirnames, filenames in os.walk(infile):
-        # Convert the absolute path to subdir + relative subdirectory.
-        relpath = dirpath[len(indir)+1:]
-        outfiles.extend(os.path.join(relpath, f) for f in filenames)
-        for index, dirname in enumerate(dirnames):
-          # Do not process blacklisted directories.
-          if blacklist(os.path.join(relpath, dirname)):
-            del dirnames[index]
-    else:
-      if not os.path.isfile(infile):
-        raise MappingError('Input file %s doesn\'t exist' % infile)
-      outfiles.append(relfile)
-  return outfiles
-
-
-def process_inputs(indir, infiles, need_hash, read_only):
-  """Returns a dictionary of input files, populated with the files' mode and
-  hash.
-
-  The file mode is manipulated if read_only is True. In practice, we only save
-  one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r).
-  """
-  outdict = {}
-  for infile in infiles:
-    filepath = os.path.join(indir, infile)
-    filemode = stat.S_IMODE(os.stat(filepath).st_mode)
-    # Remove write access for non-owner.
-    filemode &= ~(stat.S_IWGRP | stat.S_IWOTH)
-    if read_only:
-      filemode &= ~stat.S_IWUSR
-    if filemode & stat.S_IXUSR:
-      filemode |= (stat.S_IXGRP | stat.S_IXOTH)
-    else:
-      filemode &= ~(stat.S_IXGRP | stat.S_IXOTH)
-    outdict[infile] = {
-      'mode': filemode,
-    }
-    if need_hash:
-      h = hashlib.sha1()
-      with open(filepath, 'rb') as f:
-        h.update(f.read())
-      outdict[infile]['sha-1'] = h.hexdigest()
-  return outdict
-
-
 def link_file(outfile, infile, action):
   """Links a file. The type of link depends on |action|."""
   logging.debug('Mapping %s to %s' % (infile, outfile))
+  if action not in (HARDLINK, SYMLINK, COPY):
+    raise ValueError('Unknown mapping action %s' % action)
   if os.path.isfile(outfile):
     raise MappingError('%s already exist' % outfile)
 
   if action == COPY:
     shutil.copy(infile, outfile)
   elif action == SYMLINK and sys.platform != 'win32':
+    # On windows, symlink are converted to hardlink and fails over to copy.
     os.symlink(infile, outfile)
-  elif action == HARDLINK:
+  else:
     try:
       os_link(infile, outfile)
     except OSError:
@@ -116,38 +63,6 @@ def link_file(outfile, infile, action):
           'Failed to hardlink, failing back to copy %s to %s' % (
             infile, outfile))
       shutil.copy(infile, outfile)
-  else:
-    raise ValueError('Unknown mapping action %s' % action)
-
-
-def recreate_tree(outdir, indir, infiles, action):
-  """Creates a new tree with only the input files in it.
-
-  Arguments:
-    outdir:    Output directory to create the files in.
-    indir:     Root directory the infiles are based in.
-    infiles:   List of files to map from |indir| to |outdir|.
-    action:    See assert below.
-  """
-  logging.debug(
-      'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, infiles, action))
-  logging.info('Mapping from %s to %s' % (indir, outdir))
-
-  assert action in (HARDLINK, SYMLINK, COPY)
-  outdir = os.path.normpath(outdir)
-  if not os.path.isdir(outdir):
-    logging.info ('Creating %s' % outdir)
-    os.makedirs(outdir)
-  # Do not call abspath until the directory exists.
-  outdir = os.path.abspath(outdir)
-
-  for relfile in infiles:
-    infile = os.path.join(indir, relfile)
-    outfile = os.path.join(outdir, relfile)
-    outsubdir = os.path.dirname(outfile)
-    if not os.path.isdir(outsubdir):
-      os.makedirs(outsubdir)
-    link_file(outfile, infile, action)
 
 
 def _set_write_bit(path, read_only):
@@ -177,6 +92,7 @@ def make_writable(root, read_only):
 
 def rmtree(root):
   """Wrapper around shutil.rmtree() to retry automatically on Windows."""
+  make_writable(root, False)
   if sys.platform == 'win32':
     for i in range(3):
       try:
@@ -189,3 +105,206 @@ def rmtree(root):
         time.sleep(delay)
   else:
     shutil.rmtree(root)
+
+
+def open_remote(file_or_url):
+  """Reads a file or url."""
+  if re.match(r'^https?://.+$', file_or_url):
+    return urllib.urlopen(file_or_url)
+  return open(file_or_url, 'rb')
+
+
+def download_or_copy(file_or_url, dest):
+  """Copies a file or download an url."""
+  if re.match(r'^https?://.+$', file_or_url):
+    urllib.urlretrieve(file_or_url, dest)
+  else:
+    shutil.copy(file_or_url, dest)
+
+
+def get_free_space(path):
+  """Returns the number of free bytes."""
+  if sys.platform == 'win32':
+    free_bytes = ctypes.c_ulonglong(0)
+    ctypes.windll.kernel32.GetDiskFreeSpaceExW(
+        ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes))
+    return free_bytes.value
+  f = os.statvfs(path)
+  return f.f_bfree * f.f_frsize
+
+
+class Cache(object):
+  """Stateful LRU cache.
+
+  Saves its state as json file.
+  """
+  STATE_FILE = 'state.json'
+
+  def __init__(self, cache_dir, remote, max_cache_size, min_free_space):
+    """
+    Arguments:
+    - cache_dir: Directory where to place the cache.
+    - remote: Remote directory (NFS, SMB, etc) or HTTP url to fetch the objects
+              from
+    - max_cache_size: Trim if the cache gets larger than this value. If 0, the
+                      cache is effectively a leak.
+    - min_free_space: Trim if disk free space becomes lower than this value. If
+                      0, it unconditionally fill the disk.
+    """
+    self.cache_dir = cache_dir
+    self.remote = remote
+    self.max_cache_size = max_cache_size
+    self.min_free_space = min_free_space
+    self.state_file = os.path.join(cache_dir, self.STATE_FILE)
+    # The files are kept as an array in a LRU style. E.g. self.state[0] is the
+    # oldest item.
+    self.state = []
+
+    if not os.path.isdir(self.cache_dir):
+      os.makedirs(self.cache_dir)
+    if os.path.isfile(self.state_file):
+      try:
+        self.state = json.load(open(self.state_file, 'rb'))
+      except ValueError:
+        # Too bad. The file will be overwritten and the cache cleared.
+        pass
+    self.trim()
+
+  def trim(self):
+    """Trims anything we don't know, make sure enough free space exists."""
+    for f in os.listdir(self.cache_dir):
+      if f == self.STATE_FILE or f in self.state:
+        continue
+      logging.warn('Unknown file %s from cache' % f)
+      # Insert as the oldest file. It will be deleted eventually if not
+      # accessed.
+      self.state.insert(0, f)
+
+    # Ensure enough free space.
+    while (
+        self.min_free_space and
+        self.state and
+        get_free_space(self.cache_dir) < self.min_free_space):
+      os.remove(self.path(self.state.pop(0)))
+
+    # Ensure maximum cache size.
+    if self.max_cache_size and self.state:
+      sizes = [os.stat(self.path(f)).st_size for f in self.state]
+      while sizes and sum(sizes) > self.max_cache_size:
+        # Delete the oldest item.
+        os.remove(self.path(self.state.pop(0)))
+        sizes.pop(0)
+
+    self.save()
+
+  def retrieve(self, item):
+    """Retrieves a file from the remote and add it to the cache."""
+    assert not '/' in item
+    try:
+      index = self.state.index(item)
+      # Was already in cache. Update it's LRU value.
+      self.state.pop(index)
+      self.state.append(item)
+      return False
+    except ValueError:
+      out = self.path(item)
+      download_or_copy(os.path.join(self.remote, item), out)
+      self.state.append(item)
+      return True
+    finally:
+      self.save()
+
+  def path(self, item):
+    """Returns the path to one item."""
+    return os.path.join(self.cache_dir, item)
+
+  def save(self):
+    """Saves the LRU ordering."""
+    json.dump(self.state, open(self.state_file, 'wb'))
+
+
+def run_tha_test(manifest, cache_dir, remote, max_cache_size, min_free_space):
+  """Downloads the dependencies in the cache, hardlinks them into a temporary
+  directory and runs the executable.
+  """
+  cache = Cache(cache_dir, remote, max_cache_size, min_free_space)
+  outdir = tempfile.mkdtemp(prefix='run_tha_test')
+  try:
+    for filepath, properties in manifest['files'].iteritems():
+      infile = properties['sha-1']
+      outfile = os.path.join(outdir, filepath)
+      cache.retrieve(infile)
+      outfiledir = os.path.dirname(outfile)
+      if not os.path.isdir(outfiledir):
+        os.makedirs(outfiledir)
+      link_file(outfile, cache.path(infile), HARDLINK)
+      os.chmod(outfile, properties['mode'])
+
+    cwd = os.path.join(outdir, manifest['relative_cwd'])
+    if not os.path.isdir(cwd):
+      os.makedirs(cwd)
+    if manifest.get('read_only'):
+      make_writable(outdir, True)
+    cmd = manifest['command']
+    logging.info('Running %s, cwd=%s' % (cmd, cwd))
+    return subprocess.call(cmd, cwd=cwd)
+  finally:
+    # Save first, in case an exception occur in the following lines, then clean
+    # up.
+    cache.save()
+    rmtree(outdir)
+    cache.trim()
+
+
+def main():
+  parser = optparse.OptionParser(
+      usage='%prog <options>', description=sys.modules[__name__].__doc__)
+  parser.add_option(
+      '-v', '--verbose', action='count', default=0, help='Use multiple times')
+  parser.add_option(
+      '-m', '--manifest',
+      metavar='FILE',
+      help='File/url describing what to map or run')
+  parser.add_option('--no-run', action='store_true', help='Skip the run part')
+  parser.add_option(
+      '--cache',
+      default='cache',
+      metavar='DIR',
+      help='Cache directory, default=%default')
+  parser.add_option(
+      '-r', '--remote', metavar='URL', help='Remote where to get the items')
+  parser.add_option(
+      '--max-cache-size',
+      type='int',
+      metavar='NNN',
+      default=20*1024*1024*1024,
+      help='Trim if the cache gets larger than this value, default=%default')
+  parser.add_option(
+      '--min-free-space',
+      type='int',
+      metavar='NNN',
+      default=1*1024*1024*1024,
+      help='Trim if disk free space becomes lower than this value, '
+           'default=%default')
+
+  options, args = parser.parse_args()
+  level = [logging.ERROR, logging.INFO, logging.DEBUG][min(2, options.verbose)]
+  logging.basicConfig(
+      level=level,
+      format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s')
+
+  if not options.manifest:
+    parser.error('--manifest is required.')
+  if not options.remote:
+    parser.error('--remote is required.')
+  if args:
+    parser.error('Unsupported args %s' % ' '.join(args))
+
+  manifest = json.load(open_remote(options.manifest))
+  return run_tha_test(
+      manifest, os.path.abspath(options.cache), options.remote,
+      options.max_cache_size, options.min_free_space)
+
+
+if __name__ == '__main__':
+  sys.exit(main())