diff options
Diffstat (limited to 'tools/isolate/isolate.py')
| -rwxr-xr-x | tools/isolate/isolate.py | 178 |
1 files changed, 113 insertions, 65 deletions
diff --git a/tools/isolate/isolate.py b/tools/isolate/isolate.py index be4f076..d70296a 100755 --- a/tools/isolate/isolate.py +++ b/tools/isolate/isolate.py @@ -4,14 +4,21 @@ # found in the LICENSE file. """Does one of the following depending on the --mode argument: - check verify all the inputs exist, touches the file specified with - --result and exits. - run recreates a tree with all the inputs files and run the executable - in it. - remap stores all the inputs files in a directory without running the - executable. + check verify all the inputs exist, touches the file specified with + --result and exits. + hashtable puts a manifest file and hard links each of the inputs into the + output directory. + remap stores all the inputs files in a directory without running the + executable. + run recreates a tree with all the inputs files and run the executable + in it. + +See more information at +http://dev.chromium.org/developers/testing/isolated-testing """ +import hashlib +import json import logging import optparse import os @@ -24,6 +31,9 @@ import time import tree_creator +# Needs to be coherent with the file's docstring above. +VALID_MODES = ('check', 'hashtable', 'remap', 'run') + def touch(filename): """Implements the equivalent of the 'touch' command.""" @@ -48,75 +58,104 @@ def rmtree(root): shutil.rmtree(root) -def isolate(outdir, root, resultfile, mode, read_only, args): - """Main function to isolate a target with its dependencies.""" +def relpath(path, root): + """os.path.relpath() that keeps trailing slash.""" + out = os.path.relpath(path, root) + if path.endswith('/'): + out += '/' + return out + + +def separate_inputs_command(args, root): + """Strips off the command line from the inputs. + + gyp provides input paths relative to cwd. Convert them to be relative to root. + """ cmd = [] if '--' in args: - # Strip off the command line from the inputs. i = args.index('--') cmd = args[i+1:] args = args[:i] - - # gyp provides paths relative to cwd. Convert them to be relative to - # root. cwd = os.getcwd() + return [relpath(os.path.join(cwd, arg), root) for arg in args], cmd - def make_relpath(i): - """Makes an input file a relative path but keeps any trailing slash.""" - out = os.path.relpath(os.path.join(cwd, i), root) - if i.endswith('/'): - out += '/' - return out - infiles = [make_relpath(i) for i in args] +def isolate(outdir, resultfile, indir, infiles, mode, read_only, cmd): + """Main function to isolate a target with its dependencies. - if not infiles: - raise ValueError('Need at least one input file to map') + It's behavior depends on |mode|. + """ + if mode == 'run': + return run(outdir, resultfile, indir, infiles, read_only, cmd) + + if mode == 'hashtable': + return hashtable(outdir, resultfile, indir, infiles) + + assert mode in ('check', 'remap'), mode + if mode == 'remap': + if not outdir: + outdir = tempfile.mkdtemp(prefix='isolate') + tree_creator.recreate_tree( + outdir, indir, infiles, tree_creator.HARDLINK) + if read_only: + tree_creator.make_writable(outdir, True) - # Other modes ignore the command. - if mode == 'run' and not cmd: + if resultfile: + # Signal the build tool that the test succeeded. + with open(resultfile, 'wb') as f: + for infile in infiles: + f.write(infile.encode('utf-8')) + f.write('\n') + + +def run(outdir, resultfile, indir, infiles, read_only, cmd): + """Implements the 'run' mode.""" + if not cmd: print >> sys.stderr, 'Using first input %s as executable' % infiles[0] cmd = [infiles[0]] - - tempdir = None + outdir = None try: - if not outdir and mode != 'check': - tempdir = tempfile.mkdtemp(prefix='isolate') - outdir = tempdir - elif outdir: - outdir = os.path.abspath(outdir) - + outdir = tempfile.mkdtemp(prefix='isolate') tree_creator.recreate_tree( - outdir, - os.path.abspath(root), - infiles, - tree_creator.DRY_RUN if mode == 'check' else tree_creator.HARDLINK, - lambda x: re.match(r'.*\.(svn|pyc)$', x)) - - if mode != 'check' and read_only: + outdir, indir, infiles, tree_creator.HARDLINK) + if read_only: tree_creator.make_writable(outdir, True) - if mode in ('check', 'remap'): - result = 0 - else: - # TODO(maruel): Remove me. Layering violation. Used by - # base/base_paths_linux.cc - env = os.environ.copy() - env['CR_SOURCE_ROOT'] = outdir.encode() - # Rebase the command to the right path. - cmd[0] = os.path.join(outdir, cmd[0]) - logging.info('Running %s' % cmd) - result = subprocess.call(cmd, cwd=outdir, env=env) - + # TODO(maruel): Remove me. Layering violation. Used by + # base/base_paths_linux.cc + env = os.environ.copy() + env['CR_SOURCE_ROOT'] = outdir.encode() + # Rebase the command to the right path. + cmd[0] = os.path.join(outdir, cmd[0]) + logging.info('Running %s' % cmd) + result = subprocess.call(cmd, cwd=outdir, env=env) if not result and resultfile: # Signal the build tool that the test succeeded. touch(resultfile) return result finally: - if tempdir and mode == 'isolate': - if read_only: - tree_creator.make_writable(tempdir, False) - rmtree(tempdir) + if read_only: + tree_creator.make_writable(outdir, False) + rmtree(outdir) + + +def hashtable(outdir, resultfile, indir, infiles): + """Implements the 'hashtable' mode.""" + results = {} + for relfile in infiles: + infile = os.path.join(indir, relfile) + h = hashlib.sha1() + with open(infile, 'rb') as f: + h.update(f.read()) + digest = h.hexdigest() + outfile = os.path.join(outdir, digest) + tree_creator.process_file(outfile, infile, tree_creator.HARDLINK) + results[relfile] = {'sha1': digest} + json.dump( + { + 'files': results, + }, + open(resultfile, 'wb')) def main(): @@ -128,16 +167,17 @@ def main(): parser.add_option( '-v', '--verbose', action='count', default=0, help='Use multiple times') parser.add_option( - '--mode', choices=['remap', 'check', 'run'], - help='Determines the action to be taken') + '--mode', choices=VALID_MODES, + help='Determines the action to be taken: %s' % ', '.join(VALID_MODES)) parser.add_option( - '--result', metavar='X', + '--result', metavar='FILE', help='File to be touched when the command succeeds') - parser.add_option('--root', help='Base directory to fetch files, required') parser.add_option( - '--outdir', metavar='X', - help='Directory used to recreate the tree. Defaults to a /tmp ' - 'subdirectory') + '--root', metavar='DIR', help='Base directory to fetch files, required') + parser.add_option( + '--outdir', metavar='DIR', + help='Directory used to recreate the tree or store the hash table. ' + 'Defaults to a /tmp subdirectory for modes run and remap.') parser.add_option( '--read-only', action='store_true', help='Make the temporary tree read-only') @@ -151,16 +191,24 @@ def main(): if not options.root: parser.error('--root is required.') + infiles, cmd = separate_inputs_command(args, options.root) + if not infiles: + parser.error('Need at least one input file to map') + # Preprocess the input files. try: + infiles, root = tree_creator.preprocess_inputs( + options.root, infiles, lambda x: re.match(r'.*\.(svn|pyc)$', x)) return isolate( options.outdir, - options.root, options.result, + root, + infiles, options.mode, options.read_only, - args) - except ValueError, e: - parser.error(str(e)) + cmd) + except tree_creator.MappingError, e: + print >> sys.stderr, str(e) + return 1 if __name__ == '__main__': |