diff options
author | maruel@chromium.org <maruel@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-04-11 14:40:14 +0000 |
---|---|---|
committer | maruel@chromium.org <maruel@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-04-11 14:40:14 +0000 |
commit | 8bba0da89ce9a659c1979d496cb6d5d0f409576e (patch) | |
tree | 2af1bbbe8f4c8ab391b90dc7668c622b8f6bb1d4 /tools | |
parent | 7958385b0871c5651e7d0c5e1c63233638013d11 (diff) | |
download | chromium_src-8bba0da89ce9a659c1979d496cb6d5d0f409576e.zip chromium_src-8bba0da89ce9a659c1979d496cb6d5d0f409576e.tar.gz chromium_src-8bba0da89ce9a659c1979d496cb6d5d0f409576e.tar.bz2 |
Add trace_inputs.py support for Windows. Fix isolate.py on windows.
Uses logman.exe, tracerpt.exe and CSV parsing.
R=rogerta@chromium.org
BUG=98834
TEST=All tests passes on linux&mac&windows. Tracing a test with
isolate.py --mode=trace should work on Windows.
Review URL: https://chromiumcodereview.appspot.com/9958115
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@131765 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/isolate/isolate.py | 20 | ||||
-rwxr-xr-x | tools/isolate/isolate_test.py | 52 | ||||
-rw-r--r-- | tools/isolate/pylintrc | 3 | ||||
-rwxr-xr-x | tools/isolate/trace_inputs.py | 457 | ||||
-rwxr-xr-x | tools/isolate/trace_inputs_smoke_test.py | 60 |
5 files changed, 540 insertions, 52 deletions
diff --git a/tools/isolate/isolate.py b/tools/isolate/isolate.py index 5db2f17..91ef6e5 100755 --- a/tools/isolate/isolate.py +++ b/tools/isolate/isolate.py @@ -37,13 +37,20 @@ import tree_creator def relpath(path, root): """os.path.relpath() that keeps trailing slash.""" out = os.path.relpath(path, root) - if path.endswith('/'): - out += '/' + if path.endswith(os.path.sep): + out += os.path.sep + elif sys.platform == 'win32' and path.endswith('/'): + # TODO(maruel): Temporary. + out += os.path.sep return out def to_relative(path, root, relative): """Converts any absolute path to a relative path, only if under root.""" + if sys.platform == 'win32': + path = path.lower() + root = root.lower() + relative = relative.lower() if path.startswith(root): logging.info('%s starts with %s' % (path, root)) path = os.path.relpath(path, relative) @@ -64,7 +71,7 @@ def expand_directories(indir, infiles, blacklist): raise tree_creator.MappingError( 'Can\'t map file %s outside %s' % (infile, indir)) - if relfile.endswith('/'): + if relfile.endswith(os.path.sep): if not os.path.isdir(infile): raise tree_creator.MappingError( 'Input directory %s must have a trailing slash' % infile) @@ -289,12 +296,17 @@ def MODEtrace( checkout at src/. """ logging.info('Running %s, cwd=%s' % (cmd, os.path.join(indir, relative_cwd))) + try: + # Guesswork here. + product_dir = os.path.relpath(os.path.dirname(resultfile), indir) + except ValueError: + product_dir = '' return trace_inputs.trace_inputs( '%s.log' % resultfile, cmd, indir, relative_cwd, - os.path.relpath(os.path.dirname(resultfile), indir), # Guesswork here. + product_dir, False) diff --git a/tools/isolate/isolate_test.py b/tools/isolate/isolate_test.py index 3e05c226..7eaefc3 100755 --- a/tools/isolate/isolate_test.py +++ b/tools/isolate/isolate_test.py @@ -21,9 +21,15 @@ VERBOSE = False class CalledProcessError(subprocess.CalledProcessError): """Makes 2.6 version act like 2.7""" - def __init__(self, returncode, cmd, output): + def __init__(self, returncode, cmd, output, cwd): super(CalledProcessError, self).__init__(returncode, cmd) self.output = output + self.cwd = cwd + + def __str__(self): + return super(CalledProcessError, self).__str__() + ( + '\n' + 'cwd=%s\n%s') % (self.cwd, self.output) class Isolate(unittest.TestCase): @@ -44,12 +50,15 @@ class Isolate(unittest.TestCase): self.assertEquals(sorted(files), sorted(os.listdir(self.tempdir))) def _expected_result(self, with_hash, files, args, read_only): - # 4 modes are supported, 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r) - min_mode = 0444 - if not read_only: - min_mode |= 0200 - def mode(filename): - return (min_mode | 0111) if filename.endswith('.py') else min_mode + if sys.platform == 'win32': + mode = lambda _: 420 + else: + # 4 modes are supported, 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r) + min_mode = 0444 + if not read_only: + min_mode |= 0200 + def mode(filename): + return (min_mode | 0111) if filename.endswith('.py') else min_mode expected = { u'command': [unicode(sys.executable)] + @@ -82,11 +91,16 @@ class Isolate(unittest.TestCase): cmd.extend(['-v'] * 3) stdout = None stderr = None + cwd = ROOT_DIR p = subprocess.Popen( - cmd + args, stdout=stdout, stderr=stderr, cwd=ROOT_DIR) + cmd + args, + stdout=stdout, + stderr=stderr, + cwd=cwd, + universal_newlines=True) out = p.communicate()[0] if p.returncode: - raise CalledProcessError(p.returncode, cmd, out) + raise CalledProcessError(p.returncode, cmd, out, cwd) return out def test_help_modes(self): @@ -118,7 +132,10 @@ class Isolate(unittest.TestCase): self._execute(cmd) self._expected_tree(['result']) self._expected_result( - False, ['isolate_test.py'], ['./isolate_test.py'], False) + False, + ['isolate_test.py'], + [os.path.join('.', 'isolate_test.py')], + False) def test_check_non_existant(self): cmd = [ @@ -162,7 +179,7 @@ class Isolate(unittest.TestCase): '--mode', 'hashtable', '--outdir', self.tempdir, 'isolate_test.py', - os.path.join('data', 'isolate') + '/', + os.path.join('data', 'isolate') + os.path.sep, ] self._execute(cmd) files = [ @@ -170,7 +187,8 @@ class Isolate(unittest.TestCase): os.path.join('data', 'isolate', 'test_file1.txt'), os.path.join('data', 'isolate', 'test_file2.txt'), ] - data = self._expected_result(True, files, ['./isolate_test.py'], False) + data = self._expected_result( + True, files, [os.path.join('.', 'isolate_test.py')], False) self._expected_tree( [f['sha-1'] for f in data['files'].itervalues()] + ['result']) @@ -183,7 +201,10 @@ class Isolate(unittest.TestCase): self._execute(cmd) self._expected_tree(['isolate_test.py', 'result']) self._expected_result( - False, ['isolate_test.py'], ['./isolate_test.py'], False) + False, + ['isolate_test.py'], + [os.path.join('.', 'isolate_test.py')], + False) def test_run(self): cmd = [ @@ -220,7 +241,10 @@ class Isolate(unittest.TestCase): sys.executable, os.path.join(ROOT_DIR, 'isolate_test.py'), '--ok', ] out = self._execute(cmd, True) - self._expected_tree(['result', 'result.log']) + expected_tree = ['result', 'result.log'] + if sys.platform == 'win32': + expected_tree.append('result.log.etl') + self._expected_tree(expected_tree) # The 'result.log' log is OS-specific so we can't read it but we can read # the gyp result. # cmd[0] is not generated from infiles[0] so it's not using a relative path. diff --git a/tools/isolate/pylintrc b/tools/isolate/pylintrc index 964b05c..339a12e 100644 --- a/tools/isolate/pylintrc +++ b/tools/isolate/pylintrc @@ -51,10 +51,11 @@ load-plugins= # W0122: Use of the exec statement # W0141: Used builtin function '' # W0142: Used * or ** magic +# W0232: Class has no __init__ method # W0511: TODO # W0603: Using the global statement # W1201: Specify string format arguments as logging function parameters -disable=C0103,C0111,C0302,I0011,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0922,W0122,W0141,W0142,W0511,W0603,W1201 +disable=C0103,C0111,C0302,I0011,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0922,W0122,W0141,W0142,W0232,W0511,W0603,W1201 [REPORTS] diff --git a/tools/isolate/trace_inputs.py b/tools/isolate/trace_inputs.py index 820e533..7de92f7 100755 --- a/tools/isolate/trace_inputs.py +++ b/tools/isolate/trace_inputs.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# coding=utf-8 # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,6 +11,8 @@ Automatically extracts directories where all the files are used to make the dependencies list more compact. """ +import codecs +import csv import logging import optparse import os @@ -22,6 +25,83 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(os.path.dirname(BASE_DIR)) +if sys.platform == 'win32': + from ctypes.wintypes import create_unicode_buffer + from ctypes.wintypes import windll, FormatError # pylint: disable=E0611 + from ctypes.wintypes import GetLastError # pylint: disable=E0611 + + + def QueryDosDevice(drive_letter): + """Returns the Windows 'native' path for a DOS drive letter.""" + assert re.match(r'^[a-zA-Z]:$', drive_letter), drive_letter + # Guesswork. QueryDosDeviceW never returns the required number of bytes. + chars = 1024 + drive_letter = unicode(drive_letter) + p = create_unicode_buffer(chars) + if 0 == windll.kernel32.QueryDosDeviceW(drive_letter, p, chars): + err = GetLastError() + if err: + # pylint: disable=E0602 + raise WindowsError( + err, + 'QueryDosDevice(%s): %s (%d)' % ( + str(drive_letter), FormatError(err), err)) + return p.value + + + def GetShortPathName(long_path): + """Returns the Windows short path equivalent for a 'long' path.""" + long_path = unicode(long_path) + chars = windll.kernel32.GetShortPathNameW(long_path, None, 0) + if chars: + p = create_unicode_buffer(chars) + if windll.kernel32.GetShortPathNameW(long_path, p, chars): + return p.value + + err = GetLastError() + if err: + # pylint: disable=E0602 + raise WindowsError( + err, + 'GetShortPathName(%s): %s (%d)' % ( + str(long_path), FormatError(err), err)) + + + def get_current_encoding(): + """Returns the 'ANSI' code page associated to the process.""" + return 'cp%d' % int(windll.kernel32.GetACP()) + + + class DosDriveMap(object): + """Maps \Device\HarddiskVolumeN to N: on Windows.""" + # Keep one global cache. + _MAPPING = {} + + def __init__(self): + if not self._MAPPING: + for letter in (chr(l) for l in xrange(ord('C'), ord('Z')+1)): + try: + letter = '%s:' % letter + mapped = QueryDosDevice(letter) + # It can happen. Assert until we see it happens in the wild. In + # practice, prefer the lower drive letter. + assert mapped not in self._MAPPING + if mapped not in self._MAPPING: + self._MAPPING[mapped] = letter + except WindowsError: # pylint: disable=E0602 + pass + + def to_dos(self, path): + """Converts a native NT path to DOS path.""" + m = re.match(r'(^\\Device\\[a-zA-Z0-9]+)(\\.*)?$', path) + if not m or m.group(1) not in self._MAPPING: + assert False, path + drive = self._MAPPING[m.group(1)] + if not m.group(2): + return drive + return drive + m.group(2) + + def get_flavor(): """Returns the system default flavor. Copied from gyp/pylib/gyp/common.py.""" flavors = { @@ -623,6 +703,352 @@ class Dtrace(object): logfile.write(''.join(lines)) +class LogmanTrace(object): + """Uses the native Windows ETW based tracing functionality to trace a child + process. + """ + class _Context(object): + """Processes a ETW log line and keeps the list of existent and non + existent files accessed. + + Ignores directories. + """ + + EVENT_NAME = 0 + TYPE = 1 + PID = 9 + CHILD_PID = 20 + PARENT_PID = 21 + FILE_PATH = 25 + PROC_NAME = 26 + CMD_LINE = 27 + + def __init__(self, blacklist): + self.blacklist = blacklist + self.files = set() + self.non_existent = set() + + self._processes = set() + self._drive_map = DosDriveMap() + self._first_line = False + + def on_csv_line(self, line): + """Processes a CSV Event line.""" + # So much white space! + line = [i.strip() for i in line] + if not self._first_line: + assert line == [ + u'Event Name', + u'Type', + u'Event ID', + u'Version', + u'Channel', + u'Level', # 5 + u'Opcode', + u'Task', + u'Keyword', + u'PID', + u'TID', # 10 + u'Processor Number', + u'Instance ID', + u'Parent Instance ID', + u'Activity ID', + u'Related Activity ID', # 15 + u'Clock-Time', + u'Kernel(ms)', + u'User(ms)', + u'User Data', + ] + self._first_line = True + return + + # As you can see, the CSV is full of useful non-redundant information: + # Event ID + assert line[2] == '0' + # Version + assert line[3] in ('2', '3'), line[3] + # Channel + assert line[4] == '0' + # Level + assert line[5] == '0' + # Task + assert line[7] == '0' + # Keyword + assert line[8] == '0x0000000000000000' + # Instance ID + assert line[12] == '' + # Parent Instance ID + assert line[13] == '' + # Activity ID + assert line[14] == '{00000000-0000-0000-0000-000000000000}' + # Related Activity ID + assert line[15] == '' + + if line[0].startswith('{'): + # Skip GUIDs. + return + + # Convert the PID in-place from hex. + line[self.PID] = int(line[self.PID], 16) + + # By Opcode + handler = getattr( + self, + 'handle_%s_%s' % (line[self.EVENT_NAME], line[self.TYPE]), + None) + if not handler: + # Try to get an universal fallback + handler = getattr(self, 'handle_%s_Any' % line[self.EVENT_NAME], None) + if handler: + handler(line) + else: + assert False, '%s_%s' % (line[self.EVENT_NAME], line[self.TYPE]) + + def handle_EventTrace_Any(self, line): + pass + + def handle_FileIo_Create(self, line): + m = re.match(r'^\"(.+)\"$', line[self.FILE_PATH]) + self._handle_file(self._drive_map.to_dos(m.group(1)).lower()) + + def handle_FileIo_Rename(self, line): + # TODO(maruel): Handle? + pass + + def handle_FileIo_Any(self, line): + pass + + def handle_Image_DCStart(self, line): + # TODO(maruel): Handle? + pass + + def handle_Image_Load(self, line): + # TODO(maruel): Handle? + pass + + def handle_Image_Any(self, line): + # TODO(maruel): Handle? + pass + + def handle_Process_Any(self, line): + pass + + def handle_Process_DCStart(self, line): + """Gives historic information about the process tree. + + Use it to extract the pid of the trace_inputs.py parent process that + started logman.exe. + """ + ppid = int(line[self.PARENT_PID], 16) + if line[self.PROC_NAME] == '"logman.exe"': + # logman's parent is us. + self._processes.add(ppid) + logging.info('Found logman\'s parent at %d' % ppid) + + def handle_Process_End(self, line): + # Look if it is logman terminating, if so, grab the parent's process pid + # and inject cwd. + if line[self.PID] in self._processes: + logging.info('Terminated: %d' % line[self.PID]) + self._processes.remove(line[self.PID]) + + def handle_Process_Start(self, line): + """Handles a new child process started by PID.""" + ppid = line[self.PID] + pid = int(line[self.CHILD_PID], 16) + if ppid in self._processes: + if line[self.PROC_NAME] == '"logman.exe"': + # Skip the shutdown call. + return + self._processes.add(pid) + logging.info( + 'New child: %d -> %d %s' % (ppid, pid, line[self.PROC_NAME])) + + def handle_SystemConfig_Any(self, line): + pass + + def _handle_file(self, filename): + """Handles a file that was touched. + + Interestingly enough, the file is always with an absolute path. + """ + if (self.blacklist(filename) or + os.path.isdir(filename) or + filename in self.files or + filename in self.non_existent): + return + logging.debug('_handle_file(%s)' % filename) + if os.path.isfile(filename): + self.files.add(filename) + else: + self.non_existent.add(filename) + + def __init__(self): + # Most ignores need to be determined at runtime. + self.IGNORED = set([os.path.dirname(sys.executable).lower()]) + # Add many directories from environment variables. + vars_to_ignore = ( + 'APPDATA', + 'LOCALAPPDATA', + 'ProgramData', + 'ProgramFiles', + 'ProgramFiles(x86)', + 'ProgramW6432', + 'SystemRoot', + 'TEMP', + 'TMP', + ) + for i in vars_to_ignore: + if os.environ.get(i): + self.IGNORED.add(os.environ[i].lower()) + + # Also add their short path name equivalents. + for i in list(self.IGNORED): + self.IGNORED.add(GetShortPathName(i).lower()) + + # Add this one last since it has no short path name equivalent. + self.IGNORED.add('\\systemroot') + self.IGNORED = tuple(sorted(self.IGNORED)) + + @classmethod + def gen_trace(cls, cmd, cwd, logname): + logging.info('gen_trace(%s, %s, %s)' % (cmd, cwd, logname)) + # Use "logman -?" for help. + + etl = logname + '.etl' + + silent = not isEnabledFor(logging.INFO) + stdout = stderr = None + if silent: + stdout = subprocess.PIPE + stderr = subprocess.PIPE + + # 1. Start the log collection. Requires administrative access. logman.exe is + # synchronous so no need for a "warmup" call. + # 'Windows Kernel Trace' is *localized* so use its GUID instead. + # The GUID constant name is SystemTraceControlGuid. Lovely. + cmd_start = [ + 'logman.exe', + 'start', + 'NT Kernel Logger', + '-p', '{9e814aad-3204-11d2-9a82-006008a86939}', + '(process,img,file,fileio)', + '-o', etl, + '-ets', # Send directly to kernel + ] + logging.debug('Running: %s' % cmd_start) + subprocess.check_call(cmd_start, stdout=stdout, stderr=stderr) + + try: + # 2. Run the child process. + logging.debug('Running: %s' % cmd) + result = subprocess.call(cmd, cwd=cwd, stdout=stdout, stderr=stderr) + finally: + # 3. Stop the log collection. + cmd_stop = [ + 'logman.exe', + 'stop', + 'NT Kernel Logger', + '-ets', # Send directly to kernel + ] + logging.debug('Running: %s' % cmd_stop) + subprocess.check_call(cmd_stop, stdout=stdout, stderr=stderr) + + # 4. Convert the traces to text representation. + # Use "tracerpt -?" for help. + LOCALE_INVARIANT = 0x7F + windll.kernel32.SetThreadLocale(LOCALE_INVARIANT) + cmd_convert = [ + 'tracerpt.exe', + '-l', etl, + '-o', logname, + '-gmt', # Use UTC + '-y', # No prompt + ] + + # Normally, 'csv' is sufficient. If complex scripts are used (like eastern + # languages), use 'csv_unicode'. If localization gets in the way, use 'xml'. + logformat = 'csv' + + if logformat == 'csv': + # tracerpt localizes the 'Type' column, for major brainfuck + # entertainment. I can't imagine any sane reason to do that. + cmd_convert.extend(['-of', 'CSV']) + elif logformat == 'csv_utf16': + # This causes it to use UTF-16, which doubles the log size but ensures the + # log is readable for non-ASCII characters. + cmd_convert.extend(['-of', 'CSV', '-en', 'Unicode']) + elif logformat == 'xml': + cmd_convert.extend(['-of', 'XML']) + else: + assert False, logformat + logging.debug('Running: %s' % cmd_convert) + subprocess.check_call(cmd_convert, stdout=stdout, stderr=stderr) + return result + + @classmethod + def parse_log(cls, filename, blacklist): + logging.info('parse_log(%s, %s)' % (filename, blacklist)) + + # Auto-detect the log format + with open(filename, 'rb') as f: + hdr = f.read(2) + assert len(hdr) == 2 + if hdr == '<E': + # It starts with <Events> + logformat = 'xml' + elif hdr == '\xFF\xEF': + # utf-16 BOM. + logformat = 'csv_utf16' + else: + logformat = 'csv' + + context = cls._Context(blacklist) + + if logformat == 'csv_utf16': + def utf_8_encoder(unicode_csv_data): + """Encodes the unicode object as utf-8 encoded str instance""" + for line in unicode_csv_data: + yield line.encode('utf-8') + + def unicode_csv_reader(unicode_csv_data, **kwargs): + """Encodes temporarily as UTF-8 since csv module doesn't do unicode.""" + csv_reader = csv.reader(utf_8_encoder(unicode_csv_data), **kwargs) + for row in csv_reader: + # Decode str utf-8 instances back to unicode instances, cell by cell: + yield [cell.decode('utf-8') for cell in row] + + # The CSV file is UTF-16 so use codecs.open() to load the file into the + # python internal unicode format (utf-8). Then explicitly re-encode as + # utf8 as str instances so csv can parse it fine. Then decode the utf-8 + # str back into python unicode instances. This sounds about right. + for line in unicode_csv_reader(codecs.open(filename, 'r', 'utf-16')): + # line is a list of unicode objects + context.on_csv_line(line) + + elif logformat == 'csv': + def ansi_csv_reader(ansi_csv_data, **kwargs): + """Loads an 'ANSI' code page and returns unicode() objects.""" + assert sys.getfilesystemencoding() == 'mbcs' + encoding = get_current_encoding() + for row in csv.reader(ansi_csv_data, **kwargs): + # Decode str 'ansi' instances to unicode instances, cell by cell: + yield [cell.decode(encoding) for cell in row] + + # The fastest and smallest format but only supports 'ANSI' file paths. + # E.g. the filenames are encoding in the 'current' encoding. + for line in ansi_csv_reader(open(filename)): + # line is a list of unicode objects + context.on_csv_line(line) + + else: + raise NotImplementedError('Implement %s' % logformat) + + return ( + set(os.path.realpath(f) for f in context.files), + set(os.path.realpath(f) for f in context.non_existent)) + + def relevant_files(files, root): """Trims the list of files to keep the expected files and unexpected files. @@ -654,7 +1080,7 @@ def extract_directories(files, root): ) if not (actual - files): files -= actual - files.add(directory + '/') + files.add(directory + os.path.sep) return sorted(files) @@ -744,6 +1170,16 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace): # Resolve any symlink root_dir = os.path.realpath(root_dir) + if sys.platform == 'win32': + # Help ourself and lowercase all the paths. + # TODO(maruel): handle short path names by converting them to long path name + # as needed. + root_dir = root_dir.lower() + if cwd_dir: + cwd_dir = cwd_dir.lower() + if product_dir: + product_dir = product_dir.lower() + def print_if(txt): if cwd_dir is None: print(txt) @@ -753,6 +1189,8 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace): api = Strace() elif flavor == 'mac': api = Dtrace() + elif sys.platform == 'win32': + api = LogmanTrace() else: print >> sys.stderr, 'Unsupported platform %s' % sys.platform return 1 @@ -781,7 +1219,8 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace): for f in non_existent: print_if(' %s' % f) - expected, unexpected = relevant_files(files, root_dir.rstrip('/') + '/') + expected, unexpected = relevant_files( + files, root_dir.rstrip(os.path.sep) + os.path.sep) if unexpected: print_if('Unexpected: %d' % len(unexpected)) for f in unexpected: @@ -794,9 +1233,10 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace): if cwd_dir is not None: def cleanuppath(x): - """Cleans up a relative path.""" + """Cleans up a relative path. Converts any os.path.sep to '/' on Windows. + """ if x: - x = x.rstrip('/') + x = x.rstrip(os.path.sep).replace(os.path.sep, '/') if x == '.': x = '' if x: @@ -810,6 +1250,10 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace): def fix(f): """Bases the file on the most restrictive variable.""" logging.debug('fix(%s)' % f) + # Important, GYP stores the files with / and not \. + if sys.platform == 'win32': + f = f.replace('\\', '/') + if product_dir and f.startswith(product_dir): return '<(PRODUCT_DIR)/%s' % f[len(product_dir):] elif cwd_dir and f.startswith(cwd_dir): @@ -858,7 +1302,10 @@ def main(): '--root-dir', default=ROOT_DIR, help='Root directory to base everything off. Default: %default') parser.add_option( - '-f', '--force', action='store_true', help='Force to retrace the file') + '-f', '--force', + action='store_true', + default=False, + help='Force to retrace the file') options, args = parser.parse_args() level = [logging.ERROR, logging.INFO, logging.DEBUG][min(2, options.verbose)] diff --git a/tools/isolate/trace_inputs_smoke_test.py b/tools/isolate/trace_inputs_smoke_test.py index 13bb1e9..be55ad2 100755 --- a/tools/isolate/trace_inputs_smoke_test.py +++ b/tools/isolate/trace_inputs_smoke_test.py @@ -41,7 +41,7 @@ class TraceInputs(unittest.TestCase): def _execute(self, is_gyp): cmd = [ - sys.executable, os.path.join('..', 'trace_inputs.py'), + sys.executable, os.path.join('..', '..', 'trace_inputs.py'), '--log', self.log, '--root-dir', ROOT_DIR, ] @@ -51,48 +51,52 @@ class TraceInputs(unittest.TestCase): '--cwd', 'data', '--product', '.', # Not tested. ]) - cmd.append(os.path.join('..', FILENAME)) + cmd.append(sys.executable) if is_gyp: + # When the gyp argument is specified, the command is started from --cwd + # directory. In this case, 'data'. + cmd.extend([os.path.join('..', FILENAME), '--child-gyp']) + else: # When the gyp argument is not specified, the command is started from # --root-dir directory. - cmd.append('--child-gyp') - else: - # When the gyp argument is specified, the command is started from --cwd - # directory. - cmd.append('--child') + cmd.extend([FILENAME, '--child']) - cwd = 'data' + # The current directory doesn't matter, the traced process will be called + # from the correct cwd. + cwd = os.path.join('data', 'trace_inputs') + # Ignore stderr. + logging.info('Command: %s' % ' '.join(cmd)) p = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) - out = p.communicate()[0] + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, + universal_newlines=True) + out, err = p.communicate() if p.returncode: - raise CalledProcessError(p.returncode, cmd, out, cwd) - return out + raise CalledProcessError(p.returncode, cmd, out + err, cwd) + return out or '' def test_trace(self): - if sys.platform not in ('linux2', 'darwin'): - print 'WARNING: unsupported: %s' % sys.platform - return expected_end = [ - "Interesting: 4 reduced to 3", - " data/trace_inputs/", - " trace_inputs.py", - " %s" % FILENAME, + 'Interesting: 4 reduced to 3', + ' data/trace_inputs/'.replace('/', os.path.sep), + ' trace_inputs.py', + ' %s' % FILENAME, ] actual = self._execute(False).splitlines() - self.assertTrue(actual[0].startswith('Tracing... [')) - self.assertTrue(actual[1].startswith('Loading traces... ')) - self.assertTrue(actual[2].startswith('Total: ')) - self.assertEquals("Non existent: 0", actual[3]) + self.assertTrue(actual[0].startswith('Tracing... ['), actual) + self.assertTrue(actual[1].startswith('Loading traces... '), actual) + self.assertTrue(actual[2].startswith('Total: '), actual) + if sys.platform == 'win32': + # On windows, python searches the current path for python stdlib like + # subprocess.py and others, I'm not sure why. + self.assertTrue(actual[3].startswith('Non existent: '), actual[3]) + else: + self.assertEquals('Non existent: 0', actual[3]) # Ignore any Unexpected part. # TODO(maruel): Make sure there is no Unexpected part, even in the case of # virtualenv usage. self.assertEquals(expected_end, actual[-len(expected_end):]) def test_trace_gyp(self): - if sys.platform not in ('linux2', 'darwin'): - print 'WARNING: unsupported: %s' % sys.platform - return import trace_inputs expected_value = { 'conditions': [ @@ -120,7 +124,7 @@ def child(): """When the gyp argument is not specified, the command is started from --root-dir directory. """ - print 'child' + print 'child from %s' % os.getcwd() # Force file opening with a non-normalized path. open(os.path.join('data', '..', 'trace_inputs.py'), 'rb').close() # Do not wait for the child to exit. @@ -134,7 +138,7 @@ def child_gyp(): """When the gyp argument is specified, the command is started from --cwd directory. """ - print 'child_gyp' + print 'child_gyp from %s' % os.getcwd() # Force file opening. open(os.path.join('..', 'trace_inputs.py'), 'rb').close() # Do not wait for the child to exit. |