#!/usr/bin/env python # Copyright (c) 2011 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Usage: This tool will take a PDB on the command line, extract the source files that were used in building the PDB, query the source server for which repository and revision these files are at, and then finally write this information back into the PDB in a format that the debugging tools understand. This allows for automatic source debugging, as all of the information is contained in the PDB, and the debugger can go out and fetch the source files. You most likely want to run these immediately after a build, since the source input files need to match the generated PDB, and we want the correct revision information for the exact files that were used for the build. The following files from a windbg + source server installation are expected to reside in the same directory as this python script: dbghelp.dll pdbstr.exe srctool.exe NOTE: Expected to run under a native win32 python, NOT cygwin. All paths are dealt with as win32 paths, since we have to interact with the Microsoft tools. """ import os import optparse import sys import tempfile import time import subprocess import win32api from collections import namedtuple # This serves two purposes. First, it acts as a whitelist, and only files # from repositories listed here will be source indexed. Second, it allows us # to map from one URL to another, so we can map to external source servers. It # also indicates if the source for this project will be retrieved in a base64 # encoded format. # TODO(sebmarchand): Initialize this variable in the main function and pass it # to the sub functions instead of having a global variable. REPO_MAP = { 'http://src.chromium.org/svn': { 'url': 'https://src.chromium.org/chrome/' '{file_path}?revision={revision}', 'base64': False }, 'https://src.chromium.org/svn': { 'url': 'https://src.chromium.org/chrome/' '{file_path}?revision={revision}', 'base64': False } } PROJECT_GROUPS = [ # Googlecode SVN projects { 'projects': [ 'angleproject', 'google-breakpad', 'google-cache-invalidation-api', 'google-url', 'googletest', 'leveldb', 'libphonenumber', 'libyuv', 'open-vcdiff', 'ots', 'sawbuck', 'sfntly', 'smhasher', 'v8', 'v8-i18n', 'webrtc', ], 'public_url': 'https://%s.googlecode.com/svn-history/' \ 'r{revision}/{file_path}', 'svn_urls': [ 'svn://svn-mirror.golo.chromium.org/%s', 'http://src.chromium.org/%s', 'https://src.chromium.org/%s', 'http://%s.googlecode.com/svn', 'https://%s.googlecode.com/svn', ], }, # Googlecode Git projects { 'projects': [ 'syzygy', ], 'public_url': 'https://%s.googlecode.com/git-history/' \ '{revision}/{file_path}', 'svn_urls': [ 'https://code.google.com/p/%s/', ], }, # Chrome projects { 'projects': [ 'blink', 'chrome', 'multivm', 'native_client', ], 'public_url': 'https://src.chromium.org/%s/' \ '{file_path}?revision={revision}', 'svn_urls': [ 'svn://chrome-svn/%s', 'svn://chrome-svn.corp.google.com/%s', 'svn://svn-mirror.golo.chromium.org/%s', 'svn://svn.chromium.org/%s', ], }, ] # A named tuple used to store the information about a repository. # # It contains the following members: # - repo: The URL of the repository; # - rev: The revision (or hash) of the current checkout. # - file_list: The list of files coming from this repository. # - root_path: The root path of this checkout. # - path_prefix: A prefix to apply to the filename of the files coming from # this repository. RevisionInfo = namedtuple('RevisionInfo', ['repo', 'rev', 'files', 'root_path', 'path_prefix']) def GetCasedFilePath(filename): """Return the correctly cased path for a given filename""" return win32api.GetLongPathName(win32api.GetShortPathName(unicode(filename))) def FillRepositoriesMap(): """ Fill the repositories map with the whitelisted projects. """ for project_group in PROJECT_GROUPS: for project in project_group['projects']: for svn_url in project_group['svn_urls']: REPO_MAP[svn_url % project] = { 'url': project_group['public_url'] % project, 'base64': False } REPO_MAP[project_group['public_url'] % project] = None FillRepositoriesMap() def FindFile(filename): """Return the full windows path to a file in the same dir as this code.""" thisdir = os.path.dirname(os.path.join(os.path.curdir, __file__)) return os.path.abspath(os.path.join(thisdir, filename)) def RunCommand(*cmd, **kwargs): """Runs a command. Returns what have been printed to stdout by this command. kwargs: raise_on_failure: Indicates if an exception should be raised on failure, if set to false then the function will return None. """ kwargs.setdefault('stdin', subprocess.PIPE) kwargs.setdefault('stdout', subprocess.PIPE) kwargs.setdefault('stderr', subprocess.PIPE) kwargs.setdefault('universal_newlines', True) raise_on_failure = kwargs.pop('raise_on_failure', True) proc = subprocess.Popen(cmd, **kwargs) ret, err = proc.communicate() if proc.returncode != 0: if raise_on_failure: print 'Error: %s' % err raise subprocess.CalledProcessError(proc.returncode, cmd) return ret = (ret or '').rstrip('\n') return ret def ExtractSourceFiles(pdb_filename): """Extract a list of local paths of the source files from a PDB.""" src_files = RunCommand(FindFile('srctool.exe'), '-r', pdb_filename) if not src_files or src_files.startswith("srctool: "): raise Exception("srctool failed: " + src_files) return set(x.lower() for x in src_files.split('\n') if len(x) != 0) def ReadSourceStream(pdb_filename): """Read the contents of the source information stream from a PDB.""" srctool = subprocess.Popen([FindFile('pdbstr.exe'), '-r', '-s:srcsrv', '-p:%s' % pdb_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) data, _ = srctool.communicate() if ((srctool.returncode != 0 and srctool.returncode != -1) or data.startswith("pdbstr: ")): raise Exception("pdbstr failed: " + data) return data def WriteSourceStream(pdb_filename, data): """Write the contents of the source information stream to a PDB.""" # Write out the data to a temporary filename that we can pass to pdbstr. (f, fname) = tempfile.mkstemp() f = os.fdopen(f, "wb") f.write(data) f.close() srctool = subprocess.Popen([FindFile('pdbstr.exe'), '-w', '-s:srcsrv', '-i:%s' % fname, '-p:%s' % pdb_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) data, _ = srctool.communicate() if ((srctool.returncode != 0 and srctool.returncode != -1) or data.startswith("pdbstr: ")): raise Exception("pdbstr failed: " + data) os.unlink(fname) def GetSVNRepoInfo(local_path): """Calls svn info to extract the SVN information about a path.""" # We call svn.bat to make sure and get the depot tools SVN and not cygwin. info = RunCommand('svn.bat', 'info', local_path, raise_on_failure=False) if not info: return # Hack up into a dictionary of the fields printed by svn info. vals = dict((y.split(': ', 2) for y in info.split('\n') if y)) return vals def ExtractSVNInfo(local_filename): """Checks if a file is coming from a svn repository and if so returns some information about it. Args: local_filename: The name of the file that we want to check. Returns: None if the file doesn't come from a svn repository, otherwise it returns a RevisionInfo tuple. """ # Try to get the svn information about this file. vals = GetSVNRepoInfo(local_filename) if not vals: return repo = vals['Repository Root'] if not vals['URL'].startswith(repo): raise Exception("URL is not inside of the repository root?!?") rev = vals['Revision'] svn_local_root = os.path.split(local_filename)[0] # We need to look at the SVN URL of the current path to handle the case when # we do a partial SVN checkout inside another checkout of the same repository. # This happens in Chromium where we do some checkout of # '/trunk/deps/third_party' in 'src/third_party'. svn_root_url = os.path.dirname(vals['URL']) # Don't try to list all the files from this repository as this seem to slow # down the indexing, instead index one file at a time. file_list = [local_filename.replace(svn_local_root, '').lstrip(os.path.sep)] return RevisionInfo(repo=repo, rev=rev, files=file_list, root_path=svn_local_root, path_prefix=svn_root_url.replace(repo, '')) def ExtractGitInfo(local_filename): """Checks if a file is coming from a git repository and if so returns some information about it. Args: local_filename: The name of the file that we want to check. Returns: None if the file doesn't come from a git repository, otherwise it returns a RevisionInfo tuple. """ # Starts by checking if this file is coming from a git repository. For that # we'll start by calling 'git info' on this file; for this to work we need to # make sure that the current working directory is correctly cased. It turns # out that even on Windows the casing of the path passed in the |cwd| argument # of subprocess.Popen matters and if it's not correctly cased then 'git info' # will return None even if the file is coming from a git repository. This # is not the case if we're just interested in checking if the path containing # |local_filename| is coming from a git repository, in this case the casing # doesn't matter. local_filename = GetCasedFilePath(local_filename) local_file_basename = os.path.basename(local_filename) local_file_dir = os.path.dirname(local_filename) file_info = RunCommand('git.bat', 'log', '-n', '1', local_file_basename, cwd=local_file_dir, raise_on_failure=False) if not file_info: return # Get the revision of the master branch. rev = RunCommand('git.bat', 'rev-parse', 'HEAD', cwd=local_file_dir) # Get the url of the remote repository. repo = RunCommand('git.bat', 'config', '--get', 'remote.origin.url', cwd=local_file_dir) # If the repository point to a local directory then we need to run this # command one more time from this directory to get the repository url. if os.path.isdir(repo): repo = RunCommand('git.bat', 'config', '--get', 'remote.origin.url', cwd=repo) # Don't use the authenticated path. repo = repo.replace('googlesource.com/a/', 'googlesource.com/') # Get the relative file path for this file in the git repository. git_path = RunCommand('git.bat', 'ls-tree', '--full-name', '--name-only', 'HEAD', local_file_basename, cwd=local_file_dir).replace('/','\\') if not git_path: return git_root_path = local_filename.replace(git_path, '') if repo not in REPO_MAP: # Automatically adds the project coming from a git GoogleCode repository to # the repository map. The files from these repositories are accessible via # gitiles in a base64 encoded format. if 'chromium.googlesource.com' in repo: REPO_MAP[repo] = { 'url': '%s/+/{revision}/{file_path}?format=TEXT' % repo, 'base64': True } # Get the list of files coming from this repository. git_file_list = RunCommand('git.bat', 'ls-tree', '--full-name', '--name-only', 'HEAD', '-r', cwd=git_root_path) file_list = [x for x in git_file_list.splitlines() if len(x) != 0] return RevisionInfo(repo=repo, rev=rev, files=file_list, root_path=git_root_path, path_prefix=None) def IndexFilesFromRepo(local_filename, file_list, output_lines): """Checks if a given file is a part of a revision control repository (svn or git) and index all the files from this repository if it's the case. Args: local_filename: The filename of the current file. file_list: The list of files that should be indexed. output_lines: The source indexing lines that will be appended to the PDB. Returns the number of indexed files. """ indexed_files = 0 # Try to extract the revision info for the current file. info = ExtractGitInfo(local_filename) if not info: info = ExtractSVNInfo(local_filename) repo = info.repo rev = info.rev files = info.files root_path = info.root_path.lower() # Checks if we should index this file and if the source that we'll retrieve # will be base64 encoded. should_index = False base_64 = False if repo in REPO_MAP: should_index = True base_64 = REPO_MAP[repo].get('base64') else: repo = None # Iterates over the files from this repo and index them if needed. for file_iter in files: current_filename = file_iter.lower() full_file_path = os.path.normpath(os.path.join(root_path, current_filename)) # Checks if the file is in the list of files to be indexed. if full_file_path in file_list: if should_index: source_url = '' current_file = file_iter # Prefix the filename with the prefix for this repository if needed. if info.path_prefix: current_file = os.path.join(info.path_prefix, current_file) source_url = REPO_MAP[repo].get('url').format(revision=rev, file_path=os.path.normpath(current_file).replace('\\', '/')) output_lines.append('%s*%s*%s*%s*%s' % (full_file_path, current_file, rev, source_url, 'base64.b64decode' if base_64 else '')) indexed_files += 1 file_list.remove(full_file_path) # The input file should have been removed from the list of files to index. if indexed_files and local_filename in file_list: print '%s shouldn\'t be in the list of files to index anymore.' % \ local_filename # TODO(sebmarchand): Turn this into an exception once I've confirmed that # this doesn't happen on the official builder. file_list.remove(local_filename) return indexed_files def DirectoryIsUnderPublicVersionControl(local_dir): # Checks if this directory is from a Git checkout. info = RunCommand('git.bat', 'config', '--get', 'remote.origin.url', cwd=local_dir, raise_on_failure=False) if info: return True # If not checks if it's from a SVN checkout. info = GetSVNRepoInfo(local_dir) if info: return True return False def UpdatePDB(pdb_filename, verbose=True, build_dir=None, toolchain_dir=None): """Update a pdb file with source information.""" dir_blacklist = { } if build_dir: # Blacklisting the build directory allows skipping the generated files, for # Chromium this makes the indexing ~10x faster. build_dir = (os.path.normpath(build_dir)).lower() for directory, _, _ in os.walk(build_dir): dir_blacklist[directory.lower()] = True dir_blacklist[build_dir.lower()] = True if toolchain_dir: # Blacklisting the directories from the toolchain as we don't have revision # info for them. toolchain_dir = (os.path.normpath(toolchain_dir)).lower() for directory, _, _ in os.walk(build_dir): dir_blacklist[directory.lower()] = True dir_blacklist[toolchain_dir.lower()] = True # Writes the header of the source index stream. # # Here's the description of the variables used in the SRC_* macros (those # variables have to be defined for every source file that we want to index): # var1: The file path. # var2: The name of the file without its path. # var3: The revision or the hash of this file's repository. # var4: The URL to this file. # var5: (optional) The python method to call to decode this file, e.g. for # a base64 encoded file this value should be 'base64.b64decode'. lines = [ 'SRCSRV: ini ------------------------------------------------', 'VERSION=1', 'INDEXVERSION=2', 'VERCTRL=Subversion', 'DATETIME=%s' % time.asctime(), 'SRCSRV: variables ------------------------------------------', 'SRC_EXTRACT_TARGET_DIR=%targ%\%fnbksl%(%var2%)\%var3%', 'SRC_EXTRACT_TARGET=%SRC_EXTRACT_TARGET_DIR%\%fnfile%(%var1%)', 'SRC_EXTRACT_CMD=cmd /c "mkdir "%SRC_EXTRACT_TARGET_DIR%" & python -c ' '"import urllib2, base64;' 'url = \\\"%var4%\\\";' 'u = urllib2.urlopen(url);' 'print %var5%(u.read());" > "%SRC_EXTRACT_TARGET%""', 'SRCSRVTRG=%SRC_EXTRACT_TARGET%', 'SRCSRVCMD=%SRC_EXTRACT_CMD%', 'SRCSRV: source files ---------------------------------------', ] if ReadSourceStream(pdb_filename): raise Exception("PDB already has source indexing information!") filelist = ExtractSourceFiles(pdb_filename) number_of_files = len(filelist) indexed_files_total = 0 while filelist: filename = next(iter(filelist)) filedir = os.path.dirname(filename) if verbose: print "[%d / %d] Processing: %s" % (number_of_files - len(filelist), number_of_files, filename) # This directory is blacklisted, either because it's not part of a # repository, or from one we're not interested in indexing. if dir_blacklist.get(filedir, False): if verbose: print " skipping, directory is blacklisted." filelist.remove(filename) continue # Skip the files that don't exist on the current machine. if not os.path.exists(filename): filelist.remove(filename) continue # Try to index the current file and all the ones coming from the same # repository. indexed_files = IndexFilesFromRepo(filename, filelist, lines) if not indexed_files: if not DirectoryIsUnderPublicVersionControl(filedir): dir_blacklist[filedir] = True if verbose: print "Adding %s to the blacklist." % filedir filelist.remove(filename) continue indexed_files_total += indexed_files if verbose: print " %d files have been indexed." % indexed_files lines.append('SRCSRV: end ------------------------------------------------') WriteSourceStream(pdb_filename, '\r\n'.join(lines)) if verbose: print "%d / %d files have been indexed." % (indexed_files_total, number_of_files) def main(): parser = optparse.OptionParser() parser.add_option('-v', '--verbose', action='store_true', default=False) parser.add_option('--build-dir', help='The original build directory, if set ' 'all the files present in this directory (or one of its subdirectories) ' 'will be skipped.') parser.add_option('--toolchain-dir', help='The directory containing the ' 'toolchain that has been used for this build. If set all the files ' 'present in this directory (or one of its subdirectories) will be ' 'skipped.') options, args = parser.parse_args() if not args: parser.error('Specify a pdb') for pdb in args: UpdatePDB(pdb, options.verbose, options.build_dir) return 0 if __name__ == '__main__': sys.exit(main())