#!/usr/bin/env python # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import argparse import collections import logging import os import re import subprocess import sys import time SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir)) sys.path.insert(0, os.path.join(SRC_DIR, 'build')) import find_depot_tools find_depot_tools.add_depot_tools_to_path() import rietveld import roll_dep_svn from gclient import GClientKeywords from third_party import upload # Avoid depot_tools/third_party/upload.py print verbose messages. upload.verbosity = 0 # Errors only. CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git' COMMIT_POSITION_RE = re.compile('^Cr-Original-Commit-Position: .*#([0-9]+).*$') CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$') RIETVELD_URL_RE = re.compile('^https?://(.*)/(.*)') ROLL_BRANCH_NAME = 'special_webrtc_roll_branch' TRYJOB_STATUS_SLEEP_SECONDS = 30 # Use a shell for subcommands on Windows to get a PATH search. IS_WIN = sys.platform.startswith('win') WEBRTC_PATH = os.path.join('third_party', 'webrtc') LIBJINGLE_PATH = os.path.join('third_party', 'libjingle', 'source', 'talk') LIBJINGLE_README = os.path.join('third_party', 'libjingle', 'README.chromium') # Result codes from build/third_party/buildbot_8_4p1/buildbot/status/results.py # plus the -1 code which is used when there's no result yet. TRYJOB_STATUS = { -1: 'RUNNING', 0: 'SUCCESS', 1: 'WARNINGS', 2: 'FAILURE', 3: 'SKIPPED', 4: 'EXCEPTION', 5: 'RETRY', } SUCCESS_STATUS = (0, 1, 3) FAILURE_STATUS = (2, 4, 5) CommitInfo = collections.namedtuple('CommitInfo', ['commit_position', 'git_commit', 'git_repo_url']) CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'rietveld_server']) def _PosixPath(path): """Convert a possibly-Windows path to a posix-style path.""" (_, path) = os.path.splitdrive(path) return path.replace(os.sep, '/') def _ParseGitCommitPosition(description): for line in reversed(description.splitlines()): m = COMMIT_POSITION_RE.match(line.strip()) if m: return m.group(1) logging.error('Failed to parse svn revision id from:\n%s\n', description) sys.exit(-1) def _ParseGitCommitHash(description): for line in description.splitlines(): if line.startswith('commit '): return line.split()[1] logging.error('Failed to parse git commit id from:\n%s\n', description) sys.exit(-1) return None def _ParseDepsFile(filename): with open(filename, 'rb') as f: deps_content = f.read() return _ParseDepsDict(deps_content) def _ParseDepsDict(deps_content): local_scope = {} var = GClientKeywords.VarImpl({}, local_scope) global_scope = { 'File': GClientKeywords.FileImpl, 'From': GClientKeywords.FromImpl, 'Var': var.Lookup, 'deps_os': {}, } exec(deps_content, global_scope, local_scope) return local_scope def _WaitForTrybots(issue, rietveld_server): """Wait until all trybots have passed or at least one have failed. Returns: An exit code of 0 if all trybots passed or non-zero otherwise. """ assert type(issue) is int print 'Trybot status for https://%s/%d:' % (rietveld_server, issue) remote = rietveld.Rietveld('https://' + rietveld_server, None, None) attempt = 0 max_tries = 60*60/TRYJOB_STATUS_SLEEP_SECONDS # Max one hour while attempt < max_tries: # Get patches for the issue so we can use the latest one. data = remote.get_issue_properties(issue, messages=False) patchsets = data['patchsets'] # Get trybot status for the latest patch set. data = remote.get_patchset_properties(issue, patchsets[-1]) tryjob_results = data['try_job_results'] if len(tryjob_results) == 0: logging.debug('No trybots have yet been triggered for https://%s/%d' , rietveld_server, issue) else: _PrintTrybotsStatus(tryjob_results) if any(r['result'] in FAILURE_STATUS for r in tryjob_results): logging.error('Found failing tryjobs (see above)') return 1 if all(r['result'] in SUCCESS_STATUS for r in tryjob_results): return 0 logging.debug('Waiting for %d seconds before next check...', TRYJOB_STATUS_SLEEP_SECONDS) time.sleep(TRYJOB_STATUS_SLEEP_SECONDS) attempt += 1 def _PrintTrybotsStatus(tryjob_results): status_to_name = {} for trybot_result in tryjob_results: status = TRYJOB_STATUS.get(trybot_result['result'], 'UNKNOWN') status_to_name.setdefault(status, []) status_to_name[status].append(trybot_result['builder']) print '\n========== TRYJOBS STATUS ==========' for status,name_list in status_to_name.iteritems(): print '%s: %s' % (status, ','.join(sorted(name_list))) def _GenerateCLDescriptionCommand(webrtc_current, libjingle_current, webrtc_new, libjingle_new): delim = '' webrtc_str = '' def GetChangeLogURL(git_repo_url, current_hash, new_hash): return '%s/+log/%s..%s' % (git_repo_url, current_hash[0:7], new_hash[0:7]) if webrtc_current.git_commit != webrtc_new.git_commit: webrtc_str = 'WebRTC %s:%s' % (webrtc_current.commit_position, webrtc_new.commit_position) webrtc_changelog_url = GetChangeLogURL(webrtc_current.git_repo_url, webrtc_current.git_commit, webrtc_new.git_commit) libjingle_str = '' if libjingle_current.git_commit != libjingle_new.git_commit: if webrtc_str: delim += ', ' libjingle_str = 'Libjingle %s:%s' % (libjingle_current.commit_position, libjingle_new.commit_position) libjingle_changelog_url = GetChangeLogURL(libjingle_current.git_repo_url, libjingle_current.git_commit, libjingle_new.git_commit) description = [ '-m', 'Roll ' + webrtc_str + delim + libjingle_str ] if webrtc_str: description.extend(['-m', webrtc_str]) description.extend(['-m', 'Changes: %s' % webrtc_changelog_url]) if libjingle_str: description.extend(['-m', libjingle_str]) description.extend(['-m', 'Changes: %s' % libjingle_changelog_url]) description.extend(['-m', 'TBR=']) return description class AutoRoller(object): def __init__(self, chromium_src): self._chromium_src = chromium_src def _RunCommand(self, command, working_dir=None, ignore_exit_code=False, extra_env=None): """Runs a command and returns the stdout from that command. If the command fails (exit code != 0), the function will exit the process. """ working_dir = working_dir or self._chromium_src logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir) env = os.environ.copy() if extra_env: logging.debug('extra env: %s', extra_env) env.update(extra_env) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=IS_WIN, env=env, cwd=working_dir, universal_newlines=True) output = p.stdout.read() p.wait() p.stdout.close() p.stderr.close() if not ignore_exit_code and p.returncode != 0: logging.error('Command failed: %s\n%s', str(command), output) sys.exit(p.returncode) return output def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None): working_dir = os.path.join(self._chromium_src, path_below_src) self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir) revision_range = git_hash or 'origin' ret = self._RunCommand( ['git', '--no-pager', 'log', revision_range, '--pretty=full', '-1'], working_dir=working_dir) return CommitInfo(_ParseGitCommitPosition(ret), _ParseGitCommitHash(ret), git_repo_url) def _GetDepsCommitInfo(self, deps_dict, path_below_src): entry = deps_dict['deps'][_PosixPath('src/%s' % path_below_src)] at_index = entry.find('@') git_repo_url = entry[:at_index] git_hash = entry[at_index + 1:] return self._GetCommitInfo(path_below_src, git_hash, git_repo_url) def _GetCLInfo(self): cl_output = self._RunCommand(['git', 'cl', 'issue']) m = CL_ISSUE_RE.match(cl_output.strip()) if not m: logging.error('Cannot find any CL info. Output was:\n%s', cl_output) sys.exit(-1) issue_number = int(m.group(1)) url = m.group(2) # Parse the Rietveld host from the URL. m = RIETVELD_URL_RE.match(url) if not m: logging.error('Cannot parse Rietveld host from URL: %s', url) sys.exit(-1) rietveld_server = m.group(1) return CLInfo(issue_number, url, rietveld_server) def _GetCurrentBranchName(self): return self._RunCommand( ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0] def _IsTreeClean(self): lines = self._RunCommand(['git', 'status', '--porcelain']).splitlines() if len(lines) == 0: return True logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines)) return False def _UpdateReadmeFile(self, readme_path, new_revision): readme = open(os.path.join(self._chromium_src, readme_path), 'r+') txt = readme.read() m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE), ('Revision: %s' % new_revision), txt) readme.seek(0) readme.write(m) readme.truncate() def PrepareRoll(self, dry_run, ignore_checks, no_commit, close_previous_roll): # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for # cross platform compatibility. if not ignore_checks: if self._GetCurrentBranchName() != 'master': logging.error('Please checkout the master branch.') return -1 if not self._IsTreeClean(): logging.error('Please make sure you don\'t have any modified files.') return -1 logging.debug('Checking for a previous roll branch.') if close_previous_roll: self.Abort() logging.debug('Pulling latest changes') if not ignore_checks: self._RunCommand(['git', 'pull']) self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) # Modify Chromium's DEPS file. # Parse current hashes. deps_filename = os.path.join(self._chromium_src, 'DEPS') deps = _ParseDepsFile(deps_filename) webrtc_current = self._GetDepsCommitInfo(deps, WEBRTC_PATH) libjingle_current = self._GetDepsCommitInfo(deps, LIBJINGLE_PATH) # Find ToT revisions. webrtc_latest = self._GetCommitInfo(WEBRTC_PATH) libjingle_latest = self._GetCommitInfo(LIBJINGLE_PATH) if IS_WIN: # Make sure the roll script doesn't use Windows line endings. self._RunCommand(['git', 'config', 'core.autocrlf', 'true']) self._UpdateDep(deps_filename, WEBRTC_PATH, webrtc_latest) self._UpdateDep(deps_filename, LIBJINGLE_PATH, libjingle_latest) if self._IsTreeClean(): print 'The latest revision is already rolled for WebRTC and libjingle.' self._DeleteRollBranch() else: self._UpdateReadmeFile(LIBJINGLE_README, libjingle_latest.commit_position) description = _GenerateCLDescriptionCommand( webrtc_current, libjingle_current, webrtc_latest, libjingle_latest) logging.debug('Committing changes locally.') self._RunCommand(['git', 'add', '--update', '.']) self._RunCommand(['git', 'commit'] + description) logging.debug('Uploading changes...') self._RunCommand(['git', 'cl', 'upload'], extra_env={'EDITOR': 'true'}) cl_info = self._GetCLInfo() logging.debug('Issue: %d URL: %s', cl_info.issue, cl_info.url) if not dry_run and not no_commit: logging.debug('Sending the CL to the CQ...') self._RunCommand(['git', 'cl', 'set_commit']) logging.debug('Sent the CL to the CQ. Monitor here: %s', cl_info.url) # TODO(kjellander): Checkout masters/previous branches again. return 0 def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info): dep_name = os.path.join('src', dep_relative_to_src) comment = 'commit position %s' % commit_info.commit_position # roll_dep_svn.py relies on cwd being the Chromium checkout, so let's # temporarily change the working directory and then change back. cwd = os.getcwd() os.chdir(os.path.dirname(deps_filename)) roll_dep_svn.update_deps(deps_filename, dep_relative_to_src, dep_name, commit_info.git_commit, comment) os.chdir(cwd) def _DeleteRollBranch(self): self._RunCommand(['git', 'checkout', 'master']) self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME) def _GetBranches(self): """Returns a tuple of active,branches. The 'active' is the name of the currently active branch and 'branches' is a list of all branches. """ lines = self._RunCommand(['git', 'branch']).split('\n') branches = [] active = '' for l in lines: if '*' in l: # The assumption is that the first char will always be the '*'. active = l[1:].strip() branches.append(active) else: b = l.strip() if b: branches.append(b) return (active, branches) def Abort(self): active_branch, branches = self._GetBranches() if active_branch == ROLL_BRANCH_NAME: active_branch = 'master' if ROLL_BRANCH_NAME in branches: print 'Aborting pending roll.' self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME]) # Ignore an error here in case an issue wasn't created for some reason. self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True) self._RunCommand(['git', 'checkout', active_branch]) self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) return 0 def WaitForTrybots(self): active_branch, _ = self._GetBranches() if active_branch != ROLL_BRANCH_NAME: self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME]) cl_info = self._GetCLInfo() return _WaitForTrybots(cl_info.issue, cl_info.rietveld_server) def main(): parser = argparse.ArgumentParser( description='Find webrtc and libjingle revisions for roll.') parser.add_argument('--abort', help=('Aborts a previously prepared roll. ' 'Closes any associated issues and deletes the roll branches'), action='store_true') parser.add_argument('--no-commit', help=('Don\'t send the CL to the CQ. This is useful if additional changes ' 'are needed to the CL (like for API changes).'), action='store_true') parser.add_argument('--wait-for-trybots', help=('Waits until all trybots from a previously created roll are either ' 'successful or at least one has failed. This is useful to be able to ' 'continuously run this script but not initiating new rolls until a ' 'previous one is known to have passed or failed.'), action='store_true') parser.add_argument('--close-previous-roll', action='store_true', help='Abort a previous roll if one exists.') parser.add_argument('--dry-run', action='store_true', default=False, help='Create branches and CLs but doesn\'t send tryjobs or commit.') parser.add_argument('--ignore-checks', action='store_true', default=False, help=('Skips checks for being on the master branch, dirty workspaces and ' 'the updating of the checkout. Will still delete and create local ' 'Git branches.')) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Be extra verbose in printing of log messages.') args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.ERROR) autoroller = AutoRoller(SRC_DIR) if args.abort: return autoroller.Abort() elif args.wait_for_trybots: return autoroller.WaitForTrybots() else: return autoroller.PrepareRoll(args.dry_run, args.ignore_checks, args.no_commit, args.close_previous_roll) if __name__ == '__main__': sys.exit(main())