#!/usr/bin/env python # # Copyright (c) 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Updates the Chrome reference builds. Usage: $ cd /tmp $ /path/to/update_reference_build.py -r $ cd reference_builds/reference_builds $ gcl change $ gcl upload $ gcl commit """ import errno import logging import optparse import os import shutil import subprocess import sys import time import urllib import urllib2 import zipfile class BuildUpdater(object): _PLATFORM_FILES_MAP = { 'Win': [ 'chrome-win32.zip', 'chrome-win32-syms.zip', 'chrome-win32.test/_pyautolib.pyd', 'chrome-win32.test/pyautolib.py', ], 'Mac': [ 'chrome-mac.zip', 'chrome-mac.test/_pyautolib.so', 'chrome-mac.test/pyautolib.py', ], 'Linux': [ 'chrome-linux.zip', ], 'Linux_x64': [ 'chrome-linux.zip', ], } _PLATFORM_DEST_MAP = { 'Linux': 'chrome_linux', 'Linux_x64': 'chrome_linux64', 'Win': 'chrome_win', 'Mac': 'chrome_mac', } def __init__(self, options): self._platforms = options.platforms.split(',') self._revision = int(options.revision) @staticmethod def _GetCmdStatusAndOutput(args, cwd=None, shell=False): """Executes a subprocess and returns its exit code and output. Args: args: A string or a sequence of program arguments. cwd: If not None, the subprocess's current directory will be changed to |cwd| before it's executed. shell: Whether to execute args as a shell command. Returns: The tuple (exit code, output). """ logging.info(str(args) + ' ' + (cwd or '')) p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell) stdout, stderr = p.communicate() exit_code = p.returncode if stderr: logging.critical(stderr) logging.info(stdout) return (exit_code, stdout) def _GetBuildUrl(self, platform, revision, filename): URL_FMT = ('http://commondatastorage.googleapis.com/' 'chromium-browser-snapshots/%s/%s/%s') return URL_FMT % (urllib.quote_plus(platform), revision, filename) def _FindBuildRevision(self, platform, revision, filename): MAX_REVISIONS_PER_BUILD = 100 for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD): r = urllib2.Request(self._GetBuildUrl(platform, revision_guess, filename)) r.get_method = lambda: 'HEAD' try: response = urllib2.urlopen(r) return revision_guess except urllib2.HTTPError, err: if err.code == 404: time.sleep(.1) continue return None def _DownloadBuilds(self): for platform in self._platforms: for f in BuildUpdater._PLATFORM_FILES_MAP[platform]: output = os.path.join('dl', platform, '%s_%s_%s' % (platform, self._revision, f)) if os.path.exists(output): logging.info('%s alread exists, skipping download' % output) continue build_revision = self._FindBuildRevision(platform, self._revision, f) if not build_revision: logging.critical('Failed to find %s build for r%s\n' % ( platform, self._revision)) sys.exit(1) dirname = os.path.dirname(output) if dirname and not os.path.exists(dirname): os.makedirs(dirname) url = self._GetBuildUrl(platform, build_revision, f) logging.info('Downloading %s, saving to %s' % (url, output)) r = urllib2.urlopen(url) with file(output, 'wb') as f: f.write(r.read()) def _FetchSvnRepos(self): if not os.path.exists('reference_builds'): os.makedirs('reference_builds') BuildUpdater._GetCmdStatusAndOutput( ['gclient', 'config', 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'], 'reference_builds') BuildUpdater._GetCmdStatusAndOutput( ['gclient', 'sync'], 'reference_builds') def _UnzipFile(self, dl_file, dest_dir): if not zipfile.is_zipfile(dl_file): return False logging.info('Opening %s' % dl_file) with zipfile.ZipFile(dl_file, 'r') as z: for content in z.namelist(): dest = os.path.join(dest_dir, content[content.find('/')+1:]) if not os.path.basename(dest): if not os.path.isdir(dest): os.makedirs(dest) continue with z.open(content) as unzipped_content: logging.info('Extracting %s to %s (%s)' % (content, dest, dl_file)) with file(dest, 'wb') as dest_file: dest_file.write(unzipped_content.read()) permissions = z.getinfo(content).external_attr >> 16 if permissions: os.chmod(dest, permissions) return True def _ClearDir(self, dir): """Clears all files in |dir| except for hidden files and folders.""" for root, dirs, files in os.walk(dir): # Skip hidden files and folders (like .svn and .git). files = [f for f in files if f[0] != '.'] dirs[:] = [d for d in dirs if d[0] != '.'] for f in files: os.remove(os.path.join(root, f)) def _ExtractBuilds(self): for platform in self._platforms: if os.path.exists('tmp_unzip'): os.path.unlink('tmp_unzip') dest_dir = os.path.join('reference_builds', 'reference_builds', BuildUpdater._PLATFORM_DEST_MAP[platform]) self._ClearDir(dest_dir) for root, _, dl_files in os.walk(os.path.join('dl', platform)): for dl_file in dl_files: dl_file = os.path.join(root, dl_file) if not self._UnzipFile(dl_file, dest_dir): logging.info('Copying %s to %s' % (dl_file, dest_dir)) shutil.copy(dl_file, dest_dir) def _SvnAddAndRemove(self): svn_dir = os.path.join('reference_builds', 'reference_builds') stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat'], svn_dir)[1] for line in stat.splitlines(): action, filename = line.split(None, 1) if action == '?': BuildUpdater._GetCmdStatusAndOutput( ['svn', 'add', filename], svn_dir) elif action == '!': BuildUpdater._GetCmdStatusAndOutput( ['svn', 'delete', filename], svn_dir) filepath = os.path.join(svn_dir, filename) if not os.path.isdir(filepath) and os.access(filepath, os.X_OK): BuildUpdater._GetCmdStatusAndOutput( ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir) def DownloadAndUpdateBuilds(self): self._DownloadBuilds() self._FetchSvnRepos() self._ExtractBuilds() self._SvnAddAndRemove() def ParseOptions(argv): parser = optparse.OptionParser() usage = 'usage: %prog ' parser.set_usage(usage) parser.add_option('-r', dest='revision', help='Revision to pickup') parser.add_option('-p', dest='platforms', default='Win,Mac,Linux,Linux_x64', help='Comma separated list of platforms to download ' '(as defined by the chromium builders).') (options, _) = parser.parse_args(argv) if not options.revision: logging.critical('Must specify -r\n') sys.exit(1) return options def main(argv): logging.getLogger().setLevel(logging.DEBUG) options = ParseOptions(argv) b = BuildUpdater(options) b.DownloadAndUpdateBuilds() logging.info('Successfully updated reference builds. Move to ' 'reference_builds/reference_builds and make a change with gcl.') if __name__ == '__main__': sys.exit(main(sys.argv))