diff options
author | binji@chromium.org <binji@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-05-23 20:43:45 +0000 |
---|---|---|
committer | binji@chromium.org <binji@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-05-23 20:43:45 +0000 |
commit | e83f4a2fe1c92de2711510ca1c7f44c8c5fabd13 (patch) | |
tree | bfeab9088300ede9e2ad960f5eda21d5f3f5a8d6 /native_client_sdk | |
parent | 94e2f43351f936d6c1cbd01cdf6d3c8969674506 (diff) | |
download | chromium_src-e83f4a2fe1c92de2711510ca1c7f44c8c5fabd13.zip chromium_src-e83f4a2fe1c92de2711510ca1c7f44c8c5fabd13.tar.gz chromium_src-e83f4a2fe1c92de2711510ca1c7f44c8c5fabd13.tar.bz2 |
[NaCl SDK] Automatically update the sdk_update utility.
Prior to this change a user would have to run:
naclsdk update sdk_tools
naclsdk <command>
BUG=125788
TEST=tests/test_auto_update_sdktools.py
Review URL: https://chromiumcodereview.appspot.com/10332183
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@138593 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'native_client_sdk')
10 files changed, 1206 insertions, 801 deletions
diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py index 0a215eb..cad16a3 100755 --- a/native_client_sdk/src/build_tools/build_sdk.py +++ b/native_client_sdk/src/build_tools/build_sdk.py @@ -17,7 +17,6 @@ and whether it should upload an SDK to file storage (GSTORE) # std python includes -import multiprocessing import optparse import os import platform @@ -26,9 +25,10 @@ import sys # local includes import buildbot_common +import build_updater import build_utils -import lastchange import manifest_util +from tests import test_server # Create the various paths of interest SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -55,65 +55,6 @@ MAKE = 'nacl_sdk/make_3_81/make.exe' CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py') -def HTTPServerProcess(conn, serve_dir): - """Run a local httpserver with a randomly-chosen port. - - This function assumes it is run as a child process using multiprocessing. - - Args: - conn: A connection to the parent process. The child process sends - the local port, and waits for a message from the parent to - stop serving. - serve_dir: The directory to serve. All files are accessible through - http://localhost:<port>/path/to/filename. - """ - import BaseHTTPServer - import SimpleHTTPServer - - os.chdir(serve_dir) - httpd = BaseHTTPServer.HTTPServer(('', 0), - SimpleHTTPServer.SimpleHTTPRequestHandler) - conn.send(httpd.server_address[1]) # the chosen port number - httpd.timeout = 0.5 # seconds - running = True - while running: - httpd.handle_request() - if conn.poll(): - running = conn.recv() - conn.close() - - -class LocalHTTPServer(object): - """Class to start a local HTTP server as a child process.""" - - def __init__(self, serve_dir): - parent_conn, child_conn = multiprocessing.Pipe() - self.process = multiprocessing.Process(target=HTTPServerProcess, - args=(child_conn, serve_dir)) - self.process.start() - if parent_conn.poll(10): # wait 10 seconds - self.port = parent_conn.recv() - else: - raise Exception('Unable to launch HTTP server.') - - self.conn = parent_conn - - def Shutdown(self): - """Send a message to the child HTTP server process and wait for it to - finish.""" - self.conn.send(False) - self.process.join() - - def GetURL(self, rel_url): - """Get the full url for a file on the local HTTP server. - - Args: - rel_url: A URL fragment to convert to a full URL. For example, - GetURL('foobar.baz') -> 'http://localhost:1234/foobar.baz' - """ - return 'http://localhost:%d/%s' % (self.port, rel_url) - - def AddMakeBat(pepperdir, makepath): """Create a simple batch file to execute Make. @@ -501,64 +442,6 @@ def CopyExamples(pepperdir, toolchains): GenerateExamplesMakefile(os.path.join(SDK_EXAMPLE_DIR, 'Makefile'), out_makefile, examples) -UPDATER_FILES = [ - # launch scripts - ('build_tools/naclsdk', 'nacl_sdk/naclsdk'), - ('build_tools/naclsdk.bat', 'nacl_sdk/naclsdk.bat'), - - # base manifest - ('build_tools/json/naclsdk_manifest0.json', - 'nacl_sdk/sdk_cache/naclsdk_manifest2.json'), - - # SDK tools - ('build_tools/sdk_tools/cacerts.txt', 'nacl_sdk/sdk_tools/cacerts.txt'), - ('build_tools/sdk_tools/sdk_update.py', 'nacl_sdk/sdk_tools/sdk_update.py'), - ('build_tools/manifest_util.py', 'nacl_sdk/sdk_tools/manifest_util.py'), - ('build_tools/sdk_tools/third_party/__init__.py', - 'nacl_sdk/sdk_tools/third_party/__init__.py'), - ('build_tools/sdk_tools/third_party/fancy_urllib/__init__.py', - 'nacl_sdk/sdk_tools/third_party/fancy_urllib/__init__.py'), - ('build_tools/sdk_tools/third_party/fancy_urllib/README', - 'nacl_sdk/sdk_tools/third_party/fancy_urllib/README'), - ('LICENSE', 'nacl_sdk/sdk_tools/LICENSE'), - (CYGTAR, 'nacl_sdk/sdk_tools/cygtar.py'), -] - -def CopyFiles(files): - for in_file, out_file in files: - if not os.path.isabs(in_file): - in_file = os.path.join(SDK_SRC_DIR, in_file) - out_file = os.path.join(OUT_DIR, out_file) - buildbot_common.MakeDir(os.path.dirname(out_file)) - buildbot_common.CopyFile(in_file, out_file) - -def BuildUpdater(): - buildbot_common.BuildStep('Create Updater') - - # Build SDK directory - buildbot_common.RemoveDir(os.path.join(OUT_DIR, 'nacl_sdk')) - - CopyFiles(UPDATER_FILES) - - # Make zip - buildbot_common.RemoveFile(os.path.join(OUT_DIR, 'nacl_sdk.zip')) - buildbot_common.Run([sys.executable, oshelpers.__file__, 'zip', - 'nacl_sdk.zip'] + - [out_file for in_file, out_file in UPDATER_FILES], - cwd=OUT_DIR) - - # Tar of all files under nacl_sdk/sdk_tools - sdktoolsdir = 'nacl_sdk/sdk_tools' - tarname = os.path.join(OUT_DIR, 'sdk_tools.tgz') - files_to_tar = [os.path.relpath(out_file, sdktoolsdir) for in_file, out_file - in UPDATER_FILES if out_file.startswith(sdktoolsdir)] - buildbot_common.RemoveFile(tarname) - buildbot_common.Run([sys.executable, CYGTAR, '-C', - os.path.join(OUT_DIR, sdktoolsdir), '-czf', tarname] + files_to_tar, - cwd=NACL_DIR) - sys.stdout.write('\n') - - def main(args): parser = optparse.OptionParser() parser.add_option('--pnacl', help='Enable pnacl build.', @@ -601,7 +484,7 @@ def main(args): pepper_ver = str(int(build_utils.ChromeMajorVersion())) pepper_old = str(int(build_utils.ChromeMajorVersion()) - 1) - clnumber = lastchange.FetchVersionInfo(None).revision + clnumber = build_utils.ChromeRevision() if options.release: pepper_ver = options.release print 'Building PEPPER %s at %s' % (pepper_ver, clnumber) @@ -669,7 +552,7 @@ def main(args): # build sdk update if not skip_update: - BuildUpdater() + build_updater.BuildUpdater(OUT_DIR) # start local server sharing a manifest + the new bundle if not skip_test_updater and not skip_tar: @@ -681,7 +564,7 @@ def main(args): server = None try: buildbot_common.BuildStep('Run local server') - server = LocalHTTPServer(SERVER_DIR) + server = test_server.LocalHTTPServer(SERVER_DIR) buildbot_common.BuildStep('Generate manifest') with open(tarfile, 'rb') as tarfile_stream: @@ -710,9 +593,10 @@ def main(args): # use newly built sdk updater to pull this bundle buildbot_common.BuildStep('Update from local server') - updater_py = os.path.join(OUT_DIR, 'nacl_sdk', 'sdk_tools', - 'sdk_update.py') - buildbot_common.Run([sys.executable, updater_py, '-U', + naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk') + if platform == 'win': + naclsdk_sh += '.bat' + buildbot_common.Run([naclsdk_sh, '-U', server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver]) # If we are testing examples, do it in the newly pulled directory. diff --git a/native_client_sdk/src/build_tools/build_updater.py b/native_client_sdk/src/build_tools/build_updater.py new file mode 100755 index 0000000..3df0c39 --- /dev/null +++ b/native_client_sdk/src/build_tools/build_updater.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# copyright (c) 2012 the chromium authors. all rights reserved. +# use of this source code is governed by a bsd-style license that can be +# found in the license file. + +"""Build script to generate a new sdk_tools bundle. + +This script packages the files necessary to generate the SDK updater -- the +tool users run to download new bundles, update existing bundles, etc. +""" + +import buildbot_common +import build_utils +import cStringIO +import optparse +import os +import re +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR) +SDK_DIR = os.path.dirname(SDK_SRC_DIR) +SRC_DIR = os.path.dirname(SDK_DIR) +NACL_DIR = os.path.join(SRC_DIR, 'native_client') +CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py') + +sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) + +import oshelpers + + +UPDATER_FILES = [ + # launch scripts + ('build_tools/naclsdk', 'nacl_sdk/naclsdk'), + ('build_tools/naclsdk.bat', 'nacl_sdk/naclsdk.bat'), + + # base manifest + ('build_tools/json/naclsdk_manifest0.json', + 'nacl_sdk/sdk_cache/naclsdk_manifest2.json'), + + # SDK tools + ('build_tools/sdk_tools/cacerts.txt', 'nacl_sdk/sdk_tools/cacerts.txt'), + ('build_tools/sdk_tools/sdk_update.py', 'nacl_sdk/sdk_tools/sdk_update.py'), + ('build_tools/sdk_tools/sdk_update_common.py', + 'nacl_sdk/sdk_tools/sdk_update_common.py'), + ('build_tools/sdk_tools/sdk_update_main.py', + 'nacl_sdk/sdk_tools/sdk_update_main.py'), + ('build_tools/manifest_util.py', 'nacl_sdk/sdk_tools/manifest_util.py'), + ('build_tools/sdk_tools/third_party/__init__.py', + 'nacl_sdk/sdk_tools/third_party/__init__.py'), + ('build_tools/sdk_tools/third_party/fancy_urllib/__init__.py', + 'nacl_sdk/sdk_tools/third_party/fancy_urllib/__init__.py'), + ('build_tools/sdk_tools/third_party/fancy_urllib/README', + 'nacl_sdk/sdk_tools/third_party/fancy_urllib/README'), + ('LICENSE', 'nacl_sdk/sdk_tools/LICENSE'), + (CYGTAR, 'nacl_sdk/sdk_tools/cygtar.py'), +] + + +def MakeUpdaterFilesAbsolute(out_dir): + """Return the result of changing all relative paths in UPDATER_FILES to + absolute paths. + + Args: + out_dir: The output directory. + Returns: + A list of 2-tuples. The first element in each tuple is the source path and + the second is the destination path. + """ + assert os.path.isabs(out_dir) + + result = [] + for in_file, out_file in UPDATER_FILES: + if not os.path.isabs(in_file): + in_file = os.path.join(SDK_SRC_DIR, in_file) + out_file = os.path.join(out_dir, out_file) + result.append((in_file, out_file)) + return result + + +def CopyFiles(files): + """Given a list of 2-tuples (source, dest), copy each source file to a dest + file. + + Args: + files: A list of 2-tuples.""" + for in_file, out_file in files: + buildbot_common.MakeDir(os.path.dirname(out_file)) + buildbot_common.CopyFile(in_file, out_file) + + +def UpdateRevisionNumber(out_dir, revision_number): + """Update the sdk_tools bundle to have the given revision number. + + This function finds all occurrences of the string "{REVISION}" in + sdk_update_main.py and replaces them with |revision_number|. The only + observable effect of this change should be that running: + + naclsdk -v + + will contain the new |revision_number|. + + Args: + out_dir: The output directory containing the scripts to update. + revision_number: The revision number as an integer, or None to use the + current Chrome revision (as retrieved through svn/git). + """ + if revision_number is None: + revision_number = build_utils.ChromeRevision() + + SDK_UPDATE_MAIN = os.path.join(out_dir, + 'nacl_sdk/sdk_tools/sdk_update_main.py') + + file = open(SDK_UPDATE_MAIN, 'r').read().replace( + '{REVISION}', str(revision_number)) + open(SDK_UPDATE_MAIN, 'w').write(file) + + +def BuildUpdater(out_dir, revision_number=None): + """Build naclsdk.zip and sdk_tools.tgz in |out_dir|. + + Args: + out_dir: The output directory. + revision_number: The revision number of this updater, as an integer. Or + None, to use the current Chrome revision.""" + buildbot_common.BuildStep('Create Updater') + + out_dir = os.path.abspath(out_dir) + + # Build SDK directory + buildbot_common.RemoveDir(os.path.join(out_dir, 'nacl_sdk')) + + updater_files = MakeUpdaterFilesAbsolute(out_dir) + out_files = [out_file for in_file, out_file in updater_files] + + CopyFiles(updater_files) + UpdateRevisionNumber(out_dir, revision_number) + + # Make zip + buildbot_common.RemoveFile(os.path.join(out_dir, 'nacl_sdk.zip')) + buildbot_common.Run([sys.executable, oshelpers.__file__, 'zip', + 'nacl_sdk.zip'] + out_files, + cwd=out_dir) + + # Tar of all files under nacl_sdk/sdk_tools + sdktoolsdir = os.path.join(out_dir, 'nacl_sdk/sdk_tools') + tarname = os.path.join(out_dir, 'sdk_tools.tgz') + files_to_tar = [os.path.relpath(out_file, sdktoolsdir) + for out_file in out_files if out_file.startswith(sdktoolsdir)] + buildbot_common.RemoveFile(tarname) + buildbot_common.Run([sys.executable, CYGTAR, '-C', + os.path.join(out_dir, sdktoolsdir), '-czf', tarname] + files_to_tar) + sys.stdout.write('\n') + + +def main(args): + parser = optparse.OptionParser() + parser.add_option('-o', '--out', help='output directory', + dest='out_dir', default='out') + options, args = parser.parse_args(args[1:]) + + BuildUpdater(options.out_dir) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/native_client_sdk/src/build_tools/build_utils.py b/native_client_sdk/src/build_tools/build_utils.py index a521a96..5bad1a4 100644 --- a/native_client_sdk/src/build_tools/build_utils.py +++ b/native_client_sdk/src/build_tools/build_utils.py @@ -52,6 +52,15 @@ def ChromeMajorVersion(): return str(MAJOR) +def ChromeRevision(): + '''Extract chrome revision from svn. + + Returns: + The Chrome revision as a string. e.g. "12345" + ''' + return lastchange.FetchVersionInfo(None).revision + + #------------------------------------------------------------------------------ # Parameters diff --git a/native_client_sdk/src/build_tools/manifest_util.py b/native_client_sdk/src/build_tools/manifest_util.py index 9ce6495..6b6acca 100644 --- a/native_client_sdk/src/build_tools/manifest_util.py +++ b/native_client_sdk/src/build_tools/manifest_util.py @@ -174,6 +174,7 @@ class Archive(dict): # special case, self.checksum returns the sha1, not the checksum dict. if name == 'checksum': self.setdefault('checksum', {})['sha1'] = value + return return self.__setitem__(name, value) def GetChecksum(self, type='sha1'): diff --git a/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py b/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py index b7949cf..14dec4c 100755 --- a/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py +++ b/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py @@ -3,705 +3,106 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -'''A simple tool to update the Native Client SDK to the latest version''' - -import cStringIO -import cygtar -import errno -import exceptions -import hashlib -import json -import manifest_util -import optparse import os -import shutil +import re import subprocess -import sys +from sdk_update_common import * import tempfile -from third_party import fancy_urllib -import time -import urllib2 -import urlparse - - -#------------------------------------------------------------------------------ -# Constants - -# Bump the MINOR_REV every time you check this file in. -MAJOR_REV = 2 -MINOR_REV = 17 - -GLOBAL_HELP = '''Usage: naclsdk [options] command [command_options] - -naclsdk is a simple utility that updates the Native Client (NaCl) -Software Developer's Kit (SDK). Each component is kept as a 'bundle' that -this utility can download as as subdirectory into the SDK. - -Commands: - help [command] - Get either general or command-specific help - list - Lists the available bundles - update/install - Updates/installs bundles in the SDK - sources - Manage external package sources - -Example Usage: - naclsdk list - naclsdk update --force pepper_17 - naclsdk install recommended - naclsdk help update - naclsdk sources --list''' -CONFIG_FILENAME='naclsdk_config.json' -MANIFEST_FILENAME='naclsdk_manifest2.json' -SDK_TOOLS='sdk_tools' # the name for this tools directory -USER_DATA_DIR='sdk_cache' +"""Shim script for the SDK updater, to allow automatic updating. -HTTP_CONTENT_LENGTH = 'Content-Length' # HTTP Header field for content length +The purpose of this script is to be a shim which automatically updates +sdk_tools (the bundle containing the updater scripts) whenever this script is +run. +When the sdk_tools bundle has been updated to the most recent version, this +script forwards its arguments to sdk_updater_main.py. +""" -#------------------------------------------------------------------------------ -# General Utilities +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SDK_UPDATE_MAIN = os.path.join(SCRIPT_DIR, 'sdk_update_main.py') +SDK_ROOT_DIR = os.path.dirname(SCRIPT_DIR) +NACLSDK_SHELL_SCRIPT = os.path.join(SDK_ROOT_DIR, 'naclsdk') +SDK_TOOLS_DIR = os.path.join(SDK_ROOT_DIR, 'sdk_tools') +SDK_TOOLS_UPDATE_DIR = os.path.join(SDK_ROOT_DIR, 'sdk_tools_update') -_debug_mode = False -_quiet_mode = False - -def DebugPrint(msg): - '''Display a message to stderr if debug printing is enabled - - Note: This function appends a newline to the end of the string +def MakeSdkUpdateMainCmd(args): + """Returns a list of command line arguments to run sdk_update_main. Args: - msg: A string to send to stderr in debug mode''' - if _debug_mode: - sys.stderr.write("%s\n" % msg) - sys.stderr.flush() + args: A list of arguments to pass to sdk_update_main.py + Returns: + A new list that can be passed to subprocess.call, subprocess.Popen, etc. + """ + return [sys.executable, SDK_UPDATE_MAIN] + args -def InfoPrint(msg): - '''Display an informational message to stdout if not in quiet mode - - Note: This function appends a newline to the end of the string +def UpdateSDKTools(args): + """Run sdk_update_main to update sdk_tools bundle. Return True if it is + updated. Args: - mgs: A string to send to stdio when not in quiet mode''' - if not _quiet_mode: - sys.stdout.write("%s\n" % msg) - sys.stdout.flush() - - -def WarningPrint(msg): - '''Display an informational message to stderr. - - Note: This function appends a newline to the end of the string - - Args: - mgs: A string to send to stderr.''' - sys.stderr.write("WARNING: %s\n" % msg) - sys.stderr.flush() - - -class Error(Exception): - '''Generic error/exception for sdk_update module''' - pass - - -def UrlOpen(url): - request = fancy_urllib.FancyRequest(url) - ca_certs = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'cacerts.txt') - request.set_ssl_info(ca_certs=ca_certs) - url_opener = urllib2.build_opener( - fancy_urllib.FancyProxyHandler(), - fancy_urllib.FancyRedirectHandler(), - fancy_urllib.FancyHTTPSHandler()) - return url_opener.open(request) - -def ExtractInstaller(installer, outdir): - '''Extract the SDK installer into a given directory - - If the outdir already exists, then this function deletes it - - Args: - installer: full path of the SDK installer - outdir: output directory where to extract the installer - - Raises: - CalledProcessError - if the extract operation fails''' - RemoveDir(outdir) - - if os.path.splitext(installer)[1] == '.exe': - # If the installer has extension 'exe', assume it's a Windows NSIS-style - # installer that handles silent (/S) and relocated (/D) installs. - command = [installer, '/S', '/D=%s' % outdir] - subprocess.check_call(command) + args: The arguments to pass to sdk_update_main.py. We need to keep this to + ensure sdk_update_main is called correctly; some parameters specify + URLS or directories to use. + Returns: + True if the sdk_tools bundle was updated. + """ + cmd = MakeSdkUpdateMainCmd(['--update-sdk-tools'] + args) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + stdout, _ = process.communicate() + if process.returncode == 0: + return stdout.find('sdk_tools is already up-to-date.') == -1 else: - os.mkdir(outdir) - tar_file = None - curpath = os.getcwd() - try: - tar_file = cygtar.CygTar(installer, 'r', verbose=True) - if outdir: os.chdir(outdir) - tar_file.Extract() - finally: - if tar_file: - tar_file.Close() - os.chdir(curpath) - - -def RemoveDir(outdir): - '''Removes the given directory + # Updating sdk_tools could fail for any number of reasons. Regardless, it + # should be safe to try to run the user's command. + return False - On Unix systems, this just runs shutil.rmtree, but on Windows, this doesn't - work when the directory contains junctions (as does our SDK installer). - Therefore, on Windows, it runs rmdir /S /Q as a shell command. This always - does the right thing on Windows. If the directory already didn't exist, - RemoveDir will return successfully without taking any action. - Args: - outdir: The directory to delete - - Raises: - CalledProcessError - if the delete operation fails on Windows - OSError - if the delete operation fails on Linux - ''' - - DebugPrint('Removing %s' % outdir) +def RenameSdkToolsDirectory(): + """Rename sdk_tools_update to sdk_tools.""" try: - shutil.rmtree(outdir) - except: - if not os.path.exists(outdir): - return - # On Windows this could be an issue with junctions, so try again with rmdir - if sys.platform == 'win32': - subprocess.check_call(['rmdir', '/S', '/Q', outdir], shell=True) - - -def RenameDir(srcdir, destdir): - '''Renames srcdir to destdir. Removes destdir before doing the - rename if it already exists.''' - - max_tries = 5 - for num_tries in xrange(max_tries): + tempdir = tempfile.mkdtemp() + temp_sdktools = os.path.join(tempdir, 'sdk_tools') try: - RemoveDir(destdir) - os.rename(srcdir, destdir) - return - except OSError as err: - if err.errno != errno.EACCES: - raise err - # If we are here, we didn't exit due to raised exception, so we are - # handling a Windows flaky access error. Sleep one second and try - # again. - time.sleep(num_tries + 1) - # end of while loop -- could not RenameDir - raise Error('Could not RenameDir %s => %s after %d tries.\n' % - 'Please check that no shells or applications ' - 'are accessing files in %s.' - % (srcdir, destdir, num_tries, destdir)) - - -class ProgressFunction(object): - '''Create a progress function for a file with a given size''' - - def __init__(self, file_size=0): - '''Constructor - - Args: - file_size: number of bytes in file. 0 indicates unknown''' - self.dots = 0 - self.file_size = int(file_size) - - def GetProgressFunction(self): - '''Returns a progress function based on a known file size''' - def ShowKnownProgress(progress): - if progress == 0: - sys.stdout.write('|%s|\n' % ('=' * 48)) - else: - new_dots = progress * 50 / self.file_size - self.dots - sys.stdout.write('.' * new_dots) - self.dots += new_dots - if progress == self.file_size: - sys.stdout.write('\n') - sys.stdout.flush() - - return ShowKnownProgress - - -def DownloadArchiveToFile(archive, dest_path): - '''Download the archive's data to a file at dest_path. - - As a side effect, computes the sha1 hash and data size, both returned as a - tuple. Raises an Error if the url can't be opened, or an IOError exception if - dest_path can't be opened. - - Args: - dest_path: Path for the file that will receive the data. - Return: - A tuple (sha1, size) with the sha1 hash and data size respectively.''' - sha1 = None - size = 0 - with open(dest_path, 'wb') as to_stream: - from_stream = None - try: - from_stream = UrlOpen(archive.url) - except urllib2.URLError: - raise Error('Cannot open "%s" for archive %s' % - (archive.url, archive.host_os)) - try: - content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH]) - progress_function = ProgressFunction(content_length).GetProgressFunction() - InfoPrint('Downloading %s' % archive.url) - sha1, size = manifest_util.DownloadAndComputeHash( - from_stream, - to_stream=to_stream, - progress_func=progress_function) - if size != content_length: - raise Error('Download size mismatch for %s.\n' - 'Expected %s bytes but got %s' % - (archive.url, content_length, size)) - finally: - if from_stream: from_stream.close() - return sha1, size - - -def LoadFromFile(path, obj): - '''Returns a manifest loaded from the JSON file at |path|. - - If the path does not exist or is invalid, returns unmodified object.''' - methodlist = [m for m in dir(obj) if callable(getattr(obj, m))] - if 'LoadDataFromString' not in methodlist: - return obj - if not os.path.exists(path): - return obj - - with open(path, 'r') as f: - json_string = f.read() - if not json_string: - return obj - - obj.LoadDataFromString(json_string) - return obj - - -def LoadManifestFromURLs(urls): - '''Returns a manifest loaded from |urls|, merged into one manifest.''' - manifest = manifest_util.SDKManifest() - for url in urls: - try: - url_stream = UrlOpen(url) - except urllib2.URLError as e: - raise Error('Unable to open %s. [%s]' % (url, e)) - - manifest_stream = cStringIO.StringIO() - sha1, size = manifest_util.DownloadAndComputeHash(url_stream, - manifest_stream) - temp_manifest = manifest_util.SDKManifest() - temp_manifest.LoadDataFromString(manifest_stream.getvalue()) - - manifest.MergeManifest(temp_manifest) + RenameDir(SDK_TOOLS_DIR, temp_sdktools) + except Error: + # The user is probably on Windows, and the directory is locked. + sys.stderr.write('Cannot rename directory "%s". Make sure no programs are' + ' viewing or accessing this directory and try again.\n' % ( + SDK_TOOLS_DIR,)) + sys.exit(1) - def BundleFilter(bundle): - # Only add this bundle if it's supported on this platform. - return bundle.GetHostOSArchive() - - manifest.FilterBundles(BundleFilter) - return manifest - - -def WriteToFile(path, obj): - '''Write |manifest| to a JSON file at |path|.''' - methodlist = [m for m in dir(obj) if callable(getattr(obj, m))] - if 'GetDataAsString' not in methodlist: - raise Error('Unable to write object to file') - json_string = obj.GetDataAsString() - - # Write the JSON data to a temp file. - temp_file_name = None - # TODO(dspringer): Use file locks here so that multiple sdk_updates can - # run at the same time. - with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - f.write(json_string) - temp_file_name = f.name - # Move the temp file to the actual file. - if os.path.exists(path): - os.remove(path) - shutil.move(temp_file_name, path) - - -class SDKConfig(object): - '''This class contains utilities for manipulating an SDK config - ''' - - def __init__(self): - '''Create a new SDKConfig object with default contents''' - self._data = { - 'sources': [], - } - - def AddSource(self, string): - '''Add a source file to load packages from. - - Args: - string: a URL to an external package manifest file.''' - # For now whitelist only the following location for external sources: - # https://commondatastorage.googleapis.com/nativeclient-mirror/nacl/nacl_sdk - (scheme, host, path, _, _, _) = urlparse.urlparse(string) - if (host != 'commondatastorage.googleapis.com' or - scheme != 'https' or - not path.startswith('/nativeclient-mirror/nacl/nacl_sdk')): - WarningPrint('Only whitelisted sources from ' - '\'https://commondatastorage.googleapis.com/nativeclient-' - 'mirror/nacl/nacl_sdk\' are currently allowed.') - return - if string in self._data['sources']: - WarningPrint('source \''+string+'\' already exists in config.') - return try: - url_stream = UrlOpen(string) - except urllib2.URLError: - WarningPrint('Unable to fetch manifest URL \'%s\'. Exiting...' % string) - return - - self._data['sources'].append(string) - InfoPrint('source \''+string+'\' added to config.') - - def RemoveSource(self, string): - '''Remove a source file to load packages from. - - Args: - string: a URL to an external SDK manifest file.''' - if string not in self._data['sources']: - WarningPrint('source \''+string+'\' doesn\'t exist in config.') - else: - self._data['sources'].remove(string) - InfoPrint('source \''+string+'\' removed from config.') - - def RemoveAllSources(self): - if len(self.GetSources()) == 0: - InfoPrint('There are no external sources to remove.') - # Copy the list because RemoveSource modifies the underlying list - sources = list(self.GetSources()) - for source in sources: - self.RemoveSource(source) - - - def ListSources(self): - '''List all external sources in config.''' - if len(self._data['sources']): - InfoPrint('Installed sources:') - for s in self._data['sources']: - InfoPrint(' '+s) - else: - InfoPrint('No external sources installed') - - def GetSources(self): - '''Return a list of external sources''' - return self._data['sources'] - - def LoadDataFromString(self, string): - ''' Load a JSON config string. Raises an exception if string - is not well-formed JSON. - - Args: - string: a JSON-formatted string containing the previous config''' - self._data = json.loads(string) - - - def GetDataAsString(self): - '''Returns the current JSON manifest object, pretty-printed''' - pretty_string = json.dumps(self._data, sort_keys=False, indent=2) - # json.dumps sometimes returns trailing whitespace and does not put - # a newline at the end. This code fixes these problems. - pretty_lines = pretty_string.split('\n') - return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n' - - -#------------------------------------------------------------------------------ -# Commands - - -def List(options, argv, config): - '''Usage: %prog [options] list - - Lists the available SDK bundles that are available for download.''' - def PrintBundles(bundles): - for bundle in bundles: - InfoPrint(' %s' % bundle.name) - for key, value in bundle.iteritems(): - if key not in (manifest_util.ARCHIVES_KEY, manifest_util.NAME_KEY): - InfoPrint(' %s: %s' % (key, value)) - - DebugPrint("Running List command with: %s, %s" %(options, argv)) - - parser = optparse.OptionParser(usage=List.__doc__) - (list_options, args) = parser.parse_args(argv) - manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources()) - InfoPrint('Available bundles:') - PrintBundles(manifest.GetBundles()) - # Print the local information. - manifest_path = os.path.join(options.user_data_dir, options.manifest_filename) - local_manifest = LoadFromFile(manifest_path, manifest_util.SDKManifest()) - InfoPrint('\nCurrently installed bundles:') - PrintBundles(local_manifest.GetBundles()) - - -def Update(options, argv, config): - '''Usage: %prog [options] update [target] - - Updates the Native Client SDK to a specified version. By default, this - command updates all the recommended components. The update process works - like this: - 1. Fetch the manifest from the mirror. - 2. Load manifest from USER_DATA_DIR - if there is no local manifest file, - make an empty manifest object. - 3. Update each the bundle: - for bundle in bundles: - # Compare bundle versions & revisions. - # Test if local version.revision < mirror OR local doesn't exist. - if local_manifest < mirror_manifest: - update(bundle) - update local_manifest with mirror_manifest for bundle - write manifest to disk. Use locks. - else: - InfoPrint('bundle is up-to-date') - - Targets: - recommended: (default) Install/Update all recommended components - all: Install/Update all available components - bundle_name: Install/Update only the given bundle - ''' - DebugPrint("Running Update command with: %s, %s" % (options, argv)) - ALL='all' # Update all bundles - RECOMMENDED='recommended' # Only update the bundles with recommended=yes - - parser = optparse.OptionParser(usage=Update.__doc__) - parser.add_option( - '-F', '--force', dest='force', - default=False, action='store_true', - help='Force updating existing components that already exist') - (update_options, args) = parser.parse_args(argv) - if len(args) == 0: - args = [RECOMMENDED] - manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources()) - bundles = manifest.GetBundles() - local_manifest_path = os.path.join(options.user_data_dir, - options.manifest_filename) - local_manifest = LoadFromFile(local_manifest_path, - manifest_util.SDKManifest()) - - # Validate the arg list against the available bundle names. Raises an - # error if any invalid bundle names or args are detected. - valid_args = set([ALL, RECOMMENDED] + [bundle.name for bundle in bundles]) - bad_args = set(args) - valid_args - if len(bad_args) > 0: - raise Error("Unrecognized bundle name or argument: '%s'" % - ', '.join(bad_args)) - - for bundle in bundles: - bundle_path = os.path.join(options.sdk_root_dir, bundle.name) - bundle_update_path = '%s_update' % bundle_path - if not (bundle.name in args or - ALL in args or (RECOMMENDED in args and - bundle[RECOMMENDED] == 'yes')): - continue - def UpdateBundle(): - '''Helper to install a bundle''' - archive = bundle.GetHostOSArchive() - (scheme, host, path, _, _, _) = urlparse.urlparse(archive['url']) - dest_filename = os.path.join(options.user_data_dir, path.split('/')[-1]) - sha1, size = DownloadArchiveToFile(archive, dest_filename) - if sha1 != archive.GetChecksum(): - raise Error("SHA1 checksum mismatch on '%s'. Expected %s but got %s" % - (bundle.name, archive.GetChecksum(), sha1)) - if size != archive.size: - raise Error("Size mismatch on Archive. Expected %s but got %s bytes" % - (archive.size, size)) - InfoPrint('Updating bundle %s to version %s, revision %s' % ( - (bundle.name, bundle.version, bundle.revision))) - ExtractInstaller(dest_filename, bundle_update_path) - if bundle.name != SDK_TOOLS: - repath = bundle.get('repath', None) - if repath: - bundle_move_path = os.path.join(bundle_update_path, repath) - else: - bundle_move_path = bundle_update_path - RenameDir(bundle_move_path, bundle_path) - if os.path.exists(bundle_update_path): - RemoveDir(bundle_update_path) - os.remove(dest_filename) - local_manifest.MergeBundle(bundle) - WriteToFile(local_manifest_path, local_manifest) - # Test revision numbers, update the bundle accordingly. - # TODO(dspringer): The local file should be refreshed from disk each - # iteration thought this loop so that multiple sdk_updates can run at the - # same time. - if local_manifest.BundleNeedsUpdate(bundle): - if (not update_options.force and os.path.exists(bundle_path) and - bundle.name != SDK_TOOLS): - WarningPrint('%s already exists, but has an update available.\n' - 'Run update with the --force option to overwrite the ' - 'existing directory.\nWarning: This will overwrite any ' - 'modifications you have made within this directory.' - % bundle.name) - else: - UpdateBundle() - else: - InfoPrint('%s is already up-to-date.' % bundle.name) - -def Sources(options, argv, config): - '''Usage: %prog [options] sources [--list,--add URL,--remove URL] - - Manage additional package sources. URL should point to a valid package - manifest file for download. - ''' - DebugPrint("Running Sources command with: %s, %s" % (options, argv)) - - parser = optparse.OptionParser(usage=Sources.__doc__) - parser.add_option( - '-a', '--add', dest='url_to_add', - default=None, - help='Add additional package source') - parser.add_option( - '-r', '--remove', dest='url_to_remove', - default=None, - help='Remove package source (use \'all\' for all additional sources)') - parser.add_option( - '-l', '--list', dest='do_list', - default=False, action='store_true', - help='List additional package sources') - (source_options, args) = parser.parse_args(argv) - - write_config = False - if source_options.url_to_add: - config.AddSource(source_options.url_to_add) - write_config = True - elif source_options.url_to_remove: - if source_options.url_to_remove == 'all': - config.RemoveAllSources() - else: - config.RemoveSource(source_options.url_to_remove) - write_config = True - elif source_options.do_list: - config.ListSources() + RenameDir(SDK_TOOLS_UPDATE_DIR, SDK_TOOLS_DIR) + except Error: + # Failed for some reason, move the old dir back. + try: + RenameDir(temp_sdktools, SDK_TOOLS_DIR) + except: + # Not much to do here. sdk_tools won't exist, but sdk_tools_update + # should. Hopefully running the batch script again will move + # sdk_tools_update -> sdk_tools and it will work this time... + sys.stderr.write('Unable to restore directory "%s" while auto-updating.' + 'Make sure no programs are viewing or accessing this directory and' + 'try again.\n' % (SDK_TOOLS_DIR,)) + sys.exit(1) + finally: + RemoveDir(tempdir) + + +def main(): + args = sys.argv[1:] + if UpdateSDKTools(args): + RenameSdkToolsDirectory() + # Call the shell script, just in case this script was updated in the next + # version of sdk_tools + return subprocess.call([NACLSDK_SHELL_SCRIPT] + args) else: - parser.print_help() - - if write_config: - WriteToFile(os.path.join(options.user_data_dir, options.config_filename), - config) - -#------------------------------------------------------------------------------ -# Command-line interface - - -def main(argv): - '''Main entry for the sdk_update utility''' - parser = optparse.OptionParser(usage=GLOBAL_HELP) - DEFAULT_SDK_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - parser.add_option( - '-U', '--manifest-url', dest='manifest_url', - default='https://commondatastorage.googleapis.com/nativeclient-mirror/' - 'nacl/nacl_sdk/%s' % MANIFEST_FILENAME, - help='override the default URL for the NaCl manifest file') - parser.add_option( - '-d', '--debug', dest='debug', - default=False, action='store_true', - help='enable displaying debug information to stderr') - parser.add_option( - '-q', '--quiet', dest='quiet', - default=False, action='store_true', - help='suppress displaying informational prints to stdout') - parser.add_option( - '-u', '--user-data-dir', dest='user_data_dir', - # TODO(mball): the default should probably be in something like - # ~/.naclsdk (linux), or ~/Library/Application Support/NaClSDK (mac), - # or %HOMEPATH%\Application Data\NaClSDK (i.e., %APPDATA% on windows) - default=os.path.join(DEFAULT_SDK_ROOT, USER_DATA_DIR), - help="specify location of NaCl SDK's data directory") - parser.add_option( - '-s', '--sdk-root-dir', dest='sdk_root_dir', - default=DEFAULT_SDK_ROOT, - help="location where the SDK bundles are installed") - parser.add_option( - '-v', '--version', dest='show_version', - action='store_true', - help='show version information and exit') - parser.add_option( - '-m', '--manifest', dest='manifest_filename', - default=MANIFEST_FILENAME, - help="name of local manifest file relative to user-data-dir") - parser.add_option( - '-c', '--config', dest='config_filename', - default=CONFIG_FILENAME, - help="name of the local config file relative to user-data-dir") - - - COMMANDS = { - 'list': List, - 'update': Update, - 'install': Update, - 'sources': Sources, - } - - # Separate global options from command-specific options - global_argv = argv - command_argv = [] - for index, arg in enumerate(argv): - if arg in COMMANDS: - global_argv = argv[:index] - command_argv = argv[index:] - break - - (options, args) = parser.parse_args(global_argv) - args += command_argv - - global _debug_mode, _quiet_mode - _debug_mode = options.debug - _quiet_mode = options.quiet - - def PrintHelpAndExit(unused_options=None, unused_args=None): - parser.print_help() - exit(1) - - if options.show_version: - print "Native Client SDK Updater, version %s.%s" % (MAJOR_REV, MINOR_REV) - exit(0) - - if not args: - print "Need to supply a command" - PrintHelpAndExit() - - def DefaultHandler(unused_options=None, unused_args=None): - print "Unknown Command: %s" % args[0] - PrintHelpAndExit() - - def InvokeCommand(args): - command = COMMANDS.get(args[0], DefaultHandler) - # Load the config file before running commands - config = LoadFromFile(os.path.join(options.user_data_dir, - options.config_filename), - SDKConfig()) - command(options, args[1:], config) - - if args[0] == 'help': - if len(args) == 1: - PrintHelpAndExit() - else: - InvokeCommand([args[1], '-h']) - else: - # Make sure the user_data_dir exists. - if not os.path.exists(options.user_data_dir): - os.makedirs(options.user_data_dir) - InvokeCommand(args) - - return 0 # Success - + return subprocess.call(MakeSdkUpdateMainCmd(args)) + if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except Error as error: - print "Error: %s" % error - sys.exit(1) + sys.exit(main()) diff --git a/native_client_sdk/src/build_tools/sdk_tools/sdk_update_common.py b/native_client_sdk/src/build_tools/sdk_tools/sdk_update_common.py new file mode 100644 index 0000000..7cf0a88 --- /dev/null +++ b/native_client_sdk/src/build_tools/sdk_tools/sdk_update_common.py @@ -0,0 +1,68 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import errno +import os +import shutil +import subprocess +import sys +import time + +"""Utility functions for sdk_update.py and sdk_update_main.py.""" + + +class Error(Exception): + """Generic error/exception for sdk_update module""" + pass + + +def RemoveDir(outdir): + """Removes the given directory + + On Unix systems, this just runs shutil.rmtree, but on Windows, this doesn't + work when the directory contains junctions (as does our SDK installer). + Therefore, on Windows, it runs rmdir /S /Q as a shell command. This always + does the right thing on Windows. If the directory already didn't exist, + RemoveDir will return successfully without taking any action. + + Args: + outdir: The directory to delete + + Raises: + CalledProcessError - if the delete operation fails on Windows + OSError - if the delete operation fails on Linux + """ + + try: + shutil.rmtree(outdir) + except: + if not os.path.exists(outdir): + return + # On Windows this could be an issue with junctions, so try again with rmdir + if sys.platform == 'win32': + subprocess.check_call(['rmdir', '/S', '/Q', outdir], shell=True) + + +def RenameDir(srcdir, destdir): + """Renames srcdir to destdir. Removes destdir before doing the + rename if it already exists.""" + + max_tries = 5 + for num_tries in xrange(max_tries): + try: + RemoveDir(destdir) + os.rename(srcdir, destdir) + return + except OSError as err: + if err.errno != errno.EACCES: + raise err + # If we are here, we didn't exit due to raised exception, so we are + # handling a Windows flaky access error. Sleep one second and try + # again. + time.sleep(num_tries + 1) + # end of while loop -- could not RenameDir + raise Error('Could not RenameDir %s => %s after %d tries.\n' % + 'Please check that no shells or applications ' + 'are accessing files in %s.' + % (srcdir, destdir, num_tries, destdir)) diff --git a/native_client_sdk/src/build_tools/sdk_tools/sdk_update_main.py b/native_client_sdk/src/build_tools/sdk_tools/sdk_update_main.py new file mode 100755 index 0000000..c5351ff --- /dev/null +++ b/native_client_sdk/src/build_tools/sdk_tools/sdk_update_main.py @@ -0,0 +1,667 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +'''A simple tool to update the Native Client SDK to the latest version''' + +import cStringIO +import cygtar +import json +import manifest_util +import optparse +import os +from sdk_update_common import * +import shutil +import subprocess +import sys +import tempfile +from third_party import fancy_urllib +import urllib2 +import urlparse + + +#------------------------------------------------------------------------------ +# Constants + +# This revision number is autogenerated from the Chrome revision. +REVISION = '{REVISION}' + +GLOBAL_HELP = '''Usage: naclsdk [options] command [command_options] + +naclsdk is a simple utility that updates the Native Client (NaCl) +Software Developer's Kit (SDK). Each component is kept as a 'bundle' that +this utility can download as as subdirectory into the SDK. + +Commands: + help [command] - Get either general or command-specific help + list - Lists the available bundles + update/install - Updates/installs bundles in the SDK + sources - Manage external package sources + +Example Usage: + naclsdk list + naclsdk update --force pepper_17 + naclsdk install recommended + naclsdk help update + naclsdk sources --list''' + +CONFIG_FILENAME='naclsdk_config.json' +MANIFEST_FILENAME='naclsdk_manifest2.json' +SDK_TOOLS='sdk_tools' # the name for this tools directory +USER_DATA_DIR='sdk_cache' + +HTTP_CONTENT_LENGTH = 'Content-Length' # HTTP Header field for content length + + +#------------------------------------------------------------------------------ +# General Utilities + + +_debug_mode = False +_quiet_mode = False + + +def DebugPrint(msg): + '''Display a message to stderr if debug printing is enabled + + Note: This function appends a newline to the end of the string + + Args: + msg: A string to send to stderr in debug mode''' + if _debug_mode: + sys.stderr.write("%s\n" % msg) + sys.stderr.flush() + + +def InfoPrint(msg): + '''Display an informational message to stdout if not in quiet mode + + Note: This function appends a newline to the end of the string + + Args: + mgs: A string to send to stdio when not in quiet mode''' + if not _quiet_mode: + sys.stdout.write("%s\n" % msg) + sys.stdout.flush() + + +def WarningPrint(msg): + '''Display an informational message to stderr. + + Note: This function appends a newline to the end of the string + + Args: + mgs: A string to send to stderr.''' + sys.stderr.write("WARNING: %s\n" % msg) + sys.stderr.flush() + + +def UrlOpen(url): + request = fancy_urllib.FancyRequest(url) + ca_certs = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'cacerts.txt') + request.set_ssl_info(ca_certs=ca_certs) + url_opener = urllib2.build_opener( + fancy_urllib.FancyProxyHandler(), + fancy_urllib.FancyRedirectHandler(), + fancy_urllib.FancyHTTPSHandler()) + return url_opener.open(request) + +def ExtractInstaller(installer, outdir): + '''Extract the SDK installer into a given directory + + If the outdir already exists, then this function deletes it + + Args: + installer: full path of the SDK installer + outdir: output directory where to extract the installer + + Raises: + CalledProcessError - if the extract operation fails''' + RemoveDir(outdir) + + if os.path.splitext(installer)[1] == '.exe': + # If the installer has extension 'exe', assume it's a Windows NSIS-style + # installer that handles silent (/S) and relocated (/D) installs. + command = [installer, '/S', '/D=%s' % outdir] + subprocess.check_call(command) + else: + os.mkdir(outdir) + tar_file = None + curpath = os.getcwd() + try: + tar_file = cygtar.CygTar(installer, 'r', verbose=True) + if outdir: os.chdir(outdir) + tar_file.Extract() + finally: + if tar_file: + tar_file.Close() + os.chdir(curpath) + + +class ProgressFunction(object): + '''Create a progress function for a file with a given size''' + + def __init__(self, file_size=0): + '''Constructor + + Args: + file_size: number of bytes in file. 0 indicates unknown''' + self.dots = 0 + self.file_size = int(file_size) + + def GetProgressFunction(self): + '''Returns a progress function based on a known file size''' + def ShowKnownProgress(progress): + if progress == 0: + sys.stdout.write('|%s|\n' % ('=' * 48)) + else: + new_dots = progress * 50 / self.file_size - self.dots + sys.stdout.write('.' * new_dots) + self.dots += new_dots + if progress == self.file_size: + sys.stdout.write('\n') + sys.stdout.flush() + + return ShowKnownProgress + + +def DownloadArchiveToFile(archive, dest_path): + '''Download the archive's data to a file at dest_path. + + As a side effect, computes the sha1 hash and data size, both returned as a + tuple. Raises an Error if the url can't be opened, or an IOError exception if + dest_path can't be opened. + + Args: + dest_path: Path for the file that will receive the data. + Return: + A tuple (sha1, size) with the sha1 hash and data size respectively.''' + sha1 = None + size = 0 + with open(dest_path, 'wb') as to_stream: + from_stream = None + try: + from_stream = UrlOpen(archive.url) + except urllib2.URLError: + raise Error('Cannot open "%s" for archive %s' % + (archive.url, archive.host_os)) + try: + content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH]) + progress_function = ProgressFunction(content_length).GetProgressFunction() + InfoPrint('Downloading %s' % archive.url) + sha1, size = manifest_util.DownloadAndComputeHash( + from_stream, + to_stream=to_stream, + progress_func=progress_function) + if size != content_length: + raise Error('Download size mismatch for %s.\n' + 'Expected %s bytes but got %s' % + (archive.url, content_length, size)) + finally: + if from_stream: from_stream.close() + return sha1, size + + +def LoadFromFile(path, obj): + '''Returns a manifest loaded from the JSON file at |path|. + + If the path does not exist or is invalid, returns unmodified object.''' + methodlist = [m for m in dir(obj) if callable(getattr(obj, m))] + if 'LoadDataFromString' not in methodlist: + return obj + if not os.path.exists(path): + return obj + + with open(path, 'r') as f: + json_string = f.read() + if not json_string: + return obj + + obj.LoadDataFromString(json_string) + return obj + + +def LoadManifestFromURLs(urls): + '''Returns a manifest loaded from |urls|, merged into one manifest.''' + manifest = manifest_util.SDKManifest() + for url in urls: + try: + url_stream = UrlOpen(url) + except urllib2.URLError as e: + raise Error('Unable to open %s. [%s]' % (url, e)) + + manifest_stream = cStringIO.StringIO() + sha1, size = manifest_util.DownloadAndComputeHash(url_stream, + manifest_stream) + temp_manifest = manifest_util.SDKManifest() + temp_manifest.LoadDataFromString(manifest_stream.getvalue()) + + manifest.MergeManifest(temp_manifest) + + def BundleFilter(bundle): + # Only add this bundle if it's supported on this platform. + return bundle.GetHostOSArchive() + + manifest.FilterBundles(BundleFilter) + return manifest + + +def WriteToFile(path, obj): + '''Write |manifest| to a JSON file at |path|.''' + methodlist = [m for m in dir(obj) if callable(getattr(obj, m))] + if 'GetDataAsString' not in methodlist: + raise Error('Unable to write object to file') + json_string = obj.GetDataAsString() + + # Write the JSON data to a temp file. + temp_file_name = None + # TODO(dspringer): Use file locks here so that multiple sdk_updates can + # run at the same time. + with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: + f.write(json_string) + temp_file_name = f.name + # Move the temp file to the actual file. + if os.path.exists(path): + os.remove(path) + shutil.move(temp_file_name, path) + + +class SDKConfig(object): + '''This class contains utilities for manipulating an SDK config + ''' + + def __init__(self): + '''Create a new SDKConfig object with default contents''' + self._data = { + 'sources': [], + } + + def AddSource(self, string): + '''Add a source file to load packages from. + + Args: + string: a URL to an external package manifest file.''' + # For now whitelist only the following location for external sources: + # https://commondatastorage.googleapis.com/nativeclient-mirror/nacl/nacl_sdk + (scheme, host, path, _, _, _) = urlparse.urlparse(string) + if (host != 'commondatastorage.googleapis.com' or + scheme != 'https' or + not path.startswith('/nativeclient-mirror/nacl/nacl_sdk')): + WarningPrint('Only whitelisted sources from ' + '\'https://commondatastorage.googleapis.com/nativeclient-' + 'mirror/nacl/nacl_sdk\' are currently allowed.') + return + if string in self._data['sources']: + WarningPrint('source \''+string+'\' already exists in config.') + return + try: + url_stream = UrlOpen(string) + except urllib2.URLError: + WarningPrint('Unable to fetch manifest URL \'%s\'. Exiting...' % string) + return + + self._data['sources'].append(string) + InfoPrint('source \''+string+'\' added to config.') + + def RemoveSource(self, string): + '''Remove a source file to load packages from. + + Args: + string: a URL to an external SDK manifest file.''' + if string not in self._data['sources']: + WarningPrint('source \''+string+'\' doesn\'t exist in config.') + else: + self._data['sources'].remove(string) + InfoPrint('source \''+string+'\' removed from config.') + + def RemoveAllSources(self): + if len(self.GetSources()) == 0: + InfoPrint('There are no external sources to remove.') + # Copy the list because RemoveSource modifies the underlying list + sources = list(self.GetSources()) + for source in sources: + self.RemoveSource(source) + + + def ListSources(self): + '''List all external sources in config.''' + if len(self._data['sources']): + InfoPrint('Installed sources:') + for s in self._data['sources']: + InfoPrint(' '+s) + else: + InfoPrint('No external sources installed') + + def GetSources(self): + '''Return a list of external sources''' + return self._data['sources'] + + def LoadDataFromString(self, string): + ''' Load a JSON config string. Raises an exception if string + is not well-formed JSON. + + Args: + string: a JSON-formatted string containing the previous config''' + self._data = json.loads(string) + + + def GetDataAsString(self): + '''Returns the current JSON manifest object, pretty-printed''' + pretty_string = json.dumps(self._data, sort_keys=False, indent=2) + # json.dumps sometimes returns trailing whitespace and does not put + # a newline at the end. This code fixes these problems. + pretty_lines = pretty_string.split('\n') + return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n' + + +#------------------------------------------------------------------------------ +# Commands + + +def List(options, argv, config): + '''Usage: %prog [options] list + + Lists the available SDK bundles that are available for download.''' + def PrintBundles(bundles): + for bundle in bundles: + InfoPrint(' %s' % bundle.name) + for key, value in bundle.iteritems(): + if key not in (manifest_util.ARCHIVES_KEY, manifest_util.NAME_KEY): + InfoPrint(' %s: %s' % (key, value)) + + DebugPrint("Running List command with: %s, %s" %(options, argv)) + + parser = optparse.OptionParser(usage=List.__doc__) + (list_options, args) = parser.parse_args(argv) + manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources()) + InfoPrint('Available bundles:') + PrintBundles(manifest.GetBundles()) + # Print the local information. + manifest_path = os.path.join(options.user_data_dir, options.manifest_filename) + local_manifest = LoadFromFile(manifest_path, manifest_util.SDKManifest()) + InfoPrint('\nCurrently installed bundles:') + PrintBundles(local_manifest.GetBundles()) + + +def Update(options, argv, config): + '''Usage: %prog [options] update [target] + + Updates the Native Client SDK to a specified version. By default, this + command updates all the recommended components. The update process works + like this: + 1. Fetch the manifest from the mirror. + 2. Load manifest from USER_DATA_DIR - if there is no local manifest file, + make an empty manifest object. + 3. Update each the bundle: + for bundle in bundles: + # Compare bundle versions & revisions. + # Test if local version.revision < mirror OR local doesn't exist. + if local_manifest < mirror_manifest: + update(bundle) + update local_manifest with mirror_manifest for bundle + write manifest to disk. Use locks. + else: + InfoPrint('bundle is up-to-date') + + Targets: + recommended: (default) Install/Update all recommended components + all: Install/Update all available components + bundle_name: Install/Update only the given bundle + ''' + DebugPrint("Running Update command with: %s, %s" % (options, argv)) + ALL='all' # Update all bundles + RECOMMENDED='recommended' # Only update the bundles with recommended=yes + + parser = optparse.OptionParser(usage=Update.__doc__) + parser.add_option( + '-F', '--force', dest='force', + default=False, action='store_true', + help='Force updating existing components that already exist') + (update_options, args) = parser.parse_args(argv) + + if len(args) == 0: + args = [RECOMMENDED] + + manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources()) + bundles = manifest.GetBundles() + local_manifest_path = os.path.join(options.user_data_dir, + options.manifest_filename) + local_manifest = LoadFromFile(local_manifest_path, + manifest_util.SDKManifest()) + + # Validate the arg list against the available bundle names. Raises an + # error if any invalid bundle names or args are detected. + valid_args = set([ALL, RECOMMENDED] + [bundle.name for bundle in bundles]) + bad_args = set(args) - valid_args + if len(bad_args) > 0: + raise Error("Unrecognized bundle name or argument: '%s'" % + ', '.join(bad_args)) + + for bundle in bundles: + bundle_path = os.path.join(options.sdk_root_dir, bundle.name) + bundle_update_path = '%s_update' % bundle_path + if bundle.name == SDK_TOOLS and not options.update_sdk_tools: + # We only want sdk_tools to updated by sdk_update.py. If the + # user tries to update directly, we just ignore the request. + InfoPrint('Updating sdk_tools happens automatically.\n' + 'Ignoring manual update request.') + continue + + if not (bundle.name in args or + ALL in args or (RECOMMENDED in args and + bundle[RECOMMENDED] == 'yes')): + continue + def UpdateBundle(): + '''Helper to install a bundle''' + archive = bundle.GetHostOSArchive() + (scheme, host, path, _, _, _) = urlparse.urlparse(archive['url']) + dest_filename = os.path.join(options.user_data_dir, path.split('/')[-1]) + sha1, size = DownloadArchiveToFile(archive, dest_filename) + if sha1 != archive.GetChecksum(): + raise Error("SHA1 checksum mismatch on '%s'. Expected %s but got %s" % + (bundle.name, archive.GetChecksum(), sha1)) + if size != archive.size: + raise Error("Size mismatch on Archive. Expected %s but got %s bytes" % + (archive.size, size)) + InfoPrint('Updating bundle %s to version %s, revision %s' % ( + (bundle.name, bundle.version, bundle.revision))) + ExtractInstaller(dest_filename, bundle_update_path) + if bundle.name != SDK_TOOLS: + repath = bundle.get('repath', None) + if repath: + bundle_move_path = os.path.join(bundle_update_path, repath) + else: + bundle_move_path = bundle_update_path + RenameDir(bundle_move_path, bundle_path) + if os.path.exists(bundle_update_path): + RemoveDir(bundle_update_path) + os.remove(dest_filename) + local_manifest.MergeBundle(bundle) + WriteToFile(local_manifest_path, local_manifest) + # Test revision numbers, update the bundle accordingly. + # TODO(dspringer): The local file should be refreshed from disk each + # iteration thought this loop so that multiple sdk_updates can run at the + # same time. + if local_manifest.BundleNeedsUpdate(bundle): + if (not update_options.force and os.path.exists(bundle_path) and + bundle.name != SDK_TOOLS): + WarningPrint('%s already exists, but has an update available.\n' + 'Run update with the --force option to overwrite the ' + 'existing directory.\nWarning: This will overwrite any ' + 'modifications you have made within this directory.' + % bundle.name) + else: + UpdateBundle() + else: + InfoPrint('%s is already up-to-date.' % bundle.name) + +def Sources(options, argv, config): + '''Usage: %prog [options] sources [--list,--add URL,--remove URL] + + Manage additional package sources. URL should point to a valid package + manifest file for download. + ''' + DebugPrint("Running Sources command with: %s, %s" % (options, argv)) + + parser = optparse.OptionParser(usage=Sources.__doc__) + parser.add_option( + '-a', '--add', dest='url_to_add', + default=None, + help='Add additional package source') + parser.add_option( + '-r', '--remove', dest='url_to_remove', + default=None, + help='Remove package source (use \'all\' for all additional sources)') + parser.add_option( + '-l', '--list', dest='do_list', + default=False, action='store_true', + help='List additional package sources') + (source_options, args) = parser.parse_args(argv) + + write_config = False + if source_options.url_to_add: + config.AddSource(source_options.url_to_add) + write_config = True + elif source_options.url_to_remove: + if source_options.url_to_remove == 'all': + config.RemoveAllSources() + else: + config.RemoveSource(source_options.url_to_remove) + write_config = True + elif source_options.do_list: + config.ListSources() + else: + parser.print_help() + + if write_config: + WriteToFile(os.path.join(options.user_data_dir, options.config_filename), + config) + +#------------------------------------------------------------------------------ +# Command-line interface + + +def main(argv): + '''Main entry for the sdk_update utility''' + parser = optparse.OptionParser(usage=GLOBAL_HELP) + DEFAULT_SDK_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + parser.add_option( + '-U', '--manifest-url', dest='manifest_url', + default='https://commondatastorage.googleapis.com/nativeclient-mirror/' + 'nacl/nacl_sdk/%s' % MANIFEST_FILENAME, + help='override the default URL for the NaCl manifest file') + parser.add_option( + '-d', '--debug', dest='debug', + default=False, action='store_true', + help='enable displaying debug information to stderr') + parser.add_option( + '-q', '--quiet', dest='quiet', + default=False, action='store_true', + help='suppress displaying informational prints to stdout') + parser.add_option( + '-u', '--user-data-dir', dest='user_data_dir', + # TODO(mball): the default should probably be in something like + # ~/.naclsdk (linux), or ~/Library/Application Support/NaClSDK (mac), + # or %HOMEPATH%\Application Data\NaClSDK (i.e., %APPDATA% on windows) + default=os.path.join(DEFAULT_SDK_ROOT, USER_DATA_DIR), + help="specify location of NaCl SDK's data directory") + parser.add_option( + '-s', '--sdk-root-dir', dest='sdk_root_dir', + default=DEFAULT_SDK_ROOT, + help="location where the SDK bundles are installed") + parser.add_option( + '-v', '--version', dest='show_version', + action='store_true', + help='show version information and exit') + parser.add_option( + '-m', '--manifest', dest='manifest_filename', + default=MANIFEST_FILENAME, + help="name of local manifest file relative to user-data-dir") + parser.add_option( + '-c', '--config', dest='config_filename', + default=CONFIG_FILENAME, + help="name of the local config file relative to user-data-dir") + parser.add_option( + '--update-sdk-tools', dest='update_sdk_tools', + default=False, action='store_true') + + + COMMANDS = { + 'list': List, + 'update': Update, + 'install': Update, + 'sources': Sources, + } + + # Separate global options from command-specific options + global_argv = argv + command_argv = [] + for index, arg in enumerate(argv): + if arg in COMMANDS: + global_argv = argv[:index] + command_argv = argv[index:] + break + + (options, args) = parser.parse_args(global_argv) + args += command_argv + + global _debug_mode, _quiet_mode + _debug_mode = options.debug + _quiet_mode = options.quiet + + def PrintHelpAndExit(unused_options=None, unused_args=None): + parser.print_help() + exit(1) + + if options.update_sdk_tools: + # Ignore all other commands, and just update the sdk tools. + args = ['update', 'sdk_tools'] + # Leave the rest of the options alone -- they may be needed to update + # correctly. + options.show_version = False + options.sdk_root_dir = DEFAULT_SDK_ROOT + + if options.show_version: + print "Native Client SDK Updater, version r%s" % (REVISION,) + exit(0) + + + if not args: + print "Need to supply a command" + PrintHelpAndExit() + + def DefaultHandler(unused_options=None, unused_args=None, unused_config=None): + print "Unknown Command: %s" % args[0] + PrintHelpAndExit() + + def InvokeCommand(args): + command = COMMANDS.get(args[0], DefaultHandler) + # Load the config file before running commands + config = LoadFromFile(os.path.join(options.user_data_dir, + options.config_filename), + SDKConfig()) + command(options, args[1:], config) + + if args[0] == 'help': + if len(args) == 1: + PrintHelpAndExit() + else: + InvokeCommand([args[1], '-h']) + else: + # Make sure the user_data_dir exists. + if not os.path.exists(options.user_data_dir): + os.makedirs(options.user_data_dir) + InvokeCommand(args) + + return 0 # Success + + +if __name__ == '__main__': + try: + sys.exit(main(sys.argv[1:])) + except Error as error: + print "Error: %s" % error + sys.exit(1) diff --git a/native_client_sdk/src/build_tools/tests/__init__.py b/native_client_sdk/src/build_tools/tests/__init__.py new file mode 100644 index 0000000..4e68b4f --- /dev/null +++ b/native_client_sdk/src/build_tools/tests/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""tests package.""" diff --git a/native_client_sdk/src/build_tools/tests/test_auto_update_sdktools.py b/native_client_sdk/src/build_tools/tests/test_auto_update_sdktools.py new file mode 100755 index 0000000..cc2faba --- /dev/null +++ b/native_client_sdk/src/build_tools/tests/test_auto_update_sdktools.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# copyright (c) 2012 the chromium authors. all rights reserved. +# use of this source code is governed by a bsd-style license that can be +# found in the license file. + +import os +import re +import shutil +import subprocess +import sys +import tempfile +import test_server +import unittest + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +BUILD_TOOLS_DIR = os.path.dirname(SCRIPT_DIR) + +sys.path.append(BUILD_TOOLS_DIR) +import build_utils +import build_updater +import manifest_util + + +MANIFEST_BASENAME = 'naclsdk_manifest2.json' + + +class TestAutoUpdateSdkTools(unittest.TestCase): + def setUp(self): + self.basedir = tempfile.mkdtemp() + build_updater.BuildUpdater(self.basedir) + self._LoadCacheManifest() + self.current_revision = int(build_utils.ChromeRevision()) + self.server = test_server.LocalHTTPServer(self.basedir) + + def tearDown(self): + if self.server: + self.server.Shutdown() + shutil.rmtree(self.basedir) + + def _LoadCacheManifest(self): + """Read the manifest from nacl_sdk/sdk_cache. + + This manifest should only contain the sdk_tools bundle. + """ + manifest_filename = os.path.join(self.basedir, 'nacl_sdk', 'sdk_cache', + MANIFEST_BASENAME) + self.manifest = manifest_util.SDKManifest() + self.manifest.LoadDataFromString(open(manifest_filename, 'r').read()) + self.sdk_tools_bundle = self.manifest.GetBundle('sdk_tools') + + def _WriteManifest(self): + with open(os.path.join(self.basedir, MANIFEST_BASENAME), 'w') as stream: + stream.write(self.manifest.GetDataAsString()) + + def _BuildUpdaterArchive(self, rel_path, revision): + """Build a new sdk_tools bundle. + + Args: + rel_path: The relative path to build the updater. + revision: The revision number to give to this bundle. + Returns: + A manifest_util.Archive() that points to this new bundle on the local + server. + """ + build_updater.BuildUpdater(os.path.join(self.basedir, rel_path), revision) + + new_sdk_tools_tgz = os.path.join(self.basedir, rel_path, 'sdk_tools.tgz') + with open(new_sdk_tools_tgz, 'r') as sdk_tools_stream: + archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( + sdk_tools_stream) + + archive = manifest_util.Archive('all') + archive.url = self.server.GetURL('%s/sdk_tools.tgz' % (rel_path,)) + archive.checksum = archive_sha1 + archive.size = archive_size + return archive + + def _Run(self, args): + naclsdk_shell_script = os.path.join(self.basedir, 'nacl_sdk', 'naclsdk') + cmd = [naclsdk_shell_script, '-U', self.server.GetURL(MANIFEST_BASENAME)] + cmd.extend(args) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + stdout, _ = process.communicate() + self.assertEqual(process.returncode, 0) + return stdout + + def _RunAndExtractRevision(self): + stdout = self._Run(['-v']) + match = re.search('version r(\d+)', stdout) + self.assertTrue(match is not None) + return int(match.group(1)) + + def testNoUpdate(self): + """If the current revision is the newest, the shell script will run + normally. + """ + self._WriteManifest() + revision = self._RunAndExtractRevision() + self.assertEqual(revision, self.current_revision) + + def testUpdate(self): + """Create a new bundle with a bumped revision number. + When we run the shell script, we should see the new revision number. + """ + new_revision = self.current_revision + 1 + archive = self._BuildUpdaterArchive('new', new_revision) + self.sdk_tools_bundle.AddArchive(archive) + self.sdk_tools_bundle.revision = new_revision + self._WriteManifest() + + revision = self._RunAndExtractRevision() + self.assertEqual(revision, new_revision) + + def testManualUpdateIsIgnored(self): + """If the sdk_tools bundle was updated normally (i.e. the old way), it would + leave a sdk_tools_update folder that would then be copied over on a + subsequent run. This test ensures that there is no folder made. + """ + new_revision = self.current_revision + 1 + archive = self._BuildUpdaterArchive('new', new_revision) + self.sdk_tools_bundle.AddArchive(archive) + self.sdk_tools_bundle.revision = new_revision + self._WriteManifest() + + stdout = self._Run(['update', 'sdk_tools']) + self.assertTrue(stdout.find('Ignoring manual update request.') != -1) + sdk_tools_update_dir = os.path.join(self.basedir, 'nacl_sdk', + 'sdk_tools_update') + self.assertFalse(os.path.exists(sdk_tools_update_dir)) + + +def main(): + suite = unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__]) + result = unittest.TextTestRunner(verbosity=2).run(suite) + + return int(not result.wasSuccessful()) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/native_client_sdk/src/build_tools/tests/test_server.py b/native_client_sdk/src/build_tools/tests/test_server.py new file mode 100644 index 0000000..bee9fc9 --- /dev/null +++ b/native_client_sdk/src/build_tools/tests/test_server.py @@ -0,0 +1,65 @@ +# copyright (c) 2012 the chromium authors. all rights reserved. +# use of this source code is governed by a bsd-style license that can be +# found in the license file. + +import multiprocessing +import os + + +class LocalHTTPServer(object): + """Class to start a local HTTP server as a child process.""" + + def __init__(self, serve_dir): + parent_conn, child_conn = multiprocessing.Pipe() + self.process = multiprocessing.Process(target=_HTTPServerProcess, + args=(child_conn, serve_dir)) + self.process.start() + if parent_conn.poll(10): # wait 10 seconds + self.port = parent_conn.recv() + else: + raise Exception('Unable to launch HTTP server.') + + self.conn = parent_conn + + def Shutdown(self): + """Send a message to the child HTTP server process and wait for it to + finish.""" + self.conn.send(False) + self.process.join() + + def GetURL(self, rel_url): + """Get the full url for a file on the local HTTP server. + + Args: + rel_url: A URL fragment to convert to a full URL. For example, + GetURL('foobar.baz') -> 'http://localhost:1234/foobar.baz' + """ + return 'http://localhost:%d/%s' % (self.port, rel_url) + + +def _HTTPServerProcess(conn, serve_dir): + """Run a local httpserver with a randomly-chosen port. + + This function assumes it is run as a child process using multiprocessing. + + Args: + conn: A connection to the parent process. The child process sends + the local port, and waits for a message from the parent to + stop serving. + serve_dir: The directory to serve. All files are accessible through + http://localhost:<port>/path/to/filename. + """ + import BaseHTTPServer + import SimpleHTTPServer + + os.chdir(serve_dir) + httpd = BaseHTTPServer.HTTPServer(('', 0), + SimpleHTTPServer.SimpleHTTPRequestHandler) + conn.send(httpd.server_address[1]) # the chosen port number + httpd.timeout = 0.5 # seconds + running = True + while running: + httpd.handle_request() + if conn.poll(): + running = conn.recv() + conn.close() |