diff options
author | binji@chromium.org <binji@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-04-25 20:12:17 +0000 |
---|---|---|
committer | binji@chromium.org <binji@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-04-25 20:12:17 +0000 |
commit | c11498c42e47b46875a8f821c42221ad18a3b990 (patch) | |
tree | 112c42726d886e70392af9c2cc1b6b0842d67759 /native_client_sdk/src/build_tools | |
parent | 0c30f204c08416d146c19a3f72e45f4b216092ac (diff) | |
download | chromium_src-c11498c42e47b46875a8f821c42221ad18a3b990.zip chromium_src-c11498c42e47b46875a8f821c42221ad18a3b990.tar.gz chromium_src-c11498c42e47b46875a8f821c42221ad18a3b990.tar.bz2 |
[NaCl SDK] Change build script to test sdk_updater, also push manifest snippets per bot.
BUG=none
TEST=none
Review URL: https://chromiumcodereview.appspot.com/10065030
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@133963 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'native_client_sdk/src/build_tools')
-rwxr-xr-x | native_client_sdk/src/build_tools/build_sdk.py | 219 | ||||
-rw-r--r-- | native_client_sdk/src/build_tools/buildbot_common.py | 7 | ||||
-rw-r--r-- | native_client_sdk/src/build_tools/manifest_util.py | 92 | ||||
-rwxr-xr-x | native_client_sdk/src/build_tools/sdk_tools/sdk_update.py | 136 |
4 files changed, 300 insertions, 154 deletions
diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py index a0df2a1..8894c4c 100755 --- a/native_client_sdk/src/build_tools/build_sdk.py +++ b/native_client_sdk/src/build_tools/build_sdk.py @@ -17,15 +17,18 @@ and whether it should upload an SDK to file storage (GSTORE) # std python includes +import multiprocessing import optparse import os import platform +import subprocess import sys # local includes import buildbot_common import build_utils import lastchange +import manifest_util # Create the various paths of interest SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -36,6 +39,7 @@ SRC_DIR = os.path.dirname(SDK_DIR) NACL_DIR = os.path.join(SRC_DIR, 'native_client') OUT_DIR = os.path.join(SRC_DIR, 'out') PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi') +SERVER_DIR = os.path.join(OUT_DIR, 'local_server') # Add SDK make tools scripts to the python path. @@ -51,17 +55,76 @@ MAKE = 'nacl_sdk/make_3_81/make.exe' CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py') +def HTTPServerProcess(conn, serve_dir): + """Run a local httpserver with a randomly-chosen port. + + This function assumes it is run as a child process using multiprocessing. + + Args: + conn: A connection to the parent process. The child process sends + the local port, and waits for a message from the parent to + stop serving. + serve_dir: The directory to serve. All files are accessible through + http://localhost:<port>/path/to/filename. + """ + import BaseHTTPServer + import SimpleHTTPServer + + os.chdir(serve_dir) + httpd = BaseHTTPServer.HTTPServer(('', 0), + SimpleHTTPServer.SimpleHTTPRequestHandler) + conn.send(httpd.server_address[1]) # the chosen port number + httpd.timeout = 0.5 # seconds + running = True + while running: + httpd.handle_request() + if conn.poll(): + running = conn.recv() + conn.close() + + +class LocalHTTPServer(object): + """Class to start a local HTTP server as a child process.""" + + def __init__(self, serve_dir): + parent_conn, child_conn = multiprocessing.Pipe() + self.process = multiprocessing.Process(target=HTTPServerProcess, + args=(child_conn, serve_dir)) + self.process.start() + if parent_conn.poll(10): # wait 10 seconds + self.port = parent_conn.recv() + else: + raise Exception('Unable to launch HTTP server.') + + self.conn = parent_conn + + def Shutdown(self): + """Send a message to the child HTTP server process and wait for it to + finish.""" + self.conn.send(False) + self.process.join() + + def GetURL(self, rel_url): + """Get the full url for a file on the local HTTP server. + + Args: + rel_url: A URL fragment to convert to a full URL. For example, + GetURL('foobar.baz') -> 'http://localhost:1234/foobar.baz' + """ + return 'http://localhost:%d/%s' % (self.port, rel_url) + + def AddMakeBat(pepperdir, makepath): """Create a simple batch file to execute Make. - + Creates a simple batch file named make.bat for the Windows platform at the given path, pointing to the Make executable in the SDK.""" - + makepath = os.path.abspath(makepath) if not makepath.startswith(pepperdir): buildbot_common.ErrorExit('Make.bat not relative to Pepper directory: ' + makepath) - + makeexe = os.path.abspath(os.path.join(pepperdir, 'tools')) relpath = os.path.relpath(makeexe, makepath) @@ -202,7 +265,7 @@ def InstallHeaders(tc_dst_inc, pepper_ver, tc_name): buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'c', 'dev', '*.h'), os.path.join(ppapi, 'c', 'dev')) - # Run the generator to overwrite IDL files + # Run the generator to overwrite IDL files buildbot_common.Run([sys.executable, 'generator.py', '--wnone', '--cgen', '--release=M' + pepper_ver, '--verbose', '--dstroot=%s/c' % ppapi], cwd=os.path.join(PPAPI_DIR, 'generators')) @@ -259,7 +322,7 @@ def UntarToolchains(pepperdir, platform, arch, toolchains): tarfile = GetNewlibToolchain(platform, arch) buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], cwd=NACL_DIR) - + # Then rename/move it to the pepper toolchain directory srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk') newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib') @@ -270,7 +333,7 @@ def UntarToolchains(pepperdir, platform, arch, toolchains): tarfile = GetGlibcToolchain(platform, arch) buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile], cwd=NACL_DIR) - + # Then rename/move it to the pepper toolchain directory srcdir = os.path.join(tmpdir, 'toolchain', tcname) glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc') @@ -349,7 +412,7 @@ EXAMPLE_MAP = { 'fullscreen_tumbler', 'gamepad', 'geturl', - 'hello_world_interactive', + 'hello_world_interactive', 'hello_world_newlib', 'input_events', 'load_progress', @@ -458,17 +521,17 @@ def main(args): parser = optparse.OptionParser() parser.add_option('--pnacl', help='Enable pnacl build.', action='store_true', dest='pnacl', default=False) - parser.add_option('--examples', help='Rebuild the examples.', - action='store_true', dest='examples', default=False) - parser.add_option('--update', help='Rebuild the updater.', - action='store_true', dest='update', default=False) + parser.add_option('--examples', help='Only build the examples.', + action='store_true', dest='only_examples', default=False) + parser.add_option('--update', help='Only build the updater.', + action='store_true', dest='only_updater', default=False) parser.add_option('--skip-tar', help='Skip generating a tarball.', action='store_true', dest='skip_tar', default=False) parser.add_option('--archive', help='Force the archive step.', action='store_true', dest='archive', default=False) parser.add_option('--release', help='PPAPI release version.', dest='release', default=None) - + options, args = parser.parse_args(args[1:]) platform = getos.GetPlatform() arch = 'x86' @@ -482,19 +545,16 @@ def main(args): else: toolchains = ['newlib', 'glibc'] print 'Building: ' + ' '.join(toolchains) - skip = options.examples or options.update + skip = options.only_examples or options.only_updater - skip_examples = skip - skip_update = skip + skip_examples = skip and not options.only_examples + skip_update = skip and not options.only_updater skip_untar = skip skip_build = skip + skip_test_updater = skip skip_tar = skip or options.skip_tar - - if options.examples: skip_examples = False - skip_update = not options.update - - if options.archive and (options.examples or options.skip_tar): + if options.archive and (options.only_examples or options.skip_tar): parser.error('Incompatible arguments with archive.') pepper_ver = str(int(build_utils.ChromeMajorVersion())) @@ -508,17 +568,18 @@ def main(args): buildbot_common.Run(['gclient', 'runhooks'], cwd=SRC_DIR, shell=(platform=='win')) - buildbot_common.BuildStep('Clean Pepper Dir') pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver) if not skip_untar: + buildbot_common.BuildStep('Clean Pepper Dir') buildbot_common.RemoveDir(pepperdir) buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain')) buildbot_common.MakeDir(os.path.join(pepperdir, 'tools')) - buildbot_common.BuildStep('Add Text Files') - files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README'] - files = [os.path.join(SDK_SRC_DIR, filename) for filename in files] - oshelpers.Copy(['-v'] + files + [pepperdir]) + if not skip_build: + buildbot_common.BuildStep('Add Text Files') + files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README'] + files = [os.path.join(SDK_SRC_DIR, filename) for filename in files] + oshelpers.Copy(['-v'] + files + [pepperdir]) # Clean out the temporary toolchain untar directory @@ -528,33 +589,85 @@ def main(args): if not skip_build: BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains) - buildbot_common.BuildStep('Copy make OS helpers') - buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), - os.path.join(pepperdir, 'tools')) - if platform == 'win': - buildbot_common.BuildStep('Add MAKE') - http_download.HttpDownload(GSTORE + MAKE, - os.path.join(pepperdir, 'tools' ,'make.exe')) + if not skip_build: + buildbot_common.BuildStep('Copy make OS helpers') + buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), + os.path.join(pepperdir, 'tools')) + if platform == 'win': + buildbot_common.BuildStep('Add MAKE') + http_download.HttpDownload(GSTORE + MAKE, + os.path.join(pepperdir, 'tools' ,'make.exe')) if not skip_examples: CopyExamples(pepperdir, toolchains) + tarname = 'naclsdk_' + platform + '.bz2' + if 'pnacl' in toolchains: + tarname = 'p' + tarname + tarfile = os.path.join(OUT_DIR, tarname) + if not skip_tar: buildbot_common.BuildStep('Tar Pepper Bundle') - tarname = 'naclsdk_' + platform + '.bz2' - if 'pnacl' in toolchains: - tarname = 'p' + tarname - tarfile = os.path.join(OUT_DIR, tarname) buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile, 'pepper_' + pepper_ver], cwd=NACL_DIR) - # Archive on non-trybots. - if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''): - buildbot_common.BuildStep('Archive build') - buildbot_common.Archive(tarname, - 'nativeclient-mirror/nacl/nacl_sdk/%s' % build_utils.ChromeVersion(), - OUT_DIR) + # build sdk update + if not skip_update: + BuildUpdater() + # start local server sharing a manifest + the new bundle + if not skip_test_updater: + buildbot_common.BuildStep('Move bundle to localserver dir') + buildbot_common.MakeDir(SERVER_DIR) + buildbot_common.Move(tarfile, SERVER_DIR) + tarfile = os.path.join(SERVER_DIR, tarname) + + server = None + try: + buildbot_common.BuildStep('Run local server') + server = LocalHTTPServer(SERVER_DIR) + + buildbot_common.BuildStep('Generate manifest') + with open(tarfile, 'rb') as tarfile_stream: + archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( + tarfile_stream) + archive = manifest_util.Archive(manifest_util.GetHostOS()) + archive.CopyFrom({'url': server.GetURL(tarname), + 'size': archive_size, + 'checksum': {'sha1': archive_sha1}}) + bundle = manifest_util.Bundle('pepper_' + pepper_ver) + bundle.CopyFrom({ + 'revision': clnumber, + 'repath': 'pepper_' + pepper_ver, + 'version': pepper_ver, + 'description': 'Chrome %s bundle, revision %s' % ( + pepper_ver, clnumber), + 'stability': 'dev', + 'recommended': 'no', + 'archives': [archive]}) + manifest = manifest_util.SDKManifest() + manifest.SetBundle(bundle) + manifest_name = 'naclsdk_manifest2.json' + with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \ + manifest_stream: + manifest_stream.write(manifest.GetManifestString()) + + # use newly built sdk updater to pull this bundle + buildbot_common.BuildStep('Update from local server') + updater_py = os.path.join(OUT_DIR, 'nacl_sdk', 'sdk_tools', + 'sdk_update.py') + buildbot_common.Run([sys.executable, updater_py, '-U', + server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver]) + + # If we are testing examples, do it in the newly pulled directory. + pepperdir = os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver) + + # kill server + finally: + if server: + server.Shutdown() + + # build examples. if not skip_examples: buildbot_common.BuildStep('Test Build Examples') filelist = os.listdir(os.path.join(pepperdir, 'examples')) @@ -566,9 +679,25 @@ def main(args): buildbot_common.Run(['make', 'all', '-j8'], cwd=os.path.abspath(dirnode), shell=True) - # Build SDK Tools - if not skip_update: - BuildUpdater() + # Archive on non-trybots. + if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''): + buildbot_common.BuildStep('Archive build') + bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \ + build_utils.ChromeVersion() + buildbot_common.Archive(tarname, bucket_path, OUT_DIR) + + # generate "manifest snippet" for this archive. + if not skip_test_updater: + archive = bundle.GetArchive(manifest_util.GetHostOS()) + archive.url = 'https://commondatastorage.googleapis.com/' \ + 'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % ( + build_utils.ChromeVersion(), tarname) + manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json') + with open(manifest_snippet_file, 'wb') as manifest_snippet_stream: + manifest_snippet_stream.write(bundle.ToJSON()) + + buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR, + step_link=False) return 0 diff --git a/native_client_sdk/src/build_tools/buildbot_common.py b/native_client_sdk/src/build_tools/buildbot_common.py index ea978e1..bdf0aea 100644 --- a/native_client_sdk/src/build_tools/buildbot_common.py +++ b/native_client_sdk/src/build_tools/buildbot_common.py @@ -96,7 +96,7 @@ def GetGsutil(): return LOCAL_GSUTIL -def Archive(filename, bucket_path, cwd=None): +def Archive(filename, bucket_path, cwd=None, step_link=True): """Upload the given filename to Google Store.""" full_dst = 'gs://%s/%s' % (bucket_path, filename) @@ -106,5 +106,6 @@ def Archive(filename, bucket_path, cwd=None): cwd=cwd) url = 'https://commondatastorage.googleapis.com/'\ '%s/%s' % (bucket_path, filename) - print '@@@STEP_LINK@download@%s@@@' % url - sys.stdout.flush() + if step_link: + print '@@@STEP_LINK@download@%s@@@' % url + sys.stdout.flush() diff --git a/native_client_sdk/src/build_tools/manifest_util.py b/native_client_sdk/src/build_tools/manifest_util.py index c39597c..0bdafc3 100644 --- a/native_client_sdk/src/build_tools/manifest_util.py +++ b/native_client_sdk/src/build_tools/manifest_util.py @@ -2,7 +2,9 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import hashlib import json +import sys MANIFEST_VERSION = 2 @@ -33,6 +35,64 @@ VALID_BUNDLES_KEYS = frozenset([ VALID_MANIFEST_KEYS = frozenset(['manifest_version', BUNDLES_KEY]) +def GetHostOS(): + '''Returns the host_os value that corresponds to the current host OS''' + return { + 'linux2': 'linux', + 'darwin': 'mac', + 'cygwin': 'win', + 'win32': 'win' + }[sys.platform] + + +def DictToJSON(dict): + """Convert a dict to a JSON-formatted string.""" + pretty_string = json.dumps(dict, sort_keys=False, indent=2) + # json.dumps sometimes returns trailing whitespace and does not put + # a newline at the end. This code fixes these problems. + pretty_lines = pretty_string.split('\n') + return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n' + + +def DownloadAndComputeHash(from_stream, to_stream=None, progress_func=None): + ''' Download the archive data from from-stream and generate sha1 and + size info. + + Args: + from_stream: An input stream that supports read. + to_stream: [optional] the data is written to to_stream if it is + provided. + progress_func: [optional] A function used to report download progress. If + provided, progress_func is called with progress=0 at the + beginning of the download, periodically with progress=1 + during the download, and progress=100 at the end. + + Return + A tuple (sha1, size) where sha1 is a sha1-hash for the archive data and + size is the size of the archive data in bytes.''' + # Use a no-op progress function if none is specified. + def progress_no_op(progress): + pass + if not progress_func: + progress_func = progress_no_op + + sha1_hash = hashlib.sha1() + size = 0 + progress_func(progress=0) + while(1): + data = from_stream.read(32768) + if not data: + break + sha1_hash.update(data) + size += len(data) + if to_stream: + to_stream.write(data) + progress_func(size) + + progress_func(progress=100) + return sha1_hash.hexdigest(), size + + class Error(Exception): """Generic error/exception for manifest_util module""" pass @@ -69,6 +129,15 @@ class Archive(dict): host_os = 'all (default)' if not self.get('url', None): raise Error('Archive "%s" has no URL' % host_os) + if not self.get('size', None): + raise Error('Archive "%s" has no size' % host_os) + checksum = self.get('checksum', None) + if not checksum: + raise Error('Archive "%s" has no checksum' % host_os) + elif not isinstance(checksum, dict): + raise Error('Archive "%s" has a checksum, but it is not a dict' % host_os) + elif not len(checksum): + raise Error('Archive "%s" has an empty checksum dict' % host_os) # Verify that all key names are valid. for key, val in self.iteritems(): if key not in VALID_ARCHIVE_KEYS: @@ -79,6 +148,11 @@ class Archive(dict): """Returns the URL of this archive""" return self['url'] + @url.setter + def url(self, url): + """Set the URL of this archive""" + self['url'] = url + @property def size(self): """Returns the size of this archive, in bytes""" @@ -123,6 +197,14 @@ class Bundle(dict): """ return Bundle(self.items() + bundle.items()) + def ToJSON(self): + """Convert this bundle to a JSON-formatted string.""" + return DictToJSON(self) + + def FromJSON(self, json_string): + """Parse and load bundle data from a JSON-formatted string.""" + self.CopyFrom(json.loads(json_string)) + def CopyFrom(self, dict): """Update the content of the bundle by copying values from the given dictionary. @@ -186,6 +268,10 @@ class Bundle(dict): return archive return None + def GetHostOSArchive(self): + """Retrieve the archive for the current host os.""" + return self.GetArchive(GetHostOS()) + def GetArchives(self): """Returns all the archives in this bundle""" return self[ARCHIVES_KEY] @@ -345,8 +431,4 @@ class SDKManifest(object): def GetManifestString(self): """Returns the current JSON manifest object, pretty-printed""" - pretty_string = json.dumps(self._manifest_data, sort_keys=False, indent=2) - # json.dumps sometimes returns trailing whitespace and does not put - # a newline at the end. This code fixes these problems. - pretty_lines = pretty_string.split('\n') - return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n' + return DictToJSON(self._manifest_data) diff --git a/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py b/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py index 878e21e..ba06afe 100755 --- a/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py +++ b/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py @@ -103,15 +103,6 @@ class Error(Exception): pass -def GetHostOS(): - '''Returns the host_os value that corresponds to the current host OS''' - return { - 'linux2': 'linux', - 'darwin': 'mac', - 'cygwin': 'win', - 'win32': 'win' - }[sys.platform] - def UrlOpen(url): request = fancy_urllib.FancyRequest(url) ca_certs = os.path.join(os.path.dirname(os.path.abspath(__file__)), @@ -207,23 +198,6 @@ def RenameDir(srcdir, destdir): % (srcdir, destdir, num_tries, destdir)) -def ShowProgress(progress): - ''' A download-progress function used by class Archive. - (See DownloadAndComputeHash).''' - global count # A divider, so we don't emit dots too often. - - if progress == 0: - count = 0 - elif progress == 100: - sys.stdout.write('\n') - else: - count = count + 1 - if count > 10: - sys.stdout.write('.') - sys.stdout.flush() - count = 0 - - class ProgressFunction(object): '''Create a progress function for a file with a given size''' @@ -251,43 +225,41 @@ class ProgressFunction(object): return ShowKnownProgress -def DownloadAndComputeHash(from_stream, to_stream=None, progress_func=None): - ''' Download the archive data from from-stream and generate sha1 and - size info. +def DownloadArchiveToFile(archive, dest_path): + '''Download the archive's data to a file at dest_path. + + As a side effect, computes the sha1 hash and data size, both returned as a + tuple. Raises an Error if the url can't be opened, or an IOError exception if + dest_path can't be opened. Args: - from_stream: An input stream that supports read. - to_stream: [optional] the data is written to to_stream if it is - provided. - progress_func: [optional] A function used to report download progress. If - provided, progress_func is called with progress=0 at the - beginning of the download, periodically with progress=1 - during the download, and progress=100 at the end. - - Return - A tuple (sha1, size) where sha1 is a sha1-hash for the archive data and - size is the size of the archive data in bytes.''' - # Use a no-op progress function if none is specified. - def progress_no_op(progress): - pass - if not progress_func: - progress_func = progress_no_op - - sha1_hash = hashlib.sha1() + dest_path: Path for the file that will receive the data. + Return: + A tuple (sha1, size) with the sha1 hash and data size respectively.''' + sha1 = None size = 0 - progress_func(progress=0) - while(1): - data = from_stream.read(32768) - if not data: - break - sha1_hash.update(data) - size += len(data) - if to_stream: - to_stream.write(data) - progress_func(size) - - progress_func(progress=100) - return sha1_hash.hexdigest(), size + with open(dest_path, 'wb') as to_stream: + from_stream = None + try: + from_stream = UrlOpen(archive.url) + except urllib2.URLError: + raise Error('Cannot open "%s" for archive %s' % + (archive.url, archive.host_os)) + try: + content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH]) + progress_function = ProgressFunction(content_length).GetProgressFunction() + InfoPrint('Downloading %s' % archive.url) + sha1, size = manifest_util.DownloadAndComputeHash( + from_stream, + to_stream=to_stream, + progress_func=progress_function) + if size != content_length: + raise Error('Download size mismatch for %s.\n' + 'Expected %s bytes but got %s' % + (archive.url, content_length, size)) + finally: + if from_stream: from_stream.close() + return sha1, size def LoadManifestFromFile(path): @@ -315,14 +287,13 @@ def LoadManifestFromURL(url): raise Error('Unable to open %s. [%s]' % (url, e)) manifest_stream = cStringIO.StringIO() - sha1, size = DownloadAndComputeHash( - url_stream, manifest_stream) + sha1, size = manifest_util.DownloadAndComputeHash(url_stream, manifest_stream) manifest = manifest_util.SDKManifest() manifest.LoadManifestString(manifest_stream.getvalue()) def BundleFilter(bundle): # Only add this bundle if it's supported on this platform. - return bundle.GetArchive(GetHostOS()) + return bundle.GetHostOSArchive() manifest.FilterBundles(BundleFilter) return manifest @@ -345,43 +316,6 @@ def WriteManifestToFile(manifest, path): shutil.move(temp_file_name, path) -def DownloadArchiveToFile(archive, dest_path): - '''Download the archive's data to a file at dest_path. - - As a side effect, computes the sha1 hash and data size, both returned as a - tuple. Raises an Error if the url can't be opened, or an IOError exception if - dest_path can't be opened. - - Args: - dest_path: Path for the file that will receive the data. - Return: - A tuple (sha1, size) with the sha1 hash and data size respectively.''' - sha1 = None - size = 0 - with open(dest_path, 'wb') as to_stream: - from_stream = None - try: - from_stream = UrlOpen(archive.url) - except urllib2.URLError: - raise Error('Cannot open "%s" for archive %s' % - (archive.url, archive.host_os)) - try: - content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH]) - progress_function = ProgressFunction(content_length).GetProgressFunction() - InfoPrint('Downloading %s' % archive.url) - sha1, size = DownloadAndComputeHash( - from_stream, - to_stream=to_stream, - progress_func=progress_function) - if size != content_length: - raise Error('Download size mismatch for %s.\n' - 'Expected %s bytes but got %s' % - (archive.url, content_length, size)) - finally: - if from_stream: from_stream.close() - return sha1, size - - #------------------------------------------------------------------------------ # Commands @@ -471,7 +405,7 @@ def Update(options, argv): continue def UpdateBundle(): '''Helper to install a bundle''' - archive = bundle.GetArchive(GetHostOS()) + archive = bundle.GetHostOSArchive() (scheme, host, path, _, _, _) = urlparse.urlparse(archive['url']) dest_filename = os.path.join(options.user_data_dir, path.split('/')[-1]) sha1, size = DownloadArchiveToFile(archive, dest_filename) |