[NaCl SDK] Change build script to test sdk_updater, also push manifest snippets per bot.

BUG=none
TEST=none

Review URL: https://chromiumcodereview.appspot.com/10065030

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@133963 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 300 insertions, 154 deletions

diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py
index a0df2a1..8894c4c 100755
--- a/native_client_sdk/src/build_tools/build_sdk.py
+++ b/native_client_sdk/src/build_tools/build_sdk.py
@@ -17,15 +17,18 @@ and whether it should upload an SDK to file storage (GSTORE)
 
 
 # std python includes
+import multiprocessing
 import optparse
 import os
 import platform
+import subprocess
 import sys
 
 # local includes
 import buildbot_common
 import build_utils
 import lastchange
+import manifest_util
 
 # Create the various paths of interest
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -36,6 +39,7 @@ SRC_DIR = os.path.dirname(SDK_DIR)
 NACL_DIR = os.path.join(SRC_DIR, 'native_client')
 OUT_DIR = os.path.join(SRC_DIR, 'out')
 PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
+SERVER_DIR = os.path.join(OUT_DIR, 'local_server')
 
 
 # Add SDK make tools scripts to the python path.
@@ -51,17 +55,76 @@ MAKE = 'nacl_sdk/make_3_81/make.exe'
 CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py')
 
 
+def HTTPServerProcess(conn, serve_dir):
+  """Run a local httpserver with a randomly-chosen port.
+
+  This function assumes it is run as a child process using multiprocessing.
+
+  Args:
+    conn: A connection to the parent process. The child process sends
+        the local port, and waits for a message from the parent to
+        stop serving.
+    serve_dir: The directory to serve. All files are accessible through
+       http://localhost:<port>/path/to/filename.
+  """
+  import BaseHTTPServer
+  import SimpleHTTPServer
+
+  os.chdir(serve_dir)
+  httpd = BaseHTTPServer.HTTPServer(('', 0),
+                                    SimpleHTTPServer.SimpleHTTPRequestHandler)
+  conn.send(httpd.server_address[1])  # the chosen port number
+  httpd.timeout = 0.5  # seconds
+  running = True
+  while running:
+    httpd.handle_request()
+    if conn.poll():
+      running = conn.recv()
+  conn.close()
+
+
+class LocalHTTPServer(object):
+  """Class to start a local HTTP server as a child process."""
+
+  def __init__(self, serve_dir):
+    parent_conn, child_conn = multiprocessing.Pipe()
+    self.process = multiprocessing.Process(target=HTTPServerProcess,
+                                           args=(child_conn, serve_dir))
+    self.process.start()
+    if parent_conn.poll(10):  # wait 10 seconds
+      self.port = parent_conn.recv()
+    else:
+      raise Exception('Unable to launch HTTP server.')
+
+    self.conn = parent_conn
+
+  def Shutdown(self):
+    """Send a message to the child HTTP server process and wait for it to
+        finish."""
+    self.conn.send(False)
+    self.process.join()
+
+  def GetURL(self, rel_url):
+    """Get the full url for a file on the local HTTP server.
+
+    Args:
+      rel_url: A URL fragment to convert to a full URL. For example,
+          GetURL('foobar.baz') -> 'http://localhost:1234/foobar.baz'
+    """
+    return 'http://localhost:%d/%s' % (self.port, rel_url)
+
+
 def AddMakeBat(pepperdir, makepath):
   """Create a simple batch file to execute Make.
-  
+
   Creates a simple batch file named make.bat for the Windows platform at the
   given path, pointing to the Make executable in the SDK."""
-  
+
   makepath = os.path.abspath(makepath)
   if not makepath.startswith(pepperdir):
     buildbot_common.ErrorExit('Make.bat not relative to Pepper directory: ' +
                               makepath)
-  
+
   makeexe = os.path.abspath(os.path.join(pepperdir, 'tools'))
   relpath = os.path.relpath(makeexe, makepath)
 
@@ -202,7 +265,7 @@ def InstallHeaders(tc_dst_inc, pepper_ver, tc_name):
   buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'c', 'dev', '*.h'),
           os.path.join(ppapi, 'c', 'dev'))
 
-  # Run the generator to overwrite IDL files  
+  # Run the generator to overwrite IDL files
   buildbot_common.Run([sys.executable, 'generator.py', '--wnone', '--cgen',
        '--release=M' + pepper_ver, '--verbose', '--dstroot=%s/c' % ppapi],
        cwd=os.path.join(PPAPI_DIR, 'generators'))
@@ -259,7 +322,7 @@ def UntarToolchains(pepperdir, platform, arch, toolchains):
     tarfile = GetNewlibToolchain(platform, arch)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                         cwd=NACL_DIR)
-  
+
     # Then rename/move it to the pepper toolchain directory
     srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk')
     newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
@@ -270,7 +333,7 @@ def UntarToolchains(pepperdir, platform, arch, toolchains):
     tarfile = GetGlibcToolchain(platform, arch)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                         cwd=NACL_DIR)
-  
+
     # Then rename/move it to the pepper toolchain directory
     srcdir = os.path.join(tmpdir, 'toolchain', tcname)
     glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
@@ -349,7 +412,7 @@ EXAMPLE_MAP = {
     'fullscreen_tumbler',
     'gamepad',
     'geturl',
-    'hello_world_interactive', 
+    'hello_world_interactive',
     'hello_world_newlib',
     'input_events',
     'load_progress',
@@ -458,17 +521,17 @@ def main(args):
   parser = optparse.OptionParser()
   parser.add_option('--pnacl', help='Enable pnacl build.',
       action='store_true', dest='pnacl', default=False)
-  parser.add_option('--examples', help='Rebuild the examples.',
-      action='store_true', dest='examples', default=False)
-  parser.add_option('--update', help='Rebuild the updater.',
-      action='store_true', dest='update', default=False)
+  parser.add_option('--examples', help='Only build the examples.',
+      action='store_true', dest='only_examples', default=False)
+  parser.add_option('--update', help='Only build the updater.',
+      action='store_true', dest='only_updater', default=False)
   parser.add_option('--skip-tar', help='Skip generating a tarball.',
       action='store_true', dest='skip_tar', default=False)
   parser.add_option('--archive', help='Force the archive step.',
       action='store_true', dest='archive', default=False)
   parser.add_option('--release', help='PPAPI release version.',
       dest='release', default=None)
-  
+
   options, args = parser.parse_args(args[1:])
   platform = getos.GetPlatform()
   arch = 'x86'
@@ -482,19 +545,16 @@ def main(args):
   else:
     toolchains = ['newlib', 'glibc']
   print 'Building: ' + ' '.join(toolchains)
-  skip = options.examples or options.update
+  skip = options.only_examples or options.only_updater
 
-  skip_examples = skip
-  skip_update = skip
+  skip_examples = skip and not options.only_examples
+  skip_update = skip and not options.only_updater
   skip_untar = skip
   skip_build = skip
+  skip_test_updater = skip
   skip_tar = skip or options.skip_tar
 
-
-  if options.examples: skip_examples = False
-  skip_update = not options.update
-
-  if options.archive and (options.examples or options.skip_tar):
+  if options.archive and (options.only_examples or options.skip_tar):
     parser.error('Incompatible arguments with archive.')
 
   pepper_ver = str(int(build_utils.ChromeMajorVersion()))
@@ -508,17 +568,18 @@ def main(args):
     buildbot_common.Run(['gclient', 'runhooks'],
                         cwd=SRC_DIR, shell=(platform=='win'))
 
-  buildbot_common.BuildStep('Clean Pepper Dir')
   pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver)
   if not skip_untar:
+    buildbot_common.BuildStep('Clean Pepper Dir')
     buildbot_common.RemoveDir(pepperdir)
     buildbot_common.MakeDir(os.path.join(pepperdir, 'toolchain'))
     buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
 
-  buildbot_common.BuildStep('Add Text Files')
-  files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
-  files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
-  oshelpers.Copy(['-v'] + files + [pepperdir])
+  if not skip_build:
+    buildbot_common.BuildStep('Add Text Files')
+    files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE', 'README']
+    files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
+    oshelpers.Copy(['-v'] + files + [pepperdir])
 
 
   # Clean out the temporary toolchain untar directory
@@ -528,33 +589,85 @@ def main(args):
   if not skip_build:
     BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains)
 
-  buildbot_common.BuildStep('Copy make OS helpers')
-  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
-      os.path.join(pepperdir, 'tools'))
-  if platform == 'win':
-    buildbot_common.BuildStep('Add MAKE')
-    http_download.HttpDownload(GSTORE + MAKE,
-                             os.path.join(pepperdir, 'tools' ,'make.exe'))
+  if not skip_build:
+    buildbot_common.BuildStep('Copy make OS helpers')
+    buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
+        os.path.join(pepperdir, 'tools'))
+    if platform == 'win':
+      buildbot_common.BuildStep('Add MAKE')
+      http_download.HttpDownload(GSTORE + MAKE,
+                                 os.path.join(pepperdir, 'tools' ,'make.exe'))
 
   if not skip_examples:
     CopyExamples(pepperdir, toolchains)
 
+  tarname = 'naclsdk_' + platform + '.bz2'
+  if 'pnacl' in toolchains:
+    tarname = 'p' + tarname
+  tarfile = os.path.join(OUT_DIR, tarname)
+
   if not skip_tar:
     buildbot_common.BuildStep('Tar Pepper Bundle')
-    tarname = 'naclsdk_' + platform + '.bz2'
-    if 'pnacl' in toolchains:
-      tarname = 'p' + tarname
-    tarfile = os.path.join(OUT_DIR, tarname)
     buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
          'pepper_' + pepper_ver], cwd=NACL_DIR)
 
-  # Archive on non-trybots.
-  if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''):
-    buildbot_common.BuildStep('Archive build')
-    buildbot_common.Archive(tarname,
-        'nativeclient-mirror/nacl/nacl_sdk/%s' % build_utils.ChromeVersion(),
-        OUT_DIR)
+  # build sdk update
+  if not skip_update:
+    BuildUpdater()
 
+  # start local server sharing a manifest + the new bundle
+  if not skip_test_updater:
+    buildbot_common.BuildStep('Move bundle to localserver dir')
+    buildbot_common.MakeDir(SERVER_DIR)
+    buildbot_common.Move(tarfile, SERVER_DIR)
+    tarfile = os.path.join(SERVER_DIR, tarname)
+
+    server = None
+    try:
+      buildbot_common.BuildStep('Run local server')
+      server = LocalHTTPServer(SERVER_DIR)
+
+      buildbot_common.BuildStep('Generate manifest')
+      with open(tarfile, 'rb') as tarfile_stream:
+        archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
+            tarfile_stream)
+      archive = manifest_util.Archive(manifest_util.GetHostOS())
+      archive.CopyFrom({'url': server.GetURL(tarname),
+                        'size': archive_size,
+                        'checksum': {'sha1': archive_sha1}})
+      bundle = manifest_util.Bundle('pepper_' + pepper_ver)
+      bundle.CopyFrom({
+          'revision': clnumber,
+          'repath': 'pepper_' + pepper_ver,
+          'version': pepper_ver,
+          'description': 'Chrome %s bundle, revision %s' % (
+              pepper_ver, clnumber),
+          'stability': 'dev',
+          'recommended': 'no',
+          'archives': [archive]})
+      manifest = manifest_util.SDKManifest()
+      manifest.SetBundle(bundle)
+      manifest_name = 'naclsdk_manifest2.json'
+      with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \
+          manifest_stream:
+        manifest_stream.write(manifest.GetManifestString())
+
+      # use newly built sdk updater to pull this bundle
+      buildbot_common.BuildStep('Update from local server')
+      updater_py = os.path.join(OUT_DIR, 'nacl_sdk', 'sdk_tools',
+          'sdk_update.py')
+      buildbot_common.Run([sys.executable, updater_py, '-U',
+          server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver])
+
+      # If we are testing examples, do it in the newly pulled directory.
+      pepperdir = os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver)
+
+    # kill server
+    finally:
+      if server:
+        server.Shutdown()
+
+  # build examples.
   if not skip_examples:
     buildbot_common.BuildStep('Test Build Examples')
     filelist = os.listdir(os.path.join(pepperdir, 'examples'))
@@ -566,9 +679,25 @@ def main(args):
         buildbot_common.Run(['make', 'all', '-j8'],
                             cwd=os.path.abspath(dirnode), shell=True)
 
-  # Build SDK Tools
-  if not skip_update:
-    BuildUpdater()
+  # Archive on non-trybots.
+  if options.archive or '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''):
+    buildbot_common.BuildStep('Archive build')
+    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % \
+        build_utils.ChromeVersion()
+    buildbot_common.Archive(tarname, bucket_path, OUT_DIR)
+
+    # generate "manifest snippet" for this archive.
+    if not skip_test_updater:
+      archive = bundle.GetArchive(manifest_util.GetHostOS())
+      archive.url = 'https://commondatastorage.googleapis.com/' \
+          'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % (
+              build_utils.ChromeVersion(), tarname)
+      manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
+      with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
+        manifest_snippet_stream.write(bundle.ToJSON())
+
+      buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
+                              step_link=False)
 
   return 0
 
diff --git a/native_client_sdk/src/build_tools/buildbot_common.py b/native_client_sdk/src/build_tools/buildbot_common.py
index ea978e1..bdf0aea 100644
--- a/native_client_sdk/src/build_tools/buildbot_common.py
+++ b/native_client_sdk/src/build_tools/buildbot_common.py
@@ -96,7 +96,7 @@ def GetGsutil():
     return LOCAL_GSUTIL
 
 
-def Archive(filename, bucket_path, cwd=None):
+def Archive(filename, bucket_path, cwd=None, step_link=True):
   """Upload the given filename to Google Store."""
   full_dst = 'gs://%s/%s' % (bucket_path, filename)
 
@@ -106,5 +106,6 @@ def Archive(filename, bucket_path, cwd=None):
       cwd=cwd)
   url = 'https://commondatastorage.googleapis.com/'\
         '%s/%s' % (bucket_path, filename)
-  print '@@@STEP_LINK@download@%s@@@' % url
-  sys.stdout.flush()
+  if step_link:
+    print '@@@STEP_LINK@download@%s@@@' % url
+    sys.stdout.flush()
diff --git a/native_client_sdk/src/build_tools/manifest_util.py b/native_client_sdk/src/build_tools/manifest_util.py
index c39597c..0bdafc3 100644
--- a/native_client_sdk/src/build_tools/manifest_util.py
+++ b/native_client_sdk/src/build_tools/manifest_util.py
@@ -2,7 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import hashlib
 import json
+import sys
 
 MANIFEST_VERSION = 2
 
@@ -33,6 +35,64 @@ VALID_BUNDLES_KEYS = frozenset([
 VALID_MANIFEST_KEYS = frozenset(['manifest_version', BUNDLES_KEY])
 
 
+def GetHostOS():
+  '''Returns the host_os value that corresponds to the current host OS'''
+  return {
+      'linux2': 'linux',
+      'darwin': 'mac',
+      'cygwin': 'win',
+      'win32':  'win'
+      }[sys.platform]
+
+
+def DictToJSON(dict):
+  """Convert a dict to a JSON-formatted string."""
+  pretty_string = json.dumps(dict, sort_keys=False, indent=2)
+  # json.dumps sometimes returns trailing whitespace and does not put
+  # a newline at the end.  This code fixes these problems.
+  pretty_lines = pretty_string.split('\n')
+  return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n'
+
+
+def DownloadAndComputeHash(from_stream, to_stream=None, progress_func=None):
+  ''' Download the archive data from from-stream and generate sha1 and
+      size info.
+
+  Args:
+    from_stream:   An input stream that supports read.
+    to_stream:     [optional] the data is written to to_stream if it is
+                   provided.
+    progress_func: [optional] A function used to report download progress. If
+                   provided, progress_func is called with progress=0 at the
+                   beginning of the download, periodically with progress=1
+                   during the download, and progress=100 at the end.
+
+  Return
+    A tuple (sha1, size) where sha1 is a sha1-hash for the archive data and
+    size is the size of the archive data in bytes.'''
+  # Use a no-op progress function if none is specified.
+  def progress_no_op(progress):
+    pass
+  if not progress_func:
+    progress_func = progress_no_op
+
+  sha1_hash = hashlib.sha1()
+  size = 0
+  progress_func(progress=0)
+  while(1):
+    data = from_stream.read(32768)
+    if not data:
+      break
+    sha1_hash.update(data)
+    size += len(data)
+    if to_stream:
+      to_stream.write(data)
+    progress_func(size)
+
+  progress_func(progress=100)
+  return sha1_hash.hexdigest(), size
+
+
 class Error(Exception):
   """Generic error/exception for manifest_util module"""
   pass
@@ -69,6 +129,15 @@ class Archive(dict):
       host_os = 'all (default)'
     if not self.get('url', None):
       raise Error('Archive "%s" has no URL' % host_os)
+    if not self.get('size', None):
+      raise Error('Archive "%s" has no size' % host_os)
+    checksum = self.get('checksum', None)
+    if not checksum:
+      raise Error('Archive "%s" has no checksum' % host_os)
+    elif not isinstance(checksum, dict):
+      raise Error('Archive "%s" has a checksum, but it is not a dict' % host_os)
+    elif not len(checksum):
+      raise Error('Archive "%s" has an empty checksum dict' % host_os)
     # Verify that all key names are valid.
     for key, val in self.iteritems():
       if key not in VALID_ARCHIVE_KEYS:
@@ -79,6 +148,11 @@ class Archive(dict):
     """Returns the URL of this archive"""
     return self['url']
 
+  @url.setter
+  def url(self, url):
+    """Set the URL of this archive"""
+    self['url'] = url
+
   @property
   def size(self):
     """Returns the size of this archive, in bytes"""
@@ -123,6 +197,14 @@ class Bundle(dict):
     """
     return Bundle(self.items() + bundle.items())
 
+  def ToJSON(self):
+    """Convert this bundle to a JSON-formatted string."""
+    return DictToJSON(self)
+
+  def FromJSON(self, json_string):
+    """Parse and load bundle data from a JSON-formatted string."""
+    self.CopyFrom(json.loads(json_string))
+
   def CopyFrom(self, dict):
     """Update the content of the bundle by copying values from the given
        dictionary.
@@ -186,6 +268,10 @@ class Bundle(dict):
         return archive
     return None
 
+  def GetHostOSArchive(self):
+    """Retrieve the archive for the current host os."""
+    return self.GetArchive(GetHostOS())
+
   def GetArchives(self):
     """Returns all the archives in this bundle"""
     return self[ARCHIVES_KEY]
@@ -345,8 +431,4 @@ class SDKManifest(object):
 
   def GetManifestString(self):
     """Returns the current JSON manifest object, pretty-printed"""
-    pretty_string = json.dumps(self._manifest_data, sort_keys=False, indent=2)
-    # json.dumps sometimes returns trailing whitespace and does not put
-    # a newline at the end.  This code fixes these problems.
-    pretty_lines = pretty_string.split('\n')
-    return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n'
+    return DictToJSON(self._manifest_data)
diff --git a/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py b/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py
index 878e21e..ba06afe 100755
--- a/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py
+++ b/native_client_sdk/src/build_tools/sdk_tools/sdk_update.py
@@ -103,15 +103,6 @@ class Error(Exception):
   pass
 
 
-def GetHostOS():
-  '''Returns the host_os value that corresponds to the current host OS'''
-  return {
-      'linux2': 'linux',
-      'darwin': 'mac',
-      'cygwin': 'win',
-      'win32':  'win'
-      }[sys.platform]
-
 def UrlOpen(url):
   request = fancy_urllib.FancyRequest(url)
   ca_certs = os.path.join(os.path.dirname(os.path.abspath(__file__)),
@@ -207,23 +198,6 @@ def RenameDir(srcdir, destdir):
               % (srcdir, destdir, num_tries, destdir))
 
 
-def ShowProgress(progress):
-  ''' A download-progress function used by class Archive.
-      (See DownloadAndComputeHash).'''
-  global count  # A divider, so we don't emit dots too often.
-
-  if progress == 0:
-    count = 0
-  elif progress == 100:
-    sys.stdout.write('\n')
-  else:
-    count = count + 1
-    if count > 10:
-      sys.stdout.write('.')
-      sys.stdout.flush()
-      count = 0
-
-
 class ProgressFunction(object):
   '''Create a progress function for a file with a given size'''
 
@@ -251,43 +225,41 @@ class ProgressFunction(object):
     return ShowKnownProgress
 
 
-def DownloadAndComputeHash(from_stream, to_stream=None, progress_func=None):
-  ''' Download the archive data from from-stream and generate sha1 and
-      size info.
+def DownloadArchiveToFile(archive, dest_path):
+  '''Download the archive's data to a file at dest_path.
+
+  As a side effect, computes the sha1 hash and data size, both returned as a
+  tuple. Raises an Error if the url can't be opened, or an IOError exception if
+  dest_path can't be opened.
 
   Args:
-    from_stream:   An input stream that supports read.
-    to_stream:     [optional] the data is written to to_stream if it is
-                   provided.
-    progress_func: [optional] A function used to report download progress. If
-                   provided, progress_func is called with progress=0 at the
-                   beginning of the download, periodically with progress=1
-                   during the download, and progress=100 at the end.
-
-  Return
-    A tuple (sha1, size) where sha1 is a sha1-hash for the archive data and
-    size is the size of the archive data in bytes.'''
-  # Use a no-op progress function if none is specified.
-  def progress_no_op(progress):
-    pass
-  if not progress_func:
-    progress_func = progress_no_op
-
-  sha1_hash = hashlib.sha1()
+    dest_path: Path for the file that will receive the data.
+  Return:
+    A tuple (sha1, size) with the sha1 hash and data size respectively.'''
+  sha1 = None
   size = 0
-  progress_func(progress=0)
-  while(1):
-    data = from_stream.read(32768)
-    if not data:
-      break
-    sha1_hash.update(data)
-    size += len(data)
-    if to_stream:
-      to_stream.write(data)
-    progress_func(size)
-
-  progress_func(progress=100)
-  return sha1_hash.hexdigest(), size
+  with open(dest_path, 'wb') as to_stream:
+    from_stream = None
+    try:
+      from_stream = UrlOpen(archive.url)
+    except urllib2.URLError:
+      raise Error('Cannot open "%s" for archive %s' %
+          (archive.url, archive.host_os))
+    try:
+      content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH])
+      progress_function = ProgressFunction(content_length).GetProgressFunction()
+      InfoPrint('Downloading %s' % archive.url)
+      sha1, size = manifest_util.DownloadAndComputeHash(
+          from_stream,
+          to_stream=to_stream,
+          progress_func=progress_function)
+      if size != content_length:
+        raise Error('Download size mismatch for %s.\n'
+                    'Expected %s bytes but got %s' %
+                    (archive.url, content_length, size))
+    finally:
+      if from_stream: from_stream.close()
+  return sha1, size
 
 
 def LoadManifestFromFile(path):
@@ -315,14 +287,13 @@ def LoadManifestFromURL(url):
     raise Error('Unable to open %s. [%s]' % (url, e))
 
   manifest_stream = cStringIO.StringIO()
-  sha1, size = DownloadAndComputeHash(
-      url_stream, manifest_stream)
+  sha1, size = manifest_util.DownloadAndComputeHash(url_stream, manifest_stream)
   manifest = manifest_util.SDKManifest()
   manifest.LoadManifestString(manifest_stream.getvalue())
 
   def BundleFilter(bundle):
     # Only add this bundle if it's supported on this platform.
-    return bundle.GetArchive(GetHostOS())
+    return bundle.GetHostOSArchive()
 
   manifest.FilterBundles(BundleFilter)
   return manifest
@@ -345,43 +316,6 @@ def WriteManifestToFile(manifest, path):
   shutil.move(temp_file_name, path)
 
 
-def DownloadArchiveToFile(archive, dest_path):
-  '''Download the archive's data to a file at dest_path.
-
-  As a side effect, computes the sha1 hash and data size, both returned as a
-  tuple. Raises an Error if the url can't be opened, or an IOError exception if
-  dest_path can't be opened.
-
-  Args:
-    dest_path: Path for the file that will receive the data.
-  Return:
-    A tuple (sha1, size) with the sha1 hash and data size respectively.'''
-  sha1 = None
-  size = 0
-  with open(dest_path, 'wb') as to_stream:
-    from_stream = None
-    try:
-      from_stream = UrlOpen(archive.url)
-    except urllib2.URLError:
-      raise Error('Cannot open "%s" for archive %s' %
-          (archive.url, archive.host_os))
-    try:
-      content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH])
-      progress_function = ProgressFunction(content_length).GetProgressFunction()
-      InfoPrint('Downloading %s' % archive.url)
-      sha1, size = DownloadAndComputeHash(
-          from_stream,
-          to_stream=to_stream,
-          progress_func=progress_function)
-      if size != content_length:
-        raise Error('Download size mismatch for %s.\n'
-                    'Expected %s bytes but got %s' %
-                    (archive.url, content_length, size))
-    finally:
-      if from_stream: from_stream.close()
-  return sha1, size
-
-
 #------------------------------------------------------------------------------
 # Commands
 
@@ -471,7 +405,7 @@ def Update(options, argv):
       continue
     def UpdateBundle():
       '''Helper to install a bundle'''
-      archive = bundle.GetArchive(GetHostOS())
+      archive = bundle.GetHostOSArchive()
       (scheme, host, path, _, _, _) = urlparse.urlparse(archive['url'])
       dest_filename = os.path.join(options.user_data_dir, path.split('/')[-1])
       sha1, size = DownloadArchiveToFile(archive, dest_filename)