[NaCl SDK] Add tests for update_nacl_manifest.py

BUG=none
TEST=none

Review URL: https://chromiumcodereview.appspot.com/10332023

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@136435 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 811 insertions, 269 deletions

diff --git a/native_client_sdk/src/build_tools/manifest_util.py b/native_client_sdk/src/build_tools/manifest_util.py
index 890d8f9..9ce6495 100644
--- a/native_client_sdk/src/build_tools/manifest_util.py
+++ b/native_client_sdk/src/build_tools/manifest_util.py
@@ -2,6 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import copy
 import hashlib
 import json
 import sys
@@ -144,25 +145,36 @@ class Archive(dict):
       if key not in VALID_ARCHIVE_KEYS:
         raise Error('Archive "%s" has invalid attribute "%s"' % (host_os, key))
 
-  @property
-  def url(self):
-    """Returns the URL of this archive"""
-    return self['url']
+  def __getattr__(self, name):
+    """Retrieve values from this dict using attributes.
 
-  @url.setter
-  def url(self, url):
-    """Set the URL of this archive"""
-    self['url'] = url
+    This allows for foo.bar instead of foo['bar'].
 
-  @property
-  def size(self):
-    """Returns the size of this archive, in bytes"""
-    return self['size']
+    Args:
+      name: the name of the key, 'bar' in the example above.
+    Returns:
+      The value associated with that key."""
+    if name not in VALID_ARCHIVE_KEYS:
+      raise AttributeError(name)
+    # special case, self.checksum returns the sha1, not the checksum dict.
+    if name == 'checksum':
+      return self.GetChecksum()
+    return self.__getitem__(name)
+
+  def __setattr__(self, name, value):
+    """Set values in this dict using attributes.
 
-  @property
-  def host_os(self):
-    """Returns the host OS of this archive"""
-    return self['host_os']
+    This allows for foo.bar instead of foo['bar'].
+
+    Args:
+      name: The name of the key, 'bar' in the example above.
+      value: The value to associate with that key."""
+    if name not in VALID_ARCHIVE_KEYS:
+      raise AttributeError(name)
+    # special case, self.checksum returns the sha1, not the checksum dict.
+    if name == 'checksum':
+      self.setdefault('checksum', {})['sha1'] = value
+    return self.__setitem__(name, value)
 
   def GetChecksum(self, type='sha1'):
     """Returns a given cryptographic checksum of the archive"""
@@ -285,51 +297,79 @@ class Bundle(dict):
     """Returns all the archives in this bundle"""
     return self[ARCHIVES_KEY]
 
+  def AddArchive(self, archive):
+    """Add an archive to this bundle."""
+    self.RemoveArchive(archive.host_os)
+    self[ARCHIVES_KEY].append(archive)
+
   def RemoveArchive(self, host_os_name):
     """Remove an archive from this Bundle."""
-    for i, archive in enumerate(self[ARCHIVES_KEY]):
-      if archive.host_os == host_os_name:
-        del self[ARCHIVES_KEY][i]
-
-  @property
-  def name(self):
-    """Returns the name of this bundle"""
-    return self[NAME_KEY]
-
-  @name.setter
-  def name(self, name):
-    """Set the name of this bundle"""
-    self[NAME_KEY] = name
-
-  @property
-  def version(self):
-    """Returns the version of this bundle"""
-    return self[VERSION_KEY]
-
-  @property
-  def revision(self):
-    """Returns the revision of this bundle"""
-    return self[REVISION_KEY]
-
-  @property
-  def recommended(self):
-    """Returns whether this bundle is recommended"""
-    return self['recommended']
-
-  @recommended.setter
-  def recommended(self, value):
-    """Sets whether this bundle is recommended"""
-    self['recommended'] = value
-
-  @property
-  def stability(self):
-    """Returns the stability of this bundle"""
-    return self['stability']
-  
-  @stability.setter
-  def stability(self, value):
-    """Sets the stability of this bundle"""
-    self['stability'] = value
+    if host_os_name == 'all':
+      del self[ARCHIVES_KEY][:]
+    else:
+      for i, archive in enumerate(self[ARCHIVES_KEY]):
+        if archive.host_os == host_os_name:
+          del self[ARCHIVES_KEY][i]
+
+  def __getattr__(self, name):
+    """Retrieve values from this dict using attributes.
+
+    This allows for foo.bar instead of foo['bar'].
+
+    Args:
+      name: the name of the key, 'bar' in the example above.
+    Returns:
+      The value associated with that key."""
+    if name not in VALID_BUNDLES_KEYS:
+      raise AttributeError(name)
+    return self.__getitem__(name)
+
+  def __setattr__(self, name, value):
+    """Set values in this dict using attributes.
+
+    This allows for foo.bar instead of foo['bar'].
+
+    Args:
+      name: The name of the key, 'bar' in the example above.
+      value: The value to associate with that key."""
+    if name not in VALID_BUNDLES_KEYS:
+      raise AttributeError(name)
+    self.__setitem__(name, value)
+
+  def __eq__(self, bundle):
+    """Test if two bundles are equal.
+
+    Normally the default comparison for two dicts is fine, but in this case we
+    don't care about the list order of the archives.
+
+    Args:
+      bundle: The other bundle to compare against.
+    Returns:
+      True if the bundles are equal."""
+    if not isinstance(bundle, Bundle):
+      return False
+    if len(self.keys()) != len(bundle.keys()):
+      return False
+    for key in self.keys():
+      if key not in bundle:
+        return False
+      # special comparison for ARCHIVE_KEY because we don't care about the list
+      # ordering.
+      if key == ARCHIVES_KEY:
+        if len(self[key]) != len(bundle[key]):
+          return False
+        for archive in self[key]:
+          if archive != bundle.GetArchive(archive.host_os):
+            return False
+      elif self[key] != bundle[key]:
+        return False
+    return True
+
+  def __ne__(self, bundle):
+    """Test if two bundles are unequal.
+
+    See __eq__ for more info."""
+    return not self.__eq__(bundle)
 
 
 class SDKManifest(object):
@@ -394,7 +434,7 @@ class SDKManifest(object):
     for i, bundle in enumerate(bundles):
       if bundle[NAME_KEY] == name:
         del bundles[i]
-    bundles.append(new_bundle)
+    bundles.append(copy.deepcopy(new_bundle))
 
   def BundleNeedsUpdate(self, bundle):
     """Decides if a bundle needs to be updated.
diff --git a/native_client_sdk/src/build_tools/tests/test_update_manifest.py b/native_client_sdk/src/build_tools/tests/test_update_manifest.py
new file mode 100755
index 0000000..3bcfce5
--- /dev/null
+++ b/native_client_sdk/src/build_tools/tests/test_update_manifest.py
@@ -0,0 +1,366 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import copy
+import datetime
+import os
+import posixpath
+import subprocess
+import sys
+import unittest
+import urlparse
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+BUILD_TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
+
+sys.path.append(BUILD_TOOLS_DIR)
+import manifest_util
+import update_nacl_manifest
+
+
+HTTPS_BASE_URL = 'https://commondatastorage.googleapis.com' \
+    '/nativeclient_mirror/nacl/nacl_sdk/'
+
+OS_CR = ('cros',)
+OS_M = ('mac',)
+OS_ML = ('mac', 'linux')
+OS_MLW = ('mac', 'linux', 'win')
+STABLE = 'stable'
+BETA = 'beta'
+DEV = 'dev'
+CANARY = 'canary'
+
+
+def GetArchiveUrl(host_os, version):
+  basename = 'naclsdk_%s.bz2' % (host_os,)
+  return urlparse.urljoin(HTTPS_BASE_URL, posixpath.join(version, basename))
+
+
+def GetPathFromGsUrl(url):
+  assert url.startswith(update_nacl_manifest.GS_BUCKET_PATH)
+  return url[len(update_nacl_manifest.GS_BUCKET_PATH):]
+
+
+def GetPathFromHttpsUrl(url):
+  assert url.startswith(HTTPS_BASE_URL)
+  return url[len(HTTPS_BASE_URL):]
+
+
+def MakeArchive(host_os, version):
+  archive = manifest_util.Archive(host_os)
+  archive.url = GetArchiveUrl(host_os, version)
+  # dummy values that won't succeed if we ever use them, but will pass
+  # validation. :)
+  archive.checksum = {'sha1': 'foobar'}
+  archive.size = 1
+  return archive
+
+
+def MakeNonPepperBundle(name, with_archives=False):
+  bundle = manifest_util.Bundle(name)
+  bundle.version = 1
+  bundle.revision = 1
+  bundle.description = 'Dummy bundle'
+  bundle.recommended = 'yes'
+  bundle.stability = 'stable'
+
+  if with_archives:
+    for host_os in OS_MLW:
+      archive = manifest_util.Archive(host_os)
+      archive.url = 'http://example.com'
+      archive.checksum = {'sha1': 'blah'}
+      archive.size = 2
+      bundle.AddArchive(archive)
+  return bundle
+
+
+def MakeBundle(major_version, revision, version=None, host_oses=None):
+  assert version is None or version.split('.')[0] == major_version
+  bundle_name = 'pepper_' + major_version
+  bundle = manifest_util.Bundle(bundle_name)
+  bundle.version = int(major_version)
+  bundle.revision = revision
+  bundle.description = 'Chrome %s bundle, revision %s' % (major_version,
+      revision)
+  bundle.repath = bundle_name
+  bundle.recommended = 'no'
+  bundle.stability = 'dev'
+
+  if host_oses:
+    for host_os in host_oses:
+      bundle.AddArchive(MakeArchive(host_os, version))
+  return bundle
+
+
+class MakeManifest(manifest_util.SDKManifest):
+  def __init__(self, *args):
+    manifest_util.SDKManifest.__init__(self)
+
+    for bundle in args:
+      self.AddBundle(bundle)
+
+  def AddBundle(self, bundle):
+    self.MergeBundle(bundle, allow_existing=False)
+
+
+class MakeHistory(object):
+  def __init__(self):
+    # used for a dummy timestamp
+    self.datetime = datetime.datetime.utcnow()
+    self.history = []
+
+  def Add(self, host_oses, channel, version):
+    for host_os in host_oses:
+      timestamp = self.datetime.strftime('%Y-%m-%d %H:%M:%S.%f')
+      self.history.append((host_os, channel, version, timestamp))
+      self.datetime += datetime.timedelta(0, -3600) # one hour earlier
+    self.datetime += datetime.timedelta(-1) # one day earlier
+
+
+class MakeFiles(dict):
+  def Add(self, bundle, add_archive_for_os=OS_MLW, add_json_for_os=OS_MLW):
+    for archive in bundle.GetArchives():
+      if not archive.host_os in add_archive_for_os:
+        continue
+
+      # add a dummy file for each archive
+      path = GetPathFromHttpsUrl(archive.url)
+      self[path] = 'My Dummy Archive'
+
+      if archive.host_os in add_json_for_os:
+        # add .json manifest snippet, it should look like a normal Bundle, but
+        # only has one archive.
+        new_bundle = manifest_util.Bundle('')
+        new_bundle.CopyFrom(bundle)
+        del new_bundle.archives[:]
+        new_bundle.AddArchive(archive)
+        self[path + '.json'] = new_bundle.GetDataAsString()
+
+
+class TestDelegate(update_nacl_manifest.Delegate):
+  def __init__(self, manifest, history, files):
+    self.manifest = manifest
+    self.history = history
+    self.files = files
+
+  def GetRepoManifest(self):
+    return self.manifest
+
+  def GetHistory(self):
+    return self.history
+
+  def GsUtil_ls(self, url):
+    path = GetPathFromGsUrl(url)
+    result = []
+    for filename, _ in self.files.iteritems():
+      if filename.startswith(path):
+        result.append(filename)
+    return result
+
+  def GsUtil_cat(self, url):
+    path = GetPathFromGsUrl(url)
+    if path not in self.files:
+      raise subprocess.CalledProcessError(1, 'gsutil cat %s' % (url,))
+    return self.files[path]
+
+  def GsUtil_cp(self, src, dest, stdin=None):
+    dest_path = GetPathFromGsUrl(dest)
+    if src == '-':
+      self.files[dest_path] = stdin
+    else:
+      src_path = GetPathFromGsUrl(src)
+      if src_path not in self.files:
+        raise subprocess.CalledProcessError(1, 'gsutil cp %s %s' % (src, dest))
+      self.files[dest_path] = self.files[src_path]
+
+  def Print(self, *args):
+    # eat all informational messages
+    pass
+
+
+# Shorthand for premade bundles/versions
+V18_0_1025_163 = '18.0.1025.163'
+V18_0_1025_175 = '18.0.1025.175'
+V18_0_1025_184 = '18.0.1025.184'
+V19_0_1084_41 = '19.0.1084.41'
+V19_0_1084_67 = '19.0.1084.67'
+B18_0_1025_163_R1_MLW = MakeBundle('18', '1', V18_0_1025_163, OS_MLW)
+B18_0_1025_184_R1_MLW = MakeBundle('18', '1', V18_0_1025_184, OS_MLW)
+B18_R1_NONE = MakeBundle('18', '1')
+B19_0_1084_41_R1_MLW = MakeBundle('19', '1', V19_0_1084_41, OS_MLW)
+B19_0_1084_67_R1_MLW = MakeBundle('19', '1', V19_0_1084_67, OS_MLW)
+B19_R1_NONE = MakeBundle('19', '1')
+NON_PEPPER_BUNDLE_NOARCHIVES = MakeNonPepperBundle('foo')
+NON_PEPPER_BUNDLE_ARCHIVES = MakeNonPepperBundle('bar', with_archives=True)
+
+
+class TestUpdateManifest(unittest.TestCase):
+  def setUp(self):
+    self.history = MakeHistory()
+    self.files = MakeFiles()
+    self.delegate = None
+    self.uploaded_manifest = None
+    self.manifest = None
+
+  def _MakeDelegate(self):
+    self.delegate = TestDelegate(self.manifest, self.history.history,
+        self.files)
+
+  def _Run(self, host_oses):
+    update_nacl_manifest.Run(self.delegate, host_oses)
+
+  def _HasUploadedManifest(self):
+    return 'naclsdk_manifest2.json' in self.files
+
+  def _ReadUploadedManifest(self):
+    self.uploaded_manifest = manifest_util.SDKManifest()
+    self.uploaded_manifest.LoadDataFromString(
+        self.files['naclsdk_manifest2.json'])
+
+  def _AssertUploadedManifestHasBundle(self, bundle, stability):
+    uploaded_manifest_bundle = self.uploaded_manifest.GetBundle(bundle.name)
+    # Bundles that we create in the test (and in the manifest snippets) have
+    # their stability set to "dev". update_nacl_manifest correctly updates it.
+    # So we have to force the stability of |bundle| so they compare equal.
+    test_bundle = copy.copy(bundle)
+    test_bundle.stability = stability
+    self.assertEqual(uploaded_manifest_bundle, test_bundle)
+
+  def testNoUpdateNeeded(self):
+    self.manifest = MakeManifest(B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self.assertEqual(self._HasUploadedManifest(), False)
+
+    # Add another bundle, make sure it still doesn't update
+    self.manifest.AddBundle(B19_0_1084_41_R1_MLW)
+    self._Run(OS_MLW)
+    self.assertEqual(self._HasUploadedManifest(), False)
+
+  def testSimpleUpdate(self):
+    self.manifest = MakeManifest(B18_R1_NONE)
+    self.history.Add(OS_MLW, BETA, V18_0_1025_163)
+    self.files.Add(B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self._ReadUploadedManifest()
+    self._AssertUploadedManifestHasBundle(B18_0_1025_163_R1_MLW, BETA)
+    self.assertEqual(len(self.uploaded_manifest.GetBundles()), 1)
+
+  def testOnePlatformHasNewerRelease(self):
+    self.manifest = MakeManifest(B18_R1_NONE)
+    self.history.Add(OS_M, BETA, V18_0_1025_175)  # Mac has newer version
+    self.history.Add(OS_MLW, BETA, V18_0_1025_163)
+    self.files.Add(B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self._ReadUploadedManifest()
+    self._AssertUploadedManifestHasBundle(B18_0_1025_163_R1_MLW, BETA)
+    self.assertEqual(len(self.uploaded_manifest.GetBundles()), 1)
+
+  def testMultipleMissingPlatformsInHistory(self):
+    self.manifest = MakeManifest(B18_R1_NONE)
+    self.history.Add(OS_ML, BETA, V18_0_1025_184)
+    self.history.Add(OS_M, BETA, V18_0_1025_175)
+    self.history.Add(OS_MLW, BETA, V18_0_1025_163)
+    self.files.Add(B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self._ReadUploadedManifest()
+    self._AssertUploadedManifestHasBundle(B18_0_1025_163_R1_MLW, BETA)
+    self.assertEqual(len(self.uploaded_manifest.GetBundles()), 1)
+
+  def testUpdateOnlyOneBundle(self):
+    self.manifest = MakeManifest(B18_R1_NONE, B19_0_1084_41_R1_MLW)
+    self.history.Add(OS_MLW, BETA, V18_0_1025_163)
+    self.files.Add(B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self._ReadUploadedManifest()
+    self._AssertUploadedManifestHasBundle(B18_0_1025_163_R1_MLW, BETA)
+    self._AssertUploadedManifestHasBundle(B19_0_1084_41_R1_MLW, DEV)
+    self.assertEqual(len(self.uploaded_manifest.GetBundles()), 2)
+
+  def testUpdateTwoBundles(self):
+    self.manifest = MakeManifest(B18_R1_NONE, B19_R1_NONE)
+    self.history.Add(OS_MLW, BETA, V19_0_1084_41)
+    self.history.Add(OS_MLW, STABLE, V18_0_1025_163)
+    self.files.Add(B18_0_1025_163_R1_MLW)
+    self.files.Add(B19_0_1084_41_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self._ReadUploadedManifest()
+    self._AssertUploadedManifestHasBundle(B18_0_1025_163_R1_MLW, STABLE)
+    self._AssertUploadedManifestHasBundle(B19_0_1084_41_R1_MLW, BETA)
+    self.assertEqual(len(self.uploaded_manifest.GetBundles()), 2)
+
+  def testUpdateWithMissingPlatformsInArchives(self):
+    self.manifest = MakeManifest(B18_R1_NONE)
+    self.history.Add(OS_MLW, BETA, V18_0_1025_184)
+    self.history.Add(OS_MLW, BETA, V18_0_1025_163)
+    self.files.Add(B18_0_1025_184_R1_MLW, add_archive_for_os=OS_M)
+    self.files.Add(B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self._ReadUploadedManifest()
+    self._AssertUploadedManifestHasBundle(B18_0_1025_163_R1_MLW, BETA)
+    self.assertEqual(len(self.uploaded_manifest.GetBundles()), 1)
+
+  def testUpdateWithMissingManifestSnippets(self):
+    self.manifest = MakeManifest(B18_R1_NONE)
+    self.history.Add(OS_MLW, BETA, V18_0_1025_184)
+    self.history.Add(OS_MLW, BETA, V18_0_1025_163)
+    self.files.Add(B18_0_1025_184_R1_MLW, add_json_for_os=OS_ML)
+    self.files.Add(B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self._ReadUploadedManifest()
+    self._AssertUploadedManifestHasBundle(B18_0_1025_163_R1_MLW, BETA)
+    self.assertEqual(len(self.uploaded_manifest.GetBundles()), 1)
+
+  def testRecommendedIsMaintained(self):
+    for recommended in 'yes', 'no':
+      self.setUp()
+      bundle = copy.deepcopy(B18_R1_NONE)
+      bundle.recommended = recommended
+      self.manifest = MakeManifest(bundle)
+      self.history.Add(OS_MLW, BETA, V18_0_1025_163)
+      self.files.Add(B18_0_1025_163_R1_MLW)
+      self._MakeDelegate()
+      self._Run(OS_MLW)
+      self._ReadUploadedManifest()
+      self.assertEqual(len(self.uploaded_manifest.GetBundles()), 1)
+      uploaded_bundle = self.uploaded_manifest.GetBundle('pepper_18')
+      self.assertEqual(uploaded_bundle.recommended, recommended)
+
+  def testNoUpdateWithNonPepperBundle(self):
+    self.manifest = MakeManifest(NON_PEPPER_BUNDLE_NOARCHIVES,
+        B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self.assertEqual(self._HasUploadedManifest(), False)
+
+  def testUpdateWithHistoryWithExtraneousPlatforms(self):
+    self.manifest = MakeManifest(B18_R1_NONE)
+    self.history.Add(OS_ML, BETA, V18_0_1025_184)
+    self.history.Add(OS_CR, BETA, V18_0_1025_184)
+    self.history.Add(OS_CR, BETA, V18_0_1025_175)
+    self.history.Add(OS_MLW, BETA, V18_0_1025_163)
+    self.files.Add(B18_0_1025_163_R1_MLW)
+    self._MakeDelegate()
+    self._Run(OS_MLW)
+    self._ReadUploadedManifest()
+    self._AssertUploadedManifestHasBundle(B18_0_1025_163_R1_MLW, BETA)
+    self.assertEqual(len(self.uploaded_manifest.GetBundles()), 1)
+
+
+def main():
+  suite = unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__])
+  result = unittest.TextTestRunner(verbosity=2).run(suite)
+
+  return int(not result.wasSuccessful())
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/native_client_sdk/src/build_tools/update_nacl_manifest.py b/native_client_sdk/src/build_tools/update_nacl_manifest.py
index ac37553..b72ef88 100755
--- a/native_client_sdk/src/build_tools/update_nacl_manifest.py
+++ b/native_client_sdk/src/build_tools/update_nacl_manifest.py
@@ -11,6 +11,7 @@ import buildbot_common
 import csv
 import manifest_util
 import os
+import posixpath
 import re
 import subprocess
 import sys
@@ -21,13 +22,16 @@ import urllib2
 # TODO(binji) handle pushing pepper_trunk
 
 
-BUCKET_PATH = 'nativeclient-mirror/nacl/nacl_sdk'
-GS_SDK_MANIFEST = 'gs://%s/naclsdk_manifest2.json' % (BUCKET_PATH,)
+MANIFEST_BASENAME = 'naclsdk_manifest2.json'
+SCRIPT_DIR = os.path.dirname(__file__)
+REPO_MANIFEST = os.path.join(SCRIPT_DIR, 'json', MANIFEST_BASENAME)
+GS_BUCKET_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/'
+GS_SDK_MANIFEST = GS_BUCKET_PATH + MANIFEST_BASENAME
 
 
 def SplitVersion(version_string):
   """Split a version string (e.g. "18.0.1025.163") into its components.
-  
+
   Note that this function doesn't handle versions in the form "trunk.###".
   """
   return tuple(map(int, version_string.split('.')))
@@ -41,243 +45,375 @@ def JoinVersion(version_tuple):
   return '.'.join(map(str, version_tuple))
 
 
-def GetChromeRepoSDKManifest():
-  with open(os.path.join('json', 'naclsdk_manifest2.json'), 'r') as sdk_stream:
-    sdk_json_string = sdk_stream.read()
-
-  manifest = manifest_util.SDKManifest()
-  manifest.LoadDataFromString(sdk_json_string)
-  return manifest
-
-
-def GetChromeVersionHistory():
-  """Read Chrome version history from omahaproxy.appspot.com/history.
-
-  Here is an example of data from this URL:
-    cros,stable,18.0.1025.168,2012-05-01 17:04:05.962578\n
-    win,canary,20.0.1123.0,2012-05-01 13:59:31.703020\n
-    mac,canary,20.0.1123.0,2012-05-01 11:54:13.041875\n
-    win,stable,18.0.1025.168,2012-04-30 20:34:56.078490\n
-    mac,stable,18.0.1025.168,2012-04-30 20:34:55.231141\n
-    ...
-  Where each line has comma separated values in the following format:
-  platform, channel, version, date/time\n
-
-  Returns:
-    A list where each element is a line from the document, represented as a
-    tuple. The version number has also been split at '.', and converted to ints
-    for easier comparisons.
-  """
-  url_stream = urllib2.urlopen('https://omahaproxy.appspot.com/history')
-  return [(platform, channel, SplitVersion(version), date)
-      for platform, channel, version, date in csv.reader(url_stream)]
-
-
-def GetPlatformMajorVersionHistory(history, with_platform, with_major_version):
-  """Yields Chrome history for a given platform and major version.
-
-  Args:
-    history: Chrome update history as returned from GetChromeVersionHistory.
-    with_platform: The name of the platform to filter for.
-    with_major_version: The major version to filter for.
-  Returns:
-    A generator that yields a tuple (channel, version) for each version that
-    matches the platform and major version. The version returned is a tuple as
-    returned from SplitVersion.
-  """
-  for platform, channel, version, _ in history:
-    if with_platform == platform and with_major_version == version[0]:
-      yield channel, version
-
-
-def FindNextSharedVersion(history, major_version, platforms):
-  """Yields versions of Chrome that exist on all given platforms, in order of
-     newest to oldest.
-
-  Versions are compared in reverse order of release. That is, the most recently
-  updated version will be tested first.
+def GetTimestampManifestName():
+  """Create a manifest name with a timestamp.
 
-  Args:
-    history: Chrome update history as returned from GetChromeVersionHistory.
-    major_version: The major version to filter for.
-    platforms: A sequence of platforms to filter for. Any other platforms will
-        be ignored.
   Returns:
-    A generator that yields a tuple (version, channel) for each version that
-    matches all platforms and the major version. The version returned is a
-    string (e.g. "18.0.1025.164").
+    A manifest name with an embedded date. This should make it easier to roll
+    back if necessary.
   """
-  platform_generators = []
-  for platform in platforms:
-    platform_generators.append(GetPlatformMajorVersionHistory(history, platform,
-                                                              major_version))
-
-  shared_version = None
-  platform_versions = [(tuple(), '')] * len(platforms)
-  while True:
-    try:
-      for i, platform_gen in enumerate(platform_generators):
-        if platform_versions[i][1] != shared_version:
-            platform_versions[i] = platform_gen.next()
-    except StopIteration:
-      return
-
-    shared_version = min(v for c, v in platform_versions)
-
-    if all(v == shared_version for c, v in platform_versions):
-      # grab the channel from an arbitrary platform
-      first_platform = platform_versions[0]
-      channel = first_platform[0]
-      yield JoinVersion(shared_version), channel
-
-      # force increment to next version for all platforms
-      shared_version = None
+  return time.strftime('naclsdk_manifest2.%Y_%m_%d_%H_%M_%S.json',
+      time.gmtime())
 
 
-def GetAvailableNaClSDKArchivesFor(version_string):
-  """Downloads a list of all available archives for a given version.
+def GetPlatformFromArchiveUrl(url):
+  """Get the platform name given an archive url.
 
   Args:
-    version_string: The version to find archives for. (e.g. "18.0.1025.164")
+    url: An archive url.
   Returns:
-    A list of strings, each of which is a platform-specific archive URL. (e.g.
-    "https://commondatastorage.googleapis.com/nativeclient_mirror/nacl/"
-    "nacl_sdk/18.0.1025.164/naclsdk_linux.bz2".
-  """
-  gsutil = buildbot_common.GetGsutil()
-  path = 'gs://%s/%s' % (BUCKET_PATH, version_string,)
-  cmd = [gsutil, 'ls', path]
-  process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-  stdout, stderr = process.communicate()
-  if process.returncode != 0:
-    raise subprocess.CalledProcessError(process.returncode, ''.join(cmd))
-
-  # filter out empty lines
-  files = filter(lambda x: x, stdout.split('\n'))
-  archives = [file for file in files if not file.endswith('.json')]
-  manifests = [file for file in files if file.endswith('.json')]
-
-  # don't include any archives that don't have an associated manifest.
-  return filter(lambda a: a + '.json' in manifests, archives)
+    A platform name (e.g. 'linux')."""
+  match = re.match(r'naclsdk_(.*)\.bz2', posixpath.basename(url))
+  if not match:
+    return None
+  return match.group(1)
 
 
-def GetPlatformsFromArchives(archives):
-  """Get the platform names for a sequence of archives.
+def GetPlatformsFromArchives(archive_urls):
+  """Get the platform names for a sequence of archive urls.
 
   Args:
-    archives: A sequence of archives.
+    archives: A sequence of archive urls.
   Returns:
-    A list of platforms, one for each archive in |archives|."""
+    A list of platforms, one for each url in |archive_urls|."""
   platforms = []
-  for path in archives:
-    match = re.match(r'naclsdk_(.*)\.bz2', os.path.basename(path))
-    if match:
-      platforms.append(match.group(1))
+  for url in archive_urls:
+    platform = GetPlatformFromArchiveUrl(url)
+    if platform:
+      platforms.append(platform)
   return platforms
 
 
-def GetPlatformArchiveBundle(archive):
-  """Downloads the manifest "snippet" for an archive, and reads it as a Bundle.
+class Delegate(object):
+  """Delegate all external access; reading/writing to filesystem, gsutil etc."""
+  def GetRepoManifest(self):
+    """Read the manifest file from the NaCl SDK repository.
 
-  Args:
-    archive: The URL of a platform-specific archive.
-  Returns:
-    An object of type manifest_util.Bundle, read from a JSON file storing
-    metadata for this archive.
-  """
-  gsutil = buildbot_common.GetGsutil()
-  cmd = [gsutil, 'cat', archive + '.json']
-  process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-  stdout, stderr = process.communicate()
-  if process.returncode != 0:
-    return None
+    This manifest is used as a template for the auto updater; only pepper
+    bundles with no archives are considered for auto updating.
 
-  bundle = manifest_util.Bundle('')
-  bundle.LoadDataFromString(stdout)
-  return bundle
+    Returns:
+      A manifest_util.SDKManifest object read from the NaCl SDK repo."""
+    raise NotImplementedError()
 
+  def GetHistory(self):
+    """Read Chrome release history from omahaproxy.appspot.com
 
-def GetTimestampManifestName():
-  """Create a manifest name with a timestamp.
-
-  Returns:
-    A manifest name with an embedded date. This should make it easier to roll
-    back if necessary.
-  """
-  return time.strftime('naclsdk_manifest2.%Y_%m_%d_%H_%M_%S.json',
-      time.gmtime())
+    Here is an example of data from this URL:
+      cros,stable,18.0.1025.168,2012-05-01 17:04:05.962578\n
+      win,canary,20.0.1123.0,2012-05-01 13:59:31.703020\n
+      mac,canary,20.0.1123.0,2012-05-01 11:54:13.041875\n
+      win,stable,18.0.1025.168,2012-04-30 20:34:56.078490\n
+      mac,stable,18.0.1025.168,2012-04-30 20:34:55.231141\n
+      ...
+    Where each line has comma separated values in the following format:
+    platform, channel, version, date/time\n
 
+    Returns:
+      A list where each element is a line from the document, represented as a
+      tuple."""
+    raise NotImplementedError()
 
-def UploadManifest(manifest):
-  """Upload a serialized manifest_util.SDKManifest object.
+  def GsUtil_ls(self, url):
+    """Runs gsutil ls |url|
 
-  Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to
-  gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json.
+    Args:
+      url: The commondatastorage url to list."""
+    raise NotImplementedError()
 
-  Args:
-    manifest: The new manifest to upload.
-  """
-  manifest_string = manifest.GetDataAsString()
-  gsutil = buildbot_common.GetGsutil()
-  timestamp_manifest_path = 'gs://%s/manifest_backups/%s' % (
-      BUCKET_PATH, GetTimestampManifestName())
-
-  cmd = [gsutil, 'cp', '-a', 'public-read', '-', timestamp_manifest_path]
-  process = subprocess.Popen(cmd, stdin=subprocess.PIPE)
-  stdout, stderr = process.communicate(manifest_string)
-
-  # copy from timestampped copy over the official manifest.
-  cmd = [gsutil, 'cp', '-a', 'public-read', timestamp_manifest_path,
-      GS_SDK_MANIFEST]
-  subprocess.check_call(cmd)
-
-
-def main(args):
-  manifest = GetChromeRepoSDKManifest()
+  def GsUtil_cat(self, url):
+    """Runs gsutil cat |url|
+
+    Args:
+      url: The commondatastorage url to read from."""
+    raise NotImplementedError()
+
+  def GsUtil_cp(self, src, dest, stdin=None):
+    """Runs gsutil cp |src| |dest|
+
+    Args:
+      src: The file path or url to copy from.
+      dest: The file path or url to copy to.
+      stdin: If src is '-', this is used as the stdin to give to gsutil. The
+          effect is that text in stdin is copied to |dest|."""
+    raise NotImplementedError()
+
+
+  def Print(self, *args):
+    """Print a message."""
+    raise NotImplementedError()
+
+
+class RealDelegate(Delegate):
+  def __init__(self):
+    pass
+
+  def GetRepoManifest(self):
+    """See Delegate.GetRepoManifest"""
+    with open(REPO_MANIFEST, 'r') as sdk_stream:
+      sdk_json_string = sdk_stream.read()
+
+    manifest = manifest_util.SDKManifest()
+    manifest.LoadDataFromString(sdk_json_string)
+    return manifest
+
+  def GetHistory(self):
+    """See Delegate.GetHistory"""
+    url_stream = urllib2.urlopen('https://omahaproxy.appspot.com/history')
+    return [(platform, channel, version, date)
+        for platform, channel, version, date in csv.reader(url_stream)]
+
+  def GsUtil_ls(self, url):
+    """See Delegate.GsUtil_ls"""
+    stdout = self._RunGsUtil(None, 'ls', url)
+
+    # filter out empty lines
+    return filter(None, stdout.split('\n'))
+
+  def GsUtil_cat(self, url):
+    """See Delegate.GsUtil_cat"""
+    return self._RunGsUtil(None, 'cat', url)
+
+  def GsUtil_cp(self, src, dest, stdin=None):
+    """See Delegate.GsUtil_cp"""
+    return self._RunGsUtil(stdin, 'cp', '-a', 'public-read', src, dest)
+
+  def Print(self, *args):
+    sys.stdout.write(' '.join(map(str, args)) + '\n')
+
+  def _RunGsUtil(self, stdin, *args):
+    """Run gsutil as a subprocess.
+
+    Args:
+      stdin: If non-None, used as input to the process.
+      *args: Arguments to pass to gsutil. The first argument should be an
+          operation such as ls, cp or cat.
+    Returns:
+      The stdout from the process."""
+    cmd = [buildbot_common.GetGsutil()] + list(args)
+    if stdin:
+      stdin_pipe = subprocess.PIPE
+    else:
+      stdin_pipe = None
+
+    process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE)
+    stdout, _ = process.communicate(stdin)
+
+    if process.returncode != 0:
+      raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd))
+    return stdout
+
+
+class VersionFinder(object):
+  """Finds a version of a pepper bundle that all desired platforms share."""
+  def __init__(self, delegate):
+    self.delegate = delegate
+    self.history = delegate.GetHistory()
+
+  def GetMostRecentSharedVersion(self, major_version, platforms):
+    """Returns the most recent version of a pepper bundle that exists on all
+    given platforms.
+
+    Specifically, the resulting version should be the most recently released
+    (meaning closest to the top of the listing on
+    omahaproxy.appspot.com/history) version that has a Chrome release on all
+    given platforms, and has a pepper bundle archive for each platform as well.
+
+    Args:
+      major_version: The major version of the pepper bundle, e.g. 19.
+      platforms: A sequence of platforms to consider, e.g.
+          ('mac', 'linux', 'win')
+    Returns:
+      A tuple (version, channel, archives). The version is a string such as
+      "19.0.1084.41". The channel is one of ('stable', 'beta', or 'dev').
+      |archives| is a list of archive URLs."""
+    shared_version_generator = self._FindNextSharedVersion(major_version,
+                                                          platforms)
+    version = None
+    while True:
+      try:
+        version, channel = shared_version_generator.next()
+      except StopIteration:
+        raise Exception('No shared version for major_version %s, '
+            'platforms: %s. Last version checked = %s' % (
+            major_version, ', '.join(platforms), version))
+
+      archives = self._GetAvailableNaClSDKArchivesFor(version)
+      archive_platforms = GetPlatformsFromArchives(archives)
+      if set(archive_platforms) == set(platforms):
+        return version, channel, archives
+
+  def _GetPlatformMajorVersionHistory(self, with_major_version, with_platform):
+    """Yields Chrome history for a given platform and major version.
+
+    Args:
+      with_platform: The name of the platform to filter for.
+    Returns:
+      A generator that yields a tuple (channel, version) for each version that
+      matches the platform and major version. The version returned is a tuple as
+      returned from SplitVersion.
+    """
+    for platform, channel, version, _ in self.history:
+      version = SplitVersion(version)
+      if with_platform == platform and with_major_version == version[0]:
+        yield channel, version
+
+  def _FindNextSharedVersion(self, major_version, platforms):
+    """Yields versions of Chrome that exist on all given platforms, in order of
+       newest to oldest.
+
+    Versions are compared in reverse order of release. That is, the most
+    recently updated version will be tested first.
+
+    Args:
+      platforms: A sequence of platforms to filter for. Any other platforms will
+          be ignored.
+    Returns:
+      A generator that yields a tuple (version, channel) for each version that
+      matches all platforms and the major version. The version returned is a
+      string (e.g. "18.0.1025.164").
+    """
+    platform_generators = []
+    for platform in platforms:
+      platform_generators.append(self._GetPlatformMajorVersionHistory(
+          major_version, platform))
+
+    shared_version = None
+    platform_versions = [(tuple(), '')] * len(platforms)
+    while True:
+      try:
+        for i, platform_gen in enumerate(platform_generators):
+          if platform_versions[i][1] != shared_version:
+              platform_versions[i] = platform_gen.next()
+      except StopIteration:
+        return
+
+      shared_version = min(v for c, v in platform_versions)
+
+      if all(v == shared_version for c, v in platform_versions):
+        # grab the channel from an arbitrary platform
+        first_platform = platform_versions[0]
+        channel = first_platform[0]
+        yield JoinVersion(shared_version), channel
+
+        # force increment to next version for all platforms
+        shared_version = None
+
+  def _GetAvailableNaClSDKArchivesFor(self, version_string):
+    """Downloads a list of all available archives for a given version.
+
+    Args:
+      version_string: The version to find archives for. (e.g. "18.0.1025.164")
+    Returns:
+      A list of strings, each of which is a platform-specific archive URL. (e.g.
+      "https://commondatastorage.googleapis.com/nativeclient_mirror/nacl/"
+      "nacl_sdk/18.0.1025.164/naclsdk_linux.bz2".
+    """
+    files = self.delegate.GsUtil_ls(GS_BUCKET_PATH + version_string)
+    archives = [file for file in files if not file.endswith('.json')]
+    manifests = [file for file in files if file.endswith('.json')]
+
+    # don't include any archives that don't have an associated manifest.
+    return filter(lambda a: a + '.json' in manifests, archives)
+
+
+class Updater(object):
+  def __init__(self, delegate):
+    self.delegate = delegate
+    self.versions_to_update = []
+
+  def AddVersionToUpdate(self, bundle_name, version, channel, archives):
+    """Add a pepper version to update in the uploaded manifest.
+
+    Args:
+      bundle_name: The name of the pepper bundle, e.g. 'pepper_18'
+      version: The version of the pepper bundle, e.g. '18.0.1025.64'
+      channel: The stability of the pepper bundle, e.g. 'beta'
+      archives: A sequence of archive URLs for this bundle."""
+    self.versions_to_update.append((bundle_name, version, channel, archives))
+
+  def Update(self, manifest):
+    """Update a manifest and upload it.
+
+    Args:
+      manifest: The manifest used as a template for updating. Only pepper
+      bundles that contain no archives will be considered for auto-updating."""
+    for bundle_name, version, channel, archives in self.versions_to_update:
+      self.delegate.Print('Updating %s to %s...' % (bundle_name, version))
+      bundle = manifest.GetBundle(bundle_name)
+      bundle_recommended = bundle.recommended
+      for archive in archives:
+        platform_bundle = self._GetPlatformArchiveBundle(archive)
+        bundle.MergeWithBundle(platform_bundle)
+      bundle.stability = channel
+      bundle.recommended = bundle_recommended
+      manifest.MergeBundle(bundle)
+    self._UploadManifest(manifest)
+    self.delegate.Print('Done.')
+
+  def _GetPlatformArchiveBundle(self, archive):
+    """Downloads the manifest "snippet" for an archive, and reads it as a
+       Bundle.
+
+    Args:
+      archive: The URL of a platform-specific archive.
+    Returns:
+      An object of type manifest_util.Bundle, read from a JSON file storing
+      metadata for this archive.
+    """
+    stdout = self.delegate.GsUtil_cat(GS_BUCKET_PATH + archive + '.json')
+    bundle = manifest_util.Bundle('')
+    bundle.LoadDataFromString(stdout)
+    return bundle
+
+  def _UploadManifest(self, manifest):
+    """Upload a serialized manifest_util.SDKManifest object.
+
+    Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to
+    gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json.
+
+    Args:
+      manifest: The new manifest to upload.
+    """
+    timestamp_manifest_path = GS_BUCKET_PATH + GetTimestampManifestName()
+    self.delegate.GsUtil_cp('-', timestamp_manifest_path,
+        stdin=manifest.GetDataAsString())
+
+    # copy from timestampped copy over the official manifest.
+    self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST)
+
+
+def Run(delegate, platforms):
+  """Entry point for the auto-updater."""
+  manifest = delegate.GetRepoManifest()
   auto_update_major_versions = []
   for bundle in manifest.GetBundles():
+    if not bundle.name.startswith('pepper_'):
+      continue
     archives = bundle.GetArchives()
     if not archives:
       auto_update_major_versions.append(bundle.version)
 
-  if auto_update_major_versions:
-    history = GetChromeVersionHistory()
-    platforms = ('mac', 'win', 'linux')
-
-    versions_to_update = []
-    for major_version in auto_update_major_versions:
-      shared_version_generator = FindNextSharedVersion(history, major_version,
-                                                       platforms)
-      version = None
-      while True:
-        try:
-          version, channel = shared_version_generator.next()
-        except StopIteration:
-          raise Exception('No shared version for major_version %s, '
-              'platforms: %s. Last version checked = %s' % (
-              major_version, ', '.join(platforms), version))
-
-        archives = GetAvailableNaClSDKArchivesFor(version)
-        archive_platforms = GetPlatformsFromArchives(archives)
-        if set(archive_platforms) == set(platforms):
-          versions_to_update.append((major_version, version, channel, archives))
-          break
-
-    if versions_to_update:
-      for major_version, version, channel, archives in versions_to_update:
-        bundle_name = 'pepper_%s' % (major_version,)
-        bundle = manifest.GetBundle(bundle_name)
-        bundle_recommended = bundle.recommended
-        for archive in archives:
-          platform_bundle = GetPlatformArchiveBundle(archive)
-          bundle.MergeWithBundle(platform_bundle)
-        bundle.stability = channel
-        bundle.recommended = bundle_recommended
-        manifest.MergeBundle(bundle)
-    UploadManifest(manifest)
-
-  else:
-    print 'No versions need auto-updating.'
+  if not auto_update_major_versions:
+    delegate.Print('No versions need auto-updating.')
+    return
+
+  version_finder = VersionFinder(delegate)
+  updater = Updater(delegate)
+
+  for major_version in auto_update_major_versions:
+    version, channel, archives = version_finder.GetMostRecentSharedVersion(
+        major_version, platforms)
+
+    bundle_name = 'pepper_' + str(major_version)
+    updater.AddVersionToUpdate(bundle_name, version, channel, archives)
+
+  updater.Update(manifest)
+
+
+def main(args):
+  delegate = RealDelegate()
+  Run(delegate, ('mac', 'win', 'linux'))
 
 
 if __name__ == '__main__':