From 66d4cc107df0ab12f77705e562d8b6e9e6a10070 Mon Sep 17 00:00:00 2001 From: "binji@chromium.org" Date: Mon, 4 Jun 2012 22:46:16 +0000 Subject: [NaCl SDK] Add support for pepper_canary bundle. Also some stability changes in naclsdk_manifest2.json. BUG=126288 TEST=test_update_manifest.UpdateCanary* Review URL: https://chromiumcodereview.appspot.com/10442047 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@140410 0039d316-1c4b-4281-b951-d872f2087c98 --- .../src/build_tools/json/naclsdk_manifest2.json | 27 ++-- .../src/build_tools/tests/test_update_manifest.py | 140 ++++++++++++++++-- .../src/build_tools/update_nacl_manifest.py | 156 ++++++++++++++++++--- 3 files changed, 281 insertions(+), 42 deletions(-) (limited to 'native_client_sdk') diff --git a/native_client_sdk/src/build_tools/json/naclsdk_manifest2.json b/native_client_sdk/src/build_tools/json/naclsdk_manifest2.json index 2c0db96..d4e9e9f 100644 --- a/native_client_sdk/src/build_tools/json/naclsdk_manifest2.json +++ b/native_client_sdk/src/build_tools/json/naclsdk_manifest2.json @@ -89,7 +89,7 @@ { "name": "pepper_17", "description": "Chrome 17 bundle, revision 112997", - "stability": "stable", + "stability": "post_stable", "recommended": "no", "version": 17, "archives": [ @@ -123,8 +123,8 @@ { "name": "pepper_18", "description": "Chrome 18 bundle, revision 124251", - "stability": "beta", - "recommended": "yes", + "stability": "post_stable", + "recommended": "no", "version": 18, "repath": "pepper_18", "archives": [ @@ -158,7 +158,7 @@ { "name": "pepper_19", "description": "Chrome 19 bundle, revision 129248", - "stability": "dev", + "stability": "stable", "recommended": "yes", "version": 19, "repath": "pepper_19", @@ -192,14 +192,23 @@ }, { "name": "pepper_20", - "description": "Chrome 20 bundle, revision 133336", - "stability": "dev", + "description": "Chrome 20 bundle, revision xxxxx", + "stability": "beta", "recommended": "no", "version": 20, "repath": "pepper_20", - "archives": [ - ], - "revision": 134804 + "archives": [], + "revision": 0 + }, + { + "name": "pepper_canary", + "description": "Chrome Canary", + "stability": "canary", + "recommended": "no", + "version": 0, + "repath": "", + "archives": [], + "revision": 0 } ], "manifest_version": 2 diff --git a/native_client_sdk/src/build_tools/tests/test_update_manifest.py b/native_client_sdk/src/build_tools/tests/test_update_manifest.py index 33e0f11..600e63e 100755 --- a/native_client_sdk/src/build_tools/tests/test_update_manifest.py +++ b/native_client_sdk/src/build_tools/tests/test_update_manifest.py @@ -18,6 +18,7 @@ BUILD_TOOLS_DIR = os.path.dirname(SCRIPT_DIR) sys.path.append(BUILD_TOOLS_DIR) import manifest_util import update_nacl_manifest +from update_nacl_manifest import CANARY_BUNDLE_NAME HTTPS_BASE_URL = 'https://commondatastorage.googleapis.com' \ @@ -26,6 +27,7 @@ HTTPS_BASE_URL = 'https://commondatastorage.googleapis.com' \ OS_CR = ('cros',) OS_M = ('mac',) OS_ML = ('mac', 'linux') +OS_MW = ('mac', 'win') OS_MLW = ('mac', 'linux', 'win') STABLE = 'stable' BETA = 'beta' @@ -80,17 +82,24 @@ def MakeNonPepperBundle(name, with_archives=False): return bundle -def MakeBundle(major_version, revision, version=None, host_oses=None): - assert version is None or version.split('.')[0] == str(major_version) - bundle_name = 'pepper_' + str(major_version) +def MakeBundle(major_version, revision=0, version=None, host_oses=None, + stability='dev'): + assert (version is None or + version.split('.')[0] == 'trunk' or + version.split('.')[0] == str(major_version)) + if stability == CANARY: + bundle_name = CANARY_BUNDLE_NAME + else: + bundle_name = 'pepper_' + str(major_version) + bundle = manifest_util.Bundle(bundle_name) bundle.version = major_version bundle.revision = revision bundle.description = 'Chrome %s bundle, revision %s' % (major_version, revision) - bundle.repath = bundle_name + bundle.repath = 'pepper_' + str(major_version) bundle.recommended = 'no' - bundle.stability = 'dev' + bundle.stability = stability if host_oses: for host_os in host_oses: @@ -144,10 +153,11 @@ class MakeFiles(dict): class TestDelegate(update_nacl_manifest.Delegate): - def __init__(self, manifest, history, files): + def __init__(self, manifest, history, files, version_mapping): self.manifest = manifest self.history = history self.files = files + self.version_mapping = version_mapping def GetRepoManifest(self): return self.manifest @@ -155,6 +165,9 @@ class TestDelegate(update_nacl_manifest.Delegate): def GetHistory(self): return self.history + def GetTrunkRevision(self, version): + return self.version_mapping[version] + def GsUtil_ls(self, url): path = GetPathFromGsUrl(url) result = [] @@ -190,12 +203,19 @@ V18_0_1025_175 = '18.0.1025.175' V18_0_1025_184 = '18.0.1025.184' V19_0_1084_41 = '19.0.1084.41' V19_0_1084_67 = '19.0.1084.67' +V21_0_1145_0 = '21.0.1145.0' +V21_0_1166_0 = '21.0.1166.0' +VTRUNK_138079 = 'trunk.138079' B18_0_1025_163_R1_MLW = MakeBundle(18, 1, V18_0_1025_163, OS_MLW) B18_0_1025_184_R1_MLW = MakeBundle(18, 1, V18_0_1025_184, OS_MLW) -B18_R1_NONE = MakeBundle(18, '1') +B18_R1_NONE = MakeBundle(18) B19_0_1084_41_R1_MLW = MakeBundle(19, 1, V19_0_1084_41, OS_MLW) B19_0_1084_67_R1_MLW = MakeBundle(19, 1, V19_0_1084_67, OS_MLW) -B19_R1_NONE = MakeBundle(19, '1') +B19_R1_NONE = MakeBundle(19) +BCANARY_R1_NONE = MakeBundle(0, stability=CANARY) +B21_0_1145_0_R1_MLW = MakeBundle(21, 1, V21_0_1145_0, OS_MLW) +B21_0_1166_0_R1_MW = MakeBundle(21, 1, V21_0_1166_0, OS_MW) +BTRUNK_138079_R1_MLW = MakeBundle(21, 1, VTRUNK_138079, OS_MLW) NON_PEPPER_BUNDLE_NOARCHIVES = MakeNonPepperBundle('foo') NON_PEPPER_BUNDLE_ARCHIVES = MakeNonPepperBundle('bar', with_archives=True) @@ -204,13 +224,14 @@ class TestUpdateManifest(unittest.TestCase): def setUp(self): self.history = MakeHistory() self.files = MakeFiles() + self.version_mapping = {} self.delegate = None self.uploaded_manifest = None self.manifest = None def _MakeDelegate(self): self.delegate = TestDelegate(self.manifest, self.history.history, - self.files) + self.files, self.version_mapping) def _Run(self, host_oses): update_nacl_manifest.Run(self.delegate, host_oses) @@ -224,14 +245,28 @@ class TestUpdateManifest(unittest.TestCase): self.files['naclsdk_manifest2.json']) def _AssertUploadedManifestHasBundle(self, bundle, stability): - uploaded_manifest_bundle = self.uploaded_manifest.GetBundle(bundle.name) + if stability == CANARY: + bundle_name = CANARY_BUNDLE_NAME + else: + bundle_name = bundle.name + + uploaded_manifest_bundle = self.uploaded_manifest.GetBundle(bundle_name) # Bundles that we create in the test (and in the manifest snippets) have # their stability set to "dev". update_nacl_manifest correctly updates it. # So we have to force the stability of |bundle| so they compare equal. test_bundle = copy.copy(bundle) test_bundle.stability = stability + if stability == CANARY: + test_bundle.name = CANARY_BUNDLE_NAME self.assertEqual(uploaded_manifest_bundle, test_bundle) + def _AddCsvHistory(self, history): + import csv + import cStringIO + history_stream = cStringIO.StringIO(history) + self.history.history = [(platform, channel, version, date) + for platform, channel, version, date in csv.reader(history_stream)] + def testNoUpdateNeeded(self): self.manifest = MakeManifest(B18_0_1025_163_R1_MLW) self._MakeDelegate() @@ -375,6 +410,91 @@ class TestUpdateManifest(unittest.TestCase): self.assertEqual(uploaded_bundle.revision, 1234) self.assertEqual(uploaded_bundle.version, 18) + def testUpdateCanary(self): + # Note that the bundle in naclsdk_manifest2.json will be called + # CANARY_BUNDLE_NAME, whereas the bundle in the manifest "snippet" will be + # called "pepper_21". + canary_bundle = copy.deepcopy(BCANARY_R1_NONE) + self.manifest = MakeManifest(canary_bundle) + self.history.Add(OS_MW, CANARY, V21_0_1145_0) + self.files.Add(B21_0_1145_0_R1_MLW) + self._MakeDelegate() + self._Run(OS_MLW) + self._ReadUploadedManifest() + self._AssertUploadedManifestHasBundle(B21_0_1145_0_R1_MLW, CANARY) + + def testUpdateCanaryUseTrunkArchives(self): + canary_bundle = copy.deepcopy(BCANARY_R1_NONE) + self.manifest = MakeManifest(canary_bundle) + self.history.Add(OS_MW, CANARY, V21_0_1166_0) + self.files.Add(B21_0_1166_0_R1_MW) + self.files.Add(BTRUNK_138079_R1_MLW) + self.version_mapping[V21_0_1166_0] = VTRUNK_138079 + self._MakeDelegate() + self._Run(OS_MLW) + self._ReadUploadedManifest() + + test_bundle = copy.deepcopy(B21_0_1166_0_R1_MW) + test_bundle.AddArchive(BTRUNK_138079_R1_MLW.GetArchive('linux')) + self._AssertUploadedManifestHasBundle(test_bundle, CANARY) + + def testCanaryUseOnlyTrunkArchives(self): + self.manifest = MakeManifest(copy.deepcopy(BCANARY_R1_NONE)) + history = """win,canary,21.0.1163.0,2012-06-04 12:35:44.784446 +mac,canary,21.0.1163.0,2012-06-04 11:54:09.433166""" + self._AddCsvHistory(history) + self.version_mapping['21.0.1163.0'] = 'trunk.140240' + my_bundle = MakeBundle(21, 140240, '21.0.1163.0', OS_MLW) + self.files.Add(my_bundle) + self._MakeDelegate() + self._Run(OS_MLW) + self._ReadUploadedManifest() + self._AssertUploadedManifestHasBundle(my_bundle, CANARY) + + def testCanaryShouldOnlyUseCanaryVersions(self): + canary_bundle = copy.deepcopy(BCANARY_R1_NONE) + self.manifest = MakeManifest(canary_bundle) + self.history.Add(OS_MW, CANARY, V21_0_1166_0) + self.history.Add(OS_MW, BETA, V19_0_1084_41) + self.files.Add(B19_0_1084_41_R1_MLW) + self.version_mapping[V21_0_1166_0] = VTRUNK_138079 + self._MakeDelegate() + self.assertRaises(Exception, self._Run, OS_MLW) + + def testMissingCanaryFollowedByStableShouldWork(self): + history = """win,canary,21.0.1160.0,2012-06-01 19:44:35.936109 +mac,canary,21.0.1160.0,2012-06-01 18:20:02.003123 +mac,stable,19.0.1084.52,2012-06-01 17:59:21.559710 +win,canary,21.0.1159.2,2012-06-01 02:31:43.877688 +mac,stable,19.0.1084.53,2012-06-01 01:39:57.549149 +win,canary,21.0.1158.0,2012-05-31 20:16:55.615236 +win,canary,21.0.1157.0,2012-05-31 17:41:29.516013 +mac,canary,21.0.1158.0,2012-05-31 17:41:27.591354 +mac,beta,20.0.1132.21,2012-05-30 23:45:38.535586 +linux,beta,20.0.1132.21,2012-05-30 23:45:37.025015 +cf,beta,20.0.1132.21,2012-05-30 23:45:36.767529 +win,beta,20.0.1132.21,2012-05-30 23:44:56.675123 +win,canary,21.0.1156.1,2012-05-30 22:28:01.872056 +mac,canary,21.0.1156.1,2012-05-30 21:20:29.920390 +win,canary,21.0.1156.0,2012-05-30 12:46:48.046627 +mac,canary,21.0.1156.0,2012-05-30 12:14:21.305090""" + self.manifest = MakeManifest(copy.deepcopy(BCANARY_R1_NONE)) + self._AddCsvHistory(history) + self.version_mapping = { + '21.0.1160.0': 'trunk.139984', + '21.0.1159.2': 'trunk.139890', + '21.0.1158.0': 'trunk.139740', + '21.0.1157.0': 'unknown', + '21.0.1156.1': 'trunk.139576', + '21.0.1156.0': 'trunk.139984'} + self.files.Add(MakeBundle(21, 139890, '21.0.1159.2', OS_MLW)) + self.files.Add(MakeBundle(21, 0, '21.0.1157.1', ('linux', 'win'))) + my_bundle = MakeBundle(21, 139576, '21.0.1156.1', OS_MLW) + self.files.Add(my_bundle) + self._MakeDelegate() + self._Run(OS_MLW) + self._ReadUploadedManifest() + self._AssertUploadedManifestHasBundle(my_bundle, CANARY) def main(): suite = unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__]) diff --git a/native_client_sdk/src/build_tools/update_nacl_manifest.py b/native_client_sdk/src/build_tools/update_nacl_manifest.py index e3d89bd..c2078c5 100755 --- a/native_client_sdk/src/build_tools/update_nacl_manifest.py +++ b/native_client_sdk/src/build_tools/update_nacl_manifest.py @@ -19,14 +19,15 @@ import time import urllib2 -# TODO(binji) handle pushing pepper_trunk - - MANIFEST_BASENAME = 'naclsdk_manifest2.json' SCRIPT_DIR = os.path.dirname(__file__) REPO_MANIFEST = os.path.join(SCRIPT_DIR, 'json', MANIFEST_BASENAME) GS_BUCKET_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/' GS_SDK_MANIFEST = GS_BUCKET_PATH + MANIFEST_BASENAME +GS_MANIFEST_BACKUP_DIR = GS_BUCKET_PATH + 'manifest_backups/' + +CANARY_BUNDLE_NAME = 'pepper_canary' +CANARY='canary' def SplitVersion(version_string): @@ -114,6 +115,15 @@ class Delegate(object): tuple.""" raise NotImplementedError() + def GetTrunkRevision(self, version): + """Given a Chrome version, get its trunk revision. + + Args: + version: A version string of the form '18.0.1025.64' + Returns: + The revision number for that version, as a string.""" + raise NotImplementedError() + def GsUtil_ls(self, url): """Runs gsutil ls |url| @@ -167,6 +177,11 @@ class RealDelegate(Delegate): return [(platform, channel, version, date) for platform, channel, version, date in csv.reader(url_stream)] + def GetTrunkRevision(self, version): + """See Delegate.GetTrunkRevision""" + url = 'http://omahaproxy.appspot.com/revision?version=%s' % (version,) + return 'trunk.%s' % (urllib2.urlopen(url).read(),) + def GsUtil_ls(self, url): """See Delegate.GsUtil_ls""" stdout = self._RunGsUtil(None, 'ls', url) @@ -232,26 +247,95 @@ class VersionFinder(object): A tuple (version, channel, archives). The version is a string such as "19.0.1084.41". The channel is one of ('stable', 'beta', or 'dev'). |archives| is a list of archive URLs.""" - shared_version_generator = self._FindNextSharedVersion(major_version, - platforms) + shared_version_generator = self._FindNextSharedVersion( + platforms, + lambda platform: self._GetPlatformMajorVersionHistory(major_version, + platform)) + return self._DoGetMostRecentSharedVersion(platforms, + shared_version_generator, allow_trunk_revisions=False) + + def GetMostRecentSharedCanary(self, platforms): + """Returns the most recent version of a canary pepper bundle that exists on + all given platforms. + + Canary is special-cased because we don't care about its major version; we + always use the most recent canary, regardless of major version. + + Args: + platforms: A sequence of platforms to consider, e.g. + ('mac', 'linux', 'win') + Returns: + A tuple (version, channel, archives). The version is a string such as + "19.0.1084.41". The channel is always 'canary'. |archives| is a list of + archive URLs.""" + # We don't ship canary on Linux, so it won't appear in self.history. + # Instead, we can use the matching Linux trunk build for that version. + shared_version_generator = self._FindNextSharedVersion( + set(platforms) - set(('linux',)), + self._GetPlatformCanaryHistory) + return self._DoGetMostRecentSharedVersion(platforms, + shared_version_generator, allow_trunk_revisions=True) + + def _DoGetMostRecentSharedVersion(self, platforms, shared_version_generator, + allow_trunk_revisions): + """Returns the most recent version of a pepper bundle that exists on all + given platforms. + + This function does the real work for the public GetMostRecentShared* above. + + Args: + platforms: A sequence of platforms to consider, e.g. + ('mac', 'linux', 'win') + shared_version_generator: A generator that will yield (version, channel) + tuples in order of most recent to least recent. + allow_trunk_revisions: If True, will search for archives using the + trunk revision that matches the branch version. + Returns: + A tuple (version, channel, archives). The version is a string such as + "19.0.1084.41". The channel is one of ('stable', 'beta', 'dev', + 'canary'). |archives| is a list of archive URLs.""" version = None + skipped_versions = [] while True: try: version, channel = shared_version_generator.next() except StopIteration: - raise Exception('No shared version for major_version %s, ' - 'platforms: %s. Last version checked = %s' % ( - major_version, ', '.join(platforms), version)) + msg = 'No shared version for platforms: %s\n' % (', '.join(platforms)) + msg += 'Last version checked = %s.\n' % (version,) + if skipped_versions: + msg += 'Versions skipped due to missing archives:\n' + for version, channel, archives in skipped_versions: + if archives: + archive_msg = '(%s available)' % (', '.join(archives)) + else: + archive_msg = '(none available)' + msg += ' %s (%s) %s\n' % (version, channel, archive_msg) + raise Exception(msg) archives = self._GetAvailableNaClSDKArchivesFor(version) - archive_platforms = GetPlatformsFromArchives(archives) - if set(archive_platforms) == set(platforms): + missing_platforms = set(platforms) - \ + set(GetPlatformsFromArchives(archives)) + if allow_trunk_revisions and missing_platforms: + # Try to find trunk archives for platforms that are missing archives. + trunk_version = self.delegate.GetTrunkRevision(version) + trunk_archives = self._GetAvailableNaClSDKArchivesFor(trunk_version) + for trunk_archive in trunk_archives: + trunk_archive_platform = GetPlatformFromArchiveUrl(trunk_archive) + if trunk_archive_platform in missing_platforms: + archives.append(trunk_archive) + missing_platforms.discard(trunk_archive_platform) + + if not missing_platforms: return version, channel, archives + skipped_versions.append((version, channel, archives)) + def _GetPlatformMajorVersionHistory(self, with_major_version, with_platform): """Yields Chrome history for a given platform and major version. Args: + with_major_version: The major version to filter for. If 0, match all + versions. with_platform: The name of the platform to filter for. Returns: A generator that yields a tuple (channel, version) for each version that @@ -260,10 +344,28 @@ class VersionFinder(object): """ for platform, channel, version, _ in self.history: version = SplitVersion(version) - if with_platform == platform and with_major_version == version[0]: + if (with_platform == platform and + (with_major_version == 0 or with_major_version == version[0])): + yield channel, version + + def _GetPlatformCanaryHistory(self, with_platform): + """Yields Chrome history for a given platform, but only for canary + versions. + + Args: + with_platform: The name of the platform to filter for. + Returns: + A generator that yields a tuple (channel, version) for each version that + matches the platform and uses the canary channel. The version returned is + a tuple as returned from SplitVersion. + """ + for platform, channel, version, _ in self.history: + version = SplitVersion(version) + if with_platform == platform and channel==CANARY: yield channel, version - def _FindNextSharedVersion(self, major_version, platforms): + + def _FindNextSharedVersion(self, platforms, generator_func): """Yields versions of Chrome that exist on all given platforms, in order of newest to oldest. @@ -280,8 +382,7 @@ class VersionFinder(object): """ platform_generators = [] for platform in platforms: - platform_generators.append(self._GetPlatformMajorVersionHistory( - major_version, platform)) + platform_generators.append(generator_func(platform)) shared_version = None platform_versions = [(tuple(), '')] * len(platforms) @@ -315,6 +416,7 @@ class VersionFinder(object): All returned URLs will use the gs:// schema.""" files = self.delegate.GsUtil_ls(GS_BUCKET_PATH + version_string) + assert all(file.startswith('gs://') for file in files) archives = [file for file in files if not file.endswith('.json')] @@ -351,6 +453,10 @@ class Updater(object): bundle_recommended = bundle.recommended for archive in archives: platform_bundle = self._GetPlatformArchiveBundle(archive) + # Normally the manifest snippet's bundle name matches our bundle name. + # pepper_canary, however is called "pepper_###" in the manifest + # snippet. + platform_bundle.name = bundle_name bundle.MergeWithBundle(platform_bundle) bundle.stability = channel bundle.recommended = bundle_recommended @@ -386,7 +492,8 @@ class Updater(object): Args: manifest: The new manifest to upload. """ - timestamp_manifest_path = GS_BUCKET_PATH + GetTimestampManifestName() + timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \ + GetTimestampManifestName() self.delegate.GsUtil_cp('-', timestamp_manifest_path, stdin=manifest.GetDataAsString()) @@ -397,27 +504,30 @@ class Updater(object): def Run(delegate, platforms): """Entry point for the auto-updater.""" manifest = delegate.GetRepoManifest() - auto_update_major_versions = [] + auto_update_bundles = [] for bundle in manifest.GetBundles(): if not bundle.name.startswith('pepper_'): continue archives = bundle.GetArchives() if not archives: - auto_update_major_versions.append(bundle.version) + auto_update_bundles.append(bundle) - if not auto_update_major_versions: + if not auto_update_bundles: delegate.Print('No versions need auto-updating.') return version_finder = VersionFinder(delegate) updater = Updater(delegate) - for major_version in auto_update_major_versions: - version, channel, archives = version_finder.GetMostRecentSharedVersion( - major_version, platforms) + for bundle in auto_update_bundles: + if bundle.name == CANARY_BUNDLE_NAME: + version, channel, archives = version_finder.GetMostRecentSharedCanary( + platforms) + else: + version, channel, archives = version_finder.GetMostRecentSharedVersion( + bundle.version, platforms) - bundle_name = 'pepper_' + str(major_version) - updater.AddVersionToUpdate(bundle_name, version, channel, archives) + updater.AddVersionToUpdate(bundle.name, version, channel, archives) updater.Update(manifest) -- cgit v1.1