update_nacl_manifest improvements.

Add ability to leave size and checksum blank in the
manifest and have it be filled in based by downloading
the current files.

Add -v/--verbose option which prints more information.

Add -d/--debug option which will show backtraces for
Errors() which is not normally useful.

When in --dryrun mode write the generated file locally
along with the current online one and show a diff of 
the two. 

BUG=155906

Review URL: https://chromiumcodereview.appspot.com/11200002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@162445 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 134 insertions, 56 deletions

diff --git a/native_client_sdk/src/build_tools/json/naclsdk_manifest2.json b/native_client_sdk/src/build_tools/json/naclsdk_manifest2.json
index 661e714..12e348d 100644
--- a/native_client_sdk/src/build_tools/json/naclsdk_manifest2.json
+++ b/native_client_sdk/src/build_tools/json/naclsdk_manifest2.json
@@ -61,12 +61,8 @@
       "repath": "vs_addin",
       "archives": [
         {
-          "url": "https://commondatastorage.googleapis.com/nativeclient-mirror/nacl/nacl_sdk/sdk/1449/vs_addin.tgz",
-          "checksum": {
-            "sha1": "91acaf2535ad8b464a799af5251197bab02869ee"
-          },
-          "host_os": "win",
-          "size": 86759
+          "url": "https://commondatastorage.googleapis.com/nativeclient-mirror/nacl/nacl_sdk/sdk/${revision}/vs_addin.tgz",
+          "host_os": "win"
         }
       ],
       "revision": 1449
diff --git a/native_client_sdk/src/build_tools/manifest_util.py b/native_client_sdk/src/build_tools/manifest_util.py
index 4a479d8..1e055d7 100644
--- a/native_client_sdk/src/build_tools/manifest_util.py
+++ b/native_client_sdk/src/build_tools/manifest_util.py
@@ -5,7 +5,9 @@
 import copy
 import hashlib
 import json
+import string
 import sys
+import urllib2
 
 MANIFEST_VERSION = 2
 
@@ -57,8 +59,8 @@ def DictToJSON(pydict):
 
 
 def DownloadAndComputeHash(from_stream, to_stream=None, progress_func=None):
-  ''' Download the archive data from from-stream and generate sha1 and
-      size info.
+  '''Download the archive data from from-stream and generate sha1 and
+  size info.
 
   Args:
     from_stream:   An input stream that supports read.
@@ -146,6 +148,20 @@ class Archive(dict):
       if key not in VALID_ARCHIVE_KEYS:
         raise Error('Archive "%s" has invalid attribute "%s"' % (host_os, key))
 
+  def UpdateVitals(self, revision):
+    """Update the size and checksum information for this archive
+    based on the content currently at the URL.
+
+    This allows the template mandifest to be maintained without
+    the need to size and checksums to be present.
+    """
+    template = string.Template(self['url'])
+    self['url'] = template.substitute({'revision': revision})
+    from_stream = urllib2.urlopen(self['url'])
+    sha1_hash, size = DownloadAndComputeHash(from_stream)
+    self['size'] = size
+    self['checksum'] = { 'sha1': sha1_hash }
+
   def __getattr__(self, name):
     """Retrieve values from this dict using attributes.
 
@@ -215,6 +231,9 @@ class Bundle(dict):
       else:
         self[k] = v
 
+  def __str__(self):
+    return self.GetDataAsString()
+
   def GetDataAsString(self):
     """Returns the JSON bundle object, pretty-printed"""
     return DictToJSON(self)
@@ -245,21 +264,21 @@ class Bundle(dict):
       else:
         self[key] = value
 
-  def Validate(self):
+  def Validate(self, add_missing_info=False):
     """Validate the content of the bundle. Raise an Error if an invalid or
        missing field is found. """
     # Check required fields.
-    if not self.get(NAME_KEY, None):
+    if not self.get(NAME_KEY):
       raise Error('Bundle has no name')
-    if self.get(REVISION_KEY, None) == None:
+    if self.get(REVISION_KEY) == None:
       raise Error('Bundle "%s" is missing a revision number' % self[NAME_KEY])
-    if self.get(VERSION_KEY, None) == None:
+    if self.get(VERSION_KEY) == None:
       raise Error('Bundle "%s" is missing a version number' % self[NAME_KEY])
-    if not self.get('description', None):
+    if not self.get('description'):
       raise Error('Bundle "%s" is missing a description' % self[NAME_KEY])
-    if not self.get('stability', None):
+    if not self.get('stability'):
       raise Error('Bundle "%s" is missing stability info' % self[NAME_KEY])
-    if self.get('recommended', None) == None:
+    if self.get('recommended') == None:
       raise Error('Bundle "%s" is missing the recommended field' %
                   self[NAME_KEY])
     # Check specific values
@@ -277,6 +296,8 @@ class Bundle(dict):
                     (self[NAME_KEY], key))
     # Validate the archives
     for archive in self[ARCHIVES_KEY]:
+      if add_missing_info and 'size' not in archive:
+        archive.UpdateVitals(self[REVISION_KEY])
       archive.Validate()
 
   def GetArchive(self, host_os_name):
@@ -387,7 +408,7 @@ class SDKManifest(object):
         "bundles": [],
         }
 
-  def Validate(self):
+  def Validate(self, add_missing_info=False):
     """Validate the Manifest file and raises an exception for problems"""
     # Validate the manifest top level
     if self._manifest_data["manifest_version"] > MANIFEST_VERSION:
@@ -399,7 +420,7 @@ class SDKManifest(object):
         raise Error('Manifest has invalid attribute "%s"' % key)
     # Validate each bundle
     for bundle in self._manifest_data[BUNDLES_KEY]:
-      bundle.Validate()
+      bundle.Validate(add_missing_info)
 
   def GetBundle(self, name):
     """Get a bundle from the array of bundles.
@@ -457,7 +478,7 @@ class SDKManifest(object):
            (local_bundle[VERSION_KEY], local_bundle[REVISION_KEY]) <
            (bundle[VERSION_KEY], bundle[REVISION_KEY]))
 
-  def MergeBundle(self, bundle, allow_existing = True):
+  def MergeBundle(self, bundle, allow_existing=True):
     """Merge a Bundle into this manifest.
 
     The new bundle is added if not present, or merged into the existing bundle.
@@ -483,7 +504,7 @@ class SDKManifest(object):
       manifest: The manifest to merge.
     '''
     for bundle in manifest.GetBundles():
-      self.MergeBundle(bundle, allow_existing = False)
+      self.MergeBundle(bundle, allow_existing=False)
 
   def FilterBundles(self, predicate):
     """Filter the list of bundles by |predicate|.
@@ -497,7 +518,7 @@ class SDKManifest(object):
     """
     self._manifest_data[BUNDLES_KEY] = filter(predicate, self.GetBundles())
 
-  def LoadDataFromString(self, json_string):
+  def LoadDataFromString(self, json_string, add_missing_info=False):
     """Load a JSON manifest string. Raises an exception if json_string
        is not well-formed JSON.
 
@@ -517,7 +538,10 @@ class SDKManifest(object):
         self._manifest_data[key] = bundles
       else:
         self._manifest_data[key] = value
-    self.Validate()
+    self.Validate(add_missing_info)
+
+  def __str__(self):
+    return self.GetDataAsString()
 
   def GetDataAsString(self):
     """Returns the current JSON manifest object, pretty-printed"""
diff --git a/native_client_sdk/src/build_tools/nacl-mono-buildbot.py b/native_client_sdk/src/build_tools/nacl-mono-buildbot.py
index ff849e3..7931799 100755
--- a/native_client_sdk/src/build_tools/nacl-mono-buildbot.py
+++ b/native_client_sdk/src/build_tools/nacl-mono-buildbot.py
@@ -13,7 +13,7 @@ import build_utils
 
 GS_MANIFEST_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/'
 SDK_MANIFEST = 'naclsdk_manifest2.json'
-MONO_MANIFEST = 'naclmono_manifest.json' 
+MONO_MANIFEST = 'naclmono_manifest.json'
 
 def build_and_upload_mono(sdk_revision, pepper_revision, sdk_url,
                           upload_path, args):
@@ -69,7 +69,7 @@ def get_sdk_build_info():
   manifest_file = open(MONO_MANIFEST, 'r')
   mono_manifest = json.loads(manifest_file.read())
   manifest_file.close()
-  
+
   ret = []
   mono_manifest_dirty = False
   # Check to see if we need to rebuild mono based on sdk revision
@@ -145,14 +145,14 @@ def update_mono_sdk_json(infos):
           value[loc] = bundle
         else:
           value.append(bundle)
-    
+
   # Write out the file locally, then upload to its known location.
   manifest_file = open(MONO_MANIFEST, 'w')
   manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
   manifest_file.close()
   buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
       MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
-  
+
 
 def main(args):
   args = args[1:]
@@ -176,7 +176,7 @@ def main(args):
       build_and_upload_mono(None, info['pepper_revision'], info['sdk_url'],
           upload_path, args)
     update_mono_sdk_json(infos)
-    
+
 
 
 if __name__ == '__main__':
diff --git a/native_client_sdk/src/build_tools/tests/test_update_nacl_manifest.py b/native_client_sdk/src/build_tools/tests/test_update_nacl_manifest.py
index 5d2525b..1749331 100755
--- a/native_client_sdk/src/build_tools/tests/test_update_nacl_manifest.py
+++ b/native_client_sdk/src/build_tools/tests/test_update_nacl_manifest.py
@@ -5,10 +5,12 @@
 
 import copy
 import datetime
+import hashlib
 import os
 import posixpath
 import subprocess
 import sys
+import tempfile
 import unittest
 import urlparse
 
@@ -158,6 +160,7 @@ class TestDelegate(update_nacl_manifest.Delegate):
     self.history = history
     self.files = files
     self.version_mapping = version_mapping
+    self.dryrun = 0
 
   def GetRepoManifest(self):
     return self.manifest
@@ -513,11 +516,35 @@ mac,canary,21.0.1156.0,2012-05-30 12:14:21.305090"""
     self.assertEqual(len(self.uploaded_manifest.GetBundles()), 1)
 
 
-def main():
-  suite = unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__])
-  result = unittest.TextTestRunner(verbosity=2).run(suite)
+class TestUpdateVitals(unittest.TestCase):
+  def setUp(self):
+    f = tempfile.NamedTemporaryFile('w', prefix="test_update_nacl_manifest")
+    self.test_file = f.name
+    f.close()
+    test_data = "Some test data\n"
+    self.sha1 = hashlib.sha1(test_data).hexdigest()
+    with open(self.test_file, 'w') as f:
+      f.write(test_data)
+
+  def tearDown(self):
+    os.remove(self.test_file)
+
+  def testUpdateVitals(self):
+    archive = manifest_util.Archive(manifest_util.GetHostOS())
+    archive.url = 'file://%s' % os.path.abspath(self.test_file)
+    bundle = MakeBundle(18)
+    bundle.AddArchive(archive)
+    manifest = MakeManifest(bundle)
+    archive = manifest.GetBundles()[0]['archives'][0]
+
+    self.assertTrue('size' not in archive)
+    self.assertTrue('checksum' not in archive)
+
+    manifest.Validate()
+
+    self.assertEqual(archive['size'], 15)
+    self.assertEqual(archive['checksum']['sha1'], self.sha1)
 
-  return int(not result.wasSuccessful())
 
 if __name__ == '__main__':
-  sys.exit(main())
+  unittest.main()
diff --git a/native_client_sdk/src/build_tools/update_nacl_manifest.py b/native_client_sdk/src/build_tools/update_nacl_manifest.py
index 9f2a09a..b624e23 100755
--- a/native_client_sdk/src/build_tools/update_nacl_manifest.py
+++ b/native_client_sdk/src/build_tools/update_nacl_manifest.py
@@ -167,9 +167,10 @@ class Delegate(object):
 
 
 class RealDelegate(Delegate):
-  def __init__(self, dryrun=False, gsutil=None):
+  def __init__(self, dryrun=False, gsutil=None, verbose=False):
     super(RealDelegate, self).__init__()
     self.dryrun = dryrun
+    self.verbose = verbose
     if gsutil:
       self.gsutil = gsutil
     else:
@@ -181,7 +182,7 @@ class RealDelegate(Delegate):
       sdk_json_string = sdk_stream.read()
 
     manifest = manifest_util.SDKManifest()
-    manifest.LoadDataFromString(sdk_json_string)
+    manifest.LoadDataFromString(sdk_json_string, add_missing_info=True)
     return manifest
 
   def GetHistory(self):
@@ -212,6 +213,7 @@ class RealDelegate(Delegate):
   def GsUtil_cp(self, src, dest, stdin=None):
     """See Delegate.GsUtil_cp"""
     if self.dryrun:
+      self.Trace("Skipping upload: %s -> %s" % (src, dest))
       return
 
     # -p ensures we keep permissions when copying "in-the-cloud".
@@ -220,6 +222,10 @@ class RealDelegate(Delegate):
   def Print(self, *args):
     sys.stdout.write(' '.join(map(str, args)) + '\n')
 
+  def Trace(self, *args):
+    if self.verbose:
+      self.Print(*args)
+
   def _RunGsUtil(self, stdin, *args):
     """Run gsutil as a subprocess.
 
@@ -230,16 +236,20 @@ class RealDelegate(Delegate):
     Returns:
       The stdout from the process."""
     cmd = [self.gsutil] + list(args)
+    self.Trace("Running: %s" % str(cmd))
     if stdin:
       stdin_pipe = subprocess.PIPE
     else:
       stdin_pipe = None
 
-    process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE)
-    stdout, stderr = process.communicate(stdin)
+    try:
+      process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+      stdout, stderr = process.communicate(stdin)
+    except OSError as e:
+      raise manifest_util.Error("Unable to run '%s': %s" % (cmd[0], str(e)))
 
-    if process.returncode != 0:
+    if process.returncode:
       sys.stderr.write(stderr)
       raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd))
     return stdout
@@ -518,6 +528,19 @@ class Updater(object):
     Args:
       manifest: The new manifest to upload.
     """
+    if self.delegate.dryrun:
+      name = MANIFEST_BASENAME + ".new"
+      self.delegate.Print("Writing new manifest: %s" % name)
+      with open(name, 'w') as f:
+        f.write(manifest.GetDataAsString())
+      stdout = self.delegate.GsUtil_cat(GS_SDK_MANIFEST)
+
+      online = MANIFEST_BASENAME + ".online"
+      self.delegate.Print("Writing online manifest: %s" % online)
+      with open(online, 'w') as f:
+        f.write(stdout)
+      os.system('diff -u %s %s' % (online, name))
+
     timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \
         GetTimestampManifestName()
     self.delegate.GsUtil_cp('-', timestamp_manifest_path,
@@ -597,14 +620,15 @@ class CapturedFile(object):
 
 def main(args):
   parser = optparse.OptionParser()
-  parser.add_option('--gsutil', help='path to gsutil', dest='gsutil',
-      default=None)
-  parser.add_option('--mailfrom', help='email address of sender',
-      dest='mailfrom', default=None)
-  parser.add_option('--mailto', help='send error mails to...', dest='mailto',
-      default=[], action='append')
-  parser.add_option('--dryrun', help='don\'t upload the manifest.',
-      dest='dryrun', action='store_true', default=False)
+  parser.add_option('--gsutil', help='path to gsutil.')
+  parser.add_option('-d', '--debug', help='run in debug mode.',
+      action='store_true')
+  parser.add_option('--mailfrom', help='email address of sender.')
+  parser.add_option('--mailto', help='send error mails to...', action='append')
+  parser.add_option('-n', '--dryrun', help="don't upload the manifest.",
+      action='store_true')
+  parser.add_option('-v', '--verbose', help='print more diagnotic messages.',
+      action='store_true')
   options, args = parser.parse_args(args[1:])
 
   if (options.mailfrom is None) != (not options.mailto):
@@ -618,18 +642,25 @@ def main(args):
     sys.stderr = CapturedFile(sys.stderr)
 
   try:
-    delegate = RealDelegate(dryrun=options.dryrun, gsutil=options.gsutil)
-    Run(delegate, ('mac', 'win', 'linux'))
-  except Exception:
-    if options.mailfrom and options.mailto:
-      traceback.print_exc()
-      scriptname = os.path.basename(sys.argv[0])
-      subject = '[%s] Failed to update manifest' % (scriptname,)
-      text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname, sys.stderr.getvalue())
-      SendMail(options.mailfrom, options.mailto, subject, text)
-      sys.exit(1)
-    else:
+    try:
+      delegate = RealDelegate(options.dryrun, options.gsutil, options.verbose)
+      Run(delegate, ('mac', 'win', 'linux'))
+    except Exception:
+      if options.mailfrom and options.mailto:
+        traceback.print_exc()
+        scriptname = os.path.basename(sys.argv[0])
+        subject = '[%s] Failed to update manifest' % (scriptname,)
+        text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname,
+                                                sys.stderr.getvalue())
+        SendMail(options.mailfrom, options.mailto, subject, text)
+        sys.exit(1)
+      else:
+        raise
+  except manifest_util.Error as e:
+    if options.debug:
       raise
+    print e
+    sys.exit(1)
 
 
 if __name__ == '__main__':