Add --upload option to tools/clang/package.py.

It will be primarily used by the Clang Upload trybot,
but should work locally as well.

When uploading, it will not replace the remote file if one exists.
This is controlled by -n option to gsutil.

BUG=578306

Review URL: https://codereview.chromium.org/1714523004

Cr-Commit-Position: refs/heads/master@{#376366}

1 files changed, 90 insertions, 16 deletions

diff --git a/tools/clang/scripts/package.py b/tools/clang/scripts/package.py
index 9371110..4863341 100755
--- a/tools/clang/scripts/package.py
+++ b/tools/clang/scripts/package.py
@@ -17,6 +17,7 @@ import tarfile
 
 # Path constants.
 THIS_DIR = os.path.dirname(__file__)
+CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
 THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party')
 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
@@ -57,6 +58,57 @@ def PrintTarProgress(tarinfo):
   return tarinfo
 
 
+def GetExpectedStamp():
+  rev_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'),
+             '--print-revision']
+  return subprocess.check_output(rev_cmd).rstrip()
+
+
+def GetGsutilPath():
+  if not 'find_depot_tools' in sys.modules:
+    sys.path.insert(0, os.path.join(CHROMIUM_DIR, 'build'))
+    global find_depot_tools
+    import find_depot_tools
+  depot_path = find_depot_tools.add_depot_tools_to_path()
+  if depot_path is None:
+    print ('depot_tools are not found in PATH. '
+           'Follow the instructions in this document '
+           'http://dev.chromium.org/developers/how-tos/install-depot-tools'
+           ' to install depot_tools and then try again.')
+    sys.exit(1)
+  gsutil_path = os.path.join(depot_path, 'gsutil.py')
+  return gsutil_path
+
+
+def RunGsutil(args):
+  return subprocess.call([GetGsutilPath()] + args)
+
+
+def GsutilArchiveExists(archive_name, platform):
+  gsutil_args = ['-q', 'stat',
+                 'gs://chromium-browser-clang/%s/%s.tgz' %
+                 (platform, archive_name)]
+  return RunGsutil(gsutil_args) == 0
+
+
+def MaybeUpload(args, archive_name, platform):
+  # We don't want to rewrite the file, if it already exists on the server,
+  # so -n option to gsutil is used. It will warn, if the upload was aborted.
+  gsutil_args = ['cp', '-n', '-a', 'public-read',
+                  '%s.tgz' % archive_name,
+                  'gs://chromium-browser-clang/%s/%s.tgz' %
+                 (platform, archive_name)]
+  if args.upload:
+    print 'Uploading %s to Google Cloud Storage...' % archive_name
+    exit_code = RunGsutil(gsutil_args)
+    if exit_code != 0:
+      print "gsutil failed, exit_code: %s" % exit_code
+      os.exit(exit_code)
+  else:
+    print 'To upload, run:'
+    print ('gsutil %s' % string.join(gsutil_args, ' '))
+
+
 def main():
   if sys.platform == 'win32':
     try:
@@ -66,8 +118,40 @@ def main():
       return 1
 
   parser = argparse.ArgumentParser(description='build and package clang')
+  parser.add_argument('--upload', action='store_true',
+                      help='Upload the target archive to Google Cloud Storage.')
   args = parser.parse_args()
 
+  # Check that the script is not going to upload a toolchain built from HEAD.
+  use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
+  if args.upload and use_head_revision:
+    print ("--upload and LLVM_FORCE_HEAD_REVISION could not be used "
+           "at the same time.")
+    return 1
+
+  expected_stamp = GetExpectedStamp()
+  pdir = 'clang-' + expected_stamp
+  golddir = 'llvmgold-' + expected_stamp
+  print pdir
+
+  if sys.platform == 'darwin':
+    platform = 'Mac'
+  elif sys.platform == 'win32':
+    platform = 'Win'
+  else:
+    platform = 'Linux_x64'
+
+  # Check if Google Cloud Storage already has the artifacts we want to build.
+  if (args.upload and GsutilArchiveExists(pdir, platform) and
+      not sys.platform.startswith('linux') or
+      GsutilArchiveExists(golddir, platform)):
+    print ('Desired toolchain revision %s is already available '
+           'in Google Cloud Storage:') % expected_stamp
+    print 'gs://chromium-browser-clang/%s/%s.tgz' % (platform, pdir)
+    if sys.platform.startswith('linux'):
+      print 'gs://chromium-browser-clang/%s/%s.tgz' % (platform, golddir)
+    return 1
+
   with open('buildlog.txt', 'w') as log:
     Tee('Diff in llvm:\n', log)
     TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False)
@@ -103,8 +187,10 @@ def main():
     TeeCmd(build_cmd, log)
 
   stamp = open(STAMP_FILE).read().rstrip()
-  pdir = 'clang-' + stamp
-  print pdir
+  if stamp != expected_stamp:
+    print 'Actual stamp (%s) != expected stamp (%s).' % (stamp, expected_stamp)
+    return 1
+
   shutil.rmtree(pdir, ignore_errors=True)
 
   # Copy a whitelist of files to the directory we're going to tar up.
@@ -191,20 +277,10 @@ def main():
     for entry in tar_entries:
       tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress)
 
-  if sys.platform == 'darwin':
-    platform = 'Mac'
-  elif sys.platform == 'win32':
-    platform = 'Win'
-  else:
-    platform = 'Linux_x64'
-
-  print 'To upload, run:'
-  print ('gsutil cp -a public-read %s.tgz '
-         'gs://chromium-browser-clang/%s/%s.tgz') % (pdir, platform, pdir)
+  MaybeUpload(args, pdir, platform)
 
   # Zip up gold plugin on Linux.
   if sys.platform.startswith('linux'):
-    golddir = 'llvmgold-' + stamp
     shutil.rmtree(golddir, ignore_errors=True)
     os.makedirs(os.path.join(golddir, 'lib'))
     shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'lib', 'LLVMgold.so'),
@@ -212,9 +288,7 @@ def main():
     with tarfile.open(golddir + '.tgz', 'w:gz') as tar:
       tar.add(os.path.join(golddir, 'lib'), arcname='lib',
               filter=PrintTarProgress)
-    print ('gsutil cp -a public-read %s.tgz '
-           'gs://chromium-browser-clang/%s/%s.tgz') % (golddir, platform,
-                                                       golddir)
+    MaybeUpload(args, golddir, platform)
 
   # FIXME: Warn if the file already exists on the server.