clang: Rewrite package.sh in python.

The motivation is to be able to use this on Windows, but for now this is a
pure rewrite of the existing package.sh script with no Windows-specific code
yet.  (In particular, this calls update.py with --bootstrap, which isn't
implemented on Windows yet, and the plugin bundling code isn't quite right
either since the plugins are compiled into the clang binary on Windows.  Also,
Windows doesn't do symlinks.)

BUG=483065
R=hans@chromium.org

Review URL: https://codereview.chromium.org/1149153004

Cr-Commit-Position: refs/heads/master@{#331811}

3 files changed, 233 insertions, 204 deletions

diff --git a/tools/clang/scripts/package.py b/tools/clang/scripts/package.py
new file mode 100755
index 0000000..1d6300f
--- /dev/null
+++ b/tools/clang/scripts/package.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python
+# Copyright (c) 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This script will check out llvm and clang, and then package the results up
+to a tgz file."""
+
+import argparse
+import fnmatch
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+
+# Path constants.
+THIS_DIR = os.path.dirname(__file__)
+LLVM_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party', 'llvm')
+THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party')
+LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
+LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
+                                          'llvm-bootstrap-install')
+LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build')
+LLVM_BIN_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts', 'bin')
+LLVM_LIB_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts', 'lib')
+STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
+
+
+def Tee(output, logfile):
+  logfile.write(output)
+  print output,
+
+
+def TeeCmd(cmd, logfile, fail_hard=True):
+  """Runs cmd and writes the output to both stdout and logfile."""
+  # Reading from PIPE can deadlock if one buffer is full but we wait on a
+  # different one.  To work around this, pipe the subprocess's stderr to
+  # its stdout buffer and don't give it a stdin.
+  proc = subprocess.Popen(cmd, bufsize=1, stdin=open(os.devnull),
+                          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+  for line in iter(proc.stdout.readline,''):
+    Tee(line, logfile)
+    if proc.poll() is not None:
+      break
+  exit_code = proc.wait()
+  if exit_code != 0 and fail_hard:
+    print 'Failed:', cmd
+    sys.exit(1)
+
+
+def PrintTarProgress(tarinfo):
+  print 'Adding', tarinfo.name
+  return tarinfo
+
+
+def main():
+  parser = argparse.ArgumentParser(description='build and package clang')
+  parser.add_argument('--gcc-toolchain',
+                      help="the prefix for the GCC version used for building. "
+                           "For /opt/foo/bin/gcc, pass "
+                           "'--gcc-toolchain '/opt/foo'")
+
+  args = parser.parse_args()
+
+  with open('buildlog.txt', 'w') as log:
+    Tee('Diff in llvm:\n', log)
+    TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False)
+    TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False)
+    Tee('Diff in llvm/tools/clang:\n', log)
+    TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')],
+           log, fail_hard=False)
+    TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')],
+           log, fail_hard=False)
+    Tee('Diff in llvm/compiler-rt:\n', log)
+    TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'compiler-rt')],
+           log, fail_hard=False)
+    TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'compiler-rt')],
+           log, fail_hard=False)
+    Tee('Diff in llvm/projects/libcxx:\n', log)
+    TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'libcxx')],
+           log, fail_hard=False)
+    TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'libcxx')],
+           log, fail_hard=False)
+    Tee('Diff in llvm/projects/libcxxabi:\n', log)
+    TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'libcxxabi')],
+           log, fail_hard=False)
+    TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'libcxxabi')],
+           log, fail_hard=False)
+
+    Tee('Starting build\n', log)
+
+    # Do a clobber build.
+    shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True)
+    shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True)
+    shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True)
+
+    build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'),
+                 '--bootstrap', '--force-local-build', '--run-tests',
+                 '--no-stdin-hack']
+    if args.gcc_toolchain is not None:
+      build_cmd.append(args.gcc_toolchain)
+    TeeCmd(build_cmd, log)
+
+  stamp = open(STAMP_FILE).read().rstrip()
+  pdir = 'clang-' + stamp
+  print pdir
+  shutil.rmtree(pdir, ignore_errors=True)
+  os.makedirs(os.path.join(pdir, 'bin'))
+  os.makedirs(os.path.join(pdir, 'lib'))
+
+  golddir = 'llvmgold-' + stamp
+  if sys.platform.startswith('linux'):
+    try:            os.makedirs(os.path.join(golddir, 'lib'))
+    except OSError: pass
+
+  so_ext = 'dylib' if sys.platform == 'darwin' else 'so'
+
+  # Copy buildlog over.
+  shutil.copy('buildlog.txt', pdir)
+
+  # Copy clang into pdir, symlink clang++ to it.
+  shutil.copy(os.path.join(LLVM_BIN_DIR, 'clang'), os.path.join(pdir, 'bin'))
+  os.symlink('clang', os.path.join(pdir, 'bin', 'clang++'))
+  shutil.copy(os.path.join(LLVM_BIN_DIR, 'llvm-symbolizer'),
+              os.path.join(pdir, 'bin'))
+  if sys.platform == 'darwin':
+    shutil.copy(os.path.join(LLVM_BIN_DIR, 'libc++.1.dylib'),
+                os.path.join(pdir, 'bin'))
+    os.symlink('libc++.1.dylib', os.path.join(pdir, 'bin', 'libc++.dylib'))
+
+  # Copy libc++ headers.
+  if sys.platform == 'darwin':
+    shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'),
+                    os.path.join(pdir, 'include', 'c++'))
+
+  # Copy plugins. Some of the dylibs are pretty big, so copy only the ones we
+  # care about.
+  shutil.copy(os.path.join(LLVM_LIB_DIR, 'libFindBadConstructs.' + so_ext),
+              os.path.join(pdir, 'lib'))
+  shutil.copy(os.path.join(LLVM_LIB_DIR, 'libBlinkGCPlugin.' + so_ext),
+              os.path.join(pdir, 'lib'))
+
+  # Copy gold plugin on Linux.
+  if sys.platform.startswith('linux'):
+    shutil.copy(os.path.join(LLVM_LIB_DIR, 'LLVMgold.so'),
+                os.path.join(golddir, 'lib'))
+
+  if args.gcc_toolchain is not None:
+    # Copy the stdlibc++.so.6 we linked Clang against so it can run.
+    shutil.copy(os.path.join(LLVM_LIB_DIR, 'libstdc++.so.6'),
+                os.path.join(pdir, 'lib'))
+
+  # Copy built-in headers (lib/clang/3.x.y/include).
+  # compiler-rt builds all kinds of libraries, but we want only some.
+  if sys.platform == 'darwin':
+    # Keep only the OSX (ASan and profile) and iossim (ASan) runtime libraries:
+    want = ['*/lib/darwin/*asan_osx*',
+            '*/lib/darwin/*asan_iosim*',
+            '*/lib/darwin/*profile_osx*']
+    for root, dirs, files in os.walk(os.path.join(LLVM_LIB_DIR, 'clang')):
+      for f in files:
+        qualified = os.path.join(root, f)
+        if not any(fnmatch.fnmatch(qualified, p) for p in want):
+          os.remove(os.path.join(root, f))
+        elif f.endswith('.dylib'):
+          # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
+          # @executable_path.
+          # TODO(glider): this is transitional. We'll need to fix the dylib
+          # name either in our build system, or in Clang. See also
+          # http://crbug.com/344836.
+          subprocess.call(['install_name_tool', '-id',
+                           '@executable_path/' + f, qualified])
+          subprocess.call(['strip', '-x', qualified])
+  elif sys.platform.startswith('linux'):
+    # Keep only
+    # lib/clang/*/lib/linux/libclang_rt.{[atm]san,san,ubsan,profile}-*.a ,
+    # but not dfsan.
+    want = ['*/lib/linux/*[atm]san*',
+            '*/lib/linux/*ubsan*',
+            '*/lib/linux/*libclang_rt.san*',
+            '*/lib/linux/*profile*']
+    for root, dirs, files in os.walk(os.path.join(LLVM_LIB_DIR, 'clang')):
+      for f in files:
+        qualified = os.path.join(root, f)
+        if not any(fnmatch.fnmatch(qualified, p) for p in want):
+          os.remove(os.path.join(root, f))
+        elif not f.endswith('.syms'):
+          # Strip the debug info from the runtime libraries.
+          subprocess.call(['strip', '-g', qualified])
+
+  shutil.copytree(os.path.join(LLVM_LIB_DIR, 'clang'),
+                  os.path.join(pdir, 'lib', 'clang'))
+
+  tar_entries = ['bin', 'lib', 'buildlog.txt']
+  if sys.platform == 'darwin':
+    tar_entries += ['include']
+  with tarfile.open(pdir + '.tgz', 'w:gz') as tar:
+    for entry in tar_entries:
+      tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress)
+
+  if sys.platform.startswith('linux'):
+    with tarfile.open(golddir + '.tgz', 'w:gz') as tar:
+      tar.add(os.path.join(golddir, 'lib'), arcname='lib',
+              filter=PrintTarProgress)
+
+  if sys.platform == 'darwin':
+    platform = 'Mac'
+  else:
+    platform = 'Linux_x64'
+
+  print 'To upload, run:'
+  print ('gsutil cp -a public-read %s.tgz '
+         'gs://chromium-browser-clang/%s/%s.tgz') % (pdir, platform, pdir)
+
+  if sys.platform.startswith('linux'):
+    print ('gsutil cp -a public-read %s.tgz '
+           'gs://chromium-browser-clang/%s/%s.tgz') % (golddir, platform,
+                                                       golddir)
+
+  # FIXME: Warn if the file already exists on the server.
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/tools/clang/scripts/package.sh b/tools/clang/scripts/package.sh
deleted file mode 100755
index 25cd6f0..0000000
--- a/tools/clang/scripts/package.sh
+++ /dev/null
@@ -1,200 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# This script will check out llvm and clang, and then package the results up
-# to a tgz file.
-
-gcc_toolchain=
-
-# Parse command line options.
-while [[ $# > 0 ]]; do
-  case $1 in
-    --gcc-toolchain)
-      shift
-      if [[ $# == 0 ]]; then
-        echo "--gcc-toolchain requires an argument."
-        exit 1
-      fi
-      if [[ -x "$1/bin/gcc" ]]; then
-        gcc_toolchain=$1
-      else
-        echo "Invalid --gcc-toolchain: '$1'."
-        echo "'$1/bin/gcc' does not appear to be valid."
-        exit 1
-      fi
-      ;;
-
-    --help)
-      echo "usage: $0 [--gcc-toolchain <prefix>]"
-      echo
-      echo "--gcc-toolchain: Set the prefix for which GCC version should"
-      echo "    be used for building. For example, to use gcc in"
-      echo "    /opt/foo/bin/gcc, use '--gcc-toolchain '/opt/foo"
-      echo
-      exit 1
-      ;;
-    *)
-      echo "Unknown argument: '$1'."
-      echo "Use --help for help."
-      exit 1
-      ;;
-  esac
-  shift
-done
-
-
-THIS_DIR="$(dirname "${0}")"
-LLVM_DIR="${THIS_DIR}/../../../third_party/llvm"
-LLVM_BOOTSTRAP_DIR="${THIS_DIR}/../../../third_party/llvm-bootstrap"
-LLVM_BOOTSTRAP_INSTALL_DIR="${LLVM_DIR}/../llvm-bootstrap-install"
-LLVM_BUILD_DIR="${THIS_DIR}/../../../third_party/llvm-build"
-LLVM_BIN_DIR="${LLVM_BUILD_DIR}/Release+Asserts/bin"
-LLVM_LIB_DIR="${LLVM_BUILD_DIR}/Release+Asserts/lib"
-STAMP_FILE="${LLVM_DIR}/../llvm-build/cr_build_revision"
-
-echo "Diff in llvm:" | tee buildlog.txt
-svn stat "${LLVM_DIR}" 2>&1 | tee -a buildlog.txt
-svn diff "${LLVM_DIR}" 2>&1 | tee -a buildlog.txt
-echo "Diff in llvm/tools/clang:" | tee -a buildlog.txt
-svn stat "${LLVM_DIR}/tools/clang" 2>&1 | tee -a buildlog.txt
-svn diff "${LLVM_DIR}/tools/clang" 2>&1 | tee -a buildlog.txt
-echo "Diff in llvm/compiler-rt:" | tee -a buildlog.txt
-svn stat "${LLVM_DIR}/compiler-rt" 2>&1 | tee -a buildlog.txt
-svn diff "${LLVM_DIR}/compiler-rt" 2>&1 | tee -a buildlog.txt
-echo "Diff in llvm/projects/libcxx:" | tee -a buildlog.txt
-svn stat "${LLVM_DIR}/projects/libcxx" 2>&1 | tee -a buildlog.txt
-svn diff "${LLVM_DIR}/projects/libcxx" 2>&1 | tee -a buildlog.txt
-echo "Diff in llvm/projects/libcxxabi:" | tee -a buildlog.txt
-svn stat "${LLVM_DIR}/projects/libcxxabi" 2>&1 | tee -a buildlog.txt
-svn diff "${LLVM_DIR}/projects/libcxxabi" 2>&1 | tee -a buildlog.txt
-
-
-echo "Starting build" | tee -a buildlog.txt
-
-set -exu
-set -o pipefail
-
-# Do a clobber build.
-rm -rf "${LLVM_BOOTSTRAP_DIR}"
-rm -rf "${LLVM_BOOTSTRAP_INSTALL_DIR}"
-rm -rf "${LLVM_BUILD_DIR}"
-extra_flags=
-if [[ -n "${gcc_toolchain}" ]]; then
-  extra_flags="--gcc-toolchain ${gcc_toolchain}"
-fi
-"${THIS_DIR}"/update.sh --bootstrap --force-local-build --run-tests \
-    ${extra_flags} 2>&1 | tee -a buildlog.txt
-
-R=$(cat "${STAMP_FILE}")
-
-PDIR=clang-$R
-rm -rf $PDIR
-mkdir $PDIR
-mkdir $PDIR/bin
-mkdir $PDIR/lib
-
-GOLDDIR=llvmgold-$R
-if [ "$(uname -s)" = "Linux" ]; then
-  mkdir -p $GOLDDIR/lib
-fi
-
-if [ "$(uname -s)" = "Darwin" ]; then
-  SO_EXT="dylib"
-else
-  SO_EXT="so"
-fi
-
-# Copy buildlog over.
-cp buildlog.txt $PDIR/
-
-# Copy clang into pdir, symlink clang++ to it.
-cp "${LLVM_BIN_DIR}/clang" $PDIR/bin/
-(cd $PDIR/bin && ln -sf clang clang++)
-cp "${LLVM_BIN_DIR}/llvm-symbolizer" $PDIR/bin/
-if [ "$(uname -s)" = "Darwin" ]; then
-  cp "${LLVM_BIN_DIR}/libc++.1.${SO_EXT}" $PDIR/bin/
-  (cd $PDIR/bin && ln -sf libc++.1.dylib libc++.dylib)
-fi
-
-# Copy libc++ headers.
-if [ "$(uname -s)" = "Darwin" ]; then
-  mkdir $PDIR/include
-  cp -R "${LLVM_BOOTSTRAP_INSTALL_DIR}/include/c++" $PDIR/include
-fi
-
-# Copy plugins. Some of the dylibs are pretty big, so copy only the ones we
-# care about.
-cp "${LLVM_LIB_DIR}/libFindBadConstructs.${SO_EXT}" $PDIR/lib
-cp "${LLVM_LIB_DIR}/libBlinkGCPlugin.${SO_EXT}" $PDIR/lib
-
-# Copy gold plugin on Linux.
-if [ "$(uname -s)" = "Linux" ]; then
-  cp "${LLVM_LIB_DIR}/LLVMgold.${SO_EXT}" $GOLDDIR/lib
-fi
-
-if [[ -n "${gcc_toolchain}" ]]; then
-  # Copy the stdlibc++.so.6 we linked Clang against so it can run.
-  cp "${LLVM_LIB_DIR}/libstdc++.so.6" $PDIR/lib
-fi
-
-# Copy built-in headers (lib/clang/3.x.y/include).
-# compiler-rt builds all kinds of libraries, but we want only some.
-if [ "$(uname -s)" = "Darwin" ]; then
-  # Keep only the OSX (ASan and profile) and iossim (ASan) runtime libraries:
-  # Release+Asserts/lib/clang/*/lib/darwin/libclang_rt.{asan,profile}_*
-  find "${LLVM_LIB_DIR}/clang" -type f -path '*lib/darwin*' \
-       ! -name '*asan_osx*' ! -name '*asan_iossim*' ! -name '*profile_osx*' | \
-      xargs rm
-  # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
-  # @executable_path.
-  # TODO(glider): this is transitional. We'll need to fix the dylib name
-  # either in our build system, or in Clang. See also http://crbug.com/344836.
-  ASAN_DYLIB_NAMES="libclang_rt.asan_osx_dynamic.dylib
-    libclang_rt.asan_iossim_dynamic.dylib"
-  for ASAN_DYLIB_NAME in $ASAN_DYLIB_NAMES
-  do
-    ASAN_DYLIB=$(find "${LLVM_LIB_DIR}/clang" \
-                      -type f -path "*${ASAN_DYLIB_NAME}")
-    install_name_tool -id @executable_path/${ASAN_DYLIB_NAME} "${ASAN_DYLIB}"
-    strip -x "${ASAN_DYLIB}"
-  done
-else
-  # Keep only
-  # Release+Asserts/lib/clang/*/lib/linux/libclang_rt.{[atm]san,san,ubsan,profile}-*.a
-  # , but not dfsan.
-  find "${LLVM_LIB_DIR}/clang" -type f -path '*lib/linux*' \
-       ! -name '*[atm]san*' ! -name '*ubsan*' ! -name '*libclang_rt.san*' \
-       ! -name '*profile*' | xargs rm -v
-  # Strip the debug info from the runtime libraries.
-  find "${LLVM_LIB_DIR}/clang" -type f -path '*lib/linux*' ! -name '*.syms' | xargs strip -g
-fi
-
-cp -vR "${LLVM_LIB_DIR}/clang" $PDIR/lib
-
-if [ "$(uname -s)" = "Darwin" ]; then
-  tar zcf $PDIR.tgz -C $PDIR bin include lib buildlog.txt
-else
-  tar zcf $PDIR.tgz -C $PDIR bin lib buildlog.txt
-fi
-
-if [ "$(uname -s)" = "Linux" ]; then
-  tar zcf $GOLDDIR.tgz -C $GOLDDIR lib
-fi
-
-if [ "$(uname -s)" = "Darwin" ]; then
-  PLATFORM=Mac
-else
-  PLATFORM=Linux_x64
-fi
-
-echo To upload, run:
-echo gsutil cp -a public-read $PDIR.tgz \
-     gs://chromium-browser-clang/$PLATFORM/$PDIR.tgz
-if [ "$(uname -s)" = "Linux" ]; then
-  echo gsutil cp -a public-read $GOLDDIR.tgz \
-       gs://chromium-browser-clang/$PLATFORM/$GOLDDIR.tgz
-fi
-
-# FIXME: Warn if the file already exists on the server.
diff --git a/tools/clang/scripts/update.py b/tools/clang/scripts/update.py
index 9d026e8..182973f 100755
--- a/tools/clang/scripts/update.py
+++ b/tools/clang/scripts/update.py
@@ -319,10 +319,14 @@ def main():
     # dup()ed sys.stdin is writable, try
     #   fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
     # TODO: Fix gclient instead, http://crbug.com/95350
-    try:
-      stderr = os.fdopen(os.dup(sys.stdin.fileno()))
-    except:
-      stderr = sys.stderr
+    if '--no-stdin-hack' in sys.argv:
+      sys.argv.remove('--no-stdin-hack')
+      stderr = None
+    else:
+      try:
+        stderr = os.fdopen(os.dup(sys.stdin.fileno()))
+      except:
+        stderr = sys.stderr
     return subprocess.call(
         [os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:],
         stderr=stderr)