[NaCl SDK] Add build_artifacts.py script

This is in preparation for moving NaCl SDK out of the Chrome repo.

BUG=none
R=sbc@chromium.org, noelallen@chromium.org

Review URL: https://codereview.chromium.org/702603002

Cr-Commit-Position: refs/heads/master@{#303559}

4 files changed, 898 insertions, 0 deletions

diff --git a/native_client_sdk/src/build_tools/build_artifacts.py b/native_client_sdk/src/build_tools/build_artifacts.py
new file mode 100755
index 0000000..1a9f0d8
--- /dev/null
+++ b/native_client_sdk/src/build_tools/build_artifacts.py
@@ -0,0 +1,587 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Script to build binary components of the SDK.
+
+This script builds binary components of the Native Client SDK, create tarballs
+for them, and uploads them to Google Cloud Storage.
+
+This prevents a source dependency on the Chromium/NaCl tree in the Native
+Client SDK repo.
+"""
+
+import argparse
+import datetime
+import glob
+import hashlib
+import json
+import os
+import sys
+import tempfile
+
+if sys.version_info < (2, 7, 0):
+  sys.stderr.write("python 2.7 or later is required run this script\n")
+  sys.exit(1)
+
+import buildbot_common
+import build_version
+from build_paths import NACL_DIR, OUT_DIR, SRC_DIR, SDK_SRC_DIR
+from build_paths import BUILD_ARCHIVE_DIR
+
+sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
+
+import getos
+import oshelpers
+
+BUILD_DIR = os.path.join(NACL_DIR, 'build')
+NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
+NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
+
+CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
+PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
+VERSION_JSON = os.path.join(BUILD_ARCHIVE_DIR, 'version.json')
+
+
+PLATFORM = getos.GetPlatform()
+TAR = oshelpers.FindExeInPath('tar')
+options = None
+all_archives = []
+
+
+# Mapping from toolchain name to the equivalent package_version.py directory
+# name.
+TOOLCHAIN_PACKAGE_MAP = {
+    'newlib': 'nacl_x86_newlib',
+    'bionic': 'nacl_arm_bionic',
+    'arm': 'nacl_arm_newlib',
+    'glibc': 'nacl_x86_glibc',
+    'pnacl': 'pnacl_newlib'}
+
+
+def Tar(archive_path, root, files):
+  if os.path.exists(TAR):
+    cmd = [TAR]
+  else:
+    cmd = [sys.executable, CYGTAR]
+  cmd.extend(['-cjf', archive_path])
+  cmd.extend(files)
+  buildbot_common.Run(cmd, cwd=root)
+
+
+def ComputeSha(filename):
+  with open(filename) as f:
+    return hashlib.sha1(f.read()).hexdigest()
+
+
+class TempDir(object):
+  def __init__(self, prefix=None, dont_remove=False):
+    self.prefix = prefix
+    self.name = None
+    self.dont_remove = dont_remove
+    self.created = False
+    self.destroyed = False
+
+  def Create(self):
+    assert not self.created
+    self.name = tempfile.mkdtemp(prefix=self.prefix)
+    return self
+
+  def Destroy(self):
+    assert not self.destroyed
+    if not self.dont_remove:
+      buildbot_common.RemoveDir(self.name)
+
+  def __enter__(self):
+    self.Create()
+    return self.name
+
+  def __exit__(self, exc, value, tb):
+    return self.Destroy()
+
+
+class Archive(object):
+  def __init__(self, name):
+    self.name = '%s_%s' % (PLATFORM, name)
+    self.archive_name = self.name + '.tar.bz2'
+    self.archive_path = os.path.join(BUILD_ARCHIVE_DIR, self.archive_name)
+    self.dirname = os.path.join(BUILD_ARCHIVE_DIR, self.name)
+    self._MakeDirname()
+
+  def _MakeDirname(self):
+    if os.path.exists(self.dirname):
+      buildbot_common.RemoveDir(self.dirname)
+    buildbot_common.MakeDir(self.dirname)
+
+  def Copy(self, src_root, file_list):
+    if type(file_list) is not list:
+      file_list = [file_list]
+
+    for file_spec in file_list:
+      # The list of files to install can be a simple list of
+      # strings or a list of pairs, where each pair corresponds
+      # to a mapping from source to destination names.
+      if type(file_spec) is str:
+        src_file = dest_file = file_spec
+      else:
+        src_file, dest_file = file_spec
+
+      src_file = os.path.join(src_root, src_file)
+
+      # Expand sources files using glob.
+      sources = glob.glob(src_file)
+      if not sources:
+        sources = [src_file]
+
+      if len(sources) > 1:
+        if not (dest_file.endswith('/') or dest_file == ''):
+          buildbot_common.ErrorExit(
+              "Target file %r must end in '/' or be empty when "
+              "using globbing to install files %r" % (dest_file, sources))
+
+      for source in sources:
+        if dest_file.endswith('/'):
+          dest = os.path.join(dest_file, os.path.basename(source))
+        else:
+          dest = dest_file
+        dest = os.path.join(self.dirname, dest)
+        if not os.path.isdir(os.path.dirname(dest)):
+          buildbot_common.MakeDir(os.path.dirname(dest))
+        if os.path.isdir(source):
+          buildbot_common.CopyDir(source, dest)
+        else:
+          buildbot_common.CopyFile(source, dest)
+
+  def CreateArchiveShaFile(self):
+    sha1 = ComputeSha(self.archive_path)
+    sha1_filename = self.archive_path + '.sha1'
+    with open(sha1_filename, 'w') as f:
+      f.write(sha1)
+
+  def Tar(self):
+    Tar(self.archive_path, BUILD_ARCHIVE_DIR, [
+        self.name,
+        os.path.basename(VERSION_JSON)])
+    self.CreateArchiveShaFile()
+    all_archives.append(self.archive_name)
+
+
+def MakeToolchainArchive(toolchain):
+  archive = Archive(toolchain)
+
+  build_platform = '%s_x86' % PLATFORM
+
+  with TempDir('tc_%s_' % toolchain) as tmpdir:
+    package_name = os.path.join(build_platform,
+                                TOOLCHAIN_PACKAGE_MAP.get(toolchain))
+
+    # Extract all of the packages into the temp directory.
+    buildbot_common.Run([sys.executable, PKGVER,
+                           '--packages', package_name,
+                           '--tar-dir', NACL_TOOLCHAINTARS_DIR,
+                           '--dest-dir', tmpdir,
+                           'extract'])
+
+    # Copy all the files we extracted to the correct destination.
+    archive.Copy(os.path.join(tmpdir, package_name), ('*', ''))
+
+  archive.Tar()
+
+
+def MakeNinjaRelPath(path):
+  return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
+
+
+def NinjaBuild(targets, out_dir):
+  if type(targets) is not list:
+    targets = [targets]
+  out_config_dir = os.path.join(out_dir, 'Release')
+  buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
+
+
+def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, out_dir):
+  gyp_env = dict(os.environ)
+  gyp_env['GYP_GENERATORS'] = 'ninja'
+  gyp_defines = []
+  if options.mac_sdk:
+    gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
+  if arch:
+    gyp_defines.append('target_arch=%s' % arch)
+    if arch == 'arm':
+      if PLATFORM == 'linux':
+        gyp_env['CC'] = 'arm-linux-gnueabihf-gcc'
+        gyp_env['CXX'] = 'arm-linux-gnueabihf-g++'
+        gyp_env['AR'] = 'arm-linux-gnueabihf-ar'
+        gyp_env['AS'] = 'arm-linux-gnueabihf-as'
+        gyp_env['CC_host'] = 'cc'
+        gyp_env['CXX_host'] = 'c++'
+      gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1',
+                      'arm_float_abi=hard', 'nacl_enable_arm_gcc=1']
+      if options.no_arm_trusted:
+        gyp_defines.append('disable_cross_trusted=1')
+  if PLATFORM == 'mac':
+    gyp_defines.append('clang=1')
+
+  gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
+  generator_flags = ['-G', 'output_dir=%s' % out_dir]
+  depth = '--depth=.'
+  cmd = [sys.executable, gyp_py_script, gyp_file, depth] + generator_flags
+  buildbot_common.Run(cmd, cwd=SRC_DIR, env=gyp_env)
+  NinjaBuild(targets, out_dir)
+
+
+def GetToolsFiles():
+  files = [
+    ('sel_ldr', 'sel_ldr_x86_32'),
+    ('ncval_new', 'ncval'),
+    ('irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'),
+    ('irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'),
+  ]
+
+  if PLATFORM == 'linux':
+    files.append(['nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_32'])
+    files.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
+                  'nonsfi_loader_x86_32'])
+
+  # Add .exe extensions to all windows tools
+  for pair in files:
+    if PLATFORM == 'win' and not pair[0].endswith('.nexe'):
+      pair[0] += '.exe'
+      pair[1] += '.exe'
+
+  return files
+
+
+def GetTools64Files():
+  files = []
+  if PLATFORM == 'win':
+    files.append('sel_ldr64')
+  else:
+    files.append(('sel_ldr', 'sel_ldr_x86_64'))
+
+  if PLATFORM == 'linux':
+    files.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_64'))
+
+  return files
+
+
+def GetToolsArmFiles():
+  assert PLATFORM == 'linux'
+  return [('irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'),
+          ('sel_ldr', 'sel_ldr_arm'),
+          ('nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm')]
+
+
+def GetNewlibToolchainLibs():
+  return ['crti.o',
+          'crtn.o',
+          'libminidump_generator.a',
+          'libnacl.a',
+          'libnacl_dyncode.a',
+          'libnacl_exception.a',
+          'libnacl_list_mappings.a',
+          'libnosys.a',
+          'libppapi.a',
+          'libppapi_stub.a',
+          'libpthread.a']
+
+
+def GetGlibcToolchainLibs():
+  return ['libminidump_generator.a',
+          'libminidump_generator.so',
+          'libnacl.a',
+          'libnacl_dyncode.a',
+          'libnacl_dyncode.so',
+          'libnacl_exception.a',
+          'libnacl_exception.so',
+          'libnacl_list_mappings.a',
+          'libnacl_list_mappings.so',
+          'libppapi.a',
+          'libppapi.so',
+          'libppapi_stub.a']
+
+
+def GetPNaClToolchainLibs():
+  return ['libminidump_generator.a',
+          'libnacl.a',
+          'libnacl_dyncode.a',
+          'libnacl_exception.a',
+          'libnacl_list_mappings.a',
+          'libnosys.a',
+          'libppapi.a',
+          'libppapi_stub.a',
+          'libpthread.a']
+
+
+def GetBionicToolchainLibs():
+  return ['libminidump_generator.a',
+          'libnacl_dyncode.a',
+          'libnacl_exception.a',
+          'libnacl_list_mappings.a',
+          'libppapi.a']
+
+
+def GetToolchainNaClLib(tcname, tcpath, xarch):
+  if tcname == 'pnacl':
+    return os.path.join(tcpath, 'le32-nacl', 'lib')
+  elif xarch == 'x86_32':
+    return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
+  elif xarch == 'x86_64':
+    return os.path.join(tcpath, 'x86_64-nacl', 'lib')
+  elif xarch == 'arm':
+    return os.path.join(tcpath, 'arm-nacl', 'lib')
+
+
+def GetGypBuiltLib(root, tcname, xarch=None):
+  if tcname == 'pnacl':
+    tcname = 'pnacl_newlib'
+  if xarch == 'x86_32':
+    xarch = '32'
+  elif xarch == 'x86_64':
+    xarch = '64'
+  elif not xarch:
+    xarch = ''
+  return os.path.join(root, 'Release', 'gen', 'tc_' + tcname, 'lib' + xarch)
+
+
+def GetGypToolchainLib(root, tcname, xarch):
+  if xarch == 'arm':
+    toolchain = xarch
+  else:
+    toolchain = tcname
+
+  tcpath = os.path.join(root, 'Release', 'gen', 'sdk', '%s_x86' % PLATFORM,
+                        TOOLCHAIN_PACKAGE_MAP[toolchain])
+  return GetToolchainNaClLib(tcname, tcpath, xarch)
+
+
+def MakeGypArchives():
+  join = os.path.join
+  gyp_nacl = join(NACL_DIR, 'build', 'gyp_nacl')
+  gyp_chromium = join(SRC_DIR, 'build', 'gyp_chromium')
+  nacl_core_sdk_gyp = join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
+  all_gyp = join(NACL_DIR, 'build', 'all.gyp')
+  breakpad_gyp = join(SRC_DIR, 'breakpad', 'breakpad.gyp')
+  ppapi_gyp = join(SRC_DIR, 'ppapi', 'native_client', 'native_client.gyp')
+  breakpad_targets = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
+
+  # Build
+  tmpdir_obj = TempDir('nacl_core_sdk_', dont_remove=True).Create()
+  tmpdir = tmpdir_obj.name
+  GypNinjaBuild('ia32', gyp_nacl, nacl_core_sdk_gyp, 'nacl_core_sdk', tmpdir)
+  GypNinjaBuild('ia32', gyp_nacl, all_gyp, 'ncval_new', tmpdir)
+  GypNinjaBuild('ia32', gyp_chromium, breakpad_gyp, breakpad_targets, tmpdir)
+  GypNinjaBuild('ia32', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir)
+  GypNinjaBuild('x64', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir)
+
+  tmpdir64_obj = TempDir('nacl_core_sdk_64_', dont_remove=True).Create()
+  tmpdir_64 = tmpdir64_obj.name
+  if PLATFORM == 'win':
+    GypNinjaBuild('ia32', gyp_nacl, nacl_core_sdk_gyp, 'sel_ldr64', tmpdir_64)
+  else:
+    GypNinjaBuild('x64', gyp_nacl, nacl_core_sdk_gyp, 'sel_ldr', tmpdir_64)
+
+  tmpdirarm_obj = TempDir('nacl_core_sdk_arm_', dont_remove=True).Create()
+  tmpdir_arm = tmpdirarm_obj.name
+  GypNinjaBuild('arm', gyp_nacl, nacl_core_sdk_gyp, 'nacl_core_sdk', tmpdir_arm)
+  GypNinjaBuild('arm', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir_arm)
+
+  # Tools archive
+  archive = Archive('tools')
+  archive.Copy(join(tmpdir, 'Release'), GetToolsFiles())
+  archive.Copy(join(tmpdir_64, 'Release'), GetTools64Files())
+  if PLATFORM == 'linux':
+    archive.Copy(join(tmpdir_arm, 'Release'), GetToolsArmFiles())
+  # TODO(binji): dump_syms doesn't currently build on Windows. See
+  # http://crbug.com/245456
+  if PLATFORM != 'win':
+    archive.Copy(join(tmpdir, 'Release'), breakpad_targets)
+  archive.Tar()
+
+  # newlib x86 libs archives
+  for arch in ('x86_32', 'x86_64'):
+    archive = Archive('newlib_%s_libs' % arch)
+    archive.Copy(GetGypBuiltLib(tmpdir, 'newlib', arch),
+                      GetNewlibToolchainLibs())
+    archive.Copy(GetGypToolchainLib(tmpdir, 'newlib', arch), 'crt1.o')
+    archive.Tar()
+
+  # newlib arm libs archive
+  archive = Archive('newlib_arm_libs')
+  archive.Copy(GetGypBuiltLib(tmpdir_arm, 'newlib', 'arm'),
+                    GetNewlibToolchainLibs())
+  archive.Copy(GetGypToolchainLib(tmpdir_arm, 'newlib', 'arm'), 'crt1.o')
+  archive.Tar()
+
+  # glibc x86 libs archives
+  for arch in ('x86_32', 'x86_64'):
+    archive = Archive('glibc_%s_libs' % arch)
+    archive.Copy(GetGypBuiltLib(tmpdir, 'glibc', arch), GetGlibcToolchainLibs())
+    archive.Copy(GetGypToolchainLib(tmpdir, 'glibc', arch), 'crt1.o')
+    archive.Tar()
+
+  # pnacl libs archive
+  archive = Archive('pnacl_libs')
+  archive.Copy(GetGypBuiltLib(tmpdir, 'pnacl'), GetPNaClToolchainLibs())
+  archive.Tar()
+
+  if PLATFORM == 'linux':
+    # bionic arm libs archive (use newlib-built files)
+    archive = Archive('bionic_arm_libs')
+    archive.Copy(GetGypBuiltLib(tmpdir_arm, 'newlib', 'arm'),
+                      GetBionicToolchainLibs())
+    archive.Copy(GetGypToolchainLib(tmpdir_arm, 'newlib', 'arm'), 'crt1.o')
+    archive.Tar()
+
+  # Destroy the temporary directories
+  tmpdir_obj.Destroy()
+  tmpdirarm_obj.Destroy()
+  tmpdir64_obj.Destroy()
+
+
+def MakePNaClArchives():
+  join = os.path.join
+  gyp_chromium = join(SRC_DIR, 'build', 'gyp_chromium')
+  pnacl_irt_shim_gyp = join(SRC_DIR, 'ppapi', 'native_client', 'src',
+                            'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
+
+  with TempDir('pnacl_irt_shim_ia32_') as tmpdir:
+    GypNinjaBuild('ia32', gyp_chromium, pnacl_irt_shim_gyp, 'aot', tmpdir)
+
+    archive = Archive('pnacl_translator_x86_32_libs')
+    libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
+    archive.Copy(libdir, 'libpnacl_irt_shim.a')
+    archive.Tar()
+
+    archive = Archive('pnacl_translator_x86_64_libs')
+    libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
+    archive.Copy(libdir, 'libpnacl_irt_shim.a')
+    archive.Tar()
+
+  with TempDir('pnacl_irt_shim_arm_') as tmpdir:
+    GypNinjaBuild('arm', gyp_chromium, pnacl_irt_shim_gyp, 'aot', tmpdir)
+
+    archive = Archive('pnacl_translator_arm_libs')
+    libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-arm')
+    archive.Copy(libdir, 'libpnacl_irt_shim.a')
+    archive.Tar()
+
+
+def GetNewlibHeaders():
+  return [
+      ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
+      ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
+      ('native_client/src/untrusted/irt/irt.h', ''),
+      ('native_client/src/untrusted/irt/irt_dev.h', ''),
+      ('native_client/src/untrusted/irt/irt_extension.h', ''),
+      ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
+      ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
+      ('native_client/src/untrusted/pthread/pthread.h', ''),
+      ('native_client/src/untrusted/pthread/semaphore.h', ''),
+      ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
+      ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
+
+
+def GetGlibcHeaders():
+  return [
+      ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
+      ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
+      ('native_client/src/untrusted/irt/irt.h', ''),
+      ('native_client/src/untrusted/irt/irt_dev.h', ''),
+      ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
+      ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
+      ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
+      ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
+
+
+def GetBionicHeaders():
+  return [('ppapi/nacl_irt/public/irt_ppapi.h', '')]
+
+
+def MakeToolchainHeaderArchives():
+  archive = Archive('newlib_headers')
+  archive.Copy(SRC_DIR, GetNewlibHeaders())
+  archive.Tar()
+
+  archive = Archive('glibc_headers')
+  archive.Copy(SRC_DIR, GetGlibcHeaders())
+  archive.Tar()
+
+  if PLATFORM == 'linux':
+    archive = Archive('bionic_headers')
+    archive.Copy(SRC_DIR, GetBionicHeaders())
+    archive.Tar()
+
+
+def MakePepperArchive():
+  archive = Archive('ppapi')
+  archive.Copy(os.path.join(SRC_DIR, 'ppapi'), ['c', 'cpp', 'lib', 'utility'])
+  archive.Tar()
+
+
+def UploadArchives():
+  major_version = build_version.ChromeMajorVersion()
+  chrome_revision = build_version.ChromeRevision()
+  commit_position = build_version.ChromeCommitPosition()
+  git_sha = build_version.ParseCommitPosition(commit_position)[0]
+  short_sha = git_sha[:9]
+  archive_version = '%s-%s-%s' % (major_version, chrome_revision, short_sha)
+  bucket_path = 'native-client-sdk/archives/%s' % archive_version
+  for archive_name in all_archives:
+    buildbot_common.Archive(archive_name, bucket_path,
+                            cwd=BUILD_ARCHIVE_DIR, step_link=False)
+    sha1_filename = archive_name + '.sha1'
+    buildbot_common.Archive(sha1_filename, bucket_path,
+                            cwd=BUILD_ARCHIVE_DIR, step_link=False)
+
+
+def MakeVersionJson():
+  time_format = '%Y/%m/%d %H:%M:%S'
+  data = {
+      'chrome_version': build_version.ChromeVersionNoTrunk(),
+      'chrome_revision': build_version.ChromeRevision(),
+      'chrome_commit_position': build_version.ChromeCommitPosition(),
+      'nacl_revision': build_version.NaClRevision(),
+      'build_date': datetime.datetime.now().strftime(time_format)}
+
+  dirname = os.path.dirname(VERSION_JSON)
+  if not os.path.exists(dirname):
+    buildbot_common.MakeDir(dirname)
+  with open(VERSION_JSON, 'w') as outf:
+    json.dump(data, outf, indent=2, separators=(',', ': '))
+
+
+def main(args):
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument('--mac-sdk',
+      help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
+  parser.add_argument('--no-arm-trusted', action='store_true',
+      help='Disable building of ARM trusted components (sel_ldr, etc).')
+  parser.add_argument('--upload', action='store_true',
+                      help='Upload tarballs to GCS.')
+
+  global options
+  options = parser.parse_args(args)
+
+  toolchains = ['pnacl', 'newlib', 'glibc', 'arm']
+  if PLATFORM == 'linux':
+    toolchains.append('bionic')
+
+  MakeVersionJson()
+  for tc in toolchains:
+    MakeToolchainArchive(tc)
+  MakeGypArchives()
+  MakePNaClArchives()
+  MakeToolchainHeaderArchives()
+  MakePepperArchive()
+  if options.upload:
+    UploadArchives()
+
+  return 0
+
+if __name__ == '__main__':
+  try:
+    sys.exit(main(sys.argv[1:]))
+  except KeyboardInterrupt:
+    buildbot_common.ErrorExit('build_artifacts: interrupted')
diff --git a/native_client_sdk/src/build_tools/build_paths.py b/native_client_sdk/src/build_tools/build_paths.py
index f5a0663..f24070b 100644
--- a/native_client_sdk/src/build_tools/build_paths.py
+++ b/native_client_sdk/src/build_tools/build_paths.py
@@ -14,6 +14,7 @@ SDK_DIR = os.path.dirname(SDK_SRC_DIR)
 SRC_DIR = os.path.dirname(SDK_DIR)
 NACL_DIR = os.path.join(SRC_DIR, 'native_client')
 OUT_DIR = os.path.join(SRC_DIR, 'out')
+BUILD_ARCHIVE_DIR = os.path.join(OUT_DIR, 'nacl_sdk_build')
 PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
 NACLPORTS_DIR = os.path.join(OUT_DIR, 'naclports')
 GONACL_APPENGINE_DIR = os.path.join(SDK_SRC_DIR, 'gonacl_appengine')
diff --git a/native_client_sdk/src/build_tools/tests/build_artifacts_test.py b/native_client_sdk/src/build_tools/tests/build_artifacts_test.py
new file mode 100755
index 0000000..2666fab
--- /dev/null
+++ b/native_client_sdk/src/build_tools/tests/build_artifacts_test.py
@@ -0,0 +1,309 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import ntpath
+import posixpath
+import sys
+import collections
+import unittest
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+BUILD_TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
+CHROME_SRC = os.path.dirname(os.path.dirname(os.path.dirname(BUILD_TOOLS_DIR)))
+MOCK_DIR = os.path.join(CHROME_SRC, "third_party", "pymock")
+
+# For the mock library
+sys.path.append(MOCK_DIR)
+from mock import call, patch, Mock
+
+sys.path.append(BUILD_TOOLS_DIR)
+import build_artifacts
+
+
+class BasePosixTestCase(unittest.TestCase):
+  def setUp(self):
+    self.addCleanup(patch.stopall)
+    patch('build_artifacts.PLATFORM', 'posix').start()
+    patch('build_artifacts.BUILD_ARCHIVE_DIR', '/archive_dir/').start()
+    patch('os.path.join', posixpath.join).start()
+
+
+class PosixTestCase(BasePosixTestCase):
+  def setUp(self):
+    BasePosixTestCase.setUp(self)
+
+  def testGetToolchainNaClLib(self):
+    tests = [
+        (('newlib', 'x86_32'), 'foo/x86_64-nacl/lib32'),
+        (('newlib', 'x86_64'), 'foo/x86_64-nacl/lib'),
+        (('newlib', 'arm'), 'foo/arm-nacl/lib'),
+        (('glibc', 'x86_32'), 'foo/x86_64-nacl/lib32'),
+        (('glibc', 'x86_64'), 'foo/x86_64-nacl/lib'),
+        (('bionic', 'arm'), 'foo/arm-nacl/lib'),
+        (('pnacl', None), 'foo/le32-nacl/lib'),
+    ]
+
+    for test in tests:
+      self.assertEqual(
+        build_artifacts.GetToolchainNaClLib(test[0][0], 'foo', test[0][1]),
+        test[1])
+
+  def testGetGypBuiltLib(self):
+    tests = [
+        (('newlib', 'x86_32'), 'foo/Release/gen/tc_newlib/lib32'),
+        (('newlib', 'x86_64'), 'foo/Release/gen/tc_newlib/lib64'),
+        (('newlib', 'arm'), 'foo/Release/gen/tc_newlib/libarm'),
+        (('glibc', 'x86_32'), 'foo/Release/gen/tc_glibc/lib32'),
+        (('glibc', 'x86_64'), 'foo/Release/gen/tc_glibc/lib64'),
+        (('pnacl', None), 'foo/Release/gen/tc_pnacl_newlib/lib')
+    ]
+
+    for test in tests:
+      self.assertEqual(
+        build_artifacts.GetGypBuiltLib('foo', test[0][0], test[0][1]),
+        test[1])
+
+  def testGetGypToolchainLib(self):
+    tests = [
+        (('newlib', 'x86_32'),
+         'foo/Release/gen/sdk/posix_x86/nacl_x86_newlib/x86_64-nacl/lib32'),
+        (('newlib', 'x86_64'),
+         'foo/Release/gen/sdk/posix_x86/nacl_x86_newlib/x86_64-nacl/lib'),
+        (('newlib', 'arm'),
+         'foo/Release/gen/sdk/posix_x86/nacl_arm_newlib/arm-nacl/lib'),
+        (('glibc', 'x86_32'),
+         'foo/Release/gen/sdk/posix_x86/nacl_x86_glibc/x86_64-nacl/lib32'),
+        (('glibc', 'x86_64'),
+         'foo/Release/gen/sdk/posix_x86/nacl_x86_glibc/x86_64-nacl/lib'),
+        # Bionic uses the newlib toolchain lib directory
+        (('bionic', 'arm'),
+         'foo/Release/gen/sdk/posix_x86/nacl_arm_newlib/arm-nacl/lib'),
+        (('pnacl', None),
+         'foo/Release/gen/sdk/posix_x86/pnacl_newlib/le32-nacl/lib'),
+    ]
+
+    for test in tests:
+      self.assertEqual(
+        build_artifacts.GetGypToolchainLib('foo', test[0][0], test[0][1]),
+        test[1])
+
+  @patch('build_artifacts.all_archives', ['foo.tar.bz2', 'bar.tar.bz2'])
+  @patch('build_version.ChromeMajorVersion', Mock(return_value='40'))
+  @patch('build_version.ChromeRevision', Mock(return_value='302630'))
+  @patch('build_version.ChromeCommitPosition', Mock(return_value=
+             '1492c3d296476fe12cafecabba6ebabe-refs/heads/master@{#302630}'))
+  @patch('buildbot_common.Archive')
+  def testUploadArchives(self, archive_mock):
+    build_artifacts.UploadArchives()
+    cwd = '/archive_dir/'
+    bucket_path = 'native-client-sdk/archives/40-302630-1492c3d29'
+    archive_mock.assert_has_calls([
+        call('foo.tar.bz2', bucket_path, cwd=cwd, step_link=False),
+        call('foo.tar.bz2.sha1', bucket_path, cwd=cwd, step_link=False),
+        call('bar.tar.bz2', bucket_path, cwd=cwd, step_link=False),
+        call('bar.tar.bz2.sha1', bucket_path, cwd=cwd, step_link=False)
+    ])
+
+
+class GypNinjaPosixTestCase(BasePosixTestCase):
+  def setUp(self):
+    BasePosixTestCase.setUp(self)
+    patch('sys.executable', 'python').start()
+    patch('build_artifacts.SRC_DIR', 'src_dir').start()
+    patch('os.environ', {}).start()
+    self.run_mock = patch('buildbot_common.Run').start()
+    self.options_mock = patch('build_artifacts.options').start()
+    self.options_mock.mac_sdk = False
+    self.options_mock.no_arm_trusted = False
+
+  def testSimple(self):
+    build_artifacts.GypNinjaBuild(
+        None, 'gyp.py', 'foo.gyp', 'target', 'out_dir')
+    self.run_mock.assert_has_calls([
+        call(['python', 'gyp.py', 'foo.gyp', '--depth=.', '-G',
+              'output_dir=out_dir'],
+             cwd='src_dir',
+             env={'GYP_GENERATORS': 'ninja', 'GYP_DEFINES': ''}),
+        call(['ninja', '-C', 'out_dir/Release', 'target'], cwd='src_dir')
+    ])
+
+  def testTargetArch(self):
+    build_artifacts.GypNinjaBuild(
+        'x64', 'gyp.py', 'foo.gyp', 'target', 'out_dir')
+    self.run_mock.assert_has_calls([
+        call(['python', 'gyp.py', 'foo.gyp', '--depth=.', '-G',
+              'output_dir=out_dir'],
+             cwd='src_dir',
+             env={
+                 'GYP_GENERATORS': 'ninja',
+                 'GYP_DEFINES': 'target_arch=x64'
+             }),
+        call(['ninja', '-C', 'out_dir/Release', 'target'], cwd='src_dir')
+    ])
+
+  def testMultipleTargets(self):
+    build_artifacts.GypNinjaBuild(
+        None, 'gyp.py', 'foo.gyp', ['target1', 'target2'], 'out_dir')
+    self.run_mock.assert_has_calls([
+        call(['python', 'gyp.py', 'foo.gyp', '--depth=.', '-G',
+              'output_dir=out_dir'],
+             cwd='src_dir',
+             env={'GYP_GENERATORS': 'ninja', 'GYP_DEFINES': ''}),
+        call(['ninja', '-C', 'out_dir/Release', 'target1', 'target2'],
+             cwd='src_dir')
+    ])
+
+  def testMacSdk(self):
+    build_artifacts.PLATFORM = 'mac'
+    self.options_mock.mac_sdk = '10.6'
+    build_artifacts.GypNinjaBuild(
+        None, 'gyp.py', 'foo.gyp', 'target', 'out_dir')
+    self.run_mock.assert_has_calls([
+        call(['python', 'gyp.py', 'foo.gyp', '--depth=.', '-G',
+              'output_dir=out_dir'],
+             cwd='src_dir',
+             env={
+               'GYP_GENERATORS': 'ninja',
+               'GYP_DEFINES': 'mac_sdk=10.6 clang=1'
+             }),
+        call(['ninja', '-C', 'out_dir/Release', 'target'], cwd='src_dir')
+    ])
+
+  def testArmLinux(self):
+    build_artifacts.PLATFORM = 'linux'
+    build_artifacts.GypNinjaBuild(
+        'arm', 'gyp.py', 'foo.gyp', 'target', 'out_dir')
+    self.run_mock.assert_has_calls([
+        call(['python', 'gyp.py', 'foo.gyp', '--depth=.', '-G',
+              'output_dir=out_dir'],
+             cwd='src_dir',
+             env={
+               'GYP_GENERATORS': 'ninja',
+               'GYP_DEFINES': 'target_arch=arm armv7=1 arm_thumb=0 arm_neon=1'
+                              ' arm_float_abi=hard nacl_enable_arm_gcc=1',
+               'CC': 'arm-linux-gnueabihf-gcc',
+               'CXX': 'arm-linux-gnueabihf-g++',
+               'AR': 'arm-linux-gnueabihf-ar',
+               'AS': 'arm-linux-gnueabihf-as',
+               'CC_host': 'cc',
+               'CXX_host': 'c++',
+             }),
+        call(['ninja', '-C', 'out_dir/Release', 'target'], cwd='src_dir')
+    ])
+
+  def testNoArmTrusted(self):
+    build_artifacts.PLATFORM = 'linux'
+    self.options_mock.no_arm_trusted = True
+    build_artifacts.GypNinjaBuild(
+        'arm', 'gyp.py', 'foo.gyp', 'target', 'out_dir')
+    self.run_mock.assert_has_calls([
+        call(['python', 'gyp.py', 'foo.gyp', '--depth=.', '-G',
+              'output_dir=out_dir'],
+             cwd='src_dir',
+             env={
+               'GYP_GENERATORS': 'ninja',
+               'GYP_DEFINES': 'target_arch=arm armv7=1 arm_thumb=0 arm_neon=1'
+                              ' arm_float_abi=hard nacl_enable_arm_gcc=1'
+                              ' disable_cross_trusted=1',
+               'CC': 'arm-linux-gnueabihf-gcc',
+               'CXX': 'arm-linux-gnueabihf-g++',
+               'AR': 'arm-linux-gnueabihf-ar',
+               'AS': 'arm-linux-gnueabihf-as',
+               'CC_host': 'cc',
+               'CXX_host': 'c++',
+             }),
+        call(['ninja', '-C', 'out_dir/Release', 'target'], cwd='src_dir')
+    ])
+
+
+class ArchivePosixTestCase(BasePosixTestCase):
+  def setUp(self):
+    BasePosixTestCase.setUp(self)
+    self.makedir_mock = patch('buildbot_common.MakeDir').start()
+    self.copyfile_mock = patch('buildbot_common.CopyFile').start()
+    self.copydir_mock = patch('buildbot_common.CopyDir').start()
+    self.isdir_mock = patch('os.path.isdir').start()
+    patch('os.path.exists', Mock(return_value=False)).start()
+
+    def dummy_isdir(path):
+      if path == '/archive_dir/posix_foo':
+        return True
+      return False
+    self.isdir_mock.side_effect = dummy_isdir
+
+    self.archive = build_artifacts.Archive('foo')
+
+  def testInit(self):
+    self.assertEqual(self.archive.name, 'posix_foo')
+    self.assertEqual(self.archive.archive_name, 'posix_foo.tar.bz2')
+    self.assertEqual(self.archive.archive_path,
+                     '/archive_dir/posix_foo.tar.bz2')
+    self.assertEqual(self.archive.dirname, '/archive_dir/posix_foo')
+    self.makedir_mock.assert_called_once_with('/archive_dir/posix_foo')
+
+  @patch('glob.glob', Mock(side_effect=lambda x: [x]))
+  def testCopySimple(self):
+    self.archive.Copy('/copy_from', ['file1', 'file2'])
+    self.assertEqual(self.copydir_mock.call_count, 0)
+    self.copyfile_mock.assert_has_calls([
+      call('/copy_from/file1', '/archive_dir/posix_foo/file1'),
+      call('/copy_from/file2', '/archive_dir/posix_foo/file2')])
+
+  @patch('glob.glob')
+  def testCopyGlob(self, glob_mock):
+    glob_mock.return_value = ['/copy_from/foo', '/copy_from/bar']
+    self.archive.Copy('/copy_from', [('*', '')])
+    glob_mock.assert_called_once_with('/copy_from/*')
+    self.assertEqual(self.copydir_mock.call_count, 0)
+    self.copyfile_mock.assert_has_calls([
+      call('/copy_from/foo', '/archive_dir/posix_foo/'),
+      call('/copy_from/bar', '/archive_dir/posix_foo/')])
+
+  @patch('glob.glob', Mock(side_effect=lambda x: [x]))
+  def testCopyRename(self):
+    self.archive.Copy('/copy_from', [('file1', 'file1_renamed')])
+    self.assertEqual(self.copydir_mock.call_count, 0)
+    self.copyfile_mock.assert_called_once_with(
+        '/copy_from/file1', '/archive_dir/posix_foo/file1_renamed')
+
+  @patch('glob.glob', Mock(side_effect=lambda x: [x]))
+  def testCopyNewDir(self):
+    self.archive.Copy('/copy_from', [('file1', 'todir/')])
+    self.assertEqual(self.copydir_mock.call_count, 0)
+    self.copyfile_mock.assert_called_once_with(
+        '/copy_from/file1', '/archive_dir/posix_foo/todir/file1')
+
+  @patch('glob.glob', Mock(side_effect=lambda x: [x]))
+  def testCopyDir(self):
+    self.isdir_mock.side_effect = lambda _: True
+    self.archive.Copy('/copy_from', ['dirname'])
+    self.assertEqual(self.copyfile_mock.call_count, 0)
+    self.copydir_mock.assert_called_once_with(
+        '/copy_from/dirname', '/archive_dir/posix_foo/dirname')
+
+
+class WinTestCase(unittest.TestCase):
+  def setUp(self):
+    patch('build_artifacts.PLATFORM', 'win').start()
+    patch('build_artifacts.BUILD_ARCHIVE_DIR', 'c:\\archive_dir\\').start()
+    patch('os.path.join', ntpath.join).start()
+
+  def tearDown(self):
+    patch.stopall()
+
+  @patch('os.path.exists', Mock(return_value=False))
+  @patch('buildbot_common.MakeDir')
+  def testArchiveInit(self, makedir_mock):
+    archive = build_artifacts.Archive('foo')
+    self.assertEqual(archive.name, 'win_foo')
+    self.assertEqual(archive.archive_name, 'win_foo.tar.bz2')
+    self.assertEqual(archive.archive_path, r'c:\archive_dir\win_foo.tar.bz2')
+    self.assertEqual(archive.dirname, r'c:\archive_dir\win_foo')
+    makedir_mock.assert_called_once_with(r'c:\archive_dir\win_foo')
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/native_client_sdk/src/test_all.py b/native_client_sdk/src/test_all.py
index 6f2113a..4a451f4 100755
--- a/native_client_sdk/src/test_all.py
+++ b/native_client_sdk/src/test_all.py
@@ -30,6 +30,7 @@ EXTRACT_PACKAGES = ['nacl_x86_glibc']
 TOOLCHAIN_OUT = os.path.join(build_paths.OUT_DIR, 'sdk_tests', 'toolchain')
 
 TEST_MODULES = [
+    'build_artifacts_test',
     'build_version_test',
     'create_html_test',
     'create_nmf_test',