path: root/native_client_sdk/src/build_tools/build_sdk.py

#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Entry point for both build and try bots

This script is invoked from XXX, usually without arguments
to package an SDK. It automatically determines whether
this SDK is for mac, win, linux.

The script inspects the following environment variables:

BUILDBOT_BUILDERNAME to determine whether the script is run locally
and whether it should upload an SDK to file storage (GSTORE)
"""


# std python includes
import copy
import datetime
import generate_make
import optparse
import os
import platform
import subprocess
import sys

# local includes
import buildbot_common
import build_updater
import build_utils
import manifest_util
from tests import test_server

# Create the various paths of interest
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
SDK_EXAMPLE_DIR = os.path.join(SDK_SRC_DIR, 'examples')
SDK_LIBRARY_DIR = os.path.join(SDK_SRC_DIR, 'libraries')
SDK_DIR = os.path.dirname(SDK_SRC_DIR)
SRC_DIR = os.path.dirname(SDK_DIR)
NACL_DIR = os.path.join(SRC_DIR, 'native_client')
OUT_DIR = os.path.join(SRC_DIR, 'out')
PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
SERVER_DIR = os.path.join(OUT_DIR, 'local_server')


# Add SDK make tools scripts to the python path.
sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
sys.path.append(os.path.join(NACL_DIR, 'build'))

import getos
import http_download
import oshelpers

GSTORE = 'http://commondatastorage.googleapis.com/nativeclient-mirror/nacl/'
MAKE = 'nacl_sdk/make_3_81/make.exe'
CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py')


def BuildOutputDir(*paths):
  return os.path.join(OUT_DIR, *paths)


def GetGlibcToolchain(platform, arch):
  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
  tcname = 'toolchain_%s_%s.tar.bz2' % (platform, arch)
  return os.path.join(tcdir, tcname)


def GetNewlibToolchain(platform, arch):
  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
  tcname = 'naclsdk_%s_%s.tgz' % (platform, arch)
  return os.path.join(tcdir, tcname)


def GetPNaClToolchain(os_platform, arch):
  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
  # Refine the toolchain host arch.  For linux, we happen to have
  # toolchains for 64-bit hosts.  For other OSes, we only have 32-bit binaries.
  arch = 'x86_32'
  if os_platform == 'linux' and platform.machine() == 'x86_64':
    arch = 'x86_64'
  tcname = 'naclsdk_pnacl_%s_%s.tgz' % (os_platform, arch)
  return os.path.join(tcdir, tcname)

def GetScons():
  if sys.platform in ['cygwin', 'win32']:
    return 'scons.bat'
  return './scons'


def GetArchName(arch, xarch=None):
  if xarch:
    return arch + '-' + str(xarch)
  return arch


def GetToolchainNaClInclude(tcname, tcpath, arch, xarch=None):
  if arch == 'x86':
    if tcname == 'pnacl':
      return os.path.join(tcpath, 'newlib', 'sdk', 'include')
    return os.path.join(tcpath, 'x86_64-nacl', 'include')
  else:
    buildbot_common.ErrorExit('Unknown architecture.')


def GetToolchainNaClLib(tcname, tcpath, arch, xarch):
  if arch == 'x86':
    if tcname == 'pnacl':
      return os.path.join(tcpath, 'newlib', 'sdk', 'lib')
    if str(xarch) == '32':
      return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
    if str(xarch) == '64':
      return os.path.join(tcpath, 'x86_64-nacl', 'lib')
  buildbot_common.ErrorExit('Unknown architecture.')


def GetPNaClNativeLib(tcpath, arch):
  if arch not in ['arm', 'x86-32', 'x86-64']:
    buildbot_common.ErrorExit('Unknown architecture %s.' % arch)
  return os.path.join(tcpath, 'lib-' + arch)


def GetBuildArgs(tcname, tcpath, outdir, arch, xarch=None):
  """Return list of scons build arguments to generate user libraries."""
  scons = GetScons()
  mode = '--mode=opt-host,nacl'
  arch_name = GetArchName(arch, xarch)
  plat = 'platform=' + arch_name
  bin = 'bindir=' + os.path.join(outdir, 'tools')
  lib = 'libdir=' + GetToolchainNaClLib(tcname, tcpath, arch, xarch)
  args = [scons, mode, plat, bin, lib, '-j10',
          'install_bin', 'install_lib']
  if tcname == 'glibc':
    args.append('--nacl_glibc')

  if tcname == 'pnacl':
    args.append('bitcode=1')

  print "Building %s (%s): %s" % (tcname, arch, ' '.join(args))
  return args


def BuildStepBuildToolsTests():
  buildbot_common.BuildStep('Run build_tools tests')
  buildbot_common.Run([sys.executable,
      os.path.join(SDK_SRC_DIR, 'build_tools', 'tests', 'test_all.py')])


def BuildStepDownloadToolchains(platform):
  buildbot_common.BuildStep('Rerun hooks to get toolchains')
  buildbot_common.Run(['gclient', 'runhooks'],
                      cwd=SRC_DIR, shell=(platform=='win'))


def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
  buildbot_common.BuildStep('Clean Pepper Dirs')
  buildbot_common.RemoveDir(pepperdir_old)
  buildbot_common.RemoveDir(pepperdir)
  buildbot_common.MakeDir(pepperdir)


def BuildStepMakePepperDirs(pepperdir, subdirs):
  for subdir in subdirs:
    buildbot_common.MakeDir(os.path.join(pepperdir, subdir))


def BuildStepCopyTextFiles(pepperdir, pepper_ver, revision):
  buildbot_common.BuildStep('Add Text Files')
  files = ['AUTHORS', 'COPYING', 'LICENSE', 'NOTICE']
  files = [os.path.join(SDK_SRC_DIR, filename) for filename in files]
  oshelpers.Copy(['-v'] + files + [pepperdir])

  # Replace a few placeholders in README
  readme_text = open(os.path.join(SDK_SRC_DIR, 'README'), 'rt').read()
  readme_text = readme_text.replace('${VERSION}', pepper_ver)
  readme_text = readme_text.replace('${REVISION}', revision)

  # Year/Month/Day Hour:Minute:Second
  time_format = '%Y/%m/%d %H:%M:%S'
  readme_text = readme_text.replace('${DATE}',
      datetime.datetime.now().strftime(time_format))

  open(os.path.join(pepperdir, 'README'), 'wt').write(readme_text)


def BuildStepUntarToolchains(pepperdir, platform, arch, toolchains):
  buildbot_common.BuildStep('Untar Toolchains')
  tcname = platform + '_' + arch
  tmpdir = os.path.join(SRC_DIR, 'out', 'tc_temp')
  buildbot_common.RemoveDir(tmpdir)
  buildbot_common.MakeDir(tmpdir)

  if 'newlib' in toolchains:
    # Untar the newlib toolchains
    tarfile = GetNewlibToolchain(platform, arch)
    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                        cwd=NACL_DIR)

    # Then rename/move it to the pepper toolchain directory
    srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk')
    newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
    buildbot_common.Move(srcdir, newlibdir)

  if 'glibc' in toolchains:
    # Untar the glibc toolchains
    tarfile = GetGlibcToolchain(platform, arch)
    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                        cwd=NACL_DIR)

    # Then rename/move it to the pepper toolchain directory
    srcdir = os.path.join(tmpdir, 'toolchain', tcname)
    glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
    buildbot_common.Move(srcdir, glibcdir)

  # Untar the pnacl toolchains
  if 'pnacl' in toolchains:
    tmpdir = os.path.join(tmpdir, 'pnacl')
    buildbot_common.RemoveDir(tmpdir)
    buildbot_common.MakeDir(tmpdir)
    tarfile = GetPNaClToolchain(platform, arch)
    buildbot_common.Run([sys.executable, CYGTAR, '-C', tmpdir, '-xf', tarfile],
                        cwd=NACL_DIR)

    # Then rename/move it to the pepper toolchain directory
    pnacldir = os.path.join(pepperdir, 'toolchain', tcname + '_pnacl')
    buildbot_common.Move(tmpdir, pnacldir)


HEADER_MAP = {
  'newlib': {
      'pthread.h': 'src/untrusted/pthread/pthread.h',
      'semaphore.h': 'src/untrusted/pthread/semaphore.h',
      'nacl/dynamic_annotations.h':
          'src/untrusted/valgrind/dynamic_annotations.h',
      'nacl/nacl_dyncode.h': 'src/untrusted/nacl/nacl_dyncode.h',
      'nacl/nacl_startup.h': 'src/untrusted/nacl/nacl_startup.h',
      'nacl/nacl_thread.h': 'src/untrusted/nacl/nacl_thread.h',
      'pnacl.h': 'src/untrusted/nacl/pnacl.h',
      'irt.h': 'src/untrusted/irt/irt.h',
      'irt_ppapi.h': 'src/untrusted/irt/irt_ppapi.h',
  },
  'glibc': {
      'nacl/dynamic_annotations.h':
          'src/untrusted/valgrind/dynamic_annotations.h',
      'nacl/nacl_dyncode.h': 'src/untrusted/nacl/nacl_dyncode.h',
      'nacl/nacl_startup.h': 'src/untrusted/nacl/nacl_startup.h',
      'nacl/nacl_thread.h': 'src/untrusted/nacl/nacl_thread.h',
      'pnacl.h': 'src/untrusted/nacl/pnacl.h',
      'irt.h': 'src/untrusted/irt/irt.h',
      'irt_ppapi.h': 'src/untrusted/irt/irt_ppapi.h',
  },
  'libs': {
  },
}


def InstallHeaders(tc_dst_inc, pepper_ver, tc_name):
  """Copies NaCl headers to expected locations in the toolchain."""
  tc_map = HEADER_MAP[tc_name]
  for filename in tc_map:
    src = os.path.join(NACL_DIR, tc_map[filename])
    dst = os.path.join(tc_dst_inc, filename)
    buildbot_common.MakeDir(os.path.dirname(dst))
    buildbot_common.CopyFile(src, dst)

  # Clean out per toolchain ppapi directory
  ppapi = os.path.join(tc_dst_inc, 'ppapi')
  buildbot_common.RemoveDir(ppapi)

  # Copy in c and c/dev headers
  buildbot_common.MakeDir(os.path.join(ppapi, 'c', 'dev'))
  buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'c', '*.h'),
          os.path.join(ppapi, 'c'))
  buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'c', 'dev', '*.h'),
          os.path.join(ppapi, 'c', 'dev'))

  # Run the generator to overwrite IDL files
  generator_args = [sys.executable, 'generator.py', '--wnone', '--cgen',
      '--verbose', '--dstroot=%s/c' % ppapi]
  if pepper_ver:
    generator_args.append('--release=M' + pepper_ver)
  buildbot_common.Run(generator_args, cwd=os.path.join(PPAPI_DIR, 'generators'))

  # Remove private and trusted interfaces
  buildbot_common.RemoveDir(os.path.join(ppapi, 'c', 'private'))
  buildbot_common.RemoveDir(os.path.join(ppapi, 'c', 'trusted'))

  # Copy in the C++ headers
  buildbot_common.MakeDir(os.path.join(ppapi, 'cpp', 'dev'))
  buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'cpp','*.h'),
          os.path.join(ppapi, 'cpp'))
  buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'cpp', 'dev', '*.h'),
          os.path.join(ppapi, 'cpp', 'dev'))
  buildbot_common.MakeDir(os.path.join(ppapi, 'utility', 'graphics'))
  buildbot_common.MakeDir(os.path.join(ppapi, 'utility', 'threading'))
  buildbot_common.MakeDir(os.path.join(ppapi, 'utility', 'websocket'))
  buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'utility','*.h'),
          os.path.join(ppapi, 'utility'))
  buildbot_common.CopyDir(os.path.join(PPAPI_DIR, 'utility', 'graphics', '*.h'),
          os.path.join(ppapi, 'utility', 'graphics'))
  buildbot_common.CopyDir(
          os.path.join(PPAPI_DIR, 'utility', 'threading', '*.h'),
          os.path.join(ppapi, 'utility', 'threading'))
  buildbot_common.CopyDir(
          os.path.join(PPAPI_DIR, 'utility', 'websocket', '*.h'),
          os.path.join(ppapi, 'utility', 'websocket'))

  # Copy in the gles2 headers
  buildbot_common.MakeDir(os.path.join(ppapi, 'gles2'))
  buildbot_common.CopyDir(os.path.join(PPAPI_DIR,'lib','gl','gles2','*.h'),
          os.path.join(ppapi, 'gles2'))

  # Copy the EGL headers
  buildbot_common.MakeDir(os.path.join(tc_dst_inc, 'EGL'))
  buildbot_common.CopyDir(
          os.path.join(PPAPI_DIR,'lib','gl','include','EGL', '*.h'),
          os.path.join(tc_dst_inc, 'EGL'))

  # Copy the GLES2 headers
  buildbot_common.MakeDir(os.path.join(tc_dst_inc, 'GLES2'))
  buildbot_common.CopyDir(
          os.path.join(PPAPI_DIR,'lib','gl','include','GLES2', '*.h'),
          os.path.join(tc_dst_inc, 'GLES2'))

  # Copy the KHR headers
  buildbot_common.MakeDir(os.path.join(tc_dst_inc, 'KHR'))
  buildbot_common.CopyDir(
          os.path.join(PPAPI_DIR,'lib','gl','include','KHR', '*.h'),
          os.path.join(tc_dst_inc, 'KHR'))

  # Copy the lib files
  buildbot_common.CopyDir(os.path.join(PPAPI_DIR,'lib'), 
          os.path.join(tc_dst_inc, 'ppapi'))


def BuildStepBuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains):
  buildbot_common.BuildStep('SDK Items')

  tcname = platform + '_' + arch
  newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
  glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
  pnacldir = os.path.join(pepperdir, 'toolchain', tcname + '_pnacl')

  # Run scons TC build steps
  if arch == 'x86':
    if 'newlib' in toolchains:
      buildbot_common.Run(
          GetBuildArgs('newlib', newlibdir, pepperdir, 'x86', '32'),
          cwd=NACL_DIR, shell=(platform=='win'))
      buildbot_common.Run(
          GetBuildArgs('newlib', newlibdir, pepperdir, 'x86', '64'),
          cwd=NACL_DIR, shell=(platform=='win'))
      InstallHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
                     pepper_ver,
                     'newlib')

    if 'glibc' in toolchains:
      buildbot_common.Run(
          GetBuildArgs('glibc', glibcdir, pepperdir, 'x86', '32'),
          cwd=NACL_DIR, shell=(platform=='win'))
      buildbot_common.Run(
          GetBuildArgs('glibc', glibcdir, pepperdir, 'x86', '64'),
          cwd=NACL_DIR, shell=(platform=='win'))
      InstallHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
                     pepper_ver,
                     'glibc')

    if 'pnacl' in toolchains:
      buildbot_common.Run(
          GetBuildArgs('pnacl', pnacldir, pepperdir, 'x86', '32'),
          cwd=NACL_DIR, shell=(platform=='win'))
      buildbot_common.Run(
          GetBuildArgs('pnacl', pnacldir, pepperdir, 'x86', '64'),
          cwd=NACL_DIR, shell=(platform=='win'))
      # Pnacl libraries are typically bitcode, but some are native and
      # will be looked up in the native library directory.
      buildbot_common.Move(
          os.path.join(GetToolchainNaClLib('pnacl', pnacldir, 'x86', '64'),
                       'libpnacl_irt_shim.a'),
          GetPNaClNativeLib(pnacldir, 'x86-64'))
      InstallHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
                     pepper_ver,
                     'newlib')
  else:
    buildbot_common.ErrorExit('Missing arch %s' % arch)


def BuildStepCopyBuildHelpers(pepperdir, platform):
  buildbot_common.BuildStep('Copy build helpers')
  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
      os.path.join(pepperdir, 'tools'))
  if platform == 'win':
    buildbot_common.BuildStep('Add MAKE')
    http_download.HttpDownload(GSTORE + MAKE,
                               os.path.join(pepperdir, 'tools' ,'make.exe'))
    rename_list = ['ncval_x86_32', 'ncval_x86_64',
                   'sel_ldr_x86_32', 'sel_ldr_x86_64']
    tools = os.path.join(pepperdir, 'tools')
    for name in rename_list:
        src = os.path.join(pepperdir, 'tools', name)
        dst = os.path.join(pepperdir, 'tools', name + '.exe')
        buildbot_common.Move(src, dst)


EXAMPLE_LIST = [
  'debugging',
  'file_histogram',
  'file_io',
  'fullscreen_tumbler',
  'gamepad',
  'geturl',
  'hello_world_interactive',
  'hello_world',
  'hello_world_gles',
  'input_events',
  'load_progress',
  'mouselock',
  'multithreaded_input_events',
  'pi_generator',
  'pong',
  'sine_synth',
  'tumbler',
  'websocket',
  'dlopen',
]

LIBRARY_LIST = [
  'nacl_mounts',
  'ppapi',
  'ppapi_cpp',
  'ppapi_gles2',
  'pthread',
]

LIB_DICT = {
  'linux': [],
  'mac': [],
  'win': ['x86_32']
}

def BuildStepCopyExamples(pepperdir, toolchains, build_experimental):
  buildbot_common.BuildStep('Copy examples')

  if not os.path.exists(os.path.join(pepperdir, 'tools')):
    buildbot_common.ErrorExit('Examples depend on missing tools.')
  if not os.path.exists(os.path.join(pepperdir, 'toolchain')):
    buildbot_common.ErrorExit('Examples depend on missing toolchains.')

  exampledir = os.path.join(pepperdir, 'examples')
  buildbot_common.RemoveDir(exampledir)
  buildbot_common.MakeDir(exampledir)

  libdir = os.path.join(pepperdir, 'lib')
  buildbot_common.RemoveDir(libdir)
  buildbot_common.MakeDir(libdir)

  plat = getos.GetPlatform()
  for arch in LIB_DICT[plat]:
    buildbot_common.MakeDir(os.path.join(libdir, '%s_%s_host' % (plat, arch)))
    for config in ['Debug', 'Release']:
      buildbot_common.MakeDir(os.path.join(libdir, '%s_%s_host' % (plat, arch),
			      config))
    

  srcdir = os.path.join(pepperdir, 'src')
  buildbot_common.RemoveDir(srcdir)
  buildbot_common.MakeDir(srcdir)


  # Copy individual files
  files = ['favicon.ico', 'httpd.cmd', 'httpd.py', 'index.html']
  for filename in files:
    oshelpers.Copy(['-v', os.path.join(SDK_EXAMPLE_DIR, filename), exampledir])

  args = ['--dstroot=%s' % pepperdir, '--master']
  for toolchain in toolchains:
    args.append('--' + toolchain)

  for example in EXAMPLE_LIST:
    dsc = os.path.join(SDK_EXAMPLE_DIR, example, 'example.dsc')
    args.append(dsc)

  for library in LIBRARY_LIST:
    dsc = os.path.join(SDK_LIBRARY_DIR, library, 'library.dsc')
    args.append(dsc)

  if build_experimental:
    args.append('--experimental')

  if generate_make.main(args):
    buildbot_common.ErrorExit('Failed to build examples.')


def GetWindowsEnvironment():
  sys.path.append(os.path.join(NACL_DIR, 'buildbot'))
  import buildbot_standard

  # buildbot_standard.SetupWindowsEnvironment expects a "context" object. We'll
  # fake enough of that here to work.
  class FakeContext(object):
    def __init__(self):
      self.env = os.environ

    def GetEnv(self, key):
      return self.env[key]

    def __getitem__(self, key):
      return self.env[key]
    
    def SetEnv(self, key, value):
      self.env[key] = value

    def __setitem__(self, key, value):
      self.env[key] = value

  context = FakeContext()
  buildbot_standard.SetupWindowsEnvironment(context)

  # buildbot_standard.SetupWindowsEnvironment adds the directory which contains
  # vcvarsall.bat to the path, but not the directory which contains cl.exe,
  # link.exe, etc.
  # Running vcvarsall.bat adds the correct directories to the path, which we
  # extract below.
  process = subprocess.Popen('vcvarsall.bat x86 > NUL && set',
      stdout=subprocess.PIPE, env=context.env, shell=True)
  stdout, _ = process.communicate()

  # Parse environment from "set" command above.
  # It looks like this:
  # KEY1=VALUE1\r\n
  # KEY2=VALUE2\r\n
  # ...
  return dict(line.split('=') for line in stdout.split('\r\n')[:-1])


def BuildStepMakeAll(pepperdir, platform, directory, step_name, clean=False):
  buildbot_common.BuildStep(step_name)
  make_dir = os.path.join(pepperdir, directory)
  makefile = os.path.join(make_dir, 'Makefile')
  if os.path.isfile(makefile):
    print "\n\nMake: " + make_dir
    if platform == 'win':
      # We need to modify the environment to build host on Windows.
      env = GetWindowsEnvironment()
    else:
      env = os.environ

    buildbot_common.Run(['make', '-j8'],
                        cwd=os.path.abspath(make_dir), shell=True, env=env)
    if clean:
      # Clean to remove temporary files but keep the built libraries.
      buildbot_common.Run(['make', '-j8', 'clean'],
                          cwd=os.path.abspath(make_dir), shell=True)


def BuildStepBuildLibraries(pepperdir, platform, directory):
  BuildStepMakeAll(pepperdir, platform, directory, 'Build Libraries',
      clean=True)


def BuildStepTarBundle(pepper_ver, tarfile):
  buildbot_common.BuildStep('Tar Pepper Bundle')
  buildbot_common.MakeDir(os.path.dirname(tarfile))
  buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
       'pepper_' + pepper_ver], cwd=NACL_DIR)


def GetManifestBundle(pepper_ver, revision, tarfile, archive_url):
  with open(tarfile, 'rb') as tarfile_stream:
    archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
        tarfile_stream)

  archive = manifest_util.Archive(manifest_util.GetHostOS())
  archive.url = archive_url
  archive.size = archive_size
  archive.checksum = archive_sha1

  bundle = manifest_util.Bundle('pepper_' + pepper_ver)
  bundle.revision = int(revision)
  bundle.repath = 'pepper_' + pepper_ver
  bundle.version = int(pepper_ver)
  bundle.description = 'Chrome %s bundle, revision %s' % (pepper_ver, revision)
  bundle.stability = 'dev'
  bundle.recommended = 'no'
  bundle.archives = [archive]
  return bundle


def BuildStepTestUpdater(platform, pepper_ver, revision, tarfile):
  tarname = os.path.basename(tarfile)
  server = None
  try:
    buildbot_common.BuildStep('Run local server')
    server = test_server.LocalHTTPServer(SERVER_DIR)

    buildbot_common.BuildStep('Generate manifest')
    bundle = GetManifestBundle(pepper_ver, revision, tarfile,
        server.GetURL(tarname))

    manifest = manifest_util.SDKManifest()
    manifest.SetBundle(bundle)
    manifest_name = 'naclsdk_manifest2.json'
    with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \
        manifest_stream:
      manifest_stream.write(manifest.GetDataAsString())

    # use newly built sdk updater to pull this bundle
    buildbot_common.BuildStep('Update from local server')
    naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk')
    if platform == 'win':
      naclsdk_sh += '.bat'
    buildbot_common.Run([naclsdk_sh, '-U',
        server.GetURL(manifest_name), 'update', 'pepper_' + pepper_ver])

    # Return the new pepper directory as the one inside the downloaded SDK.
    return os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver)

  # kill server
  finally:
    if server:
      server.Shutdown()


def BuildStepBuildExamples(pepperdir, platform):
  BuildStepMakeAll(pepperdir, platform, 'examples', 'Build Examples')


TEST_EXAMPLE_LIST = [
  'nacl_mounts_test',
]

TEST_LIBRARY_LIST = [
  'gtest',
]

def BuildStepCopyTests(pepperdir, toolchains, build_experimental):
  buildbot_common.BuildStep('Copy Tests')

  testingdir = os.path.join(pepperdir, 'testing')
  buildbot_common.RemoveDir(testingdir)
  buildbot_common.MakeDir(testingdir)

  args = ['--dstroot=%s' % pepperdir, '--master']
  for toolchain in toolchains:
    args.append('--' + toolchain)

  for library in TEST_LIBRARY_LIST:
    dsc = os.path.join(SDK_LIBRARY_DIR, library, 'library.dsc')
    args.append(dsc)

  for example in TEST_EXAMPLE_LIST:
    dsc = os.path.join(SDK_LIBRARY_DIR, example, 'example.dsc')
    args.append(dsc)

  if build_experimental:
    args.append('--experimental')

  if generate_make.main(args):
    buildbot_common.ErrorExit('Failed to build tests.')


def BuildStepBuildTests(pepperdir, platform):
  BuildStepMakeAll(pepperdir, platform, 'testing', 'Build Tests')


def BuildStepRunPyautoTests(pepperdir, platform, pepper_ver):
  buildbot_common.BuildStep('Test Examples')
  env = copy.copy(os.environ)
  env['PEPPER_VER'] = pepper_ver
  env['NACL_SDK_ROOT'] = pepperdir

  pyauto_script = os.path.join(SRC_DIR, 'chrome', 'test', 'functional',
      'nacl_sdk.py')
  pyauto_script_args = ['nacl_sdk.NaClSDKTest.NaClSDKExamples']

  if platform == 'linux' and buildbot_common.IsSDKBuilder():
    # linux buildbots need to run the pyauto tests through xvfb. Running
    # using runtest.py does this.
    #env['PYTHON_PATH'] = '.:' + env.get('PYTHON_PATH', '.')
    build_dir = os.path.dirname(SRC_DIR)
    runtest_py = os.path.join(build_dir, '..', '..', '..', 'scripts', 'slave',
        'runtest.py')
    buildbot_common.Run([sys.executable, runtest_py, '--target', 'Release',
        '--build-dir', 'src/build', sys.executable,
        pyauto_script] + pyauto_script_args,
        cwd=build_dir, env=env)
  else:
    buildbot_common.Run([sys.executable, 'nacl_sdk.py',
      'nacl_sdk.NaClSDKTest.NaClSDKExamples'],
      cwd=os.path.dirname(pyauto_script),
      env=env)


def BuildStepArchiveBundle(pepper_ver, revision, tarfile):
  buildbot_common.BuildStep('Archive build')
  bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
      build_utils.ChromeVersion(),)
  tarname = os.path.basename(tarfile)
  tarfile_dir = os.path.dirname(tarfile)
  buildbot_common.Archive(tarname, bucket_path, tarfile_dir)

  # generate "manifest snippet" for this archive.
  archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
      build_utils.ChromeVersion(), tarname)
  bundle = GetManifestBundle(pepper_ver, revision, tarfile, archive_url)

  manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
  with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
    manifest_snippet_stream.write(bundle.GetDataAsString())

  buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
                          step_link=False)


def BuildStepArchiveSDKTools():
  # Only push up sdk_tools.tgz on the linux buildbot.
  builder_name = os.getenv('BUILDBOT_BUILDERNAME','')
  if builder_name == 'linux-sdk-multi':
    buildbot_common.BuildStep('Archive SDK Tools')
    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
        build_utils.ChromeVersion(),)
    buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
                            step_link=False)


def main(args):
  parser = optparse.OptionParser()
  parser.add_option('--pnacl', help='Enable pnacl build.',
      action='store_true', dest='pnacl', default=False)
  parser.add_option('--examples', help='Only build the examples.',
      action='store_true', dest='only_examples', default=False)
  parser.add_option('--update', help='Only build the updater.',
      action='store_true', dest='only_updater', default=False)
  parser.add_option('--test-examples',
      help='Run the pyauto tests for examples.', action='store_true',
      dest='test_examples', default=False)
  parser.add_option('--skip-tar', help='Skip generating a tarball.',
      action='store_true', dest='skip_tar', default=False)
  parser.add_option('--archive', help='Force the archive step.',
      action='store_true', dest='archive', default=False)
  parser.add_option('--release', help='PPAPI release version.',
      dest='release', default=None)
  parser.add_option('--experimental',
      help='build experimental examples and libraries', action='store_true',
      dest='build_experimental', default=False)

  options, args = parser.parse_args(args[1:])
  platform = getos.GetPlatform()
  arch = 'x86'

  builder_name = os.getenv('BUILDBOT_BUILDERNAME','')
  if builder_name.find('pnacl') >= 0 and builder_name.find('sdk') >= 0:
    options.pnacl = True

  # TODO(binji) There is currently a hack in download_nacl_toolchains.py that
  # won't download pnacl toolchains unless the BUILDBOT_BUILDERNAME has "pnacl"
  # and "sdk" in it. Set that here, if not already set...
  if options.pnacl and not os.getenv('BUILDBOT_BUILDERNAME'):
    os.environ['BUILDBOT_BUILDERNAME'] = 'pnacl-sdk'

  # TODO(binji) for now, only test examples on non-trybots. Trybots don't build
  # pyauto Chrome.
  if buildbot_common.IsSDKBuilder():
    options.test_examples = True

  if options.pnacl:
    toolchains = ['pnacl']
  else:
    toolchains = ['newlib', 'glibc', 'host']
  print 'Building: ' + ' '.join(toolchains)

  if options.archive and (options.only_examples or options.skip_tar):
    parser.error('Incompatible arguments with archive.')

  pepper_ver = str(int(build_utils.ChromeMajorVersion()))
  pepper_old = str(int(build_utils.ChromeMajorVersion()) - 1)
  pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver)
  pepperdir_old = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_old)
  clnumber = build_utils.ChromeRevision()
  tarname = 'naclsdk_' + platform + '.tar.bz2'
  if 'pnacl' in toolchains:
    tarname = 'p' + tarname
  tarfile = os.path.join(SERVER_DIR, tarname)

  if options.release:
    pepper_ver = options.release
  print 'Building PEPPER %s at %s' % (pepper_ver, clnumber)

  if options.only_examples:
    BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental)
    BuildStepBuildLibraries(pepperdir, platform, 'src')
    BuildStepBuildExamples(pepperdir, platform)
    BuildStepCopyTests(pepperdir, toolchains, options.build_experimental)
    BuildStepBuildTests(pepperdir, platform)
    if options.test_examples:
      BuildStepRunPyautoTests(pepperdir, platform, pepper_ver)
  elif options.only_updater:
    build_updater.BuildUpdater(OUT_DIR)
  else:  # Build everything.
    BuildStepBuildToolsTests()

    BuildStepDownloadToolchains(platform)
    BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
    BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
    BuildStepCopyTextFiles(pepperdir, pepper_ver, clnumber)
    BuildStepUntarToolchains(pepperdir, platform, arch, toolchains)
    BuildStepBuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains)
    InstallHeaders(os.path.join(pepperdir, 'include'), None, 'libs')
    BuildStepCopyBuildHelpers(pepperdir, platform)
    BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental)

    # Ship with libraries prebuilt, so run that first.
    BuildStepBuildLibraries(pepperdir, platform, 'src')

    if not options.skip_tar:
      BuildStepTarBundle(pepper_ver, tarfile)
      build_updater.BuildUpdater(OUT_DIR)

      # BuildStepTestUpdater downloads the bundle to its own directory. Build
      # the examples and test from this directory instead of the original.
      pepperdir = BuildStepTestUpdater(platform, pepper_ver, clnumber, tarfile)
      BuildStepBuildExamples(pepperdir, platform)
      BuildStepCopyTests(pepperdir, toolchains, options.build_experimental)
      BuildStepBuildTests(pepperdir, platform)
      if options.test_examples:
        BuildStepRunPyautoTests(pepperdir, platform, pepper_ver)

      # Archive on non-trybots.
      if options.archive or buildbot_common.IsSDKBuilder():
        BuildStepArchiveBundle(pepper_ver, clnumber, tarfile)
        BuildStepArchiveSDKTools()

  return 0


if __name__ == '__main__':
  sys.exit(main(sys.argv))