diff options
author | binji@chromium.org <binji@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-11-16 22:10:37 +0000 |
---|---|---|
committer | binji@chromium.org <binji@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-11-16 22:10:37 +0000 |
commit | 97525370580abfdde511db31db159aa064b2cbda (patch) | |
tree | 5af540efc39dc1b0b8bfd317978069761f63913b /native_client_sdk | |
parent | a8e69ca9dce5fbfc667245888b343cea726ab6da (diff) | |
download | chromium_src-97525370580abfdde511db31db159aa064b2cbda.zip chromium_src-97525370580abfdde511db31db159aa064b2cbda.tar.gz chromium_src-97525370580abfdde511db31db159aa064b2cbda.tar.bz2 |
[NaCl SDK] Refactor build_sdk into test_sdk and build_examples.
This is part of the overall effort to reduce the amount of work done by
build_sdk.
BUG=none
R=sbc@chromium.org
TBR=noelallen@chromium.org
NOTRY=true
Review URL: https://chromiumcodereview.appspot.com/11377173
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@168303 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'native_client_sdk')
-rwxr-xr-x | native_client_sdk/src/build_tools/build_examples.py | 61 | ||||
-rwxr-xr-x | native_client_sdk/src/build_tools/build_sdk.py | 240 | ||||
-rw-r--r-- | native_client_sdk/src/build_tools/buildbot_common.py | 16 | ||||
-rwxr-xr-x | native_client_sdk/src/build_tools/test_sdk.py | 132 |
4 files changed, 256 insertions, 193 deletions
diff --git a/native_client_sdk/src/build_tools/build_examples.py b/native_client_sdk/src/build_tools/build_examples.py new file mode 100755 index 0000000..ae5bb73 --- /dev/null +++ b/native_client_sdk/src/build_tools/build_examples.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import optparse +import os +import sys + +import build_sdk +import build_utils +import test_sdk + +sys.path.append(os.path.join(build_sdk.SDK_SRC_DIR, 'tools')) +import getos + + +def main(args): + parser = optparse.OptionParser() + parser.add_option('--clobber-examples', + help='Don\'t examples directory before copying new files', + action='store_true' ) + parser.add_option('--test-examples', + help='Run the pyauto tests for examples.', action='store_true') + parser.add_option('--experimental', + help='build experimental examples and libraries', action='store_true') + parser.add_option('-t', '--toolchain', + help='Build using toolchain. Can be passed more than once.', + action='append') + + options, args = parser.parse_args(args[1:]) + + valid_toolchains = ['newlib', 'glibc', 'pnacl', 'host'] + if not options.toolchain: + toolchains = valid_toolchains + else: + invalid_toolchains = set(options.toolchain) - set(valid_toolchains) + if invalid_toolchains: + print 'Ignoring invalid toolchains: %s' % (', '.join(invalid_toolchains),) + toolchains = list(set(options.toolchain) - invalid_toolchains) + + pepper_ver = str(int(build_utils.ChromeMajorVersion())) + pepperdir = os.path.join(build_sdk.OUT_DIR, 'pepper_' + pepper_ver) + platform = getos.GetPlatform() + + build_sdk.BuildStepCopyExamples(pepperdir, toolchains, options.experimental, + options.clobber_examples) + # False = don't clean after building the libraries directory. + build_sdk.BuildStepBuildLibraries(pepperdir, platform, 'src', False) + test_sdk.BuildStepBuildExamples(pepperdir, platform) + test_sdk.BuildStepCopyTests(pepperdir, toolchains, options.experimental, + options.clobber_examples) + test_sdk.BuildStepBuildTests(pepperdir, platform) + if options.test_examples: + test_sdk.BuildStepRunPyautoTests(pepperdir, platform, pepper_ver) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py index 3b6e884..9465ef5 100755 --- a/native_client_sdk/src/build_tools/build_sdk.py +++ b/native_client_sdk/src/build_tools/build_sdk.py @@ -28,12 +28,11 @@ import sys # local includes import buildbot_common -import build_updater import build_utils import generate_make import generate_notice import manifest_util -from tests import test_server +import test_sdk # Create the various paths of interest SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -45,7 +44,6 @@ SRC_DIR = os.path.dirname(SDK_DIR) NACL_DIR = os.path.join(SRC_DIR, 'native_client') OUT_DIR = os.path.join(SRC_DIR, 'out') PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi') -SERVER_DIR = os.path.join(OUT_DIR, 'local_server') # Add SDK make tools scripts to the python path. @@ -64,10 +62,6 @@ CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py') options = None -def BuildOutputDir(*paths): - return os.path.join(OUT_DIR, *paths) - - def GetGlibcToolchain(platform, arch): tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars') tcname = 'toolchain_%s_%s.tar.bz2' % (platform, arch) @@ -144,12 +138,6 @@ def GetBuildArgs(tcname, tcpath, outdir, arch, xarch=None): return args -def BuildStepBuildToolsTests(): - buildbot_common.BuildStep('Run build_tools tests') - buildbot_common.Run([sys.executable, - os.path.join(SDK_SRC_DIR, 'build_tools', 'tests', 'test_all.py')]) - - def BuildStepDownloadToolchains(platform): buildbot_common.BuildStep('Rerun hooks to get toolchains') buildbot_common.Run(['gclient', 'runhooks'], @@ -750,6 +738,15 @@ def BuildStepTarBundle(pepper_ver, tarfile): 'pepper_' + pepper_ver], cwd=NACL_DIR) +def BuildStepRunTests(): + args = [] + if options.build_experimental: + args.append('--experimental') + if options.run_pyauto_tests: + args.append('--pyauto') + test_sdk.main(args) + + def GetManifestBundle(pepper_ver, revision, tarfile, archive_url): with open(tarfile, 'rb') as tarfile_stream: archive_sha1, archive_size = manifest_util.DownloadAndComputeHash( @@ -771,114 +768,6 @@ def GetManifestBundle(pepper_ver, revision, tarfile, archive_url): return bundle -def BuildStepTestUpdater(platform, pepper_ver, revision, tarfile): - tarname = os.path.basename(tarfile) - server = None - try: - buildbot_common.BuildStep('Run local server') - server = test_server.LocalHTTPServer(SERVER_DIR) - - buildbot_common.BuildStep('Generate manifest') - bundle = GetManifestBundle(pepper_ver, revision, tarfile, - server.GetURL(tarname)) - - manifest = manifest_util.SDKManifest() - manifest.SetBundle(bundle) - manifest_name = 'naclsdk_manifest2.json' - with open(os.path.join(SERVER_DIR, manifest_name), 'wb') as \ - manifest_stream: - manifest_stream.write(manifest.GetDataAsString()) - - # use newly built sdk updater to pull this bundle - buildbot_common.BuildStep('Update from local server') - naclsdk_sh = os.path.join(OUT_DIR, 'nacl_sdk', 'naclsdk') - if platform == 'win': - naclsdk_sh += '.bat' - buildbot_common.Run([naclsdk_sh, 'update', 'pepper_' + pepper_ver, - '-U', server.GetURL(manifest_name), '-v']) - - # Return the new pepper directory as the one inside the downloaded SDK. - return os.path.join(OUT_DIR, 'nacl_sdk', 'pepper_' + pepper_ver) - - # kill server - finally: - if server: - server.Shutdown() - - -def BuildStepBuildExamples(pepperdir, platform): - BuildStepMakeAll(pepperdir, platform, 'examples', 'Build Examples') - - -TEST_EXAMPLE_LIST = [ - 'nacl_mounts_test', -] - -TEST_LIBRARY_LIST = [ - 'gmock', - 'gtest', - 'gtest_ppapi', -] - - -def BuildStepCopyTests(pepperdir, toolchains, build_experimental, clobber): - buildbot_common.BuildStep('Copy Tests') - - MakeDirectoryOrClobber(pepperdir, 'testlibs', clobber) - MakeDirectoryOrClobber(pepperdir, 'tests', clobber) - - args = ['--dstroot=%s' % pepperdir, '--master'] - for toolchain in toolchains: - args.append('--' + toolchain) - - for library in TEST_LIBRARY_LIST: - dsc = os.path.join(SDK_LIBRARY_DIR, library, 'library.dsc') - args.append(dsc) - - for example in TEST_EXAMPLE_LIST: - dsc = os.path.join(SDK_LIBRARY_DIR, example, 'example.dsc') - args.append(dsc) - - if build_experimental: - args.append('--experimental') - - if generate_make.main(args): - buildbot_common.ErrorExit('Failed to build tests.') - - -def BuildStepBuildTests(pepperdir, platform): - BuildStepMakeAll(pepperdir, platform, 'testlibs', 'Build Test Libraries') - BuildStepMakeAll(pepperdir, platform, 'tests', 'Build Tests') - - -def BuildStepRunPyautoTests(pepperdir, platform, pepper_ver): - buildbot_common.BuildStep('Test Examples') - env = copy.copy(os.environ) - env['PEPPER_VER'] = pepper_ver - env['NACL_SDK_ROOT'] = pepperdir - - pyauto_script = os.path.join(SRC_DIR, 'chrome', 'test', 'functional', - 'nacl_sdk.py') - pyauto_script_args = ['nacl_sdk.NaClSDKTest.NaClSDKExamples'] - - if platform == 'linux' and buildbot_common.IsSDKBuilder(): - # linux buildbots need to run the pyauto tests through xvfb. Running - # using runtest.py does this. - #env['PYTHON_PATH'] = '.:' + env.get('PYTHON_PATH', '.') - build_dir = os.path.dirname(SRC_DIR) - runtest_py = os.path.join(build_dir, '..', '..', '..', 'scripts', 'slave', - 'runtest.py') - buildbot_common.Run([sys.executable, runtest_py, '--target', 'Release', - '--build-dir', 'src/build', sys.executable, - pyauto_script] + pyauto_script_args, - cwd=build_dir, env=env) - else: - buildbot_common.Run([sys.executable, 'nacl_sdk.py', - 'nacl_sdk.NaClSDKTest.NaClSDKExamples'], - cwd=os.path.dirname(pyauto_script), - env=env) - - def BuildStepArchiveBundle(pepper_ver, revision, tarfile): buildbot_common.BuildStep('Archive build') bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % ( @@ -915,20 +804,14 @@ def BuildStepArchiveSDKTools(): def main(args): parser = optparse.OptionParser() - parser.add_option('--examples', help='Only build the examples.', - action='store_true', dest='only_examples', default=False) - parser.add_option('--clobber-examples', - help='Don\'t examples directory before copying new files', - action='store_true', dest='clobber_examples', default=False) - parser.add_option('--update', help='Only build the updater.', - action='store_true', dest='only_updater', default=False) - parser.add_option('--test-examples', - help='Run the pyauto tests for examples.', action='store_true', - dest='test_examples', default=False) + parser.add_option('--run-tests', + help='Run tests. This includes building examples.', action='store_true') + parser.add_option('--run-pyauto-tests', + help='Run the pyauto tests for examples.', action='store_true') parser.add_option('--skip-tar', help='Skip generating a tarball.', - action='store_true', dest='skip_tar', default=False) + action='store_true') parser.add_option('--archive', help='Force the archive step.', - action='store_true', dest='archive', default=False) + action='store_true') parser.add_option('--gyp', help='Use gyp to build examples/libraries/Makefiles.', action='store_true') @@ -936,9 +819,9 @@ def main(args): dest='release', default=None) parser.add_option('--experimental', help='build experimental examples and libraries', action='store_true', - dest='build_experimental', default=False) + dest='build_experimental') parser.add_option('--skip-toolchain', help='Skip toolchain download/untar', - action='store_true', dest='skip_toolchain', default=False) + action='store_true') parser.add_option('--mac_sdk', help='Set the mac_sdk (e.g. 10.6) to use when building with ninja.', dest='mac_sdk') @@ -953,12 +836,17 @@ def main(args): # TODO(binji) for now, only test examples on non-trybots. Trybots don't build # pyauto Chrome. if buildbot_common.IsSDKBuilder(): - options.test_examples = True + options.run_tests = True + options.run_pyauto_tests = True + options.archive = True + + if buildbot_common.IsSDKTrybot(): + options.run_tests = True toolchains = ['newlib', 'glibc', 'pnacl', 'host'] print 'Building: ' + ' '.join(toolchains) - if options.archive and (options.only_examples or options.skip_tar): + if options.archive and options.skip_tar: parser.error('Incompatible arguments with archive.') pepper_ver = str(int(build_utils.ChromeMajorVersion())) @@ -967,7 +855,7 @@ def main(args): pepperdir_old = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_old) clnumber = build_utils.ChromeRevision() tarname = 'naclsdk_' + platform + '.tar.bz2' - tarfile = os.path.join(SERVER_DIR, tarname) + tarfile = os.path.join(OUT_DIR, tarname) if options.release: pepper_ver = options.release @@ -978,56 +866,32 @@ def main(args): # of the build. del os.environ['NACL_SDK_ROOT'] - if options.only_examples: - BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental, - options.clobber_examples) - BuildStepBuildLibraries(pepperdir, platform, 'src', False) # Don't clean. - BuildStepBuildExamples(pepperdir, platform) - BuildStepCopyTests(pepperdir, toolchains, options.build_experimental, - options.clobber_examples) - BuildStepBuildTests(pepperdir, platform) - if options.test_examples: - BuildStepRunPyautoTests(pepperdir, platform, pepper_ver) - elif options.only_updater: - build_updater.BuildUpdater(OUT_DIR) - else: # Build everything. - BuildStepBuildToolsTests() - - if not options.skip_toolchain: - BuildStepDownloadToolchains(platform) - BuildStepCleanPepperDirs(pepperdir, pepperdir_old) - BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools']) - BuildStepCopyTextFiles(pepperdir, pepper_ver, clnumber) - if not options.skip_toolchain: - BuildStepUntarToolchains(pepperdir, platform, arch, toolchains) - BuildStepBuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains) - InstallHeaders(os.path.join(pepperdir, 'include'), None, 'libs') - BuildStepCopyBuildHelpers(pepperdir, platform) - BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental, - True) - - # Ship with libraries prebuilt, so run that first. - BuildStepBuildLibraries(pepperdir, platform, 'src') - BuildStepGenerateNotice(pepperdir) - - if not options.skip_tar: - BuildStepTarBundle(pepper_ver, tarfile) - build_updater.BuildUpdater(OUT_DIR) - - # BuildStepTestUpdater downloads the bundle to its own directory. Build - # the examples and test from this directory instead of the original. - pepperdir = BuildStepTestUpdater(platform, pepper_ver, clnumber, tarfile) - BuildStepBuildExamples(pepperdir, platform) - BuildStepCopyTests(pepperdir, toolchains, options.build_experimental, - True) - BuildStepBuildTests(pepperdir, platform) - if options.test_examples: - BuildStepRunPyautoTests(pepperdir, platform, pepper_ver) - - # Archive on non-trybots. - if options.archive or buildbot_common.IsSDKBuilder(): - BuildStepArchiveBundle(pepper_ver, clnumber, tarfile) - BuildStepArchiveSDKTools() + if not options.skip_toolchain: + BuildStepDownloadToolchains(platform) + BuildStepCleanPepperDirs(pepperdir, pepperdir_old) + BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools']) + BuildStepCopyTextFiles(pepperdir, pepper_ver, clnumber) + if not options.skip_toolchain: + BuildStepUntarToolchains(pepperdir, platform, arch, toolchains) + BuildStepBuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains) + InstallHeaders(os.path.join(pepperdir, 'include'), None, 'libs') + BuildStepCopyBuildHelpers(pepperdir, platform) + BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental, True) + + # Ship with libraries prebuilt, so run that first. + BuildStepBuildLibraries(pepperdir, platform, 'src') + BuildStepGenerateNotice(pepperdir) + + if not options.skip_tar: + BuildStepTarBundle(pepper_ver, tarfile) + + if options.run_tests: + BuildStepRunTests() + + # Archive on non-trybots. + if options.archive: + BuildStepArchiveBundle(pepper_ver, clnumber, tarfile) + BuildStepArchiveSDKTools() return 0 diff --git a/native_client_sdk/src/build_tools/buildbot_common.py b/native_client_sdk/src/build_tools/buildbot_common.py index 7aba249..82bbeaa 100644 --- a/native_client_sdk/src/build_tools/buildbot_common.py +++ b/native_client_sdk/src/build_tools/buildbot_common.py @@ -24,16 +24,22 @@ def IsSDKBuilder(): False means it is either running on a trybot, or a user's machine. Trybot names: - naclsdkm-((pnacl-)?linux|mac|windows(32|64)) + (win|mac|linux)_nacl_sdk Builder names: - (pnacl-)?(windows|mac|linux)-sdk-multi(rel)? - - except there are currently no pnacl multirel bots, and - pnacl-windows-sdk-multi is actually called pnacl-win-sdk-multi.""" + (windows|mac|linux)-sdk-multi(rel)?""" return '-sdk-multi' in os.getenv('BUILDBOT_BUILDERNAME', '') +def IsSDKTrybot(): + """Returns True if this script is running on an SDK trybot. + + False means it is either running on an SDK builder, or a user's machine. + + See IsSDKBuilder above for trybot/buildbot names.""" + return '_nacl_sdk' in os.getenv('BUILDBOT_BUILDERNAME', '') + + def ErrorExit(msg): """Write and error to stderr, then exit with 1 signaling failure.""" sys.stderr.write(msg + '\n') diff --git a/native_client_sdk/src/build_tools/test_sdk.py b/native_client_sdk/src/build_tools/test_sdk.py new file mode 100755 index 0000000..91a72b1 --- /dev/null +++ b/native_client_sdk/src/build_tools/test_sdk.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +import copy +import optparse +import os +import sys + +import buildbot_common +import build_utils +import build_sdk +import generate_make + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR) +SDK_LIBRARY_DIR = os.path.join(SDK_SRC_DIR, 'libraries') +SDK_DIR = os.path.dirname(SDK_SRC_DIR) +SRC_DIR = os.path.dirname(SDK_DIR) +OUT_DIR = os.path.join(SRC_DIR, 'out') + +sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) +import getos + + +TEST_EXAMPLE_LIST = [ + 'nacl_mounts_test', +] + +TEST_LIBRARY_LIST = [ + 'gmock', + 'gtest', + 'gtest_ppapi', +] + + +def BuildStepBuildToolsTests(): + buildbot_common.BuildStep('Run build_tools tests') + test_all_py = os.path.join(SDK_SRC_DIR, 'build_tools', 'tests', 'test_all.py') + buildbot_common.Run([sys.executable, test_all_py]) + + +def BuildStepBuildExamples(pepperdir, platform): + build_sdk.BuildStepMakeAll(pepperdir, platform, 'examples', 'Build Examples') + + +def BuildStepCopyTests(pepperdir, toolchains, build_experimental, clobber): + buildbot_common.BuildStep('Copy Tests') + + build_sdk.MakeDirectoryOrClobber(pepperdir, 'testlibs', clobber) + build_sdk.MakeDirectoryOrClobber(pepperdir, 'tests', clobber) + + args = ['--dstroot=%s' % pepperdir, '--master'] + for toolchain in toolchains: + args.append('--' + toolchain) + + for library in TEST_LIBRARY_LIST: + dsc = os.path.join(SDK_LIBRARY_DIR, library, 'library.dsc') + args.append(dsc) + + for example in TEST_EXAMPLE_LIST: + dsc = os.path.join(SDK_LIBRARY_DIR, example, 'example.dsc') + args.append(dsc) + + if build_experimental: + args.append('--experimental') + + if generate_make.main(args): + buildbot_common.ErrorExit('Failed to build tests.') + + +def BuildStepBuildTests(pepperdir, platform): + build_sdk.BuildStepMakeAll(pepperdir, platform, 'testlibs', + 'Build Test Libraries') + build_sdk.BuildStepMakeAll(pepperdir, platform, 'tests', 'Build Tests') + + +def BuildStepRunPyautoTests(pepperdir, platform, pepper_ver): + buildbot_common.BuildStep('Test Examples') + env = copy.copy(os.environ) + env['PEPPER_VER'] = pepper_ver + env['NACL_SDK_ROOT'] = pepperdir + + pyauto_script = os.path.join(SRC_DIR, 'chrome', 'test', + 'functional', 'nacl_sdk.py') + pyauto_script_args = ['nacl_sdk.NaClSDKTest.NaClSDKExamples'] + + if platform == 'linux' and buildbot_common.IsSDKBuilder(): + # linux buildbots need to run the pyauto tests through xvfb. Running + # using runtest.py does this. + #env['PYTHON_PATH'] = '.:' + env.get('PYTHON_PATH', '.') + build_dir = os.path.dirname(SRC_DIR) + runtest_py = os.path.join(build_dir, '..', '..', '..', 'scripts', 'slave', + 'runtest.py') + buildbot_common.Run([sys.executable, runtest_py, '--target', 'Release', + '--build-dir', 'src/build', sys.executable, + pyauto_script] + pyauto_script_args, + cwd=build_dir, env=env) + else: + buildbot_common.Run([sys.executable, 'nacl_sdk.py', + 'nacl_sdk.NaClSDKTest.NaClSDKExamples'], + cwd=os.path.dirname(pyauto_script), + env=env) + + +def main(args): + parser = optparse.OptionParser() + parser.add_option('--experimental', help='build experimental tests', + action='store_true') + parser.add_option('--pyauto', help='Run pyauto tests', action='store_true') + + options, args = parser.parse_args(args[1:]) + + platform = getos.GetPlatform() + pepper_ver = str(int(build_utils.ChromeMajorVersion())) + pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) + toolchains = ['newlib', 'glibc', 'pnacl', 'host'] + + BuildStepBuildToolsTests() + BuildStepBuildExamples(pepperdir, platform) + BuildStepCopyTests(pepperdir, toolchains, options.experimental, True) + BuildStepBuildTests(pepperdir, platform) + if options.pyauto: + BuildStepRunPyautoTests(pepperdir, platform, pepper_ver) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) |