[NaCl SDK] Build and archive pnacl component multicrx as part of SDK build

On linux machines that run build_sdk.py with --archive
we now build and archive the pnacl component as a zip
file.  The shell script that generates this is derived
from one that has been used by jvoung to generate the
component locally.

CQ_EXTRA_TRYBOTS=tryserver.chromium.linux:linux_nacl_sdk;tryserver.chromium.mac:mac_nacl_sdk
TEST=./build_tools/build_sdk.py --tar --archive

Review URL: https://codereview.chromium.org/866723006

Cr-Commit-Position: refs/heads/master@{#324985}

3 files changed, 157 insertions, 37 deletions

diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py
index f424d3d..fc7fc3e 100755
--- a/native_client_sdk/src/build_tools/build_sdk.py
+++ b/native_client_sdk/src/build_tools/build_sdk.py
@@ -828,14 +828,21 @@ def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile,
   return bundle
 
 
+def Archive(filename, from_directory, step_link=True):
+  if buildbot_common.IsSDKBuilder():
+    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/'
+  else:
+    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk_test/'
+  bucket_path += build_version.ChromeVersion()
+  buildbot_common.Archive(filename, bucket_path, from_directory, step_link)
+
+
 def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
                            tarfile):
   buildbot_common.BuildStep('Archive %s' % name)
-  bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
-      build_version.ChromeVersion(),)
   tarname = os.path.basename(tarfile)
   tarfile_dir = os.path.dirname(tarfile)
-  buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
+  Archive(tarname, tarfile_dir)
 
   # generate "manifest snippet" for this archive.
   archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
@@ -847,24 +854,37 @@ def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
   with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
     manifest_snippet_stream.write(bundle.GetDataAsString())
 
-  buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
-                          step_link=False)
+  Archive(tarname + '.json', OUT_DIR, step_link=False)
+
+
+def BuildStepBuildPNaClComponent(version, revision):
+  # Sadly revision can go backwords for a given version since when a version
+  # is built from master, revision will be a huge number (in the hundreds of
+  # thousands.  Once the branch happens the revision will reset to zero.
+  # TODO(sbc): figure out how to compensate for this in some way such that
+  # revisions always go forward for a given version.
+  buildbot_common.BuildStep('PNaCl Component')
+  if len(revision) > 4:
+    rev_minor = revision[-4:]
+    rev_major = revision[:-4]
+    version = "0.%s.%s.%s" % (version, rev_major, rev_minor)
+  else:
+    version = "0.%s.0.%s" % (version, revision)
+  buildbot_common.Run(['./make_pnacl_component.sh', version], cwd=SCRIPT_DIR)
+
+
+def BuildStepArchivePNaClComponent():
+  buildbot_common.BuildStep('Archive PNaCl Component')
+  Archive('pnacl_multicrx.zip', OUT_DIR)
 
 
 def BuildStepArchiveSDKTools():
-  # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
-  builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
-  if builder_name == 'linux-sdk-multi':
-    buildbot_common.BuildStep('Build SDK Tools')
-    build_updater.BuildUpdater(OUT_DIR)
+  buildbot_common.BuildStep('Build SDK Tools')
+  build_updater.BuildUpdater(OUT_DIR)
 
-    buildbot_common.BuildStep('Archive SDK Tools')
-    bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
-        build_version.ChromeVersion(),)
-    buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
-                            step_link=False)
-    buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
-                            step_link=False)
+  buildbot_common.BuildStep('Archive SDK Tools')
+  Archive('sdk_tools.tgz', OUT_DIR, step_link=False)
+  Archive('nacl_sdk.zip', OUT_DIR, step_link=False)
 
 
 def BuildStepSyncNaClPorts():
@@ -1123,12 +1143,15 @@ def main(args):
   if options.tar:
     BuildStepTarBundle(pepper_ver, tarfile)
 
-  if options.build_ports and platform == 'linux':
-    ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
-    BuildStepSyncNaClPorts()
-    BuildStepBuildNaClPorts(pepper_ver, pepperdir)
-    if options.tar:
-      BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
+  if platform == 'linux':
+    BuildStepBuildPNaClComponent(pepper_ver, chrome_revision)
+
+    if options.build_ports:
+      ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
+      BuildStepSyncNaClPorts()
+      BuildStepBuildNaClPorts(pepper_ver, pepperdir)
+      if options.tar:
+        BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
 
   if options.build_app_engine and platform == 'linux':
     BuildStepBuildAppEngine(pepperdir, chrome_revision)
@@ -1137,14 +1160,17 @@ def main(args):
     qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted')
     oshelpers.Copy(['-r', qemudir, pepperdir])
 
-  # Archive on non-trybots.
+  # Archive the results on Google Cloud Storage.
   if options.archive:
     BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision,
                            tarfile)
-    if options.build_ports and platform == 'linux':
-      BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
-                             nacl_revision, ports_tarfile)
-    BuildStepArchiveSDKTools()
+    # Only archive sdk_tools/naclport/pnacl_component on linux.
+    if platform == 'linux':
+      if options.build_ports:
+        BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
+                               nacl_revision, ports_tarfile)
+      BuildStepArchiveSDKTools()
+      BuildStepArchivePNaClComponent()
 
   return 0
 
diff --git a/native_client_sdk/src/build_tools/buildbot_common.py b/native_client_sdk/src/build_tools/buildbot_common.py
index 37aace9..58ebc0a 100644
--- a/native_client_sdk/src/build_tools/buildbot_common.py
+++ b/native_client_sdk/src/build_tools/buildbot_common.py
@@ -37,15 +37,6 @@ def IsSDKBuilder():
   return '-sdk-multi' in bot or '-sdk-bionic-multi' in bot
 
 
-def IsSDKTrybot():
-  """Returns True if this script is running on an SDK trybot.
-
-  False means it is either running on an SDK builder, or a user's machine.
-
-  See IsSDKBuilder above for trybot/buildbot names."""
-  return '_nacl_sdk' in os.getenv('BUILDBOT_BUILDERNAME', '')
-
-
 def ErrorExit(msg):
   """Write and error to stderr, then exit with 1 signaling failure."""
   sys.stderr.write(str(msg) + '\n')
diff --git a/native_client_sdk/src/build_tools/make_pnacl_component.sh b/native_client_sdk/src/build_tools/make_pnacl_component.sh
new file mode 100755
index 0000000..42dc040
--- /dev/null
+++ b/native_client_sdk/src/build_tools/make_pnacl_component.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script builds out/pnacl_multicrx.zip for upload to the Chrome
+# Web Store. It runs gyp + ninja once for each architecture and assembles
+# the results along with a manifest file.
+
+# TODO(sbc): rewrite this in python
+
+set -o errexit
+set -o nounset
+
+SCRIPT_DIR="$(cd $(dirname $0) && pwd)"
+CHROME_SRC=$(dirname $(dirname $(dirname ${SCRIPT_DIR})))
+cd ${CHROME_SRC}
+
+run_gyp() {
+  # The original version of the script ran 'gclient runhooks' which run turn
+  # runs gyp_chromium. However its a lot faster and quieter to just run the
+  # gyp file containing the target we need.
+  gyp_dir=ppapi/native_client/src/untrusted/pnacl_support_extension
+  build/gyp_chromium --depth=. $gyp_dir/pnacl_support_extension.gyp
+}
+
+individual_packages() {
+  export GYP_GENERATOR_FLAGS="output_dir=out_pnacl"
+
+  # arm
+  rm -rf out_pnacl/
+  GYP_DEFINES="target_arch=arm" run_gyp
+  ninja -C out_pnacl/Release/ pnacl_support_extension
+  local target_dir=out/pnacl_arm
+  mkdir -p ${target_dir}
+  cp out_pnacl/Release/pnacl/* ${target_dir}/.
+
+  # ia32
+  rm -rf out_pnacl/
+  GYP_DEFINES="target_arch=ia32" run_gyp
+  ninja -C out_pnacl/Release/ pnacl_support_extension
+  target_dir=out/pnacl_x86_32
+  mkdir -p ${target_dir}
+  cp out_pnacl/Release/pnacl/* ${target_dir}/.
+
+  # x64
+  rm -rf out_pnacl/
+  GYP_DEFINES="target_arch=x64" run_gyp
+  ninja -C out_pnacl/Release/ pnacl_support_extension
+  target_dir=out/pnacl_x86_64
+  mkdir -p ${target_dir}
+  cp out_pnacl/Release/pnacl/* ${target_dir}/.
+}
+
+multi_crx() {
+  local version=$1
+  local target_dir=out/pnacl_multicrx
+  mkdir -p ${target_dir}
+  cat > ${target_dir}/manifest.json <<EOF
+{
+  "description": "Portable Native Client Translator Multi-CRX",
+  "name": "PNaCl Translator Multi-CRX",
+  "manifest_version": 2,
+  "minimum_chrome_version": "30.0.0.0",
+  "version": "${version}",
+  "platforms": [
+    {
+      "nacl_arch": "x86-32",
+      "sub_package_path": "_platform_specific/x86_32/"
+    },
+    {
+      "nacl_arch": "x86-64",
+      "sub_package_path": "_platform_specific/x86_64/"
+    },
+    {
+      "nacl_arch": "arm",
+      "sub_package_path": "_platform_specific/arm/"
+    }
+  ]
+}
+EOF
+
+  for arch in x86_32 x86_64 arm; do
+    local sub_dir="${target_dir}/_platform_specific/${arch}"
+    local src_dir="out/pnacl_${arch}"
+    mkdir -p ${sub_dir}
+    cp ${src_dir}/pnacl_public_* ${sub_dir}/.
+  done
+  (cd ${target_dir} && zip -r ../$(basename ${target_dir}).zip .)
+  ls -l ${target_dir}.zip
+  echo "DONE: created ${target_dir}.zip -- upload that!"
+  echo "You can also delete ${target_dir} later (the pre-zipped contents)."
+}
+
+if [ $# != 1 ]; then
+  echo "Usage: $0 <rev_number>"
+  exit 1
+fi
+
+version="$1"
+echo "Buidling pnacl_multicrx.zip version=$version"
+individual_packages
+multi_crx $version