Update SDK build scripts to green tree.

Replace multiple SDK scripts with updated buildbot_run.py
Remove Windows installer, use tgz/bz2 on all platforms.
Remove scons, replace all example build components with Make.
Add missing NMF files.
Add oshelper to make a platform independant mv, rm, cp, mkdir
This CL should only affect SDK bots, not Chrome.
Commiting TBR to green SDK tree.

BUG= 109207
TBR= bradnelson@chromium.org
Review URL: http://codereview.chromium.org/9090001

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@117813 0039d316-1c4b-4281-b951-d872f2087c98

16 files changed, 753 insertions, 76 deletions

diff --git a/native_client_sdk/src/build_tools/build_utils.py b/native_client_sdk/src/build_tools/build_utils.py
index c564590..a521a96 100644..100755
--- a/native_client_sdk/src/build_tools/build_utils.py
+++ b/native_client_sdk/src/build_tools/build_utils.py
@@ -16,8 +16,6 @@ import shutil
 import subprocess
 import sys
 
-from nacl_sdk_scons import nacl_utils
-
 
 # Reuse last change utility code.
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -115,32 +113,6 @@ def CheckPatchVersion(shell_env=None):
   return True
 
 
-# Build a toolchain path based on the platform type.  |base_dir| is the root
-# directory which includes the platform-specific toolchain.  This could be
-# something like "/usr/local/mydir/nacl_sdk/src".  If |base_dir| is None, then
-# the environment variable NACL_SDK_ROOT is used (if it's set).
-# This method assumes that the platform-specific toolchain is found under
-# <base_dir>/toolchain/<platform_variant>.
-def NormalizeToolchain(toolchain=TOOLCHAIN_AUTODETECT,
-                       base_dir=None,
-                       arch=nacl_utils.DEFAULT_TOOLCHAIN_ARCH,
-                       variant=nacl_utils.DEFAULT_TOOLCHAIN_VARIANT):
-  if toolchain == TOOLCHAIN_AUTODETECT:
-    if base_dir is None:
-      script_path = os.path.abspath(__file__)
-      script_dir = os.path.dirname(script_path)
-      sdk_src_dir = os.path.dirname(script_dir)
-      sdk_dir = os.path.dirname(sdk_src_dir)
-      src_dir = os.path.dirname(sdk_dir)
-      base_dir = os.path.join(src_dir, 'native_client')
-    normalized_toolchain = nacl_utils.ToolchainPath(base_dir=base_dir,
-                                                    arch=arch,
-                                                    variant=variant)
-  else:
-    normalized_toolchain = os.path.abspath(toolchain)
-  return normalized_toolchain
-
-
 def SupportedNexeBitWidths():
   '''Return a list of .nexe bit widths that are supported by the host.
 
diff --git a/native_client_sdk/src/build_tools/buildbot_run.py b/native_client_sdk/src/build_tools/buildbot_run.py
index 5c8ab8ef..ce94f07 100755
--- a/native_client_sdk/src/build_tools/buildbot_run.py
+++ b/native_client_sdk/src/build_tools/buildbot_run.py
@@ -5,85 +5,374 @@
 
 '''Entry point for both build and try bots'''
 
+import build_utils
+import lastchange
 import os
 import subprocess
 import sys
 
-# Add scons to the python path (as nacl_utils.py requires it).
+# Create the various paths of interest
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
 SDK_DIR = os.path.dirname(SDK_SRC_DIR)
 SRC_DIR = os.path.dirname(SDK_DIR)
-sys.path.append(os.path.join(SRC_DIR, 'third_party/scons-2.0.1/engine'))
+NACL_DIR = os.path.join(SRC_DIR, 'native_client')
+OUT_DIR = os.path.join(SRC_DIR, 'out')
+PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
 
-import build_utils
 
+# Add SDK make tools scripts to the python path.
+sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
+sys.path.append(os.path.join(NACL_DIR, 'build'))
 
-def Archive(revision, chrome_version):
-  """Archive the sdk to google storage.
 
-  Args:
-    revision: SDK svn revision number.
-    chrome_version: Chrome version number / trunk svn.
-  """
-  if sys.platform in ['cygwin', 'win32']:
-    src = 'nacl-sdk.exe'
-    dst = 'naclsdk_win.exe'
-  elif sys.platform in ['darwin']:
-    src = 'nacl-sdk.tgz'
-    dst = 'naclsdk_mac.tgz'
-  else:
-    src = 'nacl-sdk.tgz'
-    dst = 'naclsdk_linux.tgz'
+import http_download
+from getos import GetPlatform
+import oshelpers
+
+GSTORE = 'http://commondatastorage.googleapis.com/nativeclient-mirror/nacl/'
+MAKE = 'nacl_sdk/make_3_81/make.exe'
+GSUTIL = '/b/build/scripts/slave/gsutil'
+
+def ErrorExit(msg):
+  """Write and error to stderr, then exit with 1 signaling failure."""
+  sys.stderr.write(msg + '\n')
+  sys.exit(1)
+
+
+def BuildStep(name):
+  """Annotate a buildbot build step."""
+  sys.stdout.flush()
+  print '\n@@@BUILD_STEP %s@@@' % name
+  sys.stdout.flush()
+
+
+def Run(args, cwd=None, shell=False):
+  """Start a process with the provided arguments.
+  
+  Starts a process in the provided directory given the provided arguments. If
+  shell is not False, the process is launched via the shell to provide shell
+  interpretation of the arguments.  Shell behavior can differ between platforms
+  so this should be avoided when not using platform dependent shell scripts."""
+  print 'Running: ' + ' '.join(args)
+  sys.stdout.flush()
+  subprocess.check_call(args, cwd=cwd, shell=shell)
+  sys.stdout.flush()
+
+
+def Archive(filename):
+  """Upload the given filename to Google Store."""
+  chrome_version = build_utils.ChromeVersion()
   bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s/%s' % (
-      chrome_version, dst)
+      chrome_version, filename)
   full_dst = 'gs://%s' % bucket_path
   subprocess.check_call(
-      '/b/build/scripts/slave/gsutil cp -a public-read %s %s' % (
-          src, full_dst), shell=True, cwd=SDK_DIR)
+      'gsutil cp -a public-read %s %s' % (
+          filename, full_dst), shell=True, cwd=OUT_DIR)
   url = 'https://commondatastorage.googleapis.com/%s' % bucket_path
   print '@@@STEP_LINK@download@%s@@@' % url
   sys.stdout.flush()
 
 
-def main(argv):
+def AddMakeBat(makepath):
+  """Create a simple batch file to execute Make.
+  
+  Creates a simple batch file named make.bat for the Windows platform at the
+  given path, pointing to the Make executable in the SDK.""" 
+  fp = open(os.path.join(makepath, 'make.bat'), 'wb')
+  fp.write('@..\\..\\tools\\make.exe %*\n')
+  fp.close()
+
+
+def CopyDir(src, dst, excludes=['.svn']):
+  """Recursively copy a directory using."""
+  args = ['-r', src, dst]
+  for exc in excludes:
+    args.append('--exclude=' + exc)
+  print "cp -r %s %s" % (src, dst)
+  oshelpers.Copy(args)
+
+def RemoveDir(dst):
+  """Remove the provided path."""
+  print "rm -fr " + dst
+  oshelpers.Remove(['-fr', dst])
+
+def MakeDir(dst):
+  """Create the path including all parent directories as needed."""
+  print "mkdir -p " + dst
+  oshelpers.Mkdir(['-p', dst])
+
+def MoveDir(src, dst):
+  """Move the path src to dst."""
+  print "mv -fr %s %s" % (src, dst)
+  oshelpers.Move(['-f', src, dst])
+
+def BuildOutputDir(*paths):
+  return os.path.join(OUT_DIR, *paths)
+
+
+def GetGlibcToolchain(platform, arch):
+  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
+  tcname = 'toolchain_%s_%s.tar.bz2' % (platform, arch)
+  return os.path.join(tcdir, tcname)
+
+
+def GetNewlibToolchain(platform, arch):
+  tcdir = os.path.join(NACL_DIR, 'toolchain', '.tars')
+  tcname = 'naclsdk_%s_%s.tgz' % (platform, arch)
+  return os.path.join(tcdir, tcname)
+
+
+def GetScons():
   if sys.platform in ['cygwin', 'win32']:
-    # Windows build
-    command = 'scons.bat'
+    return 'scons.bat'
+  return './scons'
+
+
+def GetArchName(arch, xarch=None):
+  if xarch:
+    return arch + '-' + str(xarch)
+  return arch
+
+
+def GetToolchainNaClInclude(tcpath, arch, xarch=None):
+  if arch == 'x86':
+    return os.path.join(tcpath, 'x86_64-nacl', 'include')
   else:
-    # Linux and Mac build
-    command = 'scons'
+    ErrorExit('Unknown architecture.')
 
-  parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-  params = [os.path.join(parent_dir, command)] + argv
-  def Run(parameters):
-    print '\nRunning ', parameters
-    sys.stdout.flush()
-    subprocess.check_call(' '.join(parameters), shell=True, cwd=parent_dir)
 
-  print '@@@BUILD_STEP install third party@@@'
-  sys.stdout.flush()
-  subprocess.check_call(' '.join([
-      sys.executable,
-     'build_tools/install_third_party.py',
-     '--all-toolchains',
-     ]), shell=True, cwd=parent_dir)
+def GetToolchainNaClLib(tcpath, arch, xarch):
+  if arch == 'x86':
+    if str(xarch) == '32':
+      return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
+    if str(xarch) == '64':
+      return os.path.join(tcpath, 'x86_64-nacl', 'lib')
+  ErrorExit('Unknown architecture.')
 
-  print '@@@BUILD_STEP generate sdk@@@'
-  sys.stdout.flush()
 
-  Run(params + ['-c'])
-  Run(params + ['bot', '-j1'])
+def GetBuildArgs(tcname, tcpath, arch, xarch=None):
+  """Return list of scons build arguments to generate user libraries."""
+  scons = GetScons()
+  mode = '--mode=opt-host,nacl'
+  arch_name = GetArchName(arch, xarch)
+  plat = 'platform=' + arch_name
+  bin = ('bindir=' +
+         BuildOutputDir('pepper_' + build_utils.ChromeMajorVersion(), 'tools'))
+  lib = ('libdir=' +
+         os.path.join(GetToolchainNaClLib(tcpath, arch, xarch)))
+  args = [scons, mode, plat, bin, lib, '-j10',
+          'install_bin', 'install_lib']
+  if tcname == 'glibc':
+    args.append('--nacl_glibc')
+  return args
+
+header_map = {
+  'newlib': {
+      'pthread.h': 'src/untrusted/pthread/pthread.h',
+      'semaphore.h': 'src/untrusted/pthread/semaphore.h',
+      'dynamic_annotations.h': 'src/untrusted/valgrind/dynamic_annotations.h',
+      'dynamic_annotations.h': 'src/untrusted/valgrind/dynamic_annotations.h',
+      'nacl_dyncode.h': ' src/untrusted/nacl/nacl_dyncode.h',
+      'nacl_startup.h': 'src/untrusted/nacl/nacl_startup.h',
+      'nacl_thread.h': 'src/untrusted/nacl/nacl_thread.h',
+      'pnacl.h': 'src/untrusted/nacl/pnacl.h',
+      'irt.h': 'src/untrusted/irt/irt.h',
+      'irt_ppapi.h': 'src/untrusted/irt/irt_ppapi.h',
+  },
+  'glibc': {
+      'dynamic_annotations.h': 'src/untrusted/valgrind/dynamic_annotations.h',
+      'dynamic_annotations.h': 'src/untrusted/valgrind/dynamic_annotations.h',
+      'nacl_dyncode.h': ' src/untrusted/nacl/nacl_dyncode.h',
+      'nacl_startup.h': 'src/untrusted/nacl/nacl_startup.h',
+      'nacl_thread.h': 'src/untrusted/nacl/nacl_thread.h',
+      'pnacl.h': 'src/untrusted/nacl/pnacl.h',
+      'irt.h': 'src/untrusted/irt/irt.h',
+      'irt_ppapi.h': 'src/untrusted/irt/irt_ppapi.h',
+  },
+}
+
+
+def InstallHeaders(tc_dst_inc, pepper_ver, tc_name):
+  """Copies NaCl headers to expected locations in the toolchain."""
+  tc_map = header_map[tc_name]
+  for filename in tc_map:
+    src = os.path.join(NACL_DIR, tc_map[filename])
+    dst = os.path.join(tc_dst_inc, filename)
+    oshelpers.Copy(['-v', src, dst])
+
+  # Clean out per toolchain ppapi directory
+  ppapi = os.path.join(tc_dst_inc, 'ppapi')
+  RemoveDir(ppapi)
+
+  # Copy in c and c/dev headers
+  MakeDir(os.path.join(ppapi, 'c', 'dev'))
+  CopyDir(os.path.join(PPAPI_DIR, 'c', '*.h'),
+          os.path.join(ppapi, 'c'))
+  CopyDir(os.path.join(PPAPI_DIR, 'c', 'dev', '*.h'),
+          os.path.join(ppapi, 'c', 'dev'))
+
+  # Run the generator to overwrite IDL files  
+  Run([sys.executable, 'generator.py', '--wnone', '--cgen',
+       '--release=M' + pepper_ver, '--verbose', '--dstroot=%s/c' % ppapi],
+       cwd=os.path.join(PPAPI_DIR, 'generators'))
+
+  # Remove private and trusted interfaces
+  RemoveDir(os.path.join(ppapi, 'c', 'private'))
+  RemoveDir(os.path.join(ppapi, 'c', 'trusted'))
+
+  # Copy in the C++ headers
+  MakeDir(os.path.join(ppapi, 'cpp', 'dev'))
+  CopyDir(os.path.join(PPAPI_DIR, 'cpp','*.h'),
+          os.path.join(ppapi, 'cpp'))
+  CopyDir(os.path.join(PPAPI_DIR, 'cpp', 'dev', '*.h'),
+          os.path.join(ppapi, 'cpp', 'dev'))
+  MakeDir(os.path.join(ppapi, 'utility', 'graphics'))
+  CopyDir(os.path.join(PPAPI_DIR, 'utility','*.h'),
+          os.path.join(ppapi, 'utility'))
+  CopyDir(os.path.join(PPAPI_DIR, 'utility', 'graphics', '*.h'),
+          os.path.join(ppapi, 'utility', 'graphics'))
+
+  # Copy in the gles2 headers
+  MakeDir(os.path.join(ppapi, 'gles2'))
+  CopyDir(os.path.join(PPAPI_DIR,'lib','gl','gles2','*.h'),
+          os.path.join(ppapi, 'gles2'))
+
+  # Copy the EGL headers
+  MakeDir(os.path.join(tc_dst_inc, 'EGL'))
+  CopyDir(os.path.join(PPAPI_DIR,'lib','gl','include','EGL', '*.h'),
+          os.path.join(tc_dst_inc, 'EGL'))
+
+  # Copy the GLES2 headers
+  MakeDir(os.path.join(tc_dst_inc, 'GLES2'))
+  CopyDir(os.path.join(PPAPI_DIR,'lib','gl','include','GLES2', '*.h'),
+          os.path.join(tc_dst_inc, 'GLES2'))
+
+  # Copy the KHR headers
+  MakeDir(os.path.join(tc_dst_inc, 'KHR'))
+  CopyDir(os.path.join(PPAPI_DIR,'lib','gl','include','KHR', '*.h'),
+          os.path.join(tc_dst_inc, 'KHR'))
+
+
+def main():
+  platform = GetPlatform()
+  arch = 'x86'
+  skip_untar = 0
+  skip_build = 0
+  skip_tar = 0
+  skip_examples = 0
+  skip_headers = 0
+  skip_make = 0
+
+  pepper_ver = build_utils.ChromeMajorVersion()
+  clnumber = lastchange.FetchVersionInfo(None).revision
+  print 'Building PEPPER %s at %s' % (pepper_ver, clnumber)
+
+  BuildStep('Clean Pepper Dir')
+  pepperdir = os.path.join(SRC_DIR, 'out', 'pepper_' + pepper_ver)
+  if not skip_untar:
+    RemoveDir(pepperdir)
+    MakeDir(os.path.join(pepperdir, 'toolchain'))
+    MakeDir(os.path.join(pepperdir, 'tools'))
+
+  BuildStep('Untar Toolchains')
+  tmpdir = os.path.join(SRC_DIR, 'out', 'tc_temp')
+  tcname = platform + '_' + arch
+  tmpdir = os.path.join(SRC_DIR, 'out', 'tc_temp')
+  cygtar = os.path.join(NACL_DIR, 'build', 'cygtar.py')
+
+    # Clean out the temporary toolchain untar directory
+  if not skip_untar:
+    RemoveDir(tmpdir)
+    MakeDir(tmpdir)
+    tcname = platform + '_' + arch
+
+    # Untar the newlib toolchains
+    tarfile = GetNewlibToolchain(platform, arch)
+    Run([sys.executable, cygtar, '-C', tmpdir, '-xf', tarfile], cwd=NACL_DIR)
+
+    # Then rename/move it to the pepper toolchain directory
+    srcdir = os.path.join(tmpdir, 'sdk', 'nacl-sdk')
+    newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
+    print "BUildbot mv %s to %s" % (srcdir, newlibdir)
+    MoveDir(srcdir, newlibdir)
+    print "Done with buildbot move"
+    # Untar the glibc toolchains
+    tarfile = GetGlibcToolchain(platform, arch)
+    Run([sys.executable, cygtar, '-C', tmpdir, '-xf', tarfile], cwd=NACL_DIR)
+
+    # Then rename/move it to the pepper toolchain directory
+    srcdir = os.path.join(tmpdir, 'toolchain', tcname)
+    glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
+    MoveDir(srcdir, glibcdir)
+  else:
+    newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
+    glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
+
+  BuildStep('SDK Items')
+  if not skip_build:
+    if arch == 'x86':
+      Run(GetBuildArgs('newlib', newlibdir, 'x86', '32'), cwd=NACL_DIR)
+      Run(GetBuildArgs('newlib', newlibdir, 'x86', '64'), cwd=NACL_DIR)
+
+      Run(GetBuildArgs('glibc', glibcdir, 'x86', '32'), cwd=NACL_DIR)
+      Run(GetBuildArgs('glibc', glibcdir, 'x86', '64'), cwd=NACL_DIR)
+    else:
+      ErrorExit('Missing arch %s' % arch)
+
+  if not skip_headers:
+    BuildStep('Copy Toolchain headers')
+    if arch == 'x86':
+      InstallHeaders(GetToolchainNaClInclude(newlibdir, 'x86'),
+                     pepper_ver, 
+                     'newlib')
+      InstallHeaders(GetToolchainNaClInclude(newlibdir, 'x86'),
+                     pepper_ver,
+                     'glibc')
+    else:
+      ErrorExit('Missing arch %s' % arch)
+
+  BuildStep('Copy make helpers')
+  CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
+        os.path.join(pepperdir, 'tools'))
+  if platform == 'win':
+    BuildStep('Add MAKE')
+    http_download.HttpDownload(GSTORE + MAKE,
+                               os.path.join(pepperdir, 'tools' ,'make.exe'))
+
+  if not skip_examples:
+    BuildStep('Copy examples')
+    RemoveDir(os.path.join(pepperdir, 'examples'))
+    CopyDir(os.path.join(SDK_SRC_DIR, 'examples'), pepperdir)
+
+
+  tarname = 'naclsdk_' + platform + '.bz2'
+  BuildStep('Tar Pepper Bundle')
+  if not skip_tar:
+    tarfile = os.path.join(OUT_DIR, 'naclsdk_' + platform + '.bz2')
+    Run([sys.executable, cygtar, '-C', OUT_DIR, '-cjf', tarfile,
+         'pepper_' + pepper_ver], cwd=NACL_DIR)
 
   # Archive on non-trybots.
   if '-sdk' in os.environ.get('BUILDBOT_BUILDERNAME', ''):
-    print '@@@BUILD_STEP archive build@@@'
-    sys.stdout.flush()
-    Archive(revision=os.environ.get('BUILDBOT_GOT_REVISION'),
-            chrome_version=build_utils.ChromeVersion())
+    BuildStep('Archive build')
+    Archive(tarname)
+
+  if not skip_make:
+    BuildStep('Test Build Examples')
+    filelist = os.listdir(os.path.join(pepperdir, 'examples'))
+    for filenode in filelist:
+      dirnode = os.path.join(pepperdir, 'examples', filenode)
+      makefile = os.path.join(dirnode, 'Makefile')
+      if os.path.isfile(makefile):
+        if platform == 'win':
+          AddMakeBat(dirnode)
+        print "\n\nMake: " + dirnode
+        Run(['make', 'all', '-j8'], cwd=os.path.abspath(dirnode), shell=True)
 
   return 0
 
 
 if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
+  sys.exit(main())
+
diff --git a/native_client_sdk/src/examples/fullscreen_tumbler/fullscreen_tumbler.nmf b/native_client_sdk/src/examples/fullscreen_tumbler/fullscreen_tumbler.nmf
new file mode 100644
index 0000000..cfd0e9b
--- /dev/null
+++ b/native_client_sdk/src/examples/fullscreen_tumbler/fullscreen_tumbler.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "fullscreen_tumbler_x86_64.nexe"},
+    "x86-32": {"url": "fullscreen_tumbler_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/geturl/geturl.nmf b/native_client_sdk/src/examples/geturl/geturl.nmf
new file mode 100644
index 0000000..936e301
--- /dev/null
+++ b/native_client_sdk/src/examples/geturl/geturl.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "geturl_x86_64.nexe"},
+    "x86-32": {"url": "geturl_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/hello_world/hello_world.nmf b/native_client_sdk/src/examples/hello_world/hello_world.nmf
new file mode 100644
index 0000000..78df8e1
--- /dev/null
+++ b/native_client_sdk/src/examples/hello_world/hello_world.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "hello_world_x86_64.nexe"},
+    "x86-32": {"url": "hello_world_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/hello_world_c/hello_world_c.nmf b/native_client_sdk/src/examples/hello_world_c/hello_world_c.nmf
new file mode 100644
index 0000000..377d01e
--- /dev/null
+++ b/native_client_sdk/src/examples/hello_world_c/hello_world_c.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "hello_world_c_x86_64.nexe"},
+    "x86-32": {"url": "hello_world_c_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/input_events/input_events.nmf b/native_client_sdk/src/examples/input_events/input_events.nmf
new file mode 100644
index 0000000..ab4392f
--- /dev/null
+++ b/native_client_sdk/src/examples/input_events/input_events.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "input_events_x86_64.nexe"},
+    "x86-32": {"url": "input_events_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/load_progress/load_progress.nmf b/native_client_sdk/src/examples/load_progress/load_progress.nmf
new file mode 100644
index 0000000..4018ead
--- /dev/null
+++ b/native_client_sdk/src/examples/load_progress/load_progress.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "load_progress_x86_64.nexe"},
+    "x86-32": {"url": "load_progress_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/mouselock/mouselock.nmf b/native_client_sdk/src/examples/mouselock/mouselock.nmf
new file mode 100644
index 0000000..8c69546
--- /dev/null
+++ b/native_client_sdk/src/examples/mouselock/mouselock.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "mouselock_x86_64.nexe"},
+    "x86-32": {"url": "mouselock_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/multithreaded_input_events/mt_input_events.nmf b/native_client_sdk/src/examples/multithreaded_input_events/mt_input_events.nmf
new file mode 100644
index 0000000..513ca9e
--- /dev/null
+++ b/native_client_sdk/src/examples/multithreaded_input_events/mt_input_events.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "multithreaded_input_events_x86_64.nexe"},
+    "x86-32": {"url": "multithreaded_input_events_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/pi_generator/pi_generator.nmf b/native_client_sdk/src/examples/pi_generator/pi_generator.nmf
new file mode 100644
index 0000000..62d8d39
--- /dev/null
+++ b/native_client_sdk/src/examples/pi_generator/pi_generator.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "pi_generator_x86_64.nexe"},
+    "x86-32": {"url": "pi_generator_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/pong/pong.nmf b/native_client_sdk/src/examples/pong/pong.nmf
new file mode 100644
index 0000000..fd250d2
--- /dev/null
+++ b/native_client_sdk/src/examples/pong/pong.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "pong_x86_64.nexe"},
+    "x86-32": {"url": "pong_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/sine_synth/sine_synth.nmf b/native_client_sdk/src/examples/sine_synth/sine_synth.nmf
new file mode 100644
index 0000000..f778455
--- /dev/null
+++ b/native_client_sdk/src/examples/sine_synth/sine_synth.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "sine_synth_x86_64.nexe"},
+    "x86-32": {"url": "sine_synth_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/examples/tumbler/tumbler.nmf b/native_client_sdk/src/examples/tumbler/tumbler.nmf
new file mode 100644
index 0000000..0ac6302
--- /dev/null
+++ b/native_client_sdk/src/examples/tumbler/tumbler.nmf
@@ -0,0 +1,6 @@
+{
+  "program": {
+    "x86-64": {"url": "tumbler_x86_64.nexe"},
+    "x86-32": {"url": "tumbler_x86_32.nexe"}
+  }
+}
diff --git a/native_client_sdk/src/tools/getos.py b/native_client_sdk/src/tools/getos.py
new file mode 100644
index 0000000..f41bb47
--- /dev/null
+++ b/native_client_sdk/src/tools/getos.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Native Client Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Determine OS
+
+Determine the name of the platform used to determine the correct Toolchain to
+invoke.
+"""
+
+import sys
+
+def GetPlatform():
+  if sys.platform.startswith('cygwin') or sys.platform.startswith('win'):
+    return 'win'
+
+  if sys.platform.startswith('darwin'):
+    return 'mac'
+
+  if sys.platform.startswith('linux'):
+    return 'linux'
+  return None
+
+if __name__ == '__main__':
+  platform = GetPlatform()
+  if platform is None:
+    print 'Unknown platform.'
+    sys.exit(1)
+
+  print platform
+  sys.exit(0)
+
diff --git a/native_client_sdk/src/tools/oshelpers.py b/native_client_sdk/src/tools/oshelpers.py
new file mode 100644
index 0000000..c286326
--- /dev/null
+++ b/native_client_sdk/src/tools/oshelpers.py
@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Native Client Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import fnmatch
+import glob
+import optparse
+import os
+import shutil
+import sys
+import time
+
+
+def IncludeFiles(filters, files):
+  """Filter files based on inclusion lists
+  
+  Return a list of files which match and of the Unix shell-style wildcards
+  provided, or return all the files if no filter is provided."""
+  if not filters: 
+    return files
+  match = set()
+  for filter in filters:
+    match |= set(fnmatch.filter(files, filter))
+  return [name for name in files if name in match]
+
+
+def ExcludeFiles(filters, files):
+  """Filter files based on exclusions lists
+  
+  Return a list of files which do not match any of the Unix shell-style
+  wildcards provided, or return all the files if no filter is provided."""
+  if not filters: 
+    return files
+  match = set()
+  for filter in filters:
+    excludes = set(fnmatch.filter(files, filter))
+    match |= excludes
+  return [name for name in files if name not in match]
+
+
+def CopyPath(options, src, dst):
+  """CopyPath from src to dst
+  
+  Copy a fully specified src to a fully specified dst.  If src and dst are
+  both files, the dst file is removed first to prevent error.  If and include
+  or exclude list are provided, the destination is first matched against that
+  filter."""
+  if options.includes:
+    if not IncludeFiles(options.includes, [src]):
+      return
+
+  if options.excludes:
+    if not ExcludeFiles(options.excludes, [src]):
+      return
+
+  if options.verbose:
+    print 'cp %s %s' % (src, dst)
+
+  # If the source is a single file, copy it individually
+  if os.path.isfile(src):
+    # We can not copy over a directory with a file.
+    if os.path.exists(dst):
+      if not os.path.isfile(dst):
+        msg = "cp: cannot overwrite non-file '%s' with file." % dst
+        raise OSError(msg)
+      # If the destination exists as a file, remove it before copying to avoid
+      # 'readonly' issues.
+      os.remove(dst)
+
+    # Now copy to the non-existent fully qualified target
+    shutil.copyfile(src, dst)
+    return
+
+  # Otherwise it's a directory, ignore it unless allowed
+  if os.path.isdir(src):
+    if not options.recursive:
+      print "cp: omitting directory '%s'" % src
+      return
+
+    # We can not copy over a file with a directory.
+    if os.path.exists(dst):
+      if not os.path.isdir(dst):
+        msg = "cp: cannot overwrite non-directory '%s' with directory." % dst
+        raise OSError(msg)
+    else:
+      # if it didn't exist, create the directory
+      os.makedirs(dst)
+
+    # Now copy all members
+    for filename in os.listdir(src):
+      srcfile = os.path.join(src, filename)
+      dstfile = os.path.join(dst, filename)
+      CopyPath(options, srcfile, dstfile)
+  return
+
+
+def Copy(args):
+  """A Unix cp style copy.
+  
+  Copies multiple sources to a single destination using the normal cp 
+  semantics.  In addition, it support inclusion and exclusion filters which
+  allows the copy to skip certain types of files."""
+  parser = optparse.OptionParser(usage='usage: cp [Options] souces... dest')
+  parser.add_option(
+      '-R', '-r', '--recursive', dest='recursive', action='store_true',
+      default=False,
+      help='copy directories recursively.')
+  parser.add_option(
+      '-v', '--verbose', dest='verbose', action='store_true',
+      default=False,
+      help='verbose output.')
+  parser.add_option(
+      '--include', dest='includes', action='append', default=[],
+      help='include files matching this expression.')
+  parser.add_option(
+      '--exclude', dest='excludes', action='append', default=[],
+      help='exclude files matching this expression.')
+  options, files = parser.parse_args(args)
+  if len(files) < 2:
+    parser.error('ERROR: expecting SOURCE(s) and DEST.')
+
+  srcs = files[:-1]
+  dst = files[-1]
+
+  src_list = []
+  for src in srcs:
+    files = glob.glob(src)
+    if files:
+      src_list.extend(files)
+
+  for src in src_list:
+    if options.verbose:
+      print 'cp %s %s' % (src, dst)
+    # If the destination is a directory, then append the basename of the src
+    # to the destination.
+    if os.path.isdir(dst):
+      CopyPath(options, src, os.path.join(dst, os.path.basename(src)))
+    else:
+      CopyPath(options, src, dst)
+
+
+def Mkdir(args):
+  """A Unix style mkdir"""
+  parser = optparse.OptionParser(usage='usage: mkdir [Options] DIRECTORY...')
+  parser.add_option(
+      '-p', '--parents', dest='parents', action='store_true',
+      default=False,
+      help='ignore existing parents, create parents as needed.')
+  parser.add_option(
+      '-v', '--verbose', dest='verbose', action='store_true',
+      default=False,
+      help='verbose output.')
+
+  options, dsts = parser.parse_args(args)
+  if len(dsts) < 1:
+    parser.error('ERROR: expecting DIRECTORY...')
+
+  for dst in dsts:
+    if options.verbose:
+      print 'mkdir ' + dst
+    try:
+      os.makedirs(dst)
+    except OSError as error:
+      if os.path.isdir(dst):
+        if options.parents:
+          continue
+        raise OSError('mkdir: Already exsists: ' + dst)
+      else:
+        raise OSError('mkdir: Failed to create: ' + dst)
+  return 0
+
+
+def MovePath(options, src, dst):
+  """MovePath from src to dst
+  
+  Moves the src to the dst much like the Unix style mv command, except it
+  only handles one source at a time.  Because of possible temporary failures
+  do to locks (such as anti-virus software on Windows), the function will retry
+  up to five times.""" 
+  # if the destination is not an existing directory, then overwrite it
+  if os.path.isdir(dst):
+    dst = os.path.join(dst, os.path.basename(src))
+
+  # If the destination exists, the remove it
+  if os.path.exists(dst):
+    if options.force:
+      Remove(['-vfr', dst])
+      if os.path.exists(dst):
+        raise OSError('mv: FAILED TO REMOVE ' + dst)
+    else:
+      raise OSError('mv: already exists ' + dst)
+  for i in range(5):
+    try:
+      os.rename(src, dst)
+      return
+    except OSError as error:
+      print 'Failed on %s with %s, retrying' % (src, error)
+      time.sleep(5)
+  print 'Gave up.'
+  raise OSError('mv: ' + error)
+
+
+def Move(args):
+  parser = optparse.OptionParser(usage='usage: mv [Options] souces... dest')
+  parser.add_option(
+      '-v', '--verbose', dest='verbose', action='store_true',
+      default=False,
+      help='verbose output.')
+  parser.add_option(
+      '-f', '--force', dest='force', action='store_true',
+      default=False,
+      help='force, do not error it files already exist.')
+  options, files = parser.parse_args(args)
+  if len(files) < 2:
+    parser.error('ERROR: expecting SOURCE... and DEST.')
+  if options.verbose:
+    print 'mv %s %s' % (src, dst)
+
+  srcs = files[:-1]
+  dst = files[-1]
+
+  for src in srcs:
+    MovePath(options, src, dst)
+  return 0
+
+
+def Remove(args):
+  """A Unix style rm.
+  
+  Removes the list of paths.  Because of possible temporary failures do to locks
+  (such as anti-virus software on Windows), the function will retry up to five
+  times.""" 
+  parser = optparse.OptionParser(usage='usage: rm [Options] PATHS...')
+  parser.add_option(
+      '-R', '-r', '--recursive', dest='recursive', action='store_true',
+      default=False,
+      help='remove directories recursively.')
+  parser.add_option(
+      '-v', '--verbose', dest='verbose', action='store_true',
+      default=False,
+      help='verbose output.')
+  parser.add_option(
+      '-f', '--force', dest='force', action='store_true',
+      default=False,
+      help='force, do not error it files does not exist.')
+  options, files = parser.parse_args(args)
+  if len(files) < 1:
+    parser.error('ERROR: expecting FILE...')
+
+  try:
+    for dst in files:
+      if options.verbose:
+        print 'rm ' + dst
+
+      # Ignore non existing files when using force
+      if not os.path.exists(dst) and options.force:
+        print "rm: Skipping " + dst
+        continue
+
+      if os.path.isfile(dst) or os.path.islink(dst):
+        for i in range(5):
+          try:
+            # Check every time, since it may have been deleted after the
+            # previsou failed attempt.
+            if os.path.isfile(dst) or os.path.islink(dst):
+              os.remove(dst)
+            break
+          except OSError as error:
+            if i == 5:
+              print 'Gave up.'
+              raise OSError('rm: ' + str(error))
+            print 'Failed remove with %s, retrying' % error
+            time.sleep(5)
+
+      if options.recursive:
+        for i in range(5):
+          try:
+            if os.path.isdir(dst):
+              shutil.rmtree(dst)
+            break
+          except OSError as error:
+            if i == 5:
+              print 'Gave up.'
+              raise OSError('rm: ' + str(error))
+            print 'Failed rmtree with %s, retrying' % error
+            time.sleep(5)
+
+      print 'rm: %s is a directory.' % dst
+      return 1
+
+  except OSError as error:
+    print error
+  return 0
+
+
+FuncMap = {
+  'cp': Copy,
+  'mkdir': Mkdir,
+  'mv': Move,
+  'rm': Remove,
+}
+
+
+if __name__ == '__main__':
+  func = FuncMap.get(sys.argv[1])
+  if not func:
+    print 'Do not recognize: ' + sys.argv[1]
+    sys.exit(1)
+  sys.exit(func(sys.argv[2:]))