Remove all cyclic imports

Clean up parameter sufixes to be consistant in generate_make and easy_template.
   <x>str for cStringIO
   <x>file for File object
   <x>path for Path string

Move generate_make.RunTemplate to easy_template.RunTemplateIfChanged

Break out desc parsing and selection to a parse_desc
Break out build_sdk directories to build_paths
Move build_sdk example build code to build_examples
Move build_sdk GetWindowsEnvironment to build_common

Rename build_examples to build_projects since it builds more than examples.

Remove stale references to pyauto and makefile_gyp

BUG=232172

Review URL: https://chromiumcodereview.appspot.com/13993016

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@195597 0039d316-1c4b-4281-b951-d872f2087c98

19 files changed, 727 insertions, 675 deletions

diff --git a/native_client_sdk/src/build_tools/build_examples.py b/native_client_sdk/src/build_tools/build_examples.py
deleted file mode 100755
index 0e98521..0000000
--- a/native_client_sdk/src/build_tools/build_examples.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import optparse
-import os
-import sys
-
-import build_sdk
-import build_utils
-import test_sdk
-
-sys.path.append(os.path.join(build_sdk.SDK_SRC_DIR, 'tools'))
-import getos
-
-
-def main(args):
-  parser = optparse.OptionParser()
-  parser.add_option('--copy',
-      help='Only copy the files, don\'t build.',
-      action='store_true' )
-  parser.add_option('--clobber-examples',
-      help='Don\'t examples directory before copying new files',
-      action='store_true' )
-  parser.add_option('--test-examples',
-      help='Run the pyauto tests for examples.', action='store_true')
-  parser.add_option('--experimental',
-      help='build experimental examples and libraries', action='store_true')
-  parser.add_option('-t', '--toolchain',
-      help='Build using toolchain. Can be passed more than once.',
-      action='append')
-  parser.add_option('--gyp',
-      help='Use gyp to build examples/libraries/Makefiles.',
-      action='store_true')
-
-  options, args = parser.parse_args(args[1:])
-
-  valid_toolchains = ['newlib', 'glibc', 'pnacl', 'host']
-  if not options.toolchain:
-    toolchains = valid_toolchains
-  else:
-    invalid_toolchains = set(options.toolchain) - set(valid_toolchains)
-    if invalid_toolchains:
-      print 'Ignoring invalid toolchains: %s' % (', '.join(invalid_toolchains),)
-    toolchains = list(set(options.toolchain) - invalid_toolchains)
-
-  pepper_ver = str(int(build_utils.ChromeMajorVersion()))
-  pepperdir = os.path.join(build_sdk.OUT_DIR, 'pepper_' + pepper_ver)
-  platform = getos.GetPlatform()
-
-  build_sdk.options = options
-
-  build_sdk.BuildStepCopyBuildHelpers(pepperdir, platform)
-  build_sdk.BuildStepCopyExamples(pepperdir, toolchains, options.experimental,
-                                  options.clobber_examples)
-  test_sdk.BuildStepCopyTests(pepperdir, toolchains, options.experimental,
-                              options.clobber_examples)
-  if options.copy:
-    return 0
-
-  # False = don't clean after building the libraries directory.
-  build_sdk.BuildStepBuildLibraries(pepperdir, platform, 'src', False)
-  test_sdk.BuildStepBuildExamples(pepperdir, platform)
-  test_sdk.BuildStepBuildTests(pepperdir, platform)
-  if options.test_examples:
-    test_sdk.BuildStepRunPyautoTests(pepperdir, platform, pepper_ver)
-
-  return 0
-
-
-if __name__ == '__main__':
-  sys.exit(main(sys.argv))
diff --git a/native_client_sdk/src/build_tools/build_paths.py b/native_client_sdk/src/build_tools/build_paths.py
new file mode 100644
index 0000000..a01671b
--- /dev/null
+++ b/native_client_sdk/src/build_tools/build_paths.py
@@ -0,0 +1,19 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+
+# Create the various paths of interest
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
+SDK_EXAMPLE_DIR = os.path.join(SDK_SRC_DIR, 'examples')
+SDK_LIBRARY_DIR = os.path.join(SDK_SRC_DIR, 'libraries')
+SDK_DIR = os.path.dirname(SDK_SRC_DIR)
+SRC_DIR = os.path.dirname(SDK_DIR)
+NACL_DIR = os.path.join(SRC_DIR, 'native_client')
+OUT_DIR = os.path.join(SRC_DIR, 'out')
+PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
+NACLPORTS_DIR = os.path.join(OUT_DIR, 'naclports')
+
+GSTORE = 'https://commondatastorage.googleapis.com/nativeclient-mirror/nacl/'
diff --git a/native_client_sdk/src/build_tools/build_projects.py b/native_client_sdk/src/build_tools/build_projects.py
new file mode 100755
index 0000000..bc904a0
--- /dev/null
+++ b/native_client_sdk/src/build_tools/build_projects.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import optparse
+import os
+import sys
+
+import buildbot_common
+import build_version
+import generate_make
+import parse_dsc
+
+from build_paths import NACL_DIR, SDK_SRC_DIR, OUT_DIR, SDK_EXAMPLE_DIR
+from build_paths import GSTORE
+from generate_index import LandingPage
+
+sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
+sys.path.append(os.path.join(NACL_DIR, 'build'))
+import getos
+import http_download
+
+
+MAKE = 'nacl_sdk/make_3_81/make.exe'
+LIB_DICT = {
+  'linux': [],
+  'mac': [],
+  'win': ['x86_32']
+}
+
+
+def CopyFilesFromTo(filelist, srcdir, dstdir):
+  for filename in filelist:
+    srcpath = os.path.join(srcdir, filename)
+    dstpath = os.path.join(dstdir, filename)
+    buildbot_common.CopyFile(srcpath, dstpath)
+
+
+def UpdateHelpers(pepperdir, platform, clobber=False):
+  if not os.path.exists(os.path.join(pepperdir, 'tools')):
+    buildbot_common.ErrorExit('Examples depend on missing tools.')
+
+  exampledir = os.path.join(pepperdir, 'examples')
+  if clobber:
+    buildbot_common.RemoveDir(exampledir)
+  buildbot_common.MakeDir(exampledir)
+
+  # Copy files for individual bild and landing page
+  files = ['favicon.ico', 'httpd.cmd']
+  CopyFilesFromTo(files, SDK_EXAMPLE_DIR, exampledir)
+
+  resourcesdir = os.path.join(SDK_EXAMPLE_DIR, 'resources')
+  files = ['index.css', 'index.js', 'button_close.png',
+      'button_close_hover.png']
+  CopyFilesFromTo(files, resourcesdir, exampledir)
+
+  # Copy tools scripts and make includes
+  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
+      os.path.join(pepperdir, 'tools'))
+  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'),
+      os.path.join(pepperdir, 'tools'))
+
+  # On Windows add a prebuilt make
+  if platform == 'win':
+    buildbot_common.BuildStep('Add MAKE')
+    http_download.HttpDownload(GSTORE + MAKE,
+                               os.path.join(pepperdir, 'tools', 'make.exe'))
+
+
+def UpdateProjects(pepperdir, platform, project_tree, toolchains,
+                   clobber=False, configs=None):
+  if configs is None:
+    configs = ['Debug', 'Release']
+  if not os.path.exists(os.path.join(pepperdir, 'tools')):
+    buildbot_common.ErrorExit('Examples depend on missing tools.')
+  if not os.path.exists(os.path.join(pepperdir, 'toolchain')):
+    buildbot_common.ErrorExit('Examples depend on missing toolchains.')
+
+
+  # Create the library output directories
+  libdir = os.path.join(pepperdir, 'lib')
+  for config in configs:
+    for arch in LIB_DICT[platform]:
+      dirpath = os.path.join(libdir, '%s_%s_host' % (platform, arch), config)
+      if clobber:
+        buildbot_common.RemoveDir(dirpath)
+      buildbot_common.MakeDir(dirpath)
+
+  for branch, projects in project_tree.iteritems():
+    dirpath = os.path.join(pepperdir, branch)
+    if clobber:
+      buildbot_common.RemoveDir(dirpath)
+    buildbot_common.MakeDir(dirpath)
+    depth = len(branch.split('/'))
+    targets = [desc['NAME'] for desc in projects]
+
+    # Generate master make for this branch of projects
+    generate_make.GenerateMasterMakefile(os.path.join(pepperdir, branch),
+                                         targets, depth)
+
+    if branch == 'examples':
+      landing_page = LandingPage()
+
+    # Generate individual projects
+    for desc in projects:
+      srcroot = os.path.dirname(desc['FILEPATH'])
+      generate_make.ProcessProject(srcroot, pepperdir, desc, toolchains)
+
+      if branch == 'examples':
+        landing_page.AddDesc(desc)
+
+    if branch == 'examples':
+      # Generate the landing page text file.
+      index_html = os.path.join(pepperdir, 'examples', 'index.html')
+      example_resources_dir = os.path.join(SDK_EXAMPLE_DIR, 'resources')
+      index_template = os.path.join(example_resources_dir,
+                                    'index.html.template')
+      with open(index_html, 'w') as fh:
+        out = landing_page.GeneratePage(index_template)
+        fh.write(out)
+      print 'Generated landing page.'
+
+  #
+  # TODO(noelallen) Add back in once we move examples
+  # Generate master example make
+  # example_targets = ['api', 'demos', 'getting_started', 'tutorials']
+  # example_targets = [x for x in example_targets if 'examples/'+x in desc_tree]
+  # print 'Example targets: ' + str(example_targets)
+  # generate_make.GenerateMasterMakefile(os.path.join(pepperdir, 'examples'),
+  #                                     example_targets, 2)
+
+
+def BuildProjects(pepperdir, platform, project_tree, deps=True,
+                  clean=False, config='Debug'):
+  for branch in project_tree:
+    make_dir = os.path.join(pepperdir, branch)
+    print "\n\nMake: " + make_dir
+    if platform == 'win':
+      # We need to modify the environment to build host on Windows.
+      make = os.path.join(make_dir, 'make.bat')
+    else:
+      make = 'make'
+
+    extra_args = ['CONFIG='+config]
+    if not deps:
+      extra_args += ['IGNORE_DEPS=1']
+
+    try:
+      buildbot_common.Run([make, '-j8', 'all_versions'] + extra_args,
+                          cwd=make_dir)
+    except:
+      print 'Failed to build ' + branch
+      raise
+
+    if clean:
+      # Clean to remove temporary files but keep the built
+      buildbot_common.Run([make, '-j8', 'clean'] + extra_args,
+                          cwd=make_dir)
+
+
+def main(args):
+  parser = optparse.OptionParser()
+  parser.add_option('--clobber',
+      help='Clobber project directories before copying new files',
+      action='store_true', default=False)
+  parser.add_option('-b', '--build',
+      help='Build the projects.', action='store_true')
+  parser.add_option('-x', '--experimental',
+      help='Build experimental projects', action='store_true')
+  parser.add_option('-t', '--toolchain',
+      help='Build using toolchain. Can be passed more than once.',
+      action='append', default=[])
+  parser.add_option('-d', '--dest',
+      help='Select which build destinations (project types) are valid.',
+      action='append')
+  parser.add_option('-p', '--project',
+      help='Select which projects are valid.',
+      action='append')
+
+  options, files = parser.parse_args(args[1:])
+  if len(files):
+    parser.error('Not expecting files.')
+    return 1
+
+  pepper_ver = str(int(build_version.ChromeMajorVersion()))
+  pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
+  platform = getos.GetPlatform()
+
+  if not options.toolchain:
+    options.toolchain = ['newlib', 'glibc', 'pnacl', 'host']
+
+  if 'host' in options.toolchain:
+    options.toolchain.append(platform)
+    print 'Adding platform: ' + platform
+
+  filters = {}
+  if options.toolchain:
+    filters['TOOLS'] = options.toolchain
+    print 'Filter by toolchain: ' + str(options.toolchain)
+  if not options.experimental:
+    filters['EXPERIMENTAL'] = False
+  if options.dest:
+    filters['DEST'] = options.dest
+    print 'Filter by type: ' + str(options.dest)
+  if options.project:
+    filters['NAME'] = options.project
+    print 'Filter by name: ' + str(options.project)
+
+  project_tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, filters=filters)
+  parse_dsc.PrintProjectTree(project_tree)
+
+  UpdateHelpers(pepperdir, platform, clobber=options.clobber)
+  UpdateProjects(pepperdir, platform, project_tree, options.toolchain,
+                 clobber=options.clobber)
+  if options.build:
+    BuildProjects(pepperdir, platform, project_tree)
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))
diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py
index 5f9fbeb..7d7cd48 100755
--- a/native_client_sdk/src/build_tools/build_sdk.py
+++ b/native_client_sdk/src/build_tools/build_sdk.py
@@ -23,7 +23,6 @@ import datetime
 import optparse
 import os
 import re
-import subprocess
 import sys
 
 if sys.version_info < (2, 6, 0):
@@ -32,35 +31,25 @@ if sys.version_info < (2, 6, 0):
 
 # local includes
 import buildbot_common
+import build_projects
 import build_updater
-import build_utils
+import build_version
 import generate_make
 import generate_notice
 import manifest_util
+import parse_dsc
 import test_sdk
 
-# Create the various paths of interest
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
-SDK_EXAMPLE_DIR = os.path.join(SDK_SRC_DIR, 'examples')
-SDK_LIBRARY_DIR = os.path.join(SDK_SRC_DIR, 'libraries')
-SDK_DIR = os.path.dirname(SDK_SRC_DIR)
-SRC_DIR = os.path.dirname(SDK_DIR)
-NACL_DIR = os.path.join(SRC_DIR, 'native_client')
-OUT_DIR = os.path.join(SRC_DIR, 'out')
-PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
-NACLPORTS_DIR = os.path.join(OUT_DIR, 'naclports')
+from build_paths import SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR
+from build_paths import PPAPI_DIR, NACLPORTS_DIR, GSTORE
 
 # Add SDK make tools scripts to the python path.
 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
 sys.path.append(os.path.join(NACL_DIR, 'build'))
 
 import getos
-import http_download
 import oshelpers
 
-GSTORE = 'https://commondatastorage.googleapis.com/nativeclient-mirror/nacl/'
-MAKE = 'nacl_sdk/make_3_81/make.exe'
 CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py')
 
 NACLPORTS_URL = 'https://naclports.googlecode.com/svn/trunk/src'
@@ -594,55 +583,6 @@ def BuildStepBuildToolchains(pepperdir, platform, toolchains):
                        'newlib')
 
 
-def BuildStepCopyBuildHelpers(pepperdir, platform):
-  buildbot_common.BuildStep('Copy build helpers')
-  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
-      os.path.join(pepperdir, 'tools'))
-  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'),
-      os.path.join(pepperdir, 'tools'))
-  if platform == 'win':
-    buildbot_common.BuildStep('Add MAKE')
-    http_download.HttpDownload(GSTORE + MAKE,
-                               os.path.join(pepperdir, 'tools', 'make.exe'))
-
-
-EXAMPLE_LIST = [
-  'debugging',
-  'dlopen',
-  'file_histogram',
-  'file_io',
-  'gamepad',
-  'geturl',
-  'hello_nacl_io',
-  'hello_world_stdio',
-  'hello_world',
-  'hello_world_gles',
-  'hello_world_instance3d',
-  'hello_world_interactive',
-  'input_events',
-  'load_progress',
-  'mouselock',
-  'pi_generator',
-  'sine_synth',
-  'websocket',
-]
-
-LIBRARY_LIST = [
-  'libjpeg',
-  'nacl_io',
-  'ppapi',
-  'ppapi_cpp',
-  'ppapi_gles2',
-  'ppapi_main',
-  'pthread',
-  'zlib',
-]
-
-LIB_DICT = {
-  'linux': [],
-  'mac': [],
-  'win': ['x86_32']
-}
 
 
 def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
@@ -654,97 +594,27 @@ def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
   return dirpath
 
 
-def BuildStepCopyExamples(pepperdir, toolchains, build_experimental, clobber):
-  buildbot_common.BuildStep('Copy examples')
-
-  if not os.path.exists(os.path.join(pepperdir, 'tools')):
-    buildbot_common.ErrorExit('Examples depend on missing tools.')
-  if not os.path.exists(os.path.join(pepperdir, 'toolchain')):
-    buildbot_common.ErrorExit('Examples depend on missing toolchains.')
-
-  exampledir = MakeDirectoryOrClobber(pepperdir, 'examples', clobber)
-  libdir = MakeDirectoryOrClobber(pepperdir, 'lib', clobber)
-
-  plat = getos.GetPlatform()
-  for arch in LIB_DICT[plat]:
-    buildbot_common.MakeDir(os.path.join(libdir, '%s_%s_host' % (plat, arch)))
-    if options.gyp and plat != 'win':
-      configs = ['debug', 'release']
-    else:
-      configs = ['Debug', 'Release']
-    for config in configs:
-      buildbot_common.MakeDir(os.path.join(libdir, '%s_%s_host' % (plat, arch),
-                              config))
-
-  MakeDirectoryOrClobber(pepperdir, 'src', clobber)
-
-  # Copy individual files
-  files = ['favicon.ico', 'httpd.cmd']
-  for filename in files:
-    oshelpers.Copy(['-v', os.path.join(SDK_EXAMPLE_DIR, filename),
-                   exampledir])
-
-  args = ['--dstroot=%s' % pepperdir, '--master']
-  for toolchain in toolchains:
-    if toolchain != 'arm':
-      args.append('--' + toolchain)
-
-  for example in EXAMPLE_LIST:
-    dsc = os.path.join(SDK_EXAMPLE_DIR, example, 'example.dsc')
-    args.append(dsc)
-
-  for library in LIBRARY_LIST:
-    dsc = os.path.join(SDK_LIBRARY_DIR, library, 'library.dsc')
-    args.append(dsc)
+def BuildStepUpdateHelpers(pepperdir, platform, clobber):
+  buildbot_common.BuildStep('Update project helpers')
+  build_projects.UpdateHelpers(pepperdir, platform, clobber=clobber)
 
-  if build_experimental:
-    args.append('--experimental')
-
-  print "Generating Makefiles: %s" % str(args)
-  if generate_make.main(args):
-    buildbot_common.ErrorExit('Failed to build examples.')
-
-
-def GetWindowsEnvironment():
-  sys.path.append(os.path.join(NACL_DIR, 'buildbot'))
-  import buildbot_standard
-
-  # buildbot_standard.SetupWindowsEnvironment expects a "context" object. We'll
-  # fake enough of that here to work.
-  class FakeContext(object):
-    def __init__(self):
-      self.env = os.environ
 
-    def GetEnv(self, key):
-      return self.env[key]
+def BuildStepUpdateUserProjects(pepperdir, platform, toolchains,
+                                build_experimental, clobber):
+  buildbot_common.BuildStep('Update examples and libraries')
 
-    def __getitem__(self, key):
-      return self.env[key]
+  filters = {}
+  if not build_experimental:
+    filters['EXPERIMENTAL'] = False
+  if toolchains:
+    filters['TOOLS'] = toolchains
 
-    def SetEnv(self, key, value):
-      self.env[key] = value
+  # Update examples and libraries
+  filters['DEST'] = ['examples', 'src']
 
-    def __setitem__(self, key, value):
-      self.env[key] = value
-
-  context = FakeContext()
-  buildbot_standard.SetupWindowsEnvironment(context)
-
-  # buildbot_standard.SetupWindowsEnvironment adds the directory which contains
-  # vcvarsall.bat to the path, but not the directory which contains cl.exe,
-  # link.exe, etc.
-  # Running vcvarsall.bat adds the correct directories to the path, which we
-  # extract below.
-  process = subprocess.Popen('vcvarsall.bat x86 > NUL && set',
-      stdout=subprocess.PIPE, env=context.env, shell=True)
-  stdout, _ = process.communicate()
-
-  # Parse environment from "set" command above.
-  # It looks like this:
-  # KEY1=VALUE1\r\n
-  # KEY2=VALUE2\r\n
-  # ...
-  return dict(line.split('=') for line in stdout.split('\r\n')[:-1])
+  tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, filters=filters)
+  build_projects.UpdateProjects(pepperdir, platform, tree, clobber=clobber,
+                                toolchains=toolchains)
 
 
 def BuildStepMakeAll(pepperdir, platform, directory, step_name,
@@ -754,11 +624,8 @@ def BuildStepMakeAll(pepperdir, platform, directory, step_name,
 
   print "\n\nMake: " + make_dir
   if platform == 'win':
-    # We need to modify the environment to build host on Windows.
-    env = GetWindowsEnvironment()
     make = os.path.join(make_dir, 'make.bat')
   else:
-    env = os.environ
     make = 'make'
 
   extra_args = ['CONFIG='+config]
@@ -766,11 +633,10 @@ def BuildStepMakeAll(pepperdir, platform, directory, step_name,
     extra_args += ['IGNORE_DEPS=1']
 
   buildbot_common.Run([make, '-j8', 'all_versions'] + extra_args,
-                      cwd=make_dir, env=env)
+                      cwd=make_dir)
   if clean:
     # Clean to remove temporary files but keep the built libraries.
-    buildbot_common.Run([make, '-j8', 'clean'] + extra_args,
-                        cwd=make_dir, env=env)
+    buildbot_common.Run([make, '-j8', 'clean'] + extra_args, cwd=make_dir)
 
 
 def BuildStepBuildLibraries(pepperdir, platform, directory, clean=True):
@@ -824,8 +690,6 @@ def BuildStepTestSDK():
   args = []
   if options.build_experimental:
     args.append('--experimental')
-  if options.run_pyauto_tests:
-    args.append('--pyauto')
   test_sdk.main(args)
 
 
@@ -853,14 +717,14 @@ def GetManifestBundle(pepper_ver, revision, tarfile, archive_url):
 def BuildStepArchiveBundle(name, pepper_ver, revision, tarfile):
   buildbot_common.BuildStep('Archive %s' % name)
   bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
-      build_utils.ChromeVersion(),)
+      build_version.ChromeVersion(),)
   tarname = os.path.basename(tarfile)
   tarfile_dir = os.path.dirname(tarfile)
   buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
 
   # generate "manifest snippet" for this archive.
   archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
-      build_utils.ChromeVersion(), tarname)
+      build_version.ChromeVersion(), tarname)
   bundle = GetManifestBundle(pepper_ver, revision, tarfile, archive_url)
 
   manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
@@ -880,7 +744,7 @@ def BuildStepArchiveSDKTools():
 
     buildbot_common.BuildStep('Archive SDK Tools')
     bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
-        build_utils.ChromeVersion(),)
+        build_version.ChromeVersion(),)
     buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
                             step_link=False)
     buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
@@ -949,8 +813,6 @@ def main(args):
   parser = optparse.OptionParser()
   parser.add_option('--run-tests',
       help='Run tests. This includes building examples.', action='store_true')
-  parser.add_option('--run-pyauto-tests',
-      help='Run the pyauto tests for examples.', action='store_true')
   parser.add_option('--skip-tar', help='Skip generating a tarball.',
       action='store_true')
   parser.add_option('--archive', help='Force the archive step.',
@@ -977,12 +839,8 @@ def main(args):
   arch = 'x86'
 
   generate_make.use_gyp = options.gyp
-
-  # TODO(binji) for now, only test examples on non-trybots. Trybots don't build
-  # pyauto Chrome.
   if buildbot_common.IsSDKBuilder():
     options.run_tests = True
-    options.run_pyauto_tests = True
     options.archive = True
     options.build_ports = True
 
@@ -995,8 +853,8 @@ def main(args):
   if options.archive and options.skip_tar:
     parser.error('Incompatible arguments with archive.')
 
-  chrome_version = int(build_utils.ChromeMajorVersion())
-  clnumber = build_utils.ChromeRevision()
+  chrome_version = int(build_version.ChromeMajorVersion())
+  clnumber = build_version.ChromeRevision()
   pepper_ver = str(chrome_version)
   pepper_old = str(chrome_version - 1)
   pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
@@ -1026,8 +884,10 @@ def main(args):
   BuildStepCopyTextFiles(pepperdir, pepper_ver, clnumber)
   BuildStepBuildToolchains(pepperdir, platform, toolchains)
   InstallCommonHeaders(os.path.join(pepperdir, 'include'))
-  BuildStepCopyBuildHelpers(pepperdir, platform)
-  BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental, True)
+
+  BuildStepUpdateHelpers(pepperdir, platform, True)
+  BuildStepUpdateUserProjects(pepperdir, platform, toolchains,
+                              options.build_experimental, True)
 
   # Ship with libraries prebuilt, so run that first.
   BuildStepBuildLibraries(pepperdir, platform, 'src')
diff --git a/native_client_sdk/src/build_tools/build_version.py b/native_client_sdk/src/build_tools/build_version.py
new file mode 100644
index 0000000..ffe8cf0
--- /dev/null
+++ b/native_client_sdk/src/build_tools/build_version.py
@@ -0,0 +1,64 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Small utility library of python functions used during SDK building.
+"""
+
+import os
+import sys
+
+# pylint: disable=E0602
+
+# Reuse last change utility code.
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(SCRIPT_DIR)))
+sys.path.append(os.path.join(SRC_DIR, 'build/util'))
+import lastchange
+
+
+# Location of chrome's version file.
+VERSION_PATH = os.path.join(SRC_DIR, 'chrome', 'VERSION')
+
+
+def ChromeVersion():
+  '''Extract chrome version from src/chrome/VERSION + svn.
+
+  Returns:
+    Chrome version string or trunk + svn rev.
+  '''
+  info = lastchange.FetchVersionInfo(None)
+  if info.url.startswith('/trunk/'):
+    return 'trunk.%s' % info.revision
+  else:
+    return ChromeVersionNoTrunk()
+
+
+def ChromeVersionNoTrunk():
+  '''Extract the chrome version from src/chrome/VERSION.
+  Ignore whether this is a trunk build.
+
+  Returns:
+    Chrome version string.
+  '''
+  exec(open(VERSION_PATH).read())
+  return '%s.%s.%s.%s' % (MAJOR, MINOR, BUILD, PATCH)
+
+
+def ChromeMajorVersion():
+  '''Extract chrome major version from src/chrome/VERSION.
+
+  Returns:
+    Chrome major version.
+  '''
+  exec(open(VERSION_PATH, 'r').read())
+  return str(MAJOR)
+
+
+def ChromeRevision():
+  '''Extract chrome revision from svn.
+
+  Returns:
+    The Chrome revision as a string. e.g. "12345"
+  '''
+  return lastchange.FetchVersionInfo(None).revision
diff --git a/native_client_sdk/src/build_tools/buildbot_common.py b/native_client_sdk/src/build_tools/buildbot_common.py
index a591dae..4de37f8 100644
--- a/native_client_sdk/src/build_tools/buildbot_common.py
+++ b/native_client_sdk/src/build_tools/buildbot_common.py
@@ -10,12 +10,9 @@ import os
 import subprocess
 import sys
 
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
-SDK_DIR = os.path.dirname(SDK_SRC_DIR)
-SRC_DIR = os.path.dirname(SDK_DIR)
-sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
+from build_paths import SDK_SRC_DIR, NACL_DIR
 
+sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
 import oshelpers
 
 def IsSDKBuilder():
@@ -46,6 +43,48 @@ def ErrorExit(msg):
   sys.exit(1)
 
 
+def GetWindowsEnvironment():
+  sys.path.append(os.path.join(NACL_DIR, 'buildbot'))
+  import buildbot_standard
+
+  # buildbot_standard.SetupWindowsEnvironment expects a "context" object. We'll
+  # fake enough of that here to work.
+  class FakeContext(object):
+    def __init__(self):
+      self.env = os.environ
+
+    def GetEnv(self, key):
+      return self.env[key]
+
+    def __getitem__(self, key):
+      return self.env[key]
+
+    def SetEnv(self, key, value):
+      self.env[key] = value
+
+    def __setitem__(self, key, value):
+      self.env[key] = value
+
+  context = FakeContext()
+  buildbot_standard.SetupWindowsEnvironment(context)
+
+  # buildbot_standard.SetupWindowsEnvironment adds the directory which contains
+  # vcvarsall.bat to the path, but not the directory which contains cl.exe,
+  # link.exe, etc.
+  # Running vcvarsall.bat adds the correct directories to the path, which we
+  # extract below.
+  process = subprocess.Popen('vcvarsall.bat x86 > NUL && set',
+      stdout=subprocess.PIPE, env=context.env, shell=True)
+  stdout, _ = process.communicate()
+
+  # Parse environment from "set" command above.
+  # It looks like this:
+  # KEY1=VALUE1\r\n
+  # KEY2=VALUE2\r\n
+  # ...
+  return dict(line.split('=') for line in stdout.split('\r\n')[:-1])
+
+
 def BuildStep(name):
   """Annotate a buildbot build step."""
   sys.stdout.flush()
@@ -60,6 +99,14 @@ def Run(args, cwd=None, env=None, shell=False):
   shell is not False, the process is launched via the shell to provide shell
   interpretation of the arguments.  Shell behavior can differ between platforms
   so this should be avoided when not using platform dependent shell scripts."""
+
+  # We need to modify the environment to build host on Windows.
+  if not env:
+    if sys.platform.startswith('cygwin') or sys.platform.startswith('win'):
+      env = GetWindowsEnvironment()
+    else:
+      env = os.environ
+
   print 'Running: ' + ' '.join(args)
   sys.stdout.flush()
   sys.stderr.flush()
diff --git a/native_client_sdk/src/build_tools/easy_template.py b/native_client_sdk/src/build_tools/easy_template.py
index b2bfa71..d439ea0 100755
--- a/native_client_sdk/src/build_tools/easy_template.py
+++ b/native_client_sdk/src/build_tools/easy_template.py
@@ -6,6 +6,7 @@
 import copy
 import cStringIO
 import optparse
+import os
 import re
 import sys
 
@@ -49,27 +50,42 @@ def TemplateToPython(template, statement_re, expr_re):
   return output.getvalue()
 
 
-def RunTemplate(src, dst, template_dict, statement_re=None, expr_re=None):
+def RunTemplate(srcfile, dstfile, template_dict, statement_re=None,
+                expr_re=None):
   statement_re = statement_re or re.compile(STATEMENT_RE)
   expr_re = expr_re or re.compile(EXPR_RE)
-  script = TemplateToPython(src.read(), statement_re, expr_re)
+  script = TemplateToPython(srcfile.read(), statement_re, expr_re)
   template_dict = copy.copy(template_dict)
-  template_dict['__outfile__'] = dst
+  template_dict['__outfile__'] = dstfile
   exec script in template_dict
 
 
-def RunTemplateFile(srcfile, dstfile, template_dict, statement_re=None,
+def RunTemplateFile(srcpath, dstpath, template_dict, statement_re=None,
                     expr_re=None):
-  with open(srcfile) as src:
-    with open(dstfile, 'w') as dst:
-      RunTemplate(src, dst, template_dict, statement_re, expr_re)
+  with open(srcpath) as srcfile:
+    with open(dstpath, 'w') as dstfile:
+      RunTemplate(srcfile, dstfile, template_dict, statement_re, expr_re)
+
+
+def RunTemplateFileIfChanged(srcpath, dstpath, replace):
+  dststr = cStringIO.StringIO()
+  with open(srcpath) as srcfile:
+    RunTemplate(srcfile, dststr, replace)
+
+  if os.path.exists(dstpath):
+    with open(dstpath) as dstfile:
+      if dstfile.read() == dststr.getvalue():
+        return
+
+  with open(dstpath, 'w') as dstfile:
+    dstfile.write(dststr.getvalue())
 
 
 def RunTemplateString(src, template_dict, statement_re=None, expr_re=None):
-  srcf = cStringIO.StringIO(src)
-  dstf = cStringIO.StringIO()
-  RunTemplate(srcf, dstf, template_dict, statement_re, expr_re)
-  return dstf.getvalue()
+  srcstr = cStringIO.StringIO(src)
+  dststr = cStringIO.StringIO()
+  RunTemplate(srcstr, dststr, template_dict, statement_re, expr_re)
+  return dststr.getvalue()
 
 
 def main(args):
diff --git a/native_client_sdk/src/build_tools/generate_index.py b/native_client_sdk/src/build_tools/generate_index.py
index f5d9273..37c2a52 100644
--- a/native_client_sdk/src/build_tools/generate_index.py
+++ b/native_client_sdk/src/build_tools/generate_index.py
@@ -12,8 +12,8 @@ class LandingPage(object):
     self.section_map = collections.defaultdict(list)
 
   def GeneratePage(self, template_path):
-    with open(template_path) as src:
-      template = src.read()
+    with open(template_path) as template_file:
+      template = template_file.read()
     template_dict = { 'section_map': self.section_map }
     return easy_template.RunTemplateString(template, template_dict)
 
diff --git a/native_client_sdk/src/build_tools/generate_make.py b/native_client_sdk/src/build_tools/generate_make.py
index b7eab1b..9bd0fa0 100755
--- a/native_client_sdk/src/build_tools/generate_make.py
+++ b/native_client_sdk/src/build_tools/generate_make.py
@@ -1,31 +1,14 @@
-#!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import cStringIO
-import optparse
 import os
 import sys
 
 import buildbot_common
 from buildbot_common import ErrorExit
-import easy_template
-from generate_index import LandingPage
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
-SDK_EXAMPLE_DIR = os.path.join(SDK_SRC_DIR, 'examples')
-SDK_DIR = os.path.dirname(SDK_SRC_DIR)
-SRC_DIR = os.path.dirname(SDK_DIR)
-OUT_DIR = os.path.join(SRC_DIR, 'out')
-PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
-
-use_gyp = False
-
-# Add SDK make tools scripts to the python path.
-sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
-import getos
+from easy_template import RunTemplateFileIfChanged
+from build_paths import SCRIPT_DIR, SDK_EXAMPLE_DIR
 
 def Trace(msg):
   if Trace.verbose:
@@ -33,7 +16,6 @@ def Trace(msg):
 Trace.verbose = False
 
 
-
 def ShouldProcessHTML(desc):
   return desc['DEST'] in ('examples', 'tests')
 
@@ -53,21 +35,6 @@ def GenerateSourceCopyList(desc):
   return sources
 
 
-def RunTemplateFile(srcfile, dstfile, replace):
-  dst = cStringIO.StringIO()
-  with open(srcfile) as srcf:
-    easy_template.RunTemplate(srcf, dst, replace)
-
-  if os.path.exists(dstfile):
-    with open(dstfile) as dstf:
-      if dstf.read() == dst.getvalue():
-        Trace('Generated %s is the same. Skipping' % dstfile)
-        return
-
-  with open(dstfile, 'w') as dstf:
-    dstf.write(dst.getvalue())
-
-
 def GetSourcesDict(sources):
   source_map = {}
   for key in ['.c', '.cc']:
@@ -98,123 +65,10 @@ def GetPlatforms(plat_list, plat_filter, first_toolchain):
   return platforms
 
 
-# 'KEY' : ( <TYPE>, [Accepted Values], <Required?>)
-DSC_FORMAT = {
-    'TOOLS' : (list, ['newlib:arm', 'newlib:x64', 'newlib:x86', 'newlib',
-                      'glibc', 'pnacl', 'win', 'linux'], True),
-    'CONFIGS' : (list, ['Debug', 'Release'], False),
-    'PREREQ' : (list, '', False),
-    'TARGETS' : (list, {
-        'NAME': (str, '', True),
-        # main = nexe target
-        # lib = library target
-        # so = shared object target, automatically added to NMF
-        # so-standalone =  shared object target, not put into NMF
-        'TYPE': (str, ['main', 'lib', 'so', 'so-standalone'], True),
-        'SOURCES': (list, '', True),
-        'CCFLAGS': (list, '', False),
-        'CXXFLAGS': (list, '', False),
-        'DEFINES': (list, '', False),
-        'LDFLAGS': (list, '', False),
-        'INCLUDES': (list, '', False),
-        'LIBS' : (list, '', False),
-        'DEPS' : (list, '', False)
-    }, True),
-    'HEADERS': (list, {
-        'FILES': (list, '', True),
-        'DEST': (str, '', True),
-    }, False),
-    'SEARCH': (list, '', False),
-    'POST': (str, '', False),
-    'PRE': (str, '', False),
-    'DEST': (str, ['examples', 'src', 'testlibs', 'tests'], True),
-    'NAME': (str, '', False),
-    'DATA': (list, '', False),
-    'TITLE': (str, '', False),
-    'GROUP': (str, '', False),
-    'EXPERIMENTAL': (bool, [True, False], False)
-}
-
-
 def ErrorMsgFunc(text):
   sys.stderr.write(text + '\n')
 
 
-def ValidateFormat(src, dsc_format, ErrorMsg=ErrorMsgFunc):
-  failed = False
-
-  # Verify all required keys are there
-  for key in dsc_format:
-    (exp_type, exp_value, required) = dsc_format[key]
-    if required and key not in src:
-      ErrorMsg('Missing required key %s.' % key)
-      failed = True
-
-  # For each provided key, verify it's valid
-  for key in src:
-    # Verify the key is known
-    if key not in dsc_format:
-      ErrorMsg('Unexpected key %s.' % key)
-      failed = True
-      continue
-
-    exp_type, exp_value, required = dsc_format[key]
-    value = src[key]
-
-    # Verify the key is of the expected type
-    if exp_type != type(value):
-      ErrorMsg('Key %s expects %s not %s.' % (
-          key, exp_type.__name__.upper(), type(value).__name__.upper()))
-      failed = True
-      continue
-
-    # Verify the value is non-empty if required
-    if required and not value:
-      ErrorMsg('Expected non-empty value for %s.' % key)
-      failed = True
-      continue
-
-    # If it's a bool, the expected values are always True or False.
-    if exp_type is bool:
-      continue
-
-    # If it's a string and there are expected values, make sure it matches
-    if exp_type is str:
-      if type(exp_value) is list and exp_value:
-        if value not in exp_value:
-          ErrorMsg("Value '%s' not expected for %s." % (value, key))
-          failed = True
-      continue
-
-    # if it's a list, then we need to validate the values
-    if exp_type is list:
-      # If we expect a dictionary, then call this recursively
-      if type(exp_value) is dict:
-        for val in value:
-          if not ValidateFormat(val, exp_value, ErrorMsg):
-            failed = True
-        continue
-      # If we expect a list of strings
-      if type(exp_value) is str:
-        for val in value:
-          if type(val) is not str:
-            ErrorMsg('Value %s in %s is not a string.' % (val, key))
-            failed = True
-        continue
-      # if we expect a particular string
-      if type(exp_value) is list:
-        for val in value:
-          if val not in exp_value:
-            ErrorMsg('Value %s not expected in %s.' % (val, key))
-            failed = True
-        continue
-
-    # If we got this far, it's an unexpected type
-    ErrorMsg('Unexpected type %s for key %s.' % (str(type(src[key])), key))
-    continue
-  return not failed
-
-
 def AddMakeBat(pepperdir, makepath):
   """Create a simple batch file to execute Make.
 
@@ -261,55 +115,18 @@ def IsNexe(desc):
 def ProcessHTML(srcroot, dstroot, desc, toolchains, configs, first_toolchain):
   name = desc['NAME']
   outdir = os.path.join(dstroot, desc['DEST'], name)
-  srcfile = os.path.join(srcroot, 'index.html')
-  dstfile = os.path.join(outdir, 'index.html')
+  srcpath = os.path.join(srcroot, 'index.html')
+  dstpath = os.path.join(outdir, 'index.html')
   tools = GetPlatforms(toolchains, desc['TOOLS'], first_toolchain)
 
-  if use_gyp and getos.GetPlatform() != 'win':
-    # Make the config names lowercase when using gyp...
-    configs = [c.lower() for c in configs]
-
-  if use_gyp:
-    path = "build/{tc}-{config}"
-  else:
-    path = "{tc}/{config}"
-
+  path = "{tc}/{config}"
   replace = {
     'title': desc['TITLE'],
     'attrs':
         'data-name="%s" data-tools="%s" data-configs="%s" data-path="%s"' % (
         name, ' '.join(tools), ' '.join(configs), path),
   }
-  RunTemplateFile(srcfile, dstfile, replace)
-
-
-def LoadProject(filename, toolchains):
-  """Generate a Master Makefile that builds all examples.
-
-  Load a project description file, verifying it conforms and checking
-  if it matches the set of requested toolchains.  Return None if the
-  project is filtered out."""
-
-  Trace('Processing %s...' % filename)
-  # Default src directory is the directory the description was found in
-  desc = open(filename, 'r').read()
-  desc = eval(desc, {}, {})
-
-  # Verify the format of this file
-  if not ValidateFormat(desc, DSC_FORMAT):
-    ErrorExit('Failed to validate: ' + filename)
-
-  # Check if we are actually interested in this example
-  match = False
-  for toolchain in toolchains:
-    if toolchain in desc['TOOLS']:
-      match = True
-      break
-  if not match:
-    return None
-
-  desc['FILENAME'] = filename
-  return desc
+  RunTemplateFileIfChanged(srcpath, dstpath, replace)
 
 
 def FindAndCopyFiles(src_files, root, search_dirs, dst_dir):
@@ -327,7 +144,11 @@ def FindAndCopyFiles(src_files, root, search_dirs, dst_dir):
     buildbot_common.CopyFile(src_file, dst_file)
 
 
-def ProcessProject(srcroot, dstroot, desc, toolchains, first_toolchain):
+def ProcessProject(srcroot, dstroot, desc, toolchains, configs=None,
+                   first_toolchain=False):
+  if not configs:
+    configs = ['Debug', 'Release']
+
   name = desc['NAME']
   out_dir = os.path.join(dstroot, desc['DEST'], name)
   buildbot_common.MakeDir(out_dir)
@@ -345,180 +166,52 @@ def ProcessProject(srcroot, dstroot, desc, toolchains, first_toolchain):
 
   make_path = os.path.join(out_dir, 'Makefile')
 
-  if use_gyp:
-    # Process the dsc file to produce gyp input
-    dsc = desc['FILENAME']
-    dsc2gyp = os.path.join(SDK_SRC_DIR, 'build_tools/dsc2gyp.py')
-    gypfile = os.path.join(OUT_DIR, 'tmp', name, name + '.gyp')
-    buildbot_common.Run([sys.executable, dsc2gyp, dsc, '-o', gypfile],
-                        cwd=out_dir)
-
-    # Run gyp on the generated gyp file
-    if sys.platform == 'win32':
-      generator = 'msvs'
-    else:
-      generator = os.path.join(SCRIPT_DIR, "make_simple.py")
-    gyp = os.path.join(SDK_SRC_DIR, '..', '..', 'tools', 'gyp', 'gyp')
-    if sys.platform == 'win32':
-      gyp += '.bat'
-    buildbot_common.Run([gyp, '-Gstandalone', '--format',  generator,
-                        '--toplevel-dir=.', gypfile], cwd=out_dir)
-
-  if sys.platform == 'win32' or not use_gyp:
-    if IsNexe(desc):
-      template = os.path.join(SCRIPT_DIR, 'template.mk')
-    else:
-      template = os.path.join(SCRIPT_DIR, 'library.mk')
-
-    # Ensure the order of |tools| is the same as toolchains; that way if
-    # first_toolchain is set, it will choose based on the order of |toolchains|.
-    tools = [tool for tool in toolchains if tool in desc['TOOLS']]
-    if first_toolchain:
-      tools = [tools[0]]
-    template_dict = {
-      'pre': desc.get('PRE', ''),
-      'tools': tools,
-      'targets': desc['TARGETS'],
-    }
-    RunTemplateFile(template, make_path, template_dict)
+  if IsNexe(desc):
+    template = os.path.join(SCRIPT_DIR, 'template.mk')
+  else:
+    template = os.path.join(SCRIPT_DIR, 'library.mk')
+
+  # Ensure the order of |tools| is the same as toolchains; that way if
+  # first_toolchain is set, it will choose based on the order of |toolchains|.
+  tools = [tool for tool in toolchains if tool in desc['TOOLS']]
+  if first_toolchain:
+    tools = [tools[0]]
+  template_dict = {
+    'rel_sdk': '/'.join(['..'] * (len(desc['DEST'].split('/')) + 1)),
+    'pre': desc.get('PRE', ''),
+    'tools': tools,
+    'targets': desc['TARGETS'],
+  }
+  RunTemplateFileIfChanged(template, make_path, template_dict)
 
   outdir = os.path.dirname(os.path.abspath(make_path))
   pepperdir = os.path.dirname(os.path.dirname(outdir))
   AddMakeBat(pepperdir, outdir)
+
+  if ShouldProcessHTML(desc):
+    ProcessHTML(srcroot, dstroot, desc, toolchains, configs,
+                first_toolchain)
+
   return (name, desc['DEST'])
 
 
-def GenerateMasterMakefile(in_path, out_path, projects):
-  """Generate a Master Makefile that builds all examples. """
-  project_names = [project['NAME'] for project in projects]
-  template_dict = { 'projects': project_names }
-  RunTemplateFile(in_path, out_path, template_dict)
+def GenerateMasterMakefile(out_path, targets, depth):
+  """Generate a Master Makefile that builds all examples.
 
+  Args:
+    out_path: Root for output such that out_path+NAME = full path
+    targets: List of targets names
+    depth: How deep in from NACL_SDK_ROOT
+  """
+  in_path = os.path.join(SDK_EXAMPLE_DIR, 'Makefile')
+  out_path = os.path.join(out_path, 'Makefile')
+  template_dict = {
+    'projects': targets,
+    'rel_sdk' : '/'.join(['..'] * depth)
+  }
+  RunTemplateFileIfChanged(in_path, out_path, template_dict)
   outdir = os.path.dirname(os.path.abspath(out_path))
   pepperdir = os.path.dirname(outdir)
   AddMakeBat(pepperdir, outdir)
 
 
-def main(argv):
-  usage = "usage: generate_make [options] <dsc_file ..>"
-  parser = optparse.OptionParser(usage=usage)
-  parser.add_option('--dstroot', help='Set root for destination.',
-      default=os.path.join(OUT_DIR, 'pepper_canary'))
-  parser.add_option('--master', help='Create master Makefile.',
-      action='store_true')
-  parser.add_option('--newlib', help='Create newlib examples.',
-      action='store_true')
-  parser.add_option('--glibc', help='Create glibc examples.',
-      action='store_true')
-  parser.add_option('--pnacl', help='Create pnacl examples.',
-      action='store_true')
-  parser.add_option('--host', help='Create host examples.',
-      action='store_true')
-  parser.add_option('--config', help='Add configuration (debug/release).',
-      action='append')
-  parser.add_option('--experimental', help='Create experimental examples.',
-      action='store_true')
-  parser.add_option('--first-valid-toolchain',
-      help='Only build one toolchain, the first one that is valid.',
-      action='store_true')
-  parser.add_option('-v', '--verbose', help='Verbose output',
-      action='store_true')
-
-  toolchains = []
-  platform = getos.GetPlatform()
-
-  options, args = parser.parse_args(argv)
-  if options.verbose:
-    Trace.verbose = True
-  if options.newlib:
-    toolchains.append('newlib')
-  if options.glibc:
-    toolchains.append('glibc')
-  if options.pnacl:
-    toolchains.append('pnacl')
-  if options.host:
-    toolchains.append(platform)
-
-  if not args:
-    ErrorExit('Please specify one or more projects to generate Makefiles for.')
-
-  # By default support newlib and glibc
-  if not toolchains:
-    toolchains = ['newlib', 'glibc', 'pnacl']
-
-  valid_configs = ['Debug', 'Release']
-  if options.config:
-    configs = []
-    for config in options.config:
-      if config in valid_configs:
-        configs.append(config)
-      else:
-        ErrorExit('Invalid config: %s' % config)
-  else:
-    configs = valid_configs
-
-  master_projects = {}
-
-  for i, filename in enumerate(args):
-    if i:
-      # Print two newlines between each dsc file we process
-      Trace('\n')
-    desc = LoadProject(filename, toolchains)
-    if not desc:
-      Trace('Skipping %s, not in [%s].' % (filename, ', '.join(toolchains)))
-      continue
-
-    if desc.get('EXPERIMENTAL', False) and not options.experimental:
-      Trace('Skipping %s, experimental only.' % (filename,))
-      continue
-
-    srcroot = os.path.dirname(os.path.abspath(filename))
-    if not ProcessProject(srcroot, options.dstroot, desc, toolchains,
-        options.first_valid_toolchain):
-      ErrorExit('\n*** Failed to process project: %s ***' % filename)
-
-    # if this is an example update it's html file.
-    if ShouldProcessHTML(desc):
-      ProcessHTML(srcroot, options.dstroot, desc, toolchains, configs,
-                  options.first_valid_toolchain)
-
-    # Create a list of projects for each DEST. This will be used to generate a
-    # master makefile.
-    master_projects.setdefault(desc['DEST'], []).append(desc)
-
-
-  if master_projects.get('examples'):
-    landing_page = LandingPage()
-    for desc in master_projects.get('examples'):
-      landing_page.AddDesc(desc)
-
-    # Generate the landing page text file.
-    index_html = os.path.join(options.dstroot, 'examples', 'index.html')
-    example_resources_dir = os.path.join(SDK_EXAMPLE_DIR, 'resources')
-    index_template = os.path.join(example_resources_dir, 'index.html.template')
-    with open(index_html, 'w') as fh:
-      fh.write(landing_page.GeneratePage(index_template))
-
-    # Copy additional files needed for the landing page.
-    extra_files = ['index.css', 'index.js', 'button_close.png',
-                   'button_close_hover.png']
-    for filename in extra_files:
-      src_file = os.path.join(example_resources_dir, filename)
-      dst_file = os.path.join(options.dstroot, 'examples', filename)
-      buildbot_common.CopyFile(src_file, dst_file)
-
-  if options.master:
-    if use_gyp:
-      master_in = os.path.join(SDK_EXAMPLE_DIR, 'Makefile_gyp')
-    else:
-      master_in = os.path.join(SDK_EXAMPLE_DIR, 'Makefile')
-    for dest, projects in master_projects.iteritems():
-      master_out = os.path.join(options.dstroot, dest, 'Makefile')
-      GenerateMasterMakefile(master_in, master_out, projects)
-
-  return 0
-
-
-if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
-
diff --git a/native_client_sdk/src/build_tools/parse_dsc.py b/native_client_sdk/src/build_tools/parse_dsc.py
new file mode 100644
index 0000000..6b900bd
--- /dev/null
+++ b/native_client_sdk/src/build_tools/parse_dsc.py
@@ -0,0 +1,235 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import fnmatch
+import optparse
+import os
+import sys
+
+# 'KEY' : ( <TYPE>, [Accepted Values], <Required?>)
+DSC_FORMAT = {
+    'DISABLE': (bool, [True, False], False),
+    'TOOLS' : (list, ['newlib:arm', 'newlib:x64', 'newlib:x86', 'newlib',
+                      'glibc', 'pnacl', 'win', 'linux'], True),
+    'CONFIGS' : (list, ['Debug', 'Release'], False),
+    'PREREQ' : (list, '', False),
+    'TARGETS' : (list, {
+        'NAME': (str, '', True),
+        # main = nexe target
+        # lib = library target
+        # so = shared object target, automatically added to NMF
+        # so-standalone =  shared object target, not put into NMF
+        'TYPE': (str, ['main', 'lib', 'so', 'so-standalone'], True),
+        'SOURCES': (list, '', True),
+        'CCFLAGS': (list, '', False),
+        'CXXFLAGS': (list, '', False),
+        'DEFINES': (list, '', False),
+        'LDFLAGS': (list, '', False),
+        'INCLUDES': (list, '', False),
+        'LIBS' : (list, '', False),
+        'DEPS' : (list, '', False)
+    }, True),
+    'HEADERS': (list, {
+        'FILES': (list, '', True),
+        'DEST': (str, '', True),
+    }, False),
+    'SEARCH': (list, '', False),
+    'POST': (str, '', False),
+    'PRE': (str, '', False),
+    'DEST': (str, ['examples', 'src', 'testlibs', 'tests'], True),
+    'NAME': (str, '', False),
+    'DATA': (list, '', False),
+    'TITLE': (str, '', False),
+    'GROUP': (str, '', False),
+    'EXPERIMENTAL': (bool, [True, False], False)
+}
+
+def IgnoreMsgFunc(test):
+  pass
+
+
+def ErrorMsgFunc(text):
+  sys.stderr.write(text + '\n')
+
+
+def ValidateFormat(src, dsc_format, ErrorMsg=ErrorMsgFunc):
+  failed = False
+
+  # Verify all required keys are there
+  for key in dsc_format:
+    (exp_type, exp_value, required) = dsc_format[key]
+    if required and key not in src:
+      ErrorMsg('Missing required key %s.' % key)
+      failed = True
+
+  # For each provided key, verify it's valid
+  for key in src:
+    # Verify the key is known
+    if key not in dsc_format:
+      ErrorMsg('Unexpected key %s.' % key)
+      failed = True
+      continue
+
+    exp_type, exp_value, required = dsc_format[key]
+    value = src[key]
+
+    # Verify the key is of the expected type
+    if exp_type != type(value):
+      ErrorMsg('Key %s expects %s not %s.' % (
+          key, exp_type.__name__.upper(), type(value).__name__.upper()))
+      failed = True
+      continue
+
+    # Verify the value is non-empty if required
+    if required and not value:
+      ErrorMsg('Expected non-empty value for %s.' % key)
+      failed = True
+      continue
+
+    # If it's a bool, the expected values are always True or False.
+    if exp_type is bool:
+      continue
+
+    # If it's a string and there are expected values, make sure it matches
+    if exp_type is str:
+      if type(exp_value) is list and exp_value:
+        if value not in exp_value:
+          ErrorMsg("Value '%s' not expected for %s." % (value, key))
+          failed = True
+      continue
+
+    # if it's a list, then we need to validate the values
+    if exp_type is list:
+      # If we expect a dictionary, then call this recursively
+      if type(exp_value) is dict:
+        for val in value:
+          if not ValidateFormat(val, exp_value, ErrorMsg):
+            failed = True
+        continue
+      # If we expect a list of strings
+      if type(exp_value) is str:
+        for val in value:
+          if type(val) is not str:
+            ErrorMsg('Value %s in %s is not a string.' % (val, key))
+            failed = True
+        continue
+      # if we expect a particular string
+      if type(exp_value) is list:
+        for val in value:
+          if val not in exp_value:
+            ErrorMsg('Value %s not expected in %s.' % (val, key))
+            failed = True
+        continue
+
+    # If we got this far, it's an unexpected type
+    ErrorMsg('Unexpected type %s for key %s.' % (str(type(src[key])), key))
+    continue
+  return not failed
+
+
+def LoadProject(filename, ErrorMsg=ErrorMsgFunc, verbose=False):
+  if verbose:
+    errmsg = ErrorMsgFunc
+  else:
+    errmsg = IgnoreMsgFunc
+
+  with open(filename, 'r') as descfile:
+    desc = eval(descfile.read(), {}, {})
+    if desc.get('DISABLE', False):
+      return None
+    if not ValidateFormat(desc, DSC_FORMAT, errmsg):
+      ErrorMsg('Failed to validate: ' + filename)
+      return None
+    desc['FILEPATH'] = os.path.abspath(filename)
+  return desc
+
+
+def AcceptProject(desc, filters):
+  # Check if we should filter node this on toolchain
+  if not filters:
+    return True
+
+  for key, expected in filters.iteritems():
+    # For any filtered key which is unspecified, assumed False
+    value = desc.get(key, False)
+
+    # If we provide an expected list, match at least one
+    if type(expected) != list:
+      expected = set([expected])
+    if type(value) != list:
+      value = set([value])
+
+    if not set(expected) & set(value):
+      return False
+
+  # If we fall through, then we matched the filters
+  return True
+
+
+def PruneTree(tree, filters):
+  out = collections.defaultdict(list)
+  for branch, projects in tree.iteritems():
+    for desc in projects:
+      if AcceptProject(desc, filters):
+        out[branch].append(desc)
+
+  return out
+
+
+def LoadProjectTree(srcpath, toolchains=None, verbose=False, filters=None,
+                    ErrorMsg=ErrorMsgFunc, InfoMsg=ErrorMsgFunc):
+  # Build the tree
+  out = collections.defaultdict(list)
+  for root, _, files in os.walk(srcpath):
+    for filename in files:
+      if fnmatch.fnmatch(filename, '*.dsc'):
+        filepath = os.path.join(root, filename)
+        desc = LoadProject(filepath, ErrorMsg)
+        if desc:
+          key = desc['DEST']
+          out[key].append(desc)
+
+  # Filter if needed
+  if filters:
+    out = PruneTree(out, filters)
+  return out
+
+
+def PrintProjectTree(tree, InfoMsg=ErrorMsgFunc):
+  for key in tree:
+    print key + ':'
+    for val in tree[key]:
+      print '\t' + val['NAME']
+
+
+def main(argv):
+  parser = optparse.OptionParser()
+  parser.add_option('-e', '--experimental',
+      help='build experimental examples and libraries', action='store_true')
+  parser.add_option('-t', '--toolchain',
+      help='Build using toolchain. Can be passed more than once.',
+      action='append')
+
+  options, files = parser.parse_args(argv[1:])
+  filters = {}
+
+  if len(files):
+    parser.error('Not expecting files.')
+    return 1
+
+  if options.toolchain:
+    filters['TOOLS'] = options.toolchain
+
+  if not options.experimental:
+    filters['EXPERIMENTAL'] = False
+
+  tree = LoadProjectTree('.', filters=filters)
+  PrintProjectTree(tree)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))
diff --git a/native_client_sdk/src/build_tools/template.mk b/native_client_sdk/src/build_tools/template.mk
index d240f1f..2ae0f6f 100644
--- a/native_client_sdk/src/build_tools/template.mk
+++ b/native_client_sdk/src/build_tools/template.mk
@@ -27,7 +27,7 @@ VALID_TOOLCHAINS:={{' '.join(tools)}}
 # If NACL_SDK_ROOT is not set, then assume it can be found relative to
 # to this Makefile.
 #
-NACL_SDK_ROOT?=$(abspath $(CURDIR)/../..)
+NACL_SDK_ROOT?=$(abspath $(CURDIR)/{{rel_sdk}})
 include $(NACL_SDK_ROOT)/tools/common.mk
 
 
diff --git a/native_client_sdk/src/build_tools/test_sdk.py b/native_client_sdk/src/build_tools/test_sdk.py
index 6e7a841..bd1223a 100755
--- a/native_client_sdk/src/build_tools/test_sdk.py
+++ b/native_client_sdk/src/build_tools/test_sdk.py
@@ -4,15 +4,15 @@
 # found in the LICENSE file.
 
 
-import copy
 import optparse
 import os
 import sys
 
 import buildbot_common
-import build_utils
+import build_projects
 import build_sdk
-import generate_make
+import build_version
+import parse_dsc
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
@@ -48,26 +48,19 @@ def BuildStepBuildExamples(pepperdir, platform):
 def BuildStepCopyTests(pepperdir, toolchains, build_experimental, clobber):
   buildbot_common.BuildStep('Copy Tests')
 
-  build_sdk.MakeDirectoryOrClobber(pepperdir, 'testlibs', clobber)
-  build_sdk.MakeDirectoryOrClobber(pepperdir, 'tests', clobber)
+  # Update test libraries and test apps
+  filters = {
+    'DEST': ['testlibs', 'tests']
+  }
+  if not build_experimental:
+    filters['EXPERIMENTAL'] = False
 
-  args = ['--dstroot=%s' % pepperdir, '--master']
-  for toolchain in toolchains:
-    args.append('--' + toolchain)
-
-  for library in TEST_LIBRARY_LIST:
-    dsc = os.path.join(SDK_LIBRARY_DIR, library, 'library.dsc')
-    args.append(dsc)
-
-  for example in TEST_EXAMPLE_LIST:
-    dsc = os.path.join(SDK_LIBRARY_DIR, example, 'example.dsc')
-    args.append(dsc)
-
-  if build_experimental:
-    args.append('--experimental')
+  tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, filters=filters)
+  platform = getos.GetPlatform()
+  build_projects.UpdateHelpers(pepperdir, platform, clobber=clobber)
+  build_projects.UpdateProjects(pepperdir, platform, tree, clobber=clobber,
+                                toolchains=toolchains)
 
-  if generate_make.main(args):
-    buildbot_common.ErrorExit('Failed to build tests.')
 
 
 def BuildStepBuildTests(pepperdir, platform):
@@ -77,52 +70,21 @@ def BuildStepBuildTests(pepperdir, platform):
                              deps=False)
 
 
-def BuildStepRunPyautoTests(pepperdir, platform, pepper_ver):
-  buildbot_common.BuildStep('Test Examples')
-  env = copy.copy(os.environ)
-  env['PEPPER_VER'] = pepper_ver
-  env['NACL_SDK_ROOT'] = pepperdir
-
-  pyauto_script = os.path.join(SRC_DIR, 'chrome', 'test',
-                               'functional', 'nacl_sdk.py')
-  pyauto_script_args = ['nacl_sdk.NaClSDKTest.NaClSDKExamples']
-
-  if platform == 'linux' and buildbot_common.IsSDKBuilder():
-    # linux buildbots need to run the pyauto tests through xvfb. Running
-    # using runtest.py does this.
-    #env['PYTHON_PATH'] = '.:' + env.get('PYTHON_PATH', '.')
-    build_dir = os.path.dirname(SRC_DIR)
-    runtest_py = os.path.join(build_dir, '..', '..', '..', 'scripts', 'slave',
-                              'runtest.py')
-    buildbot_common.Run([sys.executable, runtest_py, '--target', 'Release',
-                         '--build-dir', 'src/build', sys.executable,
-                         pyauto_script] + pyauto_script_args,
-                        cwd=build_dir, env=env)
-  else:
-    buildbot_common.Run([sys.executable, 'nacl_sdk.py',
-                         'nacl_sdk.NaClSDKTest.NaClSDKExamples'],
-                        cwd=os.path.dirname(pyauto_script),
-                        env=env)
-
-
 def main(args):
   parser = optparse.OptionParser()
   parser.add_option('--experimental', help='build experimental tests',
                     action='store_true')
-  parser.add_option('--pyauto', help='Run pyauto tests', action='store_true')
 
   options, args = parser.parse_args(args[1:])
 
   platform = getos.GetPlatform()
-  pepper_ver = str(int(build_utils.ChromeMajorVersion()))
+  pepper_ver = str(int(build_version.ChromeMajorVersion()))
   pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
   toolchains = ['newlib', 'glibc', 'pnacl', 'host']
 
   BuildStepBuildExamples(pepperdir, platform)
   BuildStepCopyTests(pepperdir, toolchains, options.experimental, True)
   BuildStepBuildTests(pepperdir, platform)
-  if options.pyauto:
-    BuildStepRunPyautoTests(pepperdir, platform, pepper_ver)
 
   return 0
 
diff --git a/native_client_sdk/src/build_tools/tests/generate_make_test.py b/native_client_sdk/src/build_tools/tests/parse_dsc_test.py
index cc8dd27..d6c484e 100755
--- a/native_client_sdk/src/build_tools/tests/generate_make_test.py
+++ b/native_client_sdk/src/build_tools/tests/parse_dsc_test.py
@@ -16,6 +16,7 @@ BUILD_TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
 
 sys.path.append(BUILD_TOOLS_DIR)
 import generate_make
+import parse_dsc
 
 BASIC_DESC = {
   'TOOLS': ['newlib', 'glibc'],
@@ -35,9 +36,9 @@ class TestValidateFormat(unittest.TestCase):
     return self.result
 
   def _validate(self, src, msg):
-    format = generate_make.DSC_FORMAT
+    format = parse_dsc.DSC_FORMAT
     self.result = ''
-    result = generate_make.ValidateFormat(src, format,
+    result = parse_dsc.ValidateFormat(src, format,
         lambda msg: self._append_result(msg))
     if msg:
       self.assertEqual(self.result, msg)
diff --git a/native_client_sdk/src/examples/Makefile b/native_client_sdk/src/examples/Makefile
index 9eb356d..0218488 100644
--- a/native_client_sdk/src/examples/Makefile
+++ b/native_client_sdk/src/examples/Makefile
@@ -12,6 +12,8 @@ PROJECTS:= \
   {{project}} \
 [[]]
 
+HTTPD:={{rel_sdk}}/tools/httpd.py
+
 # Define the default target
 all:
 
@@ -54,4 +56,4 @@ clean: $(CLEAN_LIST)
 .PHONY: RUN
 RUN: all
 	@echo Starting up python webserver.
-	python ../tools/httpd.py
+	python $(HTTPD)
diff --git a/native_client_sdk/src/examples/fullscreen_tumbler/example.dsc b/native_client_sdk/src/examples/fullscreen_tumbler/example.dsc
index 7ff58d2..bad2219 100644
--- a/native_client_sdk/src/examples/fullscreen_tumbler/example.dsc
+++ b/native_client_sdk/src/examples/fullscreen_tumbler/example.dsc
@@ -1,4 +1,5 @@
 {
+  'DISABLE': True,
   'TOOLS': ['newlib', 'glibc', 'pnacl'],
   'TARGETS': [
     {
diff --git a/native_client_sdk/src/examples/pong/example.dsc b/native_client_sdk/src/examples/pong/example.dsc
index cd4dd4e..d7b0d64 100644
--- a/native_client_sdk/src/examples/pong/example.dsc
+++ b/native_client_sdk/src/examples/pong/example.dsc
@@ -1,4 +1,5 @@
 {
+  'DISABLE': True,
   'TOOLS': ['newlib', 'glibc', 'pnacl'],
   'TARGETS': [
     {
diff --git a/native_client_sdk/src/libraries/libjpeg/library.dsc b/native_client_sdk/src/libraries/libjpeg/library.dsc
index 0d40f3c..d5dd5c3 100644
--- a/native_client_sdk/src/libraries/libjpeg/library.dsc
+++ b/native_client_sdk/src/libraries/libjpeg/library.dsc
@@ -1,4 +1,5 @@
 {
+  'DISABLE': True,
   'TOOLS': ['newlib', 'glibc', 'linux', 'win'],
   'SEARCH': [
     '../../../../third_party/libjpeg',
diff --git a/native_client_sdk/src/libraries/zlib/library.dsc b/native_client_sdk/src/libraries/zlib/library.dsc
index 2ce423a..0354799 100644
--- a/native_client_sdk/src/libraries/zlib/library.dsc
+++ b/native_client_sdk/src/libraries/zlib/library.dsc
@@ -1,4 +1,5 @@
 {
+  'DISABLE': True,
   'TOOLS': ['newlib', 'glibc', 'linux', 'win'],
   'SEARCH': [
     '../../../../third_party/zlib',
diff --git a/native_client_sdk/src/test_all.py b/native_client_sdk/src/test_all.py
index 4c37c78..0a1374f 100755
--- a/native_client_sdk/src/test_all.py
+++ b/native_client_sdk/src/test_all.py
@@ -21,7 +21,7 @@ TEST_MODULES = [
     'sdktools_test',
     'sdktools_commands_test',
     'update_nacl_manifest_test',
-    'generate_make_test'
+    'parse_dsc_test'
 ]
 
 def main():