Pulling in latest software construction toolkit.

Review URL: http://codereview.chromium.org/8117

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@3861 0039d316-1c4b-4281-b951-d872f2087c98

15 files changed, 560 insertions, 106 deletions

diff --git a/site_scons/site_init.py b/site_scons/site_init.py
index 38a52e2..979cd53 100644
--- a/site_scons/site_init.py
+++ b/site_scons/site_init.py
@@ -42,7 +42,7 @@ import SCons
 
 
 # List of target groups for printing help; modified by AddTargetGroup(); used
-# by BuildComponents().
+# by BuildEnvironments().
 __target_groups = {}
 
 
@@ -51,8 +51,9 @@ def _HostPlatform():
 
   That is, the platform we're actually running SCons on.  You shouldn't use
   this inside your SConscript files; instead, include the appropriate
-  target_platform tool for your environments.  When you call BuildComponents(),
-  only environments with the current host platform will be built.
+  target_platform tool for your environments.  When you call
+  BuildEnvironments(), only environments with the current host platform will be
+  built.
 
   Returns:
     The host platform name - one of ('WINDOWS', 'LINUX', 'MAC').
@@ -179,6 +180,8 @@ def _AddTargetHelp():
     if items:
       colwidth = max(map(len, items)) + 2
       cols = 77 / colwidth
+      if cols < 1:
+          cols = 1      # If target names are really long, one per line
       rows = (len(items) + cols - 1) / cols
       items.sort()
       if xml_help:
@@ -199,8 +202,8 @@ def _AddTargetHelp():
 #------------------------------------------------------------------------------
 
 
-def BuildComponents(environments):
-  """Build a collection of components under a collection of environments.
+def BuildEnvironments(environments):
+  """Build a collection of SConscripts under a collection of environments.
 
   Only environments with HOST_PLATFORMS containing the platform specified by
   --host-platform (or the native host platform, if --host-platform was not
@@ -209,11 +212,14 @@ def BuildComponents(environments):
   Each matching environment is checked against the modes passed to the --mode
   command line argument (or 'default', if no mode(s) were specified).  If any
   of the modes match the environment's BUILD_TYPE or any of the environment's
-  BUILD_GROUPS, all the BUILD_COMPONENTS and BUILD_SCONSCRIPTS in that
-  environment will be built.
+  BUILD_GROUPS, all the BUILD_SCONSCRIPTS (and for legacy reasons,
+  BUILD_COMPONENTS) in that environment will be built.
 
   Args:
     environments: List of SCons environments.
+
+  Returns:
+    List of environments which were actually evaluated (built).
   """
   # Get options
   xml_help = SCons.Script.GetOption('xml_help')
@@ -235,45 +241,49 @@ def BuildComponents(environments):
   if xml_help:
     SCons.Script.Help('<help_from_sconscripts>\n<![CDATA[\n')
 
+  environments_to_evaluate = []
   for e in environments:
     if not e.Overlap(e['HOST_PLATFORMS'], [host_platform, '*']):
       continue      # Environment requires a host platform which isn't us
 
     if e.Overlap([e['BUILD_TYPE'], e['BUILD_GROUPS']], build_modes):
-      # Set up for deferred functions and published resources
-      e._InitializeComponentBuilders()
-      e._InitializeDefer()
-      e._InitializePublish()
-
-      # Read SConscript for each component
-      # TODO(rspangler): Remove BUILD_COMPONENTS once all projects have
-      # transitioned to the BUILD_SCONSCRIPTS nomenclature.
-      for c in e.get('BUILD_COMPONENTS', []) + e.get('BUILD_SCONSCRIPTS', []):
-        # Clone the environment so components can't interfere with each other
-        ec = e.Clone()
-
-        if ec.Entry(c).isdir():
-          # The component is a directory, so assume it contains a SConscript
-          # file.
-          c_dir = ec.Dir(c)
-
-          # Use 'build.scons' as the default filename, but if that doesn't
-          # exist, fall back to 'SConscript'.
-          c_script = c_dir.File('build.scons')
-          if not c_script.exists():
-            c_script = c_dir.File('SConscript')
-        else:
-          # The component is a SConscript file.
-          c_script = ec.File(c)
-          c_dir = c_script.dir
+      environments_to_evaluate.append(e)
+
+  for e in environments_to_evaluate:
+    # Set up for deferred functions and published resources
+    e._InitializeComponentBuilders()
+    e._InitializeDefer()
+    e._InitializePublish()
+
+    # Read SConscript for each component
+    # TODO(rspangler): Remove BUILD_COMPONENTS once all projects have
+    # transitioned to the BUILD_SCONSCRIPTS nomenclature.
+    for c in e.get('BUILD_COMPONENTS', []) + e.get('BUILD_SCONSCRIPTS', []):
+      # Clone the environment so components can't interfere with each other
+      ec = e.Clone()
+
+      if ec.Entry(c).isdir():
+        # The component is a directory, so assume it contains a SConscript
+        # file.
+        c_dir = ec.Dir(c)
+
+        # Use 'build.scons' as the default filename, but if that doesn't
+        # exist, fall back to 'SConscript'.
+        c_script = c_dir.File('build.scons')
+        if not c_script.exists():
+          c_script = c_dir.File('SConscript')
+      else:
+        # The component is a SConscript file.
+        c_script = ec.File(c)
+        c_dir = c_script.dir
 
-        ec.SConscript(c_script,
-                      build_dir='$OBJ_ROOT/' + str(c_dir),
-                      exports={'env': ec},
-                      duplicate=0)
+      ec.SConscript(c_script,
+                    build_dir='$OBJ_ROOT/' + str(c_dir),
+                    exports={'env': ec},
+                    duplicate=0)
 
-      # Execute deferred functions
-      e._ExecuteDefer()
+    # Execute deferred functions
+    e._ExecuteDefer()
 
   if xml_help:
     SCons.Script.Help(']]>\n</help_from_sconscripts>\n')
@@ -284,6 +294,9 @@ def BuildComponents(environments):
   if xml_help:
     SCons.Script.Help('</help>\n')
 
+  # Return list of environments actually evaluated
+  return environments_to_evaluate
+
 
 #------------------------------------------------------------------------------
 
@@ -365,10 +378,19 @@ Additional options for SCons:
 def SiteInitMain():
   """Main code executed in site_init."""
 
+  # Bail out if we've been here before. This is needed to handle the case where
+  # this site_init.py has been dropped into a project directory.
+  if hasattr(__builtin__, 'BuildEnvironments'):
+    return
+
   # Let people use new global methods directly.
   __builtin__.AddSiteDir = AddSiteDir
-  __builtin__.BuildComponents = BuildComponents
+  __builtin__.BuildEnvironments = BuildEnvironments
   __builtin__.AddTargetGroup = AddTargetGroup
+  # Legacy method names
+  # TODO(rspangler): Remove these once they're no longer used anywhere.
+  __builtin__.BuildComponents = BuildEnvironments
+
 
   # Set list of default tools for component_setup
   __builtin__.component_setup_tools = [
diff --git a/site_scons/site_tools/atlmfc_vc80.py b/site_scons/site_tools/atlmfc_vc80.py
index d6e31df..8d3d4b0 100644
--- a/site_scons/site_tools/atlmfc_vc80.py
+++ b/site_scons/site_tools/atlmfc_vc80.py
@@ -31,7 +31,7 @@
 """Windows ATL MFC for VC80 (Visual Studio 2005) tool for SCons.
 
 Note that ATL MFC requires the commercial (non-free) version of Visual Studio
-2005.  Using this in an open-source project thus limits the size of the
+2005.  Using this in an open-source project thus limits the size of the 
 developer community to those with the commercial version.
 """
 
@@ -40,7 +40,7 @@ import os
 
 def _FindLocalInstall():
   """Returns the directory containing the local install of the tool.
-
+  
   Returns:
     Path to tool (as a string), or None if not found.
   """
@@ -59,19 +59,22 @@ def generate(env):
 
   if not env.get('ATLMFC_VC80_DIR'):
     env['ATLMFC_VC80_DIR'] = _FindLocalInstall()
-
-  env.AppendENVPath('INCLUDE', env.Dir('$ATLMFC_VC80_DIR/include').abspath)
-  env.AppendENVPath('LIB', env.Dir('$ATLMFC_VC80_DIR/lib').abspath)
+  
+  env.Append(
+      LIBPATH=['$ATLMFC_VC80_DIR/lib'],
+      RCFLAGS=['/I"$ATLMFC_VC80_DIR/include"'],
+      CCFLAGS=['/I"$ATLMFC_VC80_DIR/include"'],
+  )
 
 
 def exists(env):
   """Returns true if tool exists."""
   # NOTE: SCons requires the use of this name, which fails gpylint.
-
+  
   # If directory explicitly specified, we exist
   if env.get('ATLMFC_VC80_DIR'):
     return 1
   elif _FindLocalInstall():
     return 1
   else:
-    return 0
+    return 0  
diff --git a/site_scons/site_tools/collada_dom.py b/site_scons/site_tools/collada_dom.py
index bafccd9..2373ea1 100644
--- a/site_scons/site_tools/collada_dom.py
+++ b/site_scons/site_tools/collada_dom.py
@@ -41,6 +41,7 @@ def generate(env):
   # TODO(rspangler): Should this really be a bit, or should we have a
   # COLLADA_DOM_VERSION variable
   env['COLLADA_DOM_VERSION'] = '1.3.0'
+
   env.Append(CPPDEFINES=['COLLADA_DOM_130'])
 
   if sys.platform in ('win32', 'cygwin'):
diff --git a/site_scons/site_tools/command_output.py b/site_scons/site_tools/command_output.py
index ce702b7..795f3eb 100644
--- a/site_scons/site_tools/command_output.py
+++ b/site_scons/site_tools/command_output.py
@@ -32,11 +32,69 @@
 
 
 import os
-import SCons.Script
+import signal
 import subprocess
+import sys
+import threading
+import time
+import SCons.Script
+
+
+# TODO(rspangler): Move KillProcessTree() and RunCommand() into their own
+# module.
+
+
+def KillProcessTree(pid):
+  """Kills the process and all of its child processes.
+
+  Args:
+    pid: process to kill.
+
+  Raises:
+    OSError: Unsupported OS.
+  """
 
+  if sys.platform in ('win32', 'cygwin'):
+    # Use Windows' taskkill utility
+    killproc_path = '%s;%s\\system32;%s\\system32\\wbem' % (
+        (os.environ['SYSTEMROOT'],) * 3)
+    killproc_cmd = 'taskkill /F /T /PID %d' % pid
+    killproc_task = subprocess.Popen(killproc_cmd, shell=True,
+                                     stdout=subprocess.PIPE,
+                                     env={'PATH':killproc_path})
+    killproc_task.communicate()
+
+  elif sys.platform in ('linux', 'linux2', 'darwin'):
+    # Use ps to get a list of processes
+    ps_task = subprocess.Popen(['/bin/ps', 'x', '-o', 'pid,ppid'], stdout=subprocess.PIPE)
+    ps_out = ps_task.communicate()[0]
+
+    # Parse out a dict of pid->ppid
+    ppid = {}
+    for ps_line in ps_out.split('\n'):
+      w = ps_line.strip().split()
+      if len(w) < 2:
+          continue      # Not enough words in this line to be a process list
+      try:
+        ppid[int(w[0])] = int(w[1])
+      except ValueError:
+        pass            # Header or footer
+
+    # For each process, kill it if it or any of its parents is our child
+    for p in ppid:
+      p2 = p
+      while p2:
+        if p2 == pid:
+          os.kill(p, signal.SIGKILL)
+          break
+        p2 = ppid.get(p2)
 
-def RunCommand(cmdargs, cwdir=None, env=None, echo_output=True):
+  else:
+    raise OSError('Unsupported OS for KillProcessTree()')
+
+
+def RunCommand(cmdargs, cwdir=None, env=None, echo_output=True, timeout=None,
+               timeout_errorlevel=14):
   """Runs an external command.
 
   Args:
@@ -45,28 +103,62 @@ def RunCommand(cmdargs, cwdir=None, env=None, echo_output=True):
     cwdir: Working directory for the command, if not None.
     env: Environment variables dict, if not None.
     echo_output: If True, output will be echoed to stdout.
+    timeout: If not None, timeout for command in seconds.  If command times
+        out, it will be killed and timeout_errorlevel will be returned.
+    timeout_errorlevel: The value to return if the command times out.
 
   Returns:
     The integer errorlevel from the command.
     The combined stdout and stderr as a string.
   """
-
   # Force unicode string in the environment to strings.
   if env:
     env = dict([(k, str(v)) for k, v in env.items()])
+  start_time = time.time()
   child = subprocess.Popen(cmdargs, cwd=cwdir, env=env, shell=True,
+                           universal_newlines=True,
                            stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
   child_out = []
   child_retcode = None
 
-  # Need to poll the child process, since the stdout pipe can fill.
+  def _ReadThread():
+    """Thread worker function to read output from child process.
+
+    Necessary since there is no cross-platform way of doing non-blocking
+    reads of the output pipe.
+    """
+    read_run = True
+    while read_run:
+      # Need to have a delay of 1 cycle between child completing and
+      # thread exit, to pick up the final output from the child.
+      if child_retcode is not None:
+        read_run = False
+      new_out = child.stdout.read()
+      if new_out:
+        if echo_output:
+          print new_out,
+        child_out.append(new_out)
+
+  read_thread = threading.Thread(target=_ReadThread)
+  read_thread.start()
+
+  # Wait for child to exit or timeout
   while child_retcode is None:
+    time.sleep(1)       # So we don't poll too frequently
     child_retcode = child.poll()
-    new_out = child.stdout.read()
-    if echo_output:
-      print new_out,
-    child_out.append(new_out)
+    if timeout and child_retcode is None:
+      elapsed = time.time() - start_time
+      if elapsed > timeout:
+        print '*** RunCommand() timeout:', cmdargs
+        KillProcessTree(child.pid)
+        child_retcode = timeout_errorlevel
+
+  # Wait for worker thread to pick up final output and die
+  read_thread.join(5)
+  if read_thread.isAlive():
+    print '*** Error: RunCommand() read thread did not exit.'
+    sys.exit(1)
 
   if echo_output:
     print   # end last line of output
@@ -100,8 +192,13 @@ def CommandOutputBuilder(target, source, env):
     env.AppendENVPath('LD_LIBRARY_PATH', cwdir)
   else:
     cwdir = None
+  cmdecho = env.get('COMMAND_OUTPUT_ECHO', True)
+  timeout = env.get('COMMAND_OUTPUT_TIMEOUT')
+  timeout_errorlevel = env.get('COMMAND_OUTPUT_TIMEOUT_ERRORLEVEL')
 
-  retcode, output = RunCommand(cmdline, cwdir=cwdir, env=env['ENV'])
+  retcode, output = RunCommand(cmdline, cwdir=cwdir, env=env['ENV'],
+                               echo_output=cmdecho, timeout=timeout,
+                               timeout_errorlevel=timeout_errorlevel)
 
   # Save command line output
   output_file = open(str(target[0]), 'w')
@@ -116,10 +213,17 @@ def generate(env):
   """SCons entry point for this tool."""
 
   # Add the builder and tell it which build environment variables we use.
-  action = SCons.Script.Action(CommandOutputBuilder, varlist=[
-      'COMMAND_OUTPUT_CMDLINE',
-      'COMMAND_OUTPUT_RUN_DIR',
-  ])
+  action = SCons.Script.Action(
+      CommandOutputBuilder,
+      'Output "$COMMAND_OUTPUT_CMDLINE" to $TARGET',
+      varlist=[
+          'COMMAND_OUTPUT_CMDLINE',
+          'COMMAND_OUTPUT_RUN_DIR',
+          'COMMAND_OUTPUT_TIMEOUT',
+          'COMMAND_OUTPUT_TIMEOUT_ERRORLEVEL',
+          # We use COMMAND_OUTPUT_ECHO also, but that doesn't change the
+          # command being run or its output.
+      ], )
   builder = SCons.Script.Builder(action = action)
   env.Append(BUILDERS={'CommandOutput': builder})
 
diff --git a/site_scons/site_tools/component_bits.py b/site_scons/site_tools/component_bits.py
index 94d8ab9..5c89a96 100644
--- a/site_scons/site_tools/component_bits.py
+++ b/site_scons/site_tools/component_bits.py
@@ -35,6 +35,7 @@ This module is automatically included by the component_setup tool.
 
 
 import __builtin__
+import types
 import SCons
 
 
@@ -45,14 +46,57 @@ _bit_exclusive_groups = {}
 #------------------------------------------------------------------------------
 
 
-def DeclareBit(bit_name, desc, exclusive_groups=tuple()):
+def _CheckDeclared(bits):
+  """Checks each of the bits to make sure it's been declared.
+
+  Args:
+    bits: List of bits to check.
+
+  Raises:
+    ValueError: A bit has not been declared.
+  """
+  for bit in bits:
+    if bit not in _bit_descriptions:
+      raise ValueError('Bit "%s" used before DeclareBit()' %
+                       bit)
+
+
+def _CheckExclusive(already_set, proposed):
+  """Checks if setting proposed bits would violate any exclusive groups.
+
+  Args:
+    already_set: List of bits already set.
+    proposed: List of bits attempting to be set.
+
+  Raises:
+    ValueError: A proposed bit belongs to an exclusive group which already has
+        a bit set.
+  """
+  # Remove any already-set bits from proposed (note this makes a copy of
+  # proposed so we don't alter the passed list).
+  proposed = [bit for bit in proposed if bit not in already_set]
+
+  for group_name, group_bits in _bit_exclusive_groups.items():
+    set_match = group_bits.intersection(already_set)
+    proposed_match = group_bits.intersection(proposed)
+    if set_match and proposed_match:
+      raise ValueError('Unable to set bit "%s" because it belongs to the same '
+                       'exclusive group "%s" as already-set bit "%s"' % (
+                       proposed_match.pop(), group_name, set_match.pop()))
+
+
+#------------------------------------------------------------------------------
+
+
+def DeclareBit(bit_name, desc, exclusive_groups=None):
   """Declares and describes the bit.
 
   Args:
     bit_name: Name of the bit being described.
     desc: Description of bit.
     exclusive_groups: Bit groups which this bit belongs to.  At most one bit
-        may be set in each exclusive group.
+        may be set in each exclusive group.  May be a string, list of string,
+        or None.
 
   Raises:
     ValueError: The bit has already been defined with a different description,
@@ -74,14 +118,16 @@ def DeclareBit(bit_name, desc, exclusive_groups=tuple()):
   _bit_descriptions[bit_name] = desc
 
   # Add bit to its exclusive groups
-  for g in exclusive_groups:
-    if g not in _bit_exclusive_groups:
-      _bit_exclusive_groups[g] = set()
-    _bit_exclusive_groups[g].add(bit_name)
+  if exclusive_groups:
+    if type(exclusive_groups) == types.StringType:
+      exclusive_groups = [exclusive_groups]
+    for g in exclusive_groups:
+      if g not in _bit_exclusive_groups:
+        _bit_exclusive_groups[g] = set()
+      _bit_exclusive_groups[g].add(bit_name)
 
 #------------------------------------------------------------------------------
 
-
 def Bit(env, bit_name):
   """Checks if the environment has the bit.
 
@@ -92,8 +138,7 @@ def Bit(env, bit_name):
   Returns:
     True if the bit is present in the environment.
   """
-  # TODO(rspangler): Add bit sanity checking (description exists, exclusive
-  # groups not violated).
+  _CheckDeclared([bit_name])
   return bit_name in env['_BITS']
 
 #------------------------------------------------------------------------------
@@ -109,7 +154,7 @@ def AllBits(env, *args):
   Returns:
     True if every bit listed is present in the environment.
   """
-  # TODO(rspangler): Add bit sanity checking
+  _CheckDeclared(args)
   return set(args).issubset(env['_BITS'])
 
 #------------------------------------------------------------------------------
@@ -125,7 +170,7 @@ def AnyBits(env, *args):
   Returns:
     True if at least one bit listed is present in the environment.
   """
-  # TODO(rspangler): Add bit sanity checking
+  _CheckDeclared(args)
   return set(args).intersection(env['_BITS'])
 
 #------------------------------------------------------------------------------
@@ -138,7 +183,8 @@ def SetBits(env, *args):
     env: Environment to check.
     args: List of bit names to set.
   """
-  # TODO(rspangler): Add bit sanity checking
+  _CheckDeclared(args)
+  _CheckExclusive(env['_BITS'], args)
   env['_BITS'] = env['_BITS'].union(args)
 
 #------------------------------------------------------------------------------
@@ -151,7 +197,7 @@ def ClearBits(env, *args):
     env: Environment to check.
     args: List of bit names to set.
   """
-  # TODO(rspangler): Add bit sanity checking
+  _CheckDeclared(args)
   env['_BITS'] = env['_BITS'].difference(args)
 
 #------------------------------------------------------------------------------
@@ -165,7 +211,7 @@ def SetBitFromOption(env, bit_name, default):
     bit_name: Name of the bit to set from a command line option.
     default: Default value for bit if command line option is not present.
   """
-  # TODO(rspangler): Add bit sanity checking
+  _CheckDeclared([bit_name])
 
   # Add the command line option, if not already present
   if bit_name not in _bits_with_options:
diff --git a/site_scons/site_tools/component_builders.py b/site_scons/site_tools/component_builders.py
index ae3e336..00de6a7 100644
--- a/site_scons/site_tools/component_builders.py
+++ b/site_scons/site_tools/component_builders.py
@@ -119,6 +119,11 @@ def _ComponentPlatformSetup(env, builder_name, **kwargs):
   for k, v in kwargs.items():
     env[k] = v
 
+  # Add compiler flags for included headers, if any
+  env['INCLUDES'] = env.Flatten(env.subst_list(['$INCLUDES']))
+  for h in env['INCLUDES']:
+    env.Append(CCFLAGS = ['${CCFLAG_INCLUDE}%s' % h])
+
   # Call platform-specific component setup function, if any
   if env.get('COMPONENT_PLATFORM_SETUP'):
     env['COMPONENT_PLATFORM_SETUP'](env, builder_name)
@@ -144,8 +149,8 @@ def ComponentPackageDeferred(env):
 
   # Install program and resources
   all_outputs = []
-  components = _RetrieveComponents(package_name,
-                                   env.get('COMPONENT_PACKAGE_FILTER'))
+  filter = env.Flatten(env.subst_list('$COMPONENT_PACKAGE_FILTER'))
+  components = _RetrieveComponents(package_name, filter)
   for resource, dest_dir in env.get('COMPONENT_PACKAGE_RESOURCES').items():
     all_outputs += env.ReplicatePublished(dest_dir, components, resource)
 
@@ -210,9 +215,14 @@ def ComponentObject(self, *args, **kwargs):
 
   # Make appropriate object type
   if env.get('COMPONENT_STATIC'):
-    return env.StaticObject(*args, **kwargs)
+    o = env.StaticObject(*args, **kwargs)
   else:
-    return env.SharedObject(*args, **kwargs)
+    o = env.SharedObject(*args, **kwargs)
+
+  # Add dependencies on includes
+  env.Depends(o, env['INCLUDES'])
+
+  return o
 
 #------------------------------------------------------------------------------
 
@@ -238,6 +248,9 @@ def ComponentLibrary(self, lib_name, *args, **kwargs):
   else:
     lib_outputs = env.SharedLibrary(lib_name, *args, **kwargs)
 
+  # Add dependencies on includes
+  env.Depends(lib_outputs, env['INCLUDES'])
+
   # Scan library outputs for files we need to link against this library, and
   # files we need to run executables linked against this library.
   need_for_link = []
@@ -307,8 +320,25 @@ def ComponentTestProgramDeferred(env):
         COMMAND_OUTPUT_CMDLINE=env['COMPONENT_TEST_CMDLINE'],
         COMMAND_OUTPUT_RUN_DIR='$TESTS_DIR',
     )
-    test_out = env.CommandOutput(
-        '$TEST_OUTPUT_DIR/${PROGRAM_BASENAME}.out.txt', test_program)
+    test_out_name = '$TEST_OUTPUT_DIR/${PROGRAM_BASENAME}.out.txt'
+    if (env.GetOption('component_test_retest')
+        and env.File(test_out_name).exists()):
+      # Delete old test results, so test will rerun.
+      env.Execute(SCons.Script.Delete(test_out_name))
+
+    # Set timeout based on test size
+    timeout = env.get('COMPONENT_TEST_TIMEOUT')
+    if type(timeout) is dict:
+      timeout = timeout.get(env.get('COMPONENT_TEST_SIZE'))
+    if timeout:
+      env['COMMAND_OUTPUT_TIMEOUT'] = timeout
+
+    # Run the test.  Note that we need to refer to the file by name, so that
+    # SCons will recreate the file node after we've deleted it; if we used the
+    # env.File() we created in the if statement above, SCons would still think
+    # it exists and not rerun the test.
+    test_out = env.CommandOutput(test_out_name, test_program)
+
     # Running the test requires the test and its libs copied to the tests dir
     env.Depends(test_out, all_outputs)
     env.ComponentTestOutput('run_' + prog_name, test_out)
@@ -337,6 +367,9 @@ def ComponentTestProgram(self, prog_name, *args, **kwargs):
   # Call env.Program()
   out_nodes = env.Program(prog_name, *args, **kwargs)
 
+  # Add dependencies on includes
+  env.Depends(out_nodes, env['INCLUDES'])
+
   # Publish output
   env.Publish(prog_name, 'run', out_nodes[0])
   env.Publish(prog_name, 'debug', out_nodes[1:])
@@ -398,6 +431,9 @@ def ComponentProgram(self, prog_name, *args, **kwargs):
   # Call env.Program()
   out_nodes = env.Program(prog_name, *args, **kwargs)
 
+  # Add dependencies on includes
+  env.Depends(out_nodes, env['INCLUDES'])
+
   # Publish output
   env.Publish(prog_name, 'run', out_nodes[0])
   env.Publish(prog_name, 'debug', out_nodes[1:])
@@ -431,9 +467,22 @@ def ComponentTestOutput(self, test_name, nodes):
     Passthrough return code from env.Alias().
   """
 
-  # Add an alias for the test outputs, and add it to the right groups
+  # Add an alias for the test output
   a = self.Alias(test_name, nodes)
-  for group in self['COMPONENT_TEST_OUTPUT_GROUPS']:
+
+  groups = self.get('COMPONENT_TEST_OUTPUT_GROUPS')
+  if not groups:
+    # Output group not explicitly specified, so automatically add to groups
+    if self.get('COMPONENT_TEST_ENABLED'):
+      # Enabled tests go in all tests, and their size category
+      groups = ['run_all_tests']
+      if self.get('COMPONENT_TEST_SIZE'):
+        groups.append(self.subst('run_${COMPONENT_TEST_SIZE}_tests'))
+    else:
+      # Disabled tests only go in their group
+      groups = ['run_disabled_tests']
+
+  for group in groups:
     SCons.Script.Alias(group, a)
 
   # Return the output node
@@ -456,7 +505,14 @@ def generate(env):
       # COMPONENT_TEST_CMDLINE='${SOURCE.abspath}',
       # (it generates a SCons error)
       COMPONENT_TEST_CMDLINE='${PROGRAM_NAME}',
-      COMPONENT_STATIC=True,  # Static linking is a sensible default.
+      # Default test size is large
+      COMPONENT_TEST_SIZE='large',
+      # Default timeouts for component tests
+      COMPONENT_TEST_TIMEOUT={'large':900, 'medium':450, 'small':180},
+      # Tests are enabled by default
+      COMPONENT_TEST_ENABLED=True,
+      # Static linking is a sensible default
+      COMPONENT_STATIC=True,
       # Don't publish libraries to the staging dir by themselves by default.
       COMPONENT_LIBRARY_PUBLISH=False,
   )
@@ -469,7 +525,6 @@ def generate(env):
       COMPONENT_LIBRARY_GROUPS=['all_libraries'],
       COMPONENT_PROGRAM_GROUPS=['all_programs'],
       COMPONENT_TEST_PROGRAM_GROUPS=['all_test_programs'],
-      COMPONENT_TEST_OUTPUT_GROUPS=['run_all_tests'],
 
       # Additional components whose resources should be copied into program
       # directories, in addition to those from LIBS and the program itself.
@@ -493,6 +548,15 @@ def generate(env):
       },
   )
 
+  # Add command line option for retest
+  SCons.Script.AddOption(
+      '--retest',
+      dest='component_test_retest',
+      action='store_true',
+      help='force all tests to rerun')
+  SCons.Script.Help('  --retest                    '
+                    'Rerun specified tests, ignoring cached results.\n')
+
   # Add our pseudo-builder methods
   env.AddMethod(_InitializeComponentBuilders)
   env.AddMethod(_StoreComponents)
@@ -509,3 +573,8 @@ def generate(env):
   AddTargetGroup('all_test_programs', 'tests can be built')
   AddTargetGroup('all_packages', 'packages can be built')
   AddTargetGroup('run_all_tests', 'tests can be run')
+  AddTargetGroup('run_disabled_tests', 'tests are disabled')
+  AddTargetGroup('run_small_tests', 'small tests can be run')
+  AddTargetGroup('run_medium_tests', 'medium tests can be run')
+  AddTargetGroup('run_large_tests', 'large tests can be run')
+
diff --git a/site_scons/site_tools/concat_source.py b/site_scons/site_tools/concat_source.py
index 9a34f50..c67463b 100644
--- a/site_scons/site_tools/concat_source.py
+++ b/site_scons/site_tools/concat_source.py
@@ -60,7 +60,7 @@ def ConcatSourceBuilder(target, source, env):
 
     source_path = str(source_file)
 
-    if 'msvc' in env['TOOLS']:
+    if env.get('CC') == 'cl':
       # Add message pragma for nicer progress indication when building with
       # MSVC.
       output_lines.append('#pragma message("--%s")' % (
@@ -68,11 +68,9 @@ def ConcatSourceBuilder(target, source, env):
 
     output_lines.append('#include "%s"' % source_path)
 
-  # Need an EOL at the end of the file for more finicky build tools
-  output_lines.append('\n')
-
   output_file = open(str(target[0]), 'w')
-  output_file.write('\n'.join(output_lines))
+  # Need an EOL at the end of the file for more finicky build tools
+  output_file.write('\n'.join(output_lines) + '\n')
   output_file.close()
 
 
diff --git a/site_scons/site_tools/defer.py b/site_scons/site_tools/defer.py
index 166f314..ffd30da 100644
--- a/site_scons/site_tools/defer.py
+++ b/site_scons/site_tools/defer.py
@@ -34,7 +34,7 @@
 import os
 import sys
 import types
-
+import SCons.Errors
 
 __defer_groups = {}
 
@@ -73,11 +73,10 @@ def _ExecuteDefer(self):
       del __defer_groups[name]
       break
     if not did_work:
-      print 'Error in _ExecuteDefer: dependency cycle detected.'
+      errmsg = 'Error in _ExecuteDefer: dependency cycle detected.\n'
       for name, group in __defer_groups.items():
-        print '   %s after: %s' % (name, group.after)
-      # TODO(rspangler): should throw exception?
-      sys.exit(1)
+        errmsg += '   %s after: %s\n' % (name, group.after)
+      raise SCons.Errors.UserError(errmsg)
 
   # Restore directory
   os.chdir(oldcwd)
@@ -152,9 +151,7 @@ def Defer(self, *args, **kwargs):
       after.append(a.__name__)
     elif a is not None:
       # Deferring
-      # TODO(rspangler): should throw an exception
-      print ('Warning: Defer can only wait for functions or names; ignoring'
-             'after = ', a)
+      raise ValueError('Defer after=%r is not a function or name' % a)
 
   # Find the deferred function
   if name not in __defer_groups:
diff --git a/site_scons/site_tools/directx_9_0_c.py b/site_scons/site_tools/directx_9_0_c.py
index 535198a..a8f45bb 100644
--- a/site_scons/site_tools/directx_9_0_c.py
+++ b/site_scons/site_tools/directx_9_0_c.py
@@ -36,8 +36,10 @@ def generate(env):
   """SCons entry point for this tool."""
 
   env['DIRECTX9_DIR'] = '$DIRECTX9_0_C_DIR'
-  env.AppendENVPath('INCLUDE', env.Dir('$DIRECTX9_DIR/include').abspath)
-  env.AppendENVPath('LIB', env.Dir('$DIRECTX9_DIR/lib').abspath)
+  env.Append(
+      LIBPATH=['$DIRECTX9_DIR/lib'],
+      CCFLAGS=['/I$DIRECTX9_DIR/include'],
+  )
 
 
 def exists(env):
diff --git a/site_scons/site_tools/distcc.py b/site_scons/site_tools/distcc.py
index 8f79042..6f4b9fc 100644
--- a/site_scons/site_tools/distcc.py
+++ b/site_scons/site_tools/distcc.py
@@ -40,6 +40,7 @@ line.
 
 import optparse
 import os
+import sys
 from SCons.compat._scons_optparse import OptionConflictError
 import SCons.Script
 
@@ -84,6 +85,9 @@ def generate(env):
   for compiler_var in ('CC', 'CXX'):
     compiler = env.get(compiler_var)
     if compiler in distcc_compilers:
+      if sys.platform == 'darwin':
+        # On Mac, distcc requires the full path to the compiler
+        compiler = env.WhereIs(compiler)
       env[compiler_var] = '$DISTCC ' + compiler
 
 
diff --git a/site_scons/site_tools/gather_inputs.py b/site_scons/site_tools/gather_inputs.py
index cae733a..f299fed 100644
--- a/site_scons/site_tools/gather_inputs.py
+++ b/site_scons/site_tools/gather_inputs.py
@@ -64,12 +64,26 @@ def GatherInputs(env, target, groups=['.*'], exclude_pattern=None):
       for t in tgt:
         _FindSources(ptrns, t, all)
     else:
+      # Get key to use for tracking whether we've seen this node
+      target_abspath = None
+      if hasattr(tgt, 'abspath'):
+        # Use target's absolute path as the key
+        target_abspath = tgt.abspath
+        target_key = target_abspath
+      else:
+        # Hope node's representation is unique enough (the default repr
+        # contains a pointer to the target as a string).  This works for
+        # Alias() nodes.
+        target_key = repr(tgt)
+
       # Skip if we have been here before
-      if tgt.abspath in all: return
+      if target_key in all: return
       # Note that we have been here
-      all[tgt.abspath] = True
+      all[target_key] = True
       # Skip ones that match an exclude pattern, if we have one.
-      if exclude_pattern and exclude_pattern.match(tgt.abspath): return
+      if (exclude_pattern and target_abspath
+          and exclude_pattern.match(target_abspath)):
+        return
 
       # Handle non-leaf nodes recursively
       lst = tgt.children(scan=1)
@@ -81,8 +95,9 @@ def GatherInputs(env, target, groups=['.*'], exclude_pattern=None):
       for pattern, lst in ptrns.items():
         # Get real file (backed by repositories).
         rfile = tgt.rfile()
-        # Add to the list for the first pattern that matches.
-        if pattern.match(rfile.path):
+        # Add files to the list for the first pattern that matches (implicitly,
+        # don't add directories).
+        if rfile.isfile() and pattern.match(rfile.path):
           lst.append(rfile.abspath)
           break
 
diff --git a/site_scons/site_tools/sdl.py b/site_scons/site_tools/sdl.py
new file mode 100755
index 0000000..bc02ad7
--- /dev/null
+++ b/site_scons/site_tools/sdl.py
@@ -0,0 +1,176 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Simple DirectMedia Layer tool for SCons.
+
+This tool sets up an environment to use the SDL library.
+"""
+
+import os
+import sys
+import SCons.Script
+
+
+def _HermeticSDL(env):
+  """Set things up if sdl is hermetically setup somewhere."""
+
+  if sys.platform in ['win32', 'cygwin']:
+    env.SetDefault(
+        SDL_DIR='$SDL_HERMETIC_WINDOWS_DIR',
+        SDL_CPPPATH=['$SDL_DIR/include'],
+        SDL_LIBPATH=['$SDL_DIR/lib'],
+        SDL_LIBS=['SDL', 'SDLmain'],
+        SDL_FRAMEWORKPATH=[],
+        SDL_FRAMEWORKS=[],
+    )
+  elif sys.platform in ['darwin']:
+    env.SetDefault(
+        SDL_DIR='$SDL_HERMETIC_MAC_DIR',
+        SDL_CPPPATH=[
+            '$SDL_DIR/SDL.framework/Headers',
+        ],
+        SDL_LIBPATH=[],
+        SDL_LIBS=[],
+        SDL_FRAMEWORKPATH=['$SDL_DIR'],
+        SDL_FRAMEWORKS=['SDL', 'Cocoa'],
+    )
+  elif sys.platform in ['linux', 'linux2', 'posix']:
+    env.SetDefault(
+        SDL_DIR='$SDL_HERMETIC_LINUX_DIR',
+        SDL_CPPPATH='$SDL_DIR/include',
+        SDL_LIBPATH='$SDL_DIR/lib',
+        SDL_LIBS=['SDL', 'SDLmain'],
+        SDL_FRAMEWORKPATH=[],
+        SDL_FRAMEWORKS=[],
+    )
+  else:
+    env.SetDefault(
+        SDL_VALIDATE_PATHS=[
+            ('unsupported_platform',
+             ('Not supported on this platform.')),
+        ],
+        SDL_IS_MISSING=True,
+    )
+
+  if not env.get('SDL_IS_MISSING', False):
+    env.SetDefault(
+        SDL_VALIDATE_PATHS=[
+            ('$SDL_DIR',
+             ('You are missing a hermetic copy of SDL...')),
+        ],
+    )
+
+
+def _LocalSDL(env):
+  """Set things up if sdl is locally installed."""
+
+  if sys.platform in ['win32', 'cygwin']:
+    env.SetDefault(
+        SDL_DIR='c:/SDL-1.2.13',
+        SDL_CPPPATH='$SDL_DIR/include',
+        SDL_LIBPATH='$SDL_DIR/lib',
+        SDL_LIBS=['SDL', 'SDLmain'],
+        SDL_FRAMEWORKPATH=[],
+        SDL_FRAMEWORKS=[],
+        SDL_VALIDATE_PATHS=[
+            ('$SDL_DIR',
+             ('You are missing SDL-1.2.13 on your system.',
+              'It was supposed to be in: ${SDL_DIR}',
+              'You can download it from:',
+              '  http://www.libsdl.org/download-1.2.php')),
+        ],
+    )
+  elif sys.platform in ['darwin']:
+    env.SetDefault(
+        SDL_CPPPATH=['/Library/Frameworks/SDL.framework/Headers'],
+        SDL_LIBPATH=[],
+        SDL_LIBS=[],
+        SDL_FRAMEWORKPATH=[],
+        SDL_FRAMEWORKS=['SDL', 'Cocoa'],
+        SDL_VALIDATE_PATHS=[
+            ('/Library/Frameworks/SDL.framework/SDL',
+             ('You are missing the SDL framework on your system.',
+              'You can download it from:',
+              'http://www.libsdl.org/download-1.2.php')),
+        ],
+    )
+  elif sys.platform in ['linux', 'linux2', 'posix']:
+    env.SetDefault(
+        SDL_CPPPATH='/usr/include/SDL',
+        SDL_LIBPATH='/usr/lib',
+        SDL_LIBS=['SDL', 'SDLmain'],
+        SDL_FRAMEWORKPATH=[],
+        SDL_FRAMEWORKS=[],
+        SDL_VALIDATE_PATHS=[
+            ('/usr/lib/libSDL.so',
+             ('You are missing SDL on your system.',
+              'Run sudo apt-get install libsdl1.2-dev.')),
+        ],
+    )
+
+
+def generate(env):
+  # NOTE: SCons requires the use of this name, which fails gpylint.
+  """SCons entry point for this tool."""
+
+  # Allow the hermetic copy to be disabled on the command line.
+  sdl_mode = SCons.Script.ARGUMENTS.get('sdl', 'hermetic')
+  if sdl_mode == 'local':
+    _LocalSDL(env)
+  elif sdl_mode == 'hermetic':
+    _HermeticSDL(env)
+  elif sdl_mode == 'none':
+    return
+  else:
+    assert False
+
+  validate_paths = env['SDL_VALIDATE_PATHS']
+
+  if not validate_paths:
+    sys.stderr.write('*' * 77 + '\n')
+    sys.stderr.write('ERROR - SDL not supported on this platform.\n')
+    sys.stderr.write('*' * 77 + '\n')
+    sys.exit(-1)
+
+  for i in validate_paths:
+    if not os.path.exists(env.subst(i[0])):
+      sys.stderr.write('*' * 77 + '\n')
+      for j in i[1]:
+        sys.stderr.write(env.subst(j) + '\n')
+      sys.stderr.write('*' * 77 + '\n')
+      sys.exit(-1)
+
+  env.Append(
+      CPPPATH=['$SDL_CPPPATH'],
+      LIBPATH=['$SDL_LIBPATH'],
+      LIBS=['$SDL_LIBS'],
+      FRAMEWORKPATH=['$SDL_FRAMEWORKPATH'],
+      FRAMEWORKS=['$SDL_FRAMEWORKS'],
+  )
diff --git a/site_scons/site_tools/target_platform_linux.py b/site_scons/site_tools/target_platform_linux.py
index af66b5d..776b09c 100644
--- a/site_scons/site_tools/target_platform_linux.py
+++ b/site_scons/site_tools/target_platform_linux.py
@@ -72,6 +72,7 @@ def generate(env):
   env.Replace(
       TARGET_PLATFORM='LINUX',
       COMPONENT_PLATFORM_SETUP=ComponentPlatformSetup,
+      CCFLAG_INCLUDE='-include',     # Command line option to include a header
   )
 
   env.Append(
diff --git a/site_scons/site_tools/target_platform_mac.py b/site_scons/site_tools/target_platform_mac.py
index c1a82a9..3034f36 100644
--- a/site_scons/site_tools/target_platform_mac.py
+++ b/site_scons/site_tools/target_platform_mac.py
@@ -134,6 +134,7 @@ def generate(env):
   env.Replace(
       TARGET_PLATFORM='MAC',
       COMPONENT_PLATFORM_SETUP=ComponentPlatformSetup,
+      CCFLAG_INCLUDE='-include',     # Command line option to include a header
   )
 
   env.Append(
diff --git a/site_scons/site_tools/target_platform_windows.py b/site_scons/site_tools/target_platform_windows.py
index 8162812..f225435 100644
--- a/site_scons/site_tools/target_platform_windows.py
+++ b/site_scons/site_tools/target_platform_windows.py
@@ -103,7 +103,7 @@ def RunManifest(target, source, env, resource_num):
   """
 
   cmdline = env.subst(
-      'mt.exe -nologo -manifest ${TARGET}.manifest -outputresource:$TARGET;%d'
+      'mt.exe -nologo -manifest "$MANIFEST_FILE" -outputresource:"$TARGET";%d'
       % resource_num,
       target=target, source=source)
   print cmdline
@@ -179,7 +179,11 @@ def generate(env):
   # NOTE: SCons requires the use of this name, which fails gpylint.
   """SCons entry point for this tool."""
 
-  # Set up environment paths first
+  # Bring in the outside PATH, INCLUDE, and LIB if not blocked.
+  if not env.get('MSVC_BLOCK_ENVIRONMENT_CHANGES'):
+    env.AppendENVPath('PATH', os.environ.get('PATH', '[]'))
+    env.AppendENVPath('INCLUDE', os.environ.get('INCLUDE', '[]'))
+    env.AppendENVPath('LIB', os.environ.get('LIB', '[]'))
 
   # Load various Visual Studio related tools.
   env.Tool('as')
@@ -192,6 +196,14 @@ def generate(env):
   env.Tool('mslib')
   env.Tool('mslink')
 
+  # Find VC80_DIR if it isn't already set.
+  if not env.get('VC80_DIR'):
+    # Look in each directory in the path for cl.exe.
+    for p in env['ENV']['PATH'].split(os.pathsep):
+      # Use the directory two layers up if it exists.
+      if os.path.exists(os.path.join(p, 'cl.exe')):
+        env['VC80_DIR'] = os.path.dirname(os.path.dirname(p))
+
   # The msvc, mslink, and mslib tools search the registry for installed copies
   # of Visual Studio and prepends them to the PATH, INCLUDE, and LIB
   # environment variables.  Block these changes if necessary.
@@ -210,6 +222,8 @@ def generate(env):
       # A better rebuild command (actually cleans, then rebuild)
       MSVSREBUILDCOM=''.join(['$MSVSSCONSCOM -c "$MSVSBUILDTARGET" && ',
                               '$MSVSSCONSCOM "$MSVSBUILDTARGET"']),
+
+      CCFLAG_INCLUDE='/FI',     # Command line option to include a header
   )
 
   env.Append(
@@ -244,6 +258,7 @@ def generate(env):
   )
 
   # Add manifests to EXEs and DLLs
+  env['MANIFEST_FILE'] = '${TARGET}.manifest'  # To allow override.
   wait_action = SCons.Script.Action(WaitForWritable,
                                     lambda target, source, env: ''),
   env['LINKCOM'] = [