Create a "no compile" drivers script in python to unittest compile time asserts.

BUG=87341
TEST=enable some of the existing no-compile tests and run on try bots.

Review URL: http://codereview.chromium.org/7458012

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@100564 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 801 insertions, 102 deletions

diff --git a/base/base.gyp b/base/base.gyp
index e8da279..e78db0a 100644
--- a/base/base.gyp
+++ b/base/base.gyp
@@ -116,6 +116,7 @@
         'atomicops_unittest.cc',
         'base64_unittest.cc',
         'bind_unittest.cc',
+        'bind_unittest.nc',
         'bits_unittest.cc',
         'callback_unittest.cc',
         'command_line_unittest.cc',
@@ -245,6 +246,11 @@
         '../third_party/icu/icu.gyp:icui18n',
         '../third_party/icu/icu.gyp:icuuc',
       ],
+      'includes': ['../build/nocompile.gypi'],
+      'variables': {
+         # TODO(ajwong): Is there a way to autodetect this?
+        'module_dir': 'base'
+      },
       'conditions': [
         ['toolkit_uses_gtk==1', {
           'sources!': [
diff --git a/base/bind_unittest.cc b/base/bind_unittest.cc
index 333512a..8718d1d 100644
--- a/base/bind_unittest.cc
+++ b/base/bind_unittest.cc
@@ -251,6 +251,7 @@ TEST_F(BindTest, ArityTest) {
 
 // Function type support.
 //   - Normal function.
+//   - Normal function bound with non-refcounted first argument.
 //   - Method bound to non-const object.
 //   - Const method bound to non-const object.
 //   - Const method bound to const object.
@@ -265,12 +266,16 @@ TEST_F(BindTest, FunctionTypeSupport) {
   EXPECT_CALL(has_ref_, VoidConstMethod0()).Times(2);
 
   Closure normal_cb = Bind(&VoidFunc0);
+  Callback<NoRef*(void)> normal_non_refcounted_cb =
+      Bind(&PolymorphicIdentity<NoRef*>, &no_ref_);
+  normal_cb.Run();
+  EXPECT_EQ(&no_ref_, normal_non_refcounted_cb.Run());
+
   Closure method_cb = Bind(&HasRef::VoidMethod0, &has_ref_);
   Closure const_method_nonconst_obj_cb = Bind(&HasRef::VoidConstMethod0,
                                               &has_ref_);
   Closure const_method_const_obj_cb = Bind(&HasRef::VoidConstMethod0,
                                            const_has_ref_ptr_);
-  normal_cb.Run();
   method_cb.Run();
   const_method_nonconst_obj_cb.Run();
   const_method_const_obj_cb.Run();
@@ -580,107 +585,6 @@ TEST_F(BindTest, ArgumentCopies) {
 //
 // TODO(ajwong): Is there actually a way to test this?
 
-// No-compile tests. These should not compile. If they do, we are allowing
-// error-prone, or incorrect behavior in the callback system.  Uncomment the
-// tests to check.
-TEST_F(BindTest, NoCompile) {
-  // - Method bound to const-object.
-  //
-  // Only const methods should be allowed to work with const objects.
-  //
-  // Callback<void(void)> method_to_const_cb =
-  //     Bind(&HasRef::VoidMethod0, const_has_ref_ptr_);
-  // method_to_const_cb.Run();
-
-  // - Method bound to non-refcounted object.
-  // - Const Method bound to non-refcounted object.
-  //
-  // We require refcounts unless you have Unretained().
-  //
-  // Callback<void(void)> no_ref_cb =
-  //     Bind(&NoRef::VoidMethod0, &no_ref_);
-  // no_ref_cb.Run();
-  // Callback<void(void)> no_ref_const_cb =
-  //     Bind(&NoRef::VoidConstMethod0, &no_ref_);
-  // no_ref_const_cb.Run();
-
-  // - Unretained() used with a refcounted object.
-  //
-  // If the object supports refcounts, unretaining it in the callback is a
-  // memory management contract break.
-  // Callback<void(void)> unretained_cb =
-  //     Bind(&HasRef::VoidConstMethod0, Unretained(&has_ref_));
-  // unretained_cb.Run();
-
-  // - Const argument used with non-const pointer parameter of same type.
-  // - Const argument used with non-const pointer parameter of super type.
-  //
-  // This is just a const-correctness check.
-  //
-  // const Parent* const_parent_ptr;
-  // const Child* const_child_ptr;
-  // Callback<Parent*(void)> pointer_same_cb =
-  //     Bind(&PolymorphicIdentity<Parent*>, const_parent_ptr);
-  // pointer_same_cb.Run();
-  // Callback<Parent*(void)> pointer_super_cb =
-  //     Bind(&PolymorphicIdentity<Parent*>, const_child_ptr);
-  // pointer_super_cb.Run();
-
-  // - Construction of Callback<A> from Callback<B> if A is supertype of B.
-  //   Specific example: Callback<void(void)> a; Callback<int(void)> b; a = b;
-  //
-  // While this is technically safe, most people aren't used to it when coding
-  // C++ so if this is happening, it is almost certainly an error.
-  //
-  // Callback<int(void)> cb_a0 = Bind(&Identity, 1);
-  // Callback<void(void)> cb_b0 = cb_a0;
-
-  // - Assignment of Callback<A> from Callback<B> if A is supertype of B.
-  // See explanation above.
-  //
-  // Callback<int(void)> cb_a1 = Bind(&Identity, 1);
-  // Callback<void(void)> cb_b1;
-  // cb_a1 = cb_b1;
-
-  // - Functions with reference parameters, unsupported.
-  //
-  // First, non-const reference parameters are disallowed by the Google
-  // style guide. Seconds, since we are doing argument forwarding it becomes
-  // very tricky to avoid copies, maintain const correctness, and not
-  // accidentally have the function be modifying a temporary, or a copy.
-  //
-  // NoRefParent p;
-  // Callback<int(Parent&)> ref_arg_cb = Bind(&UnwrapNoRefParentRef);
-  // ref_arg_cb.Run(p);
-  // Callback<int(void)> ref_cb = Bind(&UnwrapNoRefParentRef, p);
-  // ref_cb.Run();
-
-  // - A method should not be bindable with an array of objects.
-  //
-  // This is likely not wanted behavior. We specifically check for it though
-  // because it is possible, depending on how you implement prebinding, to
-  // implicitly convert an array type to a pointer type.
-  //
-  // HasRef p[10];
-  // Callback<void(void)> method_bound_to_array_cb =
-  //     Bind(&HasRef::VoidConstMethod0, p);
-  // method_bound_to_array_cb.Run();
-
-  // - Refcounted types should not be bound as a raw pointer.
-  // HasRef for_raw_ptr;
-  // Callback<void(void)> ref_count_as_raw_ptr =
-  //     Bind(&VoidPolymorphic1<HasRef*>, &for_raw_ptr);
-
-  // - WeakPtrs cannot be bound to methods with return types.
-  // HasRef for_raw_ptr;
-  // WeakPtrFactory<NoRef> weak_factory(&no_ref_);
-  // Callback<int(void)> weak_ptr_with_non_void_return_type =
-  //     Bind(&NoRef::IntMethod0, weak_factory.GetWeakPtr());
-
-  // - Bind result cannot be assigned to Callbacks with a mismatching type.
-  // Closure callback_mismatches_bind_type = Bind(&VoidPolymorphic1<int>);
-}
-
 #if defined(OS_WIN)
 int __fastcall FastCallFunc(int n) {
   return n;
diff --git a/base/bind_unittest.nc b/base/bind_unittest.nc
new file mode 100644
index 0000000..2d07cbb
--- /dev/null
+++ b/base/bind_unittest.nc
@@ -0,0 +1,223 @@
+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/callback.h"
+#include "base/bind.h"
+
+namespace base {
+
+// Do not put everything inside an anonymous namespace.  If you do, many of the
+// helper function declarations will generate unused definition warnings unless
+// unused definition warnings.
+
+static const int kParentValue = 1;
+static const int kChildValue = 2;
+
+class NoRef {
+ public:
+  void VoidMethod0() {}
+  void VoidConstMethod0() const {}
+  int IntMethod0() { return 1; }
+};
+
+class HasRef : public NoRef {
+ public:
+  void AddRef(void) const {}
+  bool Release(void) const { return true; }
+};
+
+class Parent {
+ public:
+  void AddRef(void) const {}
+  void Release(void) const {}
+  virtual void VirtualSet() { value = kParentValue; }
+  void NonVirtualSet() { value = kParentValue; }
+  int value;
+};
+
+class Child : public Parent {
+ public:
+  virtual void VirtualSet() { value = kChildValue; }
+  void NonVirtualSet() { value = kChildValue; }
+};
+
+class NoRefParent {
+ public:
+  virtual void VirtualSet() { value = kParentValue; }
+  void NonVirtualSet() { value = kParentValue; }
+  int value;
+};
+
+class NoRefChild : public NoRefParent {
+  virtual void VirtualSet() { value = kChildValue; }
+  void NonVirtualSet() { value = kChildValue; }
+};
+
+template <typename T>
+T PolymorphicIdentity(T t) {
+  return t;
+}
+
+int UnwrapParentRef(Parent& p) {
+  return p.value;
+}
+
+template <typename T>
+void VoidPolymorphic1(T t) {
+}
+
+#if defined(NCTEST_METHOD_ON_CONST_OBJECT)  // [r"invalid conversion from 'const base::NoRef\*' to 'base::NoRef\*'"]
+
+// Method bound to const-object.
+//
+// Only const methods should be allowed to work with const objects.
+void WontCompile() {
+  HasRef has_ref;
+  const HasRef* const_has_ref_ptr_ = &has_ref;
+  Callback<void(void)> method_to_const_cb =
+      Bind(&HasRef::VoidMethod0, const_has_ref_ptr_);
+  method_to_const_cb.Run();
+}
+
+#elif defined(NCTEST_CONSTRUCTION_FROM_SUBTYPE)  // [r"conversion from 'base::Callback<int\(\)>' to non-scalar type 'base::Callback<void\(\)>'"]
+
+// Construction of Callback<A> from Callback<B> if A is supertype of B.
+//
+// While this is technically safe, most people aren't used to it when coding
+// C++ so if this is happening, it is almost certainly an error.
+void WontCompile() {
+  Callback<int(void)> cb_a = Bind(&PolymorphicIdentity<int>, 1);
+  Callback<void(void)> cb_b = cb_a;
+}
+
+#elif defined(NCTEST_ASSIGNMENT_FROM_SUBTYPE)  // [r"no match for 'operator=' in 'cb_a = cb_b'"]
+
+// Assignment of Callback<A> from Callback<B> if A is supertype of B.
+// See explanation for NCTEST_CONSTRUCTION_FROM_SUBTYPE
+void WontCompile() {
+  Callback<int(void)> cb_a = Bind(&PolymorphicIdentity<int>, 1);
+  Callback<void(void)> cb_b;
+  cb_a = cb_b;
+}
+
+#elif defined(NCTEST_METHOD_BIND_NEEDS_REFCOUNTED_OBJECT)  // [r"has no member named 'AddRef'"]
+
+// Method bound to non-refcounted object.
+//
+// We require refcounts unless you have Unretained().
+void WontCompile() {
+  NoRef no_ref;
+  Callback<void(void)> no_ref_cb =
+      Bind(&NoRef::VoidMethod0, &no_ref);
+  no_ref_cb.Run();
+}
+
+#elif defined(NCTEST_CONST_METHOD_NEEDS_REFCOUNTED_OBJECT)  // [r"has no member named 'AddRef'"]
+
+// Const Method bound to non-refcounted object.
+//
+// We require refcounts unless you have Unretained().
+void WontCompile() {
+  NoRef no_ref;
+  Callback<void(void)> no_ref_const_cb =
+      Bind(&NoRef::VoidConstMethod0, &no_ref);
+  no_ref_const_cb.Run();
+}
+
+#elif defined(NCTEST_CONST_POINTER)  // [r"invalid conversion from 'const base::NoRef\*' to 'base::NoRef\*'"]
+
+// Const argument used with non-const pointer parameter of same type.
+//
+// This is just a const-correctness check.
+void WontCompile() {
+  const NoRef* const_no_ref_ptr;
+  Callback<NoRef*(void)> pointer_same_cb =
+      Bind(&PolymorphicIdentity<NoRef*>, const_no_ref_ptr);
+  pointer_same_cb.Run();
+}
+
+#elif defined(NCTEST_CONST_POINTER_SUBTYPE)  // [r"'const base::NoRefParent\*' to 'base::NoRefParent\*'"]
+
+// Const argument used with non-const pointer parameter of super type.
+//
+// This is just a const-correctness check.
+void WontCompile() {
+  const NoRefChild* const_child_ptr;
+  Callback<NoRefParent*(void)> pointer_super_cb =
+    Bind(&PolymorphicIdentity<NoRefParent*>, const_child_ptr);
+  pointer_super_cb.Run();
+}
+
+#elif defined(DISABLED_NCTEST_DISALLOW_NON_CONST_REF_PARAM)  // [r"badstring"]
+// I think there's a type safety promotion issue here where we can pass a const
+// ref to a non const-ref function, or vice versa accidentally. Or we make a
+// copy accidentally. Check.
+
+// Functions with reference parameters, unsupported.
+//
+// First, non-const reference parameters are disallowed by the Google
+// style guide. Second, since we are doing argument forwarding it becomes
+// very tricky to avoid copies, maintain const correctness, and not
+// accidentally have the function be modifying a temporary, or a copy.
+void WontCompile() {
+  Parent p;
+  Callback<int(Parent&)> ref_arg_cb = Bind(&UnwrapParentRef);
+  ref_arg_cb.Run(p);
+}
+
+#elif defined(NCTEST_DISALLOW_BIND_TO_NON_CONST_REF_PARAM)  // [r"creating array with negative size"]
+
+// Binding functions with reference parameters, unsupported.
+//
+// See comment in NCTEST_DISALLOW_NON_CONST_REF_PARAM
+void WontCompile() {
+  Parent p;
+  Callback<int(void)> ref_cb = Bind(&UnwrapParentRef, p);
+  ref_cb.Run();
+}
+
+#elif defined(NCTEST_NO_IMPLICIT_ARRAY_PTR_CONVERSION)  // [r"creating array with negative size"]
+
+// A method should not be bindable with an array of objects.
+//
+// This is likely not wanted behavior. We specifically check for it though
+// because it is possible, depending on how you implement prebinding, to
+// implicitly convert an array type to a pointer type.
+void WontCompile() {
+  HasRef p[10];
+  Callback<void(void)> method_bound_to_array_cb =
+      Bind(&HasRef::VoidMethod0, p);
+  method_bound_to_array_cb.Run();
+}
+
+#elif defined(NCTEST_NO_RAW_PTR_FOR_REFCOUNTED_TYPES)  // [r"creating array with negative size"]
+
+// Refcounted types should not be bound as a raw pointer.
+void WontCompile() {
+  HasRef for_raw_ptr;
+  Callback<void(void)> ref_count_as_raw_ptr =
+      Bind(&VoidPolymorphic1<HasRef*>, &for_raw_ptr);
+}
+
+#elif defined(NCTEST_WEAKPTR_BIND_MUST_RETURN_VOID)  // [r"creating array with negative size"]
+
+// WeakPtrs cannot be bound to methods with return types.
+void WontCompile() {
+  NoRef no_ref;
+  WeakPtrFactory<NoRef> weak_factory(&no_ref);
+  Callback<int(void)> weak_ptr_with_non_void_return_type =
+      Bind(&NoRef::IntMethod0, weak_factory.GetWeakPtr());
+  weak_ptr_with_non_void_return_type.Run();
+}
+
+#elif defined(NCTEST_DISALLOW_ASSIGN_DIFFERINT_TYPES)  // [r"creating array with negative size"]
+
+// Bind result cannot be assigned to Callbacks with a mismatching type.
+void WontCompile() {
+  Closure callback_mismatches_bind_type = Bind(&VoidPolymorphic1<int>);
+}
+
+#endif
+
+}  // namespace base
diff --git a/build/nocompile.gypi b/build/nocompile.gypi
new file mode 100644
index 0000000..88dffc7
--- /dev/null
+++ b/build/nocompile.gypi
@@ -0,0 +1,94 @@
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file is meant to be included into an target to create a unittest that
+# invokes a set of no-compile tests.  A no-compile test is a test that asserts
+# a particular construct will not compile.
+#
+# Also see:
+#   http://dev.chromium.org/developers/testing/no-compile-tests
+#
+# To use this, create a gyp target with the following form:
+# {
+#   'target_name': 'my_module_nc_unittests',
+#   'type': 'executable',
+#   'sources': [
+#     'nc_testset_1.nc',
+#     'nc_testset_2.nc',
+#   ],
+#   'includes': ['path/to/this/gypi/file'],
+# }
+#
+# The .nc files are C++ files that contain code we wish to assert will not
+# compile.  Each individual test case in the file should be put in its own
+# #ifdef section.  The expected output should be appended with a C++-style
+# comment that has a python list of regular expressions.  This will likely
+# be greater than 80-characters. Giving a solid expected output test is
+# important so that random compile failures do not cause the test to pass.
+#
+# Example .nc file:
+#
+#   #if defined(TEST_NEEDS_SEMICOLON)  // [r"expected ',' or ';' at end of input"]
+#
+#   int a = 1
+#
+#   #elif defined(TEST_NEEDS_CAST)  // [r"invalid conversion from 'void*' to 'char*'"]
+#
+#   void* a = NULL;
+#   char* b = a;
+#
+#   #endif
+#
+# If we needed disable TEST_NEEDS_SEMICOLON, then change the define to:
+#
+#   DISABLE_TEST_NEEDS_SEMICOLON
+#   TEST_NEEDS_CAST
+#
+# The lines above are parsed by a regexp so avoid getting creative with the
+# formatting or ifdef logic; it will likely just not work.
+#
+# Implementation notes:
+# The .nc files are actually processed by a python script which executes the
+# compiler and generates a .cc file that is empty on success, or will have a
+# series of #error lines on failure, and a set of trivially passing gunit
+# TEST() functions on success. This allows us to fail at the compile step when
+# something goes wrong, and know during the unittest run that the test was at
+# least processed when things go right.
+
+{
+  'conditions': [
+    [ 'OS=="linux" and clang==0', {
+      'rules': [
+        {
+          'variables': {
+            'nocompile_driver': '<(DEPTH)/tools/nocompile_driver.py',
+            'nc_result_path': ('<(INTERMEDIATE_DIR)/<(module_dir)/'
+                               '<(RULE_INPUT_ROOT)_nc.cc'),
+           },
+          'rule_name': 'run_nocompile',
+          'extension': 'nc',
+          'inputs': [
+            '<(nocompile_driver)',
+          ],
+          'outputs': [
+            '<(nc_result_path)'
+          ],
+          'action': [
+            'python',
+            '<(nocompile_driver)',
+            '4', # number of compilers to invoke in parallel.
+            '<(RULE_INPUT_PATH)',
+            '-Wall -Werror -Wfatal-errors -I<(DEPTH)',
+            '<(nc_result_path)',
+            ],
+          'message': 'Generating no compile results for <(RULE_INPUT_PATH)',
+          'process_outputs_as_sources': 1,
+        },
+      ],
+    }, {
+      'sources/': [['exclude', '\\.nc$']]
+    }],  # 'OS=="linux" and clang=="0"'
+  ],
+}
+
diff --git a/tools/nocompile_driver.py b/tools/nocompile_driver.py
new file mode 100755
index 0000000..2c5b354
--- /dev/null
+++ b/tools/nocompile_driver.py
@@ -0,0 +1,472 @@
+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Implements a simple "negative compile" test for C++ on linux.
+
+Sometimes a C++ API needs to ensure that various usages cannot compile. To
+enable unittesting of these assertions, we use this python script to
+invoke gcc on a source file and assert that compilation fails.
+
+For more info, see:
+  http://dev.chromium.org/developers/testing/no-compile-tests
+"""
+
+import ast
+import locale
+import os
+import re
+import select
+import shlex
+import subprocess
+import sys
+import time
+
+
+# Matches lines that start with #if and have the substring TEST in the
+# conditional. Also extracts the comment.  This allows us to search for
+# lines like the following:
+#
+#   #ifdef NCTEST_NAME_OF_TEST  // [r'expected output']
+#   #if defined(NCTEST_NAME_OF_TEST)  // [r'expected output']
+#   #if NCTEST_NAME_OF_TEST  // [r'expected output']
+#   #elif NCTEST_NAME_OF_TEST  // [r'expected output']
+#   #elif DISABLED_NCTEST_NAME_OF_TEST  // [r'expected output']
+#
+# inside the unittest file.
+NCTEST_CONFIG_RE = re.compile(r'^#(?:el)?if.*\s+(\S*NCTEST\S*)\s*(//.*)?')
+
+
+# Matches and removes the defined() preprocesor predicate. This is useful
+# for test cases that use the preprocessor if-statement form:
+#
+#   #if defined(NCTEST_NAME_OF_TEST)
+#
+# Should be used to post-process the results found by NCTEST_CONFIG_RE.
+STRIP_DEFINED_RE = re.compile(r'defined\((.*)\)')
+
+
+# Used to grab the expectation from comment at the end of an #ifdef.  See
+# NCTEST_CONFIG_RE's comment for examples of what the format should look like.
+#
+# The extracted substring should be a python array of regular expressions.
+EXTRACT_EXPECTATION_RE = re.compile(r'//\s*(\[.*\])')
+
+
+# The header for the result file so that it can be compiled.
+RESULT_FILE_HEADER = """
+// This file is generated by the no compile test from:
+//   %s
+
+#include "base/logging.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+"""
+
+
+# The GUnit test function to output on a successful test completion.
+SUCCESS_GUNIT_TEMPLATE = """
+TEST(%s, %s) {
+  LOG(INFO) << "Took %f secs. Started at %f, ended at %f";
+}
+"""
+
+# The GUnit test function to output for a disabled test.
+DISABLED_GUNIT_TEMPLATE = """
+TEST(%s, %s) { }
+"""
+
+
+# Timeout constants.
+NCTEST_TERMINATE_TIMEOUT_SEC = 60
+NCTEST_KILL_TIMEOUT_SEC = NCTEST_TERMINATE_TIMEOUT_SEC + 2
+BUSY_LOOP_MAX_TIME_SEC = NCTEST_KILL_TIMEOUT_SEC * 2
+
+
+def ValidateInput(parallelism, sourcefile_path, cflags, resultfile_path):
+  """Make sure the arguments being passed in are sane."""
+  assert parallelism >= 1
+  assert type(sourcefile_path) is str
+  assert type(cflags) is str
+  assert type(resultfile_path) is str
+
+
+def ParseExpectation(expectation_string):
+  """Extracts expectation definition from the trailing comment on the ifdef.
+
+  See the comment on NCTEST_CONFIG_RE for examples of the format we are parsing.
+
+  Args:
+    expectation_string: A string like "// [r'some_regex']"
+
+  Returns:
+    A list of compiled regular expressions indicating all possible valid
+    compiler outputs.  If the list is empty, all outputs are considered valid.
+  """
+  assert expectation_string is not None
+
+  match = EXTRACT_EXPECTATION_RE.match(expectation_string)
+  assert match
+
+  raw_expectation = ast.literal_eval(match.group(1))
+  assert type(raw_expectation) is list
+
+  expectation = []
+  for regex_str in raw_expectation:
+    assert type(regex_str) is str
+    expectation.append(re.compile(regex_str))
+  return expectation
+
+
+def ExtractTestConfigs(sourcefile_path):
+  """Parses the soruce file for test configurations.
+
+  Each no-compile test in the file is separated by an ifdef macro.  We scan
+  the source file with the NCTEST_CONFIG_RE to find all ifdefs that look like
+  they demark one no-compile test and try to extract the test configuration
+  from that.
+
+  Args:
+    sourcefile_path: The path to the source file.
+
+  Returns:
+    A list of test configurations. Each test configuration is a dictionary of
+    the form:
+
+      { name: 'NCTEST_NAME'
+        suite_name: 'SOURCE_FILE_NAME'
+        expectations: [re.Pattern, re.Pattern] }
+
+    The |suite_name| is used to generate a pretty gtest output on successful
+    completion of the no compile test.
+
+    The compiled regexps in |expectations| define the valid outputs of the
+    compiler.  If any one of the listed patterns matches either the stderr or
+    stdout from the compilation, and the compilation failed, then the test is
+    considered to have succeeded.  If the list is empty, than we ignore the
+    compiler output and just check for failed compilation. If |expectations|
+    is actually None, then this specifies a compiler sanity check test, which
+    should expect a SUCCESSFUL compilation.
+  """
+  sourcefile = open(sourcefile_path, 'r')
+
+  # Convert filename from underscores to CamelCase.
+  words = os.path.splitext(os.path.basename(sourcefile_path))[0].split('_')
+  words = [w.capitalize() for w in words]
+  suite_name = 'NoCompile' + ''.join(words)
+
+  # Start with at least the compiler sanity test.  You need to always have one
+  # sanity test to show that compiler flags and configuration are not just
+  # wrong.  Otherwise, having a misconfigured compiler, or an error in the
+  # shared portions of the .nc file would cause all tests to erroneously pass.
+  test_configs = [{'name': 'NCTEST_SANITY',
+                   'suite_name': suite_name,
+                   'expectations': None}]
+
+  for line in sourcefile:
+    match_result = NCTEST_CONFIG_RE.match(line)
+    if not match_result:
+      continue
+
+    groups = match_result.groups()
+
+    # Grab the name and remove the defined() predicate if there is one.
+    name = groups[0]
+    strip_result = STRIP_DEFINED_RE.match(name)
+    if strip_result:
+      name = strip_result.group(1)
+
+    # Read expectations if there are any.
+    test_configs.append({'name': name,
+                         'suite_name': suite_name,
+                         'expectations': ParseExpectation(groups[1])})
+  sourcefile.close()
+  return test_configs
+
+
+def StartTest(sourcefile_path, cflags, config):
+  """Start one negative compile test.
+
+  Args:
+    sourcefile_path: The path to the source file.
+    cflags: A string with all the CFLAGS to give to gcc. This string will be
+            split by shelex so be careful with escaping.
+    config: A dictionary describing the test.  See ExtractTestConfigs
+      for a description of the config format.
+
+  Returns:
+    A dictionary containing all the information about the started test. The
+    fields in the dictionary are as follows:
+      { 'proc': A subprocess object representing the compiler run.
+        'cmdline': The exectued command line.
+        'name': The name of the test.
+        'suite_name': The suite name to use when generating the gunit test
+                      result.
+        'terminate_timeout': The timestamp in seconds since the epoch after
+                             which the test should be terminated.
+        'kill_timeout': The timestamp in seconds since the epoch after which
+                        the test should be given a hard kill signal.
+        'started_at': A timestamp in seconds since the epoch for when this test
+                      was started.
+        'aborted_at': A timestamp in seconds since the epoch for when this test
+                      was aborted.  If the test completed successfully,
+                      this value is 0.
+        'finished_at': A timestamp in seconds since the epoch for when this
+                       test was successfully complete.  If the test is aborted,
+                       or running, this value is 0.
+        'expectations': A dictionary with the test expectations. See
+                        ParseExpectation() for the structure.
+        }
+  """
+  # TODO(ajwong): Get the compiler from gyp.
+  cmdline = ['g++']
+  cmdline.extend(shlex.split(cflags))
+  name = config['name']
+  expectations = config['expectations']
+  if expectations is not None:
+    cmdline.append('-D%s' % name)
+  cmdline.extend(['-o', '/dev/null', '-c', '-x', 'c++', sourcefile_path])
+
+  process = subprocess.Popen(cmdline, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+  now = time.time()
+  return {'proc': process,
+          'cmdline': ' '.join(cmdline),
+          'name': name,
+          'suite_name': config['suite_name'],
+          'terminate_timeout': now + NCTEST_TERMINATE_TIMEOUT_SEC,
+          'kill_timeout': now + NCTEST_KILL_TIMEOUT_SEC,
+          'started_at': now,
+          'aborted_at': 0,
+          'finished_at': 0,
+          'expectations': expectations}
+
+
+def PassTest(resultfile, test):
+  """Logs the result of a test started by StartTest(), or a disabled test
+  configuration.
+
+  Args:
+    resultfile: File object for .cc file that results are written to.
+    test: An instance of the dictionary returned by StartTest(), a
+          configuration from ExtractTestConfigs().
+  """
+  # The 'started_at' key is only added if a test has been started.
+  if 'started_at' in test:
+    resultfile.write(SUCCESS_GUNIT_TEMPLATE % (
+        test['suite_name'], test['name'],
+        test['finished_at'] - test['started_at'],
+        test['started_at'], test['finished_at']))
+  else:
+    resultfile.write(DISABLED_GUNIT_TEMPLATE % (
+        test['suite_name'], test['name']))
+
+
+def FailTest(resultfile, test, error, stdout=None, stderr=None):
+  """Logs the result of a test started by StartTest()
+
+  Args:
+    resultfile: File object for .cc file that results are written to.
+    test: An instance of the dictionary returned by StartTest()
+    error: The printable reason for the failure.
+    stdout: The test's output to stdout.
+    stderr: The test's output to stderr.
+  """
+  resultfile.write('#error %s Failed: %s\n' % (test['name'], error))
+  resultfile.write('#error compile line: %s\n' % test['cmdline'])
+  if stdout and len(stdout) != 0:
+    resultfile.write('#error %s stdout:\n' % test['name'])
+    for line in stdout.split('\n'):
+      resultfile.write('#error   %s\n' % line)
+
+  if stderr and len(stderr) != 0:
+    resultfile.write('#error %s stderr:\n' % test['name'])
+    for line in stderr.split('\n'):
+      resultfile.write('#error   %s\n' % line)
+  resultfile.write('\n')
+
+
+def WriteStats(resultfile, suite_name, timings):
+  """Logs the peformance timings for each stage of the script into a fake test.
+
+  Args:
+    resultfile: File object for .cc file that results are written to.
+    suite_name: The name of the GUnit suite this test belongs to.
+    timings: Dictionary with timestamps for each stage of the script run.
+  """
+  stats_template = ("Started %f, Ended %f, Total %fs, Extract %fs, "
+                    "Compile %fs, Process %fs")
+  total_secs = timings['results_processed'] - timings['started']
+  extract_secs = timings['extract_done'] - timings['started']
+  compile_secs = timings['compile_done'] - timings['extract_done']
+  process_secs = timings['results_processed'] - timings['compile_done']
+  resultfile.write('TEST(%s, Stats) { LOG(INFO) << "%s"; }\n' % (
+      suite_name, stats_template % (
+          timings['started'], timings['results_processed'], total_secs,
+          extract_secs, compile_secs, process_secs)))
+
+
+def ProcessTestResult(resultfile, test):
+  """Interprets and logs the result of a test started by StartTest()
+
+  Args:
+    resultfile: File object for .cc file that results are written to.
+    test: The dictionary from StartTest() to process.
+  """
+  # Snap a copy of stdout and stderr into the test dictionary immediately
+  # cause we can only call this once on the Popen object, and lots of stuff
+  # below will want access to it.
+  proc = test['proc']
+  (stdout, stderr) = proc.communicate()
+
+  if test['aborted_at'] != 0:
+    FailTest(resultfile, test, "Compile timed out. Started %f ended %f." %
+             (test['started_at'], test['aborted_at']))
+    return
+
+  if test['expectations'] is None:
+    # This signals a compiler sanity check test. Fail iff compilation failed.
+    if proc.poll() == 0:
+      PassTest(resultfile, test)
+      return
+    else:
+      FailTest(resultfile, test, 'Sanity compile failed. Is compiler borked?',
+               stdout, stderr)
+      return
+  elif proc.poll() == 0:
+    # Handle failure due to successful compile.
+    FailTest(resultfile, test,
+             'Unexpected successful compilation.',
+             stdout, stderr)
+    return
+  else:
+    # Check the output has the right expectations.  If there are no
+    # expectations, then we just consider the output "matched" by default.
+    if len(test['expectations']) == 0:
+      PassTest(resultfile, test)
+      return
+
+    # Otherwise test against all expectations.
+    for regexp in test['expectations']:
+      if (regexp.search(stdout) is not None or
+          regexp.search(stderr) is not None):
+        PassTest(resultfile, test)
+        return
+    expectation_str = ', '.join(
+        ["r'%s'" % regexp.pattern for regexp in test['expectations']])
+    FailTest(resultfile, test,
+             'Expectations [%s] did not match output.' % expectation_str,
+             stdout, stderr)
+    return
+
+
+def CompleteAtLeastOneTest(resultfile, executing_tests):
+  """Blocks until at least one task is removed from executing_tests.
+
+  This function removes completed tests from executing_tests, logging failures
+  and output.  If no tests can be removed, it will enter a poll-loop until one
+  test finishes or times out.  On a timeout, this function is responsible for
+  terminating the process in the appropriate fashion.
+
+  Args:
+    executing_tests: A dict mapping a string containing the test name to the
+                     test dict return from StartTest().
+
+  Returns:
+    A list of tests that have finished.
+  """
+  finished_tests = []
+  busy_loop_timeout = time.time() + BUSY_LOOP_MAX_TIME_SEC
+  while len(finished_tests) == 0:
+    # If we don't make progress for too long, assume the code is just dead.
+    assert busy_loop_timeout > time.time()
+
+    # Select on the output pipes.
+    read_set = []
+    for test in executing_tests.values():
+      read_set.extend([test['proc'].stderr, test['proc'].stdout])
+    result = select.select(read_set, [], read_set, NCTEST_TERMINATE_TIMEOUT_SEC)
+
+    # Now attempt to process results.
+    now = time.time()
+    for test in executing_tests.values():
+      proc = test['proc']
+      if proc.poll() is not None:
+        test['finished_at'] = now
+        finished_tests.append(test)
+      elif test['terminate_timeout'] < now:
+        proc.terminate()
+        test['aborted_at'] = now
+      elif test['kill_timeout'] < now:
+        proc.kill()
+        test['aborted_at'] = now
+
+  for test in finished_tests:
+    del executing_tests[test['name']]
+  return finished_tests
+
+
+def main():
+  if len(sys.argv) != 5:
+    print ('Usage: %s <parallelism> <sourcefile> <cflags> <resultfile>' %
+           sys.argv[0])
+    sys.exit(1)
+
+  # Force us into the "C" locale so the compiler doesn't localize its output.
+  # In particular, this stops gcc from using smart quotes when in english UTF-8
+  # locales.  This makes the expectation writing much easier.
+  os.environ['LC_ALL'] = 'C'
+
+  parallelism = int(sys.argv[1])
+  sourcefile_path = sys.argv[2]
+  cflags = sys.argv[3]
+  resultfile_path = sys.argv[4]
+
+  timings = {'started': time.time()}
+
+  ValidateInput(parallelism, sourcefile_path, cflags, resultfile_path)
+
+  test_configs = ExtractTestConfigs(sourcefile_path)
+  timings['extract_done'] = time.time()
+
+  resultfile = open(resultfile_path, 'w')
+  resultfile.write(RESULT_FILE_HEADER % sourcefile_path)
+
+  # Run the no-compile tests, but ensure we do not run more than |parallelism|
+  # tests at once.
+  timings['header_written'] = time.time()
+  executing_tests = {}
+  finished_tests = []
+  for config in test_configs:
+    # CompleteAtLeastOneTest blocks until at least one test finishes. Thus, this
+    # acts as a semaphore.  We cannot use threads + a real semaphore because
+    # subprocess forks, which can cause all sorts of hilarity with threads.
+    if len(executing_tests) >= parallelism:
+      finished_tests.extend(CompleteAtLeastOneTest(resultfile, executing_tests))
+
+    if config['name'].startswith('DISABLED_'):
+      PassTest(resultfile, config)
+    else:
+      test = StartTest(sourcefile_path, cflags, config)
+      assert test['name'] not in executing_tests
+      executing_tests[test['name']] = test
+
+  # If there are no more test to start, we still need to drain the running
+  # ones.
+  while len(executing_tests) > 0:
+    finished_tests.extend(CompleteAtLeastOneTest(resultfile, executing_tests))
+  timings['compile_done'] = time.time()
+
+  for test in finished_tests:
+    ProcessTestResult(resultfile, test)
+  timings['results_processed'] = time.time()
+
+  # We always know at least a sanity test was run.
+  WriteStats(resultfile, finished_tests[0]['suite_name'], timings)
+
+  resultfile.close()
+
+
+if __name__ == '__main__':
+  main()