[NaCl SDK] Build gtest on buildbots, but don't include it in the SDK.

**For now, gtest is marked experimental because the NaCl fix to gtest has not yet been upstreamed and DEPS'd into chrome**

* Test-only libraries and examples are build to the $(NACL_SDK_ROOT)testing/ directory.
* Added INCLUDES to .dsc file, platform-generic way to add an include directory.

BUG=none
NOTRY=true

Review URL: https://chromiumcodereview.appspot.com/10824092

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@149929 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 157 insertions, 145 deletions

diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py
index 0b2ba94..437fdaf 100755
--- a/native_client_sdk/src/build_tools/build_sdk.py
+++ b/native_client_sdk/src/build_tools/build_sdk.py
@@ -532,9 +532,9 @@ def GetWindowsEnvironment():
   return dict(line.split('=') for line in stdout.split('\r\n')[:-1])
 
 
-def BuildStepBuildLibraries(pepperdir, platform):
+def BuildStepBuildLibraries(pepperdir, platform, directory):
   buildbot_common.BuildStep('Build Libraries')
-  src_dir = os.path.join(pepperdir, 'src')
+  src_dir = os.path.join(pepperdir, directory)
   makefile = os.path.join(src_dir, 'Makefile')
   if os.path.isfile(makefile):
     print "\n\nMake: " + src_dir
@@ -630,6 +630,38 @@ def BuildStepBuildExamples(pepperdir, platform):
     buildbot_common.Run(['make', '-j8'],
                         cwd=os.path.abspath(example_dir), shell=True, env=env)
 
+TEST_EXAMPLE_LIST = [
+]
+
+TEST_LIBRARY_LIST = [
+  'gtest',
+]
+
+def BuildStepCopyTests(pepperdir, toolchains, build_experimental):
+  buildbot_common.BuildStep('Copy Tests')
+
+  testingdir = os.path.join(pepperdir, 'testing')
+  buildbot_common.RemoveDir(testingdir)
+  buildbot_common.MakeDir(testingdir)
+
+  args = ['--dstroot=%s' % pepperdir, '--master']
+  for toolchain in toolchains:
+    args.append('--' + toolchain)
+
+  for example in TEST_EXAMPLE_LIST:
+    dsc = os.path.join(SDK_EXAMPLE_DIR, example, 'example.dsc')
+    args.append(dsc)
+
+  for library in TEST_LIBRARY_LIST:
+    dsc = os.path.join(SDK_LIBRARY_DIR, library, 'library.dsc')
+    args.append(dsc)
+
+  if build_experimental:
+    args.append('--experimental')
+
+  if generate_make.main(args):
+    buildbot_common.ErrorExit('Failed to build tests.')
+
 
 def BuildStepTestExamples(pepperdir, platform, pepper_ver):
   buildbot_common.BuildStep('Test Examples')
@@ -756,8 +788,10 @@ def main(args):
 
   if options.only_examples:
     BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental)
-    BuildStepBuildLibraries(pepperdir, platform)
+    BuildStepBuildLibraries(pepperdir, platform, 'src')
     BuildStepBuildExamples(pepperdir, platform)
+    BuildStepCopyTests(pepperdir, toolchains, options.build_experimental)
+    BuildStepBuildLibraries(pepperdir, platform, 'testing')
     if options.test_examples:
       BuildStepTestExamples(pepperdir, platform, pepper_ver)
   elif options.only_updater:
@@ -776,7 +810,7 @@ def main(args):
     BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental)
 
     # Ship with libraries prebuilt, so run that first.
-    BuildStepBuildLibraries(pepperdir, platform)
+    BuildStepBuildLibraries(pepperdir, platform, 'src')
 
     if not options.skip_tar:
       BuildStepTarBundle(pepper_ver, tarfile)
@@ -786,6 +820,8 @@ def main(args):
       # the examples and test from this directory instead of the original.
       pepperdir = BuildStepTestUpdater(platform, pepper_ver, clnumber, tarfile)
       BuildStepBuildExamples(pepperdir, platform)
+      BuildStepCopyTests(pepperdir, toolchains, options.build_experimental)
+      BuildStepBuildLibraries(pepperdir, platform, 'testing')
       if options.test_examples:
         BuildStepTestExamples(pepperdir, platform, pepper_ver)
 
diff --git a/native_client_sdk/src/build_tools/generate_make.py b/native_client_sdk/src/build_tools/generate_make.py
index 8367bb3..d3abb20 100755
--- a/native_client_sdk/src/build_tools/generate_make.py
+++ b/native_client_sdk/src/build_tools/generate_make.py
@@ -10,7 +10,7 @@ import os
 import sys
 
 from make_rules import BuildDefineList, BuildLibList, BuildToolDict
-from make_rules import GetBuildRule, BUILD_RULES
+from make_rules import BuildIncludeList, GetBuildRule, BUILD_RULES
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
@@ -145,17 +145,19 @@ def GenerateCompile(target, tool, arch, srcs):
   name = target['NAME']
   object_sets = []
 
-  defines = target.get('DEFINES', [])
-  defs = BuildDefineList(tool, defines)
+  defs = BuildDefineList(tool, target.get('DEFINES', []))
+  includes = BuildIncludeList(tool, target.get('INCLUDES', []))
 
   if srcs['.c']:
-    replace = BuildToolDict(tool, name, arch, 'c', DEFLIST=defs)
+    replace = BuildToolDict(tool, name, arch, 'c',
+        DEFLIST=defs, INCLUDELIST=includes)
     compile_rule = GetBuildRule(tool, 'CC')
     rules += Replace(compile_rule, replace)
     object_sets.append('$(%s)' % replace['<OBJS>'])
 
   if srcs['.cc']:
-    replace = BuildToolDict(tool, name, arch, 'cc', DEFLIST=defs)
+    replace = BuildToolDict(tool, name, arch, 'cc',
+        DEFLIST=defs, INCLUDELIST=includes)
     compile_rule = GetBuildRule(tool, 'CXX')
     rules += Replace(compile_rule, replace)
     object_sets.append('$(%s)' % replace['<OBJS>'])
@@ -204,11 +206,11 @@ def GenerateRules(desc, tools):
         targs, link_rule = GenerateLink(target, tc, arch, objs)
         rules += comp_rule + link_rule
         clean.append(objs)
+        if target['TYPE'] == 'lib':
+          all_targets.append(targs)
 
       if target['TYPE'] == 'main':
         main = target
-      elif target['TYPE'] == 'lib':
-        all_targets.append(targs)
 
     if main:
       targs, nmf_rule = GenerateNMF(main, tc)
@@ -269,6 +271,7 @@ DSC_FORMAT = {
         'CCFLAGS': (list, '', False),
         'CXXFLAGS': (list, '', False),
         'LDFLAGS': (list, '', False),
+        'INCLUDES': (list, '', False),
         'LIBS' : (list, '', False)
     }, True),
     'HEADERS': (list, {
@@ -278,7 +281,7 @@ DSC_FORMAT = {
     'SEARCH': (list, '', False),
     'POST': (str, '', False),
     'PRE': (str, '', False),
-    'DEST': (str, ['examples', 'src'], True),
+    'DEST': (str, ['examples', 'src', 'testing'], True),
     'NAME': (str, '', False),
     'DATA': (list, '', False),
     'TITLE': (str, '', False),
@@ -326,6 +329,10 @@ def ValidateFormat(src, format, ErrorMsg=ErrorMsgFunc):
       failed = True
       continue
 
+    # If it's a bool, the expected values are always True or False.
+    if exp_type is bool:
+      continue
+
     # If it's a string and there are expected values, make sure it matches
     if exp_type is str:
       if type(exp_value) is list and exp_value:
@@ -552,8 +559,7 @@ def main(argv):
     toolchains = ['newlib', 'glibc']
     print 'Using default toolchains: ' + ' '.join(toolchains)
 
-  examples = []
-  libs = []
+  master_projects = {}
   for filename in args:
     desc = LoadProject(filename, toolchains)
     if not desc:
@@ -568,21 +574,19 @@ def main(argv):
     if not ProcessProject(srcroot, options.dstroot, desc, toolchains):
       ErrorExit('\n*** Failed to process project: %s ***' % filename)
 
-    # if this is an example add it to the master make and update the html
+    # if this is an example update the html
     if desc['DEST'] == 'examples':
-      examples.append(desc['NAME'])
       ProcessHTML(srcroot, options.dstroot, desc, toolchains)
 
-    # if this is a library add it to the master make
-    if desc['DEST'] == 'src':
-      libs.append(desc['NAME'])
+    # Create a list of projects for each DEST. This will be used to generate a
+    # master makefile.
+    master_projects.setdefault(desc['DEST'], []).append(desc['NAME'])
 
   if options.master:
     master_in = os.path.join(SDK_EXAMPLE_DIR, 'Makefile')
-    master_out = os.path.join(options.dstroot, 'examples', 'Makefile')
-    GenerateMasterMakefile(master_in, master_out, examples)
-    master_out = os.path.join(options.dstroot, 'src', 'Makefile')
-    GenerateMasterMakefile(master_in, master_out, libs)
+    for dest, projects in master_projects.iteritems():
+      master_out = os.path.join(options.dstroot, dest, 'Makefile')
+      GenerateMasterMakefile(master_in, master_out, projects)
   return 0
 
 
diff --git a/native_client_sdk/src/build_tools/make_rules.py b/native_client_sdk/src/build_tools/make_rules.py
index 7bf5631..ec72402 100755
--- a/native_client_sdk/src/build_tools/make_rules.py
+++ b/native_client_sdk/src/build_tools/make_rules.py
@@ -56,20 +56,20 @@ NACL_CC_RULE = """
 <OBJS>:=$(patsubst %.<ext>, <tc>/%_<ARCH>.o,$(<PROJ>_<EXT>))
 DEPFILES+=$(<OBJS>:.o=.d)
 $(<OBJS>) : <tc>/%_<ARCH>.o : %.<ext> $(THIS_MAKE) | <tc>
-<TAB>$(<CC>) -o $@ $< <MACH> $(<PROJ>_<EXT>FLAGS) -DTCNAME=<tc> $(<TC>_CCFLAGS) <DEFLIST>
+<TAB>$(<CC>) -o $@ $< <MACH> -DTCNAME=<tc> $(<TC>_CCFLAGS) $(<PROJ>_<EXT>FLAGS) <DEFLIST> <INCLUDELIST>
 """
 
 SO_CC_RULE = """
 <OBJS>:=$(patsubst %.<ext>, <tc>/%_<ARCH>.o,$(<PROJ>_<EXT>))
 DEPFILES+=$(<OBJS>:.o=.d)
 $(<OBJS>) : <tc>/%_<ARCH>.o : %.<ext> $(THIS_MAKE) | <tc>
-<TAB>$(<CC>) -o $@ $< <MACH> -fPIC $(<PROJ>_<EXT>FLAGS) -DTCNAME=<tc> $(<TC>_CCFLAGS) <DEFLIST>
+<TAB>$(<CC>) -o $@ $< <MACH> -fPIC -DTCNAME=<tc> $(<TC>_CCFLAGS) $(<PROJ>_<EXT>FLAGS) <DEFLIST> <INCLUDELIST>
 """
 
 WIN_CC_RULE = """
 <OBJS>:=$(patsubst %.<ext>, <tc>/%.obj,$(<PROJ>_<EXT>))
 $(<OBJS>) : <tc>/%.obj : %.<ext> $(THIS_MAKE) | <tc>
-<TAB>$(<CC>) /Fo$@ /c $< -DTCNAME=host $(WIN_CCFLAGS) <DEFLIST>
+<TAB>$(<CC>) /Fo$@ /c $< -DTCNAME=host $(WIN_CCFLAGS) <DEFLIST> <INCLUDELIST>
 """
 
 #
@@ -118,7 +118,7 @@ LAUNCH_HOST: CHECK_FOR_CHROME all
 #
 POSIX_LIB_RULE = """
 $(NACL_SDK_ROOT)/lib/$(OSNAME)_<ARCH>_<tc>/lib<proj>.a : <OBJS>
-<TAB>$(MKDIR) $(dir $@)
+<TAB>$(MKDIR) -p $(dir $@)
 <TAB>$(<LIB>) $@ $^
 """
 
@@ -152,6 +152,7 @@ EXT_MAP = {
 
 WIN_TOOL = {
   'DEFINE': '-D%s',
+  'INCLUDE': '/I%s',
   'LIBRARY': '%s.lib',
   'main': '<tc>/<proj>.dll',
   'nmf': '<tc>/<proj>.nmf',
@@ -161,6 +162,7 @@ WIN_TOOL = {
 
 NACL_TOOL = {
   'DEFINE': '-D%s',
+  'INCLUDE': '-I%s',
   'LIBRARY': '-l%s',
   'main': '<tc>/<proj>_<ARCH>.nexe',
   'nmf': '<tc>/<proj>.nmf',
@@ -246,16 +248,21 @@ def GetBuildRule(tool, ext):
   return BUILD_RULES[tool][ext]
 
 
+def BuildList(tool, key, items):
+  pattern = BUILD_RULES[tool]['TOOL'][key]
+  items = [(pattern % name) for name in items]
+  return ' '.join(items)
+
 def BuildDefineList(tool, defs):
-  pattern = BUILD_RULES[tool]['TOOL']['DEFINE']
-  defines = [(pattern % name) for name in defs]
-  return ' '.join(defines)
+  return BuildList(tool, 'DEFINE', defs)
+
+
+def BuildIncludeList(tool, includes):
+  return BuildList(tool, 'INCLUDE', includes)
 
 
 def BuildLibList(tool, libs):
-  pattern = BUILD_RULES[tool]['TOOL']['LIBRARY']
-  libraries = [(pattern % name) for name in libs]
-  return ' '.join(libraries)
+  return BuildList(tool, 'LIBRARY', libs)
 
 
 def BuildToolDict(toolchain, project, arch = {}, ext='nexe', **kwargs):
diff --git a/native_client_sdk/src/libraries/gtest/build.scons b/native_client_sdk/src/libraries/gtest/build.scons
deleted file mode 100644
index dd3b788..0000000
--- a/native_client_sdk/src/libraries/gtest/build.scons
+++ /dev/null
@@ -1,112 +0,0 @@
-# -*- python -*-
-# Copyright (c) 2011 The Chromium Authors.  All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import build_utils
-import glob
-import os
-import shutil
-import subprocess
-import sys
-
-from SCons import Script
-
-# Directories used throughout this script.
-script_dir = os.path.abspath(os.getcwd())
-sdk_root_dir = os.getenv('NACL_SDK_ROOT')
-build_tools_dir = os.path.join(sdk_root_dir, 'build_tools')
-libraries_dir = os.path.join(sdk_root_dir, 'libraries')
-
-# Add the path to build_tools to the shell's python path.
-shell_env = os.environ.copy()
-python_paths = [build_tools_dir]
-python_paths += [shell_env.get('PYTHONPATH', '')]
-shell_env['PYTHONPATH'] = os.pathsep.join(python_paths)
-
-# Argv for the install-gtest python script.
-script_argv = [
-  '--toolchain=%s' % (
-      build_utils.NormalizeToolchain(base_dir=sdk_root_dir,
-                                     arch='x86',
-                                     variant='glibc')),
-  '--toolchain=%s' % (
-      build_utils.NormalizeToolchain(base_dir=sdk_root_dir,
-                                     arch='x86',
-                                     variant='newlib')),
-  '--working_dir=%s' % script_dir
-  ]
-
-# The scons build env.
-build_env = Script.Environment().Clone()
-
-# Where the src for gtest and gmock will be found after running the install
-# script. We keep them around as a sentinel, to indicate they they have been
-# installed. (See BuildGTestLibs below.)
-gtest_src = os.path.join(script_dir, 'gtest-1.5.0')
-gmock_src = os.path.join(script_dir, 'gmock-1.5.0')
-
-
-def BuildGTestLibs(env, target, source):
-  '''Build and install the gtest/gmock libraries.
-
-  This invokes the gtest_install.py script in the build_tools directory. In turn
-  that scripts downloads, untar, patches and build the gtest/gmock libraries.
-  Finally, the libraries and related include files are copied to the toolchain.
-
-  Args:
-    env: The construction Environment() that is building the examples.
-    target: The target that triggered this build.  Not used.
-    source: The sources used for this build.  Not used.
-  '''
-  # If our sentinel, the gtest source is present, do not build.
-  if os.path.exists(gtest_src):
-    return
-  # Remove any old gmock source if still present.
-  shutil.rmtree(gmock_src, ignore_errors=True)
-  # Invoke the gtest install script.
-  script = os.path.join(build_tools_dir, 'install_gtest', 'install_gtest.py')
-  py_command = [sys.executable, script]
-  subprocess.check_call(py_command + script_argv, env=shell_env)
-
-  # Clean up: remove left-over tgz files.
-  for f in glob.iglob(os.path.join(script_dir, '*.tgz')):
-    os.remove(f)
-
-
-def CleanGTestLibs(env, target, suite_name):
-  '''Clean the gtest/gmock libraries sources.
-
-  This does a partial clean up of the gtest/gmock projects. It removes the src
-  directories. However, the actual libraries and related includes in the
-  toolchains are not removed. It is however sufficient to trigger a full
-  rebuild of gtest/gmock.
-
-  Args:
-    env: The construction Environment() that is building the examples.
-    target: The target that triggered this build.
-    suite_name: A suite name that should cause this target to be cleaned.
-  '''
-  # Only do this in 'clean' mode.
-  if not build_env.GetOption('clean'):
-    return
-  # Only clean target if it's on the cmd line or it's a clean all.
-  clean_this = True
-  if len(COMMAND_LINE_TARGETS) > 0:
-    clean_this = False
-    for cl_target in COMMAND_LINE_TARGETS:
-      if cl_target == suite_name or cl_target == target:
-        clean_this = True
-        break
-  # Delete the src trees for gtest and gmock.
-  if clean_this:
-    shutil.rmtree(gmock_src, ignore_errors=True)
-    shutil.rmtree(gtest_src, ignore_errors=True)
-
-
-gtest_libs_builder = build_env.Alias('gtest_libs', [], BuildGTestLibs)
-build_env.AlwaysBuild(gtest_libs_builder)
-CleanGTestLibs(build_env, 'gtest_libs', 'bot')
-
-# ----------------------------------------------------------------------------
-build_env.Default('gtest_libs')
diff --git a/native_client_sdk/src/libraries/gtest/library.dsc b/native_client_sdk/src/libraries/gtest/library.dsc
new file mode 100644
index 0000000..a7405a8
--- /dev/null
+++ b/native_client_sdk/src/libraries/gtest/library.dsc
@@ -0,0 +1,77 @@
+{
+  'TOOLS': ['newlib', 'glibc', 'pnacl', 'win'],
+  'SEARCH': [
+    '../../../../testing/gtest/include/gtest',
+    '../../../../testing/gtest/include/gtest/internal',
+    '../../../../testing/gtest/src',
+  ],
+  'TARGETS': [
+    {
+      'NAME' : 'gtest',
+      'TYPE' : 'lib',
+      'SOURCES' : [
+        'gtest.cc',
+        'gtest-death-test.cc',
+        'gtest-filepath.cc',
+        'gtest_main.cc',
+        'gtest-port.cc',
+        'gtest-printers.cc',
+        'gtest-test-part.cc',
+        'gtest-typed-test.cc',
+      ],
+      # Ignore warning:
+      #   gtest.cc:2555: error: enumeration value ‘COLOR_DEFAULT’ not handled in switch
+      'CXXFLAGS': ['-Wno-switch-enum'],
+      'INCLUDES': [
+        # See comment below about gtest-internal-inl.h
+        '$(NACL_SDK_ROOT)/include/gtest/internal',
+      ],
+    }
+  ],
+  'HEADERS': [
+    {
+      'FILES': [
+        'gtest-death-test.h',
+        'gtest.h',
+        'gtest-message.h',
+        'gtest-param-test.h',
+        'gtest_pred_impl.h',
+        'gtest-printers.h',
+        'gtest_prod.h',
+        'gtest-spi.h',
+        'gtest-test-part.h',
+        'gtest-typed-test.h',
+      ],
+      'DEST': 'include/gtest',
+    },
+    {
+      'FILES': [
+        'gtest-death-test-internal.h',
+        'gtest-filepath.h',
+        'gtest-internal.h',
+        'gtest-linked_ptr.h',
+        'gtest-param-util-generated.h',
+        'gtest-param-util.h',
+        'gtest-port.h',
+        'gtest-string.h',
+        'gtest-tuple.h',
+        'gtest-type-util.h',
+      ],
+      'DEST': 'include/gtest/internal',
+    },
+    {
+      # This is cheesy, but gtest.cc includes "src/gtest-internal-inl.h". Since
+      # gtest is not installed in the SDK, I don't really care about the
+      # directory layout.
+      # TODO(binji): If we decide to include gtest, put this file in a better
+      # spot.
+      'FILES': [
+        'gtest-internal-inl.h',
+      ],
+      'DEST': 'include/gtest/internal/src',
+    },
+  ],
+  'DEST': 'testing',
+  'NAME': 'gtest',
+  'EXPERIMENTAL': True,
+}