Add ability to automatically generate the makefiles for SDK

Adds the ability automatically generate the NMF for file for
both newlib and glibc cases.  Adds a DEST field to automatically copy files as needed.

Added --toolchain=glibc to fix NMF creation problem in ppapi_untrusted.gyp where auto-detect of toolchain type fails due to missing glibc in toolchain name.

BUG=130618
R=binji@chromium.org
TEST=test_generate_make.py

Adding brettw@chromium.org for ppapi_untrusted

Review URL: https://chromiumcodereview.appspot.com/10538046

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@141343 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 146 insertions, 54 deletions

diff --git a/native_client_sdk/src/build_tools/generate_make.py b/native_client_sdk/src/build_tools/generate_make.py
index f04aa47..5a07616 100755
--- a/native_client_sdk/src/build_tools/generate_make.py
+++ b/native_client_sdk/src/build_tools/generate_make.py
@@ -3,16 +3,24 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import buildbot_common
+import optparse
 import os
 import sys
 
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+SDK_SRC_DIR = os.path.dirname(SCRIPT_DIR)
+SDK_EXAMPLE_DIR = os.path.join(SDK_SRC_DIR, 'examples')
+SDK_DIR = os.path.dirname(SDK_SRC_DIR)
+SRC_DIR = os.path.dirname(SDK_DIR)
+OUT_DIR = os.path.join(SRC_DIR, 'out')
+PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
+
 ARCHITECTURES = ['32', '64']
 
 
 def WriteMakefile(srcpath, dstpath, replacements):
-  print 'opening: %s\n' % srcpath
   text = open(srcpath, 'rb').read()
   for key in replacements:
     text = text.replace(key, replacements[key])
@@ -51,18 +59,33 @@ def SetVar(varname, values):
   return out
 
 
+def GenerateCopyList(desc):
+  sources = []
+
+  # Add sources for each target
+  for target in desc['TARGETS']:
+    sources.extend(target['SOURCES'])
+
+  # And HTML and data files
+  sources.append(desc['NAME'] + '.html')
+  sources.extend(desc.get('DATA', []))
+  return sources
+
+
 def GenerateReplacements(desc):
   # Generate target settings
   tools = desc['TOOLS']
 
+  prelaunch = ''
+  prerun = ''
+
   settings = SetVar('VALID_TOOLCHAINS', tools)
   settings+= 'TOOLCHAIN?=%s\n\n' % tools[0]
 
-  targets = []
   rules = ''
-
-  for name in desc['TARGETS']:
-    target = desc['TARGETS'][name]
+  targets = []
+  for target in desc['TARGETS']:
+    name = target['NAME']
     macro = name.upper()
     ext = GetExtType(target)
 
@@ -71,16 +94,16 @@ def GenerateReplacements(desc):
     cxx_sources = [fname for fname in sources if fname.endswith('.cc')]
 
     if cc_sources:
-      flags = target.get('CCFLAGS', '')
+      flags = target.get('CCFLAGS', ['$(NACL_CCFLAGS)'])
       settings += SetVar(macro + '_CC', cc_sources)
       settings += SetVar(macro + '_CCFLAGS', flags)
 
     if cxx_sources:
-      flags = target.get('CXXFLAGS', '')
+      flags = target.get('CXXFLAGS', ['$(NACL_CXXFLAGS)'])
       settings += SetVar(macro + '_CXX', cxx_sources)
-      settings += SetVar(macro + '_CCFLAGS', flags)
+      settings += SetVar(macro + '_CXXFLAGS', flags)
 
-    flags = target.get('LDFLAGS')
+    flags = target.get('LDFLAGS', ['$(NACL_LDFLAGS)'])
     settings += SetVar(macro + '_LDFLAGS', flags)
 
     for arch in ARCHITECTURES:
@@ -103,17 +126,25 @@ def GenerateReplacements(desc):
       rules += '%s : %s\n' % (target_name, ' '.join(object_sets))
       rules += '\t$(NACL_LINK) -o $@ $^ -m%s $(%s_LDFLAGS)\n\n' % (arch, macro)
 
-  targets = 'all : '+ ' '.join(targets) 
+  nmf = desc['NAME'] + '.nmf'
+  nmfs = '%s : %s\n' % (nmf, ' '.join(targets))
+  nmfs +='\t$(NMF) $(NMF_ARGS) -o $@ $(NMF_PATHS) $^\n'
+
+  targets = 'all : '+ ' '.join(targets + [nmf]) 
   return {
       '__PROJECT_SETTINGS__' : settings,
       '__PROJECT_TARGETS__' : targets,
-      '__PROJECT_RULES__' : rules
+      '__PROJECT_RULES__' : rules,
+      '__PROJECT_NMFS__' : nmfs,
+      '__PROJECT_PRELAUNCH__' : prelaunch,
+      '__PROJECT_PRERUN__' : prerun
+
   }
 
 
 # 'KEY' : ( <TYPE>, [Accepted Values], <Required?>)
 DSC_FORMAT = {
-    'TOOLS' : (list, ['host', 'newlib', 'glibc', 'pnacl'], True),
+    'TOOLS' : (list, ['newlib', 'glibc', 'pnacl'], True),
     'TARGETS' : (list, {
         'NAME': (str, '', True),
         'TYPE': (str, ['main', 'nexe', 'so'], True),
@@ -122,8 +153,10 @@ DSC_FORMAT = {
         'CXXFLAGS': (list, '', False),
         'LDFLAGS': (list, '', False)
     }, True),
-    'PAGE': (str, '', False),
+    'DEST': (str, ['examples', 'src'], True),
     'NAME': (str, '', False),
+    'DATA': (list, '', False),
+    'TITLE': (str, '', False),
     'DESC': (str, '', False),
     'INFO': (str, '', False)
 }
@@ -204,34 +237,48 @@ def ValidateFormat(src, format, ErrorMsg=ErrorMsgFunc):
 
   return not failed
 
-# TODO(noelallen) : Remove before turning on
-testdesc = {
-    'TOOLS': ['newlib', 'glibc'],
-    'TARGETS': {
-        'hello_world' : {
-            'TYPE' : 'main',
-            'SOURCES' : ['hello_world.c'],
-            'CCFLAGS' : '',
-            'CXXFLAGS' : '',
-            'LDFLAGS' : '',
-        },
-    },
-    'PAGE': 'hello_world.html',
-    'NAME': 'Hello World',
-    'DESC': """
-The Hello World In C example demonstrates the basic structure of all
-Native Client applications. This example loads a Native Client module.  The
-page tracks the status of the module as it load.  On a successful load, the
-module will post a message containing the string "Hello World" back to
-JavaScript which will display it as an alert.""",
-    'INFO': 'Basic HTML, JavaScript, and module architecture.'
-}
+
+def ProcessProject(options, filename):
+  print 'Processing %s...' % filename
+  # Default src directory is the directory the description was found in 
+  src_dir = os.path.dirname(os.path.abspath(filename))
+  desc = open(filename, 'rb').read()
+  desc = eval(desc, {}, {})
+  if not ValidateFormat(desc, DSC_FORMAT):
+    return False
+
+  name = desc['NAME']
+  out_dir = os.path.join(options.dstroot, desc['DEST'], name)
+  buildbot_common.MakeDir(out_dir)
+
+  # Copy sources to example directory
+  sources = GenerateCopyList(desc)
+  for src_name in sources:
+    src_file = os.path.join(src_dir, src_name)
+    dst_file = os.path.join(out_dir, src_name)
+    buildbot_common.CopyFile(src_file, dst_file)
+
+  # Add Makefile
+  repdict = GenerateReplacements(desc)
+  make_path = os.path.join(out_dir, 'Makefile')
+  WriteMakefile(options.template, make_path, repdict)
+  return True
 
 
 def main(argv):
-  srcpath = os.path.join(SCRIPT_DIR, 'template.mk')
-  repdict = GenerateReplacements(testdesc)
-  WriteMakefile(srcpath, 'out.make', repdict)
+  parser = optparse.OptionParser()
+  parser.add_option('--dstroot', help='Set root for destination.',
+      dest='dstroot', default=OUT_DIR)
+  parser.add_option('--template', help='Set the makefile template.',
+      dest='template', default=os.path.join(SCRIPT_DIR, 'template.mk'))
+
+  options, args = parser.parse_args(argv)
+  for filename in args:
+    if not ProcessProject(options, filename):
+      print '\n*** Failed to process project: %s ***' % filename
+      return 1
+
+  return 0
 
 
 if __name__ == '__main__':
diff --git a/native_client_sdk/src/build_tools/template.mk b/native_client_sdk/src/build_tools/template.mk
index ae4f9b9..77d2200 100644
--- a/native_client_sdk/src/build_tools/template.mk
+++ b/native_client_sdk/src/build_tools/template.mk
@@ -7,13 +7,20 @@
 #   http://www.gnu.org/software/make/manual/make.html
 #
 
+#
+# Defaults
+#
+NACL_WARNINGS:=-Wno-long-long -Wall -Wswitch-enum -Werror -pedantic
+NACL_CCFLAGS:=-O0 -g -pthread $(NACL_WARNINGS)
+NACL_CXXFLAGS:= -O0 -g -pthread -std=gnu++98 $(NACL_WARNINGS)
+NACL_LDFLAGS:=-Wl,-as-needed -g -pthread -lppapi_cpp -lppapi 
+
 
 #
 # Project Settings
 #
 __PROJECT_SETTINGS__
 
-
 #
 # Project Targets
 #
@@ -91,15 +98,6 @@ NACL_LINK?=$(TC_PATH)/bin/i686-nacl-g++
 
 
 #
-# NaCl Flags
-#
-WARNINGS:=-Wno-long-long -Wall -Wswitch-enum -Werror -pedantic 
-NACL_CCFLAGS:= -O0 -g -pthread $(WARNINGS)
-NACL_CXXFLAGS:= -O0 -g -pthread -std=gnu++98 $(WARNINGS)
-NACL_LDFLAGS:=-g -pthread -lppapi
-
-
-#
 # Verify we can find the Chrome executable if we need to launch it.
 #
 .PHONY: CHECK_FOR_CHROME
@@ -113,12 +111,27 @@ endif
 
 
 __PROJECT_RULES__
+#
+# NMF Manifiest generation
+#
+# Use the python script create_nmf to scan the binaries for dependencies using
+# objdump.  Pass in the (-L) paths to the default library toolchains so that we
+# can find those libraries and have it automatically copy the files (-s) to
+# the target directory for us.
+NMF:=python $(NACL_SDK_ROOT)/tools/create_nmf.py
+NMF_ARGS:=-D $(TC_PATH)/x86_64-nacl/bin/objdump -s .
+NMF_PATHS:=-L $(TC_PATH)/x86_64-nacl/lib32 -L $(TC_PATH)/x86_64-nacl/lib
+
+__PROJECT_NMFS__
 
+__PROJECT_PRERUN__
 
 RUN: all
 	python ../httpd.py
 
+__PROJECT_PRELAUNCH__
 
 LAUNCH_NEXE: CHECK_FOR_CHROME all
 	$(CHROME_PATH) $(NEXE_ARGS) "localhost:5103/$(PROJECT).html"
 
+
diff --git a/native_client_sdk/src/build_tools/tests/test_generate_make.py b/native_client_sdk/src/build_tools/tests/test_generate_make.py
index 0dce260..c0d1e42 100755
--- a/native_client_sdk/src/build_tools/tests/test_generate_make.py
+++ b/native_client_sdk/src/build_tools/tests/test_generate_make.py
@@ -26,6 +26,7 @@ BASIC_DESC = {
       'SOURCES' : ['hello_world.c'],
     },
   ],
+  'DEST' : 'examples'
 }
 
 class TestFunctions(unittest.TestCase):
diff --git a/native_client_sdk/src/tools/create_nmf.py b/native_client_sdk/src/tools/create_nmf.py
index 84c6f53..a9c0e40 100755
--- a/native_client_sdk/src/tools/create_nmf.py
+++ b/native_client_sdk/src/tools/create_nmf.py
@@ -93,7 +93,8 @@ class NmfUtils(object):
   '''
 
   def __init__(self, main_files=None, objdump='x86_64-nacl-objdump',
-               lib_path=None, extra_files=None, lib_prefix=None):
+               lib_path=None, extra_files=None, lib_prefix=None,
+               toolchain=None):
     ''' Constructor
 
     Args:
@@ -104,7 +105,9 @@ class NmfUtils(object):
       extra_files: List of extra files to include in the nmf
       lib_prefix: A list of path components to prepend to the library paths,
           both for staging the libraries and for inclusion into the nmf file.
-          Examples:  ['..'], ['lib_dir'] '''
+          Examples:  ['..'], ['lib_dir']
+      toolchain: Specify which toolchain newlib|glibc|pnacl which can require
+          different forms of the NMF.'''
     self.objdump = objdump
     self.main_files = main_files or []
     self.extra_files = extra_files or []
@@ -112,6 +115,7 @@ class NmfUtils(object):
     self.manifest = None
     self.needed = None
     self.lib_prefix = lib_prefix or []
+    self.toolchain = toolchain
 
 
   def GleanFromObjdump(self, files):
@@ -237,7 +241,7 @@ class NmfUtils(object):
           os.path.normcase(os.path.abspath(destination))):
         shutil.copy2(source, destination)
 
-  def _GenerateManifest(self, runnable=True):
+  def _GenerateManifest(self):
     '''Create a JSON formatted dict containing the files
 
     NaCl will map url requests based on architecture.  The startup NEXE
@@ -250,6 +254,7 @@ class NmfUtils(object):
     manifest = { FILES_KEY: {}, PROGRAM_KEY: {} }
     needed = self.GetNeeded()
 
+    runnable = self.toolchain != 'newlib' 
     for need in needed:
       archinfo = needed[need]
       urlinfo = { URL_KEY: archinfo.url }
@@ -277,7 +282,6 @@ class NmfUtils(object):
       fileinfo = manifest[FILES_KEY].get(name, {})
       fileinfo[archinfo.arch] = urlinfo
       manifest[FILES_KEY][name] = fileinfo
-
     self.manifest = manifest
 
   def GetManifest(self):
@@ -296,6 +300,24 @@ class NmfUtils(object):
     return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n'
 
 
+def ErrorOut(text):
+  sys.stderr.write(text + '\n')
+  sys.exit(1)
+
+
+def DetermineToolchain(objdump):
+  objdump = objdump.replace('\\', '/')
+  paths = objdump.split('/')
+  count = len(paths)
+  for index in range(count - 2, 0, -1):
+    if paths[index] == 'toolchain':
+      if paths[index + 1].endswith('newlib'):
+        return 'newlib'
+      if paths[index + 1].endswith('glibc'):
+        return 'glibc'
+  ErrorOut('Could not deternime which toolchain to use.')
+
+
 def Main(argv):
   parser = optparse.OptionParser(
       usage='Usage: %prog [options] nexe [extra_libs...]')
@@ -315,7 +337,16 @@ def Main(argv):
   parser.add_option('-r', '--remove', dest='remove',
                     help='Remove the prefix from the files.',
                     metavar='PATH')
+  parser.add_option('-t', '--toolchain', dest='toolchain',
+                    help='Add DIRECTORY to library search path',
+                    default=None, metavar='TOOLCHAIN')
   (options, args) = parser.parse_args(argv)
+  
+  if not options.toolchain:
+    options.toolchain = DetermineToolchain(os.path.abspath(options.objdump))
+
+  if options.toolchain not in ['newlib', 'glibc']:
+    ErrorOut('Unknown toolchain: ' + str(options.toolchain))
 
   if len(args) < 1:
     parser.print_usage()
@@ -323,10 +354,10 @@ def Main(argv):
 
   nmf = NmfUtils(objdump=options.objdump,
                  main_files=args,
-                 lib_path=options.lib_path)
+                 lib_path=options.lib_path,
+                 toolchain=options.toolchain)
 
   manifest = nmf.GetManifest()
-
   if options.output is None:
     sys.stdout.write(nmf.GetJson())
   else: