Re-enable PNaCl

Renable examples disabled to support newlib/glibc combo.
Updates template, buildsdk, create_nmf to enable PNaCl.
Unifies macro replacement code in generate_make

BUG=130618
TBR=binji@chromium.org

Review URL: https://chromiumcodereview.appspot.com/10543175

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@142382 0039d316-1c4b-4281-b951-d872f2087c98

7 files changed, 185 insertions, 123 deletions

diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py
index 06c25f5..089b0aa 100755
--- a/native_client_sdk/src/build_tools/build_sdk.py
+++ b/native_client_sdk/src/build_tools/build_sdk.py
@@ -232,8 +232,8 @@ def InstallHeaders(tc_dst_inc, pepper_ver, tc_name):
           os.path.join(tc_dst_inc, 'KHR'))
 
   # Copy the lib files
-  buildbot_common.MakeDir(os.path.join(tc_dst_inc, 'lib'))
-  buildbot_common.CopyDir(os.path.join(PPAPI_DIR,'lib'), tc_dst_inc)
+  buildbot_common.CopyDir(os.path.join(PPAPI_DIR,'lib'), 
+          os.path.join(tc_dst_inc, 'ppapi'))
 
 
 def UntarToolchains(pepperdir, platform, arch, toolchains):
@@ -330,44 +330,31 @@ def BuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains):
   else:
     buildbot_common.ErrorExit('Missing arch %s' % arch)
 
-
-EXAMPLE_MAP = {
-  'newlib': [
-    'debugging',
-    'file_histogram',
-    'file_io',
-    'fullscreen_tumbler',
-    'gamepad',
-    'geturl',
-    'hello_world_interactive',
-    'hello_world',
-#    'hello_world_gles',
-    'input_events',
-    'load_progress',
-    'mouselock',
-    'multithreaded_input_events',
-    'pi_generator',
-    'pong',
-    'sine_synth',
-    'tumbler',
-    'websocket'
-  ],
-  'glibc': [
-    'dlopen',
-  ],
-  'pnacl': [
-#    'hello_world_pnacl',
-  ],
-}
-
-
-LIBRARY_MAP = {
-  'newlib': [
-    'gles2',
-  ],
-  'glibc': [],
-  'pnacl': []
-}
+EXAMPLE_LIST = [
+  'debugging',
+  'file_histogram',
+  'file_io',
+  'fullscreen_tumbler',
+  'gamepad',
+  'geturl',
+  'hello_world_interactive',
+  'hello_world',
+  'hello_world_gles',
+  'input_events',
+  'load_progress',
+  'mouselock',
+  'multithreaded_input_events',
+  'pi_generator',
+  'pong',
+  'sine_synth',
+  'tumbler',
+  'websocket',
+  'dlopen',
+]
+
+LIBRARY_LIST = [
+  'gles2',
+]
 
 
 def CopyExamples(pepperdir, toolchains):
@@ -387,22 +374,15 @@ def CopyExamples(pepperdir, toolchains):
   for filename in files:
     oshelpers.Copy(['-v', os.path.join(SDK_EXAMPLE_DIR, filename), exampledir])
 
-  # Add examples for supported toolchains
-  examples = []
-  for tc in toolchains:
-    examples.extend(EXAMPLE_MAP[tc])
-
-  libraries = []
-  for tc in toolchains:
-    libraries.extend(LIBRARY_MAP[tc])
-
-  print 'Process: ' + ' '.join(examples)
-  print 'Process: ' + ' '.join(libraries)
   args = ['--dstroot=%s' % pepperdir, '--master']
-  for example in examples:
+  for toolchain in toolchains:
+    args.append('--' + toolchain)
+
+  for example in EXAMPLE_LIST:
     dsc = os.path.join(SDK_EXAMPLE_DIR, example, 'example.dsc')
     args.append(dsc)
-  for library in libraries:
+
+  for library in LIBRARY_LIST:
     dsc = os.path.join(SDK_LIBRARY_DIR, library, 'library.dsc')
     args.append(dsc)
 
diff --git a/native_client_sdk/src/build_tools/generate_make.py b/native_client_sdk/src/build_tools/generate_make.py
index d6a0825..caa2242 100755
--- a/native_client_sdk/src/build_tools/generate_make.py
+++ b/native_client_sdk/src/build_tools/generate_make.py
@@ -28,6 +28,11 @@ SRC_EXT = {
 }
 
 
+def ErrorExit(text):
+  sys.stderr.write(text + '\n')
+  sys.exit(1)
+
+
 def Replace(text, replacements):
   for key in replacements:
     text = text.replace(key, replacements[key])
@@ -89,19 +94,27 @@ def GenerateCopyList(desc):
 def BuildToolDict(tc, proj, arch='', ext='.nexe', OBJS='', TARG='', REMAP=''):
   TC = tc.upper()
   PROJ = proj.upper()
+  EXT= SRC_EXT[ext]
 
   if not OBJS:
-    OBJS = '%s_%s_%s_%s_O' % (TC, PROJ, arch, ext)
+    OBJS = '%s_%s_%s_%s_O' % (TC, PROJ, arch, EXT)
   
-  if not TARG:
-    TARG = '%s_x86_%s%s' % (proj,arch,ext)
+  if tc in ['newlib', 'glibc']:
+    machine = '-m' + arch
+    if not TARG:
+      TARG = '%s_x86_%s%s' % (proj,arch,ext)
+  else:
+    machine = ''
+    if not TARG:
+      TARG = proj + '.pexe'
 
   replace = {
-    '<ARCH>': arch,
+    '<arch>': arch,
+    '<ARCH>': machine,
     '<CC>': '%s_%s' % (TC, SRC_EXT[ext]),
     '<DUMP>': '%s_DUMP' % TC,
     '<ext>' : ext,
-    '<EXT>' : SRC_EXT[ext],
+    '<EXT>' : EXT,
     '<LINK>': '%s_LINK' % TC,
     '<OBJS>' : OBJS,
     '<proj>': proj,
@@ -117,17 +130,22 @@ def BuildToolDict(tc, proj, arch='', ext='.nexe', OBJS='', TARG='', REMAP=''):
 
 def GenerateNEXE(toolchain, name, ext, cc_sources, cxx_sources):
   COMPILE_RULE = """
-<OBJS>:=$(patsubst %.<ext>, <tc>/%_<ARCH>.o,$(<PROJ>_<EXT>))
-$(<OBJS>) : <tc>/%_<ARCH>.o : %.<ext> $(THIS_MAKE) | <tc>
-<TAB>$(<CC>) -o $@ $< -m<ARCH> $(<PROJ>_<EXT>FLAGS) -DTCNAME=<tc>
+<OBJS>:=$(patsubst %.<ext>, <tc>/%_<arch>.o,$(<PROJ>_<EXT>))
+$(<OBJS>) : <tc>/%_<arch>.o : %.<ext> $(THIS_MAKE) | <tc>
+<TAB>$(<CC>) -o $@ $< <ARCH> $(<PROJ>_<EXT>FLAGS) -DTCNAME=<tc>
 """
   LINK_RULE = """
 <tc>/<TARG> : $(<OBJS>)
-<TAB>$(<LINK>) -o $@ $^ -m<ARCH> $(<PROJ>_LDFLAGS)
+<TAB>$(<LINK>) -o $@ $^ <ARCH> $(<PROJ>_LDFLAGS)
 """
   rules = ''
   targs = []
-  for arch in ARCHITECTURES:
+  if toolchain in ['newlib', 'glibc']:
+    archs = ['32', '64']
+  else:
+    archs = [ toolchain.upper() ]
+
+  for arch in archs:
     object_sets = []
     remap = ''
     if cc_sources:
@@ -148,21 +166,36 @@ $(<OBJS>) : <tc>/%_<ARCH>.o : %.<ext> $(THIS_MAKE) | <tc>
       continue
     if ext == '.so':
       remap = ' -n %s,%s.so' % (replace['<TARG>'], name)
-    rules += '%s_NMF+=%s/%s\n' % (replace['<TC>'], toolchain, replace['<TARG>'])
+    if toolchain in ['newlib', 'glibc']:
+      rules += '%s_NMF+=%s/%s\n' % (replace['<TC>'], toolchain, replace['<TARG>'])
     if remap:
       rules += '%s_REMAP+=%s\n' % (replace['<TC>'], remap)
   return rules
 
 
-def GenerateReplacements(desc):
+def GenerateReplacements(desc, toolchains):
+  TRANSLATE_RULE = """
+<tc>/<proj>_x86_32.nexe : <tc>/<proj>.pexe
+<TAB>$(TRANSLATE) -arch x86-32 $< -o $@ 
+
+<tc>/<proj>_x86_64.nexe : <tc>/<proj>.pexe
+<TAB>$(TRANSLATE) -arch x86-64 $< -o $@ 
+
+<tc>/<proj>_arm.nexe : <tc>/<proj>.pexe
+<TAB>$(TRANSLATE) -arch arm $< -o $@ 
+PNACL_NMF:=<tc>/<proj>_x86_32.nexe <tc>/<proj>_x86_64.nexe <tc>/<proj>_arm.nexe
+"""
   NMF_RULE = """
 <tc>/<proj>.nmf : $(<TC>_NMF)
 <TAB>$(NMF) -D $(<DUMP>) -o $@ $(<TC>_PATHS) $^ -t <tc> -s <tc> $(<TC>_REMAP)
 
 """
   # Generate target settings
-  tools = desc['TOOLS']
-
+  tools = []
+  for tool in desc['TOOLS']:
+    if tool in toolchains:
+      tools.append(tool)
+  
   prerun = desc.get('PRE', '')
   postlaunch = desc.get('POST', '')
   prelaunch = desc.get('LAUNCH', '')
@@ -210,6 +243,9 @@ def GenerateReplacements(desc):
         nexe = name
         targets.append('%s/%s.nmf' % (tc, name))
     if nexe: 
+      if tc == 'pnacl':
+        replace = BuildToolDict(tc, nexe)
+        rules += Replace(TRANSLATE_RULE, replace)
       replace = BuildToolDict(tc, nexe)
       rules += Replace(NMF_RULE, replace)
 
@@ -329,7 +365,6 @@ def ValidateFormat(src, format, ErrorMsg=ErrorMsgFunc):
     # If we got this far, it's an unexpected type
     ErrorMsg('Unexpected type %s for key %s.' % (str(type(src[key])), key))
     continue
-
   return not failed
 
 
@@ -376,19 +411,38 @@ def IsNexe(desc):
       return True
   return False
 
-def ProcessProject(dstroot, filename):
+
+def LoadProject(filename, toolchains):
+  """Generate a Master Makefile that builds all examples. 
+
+  Load a project desciption file, verifying it conforms and checking
+  if it matches the set of requested toolchains.  Return None if the
+  project is filtered out."""
+
   print '\n\nProcessing %s...' % filename
   # Default src directory is the directory the description was found in 
-  srcroot = os.path.dirname(os.path.abspath(filename))
-  desc = open(filename, 'rb').read()
+  desc = open(filename, 'r').read()
   desc = eval(desc, {}, {})
+
+  # Verify the format of this file
   if not ValidateFormat(desc, DSC_FORMAT):
-    return (None, None)
+    ErrorExit('Failed to validate: ' + filename)
+
+  # Check if we are actually interested in this example
+  match = False
+  for toolchain in toolchains:
+    if toolchain in desc['TOOLS']:
+      match = True
+      break
+  if not match:
+    return None
+  return desc
 
+
+def ProcessProject(srcroot, dstroot, desc, toolchains):
   name = desc['NAME']
   out_dir = os.path.join(dstroot, desc['DEST'], name)
   buildbot_common.MakeDir(out_dir)
-
   srcdirs = desc.get('SEARCH', ['.'])
 
   # Copy sources to example directory
@@ -405,9 +459,8 @@ def ProcessProject(dstroot, filename):
   else:
     template=os.path.join(SCRIPT_DIR, 'library.mk')
 
-
   # Add Makefile
-  repdict = GenerateReplacements(desc)
+  repdict = GenerateReplacements(desc, toolchains)
   make_path = os.path.join(out_dir, 'Makefile')
   WriteMakefile(template, make_path, repdict)
 
@@ -418,32 +471,14 @@ def ProcessProject(dstroot, filename):
 
 
 def GenerateExamplesMakefile(in_path, out_path, examples):
-  """Generate a Makefile that includes only the examples supported by this
-  SDK."""
-  # Line wrap the PROJECTS variable
-  wrap_width = 80
-  projects_text = SetVar('PROJECTS', examples)
-
-  out_makefile_text = ''
-  wrote_projects_text = False
-  snipping = False
-  for line in open(in_path, 'r'):
-    if line.startswith('# =SNIP='):
-      snipping = not snipping
-      continue
+  """Generate a Master Makefile that builds all examples. """
+  replace = {  '__PROJECT_LIST__' : SetVar('PROJECTS', examples) }
+  WriteMakefile(in_path, out_path, replace)
 
-    if snipping:
-      if not wrote_projects_text:
-        out_makefile_text += projects_text
-      wrote_projects_text = True
-    else:
-      out_makefile_text += line
-  open(out_path, 'w').write(out_makefile_text)
-  
   outdir = os.path.dirname(os.path.abspath(out_path))
   pepperdir = os.path.dirname(outdir)
   AddMakeBat(pepperdir, outdir)
-
+  
 
 def main(argv):
   parser = optparse.OptionParser()
@@ -451,23 +486,49 @@ def main(argv):
       dest='dstroot', default=OUT_DIR)
   parser.add_option('--master', help='Create master Makefile.',
       action='store_true', dest='master', default=False)
+  parser.add_option('--newlib', help='Create newlib examples.',
+      action='store_true', dest='newlib', default=False)
+  parser.add_option('--glibc', help='Create glibc examples.',
+      action='store_true', dest='glibc', default=False)
+  parser.add_option('--pnacl', help='Create pnacl examples.',
+      action='store_true', dest='pnacl', default=False)
+  parser.add_option('--host', help='Create host examples.',
+      action='store_true', dest='host', default=False)
+
+  toolchains = []
+  options, args = parser.parse_args(argv)
+  if options.newlib:
+    toolchains.append('newlib')
+  if options.glibc:
+    toolchains.append('glibc')
+  if options.pnacl:
+    toolchains.append('pnacl')
+  if options.host:
+    toolchains.append('host')
+
+  # By default support newlib and glibc
+  if not toolchains:
+    toolchains = ['newlib', 'glibc']
+    print 'Using default toolchains: ' + ' '.join(toolchains)
 
   examples = []
-  options, args = parser.parse_args(argv)
   for filename in args:
-    name, dest = ProcessProject(options.dstroot, filename)
-    if not name:
-      print '\n*** Failed to process project: %s ***' % filename
-      return 1
+    desc = LoadProject(filename, toolchains)
+    if not desc:
+      print 'Skipping %s, not in [%s].' % (filename, ', '.join(toolchains))
+      continue
+
+    srcroot = os.path.dirname(os.path.abspath(filename))
+    if not ProcessProject(srcroot, options.dstroot, desc, toolchains):
+      ErrorExit('\n*** Failed to process project: %s ***' % filename)
 
-    if dest == 'examples':
-      examples.append(name)
+    if desc['DEST'] == 'examples':
+      examples.append(desc['NAME'])
 
   if options.master:
     master_in = os.path.join(SDK_EXAMPLE_DIR, 'Makefile')
     master_out = os.path.join(options.dstroot, 'examples', 'Makefile')
     GenerateExamplesMakefile(master_in, master_out, examples)
-
   return 0
 
 
diff --git a/native_client_sdk/src/build_tools/template.mk b/native_client_sdk/src/build_tools/template.mk
index e178a15..c76eb17 100644
--- a/native_client_sdk/src/build_tools/template.mk
+++ b/native_client_sdk/src/build_tools/template.mk
@@ -23,7 +23,7 @@ CHROME_PATH?=Undefined
 NACL_WARNINGS:=-Wno-long-long -Wall -Wswitch-enum -Werror -pedantic
 NACL_CCFLAGS:=-O0 -g -pthread $(NACL_WARNINGS)
 NACL_CXXFLAGS:= -O0 -g -pthread -std=gnu++98 $(NACL_WARNINGS)
-NACL_LDFLAGS:=-Wl,-as-needed -g -pthread -lppapi_cpp -lppapi 
+NACL_LDFLAGS:=-g -pthread -lppapi_cpp -lppapi 
 
 #
 # Project Settings
@@ -85,17 +85,23 @@ export CYGWIN
 #
 NEWLIB_CC?=$(TC_PATH)/$(OSNAME)_x86_newlib/bin/i686-nacl-gcc -c
 NEWLIB_CXX?=$(TC_PATH)/$(OSNAME)_x86_newlib/bin/i686-nacl-g++ -c
-NEWLIB_LINK?=$(TC_PATH)/$(OSNAME)_x86_newlib/bin/i686-nacl-g++
+NEWLIB_LINK?=$(TC_PATH)/$(OSNAME)_x86_newlib/bin/i686-nacl-g++ -Wl,-as-needed
 NEWLIB_DUMP?=$(TC_PATH)/$(OSNAME)_x86_newlib/x86_64-nacl/bin/objdump
 NEWLIB_PATHS:=
 
 GLIBC_CC?=$(TC_PATH)/$(OSNAME)_x86_glibc/bin/i686-nacl-gcc -c
 GLIBC_CXX?=$(TC_PATH)/$(OSNAME)_x86_glibc/bin/i686-nacl-g++ -c
-GLIBC_LINK?=$(TC_PATH)/$(OSNAME)_x86_glibc/bin/i686-nacl-g++
+GLIBC_LINK?=$(TC_PATH)/$(OSNAME)_x86_glibc/bin/i686-nacl-g++ -Wl,-as-needed
 GLIBC_DUMP?=$(TC_PATH)/$(OSNAME)_x86_glibc/x86_64-nacl/bin/objdump
 GLIBC_PATHS:=-L $(TC_PATH)/$(OSNAME)_x86_glibc/x86_64-nacl/lib32
 GLIBC_PATHS+=-L $(TC_PATH)/$(OSNAME)_x86_glibc/x86_64-nacl/lib
 
+PNACL_CC?=$(TC_PATH)/$(OSNAME)_x86_pnacl/newlib/bin/pnacl-clang -c
+PNACL_CXX?=$(TC_PATH)/$(OSNAME)_x86_pnacl/newlib/bin/pnacl-clang -c
+PNACL_LINK?=$(TC_PATH)/$(OSNAME)_x86_pnacl/newlib/bin/pnacl-clang
+PNACL_DUMP?=$(TC_PATH)/$(OSNAME)_x86_pnacl/newlib/bin/objdump
+PNACL_PATHS:=
+TRANSLATE:=$(TC_PATH)/$(OSNAME)_x86_pnacl/newlib/bin/pnacl-translate
 #
 # NMF Manifiest generation
 #
diff --git a/native_client_sdk/src/examples/Makefile b/native_client_sdk/src/examples/Makefile
index 7ad5e22..78a80ed 100644
--- a/native_client_sdk/src/examples/Makefile
+++ b/native_client_sdk/src/examples/Makefile
@@ -7,12 +7,7 @@
 #   http://www.gnu.org/software/make/manual/make.html
 #
 
-# =SNIP=
-# This Makefile is used as a template to generate the real Makefile that is
-# included in the SDK distribution. The toolchain-specific project lists can be
-# found in build_tools/build_sdk.py.
-$(error MISSING PROJECTS)
-# =SNIP=
+__PROJECT_LIST__
 
 # Define the default target
 all:
diff --git a/native_client_sdk/src/examples/hello_world/example.dsc b/native_client_sdk/src/examples/hello_world/example.dsc
index fbc1a01..6d1cde6 100644
--- a/native_client_sdk/src/examples/hello_world/example.dsc
+++ b/native_client_sdk/src/examples/hello_world/example.dsc
@@ -1,10 +1,11 @@
 {
-  'TOOLS': ['newlib', 'glibc'],
+  'TOOLS': ['newlib', 'glibc', 'pnacl'],
   'TARGETS': [
     {
       'NAME' : 'hello_world',
       'TYPE' : 'main',
       'SOURCES' : ['hello_world.c'],
+      'LDFLAGS' : ['-g', '-pthread', '-lppapi']
     }
   ],
   'DEST': 'examples',
diff --git a/native_client_sdk/src/examples/hello_world_gles/example.dsc b/native_client_sdk/src/examples/hello_world_gles/example.dsc
index 3738886..13076162 100644
--- a/native_client_sdk/src/examples/hello_world_gles/example.dsc
+++ b/native_client_sdk/src/examples/hello_world_gles/example.dsc
@@ -7,10 +7,10 @@
       'SOURCES' : ['hello_world.cc', 'matrix.cc', 'matrix.h'],
       'CXXFLAGS': [
         '$(NACL_CXXFLAGS)', 
-        '-I../../libraries',
-        '-I$../../libraries/ppapi/lib/gl'
+        '-I../../src',
+        '-I../../src/ppapi/lib/gl'
       ],
-      'LDFLAGS': ['$(NACL_LDFLAGS)', '-lppapi_gles2']
+      'LDFLAGS': ['-g', '-pthread', '-lppapi', '-lppapi_gles2']
     }
   ],
   'DATA': [
diff --git a/native_client_sdk/src/tools/create_nmf.py b/native_client_sdk/src/tools/create_nmf.py
index 165afe1..da6924c 100755
--- a/native_client_sdk/src/tools/create_nmf.py
+++ b/native_client_sdk/src/tools/create_nmf.py
@@ -36,6 +36,12 @@ ARCH_LOCATION = {
     'x86-64': 'lib64',
 }
 
+NAME_ARCH_MAP = {
+    '32.nexe': 'x86-32',
+    '64.nexe': 'x64-64',
+    'arm.nexe': 'arm'
+}
+
 # These constants are used within nmf files.
 RUNNABLE_LD = 'runnable-ld.so'  # Name of the dynamic loader
 MAIN_NEXE = 'main.nexe'  # Name of entry point for execution
@@ -195,8 +201,12 @@ class NmfUtils(object):
           Includes the input files as well, with arch filled in if absent.
           Example: { '/path/to/my.nexe': ArchFile(my.nexe),
                      '/path/to/libfoo.so': ArchFile(libfoo.so) }'''
-    if not self.needed:
-      DebugPrint('GetNeeded(%s)' % self.main_files)
+    if self.needed:
+      return self.needed
+
+    runnable = (self.toolchain != 'newlib' and self.toolchain != 'pnacl')
+    DebugPrint('GetNeeded(%s)' % self.main_files)
+    if runnable:    
       examined = set()
       all_files, unexamined = self.GleanFromObjdump(
           dict([(file, None) for file in self.main_files]))
@@ -221,6 +231,15 @@ class NmfUtils(object):
         if arch_map.name in ldso:
           del all_files[name]
       self.needed = all_files
+    else:
+      need = {}
+      for filename in self.main_files:
+        arch = filename.split('_')[-1]
+        arch = NAME_ARCH_MAP[arch]
+        url = os.path.split(filename)[1]
+        need[filename] = ArchFile(arch=arch, name=os.path.basename(filename),
+                                  path=filename, url=url)
+      self.needed = need        
     return self.needed
 
   def StageDependencies(self, destination_dir):
@@ -255,9 +274,9 @@ class NmfUtils(object):
     FILES key mapped as 'main.exe' instead of it's original name so that the
     loader can find it.'''
     manifest = { FILES_KEY: {}, PROGRAM_KEY: {} }
-    needed = self.GetNeeded()
+    runnable = (self.toolchain != 'newlib' and self.toolchain != 'pnacl')
 
-    runnable = self.toolchain != 'newlib' 
+    needed = self.GetNeeded()
     for need in needed:
       archinfo = needed[need]
       urlinfo = { URL_KEY: archinfo.url }
@@ -352,7 +371,7 @@ def Main(argv):
   if not options.toolchain:
     options.toolchain = DetermineToolchain(os.path.abspath(options.objdump))
 
-  if options.toolchain not in ['newlib', 'glibc']:
+  if options.toolchain not in ['newlib', 'glibc', 'pnacl']:
     ErrorOut('Unknown toolchain: ' + str(options.toolchain))
 
   if len(args) < 1: