[NaCl SDK] Refactor create_nmf.py

Most of the complex logic is moved to elf.py or get_shared_deps.py (each with
their own simpler tests). These changes will make it easier to handle
generating a multi-platform NMF.

BUG=none
R=sbc@chromium.org

Review URL: https://codereview.chromium.org/161613002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@255474 0039d316-1c4b-4281-b951-d872f2087c98

16 files changed, 1140 insertions, 445 deletions

diff --git a/native_client_sdk/src/build_tools/build_projects.py b/native_client_sdk/src/build_tools/build_projects.py
index 3ca7941..9352e50 100755
--- a/native_client_sdk/src/build_tools/build_projects.py
+++ b/native_client_sdk/src/build_tools/build_projects.py
@@ -73,6 +73,12 @@ def UpdateHelpers(pepperdir, clobber=False):
   buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'),
       tools_dir)
 
+  # Copy tools/lib scripts
+  tools_lib_dir = os.path.join(pepperdir, 'tools', 'lib')
+  buildbot_common.MakeDir(tools_lib_dir)
+  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', 'lib', '*.py'),
+      tools_lib_dir)
+
   # On Windows add a prebuilt make
   if getos.GetPlatform() == 'win':
     buildbot_common.BuildStep('Add MAKE')
diff --git a/native_client_sdk/src/build_tools/sdk_files.list b/native_client_sdk/src/build_tools/sdk_files.list
index cdadcf3..00ac19e 100644
--- a/native_client_sdk/src/build_tools/sdk_files.list
+++ b/native_client_sdk/src/build_tools/sdk_files.list
@@ -421,6 +421,9 @@ tools/host_vc.mk
 tools/httpd.py
 tools/irt_core_x86_32.nexe
 tools/irt_core_x86_64.nexe
+tools/lib/elf.py
+tools/lib/get_shared_deps.py
+tools/lib/quote.py
 [win]tools/make.exe
 [linux,mac]tools/minidump_dump
 [linux,mac]tools/minidump_stackwalk
@@ -432,7 +435,6 @@ tools/nacl_llvm.mk
 tools/ncval${EXE_EXT}
 tools/ncval.py
 tools/oshelpers.py
-tools/quote.py
 tools/run.py
 tools/sel_ldr.py
 tools/sel_ldr_x86_32${EXE_EXT}
diff --git a/native_client_sdk/src/test_all.py b/native_client_sdk/src/test_all.py
index 6e433e7..c91e64e 100755
--- a/native_client_sdk/src/test_all.py
+++ b/native_client_sdk/src/test_all.py
@@ -10,14 +10,17 @@ import unittest
 # add tools folder to sys.path
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(os.path.join(SCRIPT_DIR, 'tools', 'tests'))
+sys.path.append(os.path.join(SCRIPT_DIR, 'tools', 'lib', 'tests'))
 sys.path.append(os.path.join(SCRIPT_DIR, 'build_tools', 'tests'))
 
 TEST_MODULES = [
     'create_html_test',
     'create_nmf_test',
     'easy_template_test',
+    'elf_test',
     'fix_deps_test',
     'getos_test',
+    'get_shared_deps_test',
     'httpd_test',
     'nacl_config_test',
     'oshelpers_test',
diff --git a/native_client_sdk/src/tools/create_nmf.py b/native_client_sdk/src/tools/create_nmf.py
index 35b2705..e4521b35 100755
--- a/native_client_sdk/src/tools/create_nmf.py
+++ b/native_client_sdk/src/tools/create_nmf.py
@@ -6,45 +6,32 @@
 """Tool for automatically creating .nmf files from .nexe/.pexe executables.
 
 As well as creating the nmf file this tool can also find and stage
-any shared libraries dependancies that the executables might have.
+any shared libraries dependencies that the executables might have.
 """
 
 import errno
 import json
 import optparse
 import os
-import re
+import posixpath
 import shutil
-import struct
-import subprocess
 import sys
 
 import getos
-import quote
 
 if sys.version_info < (2, 6, 0):
   sys.stderr.write("python 2.6 or later is required run this script\n")
   sys.exit(1)
 
-NeededMatcher = re.compile('^ *NEEDED *([^ ]+)\n$')
-FormatMatcher = re.compile('^(.+):\\s*file format (.+)\n$')
-
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+LIB_DIR = os.path.join(SCRIPT_DIR, 'lib')
+
+sys.path.append(LIB_DIR)
+
+import elf
+import get_shared_deps
+import quote
 
-OBJDUMP_ARCH_MAP = {
-    # Names returned by Linux's objdump:
-    'elf64-x86-64': 'x86-64',
-    'elf32-i386': 'x86-32',
-    'elf32-little': 'arm',
-    'elf32-littlearm': 'arm',
-    # Names returned by old x86_64-nacl-objdump:
-    'elf64-nacl': 'x86-64',
-    'elf32-nacl': 'x86-32',
-    # Names returned by new x86_64-nacl-objdump:
-    'elf64-x86-64-nacl': 'x86-64',
-    'elf32-x86-64-nacl': 'x86-64',
-    'elf32-i386-nacl': 'x86-32',
-}
 
 ARCH_LOCATION = {
     'x86-32': 'lib32',
@@ -64,15 +51,6 @@ PORTABLE_KEY = 'portable' # key for portable section of manifest
 TRANSLATE_KEY = 'pnacl-translate' # key for translatable objects
 
 
-# The proper name of the dynamic linker, as kept in the IRT.  This is
-# excluded from the nmf file by convention.
-LD_NACL_MAP = {
-    'x86-32': 'ld-nacl-x86-32.so.1',
-    'x86-64': 'ld-nacl-x86-64.so.1',
-    'arm': None,
-}
-
-
 def DebugPrint(message):
   if DebugPrint.debug_mode:
     sys.stderr.write('%s\n' % message)
@@ -81,6 +59,41 @@ def DebugPrint(message):
 DebugPrint.debug_mode = False  # Set to True to enable extra debug prints
 
 
+def SplitPath(path):
+  """Returns all components of a path as a list.
+
+  e.g.
+  'foo/bar/baz.blah' => ['foo', 'bar', 'baz.blah']
+  """
+  result = []
+  while path:
+    path, part = os.path.split(path)
+    result.append(part)
+  return result[::-1]  # Reverse.
+
+
+def MakePosixPath(path):
+  """Converts from the native format to posixpath format.
+
+  e.g. on Windows, "foo\\bar\\baz.blah" => "foo/bar/baz.blah"
+  on Mac/Linux this is a no-op.
+  """
+  if os.path == posixpath:
+    return path
+  return posixpath.join(*SplitPath(path))
+
+
+def PosixRelPath(path, start):
+  """Takes two paths in native format, and produces a relative path in posix
+  format.
+
+  e.g.
+  For Windows: "foo\\bar\\baz.blah", "foo" => "bar/baz.blah"
+  For Mac/Linux: "foo/bar/baz.blah", "foo" => "bar/baz.blah"
+  """
+  return MakePosixPath(os.path.relpath(path, start))
+
+
 def MakeDir(dirname):
   """Just like os.makedirs but doesn't generate errors when dirname
   already exists.
@@ -96,109 +109,31 @@ def MakeDir(dirname):
       raise
 
 
-class Error(Exception):
-  '''Local Error class for this file.'''
-  pass
-
-
 def ParseElfHeader(path):
-  """Determine properties of a nexe by parsing elf header.
-  Return tuple of architecture and boolean signalling whether
-  the executable is dynamic (has INTERP header) or static.
-  """
-  # From elf.h:
-  # typedef struct
-  # {
-  #   unsigned char e_ident[EI_NIDENT]; /* Magic number and other info */
-  #   Elf64_Half e_type; /* Object file type */
-  #   Elf64_Half e_machine; /* Architecture */
-  #   ...
-  # } Elf32_Ehdr;
-  elf_header_format = '16s2H'
-  elf_header_size = struct.calcsize(elf_header_format)
-
-  with open(path, 'rb') as f:
-    header = f.read(elf_header_size)
-
+  """Wrap elf.ParseElfHeader to return raise this module's Error on failure."""
   try:
-    header = struct.unpack(elf_header_format, header)
-  except struct.error:
-    raise Error("error parsing elf header: %s" % path)
-  e_ident, _, e_machine = header[:3]
-
-  elf_magic = '\x7fELF'
-  if e_ident[:4] != elf_magic:
-    raise Error('Not a valid NaCl executable: %s' % path)
-
-  e_machine_mapping = {
-    3 : 'x86-32',
-    40 : 'arm',
-    62 : 'x86-64'
-  }
-  if e_machine not in e_machine_mapping:
-    raise Error('Unknown machine type: %s' % e_machine)
-
-  # Set arch based on the machine type in the elf header
-  arch = e_machine_mapping[e_machine]
-
-  # Now read the full header in either 64bit or 32bit mode
-  dynamic = IsDynamicElf(path, arch == 'x86-64')
-  return arch, dynamic
-
-
-def IsDynamicElf(path, is64bit):
-  """Examine an elf file to determine if it is dynamically
-  linked or not.
-  This is determined by searching the program headers for
-  a header of type PT_INTERP.
-  """
-  if is64bit:
-    elf_header_format = '16s2HI3QI3H'
-  else:
-    elf_header_format = '16s2HI3II3H'
-
-  elf_header_size = struct.calcsize(elf_header_format)
-
-  with open(path, 'rb') as f:
-    header = f.read(elf_header_size)
-    header = struct.unpack(elf_header_format, header)
-    p_header_offset = header[5]
-    p_header_entry_size = header[9]
-    num_p_header = header[10]
-    f.seek(p_header_offset)
-    p_headers = f.read(p_header_entry_size*num_p_header)
-
-  # Read the first word of each Phdr to find out its type.
-  #
-  # typedef struct
-  # {
-  #   Elf32_Word  p_type;     /* Segment type */
-  #   ...
-  # } Elf32_Phdr;
-  elf_phdr_format = 'I'
-  PT_INTERP = 3
-
-  while p_headers:
-    p_header = p_headers[:p_header_entry_size]
-    p_headers = p_headers[p_header_entry_size:]
-    phdr_type = struct.unpack(elf_phdr_format, p_header[:4])[0]
-    if phdr_type == PT_INTERP:
-      return True
-
-  return False
+    return elf.ParseElfHeader(path)
+  except elf.Error, e:
+    raise Error(str(e))
+
+
+class Error(Exception):
+  """Local Error class for this file."""
+  pass
 
 
 class ArchFile(object):
-  '''Simple structure containing information about
+  """Simple structure containing information about an architecture-specific
+     file.
 
   Attributes:
     name: Name of this file
     path: Full path to this file on the build system
     arch: Architecture of this file (e.g., x86-32)
     url: Relative path to file in the staged web directory.
-        Used for specifying the "url" attribute in the nmf file.'''
+        Used for specifying the "url" attribute in the nmf file."""
 
-  def __init__(self, name, path, url, arch=None):
+  def __init__(self, name, path, url=None, arch=None):
     self.name = name
     self.path = path
     self.url = url
@@ -210,22 +145,18 @@ class ArchFile(object):
     return '<ArchFile %s>' % self.path
 
   def __str__(self):
-    '''Return the file path when invoked with the str() function'''
+    """Return the file path when invoked with the str() function"""
     return self.path
 
 
 class NmfUtils(object):
-  '''Helper class for creating and managing nmf files
-
-  Attributes:
-    manifest: A JSON-structured dict containing the nmf structure
-    needed: A dict with key=filename and value=ArchFile (see GetNeeded)
-  '''
+  """Helper class for creating and managing nmf files"""
 
   def __init__(self, main_files=None, objdump=None,
                lib_path=None, extra_files=None, lib_prefix=None,
-               remap=None, pnacl_optlevel=None):
-    '''Constructor
+               nexe_prefix=None, no_arch_prefix=None, remap=None,
+               pnacl_optlevel=None, nmf_root=None):
+    """Constructor
 
     Args:
       main_files: List of main entry program files.  These will be named
@@ -233,22 +164,34 @@ class NmfUtils(object):
       objdump: path to x86_64-nacl-objdump tool (or Linux equivalent)
       lib_path: List of paths to library directories
       extra_files: List of extra files to include in the nmf
-      lib_prefix: A list of path components to prepend to the library paths,
-          both for staging the libraries and for inclusion into the nmf file.
-          Examples:  ['..'], ['lib_dir']
+      lib_prefix: A path prefix to prepend to the library paths, both for
+          staging the libraries and for inclusion into the nmf file.
+          Example: '../lib_dir'
+      nexe_prefix: Like lib_prefix, but is prepended to the nexes instead.
+      no_arch_prefix: Don't prefix shared libraries by lib32/lib64.
       remap: Remaps the library name in the manifest.
       pnacl_optlevel: Optimization level for PNaCl translation.
-      '''
+      nmf_root: Directory of the NMF. All urls are relative to this directory.
+      """
+    assert len(main_files) > 0
     self.objdump = objdump
-    self.main_files = main_files or []
+    self.main_files = main_files
     self.extra_files = extra_files or []
     self.lib_path = lib_path or []
     self.manifest = None
-    self.needed = {}
-    self.lib_prefix = lib_prefix or []
+    self.needed = None
+    self.lib_prefix = lib_prefix or ''
+    self.nexe_prefix = nexe_prefix or ''
+    self.no_arch_prefix = no_arch_prefix
     self.remap = remap or {}
-    self.pnacl = main_files and main_files[0].endswith('pexe')
+    self.pnacl = main_files[0].endswith('pexe')
     self.pnacl_optlevel = pnacl_optlevel
+    if nmf_root:
+      self.nmf_root = nmf_root
+    else:
+      # To match old behavior, if there is no nmf_root, use the directory of
+      # the first nexe found in main_files.
+      self.nmf_root = os.path.dirname(main_files[0])
 
     for filename in self.main_files:
       if not os.path.exists(filename):
@@ -256,186 +199,101 @@ class NmfUtils(object):
       if not os.path.isfile(filename):
         raise Error('Input is not a file: %s' % filename)
 
-  def GleanFromObjdump(self, files, arch):
-    '''Get architecture and dependency information for given files
-
-    Args:
-      files: A list of files to examine.
-          [ '/path/to/my.nexe',
-            '/path/to/lib64/libmy.so',
-            '/path/to/mydata.so',
-            '/path/to/my.data' ]
-      arch: The architecure we are looking for, or None to accept any
-            architecture.
-
-    Returns: A tuple with the following members:
-      input_info: A dict with key=filename and value=ArchFile of input files.
-          Includes the input files as well, with arch filled in if absent.
-          Example: { '/path/to/my.nexe': ArchFile(my.nexe),
-                     '/path/to/libfoo.so': ArchFile(libfoo.so) }
-      needed: A set of strings formatted as "arch/name".  Example:
-          set(['x86-32/libc.so', 'x86-64/libgcc.so'])
-    '''
-    if not self.objdump:
-      self.objdump = FindObjdumpExecutable()
-      if not self.objdump:
-        raise Error('No objdump executable found (see --help for more info)')
-
-    full_paths = set()
-    for filename in files:
-      if os.path.exists(filename):
-        full_paths.add(filename)
-      else:
-        for path in self.FindLibsInPath(filename):
-          full_paths.add(path)
-
-    cmd = [self.objdump, '-p'] + list(full_paths)
-    DebugPrint('GleanFromObjdump[%s](%s)' % (arch, cmd))
-    env = {'LANG': 'en_US.UTF-8'}
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE, bufsize=-1,
-                            env=env)
-
-    input_info = {}
-    found_basenames = set()
-    needed = set()
-    output, err_output = proc.communicate()
-    if proc.returncode:
-      raise Error('%s\nStdError=%s\nobjdump failed with error code: %d' %
-                  (output, err_output, proc.returncode))
-
-    file_arch = None
-    for line in output.splitlines(True):
-      # Objdump should display the architecture first and then the dependencies
-      # second for each file in the list.
-      matched = FormatMatcher.match(line)
-      if matched:
-        filename = matched.group(1)
-        file_arch = OBJDUMP_ARCH_MAP[matched.group(2)]
-        if arch and file_arch != arch:
-          continue
-        name = os.path.basename(filename)
-        found_basenames.add(name)
-        input_info[filename] = ArchFile(
-            arch=file_arch,
-            name=name,
-            path=filename,
-            url='/'.join(self.lib_prefix + [ARCH_LOCATION[file_arch], name]))
-      matched = NeededMatcher.match(line)
-      if matched:
-        assert file_arch is not None
-        match = '/'.join([file_arch, matched.group(1)])
-        needed.add(match)
-        Trace("NEEDED: %s" % match)
-
-    for filename in files:
-      if os.path.basename(filename) not in found_basenames:
-        raise Error('Library not found [%s]: %s' % (arch, filename))
-
-    return input_info, needed
-
-  def FindLibsInPath(self, name):
-    '''Finds the set of libraries matching |name| within lib_path
-
-    Args:
-      name: name of library to find
-
-    Returns:
-      A list of system paths that match the given name within the lib_path'''
-    files = []
-    for dirname in self.lib_path:
-      filename = os.path.join(dirname, name)
-      if os.path.exists(filename):
-        files.append(filename)
-    if not files:
-      raise Error('cannot find library %s' % name)
-    return files
-
   def GetNeeded(self):
-    '''Collect the list of dependencies for the main_files
+    """Collect the list of dependencies for the main_files
 
     Returns:
       A dict with key=filename and value=ArchFile of input files.
           Includes the input files as well, with arch filled in if absent.
           Example: { '/path/to/my.nexe': ArchFile(my.nexe),
-                     '/path/to/libfoo.so': ArchFile(libfoo.so) }'''
+                     '/path/to/libfoo.so': ArchFile(libfoo.so) }"""
 
     if self.needed:
       return self.needed
 
     DebugPrint('GetNeeded(%s)' % self.main_files)
 
-    dynamic = any(ParseElfHeader(f)[1] for f in self.main_files)
-
-    if dynamic:
-      examined = set()
-      all_files, unexamined = self.GleanFromObjdump(self.main_files, None)
-      for arch_file in all_files.itervalues():
-        arch_file.url = arch_file.path
-        if unexamined:
-          unexamined.add('/'.join([arch_file.arch, RUNNABLE_LD]))
-
-      while unexamined:
-        files_to_examine = {}
-
-        # Take all the currently unexamined files and group them
-        # by architecture.
-        for arch_name in unexamined:
-          arch, name = arch_name.split('/')
-          files_to_examine.setdefault(arch, []).append(name)
-
-        # Call GleanFromObjdump() for each architecture.
-        needed = set()
-        for arch, files in files_to_examine.iteritems():
-          new_files, new_needed = self.GleanFromObjdump(files, arch)
-          all_files.update(new_files)
-          needed |= new_needed
-
-        examined |= unexamined
-        unexamined = needed - examined
-
-      # With the runnable-ld.so scheme we have today, the proper name of
-      # the dynamic linker should be excluded from the list of files.
-      ldso = [LD_NACL_MAP[arch] for arch in set(OBJDUMP_ARCH_MAP.values())]
-      for name, arch_file in all_files.items():
-        if arch_file.name in ldso:
-          del all_files[name]
-
-      self.needed = all_files
-    else:
-      for filename in self.main_files:
-        url = os.path.split(filename)[1]
-        archfile = ArchFile(name=os.path.basename(filename),
-                            path=filename, url=url)
-        self.needed[filename] = archfile
+    if not self.objdump:
+      self.objdump = FindObjdumpExecutable()
+
+    try:
+      all_files = get_shared_deps.GetNeeded(self.main_files, self.objdump,
+                                            self.lib_path)
+    except get_shared_deps.NoObjdumpError:
+      raise Error('No objdump executable found (see --help for more info)')
+    except get_shared_deps.Error, e:
+      raise Error(str(e))
+
+    self.needed = {}
+
+    # all_files is a dictionary mapping filename to architecture. self.needed
+    # should be a dictionary of filename to ArchFile.
+    for filename, arch in all_files.iteritems():
+      name = os.path.basename(filename)
+      self.needed[filename] = ArchFile(name=name, path=filename, arch=arch)
+
+    self._SetArchFileUrls()
 
     return self.needed
 
+  def _SetArchFileUrls(self):
+    """Fill in the url member of all ArchFiles in self.needed.
+
+    All urls are relative to the nmf_root. In addition, architecture-specific
+    files are relative to the .nexe with the matching architecture. This is
+    useful when making a multi-platform packaged app, so each architecture's
+    files are in a different directory.
+    """
+    # self.GetNeeded() should have already been called.
+    assert self.needed is not None
+
+    main_nexes = [f for f in self.main_files if f.endswith('.nexe')]
+
+    # map from each arch to its corresponding main nexe.
+    arch_to_main_dir = {}
+    for main_file in main_nexes:
+      arch, _ = ParseElfHeader(main_file)
+      main_dir = os.path.dirname(os.path.abspath(main_file))
+      main_dir = PosixRelPath(main_dir, self.nmf_root)
+      if main_dir == '.':
+        main_dir = ''
+      arch_to_main_dir[arch] = main_dir
+
+    for arch_file in self.needed.itervalues():
+      path = os.path.normcase(os.path.abspath(arch_file.path))
+      prefix = ''
+      if path.startswith(self.nmf_root):
+        # This file is already in the nmf_root tree, so it does not need to be
+        # staged. Just make the URL relative to the .nmf.
+        url = PosixRelPath(path, self.nmf_root)
+      else:
+        # This file is outside of the nmf_root subtree, so it needs to be
+        # staged. Its path should be relative to the main .nexe with the same
+        # architecture.
+        prefix = arch_to_main_dir[arch_file.arch]
+        url = os.path.basename(arch_file.path)
+
+      if arch_file.name.endswith('.nexe'):
+        prefix = posixpath.join(prefix, self.nexe_prefix)
+      elif self.no_arch_prefix:
+        prefix = posixpath.join(prefix, self.lib_prefix)
+      else:
+        prefix = posixpath.join(
+            prefix, self.lib_prefix, ARCH_LOCATION[arch_file.arch])
+      arch_file.url = posixpath.join(prefix, url)
+
   def StageDependencies(self, destination_dir):
-    '''Copies over the dependencies into a given destination directory
+    """Copies over the dependencies into a given destination directory
 
     Each library will be put into a subdirectory that corresponds to the arch.
 
     Args:
       destination_dir: The destination directory for staging the dependencies
-    '''
-    nexe_root = os.path.dirname(os.path.abspath(self.main_files[0]))
-    nexe_root = os.path.normcase(nexe_root)
-
-    needed = self.GetNeeded()
-    for arch_file in needed.itervalues():
-      urldest = arch_file.url
+    """
+    assert self.needed is not None
+    for arch_file in self.needed.itervalues():
       source = arch_file.path
-
-      # for .nexe and .so files specified on the command line stage
-      # them in paths relative to the .nexe (with the .nexe always
-      # being staged at the root).
-      if source in self.main_files:
-        absdest = os.path.normcase(os.path.abspath(urldest))
-        if absdest.startswith(nexe_root):
-          urldest = os.path.relpath(urldest, nexe_root)
-
-      destination = os.path.join(destination_dir, urldest)
+      destination = os.path.join(destination_dir, arch_file.url)
 
       if (os.path.normcase(os.path.abspath(source)) ==
           os.path.normcase(os.path.abspath(destination))):
@@ -460,7 +318,7 @@ class NmfUtils(object):
     self.manifest = manifest
 
   def _GenerateManifest(self):
-    '''Create a JSON formatted dict containing the files
+    """Create a JSON formatted dict containing the files
 
     NaCl will map url requests based on architecture.  The startup NEXE
     can always be found under the top key PROGRAM.  Additional files are under
@@ -468,7 +326,8 @@ class NmfUtils(object):
     PROGRAM key is populated with urls pointing the runnable-ld.so which acts
     as the startup nexe.  The application itself is then placed under the
     FILES key mapped as 'main.exe' instead of the original name so that the
-    loader can find it. '''
+    loader can find it.
+    """
     manifest = { FILES_KEY: {}, PROGRAM_KEY: {} }
 
     needed = self.GetNeeded()
@@ -481,8 +340,6 @@ class NmfUtils(object):
                                      url=url))
                       for key, arch, url in self.extra_files]
 
-    nexe_root = os.path.dirname(os.path.abspath(self.main_files[0]))
-
     for need, archinfo in needed.items() + extra_files_kv:
       urlinfo = { URL_KEY: archinfo.url }
       name = archinfo.name
@@ -494,11 +351,6 @@ class NmfUtils(object):
           continue
 
       if need in self.main_files:
-        # Ensure that the .nexe and .so names are relative to the root
-        # of where the .nexe lives.
-        if os.path.abspath(urlinfo[URL_KEY]).startswith(nexe_root):
-          urlinfo[URL_KEY] = os.path.relpath(urlinfo[URL_KEY], nexe_root)
-
         if need.endswith(".nexe"):
           # Place it under program if we aren't using the runnable-ld.so.
           if not runnable:
@@ -514,7 +366,7 @@ class NmfUtils(object):
     self.manifest = manifest
 
   def GetManifest(self):
-    '''Returns a JSON-formatted dict containing the NaCl dependencies'''
+    """Returns a JSON-formatted dict containing the NaCl dependencies"""
     if not self.manifest:
       if self.pnacl:
         self._GeneratePNaClManifest()
@@ -523,7 +375,7 @@ class NmfUtils(object):
     return self.manifest
 
   def GetJson(self):
-    '''Returns the Manifest as a JSON-formatted string'''
+    """Returns the Manifest as a JSON-formatted string"""
     pretty_string = json.dumps(self.GetManifest(), indent=2)
     # json.dumps sometimes returns trailing whitespace and does not put
     # a newline at the end.  This code fixes these problems.
@@ -666,11 +518,21 @@ def main(argv):
                     help='Add DIRECTORY to library search path',
                     metavar='DIRECTORY')
   parser.add_option('-P', '--path-prefix', dest='path_prefix', default='',
+                    help='Deprecated. An alias for --lib-prefix.',
+                    metavar='DIRECTORY')
+  parser.add_option('-p', '--lib-prefix', dest='lib_prefix', default='',
                     help='A path to prepend to shared libraries in the .nmf',
                     metavar='DIRECTORY')
+  parser.add_option('-N', '--nexe-prefix', dest='nexe_prefix', default='',
+                    help='A path to prepend to nexes in the .nmf',
+                    metavar='DIRECTORY')
   parser.add_option('-s', '--stage-dependencies', dest='stage_dependencies',
                     help='Destination directory for staging libraries',
                     metavar='DIRECTORY')
+  parser.add_option('--no-arch-prefix', action='store_true',
+                    help='Don\'t put shared libraries in the lib32/lib64 '
+                    'directories. Instead, they will be put in the same '
+                    'directory as the .nexe that matches its architecture.')
   parser.add_option('-t', '--toolchain', help='Legacy option, do not use')
   parser.add_option('-n', '--name', dest='name',
                     help='Rename FOO as BAR',
@@ -720,9 +582,8 @@ def main(argv):
     remap[parts[0]] = parts[1]
 
   if options.path_prefix:
-    path_prefix = options.path_prefix.split('/')
-  else:
-    path_prefix = []
+    sys.stderr.write('warning: option -P/--path-prefix is deprecated.\n')
+    options.lib_prefix = options.path_prefix
 
   for libpath in options.lib_path:
     if not os.path.exists(libpath):
@@ -743,15 +604,21 @@ def main(argv):
           'warning: PNaCl optlevel %d is unsupported (< 0 or > 3)\n' %
           pnacl_optlevel)
 
+  nmf_root = None
+  if options.output:
+    nmf_root = os.path.dirname(options.output)
+
   nmf = NmfUtils(objdump=options.objdump,
                  main_files=args,
                  lib_path=options.lib_path,
                  extra_files=canonicalized,
-                 lib_prefix=path_prefix,
+                 lib_prefix=options.lib_prefix,
+                 nexe_prefix=options.nexe_prefix,
+                 no_arch_prefix=options.no_arch_prefix,
                  remap=remap,
-                 pnacl_optlevel=pnacl_optlevel)
+                 pnacl_optlevel=pnacl_optlevel,
+                 nmf_root=nmf_root)
 
-  nmf.GetManifest()
   if not options.output:
     sys.stdout.write(nmf.GetJson())
   else:
diff --git a/native_client_sdk/src/tools/lib/elf.py b/native_client_sdk/src/tools/lib/elf.py
new file mode 100644
index 0000000..2011a2b
--- /dev/null
+++ b/native_client_sdk/src/tools/lib/elf.py
@@ -0,0 +1,99 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Helper script for extracting information from ELF files"""
+
+import struct
+
+
+class Error(Exception):
+  '''Local Error class for this file.'''
+  pass
+
+
+def ParseElfHeader(path):
+  """Determine properties of a nexe by parsing elf header.
+  Return tuple of architecture and boolean signalling whether
+  the executable is dynamic (has INTERP header) or static.
+  """
+  # From elf.h:
+  # typedef struct
+  # {
+  #   unsigned char e_ident[EI_NIDENT]; /* Magic number and other info */
+  #   Elf64_Half e_type; /* Object file type */
+  #   Elf64_Half e_machine; /* Architecture */
+  #   ...
+  # } Elf32_Ehdr;
+  elf_header_format = '16s2H'
+  elf_header_size = struct.calcsize(elf_header_format)
+
+  with open(path, 'rb') as f:
+    header = f.read(elf_header_size)
+
+  try:
+    header = struct.unpack(elf_header_format, header)
+  except struct.error:
+    raise Error("error parsing elf header: %s" % path)
+  e_ident, _, e_machine = header[:3]
+
+  elf_magic = '\x7fELF'
+  if e_ident[:4] != elf_magic:
+    raise Error('Not a valid NaCl executable: %s' % path)
+
+  e_machine_mapping = {
+    3 : 'x86-32',
+    40 : 'arm',
+    62 : 'x86-64'
+  }
+  if e_machine not in e_machine_mapping:
+    raise Error('Unknown machine type: %s' % e_machine)
+
+  # Set arch based on the machine type in the elf header
+  arch = e_machine_mapping[e_machine]
+
+  # Now read the full header in either 64bit or 32bit mode
+  dynamic = IsDynamicElf(path, arch == 'x86-64')
+  return arch, dynamic
+
+
+def IsDynamicElf(path, is64bit):
+  """Examine an elf file to determine if it is dynamically
+  linked or not.
+  This is determined by searching the program headers for
+  a header of type PT_INTERP.
+  """
+  if is64bit:
+    elf_header_format = '16s2HI3QI3H'
+  else:
+    elf_header_format = '16s2HI3II3H'
+
+  elf_header_size = struct.calcsize(elf_header_format)
+
+  with open(path, 'rb') as f:
+    header = f.read(elf_header_size)
+    header = struct.unpack(elf_header_format, header)
+    p_header_offset = header[5]
+    p_header_entry_size = header[9]
+    num_p_header = header[10]
+    f.seek(p_header_offset)
+    p_headers = f.read(p_header_entry_size*num_p_header)
+
+  # Read the first word of each Phdr to find out its type.
+  #
+  # typedef struct
+  # {
+  #   Elf32_Word  p_type;     /* Segment type */
+  #   ...
+  # } Elf32_Phdr;
+  elf_phdr_format = 'I'
+  PT_INTERP = 3
+
+  while p_headers:
+    p_header = p_headers[:p_header_entry_size]
+    p_headers = p_headers[p_header_entry_size:]
+    phdr_type = struct.unpack(elf_phdr_format, p_header[:4])[0]
+    if phdr_type == PT_INTERP:
+      return True
+
+  return False
diff --git a/native_client_sdk/src/tools/lib/get_shared_deps.py b/native_client_sdk/src/tools/lib/get_shared_deps.py
new file mode 100644
index 0000000..3287474
--- /dev/null
+++ b/native_client_sdk/src/tools/lib/get_shared_deps.py
@@ -0,0 +1,217 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Helper script to close over all transitive dependencies of a given .nexe
+executable.
+
+e.g. Given
+A -> B
+B -> C
+B -> D
+C -> E
+
+where "A -> B" means A depends on B, then GetNeeded(A) will return A, B, C, D
+and E.
+"""
+
+import os
+import re
+import subprocess
+
+import elf
+
+NeededMatcher = re.compile('^ *NEEDED *([^ ]+)\n$')
+FormatMatcher = re.compile('^(.+):\\s*file format (.+)\n$')
+
+RUNNABLE_LD = 'runnable-ld.so'  # Name of the dynamic loader
+
+OBJDUMP_ARCH_MAP = {
+    # Names returned by Linux's objdump:
+    'elf64-x86-64': 'x86-64',
+    'elf32-i386': 'x86-32',
+    'elf32-little': 'arm',
+    'elf32-littlearm': 'arm',
+    # Names returned by old x86_64-nacl-objdump:
+    'elf64-nacl': 'x86-64',
+    'elf32-nacl': 'x86-32',
+    # Names returned by new x86_64-nacl-objdump:
+    'elf64-x86-64-nacl': 'x86-64',
+    'elf32-x86-64-nacl': 'x86-64',
+    'elf32-i386-nacl': 'x86-32',
+}
+
+# The proper name of the dynamic linker, as kept in the IRT.  This is
+# excluded from the nmf file by convention.
+LD_NACL_MAP = {
+    'x86-32': 'ld-nacl-x86-32.so.1',
+    'x86-64': 'ld-nacl-x86-64.so.1',
+    'arm': None,
+}
+
+
+class Error(Exception):
+  '''Local Error class for this file.'''
+  pass
+
+
+class NoObjdumpError(Error):
+  '''Error raised when objdump is needed but not found'''
+  pass
+
+
+def GetNeeded(main_files, objdump, lib_path):
+  '''Collect the list of dependencies for the main_files
+
+  Args:
+    main_files: A list of files to find dependencies of.
+    objdump: Path to the objdump executable.
+    lib_path: A list of paths to search for shared libraries.
+
+  Returns:
+    A dict with key=filename and value=architecture. The architecture will be
+    one of ('x86_32', 'x86_64', 'arm').
+  '''
+
+  dynamic = any(elf.ParseElfHeader(f)[1] for f in main_files)
+
+  if dynamic:
+    return _GetNeededDynamic(main_files, objdump, lib_path)
+  else:
+    return _GetNeededStatic(main_files)
+
+
+def _GetNeededDynamic(main_files, objdump, lib_path):
+  examined = set()
+  all_files, unexamined = GleanFromObjdump(main_files, None, objdump, lib_path)
+  for arch in all_files.itervalues():
+    if unexamined:
+      unexamined.add((RUNNABLE_LD, arch))
+
+  while unexamined:
+    files_to_examine = {}
+
+    # Take all the currently unexamined files and group them
+    # by architecture.
+    for name, arch in unexamined:
+      files_to_examine.setdefault(arch, []).append(name)
+
+    # Call GleanFromObjdump() for each architecture.
+    needed = set()
+    for arch, files in files_to_examine.iteritems():
+      new_files, new_needed = GleanFromObjdump(files, arch, objdump, lib_path)
+      all_files.update(new_files)
+      needed |= new_needed
+
+    examined |= unexamined
+    unexamined = needed - examined
+
+  # With the runnable-ld.so scheme we have today, the proper name of
+  # the dynamic linker should be excluded from the list of files.
+  ldso = [LD_NACL_MAP[arch] for arch in set(OBJDUMP_ARCH_MAP.values())]
+  for filename, arch in all_files.items():
+    name = os.path.basename(filename)
+    if name in ldso:
+      del all_files[filename]
+
+  return all_files
+
+
+def GleanFromObjdump(files, arch, objdump, lib_path):
+  '''Get architecture and dependency information for given files
+
+  Args:
+    files: A list of files to examine.
+        [ '/path/to/my.nexe',
+          '/path/to/lib64/libmy.so',
+          '/path/to/mydata.so',
+          '/path/to/my.data' ]
+    arch: The architecure we are looking for, or None to accept any
+          architecture.
+    objdump: Path to the objdump executable.
+    lib_path: A list of paths to search for shared libraries.
+
+  Returns: A tuple with the following members:
+    input_info: A dict with key=filename and value=architecture. The
+        architecture will be one of ('x86_32', 'x86_64', 'arm').
+    needed: A set of strings formatted as "arch/name".  Example:
+        set(['x86-32/libc.so', 'x86-64/libgcc.so'])
+  '''
+  if not objdump:
+    raise NoObjdumpError('No objdump executable found!')
+
+  full_paths = set()
+  for filename in files:
+    if os.path.exists(filename):
+      full_paths.add(filename)
+    else:
+      for path in _FindLibsInPath(filename, lib_path):
+        full_paths.add(path)
+
+  cmd = [objdump, '-p'] + list(full_paths)
+  env = {'LANG': 'en_US.UTF-8'}
+  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE, bufsize=-1,
+                          env=env)
+
+  input_info = {}
+  found_basenames = set()
+  needed = set()
+  output, err_output = proc.communicate()
+  if proc.returncode:
+    raise Error('%s\nStdError=%s\nobjdump failed with error code: %d' %
+                (output, err_output, proc.returncode))
+
+  file_arch = None
+  for line in output.splitlines(True):
+    # Objdump should display the architecture first and then the dependencies
+    # second for each file in the list.
+    matched = FormatMatcher.match(line)
+    if matched:
+      filename = matched.group(1)
+      file_arch = OBJDUMP_ARCH_MAP[matched.group(2)]
+      if arch and file_arch != arch:
+        continue
+      name = os.path.basename(filename)
+      found_basenames.add(name)
+      input_info[filename] = file_arch
+    matched = NeededMatcher.match(line)
+    if matched:
+      if arch and file_arch != arch:
+        continue
+      filename = matched.group(1)
+      new_needed = (filename, file_arch)
+      needed.add(new_needed)
+
+  for filename in files:
+    if os.path.basename(filename) not in found_basenames:
+      raise Error('Library not found [%s]: %s' % (arch, filename))
+
+  return input_info, needed
+
+
+def _FindLibsInPath(name, lib_path):
+  '''Finds the set of libraries matching |name| within lib_path
+
+  Args:
+    name: name of library to find
+    lib_path: A list of paths to search for shared libraries.
+
+  Returns:
+    A list of system paths that match the given name within the lib_path'''
+  files = []
+  for dirname in lib_path:
+    filename = os.path.join(dirname, name)
+    if os.path.exists(filename):
+      files.append(filename)
+  if not files:
+    raise Error('cannot find library %s' % name)
+  return files
+
+
+def _GetNeededStatic(main_files):
+  needed = {}
+  for filename in main_files:
+    arch = elf.ParseElfHeader(filename)[0]
+    needed[filename] = arch
+  return needed
diff --git a/native_client_sdk/src/tools/quote.py b/native_client_sdk/src/tools/lib/quote.py
index 5af6400..cec0ee4 100755
--- a/native_client_sdk/src/tools/quote.py
+++ b/native_client_sdk/src/tools/lib/quote.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
diff --git a/native_client_sdk/src/tools/lib/tests/elf_test.py b/native_client_sdk/src/tools/lib/tests/elf_test.py
new file mode 100755
index 0000000..8c57e8d
--- /dev/null
+++ b/native_client_sdk/src/tools/lib/tests/elf_test.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+import unittest
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PARENT_DIR = os.path.dirname(SCRIPT_DIR)
+DATA_DIR = os.path.join(SCRIPT_DIR, 'data')
+
+sys.path.append(PARENT_DIR)
+
+import elf
+
+
+class TestIsDynamicElf(unittest.TestCase):
+  def test_arm(self):
+    static_nexe = os.path.join(DATA_DIR, 'test_static_arm.nexe')
+    self.assertFalse(elf.IsDynamicElf(static_nexe, False))
+
+  def test_x86_32(self):
+    dyn_nexe = os.path.join(DATA_DIR, 'test_dynamic_x86_32.nexe')
+    static_nexe = os.path.join(DATA_DIR, 'test_static_x86_32.nexe')
+    self.assertTrue(elf.IsDynamicElf(dyn_nexe, False))
+    self.assertFalse(elf.IsDynamicElf(static_nexe, False))
+
+  def test_x86_64(self):
+    dyn_nexe = os.path.join(DATA_DIR, 'test_dynamic_x86_64.nexe')
+    static_nexe = os.path.join(DATA_DIR, 'test_static_x86_64.nexe')
+    self.assertTrue(elf.IsDynamicElf(dyn_nexe, True))
+    self.assertFalse(elf.IsDynamicElf(static_nexe, True))
+
+
+class TestParseElfHeader(unittest.TestCase):
+  def test_invalid_elf(self):
+    self.assertRaises(elf.Error, elf.ParseElfHeader, __file__)
+
+  def test_arm_elf_parse(self):
+    """Test parsing of ARM elf header."""
+    static_nexe = os.path.join(DATA_DIR, 'test_static_arm.nexe')
+    arch, dynamic = elf.ParseElfHeader(static_nexe)
+    self.assertEqual(arch, 'arm')
+    self.assertFalse(dynamic)
+
+  def test_x86_32_elf_parse(self):
+    """Test parsing of x86-32 elf header."""
+    dyn_nexe = os.path.join(DATA_DIR, 'test_dynamic_x86_32.nexe')
+    static_nexe = os.path.join(DATA_DIR, 'test_static_x86_32.nexe')
+
+    arch, dynamic = elf.ParseElfHeader(dyn_nexe)
+    self.assertEqual(arch, 'x86-32')
+    self.assertTrue(dynamic)
+
+    arch, dynamic = elf.ParseElfHeader(static_nexe)
+    self.assertEqual(arch, 'x86-32')
+    self.assertFalse(dynamic)
+
+  def test_x86_64_elf_parse(self):
+    """Test parsing of x86-64 elf header."""
+    dyn_nexe = os.path.join(DATA_DIR, 'test_dynamic_x86_64.nexe')
+    static_nexe = os.path.join(DATA_DIR, 'test_static_x86_64.nexe')
+
+    arch, dynamic = elf.ParseElfHeader(dyn_nexe)
+    self.assertEqual(arch, 'x86-64')
+    self.assertTrue(dynamic)
+
+    arch, dynamic = elf.ParseElfHeader(static_nexe)
+    self.assertEqual(arch, 'x86-64')
+    self.assertFalse(dynamic)
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/native_client_sdk/src/tools/lib/tests/get_shared_deps_test.py b/native_client_sdk/src/tools/lib/tests/get_shared_deps_test.py
new file mode 100755
index 0000000..d891c4f
--- /dev/null
+++ b/native_client_sdk/src/tools/lib/tests/get_shared_deps_test.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+LIB_DIR = os.path.dirname(SCRIPT_DIR)
+TOOLS_DIR = os.path.dirname(LIB_DIR)
+DATA_DIR = os.path.join(SCRIPT_DIR, 'data')
+CHROME_SRC = os.path.dirname(os.path.dirname(os.path.dirname(TOOLS_DIR)))
+
+sys.path.append(LIB_DIR)
+sys.path.append(TOOLS_DIR)
+
+import getos
+import get_shared_deps
+
+
+def StripDependencies(deps):
+  '''Strip the dirnames and version suffixes from
+  a list of nexe dependencies.
+
+  e.g:
+  /path/to/libpthread.so.1a2d3fsa -> libpthread.so
+  '''
+  names = []
+  for name in deps:
+    name = os.path.basename(name)
+    if '.so.' in name:
+      name = name.rsplit('.', 1)[0]
+    names.append(name)
+  return names
+
+
+class TestGetNeeded(unittest.TestCase):
+  def setUp(self):
+    self.tempdir = None
+    toolchain = os.path.join(CHROME_SRC, 'native_client', 'toolchain')
+    self.toolchain = os.path.join(toolchain, '%s_x86' % getos.GetPlatform())
+    self.objdump = os.path.join(self.toolchain, 'bin', 'i686-nacl-objdump')
+    if os.name == 'nt':
+      self.objdump += '.exe'
+    self.Mktemp()
+    self.dyn_nexe = self.createTestNexe('test_dynamic_x86_32.nexe', 'i686')
+    self.dyn_deps = set(['libc.so', 'runnable-ld.so',
+                         'libgcc_s.so', 'libpthread.so'])
+
+  def tearDown(self):
+    if self.tempdir:
+      shutil.rmtree(self.tempdir)
+
+  def Mktemp(self):
+    self.tempdir = tempfile.mkdtemp()
+
+  def createTestNexe(self, name, arch):
+    '''Create an empty test .nexe file for use in create_nmf tests.
+
+    This is used rather than checking in test binaries since the
+    checked in binaries depend on .so files that only exist in the
+    certain SDK that built them.
+    '''
+    compiler = os.path.join(self.toolchain, 'bin', '%s-nacl-g++' % arch)
+    if os.name == 'nt':
+      compiler += '.exe'
+      os.environ['CYGWIN'] = 'nodosfilewarning'
+    program = 'int main() { return 0; }'
+    name = os.path.join(self.tempdir, name)
+    cmd = [compiler, '-pthread', '-x' , 'c', '-o', name, '-']
+    p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+    p.communicate(input=program)
+    self.assertEqual(p.returncode, 0)
+    return name
+
+  def testStatic(self):
+    nexe = os.path.join(DATA_DIR, 'test_static_x86_32.nexe')
+    # GetNeeded should not raise an error if objdump is not set, but the .nexe
+    # is statically linked.
+    objdump = None
+    lib_path = []
+    needed = get_shared_deps.GetNeeded([nexe], objdump, lib_path)
+
+    # static nexe should have exactly one needed file
+    self.assertEqual(len(needed), 1)
+    self.assertEqual(needed.keys()[0], nexe)
+
+    # arch of needed file should be x86-32
+    arch = needed.values()[0]
+    self.assertEqual(arch, 'x86-32')
+
+  def testDynamic(self):
+    libdir = os.path.join(self.toolchain, 'x86_64-nacl', 'lib32')
+    needed = get_shared_deps.GetNeeded([self.dyn_nexe],
+                                       lib_path=[libdir],
+                                       objdump=self.objdump)
+    names = needed.keys()
+
+    # this nexe has 5 dependencies
+    expected = set(self.dyn_deps)
+    expected.add(os.path.basename(self.dyn_nexe))
+
+    basenames = set(StripDependencies(names))
+    self.assertEqual(expected, basenames)
+
+  def testMissingArchLibrary(self):
+    libdir = os.path.join(self.toolchain, 'x86_64-nacl', 'lib32')
+    lib_path = [libdir]
+    nexes = ['libgcc_s.so.1']
+    # CreateNmfUtils uses the 32-bit library path, but not the 64-bit one
+    # so searching for a 32-bit library should succeed while searching for
+    # a 64-bit one should fail.
+    get_shared_deps.GleanFromObjdump(nexes, 'x86-32', self.objdump, lib_path)
+    self.assertRaises(get_shared_deps.Error,
+                      get_shared_deps.GleanFromObjdump,
+                      nexes, 'x86-64', self.objdump, lib_path)
+
+  def testCorrectArch(self):
+    lib_path = [os.path.join(self.toolchain, 'x86_64-nacl', 'lib32'),
+                os.path.join(self.toolchain, 'x86_64-nacl', 'lib')]
+
+    needed = get_shared_deps.GetNeeded([self.dyn_nexe],
+                                       lib_path=lib_path,
+                                       objdump=self.objdump)
+    for arch in needed.itervalues():
+      self.assertEqual(arch, 'x86-32')
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/native_client_sdk/src/tools/tests/quote_test.py b/native_client_sdk/src/tools/lib/tests/quote_test.py
index f5fb866..fe0e6c3 100755
--- a/native_client_sdk/src/tools/tests/quote_test.py
+++ b/native_client_sdk/src/tools/lib/tests/quote_test.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
diff --git a/native_client_sdk/src/tools/tests/create_nmf_test.py b/native_client_sdk/src/tools/tests/create_nmf_test.py
index c0855ad..61bcf48 100755
--- a/native_client_sdk/src/tools/tests/create_nmf_test.py
+++ b/native_client_sdk/src/tools/tests/create_nmf_test.py
@@ -2,7 +2,9 @@
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+import json
 import os
+import posixpath
 import shutil
 import subprocess
 import sys
@@ -10,74 +12,47 @@ import tempfile
 import unittest
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-PARENT_DIR = os.path.dirname(SCRIPT_DIR)
-DATA_DIR = os.path.join(SCRIPT_DIR, 'data')
-CHROME_SRC = os.path.dirname(os.path.dirname(os.path.dirname(PARENT_DIR)))
+TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
+DATA_DIR = os.path.join(TOOLS_DIR, 'lib', 'tests', 'data')
+CHROME_SRC = os.path.dirname(os.path.dirname(os.path.dirname(TOOLS_DIR)))
 MOCK_DIR = os.path.join(CHROME_SRC, "third_party", "pymock")
 
 # For the mock library
 sys.path.append(MOCK_DIR)
-sys.path.append(PARENT_DIR)
+sys.path.append(TOOLS_DIR)
 
 import create_nmf
 import getos
 import mock
 
 
-class TestIsDynamicElf(unittest.TestCase):
-  def test_arm(self):
-    static_nexe = os.path.join(DATA_DIR, 'test_static_arm.nexe')
-    self.assertFalse(create_nmf.IsDynamicElf(static_nexe, False))
+PosixRelPath = create_nmf.PosixRelPath
 
-  def test_x86_32(self):
-    dyn_nexe = os.path.join(DATA_DIR, 'test_dynamic_x86_32.nexe')
-    static_nexe = os.path.join(DATA_DIR, 'test_static_x86_32.nexe')
-    self.assertTrue(create_nmf.IsDynamicElf(dyn_nexe, False))
-    self.assertFalse(create_nmf.IsDynamicElf(static_nexe, False))
 
-  def test_x86_64(self):
-    dyn_nexe = os.path.join(DATA_DIR, 'test_dynamic_x86_64.nexe')
-    static_nexe = os.path.join(DATA_DIR, 'test_static_x86_64.nexe')
-    self.assertTrue(create_nmf.IsDynamicElf(dyn_nexe, True))
-    self.assertFalse(create_nmf.IsDynamicElf(static_nexe, True))
+def StripSo(name):
+  """Strip trailing hexidecimal characters from the name of a shared object.
 
+  It strips everything after the last '.' in the name, and checks that the new
+  name ends with .so.
 
-class TestParseElfHeader(unittest.TestCase):
-  def test_invalid_elf(self):
-    self.assertRaises(create_nmf.Error, create_nmf.ParseElfHeader, __file__)
+  e.g.
 
-  def test_arm_elf_parse(self):
-    """Test parsing of ARM elf header."""
-    static_nexe = os.path.join(DATA_DIR, 'test_static_arm.nexe')
-    arch, dynamic = create_nmf.ParseElfHeader(static_nexe)
-    self.assertEqual(arch, 'arm')
-    self.assertFalse(dynamic)
+  libc.so.ad6acbfa => libc.so
+  foo.bar.baz => foo.bar.baz
+  """
+  stripped_name = '.'.join(name.split('.')[:-1])
+  if stripped_name.endswith('.so'):
+    return stripped_name
+  return name
 
-  def test_x86_32_elf_parse(self):
-    """Test parsing of x86-32 elf header."""
-    dyn_nexe = os.path.join(DATA_DIR, 'test_dynamic_x86_32.nexe')
-    static_nexe = os.path.join(DATA_DIR, 'test_static_x86_32.nexe')
 
-    arch, dynamic = create_nmf.ParseElfHeader(dyn_nexe)
-    self.assertEqual(arch, 'x86-32')
-    self.assertTrue(dynamic)
-
-    arch, dynamic = create_nmf.ParseElfHeader(static_nexe)
-    self.assertEqual(arch, 'x86-32')
-    self.assertFalse(dynamic)
-
-  def test_x86_64_elf_parse(self):
-    """Test parsing of x86-64 elf header."""
-    dyn_nexe = os.path.join(DATA_DIR, 'test_dynamic_x86_64.nexe')
-    static_nexe = os.path.join(DATA_DIR, 'test_static_x86_64.nexe')
-
-    arch, dynamic = create_nmf.ParseElfHeader(dyn_nexe)
-    self.assertEqual(arch, 'x86-64')
-    self.assertTrue(dynamic)
-
-    arch, dynamic = create_nmf.ParseElfHeader(static_nexe)
-    self.assertEqual(arch, 'x86-64')
-    self.assertFalse(dynamic)
+class TestPosixRelPath(unittest.TestCase):
+  def testBasic(self):
+    # Note that PosixRelPath only converts from native path format to posix
+    # path format, that's why we have to use os.path.join here.
+    path = os.path.join(os.path.sep, 'foo', 'bar', 'baz.blah')
+    start = os.path.sep + 'foo'
+    self.assertEqual(PosixRelPath(path, start), 'bar/baz.blah')
 
 
 class TestDefaultLibpath(unittest.TestCase):
@@ -113,13 +88,9 @@ class TestNmfUtils(unittest.TestCase):
     self.objdump = os.path.join(self.toolchain, 'bin', 'i686-nacl-objdump')
     if os.name == 'nt':
       self.objdump += '.exe'
-    self.Mktemp()
-    self.dyn_nexe = self.createTestNexe('test_dynamic_x86_32.nexe', True,
-                                        'i686')
-    self.dyn_deps = set(['libc.so', 'runnable-ld.so',
-                         'libgcc_s.so', 'libpthread.so'])
+    self._Mktemp()
 
-  def createTestNexe(self, name, dynamic, arch):
+  def _CreateTestNexe(self, name, arch):
     """Create an empty test .nexe file for use in create_nmf tests.
 
     This is used rather than checking in test binaries since the
@@ -132,6 +103,9 @@ class TestNmfUtils(unittest.TestCase):
       os.environ['CYGWIN'] = 'nodosfilewarning'
     program = 'int main() { return 0; }'
     name = os.path.join(self.tempdir, name)
+    dst_dir = os.path.dirname(name)
+    if not os.path.exists(dst_dir):
+      os.makedirs(dst_dir)
     cmd = [compiler, '-pthread', '-x' , 'c', '-o', name, '-']
     p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
     p.communicate(input=program)
@@ -142,85 +116,401 @@ class TestNmfUtils(unittest.TestCase):
     if self.tempdir:
       shutil.rmtree(self.tempdir)
 
-  def Mktemp(self):
+  def _Mktemp(self):
     self.tempdir = tempfile.mkdtemp()
 
-  def CreateNmfUtils(self, libdir=None):
-    if not libdir:
-      libdir = os.path.join(self.toolchain, 'x86_64-nacl', 'lib32')
-    return create_nmf.NmfUtils([self.dyn_nexe],
-                               lib_path=[libdir],
-                               objdump=self.objdump)
+  def _CreateNmfUtils(self, nexes, **kwargs):
+    if not kwargs.get('lib_path'):
+      # Use lib instead of lib64 (lib64 is a symlink to lib).
+      kwargs['lib_path'] = [
+          os.path.join(self.toolchain, 'x86_64-nacl', 'lib'),
+          os.path.join(self.toolchain, 'x86_64-nacl', 'lib32')]
+    return create_nmf.NmfUtils(nexes,
+                               objdump=self.objdump,
+                               **kwargs)
+
+  def _CreateStatic(self, arch_path=None, **kwargs):
+    """Copy all static .nexe files from the DATA_DIR to a temporary directory.
+
+    Args:
+      arch_path: A dictionary mapping architecture to the directory to generate
+          the .nexe for the architecture in.
+      kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
+          constructor.
+
+    Returns:
+      A tuple with 2 elements:
+        * The generated NMF as a dictionary (i.e. parsed by json.loads)
+        * A list of the generated .nexe paths
+    """
+    arch_path = arch_path or {}
+    nexes = []
+    for arch in ('x86_64', 'x86_32', 'arm'):
+      nexe_name = 'test_static_%s.nexe' % arch
+      src_nexe = os.path.join(DATA_DIR, nexe_name)
+      dst_nexe = os.path.join(self.tempdir, arch_path.get(arch, ''), nexe_name)
+      dst_dir = os.path.dirname(dst_nexe)
+      if not os.path.exists(dst_dir):
+        os.makedirs(dst_dir)
+      shutil.copy(src_nexe, dst_nexe)
+      nexes.append(dst_nexe)
+
+    nexes.sort()
+    nmf_utils = self._CreateNmfUtils(nexes, **kwargs)
+    nmf = json.loads(nmf_utils.GetJson())
+    return nmf, nexes
+
+  def _CreateDynamicAndStageDeps(self, arch_path=None, **kwargs):
+    """Create dynamic .nexe files and put them in a temporary directory, with
+    their dependencies staged in the same directory.
+
+    Args:
+      arch_path: A dictionary mapping architecture to the directory to generate
+          the .nexe for the architecture in.
+      kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
+          constructor.
+
+    Returns:
+      A tuple with 2 elements:
+        * The generated NMF as a dictionary (i.e. parsed by json.loads)
+        * A list of the generated .nexe paths
+    """
+    arch_path = arch_path or {}
+    nexes = []
+    for arch in ('x86_64', 'x86_32'):
+      nexe_name = 'test_dynamic_%s.nexe' % arch
+      rel_nexe = os.path.join(arch_path.get(arch, ''), nexe_name)
+      arch_alt = 'i686' if arch == 'x86_32' else arch
+      nexe = self._CreateTestNexe(rel_nexe, arch_alt)
+      nexes.append(nexe)
+
+    nexes.sort()
+    nmf_utils = self._CreateNmfUtils(nexes, **kwargs)
+    nmf = json.loads(nmf_utils.GetJson())
+    nmf_utils.StageDependencies(self.tempdir)
+
+    return nmf, nexes
+
+  def _CreatePexe(self, **kwargs):
+    """Copy test.pexe from the DATA_DIR to a temporary directory.
+
+    Args:
+      kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
+          constructor.
+
+    Returns:
+      A tuple with 2 elements:
+        * The generated NMF as a dictionary (i.e. parsed by json.loads)
+        * A list of the generated .pexe paths
+    """
+    pexe_name = 'test.pexe'
+    src_pexe = os.path.join(DATA_DIR, pexe_name)
+    dst_pexe = os.path.join(self.tempdir, pexe_name)
+    shutil.copy(src_pexe, dst_pexe)
 
-  def testGetNeededStatic(self):
-    nexe = os.path.join(DATA_DIR, 'test_static_x86_32.nexe')
-    nmf = create_nmf.NmfUtils([nexe])
-    needed = nmf.GetNeeded()
+    pexes = [dst_pexe]
+    nmf_utils = self._CreateNmfUtils(pexes, **kwargs)
+    nmf = json.loads(nmf_utils.GetJson())
 
-    # static nexe should have exactly one needed file
-    self.assertEqual(len(needed), 1)
-    self.assertEqual(needed.keys()[0], nexe)
+    return nmf, pexes
 
-    # arch of needed file should be x86-32
-    archfile = needed.values()[0]
-    self.assertEqual(archfile.arch, 'x86-32')
+  def assertManifestEquals(self, manifest, expected):
+    """Compare two manifest dictionaries.
 
-  def StripDependencies(self, deps):
-    """Strip the dirnames and version suffixes from
-    a list of nexe dependencies.
+    The input manifest is regenerated with all string keys and values being
+    processed through StripSo, to remove the random hexidecimal characters at
+    the end of shared object names.
 
-    e.g:
-    /path/to/libpthread.so.1a2d3fsa -> libpthread.so
+    Args:
+      manifest: The generated manifest.
+      expected: The expected manifest.
+    """
+    def StripSoCopyDict(d):
+      new_d = {}
+      for k, v in d.iteritems():
+        new_k = StripSo(k)
+        if isinstance(v, (str, unicode)):
+          new_v = StripSo(v)
+        elif type(v) is list:
+          new_v = v[:]
+        elif type(v) is dict:
+          new_v = StripSoCopyDict(v)
+        else:
+          # Assume that anything else can be copied directly.
+          new_v = v
+
+        new_d[new_k] = new_v
+      return new_d
+
+    self.assertEqual(StripSoCopyDict(manifest), expected)
+
+  def assertStagingEquals(self, expected):
+    """Compare the contents of the temporary directory, to an expected
+    directory layout.
+
+    Args:
+      expected: The expected directory layout.
     """
-    names = []
-    for name in deps:
-      name = os.path.basename(name)
-      if '.so.' in name:
-        name = name.rsplit('.', 1)[0]
-      names.append(name)
-    return names
-
-  def testGetNeededDynamic(self):
-    nmf = self.CreateNmfUtils()
-    needed = nmf.GetNeeded()
-    names = needed.keys()
-
-    # this nexe has 5 dependencies
-    expected = set(self.dyn_deps)
-    expected.add(os.path.basename(self.dyn_nexe))
-
-    basenames = set(self.StripDependencies(names))
-    self.assertEqual(expected, basenames)
-
-  def testStageDependencies(self):
-    self.Mktemp()
-    nmf = self.CreateNmfUtils()
-    #create_nmf.DebugPrint.debug_mode = True
-    #create_nmf.Trace.verbose = True
-
-    # Stage dependencies
-    nmf.StageDependencies(self.tempdir)
-
-    # Verify directory contents
-    contents = set(os.listdir(self.tempdir))
-    expectedContents = set((os.path.basename(self.dyn_nexe), 'lib32'))
-    self.assertEqual(contents, expectedContents)
-
-    contents = os.listdir(os.path.join(self.tempdir, 'lib32'))
-    contents = self.StripDependencies(contents)
-    contents = set(contents)
-    expectedContents = self.dyn_deps
-    self.assertEqual(contents, expectedContents)
-
-  def testMissingArchLibrary(self):
-    self.Mktemp()
-    nmf = self.CreateNmfUtils()
-    # CreateNmfUtils uses the 32-bit library path, but not the 64-bit one
-    # so searching for a 32-bit library should succeed while searching for
-    # a 64-bit one should fail.
-    nmf.GleanFromObjdump(['libgcc_s.so.1'], 'x86-32')
-    self.assertRaises(create_nmf.Error,
-                      nmf.GleanFromObjdump, ['libgcc_s.so.1'], 'x86-64')
+    all_files = []
+    for root, _, files in os.walk(self.tempdir):
+      rel_root_posix = PosixRelPath(root, self.tempdir)
+      for f in files:
+        path = posixpath.join(rel_root_posix, StripSo(f))
+        if path.startswith('./'):
+          path = path[2:]
+        all_files.append(path)
+    self.assertEqual(set(expected), set(all_files))
+
+
+  def testStatic(self):
+    nmf, _ = self._CreateStatic()
+    expected_manifest = {
+      'files': {},
+      'program': {
+        'x86-64': {'url': 'test_static_x86_64.nexe'},
+        'x86-32': {'url': 'test_static_x86_32.nexe'},
+        'arm': {'url': 'test_static_arm.nexe'},
+      }
+    }
+    self.assertManifestEquals(nmf, expected_manifest)
+
+  def testStaticWithPath(self):
+    arch_dir = {'x86_32': 'x86_32', 'x86_64': 'x86_64', 'arm': 'arm'}
+    nmf, _ = self._CreateStatic(arch_dir, nmf_root=self.tempdir)
+    expected_manifest = {
+      'files': {},
+      'program': {
+        'x86-32': {'url': 'x86_32/test_static_x86_32.nexe'},
+        'x86-64': {'url': 'x86_64/test_static_x86_64.nexe'},
+        'arm': {'url': 'arm/test_static_arm.nexe'},
+      }
+    }
+    self.assertManifestEquals(nmf, expected_manifest)
+
+  def testStaticWithPathNoNmfRoot(self):
+    # This case is not particularly useful, but it is similar to how create_nmf
+    # used to work. If there is no nmf_root given, all paths are relative to
+    # the first nexe passed on the commandline. I believe the assumption
+    # previously was that all .nexes would be in the same directory.
+    arch_dir = {'x86_32': 'x86_32', 'x86_64': 'x86_64', 'arm': 'arm'}
+    nmf, _ = self._CreateStatic(arch_dir)
+    expected_manifest = {
+      'files': {},
+      'program': {
+        'x86-32': {'url': '../x86_32/test_static_x86_32.nexe'},
+        'x86-64': {'url': '../x86_64/test_static_x86_64.nexe'},
+        'arm': {'url': 'test_static_arm.nexe'},
+      }
+    }
+    self.assertManifestEquals(nmf, expected_manifest)
+
+  def testStaticWithNexePrefix(self):
+    nmf, _ = self._CreateStatic(nexe_prefix='foo')
+    expected_manifest = {
+      'files': {},
+      'program': {
+        'x86-64': {'url': 'foo/test_static_x86_64.nexe'},
+        'x86-32': {'url': 'foo/test_static_x86_32.nexe'},
+        'arm': {'url': 'foo/test_static_arm.nexe'},
+      }
+    }
+    self.assertManifestEquals(nmf, expected_manifest)
+
+  def testDynamic(self):
+    nmf, nexes = self._CreateDynamicAndStageDeps()
+    expected_manifest = {
+      'files': {
+        'main.nexe': {
+          'x86-32': {'url': 'test_dynamic_x86_32.nexe'},
+          'x86-64': {'url': 'test_dynamic_x86_64.nexe'},
+        },
+        'libc.so': {
+          'x86-32': {'url': 'lib32/libc.so'},
+          'x86-64': {'url': 'lib64/libc.so'},
+        },
+        'libgcc_s.so': {
+          'x86-32': {'url': 'lib32/libgcc_s.so'},
+          'x86-64': {'url': 'lib64/libgcc_s.so'},
+        },
+        'libpthread.so': {
+          'x86-32': {'url': 'lib32/libpthread.so'},
+          'x86-64': {'url': 'lib64/libpthread.so'},
+        },
+      },
+      'program': {
+        'x86-32': {'url': 'lib32/runnable-ld.so'},
+        'x86-64': {'url': 'lib64/runnable-ld.so'},
+      }
+    }
+
+    expected_staging = [os.path.basename(f) for f in nexes]
+    expected_staging.extend([
+      'lib32/libc.so',
+      'lib32/libgcc_s.so',
+      'lib32/libpthread.so',
+      'lib32/runnable-ld.so',
+      'lib64/libc.so',
+      'lib64/libgcc_s.so',
+      'lib64/libpthread.so',
+      'lib64/runnable-ld.so'])
+
+    self.assertManifestEquals(nmf, expected_manifest)
+    self.assertStagingEquals(expected_staging)
+
+  def testDynamicWithPath(self):
+    arch_dir = {'x86_64': 'x86_64', 'x86_32': 'x86_32'}
+    nmf, nexes = self._CreateDynamicAndStageDeps(arch_dir,
+                                                 nmf_root=self.tempdir)
+    expected_manifest = {
+      'files': {
+        'main.nexe': {
+          'x86-32': {'url': 'x86_32/test_dynamic_x86_32.nexe'},
+          'x86-64': {'url': 'x86_64/test_dynamic_x86_64.nexe'},
+        },
+        'libc.so': {
+          'x86-32': {'url': 'x86_32/lib32/libc.so'},
+          'x86-64': {'url': 'x86_64/lib64/libc.so'},
+        },
+        'libgcc_s.so': {
+          'x86-32': {'url': 'x86_32/lib32/libgcc_s.so'},
+          'x86-64': {'url': 'x86_64/lib64/libgcc_s.so'},
+        },
+        'libpthread.so': {
+          'x86-32': {'url': 'x86_32/lib32/libpthread.so'},
+          'x86-64': {'url': 'x86_64/lib64/libpthread.so'},
+        },
+      },
+      'program': {
+        'x86-32': {'url': 'x86_32/lib32/runnable-ld.so'},
+        'x86-64': {'url': 'x86_64/lib64/runnable-ld.so'},
+      }
+    }
+
+    expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
+    expected_staging.extend([
+      'x86_32/lib32/libc.so',
+      'x86_32/lib32/libgcc_s.so',
+      'x86_32/lib32/libpthread.so',
+      'x86_32/lib32/runnable-ld.so',
+      'x86_64/lib64/libc.so',
+      'x86_64/lib64/libgcc_s.so',
+      'x86_64/lib64/libpthread.so',
+      'x86_64/lib64/runnable-ld.so'])
+
+    self.assertManifestEquals(nmf, expected_manifest)
+    self.assertStagingEquals(expected_staging)
+
+  def testDynamicWithPathNoArchPrefix(self):
+    arch_dir = {'x86_64': 'x86_64', 'x86_32': 'x86_32'}
+    nmf, nexes = self._CreateDynamicAndStageDeps(arch_dir,
+                                                 nmf_root=self.tempdir,
+                                                 no_arch_prefix=True)
+    expected_manifest = {
+      'files': {
+        'main.nexe': {
+          'x86-32': {'url': 'x86_32/test_dynamic_x86_32.nexe'},
+          'x86-64': {'url': 'x86_64/test_dynamic_x86_64.nexe'},
+        },
+        'libc.so': {
+          'x86-32': {'url': 'x86_32/libc.so'},
+          'x86-64': {'url': 'x86_64/libc.so'},
+        },
+        'libgcc_s.so': {
+          'x86-32': {'url': 'x86_32/libgcc_s.so'},
+          'x86-64': {'url': 'x86_64/libgcc_s.so'},
+        },
+        'libpthread.so': {
+          'x86-32': {'url': 'x86_32/libpthread.so'},
+          'x86-64': {'url': 'x86_64/libpthread.so'},
+        },
+      },
+      'program': {
+        'x86-32': {'url': 'x86_32/runnable-ld.so'},
+        'x86-64': {'url': 'x86_64/runnable-ld.so'},
+      }
+    }
+
+    expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
+    expected_staging.extend([
+      'x86_32/libc.so',
+      'x86_32/libgcc_s.so',
+      'x86_32/libpthread.so',
+      'x86_32/runnable-ld.so',
+      'x86_64/libc.so',
+      'x86_64/libgcc_s.so',
+      'x86_64/libpthread.so',
+      'x86_64/runnable-ld.so'])
+
+    self.assertManifestEquals(nmf, expected_manifest)
+    self.assertStagingEquals(expected_staging)
+
+  def testDynamicWithLibPrefix(self):
+    nmf, nexes = self._CreateDynamicAndStageDeps(lib_prefix='foo')
+    expected_manifest = {
+      'files': {
+        'main.nexe': {
+          'x86-32': {'url': 'test_dynamic_x86_32.nexe'},
+          'x86-64': {'url': 'test_dynamic_x86_64.nexe'},
+        },
+        'libc.so': {
+          'x86-32': {'url': 'foo/lib32/libc.so'},
+          'x86-64': {'url': 'foo/lib64/libc.so'},
+        },
+        'libgcc_s.so': {
+          'x86-32': {'url': 'foo/lib32/libgcc_s.so'},
+          'x86-64': {'url': 'foo/lib64/libgcc_s.so'},
+        },
+        'libpthread.so': {
+          'x86-32': {'url': 'foo/lib32/libpthread.so'},
+          'x86-64': {'url': 'foo/lib64/libpthread.so'},
+        },
+      },
+      'program': {
+        'x86-32': {'url': 'foo/lib32/runnable-ld.so'},
+        'x86-64': {'url': 'foo/lib64/runnable-ld.so'},
+      }
+    }
+
+    expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
+    expected_staging.extend([
+      'foo/lib32/libc.so',
+      'foo/lib32/libgcc_s.so',
+      'foo/lib32/libpthread.so',
+      'foo/lib32/runnable-ld.so',
+      'foo/lib64/libc.so',
+      'foo/lib64/libgcc_s.so',
+      'foo/lib64/libpthread.so',
+      'foo/lib64/runnable-ld.so'])
+
+    self.assertManifestEquals(nmf, expected_manifest)
+    self.assertStagingEquals(expected_staging)
+
+  def testPexe(self):
+    nmf, _ = self._CreatePexe()
+    expected_manifest = {
+      'program': {
+        'portable': {
+          'pnacl-translate': {
+            'url': 'test.pexe'
+          }
+        }
+      }
+    }
+    self.assertManifestEquals(nmf, expected_manifest)
+
+  def testPexeOptLevel(self):
+    nmf, _ = self._CreatePexe(pnacl_optlevel=2)
+    expected_manifest = {
+      'program': {
+        'portable': {
+          'pnacl-translate': {
+            'url': 'test.pexe',
+            'optlevel': 2,
+          }
+        }
+      }
+    }
+    self.assertManifestEquals(nmf, expected_manifest)
 
 
 if __name__ == '__main__':
diff --git a/native_client_sdk/src/tools/tests/data/test_dynamic_x86_32.nexe b/native_client_sdk/src/tools/tests/data/test_dynamic_x86_32.nexe
deleted file mode 100644
index 0c46bbd..0000000
--- a/native_client_sdk/src/tools/tests/data/test_dynamic_x86_32.nexe
+++ /dev/null
diff --git a/native_client_sdk/src/tools/tests/data/test_dynamic_x86_64.nexe b/native_client_sdk/src/tools/tests/data/test_dynamic_x86_64.nexe
deleted file mode 100644
index 71a62fa..0000000
--- a/native_client_sdk/src/tools/tests/data/test_dynamic_x86_64.nexe
+++ /dev/null
diff --git a/native_client_sdk/src/tools/tests/data/test_static_arm.nexe b/native_client_sdk/src/tools/tests/data/test_static_arm.nexe
deleted file mode 100644
index ffa0e4e..0000000
--- a/native_client_sdk/src/tools/tests/data/test_static_arm.nexe
+++ /dev/null
diff --git a/native_client_sdk/src/tools/tests/data/test_static_x86_32.nexe b/native_client_sdk/src/tools/tests/data/test_static_x86_32.nexe
deleted file mode 100644
index 94f846b..0000000
--- a/native_client_sdk/src/tools/tests/data/test_static_x86_32.nexe
+++ /dev/null
diff --git a/native_client_sdk/src/tools/tests/data/test_static_x86_64.nexe b/native_client_sdk/src/tools/tests/data/test_static_x86_64.nexe
deleted file mode 100644
index 31144a2..0000000
--- a/native_client_sdk/src/tools/tests/data/test_static_x86_64.nexe
+++ /dev/null