Load static symbol information lazily with some clean-ups.

It also changes :
- dmprof messages,
- how to run subprocesses in prepare_symbol_info,
- to prepare readelf result in prepare_symbol_info,
- to rename procedure_boundaries => static_symbols,
- how to parse nm results in static_symbols,
- additional --keep option,
- to ignore empty files, and
- to store addresses in int.


BUG=123749
TEST=use dmprof.

Review URL: https://chromiumcodereview.appspot.com/10826008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@148930 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 157 insertions, 97 deletions

diff --git a/tools/find_runtime_symbols/find_runtime_symbols.py b/tools/find_runtime_symbols/find_runtime_symbols.py
index 1c96c7f..2d9f452c 100755
--- a/tools/find_runtime_symbols/find_runtime_symbols.py
+++ b/tools/find_runtime_symbols/find_runtime_symbols.py
@@ -3,14 +3,12 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import json
 import logging
 import os
 import re
 import sys
 
-from parse_proc_maps import parse_proc_maps
-from procedure_boundaries import get_procedure_boundaries_from_nm_bsd
+from static_symbols import StaticSymbols
 from util import executable_condition
 
 
@@ -50,37 +48,9 @@ class _FileOutput(object):
       self.result.write('%s\n' % symbol_name)
 
 
-def _find_runtime_symbols(
-    prepared_data_dir, addresses, outputter, loglevel=logging.WARN):
-  log = logging.getLogger('find_runtime_symbols')
-  log.setLevel(loglevel)
-  handler = logging.StreamHandler()
-  handler.setLevel(loglevel)
-  formatter = logging.Formatter('%(message)s')
-  handler.setFormatter(formatter)
-  log.addHandler(handler)
-
-  if not os.path.exists(prepared_data_dir):
-    log.warn("Nothing found: %s" % prepared_data_dir)
-    return 1
-  if not os.path.isdir(prepared_data_dir):
-    log.warn("Not a directory: %s" % prepared_data_dir)
-    return 1
-
-  with open(os.path.join(prepared_data_dir, 'maps'), mode='r') as f:
-    maps = parse_proc_maps(f)
-
-  with open(os.path.join(prepared_data_dir, 'nm.json'), mode='r') as f:
-    nm_files = json.load(f)
-
-  symbol_table = {}
-  for entry in maps.iter(executable_condition):
-    if nm_files.has_key(entry.name):
-      if nm_files[entry.name]['format'] == 'bsd':
-        with open(os.path.join(prepared_data_dir,
-                               nm_files[entry.name]['file']), mode='r') as f:
-          symbol_table[entry.name] = get_procedure_boundaries_from_nm_bsd(
-              f, nm_files[entry.name]['mangled'])
+def _find_runtime_symbols(static_symbols, addresses, outputter):
+  maps = static_symbols.maps
+  symbol_tables = static_symbols.procedure_boundaries
 
   for address in addresses:
     if isinstance(address, str):
@@ -88,8 +58,8 @@ def _find_runtime_symbols(
     is_found = False
     for entry in maps.iter(executable_condition):
       if entry.begin <= address < entry.end:
-        if entry.name in symbol_table:
-          found = symbol_table[entry.name].find_procedure(
+        if entry.name in symbol_tables:
+          found = symbol_tables[entry.name].find_procedure(
               address - (entry.begin - entry.offset))
           outputter.output(address, found)
         else:
@@ -102,21 +72,21 @@ def _find_runtime_symbols(
   return 0
 
 
-def find_runtime_symbols_list(prepared_data_dir, addresses):
+def find_runtime_symbols_list(static_symbols, addresses):
   result = []
-  _find_runtime_symbols(prepared_data_dir, addresses, _ListOutput(result))
+  _find_runtime_symbols(static_symbols, addresses, _ListOutput(result))
   return result
 
 
-def find_runtime_symbols_dict(prepared_data_dir, addresses):
+def find_runtime_symbols_dict(static_symbols, addresses):
   result = {}
-  _find_runtime_symbols(prepared_data_dir, addresses, _DictOutput(result))
+  _find_runtime_symbols(static_symbols, addresses, _DictOutput(result))
   return result
 
 
-def find_runtime_symbols_file(prepared_data_dir, addresses, f):
+def find_runtime_symbols_file(static_symbols, addresses, f):
   _find_runtime_symbols(
-      prepared_data_dir, addresses, _FileOutput(f, False))
+      static_symbols, addresses, _FileOutput(f, False))
 
 
 def main():
@@ -127,7 +97,24 @@ def main():
 """ % sys.argv[0])
     return 1
 
-  return find_runtime_symbols_file(sys.argv[1], sys.stdin, sys.stdout)
+  log = logging.getLogger('find_runtime_symbols')
+  log.setLevel(logging.WARN)
+  handler = logging.StreamHandler()
+  handler.setLevel(logging.WARN)
+  formatter = logging.Formatter('%(message)s')
+  handler.setFormatter(formatter)
+  log.addHandler(handler)
+
+  prepared_data_dir = sys.argv[1]
+  if not os.path.exists(prepared_data_dir):
+    log.warn("Nothing found: %s" % prepared_data_dir)
+    return 1
+  if not os.path.isdir(prepared_data_dir):
+    log.warn("Not a directory: %s" % prepared_data_dir)
+    return 1
+
+  static_symbols = StaticSymbols.load(prepared_data_dir)
+  return find_runtime_symbols_file(static_symbols, sys.stdin, sys.stdout)
 
 
 if __name__ == '__main__':
diff --git a/tools/find_runtime_symbols/prepare_symbol_info.py b/tools/find_runtime_symbols/prepare_symbol_info.py
index 57fcfbc..50654b1a 100755
--- a/tools/find_runtime_symbols/prepare_symbol_info.py
+++ b/tools/find_runtime_symbols/prepare_symbol_info.py
@@ -16,6 +16,39 @@ from parse_proc_maps import parse_proc_maps
 from util import executable_condition
 
 
+def _dump_command_result(command, output_dir_path, basename, suffix, log):
+  handle_out, filename_out = tempfile.mkstemp(
+      suffix=suffix, prefix=basename + '.', dir=output_dir_path)
+  handle_err, filename_err = tempfile.mkstemp(
+      suffix=suffix + '.err', prefix=basename + '.', dir=output_dir_path)
+  error = False
+  try:
+    subprocess.check_call(
+        command, stdout=handle_out, stderr=handle_err, shell=True)
+  except:
+    error = True
+  finally:
+    os.close(handle_err)
+    os.close(handle_out)
+
+  if os.path.exists(filename_err):
+    if log.getEffectiveLevel() <= logging.DEBUG:
+      with open(filename_err, 'r') as f:
+        for line in f:
+          log.debug(line.rstrip())
+    os.remove(filename_err)
+
+  if os.path.exists(filename_out) and (
+      os.path.getsize(filename_out) == 0 or error):
+    os.remove(filename_out)
+    return None
+
+  if not os.path.exists(filename_out):
+    return None
+
+  return filename_out
+
+
 def prepare_symbol_info(maps_path, output_dir_path=None, loglevel=logging.WARN):
   log = logging.getLogger('prepare_symbol_info')
   log.setLevel(loglevel)
@@ -58,42 +91,31 @@ def prepare_symbol_info(maps_path, output_dir_path=None, loglevel=logging.WARN):
     maps = parse_proc_maps(f)
 
   log.debug('Listing up symbols.')
-  nm_files = {}
+  files = {}
   for entry in maps.iter(executable_condition):
     log.debug('  %016x-%016x +%06x %s' % (
         entry.begin, entry.end, entry.offset, entry.name))
-    with tempfile.NamedTemporaryFile(
-        prefix=os.path.basename(entry.name) + '.',
-        suffix='.nm', delete=False, mode='w', dir=output_dir_path) as f:
-      nm_filename = os.path.realpath(f.name)
-      nm_succeeded = False
-      cppfilt_succeeded = False
-      p_nm = subprocess.Popen(
-          'nm -n --format bsd %s' % entry.name, shell=True,
-          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      p_cppfilt = subprocess.Popen(
-          'c++filt', shell=True,
-          stdin=p_nm.stdout, stdout=f, stderr=subprocess.PIPE)
-
-      if p_nm.wait() == 0:
-        nm_succeeded = True
-      for line in p_nm.stderr:
-        log.debug(line.rstrip())
-      if p_cppfilt.wait() == 0:
-        cppfilt_succeeded = True
-      for line in p_cppfilt.stderr:
-        log.debug(line.rstrip())
-
-    if nm_succeeded and cppfilt_succeeded:
-      nm_files[entry.name] = {
+    nm_filename = _dump_command_result(
+        'nm -n --format bsd %s | c++filt' % entry.name,
+        output_dir_path, os.path.basename(entry.name), '.nm', log)
+    if not nm_filename:
+      continue
+    readelf_e_filename = _dump_command_result(
+        'readelf -e %s' % entry.name,
+        output_dir_path, os.path.basename(entry.name), '.readelf-e', log)
+    if not readelf_e_filename:
+      continue
+
+    files[entry.name] = {}
+    files[entry.name]['nm'] = {
         'file': os.path.basename(nm_filename),
         'format': 'bsd',
         'mangled': False}
-    else:
-      os.remove(nm_filename)
+    files[entry.name]['readelf-e'] = {
+        'file': os.path.basename(readelf_e_filename)}
 
-  with open(os.path.join(output_dir_path, 'nm.json'), 'w') as f:
-    json.dump(nm_files, f, indent=2, sort_keys=True)
+  with open(os.path.join(output_dir_path, 'files.json'), 'w') as f:
+    json.dump(files, f, indent=2, sort_keys=True)
 
   log.info('Collected symbol information at "%s".' % output_dir_path)
   return 0
@@ -110,7 +132,7 @@ def main():
 """ % sys.argv[0])
     return 1
   elif len(sys.argv) == 2:
-    sys.exit(prepare_symbol_info(sys.argv[1], loglevel=logging.DEBUG))
+    sys.exit(prepare_symbol_info(sys.argv[1], loglevel=logging.INFO))
   else:
     sys.exit(prepare_symbol_info(sys.argv[1], sys.argv[2],
                                  loglevel=logging.INFO))
diff --git a/tools/find_runtime_symbols/procedure_boundaries.py b/tools/find_runtime_symbols/static_symbols.py
index be1d76c..01412021 100644
--- a/tools/find_runtime_symbols/procedure_boundaries.py
+++ b/tools/find_runtime_symbols/static_symbols.py
@@ -3,10 +3,14 @@
 # found in the LICENSE file.
 
 import bisect
+import json
 import os
 import re
 import sys
 
+from parse_proc_maps import parse_proc_maps
+from util import executable_condition
+
 
 _ARGUMENT_TYPE_PATTERN = re.compile('\([^()]*\)(\s*const)?')
 _TEMPLATE_ARGUMENT_PATTERN = re.compile('<[^<>]*>')
@@ -18,6 +22,63 @@ class ParsingException(Exception):
     return repr(self.args[0])
 
 
+class StaticSymbols(object):
+  """Represents static symbol information."""
+
+  def __init__(self, maps, procedure_boundaries):
+    self.maps = maps
+    self.procedure_boundaries = procedure_boundaries
+
+  # TODO(dmikurube): It will be deprecated.
+  @staticmethod
+  def _load_nm(prepared_data_dir, maps_filename, nm_json_filename):
+    with open(os.path.join(prepared_data_dir, maps_filename), mode='r') as f:
+      maps = parse_proc_maps(f)
+    with open(os.path.join(prepared_data_dir, nm_json_filename), mode='r') as f:
+      nm_files = json.load(f)
+
+    symbol_tables = {}
+    for entry in maps.iter(executable_condition):
+      if nm_files.has_key(entry.name):
+        if nm_files[entry.name]['format'] == 'bsd':
+          with open(os.path.join(prepared_data_dir,
+                                 nm_files[entry.name]['file']), mode='r') as f:
+            symbol_tables[entry.name] = _get_static_symbols_from_nm_bsd(
+                f, nm_files[entry.name]['mangled'])
+
+    return StaticSymbols(maps, symbol_tables)
+
+  @staticmethod
+  def _load_files(prepared_data_dir, maps_filename, files_filename):
+    with open(os.path.join(prepared_data_dir, maps_filename), mode='r') as f:
+      maps = parse_proc_maps(f)
+    with open(os.path.join(prepared_data_dir, files_filename), mode='r') as f:
+      files = json.load(f)
+
+    symbol_tables = {}
+    for entry in maps.iter(executable_condition):
+      if entry.name in files:
+        if 'nm' in files[entry.name]:
+          nm_entry = files[entry.name]['nm']
+          if nm_entry['format'] == 'bsd':
+            with open(os.path.join(prepared_data_dir, nm_entry['file']),
+                      mode='r') as f:
+              symbol_tables[entry.name] = _get_static_symbols_from_nm_bsd(
+                  f, nm_entry['mangled'])
+        if 'readelf-e' in files:
+          readelf_entry = files[entry.name]['readelf-e']
+          # TODO(dmikurube) Implement it.
+
+    return StaticSymbols(maps, symbol_tables)
+
+  @staticmethod
+  def load(prepared_data_dir):
+    if os.path.exists(os.path.join(prepared_data_dir, 'nm.json')):
+      return StaticSymbols._load_nm(prepared_data_dir, 'maps', 'nm.json')
+    else:
+      return StaticSymbols._load_files(prepared_data_dir, 'maps', 'files.json')
+
+
 class ProcedureBoundary(object):
   """A class for a procedure symbol and an address range for the symbol."""
 
@@ -65,7 +126,15 @@ def _get_short_function_name(function):
   return _LEADING_TYPE_PATTERN.sub('\g<1>', function)
 
 
-def get_procedure_boundaries_from_nm_bsd(f, mangled=False):
+def _parse_nm_bsd_line(line):
+  if line[8] == ' ':
+    return line[0:8], line[9], line[11:]
+  elif line[16] == ' ':
+    return line[0:16], line[17], line[19:]
+  raise ParsingException('Invalid nm output.')
+
+
+def _get_static_symbols_from_nm_bsd(f, mangled=False):
   """Gets procedure boundaries from a result of nm -n --format bsd.
 
   Args:
@@ -81,27 +150,9 @@ def get_procedure_boundaries_from_nm_bsd(f, mangled=False):
   routine = ''
 
   for line in f:
-    symbol_info = line.rstrip().split(None, 2)
-    if len(symbol_info) == 3:
-      if len(symbol_info[0]) == 1:
-        symbol_info = line.split(None, 1)
-        (sym_type, this_routine) = symbol_info
-        sym_value = ''
-      else:
-        (sym_value, sym_type, this_routine) = symbol_info
-    elif len(symbol_info) == 2:
-      if len(symbol_info[0]) == 1:
-        (sym_type, this_routine) = symbol_info
-        sym_value = ''
-      elif len(symbol_info[0]) == 8 or len(symbol_info[0]) == 16:
-        (sym_value, this_routine) = symbol_info
-        sym_type = ' '
-      else:
-        raise ParsingException('Invalid output 1 from (eu-)nm.')
-    else:
-      raise ParsingException('Invalid output 2 from (eu-)nm.')
+    sym_value, sym_type, sym_name = _parse_nm_bsd_line(line)
 
-    if sym_value == '':
+    if sym_value[0] == ' ':
       continue
 
     start_val = int(sym_value, 16)
@@ -123,7 +174,7 @@ def get_procedure_boundaries_from_nm_bsd(f, mangled=False):
     # got touched in the queue), and ignore the others.
     if start_val == last_start and (sym_type == 't' or sym_type == 'T'):
       # We are the 'T' symbol at this address, replace previous symbol.
-      routine = this_routine
+      routine = sym_name
       continue
     elif start_val == last_start:
       # We're not the 'T' symbol at this address, so ignore us.
@@ -133,14 +184,14 @@ def get_procedure_boundaries_from_nm_bsd(f, mangled=False):
     # has multiple occurrences of this routine.  We use a syntax
     # that resembles template paramters that are automatically
     # stripped out by ShortFunctionName()
-    this_routine += "<%016x>" % start_val
+    sym_name += "<%016x>" % start_val
 
     if not mangled:
       routine = _get_short_function_name(routine)
     symbol_table.append(ProcedureBoundary(last_start, start_val, routine))
 
     last_start = start_val
-    routine = this_routine
+    routine = sym_name
 
   if not mangled:
     routine = _get_short_function_name(routine)