Android: apply some optimizations to the stack symbolization tools.

BUG=234973

Review URL: https://chromiumcodereview.appspot.com/18715002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@211068 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 261 insertions, 95 deletions

diff --git a/third_party/android_platform/development/scripts/stack b/third_party/android_platform/development/scripts/stack
index 6bb8d0a..758c412 100755
--- a/third_party/android_platform/development/scripts/stack
+++ b/third_party/android_platform/development/scripts/stack
@@ -17,11 +17,14 @@
 """stack symbolizes native crash dumps."""
 
 import getopt
+import glob
+import os
 import sys
 
 import stack_core
+import subprocess
 import symbol
-
+import sys
 
 def PrintUsage():
   """Print usage and exit with error."""
@@ -29,6 +32,26 @@ def PrintUsage():
   print
   print "  usage: " + sys.argv[0] + " [options] [FILE]"
   print
+  print "  --symbols-dir=path"
+  print "       the path to a symbols dir, such as =/tmp/out/target/product/dream/symbols"
+  print
+  print "  --chrome-symbols-dir=path"
+  print "       the path to a Chrome symbols dir (can be absolute or relative"
+  print "       to src), such as =out/Debug/lib"
+  print "       If not specified, will look for the newest lib in out/Debug or"
+  print "       out/Release"
+  print
+  print "  --symbols-zip=path"
+  print "       the path to a symbols zip file, such as =dream-symbols-12345.zip"
+  print
+  print "  --more-info"
+  print "  --less-info"
+  print "       Change the level of detail in the output."
+  print "       --more-info is slower and more verbose, but more functions will"
+  print "       be fully qualified with namespace/classname and have full"
+  print "       argument information. Also, the 'stack data' section will be"
+  print "       printed."
+  print
   print "  --arch=arm|x86"
   print "       the target architecture"
   print
@@ -41,20 +64,78 @@ def PrintUsage():
   # pylint: enable-msg=C6310
   sys.exit(1)
 
+def UnzipSymbols(symbolfile, symdir=None):
+  """Unzips a file to DEFAULT_SYMROOT and returns the unzipped location.
+
+  Args:
+    symbolfile: The .zip file to unzip
+    symdir: Optional temporary directory to use for extraction
+
+  Returns:
+    A tuple containing (the directory into which the zip file was unzipped,
+    the path to the "symbols" directory in the unzipped file).  To clean
+    up, the caller can delete the first element of the tuple.
+
+  Raises:
+    SymbolDownloadException: When the unzip fails.
+  """
+  if not symdir:
+    symdir = "%s/%s" % (DEFAULT_SYMROOT, hash(symbolfile))
+  if not os.path.exists(symdir):
+    os.makedirs(symdir)
+
+  print "extracting %s..." % symbolfile
+  saveddir = os.getcwd()
+  os.chdir(symdir)
+  try:
+    unzipcode = subprocess.call(["unzip", "-qq", "-o", symbolfile])
+    if unzipcode > 0:
+      os.remove(symbolfile)
+      raise SymbolDownloadException("failed to extract symbol files (%s)."
+                                    % symbolfile)
+  finally:
+    os.chdir(saveddir)
+
+  android_symbols = glob.glob("%s/out/target/product/*/symbols" % symdir)
+  if android_symbols:
+    return (symdir, android_symbols[0])
+  else:
+    # This is a zip of Chrome symbols, so symbol.CHROME_SYMBOLS_DIR needs to be
+    # updated to point here.
+    symbol.CHROME_SYMBOLS_DIR = symdir
+    return (symdir, symdir)
+
 
 def main():
   try:
     options, arguments = getopt.getopt(sys.argv[1:], "",
-                                       ["arch=",
+                                       ["more-info",
+                                        "less-info",
+                                        "chrome-symbols-dir=",
+                                        "symbols-dir=",
+                                        "symbols-zip=",
+                                        "arch=",
                                         "help"])
   except getopt.GetoptError, unused_error:
     PrintUsage()
 
+  zip_arg = None
+  more_info = False
   for option, value in options:
     if option == "--help":
       PrintUsage()
+    elif option == "--symbols-dir":
+      symbol.SYMBOLS_DIR = os.path.expanduser(value)
+    elif option == "--symbols-zip":
+      zip_arg = os.path.expanduser(value)
     elif option == "--arch":
       symbol.ARCH = value
+    elif option == "--chrome-symbols-dir":
+      symbol.CHROME_SYMBOLS_DIR = os.path.join(symbol.CHROME_SYMBOLS_DIR, value)
+    elif option == "--more-info":
+      more_info = True
+    elif option == "--less-info":
+      more_info = False
 
   if len(arguments) > 1:
     PrintUsage()
@@ -69,8 +150,19 @@ def main():
   lines = f.readlines()
   f.close()
 
-  print "Reading symbols from", symbol.SYMBOLS_DIR
-  stack_core.ConvertTrace(lines)
+  rootdir = None
+  if zip_arg:
+    rootdir, symbol.SYMBOLS_DIR = UnzipSymbols(zip_arg)
+
+  print "Reading Android symbols from", symbol.SYMBOLS_DIR
+  print "Reading Chrome symbols from", symbol.CHROME_SYMBOLS_DIR
+  stack_core.ConvertTrace(lines, more_info)
+
+  if rootdir:
+    # be a good citizen and clean up...os.rmdir and os.removedirs() don't work
+    cmd = "rm -rf \"%s\"" % rootdir
+    print "\ncleaning up (%s)" % cmd
+    os.system(cmd)
 
 if __name__ == "__main__":
   main()
diff --git a/third_party/android_platform/development/scripts/stack_core.py b/third_party/android_platform/development/scripts/stack_core.py
index 42285d4..4120636 100755
--- a/third_party/android_platform/development/scripts/stack_core.py
+++ b/third_party/android_platform/development/scripts/stack_core.py
@@ -48,17 +48,23 @@ HEAP = "[heap]"
 STACK = "[stack]"
 
 
-def PrintOutput(trace_lines, value_lines):
+def PrintOutput(trace_lines, value_lines, more_info):
   if trace_lines:
     PrintTraceLines(trace_lines)
   if value_lines:
-    PrintValueLines(value_lines)
+    # TODO(cjhopman): it seems that symbol.SymbolInformation always fails to
+    # find information for addresses in value_lines in chrome libraries, and so
+    # value_lines have little value to us and merely clutter the output.
+    # Since information is sometimes contained in these lines (from system
+    # libraries), don't completely disable them.
+    if more_info:
+      PrintValueLines(value_lines)
 
 def PrintDivider():
   print
   print "-----------------------------------------------------\n"
 
-def ConvertTrace(lines):
+def ConvertTrace(lines, more_info):
   """Convert strings containing native crash to a stack."""
   process_info_line = re.compile("(pid: [0-9]+, tid: [0-9]+.*)")
   signal_line = re.compile("(signal [0-9]+ \(.*\).*)")
@@ -77,7 +83,7 @@ def ConvertTrace(lines):
   # Or lines from AndroidFeedback crash report system logs like:
   #   03-25 00:51:05.520 I/DEBUG ( 65): #00 pc 001cf42e /data/data/com.my.project/lib/libmyproject.so
   # Please note the spacing differences.
-  trace_line = re.compile("(.*)\#([0-9]+)[ \t]+(..)[ \t]+([0-9a-f]{8})[ \t]+([^\r\n \t]*)( \((.*)\))?")  # pylint: disable-msg=C6310
+  trace_line = re.compile("(.*)\#(?P<frame>[0-9]+)[ \t]+(..)[ \t]+(0x)?(?P<address>[0-9a-f]{0,8})[ \t]+(?P<lib>[^\r\n \t]*)(?P<symbol_present> \((?P<symbol_name>.*)\))?")  # pylint: disable-msg=C6310
   # Examples of matched value lines include:
   #   bea4170c  8018e4e9  /data/data/com.my.project/lib/libmyproject.so
   #   bea4170c  8018e4e9  /data/data/com.my.project/lib/libmyproject.so (symbol)
@@ -97,6 +103,31 @@ def ConvertTrace(lines):
   value_lines = []
   last_frame = -1
 
+  # It is faster to get symbol information with a single call rather than with
+  # separate calls for each line. Since symbol.SymbolInformation caches results,
+  # we can extract all the addresses that we will want symbol information for
+  # from the log and call symbol.SymbolInformation so that the results are
+  # cached in the following lookups.
+  code_addresses = {}
+  for ln in lines:
+    line = unicode(ln, errors='ignore')
+    lib, address = None, None
+
+    match = trace_line.match(line)
+    if match:
+      address, lib = match.group('address', 'lib')
+
+    match = value_line.match(line)
+    if match and not code_line.match(line):
+      (_0, _1, address, lib, _2, _3) = match.groups()
+
+    if lib:
+      code_addresses.setdefault(lib, set()).add(address)
+
+  for lib in code_addresses:
+    symbol.SymbolInformationForSet(
+        symbol.TranslateLibPath(lib), code_addresses[lib], more_info)
+
   for ln in lines:
     # AndroidFeedback adds zero width spaces into its crash reports. These
     # should be removed or the regular expresssions will fail to match.
@@ -110,7 +141,7 @@ def ConvertTrace(lines):
     if process_header or signal_header or register_header or thread_header \
         or dalvik_jni_thread_header or dalvik_native_thread_header:
       if trace_lines or value_lines:
-        PrintOutput(trace_lines, value_lines)
+        PrintOutput(trace_lines, value_lines, more_info)
         PrintDivider()
         trace_lines = []
         value_lines = []
@@ -130,11 +161,11 @@ def ConvertTrace(lines):
       continue
     if trace_line.match(line):
       match = trace_line.match(line)
-      (unused_0, frame, unused_1,
-       code_addr, area, symbol_present, symbol_name) = match.groups()
+      frame, code_addr, area, symbol_present, symbol_name = match.group(
+          'frame', 'address', 'lib', 'symbol_present', 'symbol_name')
 
       if frame <= last_frame and (trace_lines or value_lines):
-        PrintOutput(trace_lines, value_lines)
+        PrintOutput(trace_lines, value_lines, more_info)
         PrintDivider()
         trace_lines = []
         value_lines = []
@@ -145,7 +176,7 @@ def ConvertTrace(lines):
       else:
         # If a calls b which further calls c and c is inlined to b, we want to
         # display "a -> b -> c" in the stack trace instead of just "a -> c"
-        info = symbol.SymbolInformation(area, code_addr)
+        info = symbol.SymbolInformation(area, code_addr, more_info)
         nest_count = len(info) - 1
         for (source_symbol, source_location, object_symbol_with_offset) in info:
           if not source_symbol:
@@ -174,7 +205,7 @@ def ConvertTrace(lines):
       if area == UNKNOWN or area == HEAP or area == STACK or not area:
         value_lines.append((addr, value, "", area))
       else:
-        info = symbol.SymbolInformation(area, value)
+        info = symbol.SymbolInformation(area, value, more_info)
         (source_symbol, source_location, object_symbol_with_offset) = info.pop()
         if not source_symbol:
           if symbol_present:
@@ -190,7 +221,4 @@ def ConvertTrace(lines):
                             object_symbol_with_offset,
                             source_location))
 
-  PrintOutput(trace_lines, value_lines)
-
-
-# vi: ts=2 sw=2
+  PrintOutput(trace_lines, value_lines, more_info)
diff --git a/third_party/android_platform/development/scripts/symbol.py b/third_party/android_platform/development/scripts/symbol.py
index 0f58df6..0f1cf64 100755
--- a/third_party/android_platform/development/scripts/symbol.py
+++ b/third_party/android_platform/development/scripts/symbol.py
@@ -23,23 +23,11 @@ import os
 import re
 import subprocess
 
-ANDROID_BUILD_TOP = os.environ["ANDROID_BUILD_TOP"]
-if not ANDROID_BUILD_TOP:
-  ANDROID_BUILD_TOP = "."
-
-def FindSymbolsDir():
-  saveddir = os.getcwd()
-  os.chdir(ANDROID_BUILD_TOP)
-  try:
-    cmd = ("CALLED_FROM_SETUP=true BUILD_SYSTEM=build/core "
-           "SRC_TARGET_DIR=build/target make -f build/core/config.mk "
-           "dumpvar-abs-TARGET_OUT_UNSTRIPPED")
-    stream = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout
-    return os.path.join(ANDROID_BUILD_TOP, stream.read().strip())
-  finally:
-    os.chdir(saveddir)
-
-SYMBOLS_DIR = FindSymbolsDir()
+CHROME_SRC = os.path.join(os.path.realpath(os.path.dirname(__file__)),
+                          os.pardir, os.pardir, os.pardir, os.pardir)
+ANDROID_BUILD_TOP = CHROME_SRC
+SYMBOLS_DIR = CHROME_SRC
+CHROME_SYMBOLS_DIR = CHROME_SRC
 
 ARCH = "arm"
 
@@ -59,11 +47,22 @@ def Uname():
 
 def ToolPath(tool, toolchain_info=None):
   """Return a full qualified path to the specified tool"""
-  if not toolchain_info:
-    toolchain_info = FindToolchain()
-  (label, platform, target) = toolchain_info
-  return os.path.join(ANDROID_BUILD_TOP, "prebuilts/gcc", Uname(), platform, label, "bin",
-                     target + "-" + tool)
+  # ToolPath looks for the tools in the completely incorrect directory.
+  # This looks in the checked in android_tools.
+  if ARCH == "arm":
+    toolchain_source = "arm-linux-androideabi-4.6"
+    toolchain_prefix = "arm-linux-androideabi"
+  else:
+    toolchain_source = "x86-4.6"
+    toolchain_prefix = "i686-android-linux"
+
+  toolchain_subdir = (
+      "third_party/android_tools/ndk/toolchains/%s/prebuilt/linux-x86_64/bin" %
+       toolchain_source)
+
+  return os.path.join(CHROME_SRC,
+                      toolchain_subdir,
+                      toolchain_prefix + "-" + tool)
 
 def FindToolchain():
   """Look for the latest available toolchain
@@ -80,9 +79,8 @@ def FindToolchain():
 
   ## Known toolchains, newer ones in the front.
   if ARCH == "arm":
-    gcc_version = os.environ["TARGET_GCC_VERSION"]
     known_toolchains = [
-      ("arm-linux-androideabi-" + gcc_version, "arm", "arm-linux-androideabi"),
+      ("arm-linux-androideabi-4.6", "arm", "arm-linux-androideabi"),
     ]
   elif ARCH =="x86":
     known_toolchains = [
@@ -100,7 +98,35 @@ def FindToolchain():
 
   raise Exception("Could not find tool chain")
 
-def SymbolInformation(lib, addr):
+def TranslateLibPath(lib):
+  # SymbolInformation(lib, addr) receives lib as the path from symbols
+  # root to the symbols file. This needs to be translated to point to the
+  # correct .so path. If the user doesn't explicitly specify which directory to
+  # use, then use the most recently updated one in one of the known directories.
+  # If the .so is not found somewhere in CHROME_SYMBOLS_DIR, leave it
+  # untranslated in case it is an Android symbol in SYMBOLS_DIR.
+  library_name = os.path.basename(lib)
+  candidate_dirs = ['.',
+                    'out/Debug/lib',
+                    'out/Debug/lib.target',
+                    'out/Release/lib',
+                    'out/Release/lib.target',
+                    ]
+
+  candidate_libraries = map(
+      lambda d: ('%s/%s/%s' % (CHROME_SYMBOLS_DIR, d, library_name)),
+      candidate_dirs)
+  candidate_libraries = filter(os.path.exists, candidate_libraries)
+  candidate_libraries = sorted(candidate_libraries,
+                               key=os.path.getmtime, reverse=True)
+
+  if not candidate_libraries:
+    return lib
+
+  library_path = os.path.relpath(candidate_libraries[0], SYMBOLS_DIR)
+  return '/' + library_path
+
+def SymbolInformation(lib, addr, get_detailed_info):
   """Look up symbol information about an address.
 
   Args:
@@ -119,11 +145,12 @@ def SymbolInformation(lib, addr):
     Usually you want to display the source_location and
     object_symbol_with_offset from the last element in the list.
   """
-  info = SymbolInformationForSet(lib, set([addr]))
+  lib = TranslateLibPath(lib)
+  info = SymbolInformationForSet(lib, set([addr]), get_detailed_info)
   return (info and info.get(addr)) or [(None, None, None)]
 
 
-def SymbolInformationForSet(lib, unique_addrs):
+def SymbolInformationForSet(lib, unique_addrs, get_detailed_info):
   """Look up symbol information for a set of addresses from the given library.
 
   Args:
@@ -150,9 +177,12 @@ def SymbolInformationForSet(lib, unique_addrs):
   if not addr_to_line:
     return None
 
-  addr_to_objdump = CallObjdumpForSet(lib, unique_addrs)
-  if not addr_to_objdump:
-    return None
+  if get_detailed_info:
+    addr_to_objdump = CallObjdumpForSet(lib, unique_addrs)
+    if not addr_to_objdump:
+      return None
+  else:
+    addr_to_objdump = dict((addr, ("", 0)) for addr in unique_addrs)
 
   result = {}
   for addr in unique_addrs:
@@ -171,6 +201,25 @@ def SymbolInformationForSet(lib, unique_addrs):
   return result
 
 
+class MemoizedForSet(object):
+  def __init__(self, fn):
+    self.fn = fn
+    self.cache = {}
+
+  def __call__(self, lib, unique_addrs):
+    lib_cache = self.cache.setdefault(lib, {})
+
+    no_cache = filter(lambda x: x not in lib_cache, unique_addrs)
+    if no_cache:
+      lib_cache.update((k, None) for k in no_cache)
+      result = self.fn(lib, no_cache)
+      if result:
+        lib_cache.update(result)
+
+    return dict((k, lib_cache[k]) for k in unique_addrs if lib_cache[k])
+
+
+@MemoizedForSet
 def CallAddr2LineForSet(lib, unique_addrs):
   """Look up line and symbol information for a set of addresses.
 
@@ -244,6 +293,7 @@ def StripPC(addr):
     return addr & ~1
   return addr
 
+@MemoizedForSet
 def CallObjdumpForSet(lib, unique_addrs):
   """Use objdump to find out the names of the containing functions.
 
@@ -265,16 +315,7 @@ def CallObjdumpForSet(lib, unique_addrs):
   if not os.path.exists(symbols):
     return None
 
-  addrs = sorted(unique_addrs)
-  start_addr_dec = str(StripPC(int(addrs[0], 16)))
-  stop_addr_dec = str(StripPC(int(addrs[-1], 16)) + 8)
-  cmd = [ToolPath("objdump"),
-         "--section=.text",
-         "--demangle",
-         "--disassemble",
-         "--start-address=" + start_addr_dec,
-         "--stop-address=" + stop_addr_dec,
-         symbols]
+  result = {}
 
   # Function lines look like:
   #   000177b0 <android::IBinder::~IBinder()+0x2c>:
@@ -284,46 +325,51 @@ def CallObjdumpForSet(lib, unique_addrs):
   offset_regexp = re.compile("(.*)\+0x([a-f0-9]*)")
 
   # A disassembly line looks like:
-  #   177b2:	b510      	push	{r4, lr}
+  #   177b2:  b510        push  {r4, lr}
   asm_regexp = re.compile("(^[ a-f0-9]*):[ a-f0-0]*.*$")
 
-  current_symbol = None    # The current function symbol in the disassembly.
-  current_symbol_addr = 0  # The address of the current function.
-  addr_index = 0  # The address that we are currently looking for.
-
-  stream = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
-  result = {}
-  for line in stream:
-    # Is it a function line like:
-    #   000177b0 <android::IBinder::~IBinder()>:
-    components = func_regexp.match(line)
-    if components:
-      # This is a new function, so record the current function and its address.
-      current_symbol_addr = int(components.group(1), 16)
-      current_symbol = components.group(2)
-
-      # Does it have an optional offset like: "foo(..)+0x2c"?
-      components = offset_regexp.match(current_symbol)
+  for target_addr in unique_addrs:
+    start_addr_dec = str(StripPC(int(target_addr, 16)))
+    stop_addr_dec = str(StripPC(int(target_addr, 16)) + 8)
+    cmd = [ToolPath("objdump"),
+           "--section=.text",
+           "--demangle",
+           "--disassemble",
+           "--start-address=" + start_addr_dec,
+           "--stop-address=" + stop_addr_dec,
+           symbols]
+
+    current_symbol = None    # The current function symbol in the disassembly.
+    current_symbol_addr = 0  # The address of the current function.
+
+    stream = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
+    for line in stream:
+      # Is it a function line like:
+      #   000177b0 <android::IBinder::~IBinder()>:
+      components = func_regexp.match(line)
+      if components:
+        # This is a new function, so record the current function and its address.
+        current_symbol_addr = int(components.group(1), 16)
+        current_symbol = components.group(2)
+
+        # Does it have an optional offset like: "foo(..)+0x2c"?
+        components = offset_regexp.match(current_symbol)
+        if components:
+          current_symbol = components.group(1)
+          offset = components.group(2)
+          if offset:
+            current_symbol_addr -= int(offset, 16)
+
+      # Is it an disassembly line like:
+      #   177b2:  b510        push  {r4, lr}
+      components = asm_regexp.match(line)
       if components:
-        current_symbol = components.group(1)
-        offset = components.group(2)
-        if offset:
-          current_symbol_addr -= int(offset, 16)
-
-    # Is it an disassembly line like:
-    #   177b2:	b510      	push	{r4, lr}
-    components = asm_regexp.match(line)
-    if components:
-      addr = components.group(1)
-      target_addr = addrs[addr_index]
-      i_addr = int(addr, 16)
-      i_target = StripPC(int(target_addr, 16))
-      if i_addr == i_target:
-        result[target_addr] = (current_symbol, i_target - current_symbol_addr)
-        addr_index += 1
-        if addr_index >= len(addrs):
-          break
-  stream.close()
+        addr = components.group(1)
+        i_addr = int(addr, 16)
+        i_target = StripPC(int(target_addr, 16))
+        if i_addr == i_target:
+          result[target_addr] = (current_symbol, i_target - current_symbol_addr)
+    stream.close()
 
   return result