Android: adds a few third-party scripts for stack symbolization.

Pick a few utility scripts from https://android.googlesource.com/platform/development/+/master/scripts/
These are used for symbolizing crash stacks on android.

BUG=234973

Review URL: https://chromiumcodereview.appspot.com/18326020

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@210833 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 618 insertions, 0 deletions

diff --git a/third_party/android_platform/development/scripts/stack b/third_party/android_platform/development/scripts/stack
new file mode 100755
index 0000000..6bb8d0a
--- /dev/null
+++ b/third_party/android_platform/development/scripts/stack
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""stack symbolizes native crash dumps."""
+
+import getopt
+import sys
+
+import stack_core
+import symbol
+
+
+def PrintUsage():
+  """Print usage and exit with error."""
+  # pylint: disable-msg=C6310
+  print
+  print "  usage: " + sys.argv[0] + " [options] [FILE]"
+  print
+  print "  --arch=arm|x86"
+  print "       the target architecture"
+  print
+  print "  FILE should contain a stack trace in it somewhere"
+  print "       the tool will find that and re-print it with"
+  print "       source files and line numbers.  If you don't"
+  print "       pass FILE, or if file is -, it reads from"
+  print "       stdin."
+  print
+  # pylint: enable-msg=C6310
+  sys.exit(1)
+
+
+def main():
+  try:
+    options, arguments = getopt.getopt(sys.argv[1:], "",
+                                       ["arch=",
+                                        "help"])
+  except getopt.GetoptError, unused_error:
+    PrintUsage()
+
+  for option, value in options:
+    if option == "--help":
+      PrintUsage()
+    elif option == "--arch":
+      symbol.ARCH = value
+
+  if len(arguments) > 1:
+    PrintUsage()
+
+  if not arguments or arguments[0] == "-":
+    print "Reading native crash info from stdin"
+    f = sys.stdin
+  else:
+    print "Searching for native crashes in %s" % arguments[0]
+    f = open(arguments[0], "r")
+
+  lines = f.readlines()
+  f.close()
+
+  print "Reading symbols from", symbol.SYMBOLS_DIR
+  stack_core.ConvertTrace(lines)
+
+if __name__ == "__main__":
+  main()
+
+# vi: ts=2 sw=2
diff --git a/third_party/android_platform/development/scripts/stack_core.py b/third_party/android_platform/development/scripts/stack_core.py
new file mode 100755
index 0000000..42285d4
--- /dev/null
+++ b/third_party/android_platform/development/scripts/stack_core.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""stack symbolizes native crash dumps."""
+
+import re
+
+import symbol
+
+def PrintTraceLines(trace_lines):
+  """Print back trace."""
+  maxlen = max(map(lambda tl: len(tl[1]), trace_lines))
+  print
+  print "Stack Trace:"
+  print "  RELADDR   " + "FUNCTION".ljust(maxlen) + "  FILE:LINE"
+  for tl in trace_lines:
+    (addr, symbol_with_offset, location) = tl
+    print "  %8s  %s  %s" % (addr, symbol_with_offset.ljust(maxlen), location)
+  return
+
+
+def PrintValueLines(value_lines):
+  """Print stack data values."""
+  maxlen = max(map(lambda tl: len(tl[2]), value_lines))
+  print
+  print "Stack Data:"
+  print "  ADDR      VALUE     " + "FUNCTION".ljust(maxlen) + "  FILE:LINE"
+  for vl in value_lines:
+    (addr, value, symbol_with_offset, location) = vl
+    print "  %8s  %8s  %s  %s" % (addr, value, symbol_with_offset.ljust(maxlen), location)
+  return
+
+UNKNOWN = "<unknown>"
+HEAP = "[heap]"
+STACK = "[stack]"
+
+
+def PrintOutput(trace_lines, value_lines):
+  if trace_lines:
+    PrintTraceLines(trace_lines)
+  if value_lines:
+    PrintValueLines(value_lines)
+
+def PrintDivider():
+  print
+  print "-----------------------------------------------------\n"
+
+def ConvertTrace(lines):
+  """Convert strings containing native crash to a stack."""
+  process_info_line = re.compile("(pid: [0-9]+, tid: [0-9]+.*)")
+  signal_line = re.compile("(signal [0-9]+ \(.*\).*)")
+  register_line = re.compile("(([ ]*[0-9a-z]{2} [0-9a-f]{8}){4})")
+  thread_line = re.compile("(.*)(\-\-\- ){15}\-\-\-")
+  dalvik_jni_thread_line = re.compile("(\".*\" prio=[0-9]+ tid=[0-9]+ NATIVE.*)")
+  dalvik_native_thread_line = re.compile("(\".*\" sysTid=[0-9]+ nice=[0-9]+.*)")
+  # Note that both trace and value line matching allow for variable amounts of
+  # whitespace (e.g. \t). This is because the we want to allow for the stack
+  # tool to operate on AndroidFeedback provided system logs. AndroidFeedback
+  # strips out double spaces that are found in tombsone files and logcat output.
+  #
+  # Examples of matched trace lines include lines from tombstone files like:
+  #   #00  pc 001cf42e  /data/data/com.my.project/lib/libmyproject.so
+  #   #00  pc 001cf42e  /data/data/com.my.project/lib/libmyproject.so (symbol)
+  # Or lines from AndroidFeedback crash report system logs like:
+  #   03-25 00:51:05.520 I/DEBUG ( 65): #00 pc 001cf42e /data/data/com.my.project/lib/libmyproject.so
+  # Please note the spacing differences.
+  trace_line = re.compile("(.*)\#([0-9]+)[ \t]+(..)[ \t]+([0-9a-f]{8})[ \t]+([^\r\n \t]*)( \((.*)\))?")  # pylint: disable-msg=C6310
+  # Examples of matched value lines include:
+  #   bea4170c  8018e4e9  /data/data/com.my.project/lib/libmyproject.so
+  #   bea4170c  8018e4e9  /data/data/com.my.project/lib/libmyproject.so (symbol)
+  #   03-25 00:51:05.530 I/DEBUG ( 65): bea4170c 8018e4e9 /data/data/com.my.project/lib/libmyproject.so
+  # Again, note the spacing differences.
+  value_line = re.compile("(.*)([0-9a-f]{8})[ \t]+([0-9a-f]{8})[ \t]+([^\r\n \t]*)( \((.*)\))?")
+  # Lines from 'code around' sections of the output will be matched before
+  # value lines because otheriwse the 'code around' sections will be confused as
+  # value lines.
+  #
+  # Examples include:
+  #   801cf40c ffffc4cc 00b2f2c5 00b2f1c7 00c1e1a8
+  #   03-25 00:51:05.530 I/DEBUG ( 65): 801cf40c ffffc4cc 00b2f2c5 00b2f1c7 00c1e1a8
+  code_line = re.compile("(.*)[ \t]*[a-f0-9]{8}[ \t]*[a-f0-9]{8}[ \t]*[a-f0-9]{8}[ \t]*[a-f0-9]{8}[ \t]*[a-f0-9]{8}[ \t]*[ \r\n]")  # pylint: disable-msg=C6310
+
+  trace_lines = []
+  value_lines = []
+  last_frame = -1
+
+  for ln in lines:
+    # AndroidFeedback adds zero width spaces into its crash reports. These
+    # should be removed or the regular expresssions will fail to match.
+    line = unicode(ln, errors='ignore')
+    process_header = process_info_line.search(line)
+    signal_header = signal_line.search(line)
+    register_header = register_line.search(line)
+    thread_header = thread_line.search(line)
+    dalvik_jni_thread_header = dalvik_jni_thread_line.search(line)
+    dalvik_native_thread_header = dalvik_native_thread_line.search(line)
+    if process_header or signal_header or register_header or thread_header \
+        or dalvik_jni_thread_header or dalvik_native_thread_header:
+      if trace_lines or value_lines:
+        PrintOutput(trace_lines, value_lines)
+        PrintDivider()
+        trace_lines = []
+        value_lines = []
+        last_frame = -1
+      if process_header:
+        print process_header.group(1)
+      if signal_header:
+        print signal_header.group(1)
+      if register_header:
+        print register_header.group(1)
+      if thread_header:
+        print thread_header.group(1)
+      if dalvik_jni_thread_header:
+        print dalvik_jni_thread_header.group(1)
+      if dalvik_native_thread_header:
+        print dalvik_native_thread_header.group(1)
+      continue
+    if trace_line.match(line):
+      match = trace_line.match(line)
+      (unused_0, frame, unused_1,
+       code_addr, area, symbol_present, symbol_name) = match.groups()
+
+      if frame <= last_frame and (trace_lines or value_lines):
+        PrintOutput(trace_lines, value_lines)
+        PrintDivider()
+        trace_lines = []
+        value_lines = []
+      last_frame = frame
+
+      if area == UNKNOWN or area == HEAP or area == STACK:
+        trace_lines.append((code_addr, "", area))
+      else:
+        # If a calls b which further calls c and c is inlined to b, we want to
+        # display "a -> b -> c" in the stack trace instead of just "a -> c"
+        info = symbol.SymbolInformation(area, code_addr)
+        nest_count = len(info) - 1
+        for (source_symbol, source_location, object_symbol_with_offset) in info:
+          if not source_symbol:
+            if symbol_present:
+              source_symbol = symbol.CallCppFilt(symbol_name)
+            else:
+              source_symbol = UNKNOWN
+          if not source_location:
+            source_location = area
+          if nest_count > 0:
+            nest_count = nest_count - 1
+            trace_lines.append(("v------>", source_symbol, source_location))
+          else:
+            if not object_symbol_with_offset:
+              object_symbol_with_offset = source_symbol
+            trace_lines.append((code_addr,
+                                object_symbol_with_offset,
+                                source_location))
+    if code_line.match(line):
+      # Code lines should be ignored. If this were exluded the 'code around'
+      # sections would trigger value_line matches.
+      continue;
+    if value_line.match(line):
+      match = value_line.match(line)
+      (unused_, addr, value, area, symbol_present, symbol_name) = match.groups()
+      if area == UNKNOWN or area == HEAP or area == STACK or not area:
+        value_lines.append((addr, value, "", area))
+      else:
+        info = symbol.SymbolInformation(area, value)
+        (source_symbol, source_location, object_symbol_with_offset) = info.pop()
+        if not source_symbol:
+          if symbol_present:
+            source_symbol = symbol.CallCppFilt(symbol_name)
+          else:
+            source_symbol = UNKNOWN
+        if not source_location:
+          source_location = area
+        if not object_symbol_with_offset:
+          object_symbol_with_offset = source_symbol
+        value_lines.append((addr,
+                            value,
+                            object_symbol_with_offset,
+                            source_location))
+
+  PrintOutput(trace_lines, value_lines)
+
+
+# vi: ts=2 sw=2
diff --git a/third_party/android_platform/development/scripts/symbol.py b/third_party/android_platform/development/scripts/symbol.py
new file mode 100755
index 0000000..0f58df6
--- /dev/null
+++ b/third_party/android_platform/development/scripts/symbol.py
@@ -0,0 +1,344 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module for looking up symbolic debugging information.
+
+The information can include symbol names, offsets, and source locations.
+"""
+
+import os
+import re
+import subprocess
+
+ANDROID_BUILD_TOP = os.environ["ANDROID_BUILD_TOP"]
+if not ANDROID_BUILD_TOP:
+  ANDROID_BUILD_TOP = "."
+
+def FindSymbolsDir():
+  saveddir = os.getcwd()
+  os.chdir(ANDROID_BUILD_TOP)
+  try:
+    cmd = ("CALLED_FROM_SETUP=true BUILD_SYSTEM=build/core "
+           "SRC_TARGET_DIR=build/target make -f build/core/config.mk "
+           "dumpvar-abs-TARGET_OUT_UNSTRIPPED")
+    stream = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True).stdout
+    return os.path.join(ANDROID_BUILD_TOP, stream.read().strip())
+  finally:
+    os.chdir(saveddir)
+
+SYMBOLS_DIR = FindSymbolsDir()
+
+ARCH = "arm"
+
+TOOLCHAIN_INFO = None
+
+def Uname():
+  """'uname' for constructing prebuilt/<...> and out/host/<...> paths."""
+  uname = os.uname()[0]
+  if uname == "Darwin":
+    proc = os.uname()[-1]
+    if proc == "i386" or proc == "x86_64":
+      return "darwin-x86"
+    return "darwin-ppc"
+  if uname == "Linux":
+    return "linux-x86"
+  return uname
+
+def ToolPath(tool, toolchain_info=None):
+  """Return a full qualified path to the specified tool"""
+  if not toolchain_info:
+    toolchain_info = FindToolchain()
+  (label, platform, target) = toolchain_info
+  return os.path.join(ANDROID_BUILD_TOP, "prebuilts/gcc", Uname(), platform, label, "bin",
+                     target + "-" + tool)
+
+def FindToolchain():
+  """Look for the latest available toolchain
+
+  Args:
+    None
+
+  Returns:
+    A pair of strings containing toolchain label and target prefix.
+  """
+  global TOOLCHAIN_INFO
+  if TOOLCHAIN_INFO is not None:
+    return TOOLCHAIN_INFO
+
+  ## Known toolchains, newer ones in the front.
+  if ARCH == "arm":
+    gcc_version = os.environ["TARGET_GCC_VERSION"]
+    known_toolchains = [
+      ("arm-linux-androideabi-" + gcc_version, "arm", "arm-linux-androideabi"),
+    ]
+  elif ARCH =="x86":
+    known_toolchains = [
+      ("i686-android-linux-4.4.3", "x86", "i686-android-linux")
+    ]
+  else:
+    known_toolchains = []
+
+  # Look for addr2line to check for valid toolchain path.
+  for (label, platform, target) in known_toolchains:
+    toolchain_info = (label, platform, target);
+    if os.path.exists(ToolPath("addr2line", toolchain_info)):
+      TOOLCHAIN_INFO = toolchain_info
+      return toolchain_info
+
+  raise Exception("Could not find tool chain")
+
+def SymbolInformation(lib, addr):
+  """Look up symbol information about an address.
+
+  Args:
+    lib: library (or executable) pathname containing symbols
+    addr: string hexidecimal address
+
+  Returns:
+    A list of the form [(source_symbol, source_location,
+    object_symbol_with_offset)].
+
+    If the function has been inlined then the list may contain
+    more than one element with the symbols for the most deeply
+    nested inlined location appearing first.  The list is
+    always non-empty, even if no information is available.
+
+    Usually you want to display the source_location and
+    object_symbol_with_offset from the last element in the list.
+  """
+  info = SymbolInformationForSet(lib, set([addr]))
+  return (info and info.get(addr)) or [(None, None, None)]
+
+
+def SymbolInformationForSet(lib, unique_addrs):
+  """Look up symbol information for a set of addresses from the given library.
+
+  Args:
+    lib: library (or executable) pathname containing symbols
+    unique_addrs: set of hexidecimal addresses
+
+  Returns:
+    A dictionary of the form {addr: [(source_symbol, source_location,
+    object_symbol_with_offset)]} where each address has a list of
+    associated symbols and locations.  The list is always non-empty.
+
+    If the function has been inlined then the list may contain
+    more than one element with the symbols for the most deeply
+    nested inlined location appearing first.  The list is
+    always non-empty, even if no information is available.
+
+    Usually you want to display the source_location and
+    object_symbol_with_offset from the last element in the list.
+  """
+  if not lib:
+    return None
+
+  addr_to_line = CallAddr2LineForSet(lib, unique_addrs)
+  if not addr_to_line:
+    return None
+
+  addr_to_objdump = CallObjdumpForSet(lib, unique_addrs)
+  if not addr_to_objdump:
+    return None
+
+  result = {}
+  for addr in unique_addrs:
+    source_info = addr_to_line.get(addr)
+    if not source_info:
+      source_info = [(None, None)]
+    if addr in addr_to_objdump:
+      (object_symbol, object_offset) = addr_to_objdump.get(addr)
+      object_symbol_with_offset = FormatSymbolWithOffset(object_symbol,
+                                                         object_offset)
+    else:
+      object_symbol_with_offset = None
+    result[addr] = [(source_symbol, source_location, object_symbol_with_offset)
+        for (source_symbol, source_location) in source_info]
+
+  return result
+
+
+def CallAddr2LineForSet(lib, unique_addrs):
+  """Look up line and symbol information for a set of addresses.
+
+  Args:
+    lib: library (or executable) pathname containing symbols
+    unique_addrs: set of string hexidecimal addresses look up.
+
+  Returns:
+    A dictionary of the form {addr: [(symbol, file:line)]} where
+    each address has a list of associated symbols and locations
+    or an empty list if no symbol information was found.
+
+    If the function has been inlined then the list may contain
+    more than one element with the symbols for the most deeply
+    nested inlined location appearing first.
+  """
+  if not lib:
+    return None
+
+
+  symbols = SYMBOLS_DIR + lib
+  if not os.path.exists(symbols):
+    return None
+
+  (label, platform, target) = FindToolchain()
+  cmd = [ToolPath("addr2line"), "--functions", "--inlines",
+      "--demangle", "--exe=" + symbols]
+  child = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+  result = {}
+  addrs = sorted(unique_addrs)
+  for addr in addrs:
+    child.stdin.write("0x%s\n" % addr)
+    child.stdin.flush()
+    records = []
+    first = True
+    while True:
+      symbol = child.stdout.readline().strip()
+      if symbol == "??":
+        symbol = None
+      location = child.stdout.readline().strip()
+      if location == "??:0":
+        location = None
+      if symbol is None and location is None:
+        break
+      records.append((symbol, location))
+      if first:
+        # Write a blank line as a sentinel so we know when to stop
+        # reading inlines from the output.
+        # The blank line will cause addr2line to emit "??\n??:0\n".
+        child.stdin.write("\n")
+        first = False
+    result[addr] = records
+  child.stdin.close()
+  child.stdout.close()
+  return result
+
+
+def StripPC(addr):
+  """Strips the Thumb bit a program counter address when appropriate.
+
+  Args:
+    addr: the program counter address
+
+  Returns:
+    The stripped program counter address.
+  """
+  global ARCH
+
+  if ARCH == "arm":
+    return addr & ~1
+  return addr
+
+def CallObjdumpForSet(lib, unique_addrs):
+  """Use objdump to find out the names of the containing functions.
+
+  Args:
+    lib: library (or executable) pathname containing symbols
+    unique_addrs: set of string hexidecimal addresses to find the functions for.
+
+  Returns:
+    A dictionary of the form {addr: (string symbol, offset)}.
+  """
+  if not lib:
+    return None
+
+  symbols = SYMBOLS_DIR + lib
+  if not os.path.exists(symbols):
+    return None
+
+  symbols = SYMBOLS_DIR + lib
+  if not os.path.exists(symbols):
+    return None
+
+  addrs = sorted(unique_addrs)
+  start_addr_dec = str(StripPC(int(addrs[0], 16)))
+  stop_addr_dec = str(StripPC(int(addrs[-1], 16)) + 8)
+  cmd = [ToolPath("objdump"),
+         "--section=.text",
+         "--demangle",
+         "--disassemble",
+         "--start-address=" + start_addr_dec,
+         "--stop-address=" + stop_addr_dec,
+         symbols]
+
+  # Function lines look like:
+  #   000177b0 <android::IBinder::~IBinder()+0x2c>:
+  # We pull out the address and function first. Then we check for an optional
+  # offset. This is tricky due to functions that look like "operator+(..)+0x2c"
+  func_regexp = re.compile("(^[a-f0-9]*) \<(.*)\>:$")
+  offset_regexp = re.compile("(.*)\+0x([a-f0-9]*)")
+
+  # A disassembly line looks like:
+  #   177b2:	b510      	push	{r4, lr}
+  asm_regexp = re.compile("(^[ a-f0-9]*):[ a-f0-0]*.*$")
+
+  current_symbol = None    # The current function symbol in the disassembly.
+  current_symbol_addr = 0  # The address of the current function.
+  addr_index = 0  # The address that we are currently looking for.
+
+  stream = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
+  result = {}
+  for line in stream:
+    # Is it a function line like:
+    #   000177b0 <android::IBinder::~IBinder()>:
+    components = func_regexp.match(line)
+    if components:
+      # This is a new function, so record the current function and its address.
+      current_symbol_addr = int(components.group(1), 16)
+      current_symbol = components.group(2)
+
+      # Does it have an optional offset like: "foo(..)+0x2c"?
+      components = offset_regexp.match(current_symbol)
+      if components:
+        current_symbol = components.group(1)
+        offset = components.group(2)
+        if offset:
+          current_symbol_addr -= int(offset, 16)
+
+    # Is it an disassembly line like:
+    #   177b2:	b510      	push	{r4, lr}
+    components = asm_regexp.match(line)
+    if components:
+      addr = components.group(1)
+      target_addr = addrs[addr_index]
+      i_addr = int(addr, 16)
+      i_target = StripPC(int(target_addr, 16))
+      if i_addr == i_target:
+        result[target_addr] = (current_symbol, i_target - current_symbol_addr)
+        addr_index += 1
+        if addr_index >= len(addrs):
+          break
+  stream.close()
+
+  return result
+
+
+def CallCppFilt(mangled_symbol):
+  cmd = [ToolPath("c++filt")]
+  process = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  process.stdin.write(mangled_symbol)
+  process.stdin.write("\n")
+  process.stdin.close()
+  demangled_symbol = process.stdout.readline().strip()
+  process.stdout.close()
+  return demangled_symbol
+
+def FormatSymbolWithOffset(symbol, offset):
+  if offset == 0:
+    return symbol
+  return "%s+%d" % (symbol, offset)