Android: adds stack symbolization utilities.

asan_symbolize.py is used for the logcat stacks using the special ASan format.
tombstones.py is used to manage tombstones files in devices.
symbolize.py is used for stack traces generated by base/debug/stack_trace_android.cc

BUG=234973

Review URL: https://chromiumcodereview.appspot.com/18473004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@211134 0039d316-1c4b-4281-b951-d872f2087c98

7 files changed, 534 insertions, 0 deletions

diff --git a/build/android/asan_symbolize.py b/build/android/asan_symbolize.py
new file mode 100755
index 0000000..928798f
--- /dev/null
+++ b/build/android/asan_symbolize.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+#
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import collections
+import optparse
+import os
+import re
+import sys
+
+from pylib import constants
+
+# Uses symbol.py from third_party/android_platform, not python's.
+sys.path.insert(0,
+                os.path.join(constants.DIR_SOURCE_ROOT,
+                            'third_party/android_platform/development/scripts'))
+import symbol
+
+
+_RE_ASAN = re.compile(r'I/asanwrapper\.sh.*?(#\S*?) (\S*?) \((.*?)\+(.*?)\)')
+
+def _ParseAsanLogLine(line):
+  m = re.match(_RE_ASAN, line)
+  if not m:
+    return None
+  return {
+      'library': m.group(3),
+      'pos': m.group(1),
+      'rel_address': '%08x' % int(m.group(4), 16),
+  }
+
+
+def _FindASanLibraries():
+  asan_lib_dir = os.path.join(constants.DIR_SOURCE_ROOT,
+                              'third_party', 'llvm-build',
+                              'Release+Asserts', 'lib')
+  asan_libs = []
+  for src_dir, _, files in os.walk(asan_lib_dir):
+    asan_libs += [os.path.relpath(os.path.join(src_dir, f))
+                  for f in files
+                  if f.endswith('.so')]
+  return asan_libs
+
+
+def _TranslateLibPath(library, asan_libs):
+  for asan_lib in asan_libs:
+    if os.path.basename(library) == os.path.basename(asan_lib):
+      return '/' + asan_lib
+  return symbol.TranslateLibPath(library)
+
+
+def _Symbolize(input):
+  asan_libs = _FindASanLibraries()
+  libraries = collections.defaultdict(list)
+  asan_lines = []
+  for asan_log_line in [a.strip() for a in input]:
+    m = _ParseAsanLogLine(asan_log_line)
+    if m:
+      libraries[m['library']].append(m)
+    asan_lines.append({'raw_log': asan_log_line, 'parsed': m})
+
+  all_symbols = collections.defaultdict(dict)
+  original_symbols_dir = symbol.SYMBOLS_DIR
+  for library, items in libraries.iteritems():
+    libname = _TranslateLibPath(library, asan_libs)
+    lib_relative_addrs = set([i['rel_address'] for i in items])
+    info_dict = symbol.SymbolInformationForSet(libname,
+                                               lib_relative_addrs,
+                                               True)
+    if info_dict:
+      all_symbols[library]['symbols'] = info_dict
+
+  for asan_log_line in asan_lines:
+    m = asan_log_line['parsed']
+    if not m:
+      print asan_log_line['raw_log']
+      continue
+    if (m['library'] in all_symbols and
+        m['rel_address'] in all_symbols[m['library']]['symbols']):
+      s = all_symbols[m['library']]['symbols'][m['rel_address']][0]
+      print s[0], s[1], s[2]
+    else:
+      print asan_log_line['raw_log']
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('-l', '--logcat',
+                    help='File containing adb logcat output with ASan stacks. '
+                         'Use stdin if not specified.')
+  options, args = parser.parse_args()
+  if options.logcat:
+    input = file(options.logcat, 'r')
+  else:
+    input = sys.stdin
+  _Symbolize(input.readlines())
+
+
+if __name__ == "__main__":
+  sys.exit(main())
diff --git a/build/android/symbolize.py b/build/android/symbolize.py
new file mode 100755
index 0000000..d589a05
--- /dev/null
+++ b/build/android/symbolize.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+#
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Symbolizes stack traces generated by Chromium for Android.
+
+Sample usage:
+  adb logcat chromium:V | symbolize.py
+"""
+
+import os
+import re
+import sys
+
+from pylib import constants
+
+# Uses symbol.py from third_party/android_platform, not python's.
+sys.path.insert(0,
+                os.path.join(constants.DIR_SOURCE_ROOT,
+                            'third_party/android_platform/development/scripts'))
+import symbol
+
+# Sample output from base/debug/stack_trace_android.cc
+#00 pc 000634c1 /data/app-lib/org.chromium.native_test-1/libbase_unittests.so
+TRACE_LINE = re.compile('(?P<frame>\#[0-9]+) pc (?P<addr>[0-9a-f]{8,8}) '
+                        '(?P<lib>[^\r\n \t]+)')
+
+class Symbolizer(object):
+  def __init__(self, file_in, file_out):
+    self.file_in = file_in
+    self.file_out = file_out
+
+  def ProcessInput(self):
+    for line in self.file_in:
+      match = re.search(TRACE_LINE, line)
+      if not match:
+        self.file_out.write(line)
+        self.file_out.flush()
+        continue
+
+      frame = match.group('frame')
+      lib = match.group('lib')
+      addr = match.group('addr')
+
+      # TODO(scherkus): Doing a single lookup per line is pretty slow,
+      # especially with larger libraries. Consider caching strategies such as:
+      # 1) Have Python load the libraries and do symbol lookups instead of
+      #    calling out to addr2line each time.
+      # 2) Have Python keep multiple addr2line instances open as subprocesses,
+      #    piping addresses and reading back symbols as we find them
+      # 3) Read ahead the entire stack trace until we find no more, then batch
+      #    the symbol lookups.
+      #
+      # TODO(scherkus): These results are memoized, which could result in
+      # incorrect lookups when running this script on long-lived instances
+      # (e.g., adb logcat) when doing incremental development. Consider clearing
+      # the cache when modification timestamp of libraries change.
+      sym = symbol.SymbolInformation(lib, addr, False)[0][0]
+
+      if not sym:
+        self.file_out.write(line)
+        self.file_out.flush()
+        continue
+
+      pre = line[0:match.start('frame')]
+      post = line[match.end('lib'):]
+
+      self.file_out.write('%s%s pc %s %s%s' % (pre, frame, addr, sym, post))
+      self.file_out.flush()
+
+
+def main():
+  symbolizer = Symbolizer(sys.stdin, sys.stdout)
+  symbolizer.ProcessInput()
+
+
+if __name__ == '__main__':
+  main()
diff --git a/build/android/symbolize_test.py b/build/android/symbolize_test.py
new file mode 100755
index 0000000..bbd5eea
--- /dev/null
+++ b/build/android/symbolize_test.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+#
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unittest for symbolize.py.
+
+This test uses test libraries generated by the Android g++ toolchain.
+
+Should things break you can recreate the libraries and get the updated
+addresses and demangled names by running the following:
+  cd test/symbolize/
+  make
+  nm -gC *.so
+"""
+
+import sys
+import StringIO
+import unittest
+
+import symbolize
+
+def RunSymbolizer(text):
+  output = StringIO.StringIO()
+  s = symbolize.Symbolizer(StringIO.StringIO(text), output)
+  s.ProcessInput()
+  return output.getvalue()
+
+
+class SymbolizerUnittest(unittest.TestCase):
+  def testSingleLineNoMatch(self):
+    # Leading '#' is required.
+    expected = '00 pc 00000254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+
+    # Whitespace should be exactly one space.
+    expected = '#00  pc 00000254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+    expected = '#00 pc  00000254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+    expected = '#00 pc 00000254  /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+
+    # Decimal stack frame numbers are required.
+    expected = '#0a pc 00000254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+
+    # Lowercase 'pc' token is required.
+    expected = '#00 PC 00000254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+
+    # Hexadecimal addresses are required.
+    expected = '#00 pc ghijklmn /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+
+    # Addresses must be exactly 8 characters.
+    expected = '#00 pc 254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+    expected = '#00 pc 0254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+    expected = '#00 pc 00254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+    expected = '#00 pc 000254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+    expected = '#00 pc 0000254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+
+    # Addresses must not be prefixed with '0x'.
+    expected = '#00 pc 0x00000254 /build/android/tests/symbolize/liba.so\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+
+    # Library name is required.
+    expected = '#00 pc 00000254\n'
+    self.assertEqual(expected, RunSymbolizer(expected))
+
+  def testSingleLine(self):
+    text = '#00 pc 00000254 /build/android/tests/symbolize/liba.so\n'
+    expected = '#00 pc 00000254 A::Bar(char const*)\n'
+    actual = RunSymbolizer(text)
+    self.assertEqual(expected, actual)
+
+  def testSingleLineWithSurroundingText(self):
+    text = 'LEFT #00 pc 00000254 /build/android/tests/symbolize/liba.so RIGHT\n'
+    expected = 'LEFT #00 pc 00000254 A::Bar(char const*) RIGHT\n'
+    actual = RunSymbolizer(text)
+    self.assertEqual(expected, actual)
+
+  def testMultipleLinesSameLibrary(self):
+    text = '#00 pc 00000254 /build/android/tests/symbolize/liba.so\n'
+    text += '#01 pc 00000234 /build/android/tests/symbolize/liba.so\n'
+    expected = '#00 pc 00000254 A::Bar(char const*)\n'
+    expected += '#01 pc 00000234 A::Foo(int)\n'
+    actual = RunSymbolizer(text)
+    self.assertEqual(expected, actual)
+
+  def testMultipleLinesDifferentLibrary(self):
+    text = '#00 pc 00000254 /build/android/tests/symbolize/liba.so\n'
+    text += '#01 pc 00000234 /build/android/tests/symbolize/libb.so\n'
+    expected = '#00 pc 00000254 A::Bar(char const*)\n'
+    expected += '#01 pc 00000234 B::Baz(float)\n'
+    actual = RunSymbolizer(text)
+    self.assertEqual(expected, actual)
+
+  def testMultipleLinesWithSurroundingTextEverywhere(self):
+    text = 'TOP\n'
+    text += ('LEFT #00 pc 00000254 '
+             '/build/android/tests/symbolize/liba.so RIGHT\n')
+    text += ('LEFT #01 pc 00000234 '
+             '/build/android/tests/symbolize/libb.so RIGHT\n')
+    text += 'BOTTOM\n'
+    expected = 'TOP\n'
+    expected += 'LEFT #00 pc 00000254 A::Bar(char const*) RIGHT\n'
+    expected += 'LEFT #01 pc 00000234 B::Baz(float) RIGHT\n'
+    expected += 'BOTTOM\n'
+    actual = RunSymbolizer(text)
+    self.assertEqual(expected, actual)
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/build/android/tests/symbolize/Makefile b/build/android/tests/symbolize/Makefile
new file mode 100644
index 0000000..5178a04
--- /dev/null
+++ b/build/android/tests/symbolize/Makefile
@@ -0,0 +1,11 @@
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+TOOLCHAIN=../../../../third_party/android_tools/ndk/toolchains/arm-linux-androideabi-4.6/prebuilt/linux-x86_64/bin/arm-linux-androideabi-
+CXX=$(TOOLCHAIN)g++
+
+lib%.so: %.cc
+	$(CXX) -nostdlib -g -fPIC -shared $< -o $@
+
+all: liba.so libb.so
diff --git a/build/android/tests/symbolize/a.cc b/build/android/tests/symbolize/a.cc
new file mode 100644
index 0000000..f0c7ca4
--- /dev/null
+++ b/build/android/tests/symbolize/a.cc
@@ -0,0 +1,14 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class A {
+ public:
+  A();
+  void Foo(int i);
+  void Bar(const char* c);
+};
+
+A::A() {}
+void A::Foo(int i) {}
+void A::Bar(const char* c) {}
diff --git a/build/android/tests/symbolize/b.cc b/build/android/tests/symbolize/b.cc
new file mode 100644
index 0000000..db87520
--- /dev/null
+++ b/build/android/tests/symbolize/b.cc
@@ -0,0 +1,14 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class B {
+ public:
+  B();
+  void Baz(float f);
+  void Qux(double d);
+};
+
+B::B() {}
+void B::Baz(float f) {}
+void B::Qux(double d) {}
diff --git a/build/android/tombstones.py b/build/android/tombstones.py
new file mode 100755
index 0000000..b2dbfb6
--- /dev/null
+++ b/build/android/tombstones.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+#
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Find the most recent tombstone file(s) on all connected devices
+# and prints their stacks.
+#
+# Assumes tombstone file was created with current symbols.
+
+import datetime
+import logging
+import multiprocessing
+import os
+import subprocess
+import sys
+import optparse
+
+from pylib import android_commands
+
+
+def _ListTombstones(adb):
+  """List the tombstone files on the device.
+
+  Args:
+    adb: An instance of AndroidCommands.
+
+  Yields:
+    Tuples of (tombstone filename, date time of file on device).
+  """
+  lines = adb.RunShellCommand('TZ=UTC su -c ls -a -l /data/tombstones')
+  for line in lines:
+    if 'tombstone' in line and not 'No such file or directory' in line:
+      details = line.split()
+      t = datetime.datetime.strptime(details[-3] + ' ' + details[-2],
+                                     '%Y-%m-%d %H:%M')
+      yield details[-1], t
+
+
+def _GetDeviceDateTime(adb):
+  """Determine the date time on the device.
+
+  Args:
+    adb: An instance of AndroidCommands.
+
+  Returns:
+    A datetime instance.
+  """
+  device_now_string = adb.RunShellCommand('TZ=UTC date')
+  return datetime.datetime.strptime(
+      device_now_string[0], '%a %b %d %H:%M:%S %Z %Y')
+
+
+def _GetTombstoneData(adb, tombstone_file):
+  """Retrieve the tombstone data from the device
+
+  Args:
+    tombstone_file: the tombstone to retrieve
+
+  Returns:
+    A list of lines
+  """
+  return adb.GetProtectedFileContents('/data/tombstones/' + tombstone_file)
+
+
+def _EraseTombstone(adb, tombstone_file):
+  """Deletes a tombstone from the device.
+
+  Args:
+    tombstone_file: the tombstone to delete.
+  """
+  return adb.RunShellCommand('su -c rm /data/tombstones/' + tombstone_file)
+
+
+def _ResolveSymbols(tombstone_data, include_stack):
+  """Run the stack tool for given tombstone input.
+
+  Args:
+    tombstone_data: a list of strings of tombstone data.
+    include_stack: boolean whether to include stack data in output.
+
+  Yields:
+    A string for each line of resolved stack output.
+  """
+  stack_tool = os.path.join(os.path.dirname(__file__), '..', '..',
+                            'third_party', 'android_platform', 'development',
+                            'scripts', 'stack')
+  proc = subprocess.Popen(stack_tool, stdin=subprocess.PIPE,
+                          stdout=subprocess.PIPE)
+  output = proc.communicate(input='\n'.join(tombstone_data))[0]
+  for line in output.split('\n'):
+    if not include_stack and 'Stack Data:' in line:
+      break
+    yield line
+
+
+def _ResolveTombstone(tombstone):
+  lines = []
+  lines += [tombstone['file'] + ' created on ' + str(tombstone['time']) +
+            ', about this long ago: ' +
+            (str(tombstone['device_now'] - tombstone['time']) +
+            ' Device: ' + tombstone['serial'])]
+  print '\n'.join(lines)
+  print 'Resolving...'
+  lines += _ResolveSymbols(tombstone['data'], tombstone['stack'])
+  return lines
+
+
+def _ResolveTombstones(jobs, tombstones):
+  """Resolve a list of tombstones.
+
+  Args:
+    jobs: the number of jobs to use with multiprocess.
+    tombstones: a list of tombstones.
+  """
+  if not tombstones:
+    print 'No device attached?  Or no tombstones?'
+    return
+  if len(tombstones) == 1:
+    data = _ResolveTombstone(tombstones[0])
+  else:
+    pool = multiprocessing.Pool(processes=jobs)
+    data = pool.map(_ResolveTombstone, tombstones)
+    data = ['\n'.join(d) for d in data]
+  print '\n'.join(data)
+
+
+def _GetTombstonesForDevice(adb, options):
+  """Returns a list of tombstones on a given adb connection.
+
+  Args:
+    adb: An instance of Androidcommands.
+    options: command line arguments from OptParse
+  """
+  ret = []
+  all_tombstones = list(_ListTombstones(adb))
+  if not all_tombstones:
+    print 'No device attached?  Or no tombstones?'
+    return ret
+
+  # Sort the tombstones in date order, descending
+  all_tombstones.sort(cmp=lambda a, b: cmp(b[1], a[1]))
+
+  # Only resolve the most recent unless --all-tombstones given.
+  tombstones = all_tombstones if options.all_tombstones else [all_tombstones[0]]
+
+  device_now = _GetDeviceDateTime(adb)
+  for tombstone_file, tombstone_time in tombstones:
+    ret += [{'serial': adb.Adb().GetSerialNumber(),
+             'device_now': device_now,
+             'time': tombstone_time,
+             'file': tombstone_file,
+             'stack': options.stack,
+             'data': _GetTombstoneData(adb, tombstone_file)}]
+
+  # Erase all the tombstones if desired.
+  if options.wipe_tombstones:
+    for tombstone_file, _ in all_tombstones:
+      _EraseTombstone(adb, tombstone_file)
+
+  return ret
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('-a', '--all-tombstones', action='store_true',
+                    dest='all_tombstones', default=False,
+                    help="""Resolve symbols for all tombstones, rather than just
+                         the most recent""")
+  parser.add_option('-s', '--stack', action='store_true',
+                    dest='stack', default=False,
+                    help='Also include symbols for stack data')
+  parser.add_option('-w', '--wipe-tombstones', action='store_true',
+                    dest='wipe_tombstones', default=False,
+                    help='Erase all tombstones from device after processing')
+  parser.add_option('-j', '--jobs', type='int',
+                    default=4,
+                    help='Number of jobs to use when processing multiple '
+                         'crash stacks.')
+  options, args = parser.parse_args()
+
+  devices = android_commands.GetAttachedDevices()
+  tombstones = []
+  for device in devices:
+    adb = android_commands.AndroidCommands(device)
+    tombstones += _GetTombstonesForDevice(adb, options)
+
+  _ResolveTombstones(options.jobs, tombstones)
+
+if __name__ == '__main__':
+  sys.exit(main())