path: root/tools/deep_memory_profiler/dmprof

#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""The deep heap profiler script for Chrome."""

from datetime import datetime
import json
import os
import re
import subprocess
import sys
import tempfile

BUCKET_ID = 5
VIRTUAL = 0
COMMITTED = 1
ALLOC_COUNT = 2
FREE_COUNT = 3
NULL_REGEX = re.compile('')
PPROF_PATH = os.path.join(os.path.dirname(__file__),
                          os.pardir,
                          os.pardir,
                          'third_party',
                          'tcmalloc',
                          'chromium',
                          'src',
                          'pprof')

# Heap Profile Dump versions

# DUMP_DEEP_1 DOES NOT distinct mmap regions and malloc chunks.
# Their stacktraces DO contain mmap* or tc-* at their tops.
# They should be processed by POLICY_DEEP_1.
DUMP_DEEP_1 = 'DUMP_DEEP_1'

# DUMP_DEEP_2 DOES distinct mmap regions and malloc chunks.
# Their stacktraces still DO contain mmap* or tc-*.
# They should be processed by POLICY_DEEP_1.
DUMP_DEEP_2 = 'DUMP_DEEP_2'

# DUMP_DEEP_3 DOES distinct mmap regions and malloc chunks.
# Their stacktraces DO NOT contain mmap* or tc-*.
# They should be processed by POLICY_DEEP_2.
DUMP_DEEP_3 = 'DUMP_DEEP_3'

# DUMP_DEEP_4 adds some features to DUMP_DEEP_3:
# 1. Support comments starting with '#'
# 2. Support additional global stats: e.g. nonprofiled-*.
DUMP_DEEP_4 = 'DUMP_DEEP_4'

# Heap Profile Policy versions

# POLICY_DEEP_1 DOES NOT include allocation_type columns.
# mmap regions are distincted w/ mmap frames in the pattern column.
POLICY_DEEP_1 = 'POLICY_DEEP_1'

# POLICY_DEEP_2 DOES include allocation_type columns.
# mmap regions are distincted w/ the allocation_type column.
POLICY_DEEP_2 = 'POLICY_DEEP_2'

# TODO(dmikurube): Avoid global variables.
address_symbol_dict = {}
appeared_addresses = set()
components = []


class ParsingException(Exception):
  def __init__(self, value):
    self.value = value
  def __str__(self):
    return repr(self.value)


class Policy(object):

  def __init__(self, name, mmap, pattern):
    self.name = name
    self.mmap = mmap
    self.condition = re.compile(pattern + r'\Z')


def get_component(policy_list, bucket, mmap):
  """Returns a component name which a given bucket belongs to.

  Args:
      policy_list: A list containing Policy objects.  (Parsed policy data by
          parse_policy.)
      bucket: A Bucket object to be searched for.
      mmap: True if searching for a mmap region.

  Returns:
      A string representing a component name.
  """
  if not bucket:
    return 'no-bucket'
  if bucket.component:
    return bucket.component

  stacktrace = ''.join(
      address_symbol_dict[a] + ' ' for a in bucket.stacktrace).strip()

  for policy in policy_list:
    if mmap == policy.mmap and policy.condition.match(stacktrace):
      bucket.component = policy.name
      return policy.name

  assert False


class Bucket(object):

  def __init__(self, stacktrace):
    self.stacktrace = stacktrace
    self.component = ''


class Log(object):

  """A class representing one dumped log data."""
  def __init__(self, log_path):
    self.log_path = log_path
    self.log_lines = [
        l for l in open(self.log_path, 'r') if l and not l.startswith('#')]
    self.log_version = ''
    sys.stderr.write('Loading a dump: %s\n' % log_path)
    self.mmap_stacktrace_lines = []
    self.malloc_stacktrace_lines = []
    self.counters = {}
    self.log_time = os.stat(self.log_path).st_mtime

  @staticmethod
  def dump_stacktrace_lines(stacktrace_lines, buckets):
    """Prints a given stacktrace.

    Args:
        stacktrace_lines: A list of strings which are valid as stacktraces.
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
    """
    for l in stacktrace_lines:
      words = l.split()
      bucket = buckets.get(int(words[BUCKET_ID]))
      if not bucket:
        continue
      for i in range(0, BUCKET_ID - 1):
        sys.stdout.write(words[i] + ' ')
      for address in bucket.stacktrace:
        sys.stdout.write((address_symbol_dict.get(address) or address) + ' ')
      sys.stdout.write('\n')

  def dump_stacktrace(self, buckets):
    """Prints stacktraces contained in the log.

    Args:
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
    """
    self.dump_stacktrace_lines(self.mmap_stacktrace_lines, buckets)
    self.dump_stacktrace_lines(self.malloc_stacktrace_lines, buckets)

  @staticmethod
  def accumulate_size_for_pprof(stacktrace_lines, policy_list, buckets,
                                component_name, mmap):
    """Accumulates size of committed chunks and the number of allocated chunks.

    Args:
        stacktrace_lines: A list of strings which are valid as stacktraces.
        policy_list: A list containing Policy objects.  (Parsed policy data by
            parse_policy.)
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
        component_name: A name of component for filtering.
        mmap: True if searching for a mmap region.

    Returns:
        Two integers which are the accumulated size of committed regions and the
        number of allocated chunks, respectively.
    """
    com_committed = 0
    com_allocs = 0
    for l in stacktrace_lines:
      words = l.split()
      bucket = buckets.get(int(words[BUCKET_ID]))
      if (not bucket or
          (component_name and
           component_name != get_component(policy_list, bucket, mmap))):
        continue

      com_committed += int(words[COMMITTED])
      com_allocs += int(words[ALLOC_COUNT]) - int(words[FREE_COUNT])

    return com_committed, com_allocs

  @staticmethod
  def dump_stacktrace_lines_for_pprof(stacktrace_lines, policy_list,
                                      buckets, component_name, mmap):
    """Prints information of stacktrace lines for pprof.

    Args:
        stacktrace_lines: A list of strings which are valid as stacktraces.
        policy_list: A list containing Policy objects.  (Parsed policy data by
            parse_policy.)
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
        component_name: A name of component for filtering.
        mmap: True if searching for a mmap region.
    """
    for l in stacktrace_lines:
      words = l.split()
      bucket = buckets.get(int(words[BUCKET_ID]))
      if (not bucket or
          (component_name and
           component_name != get_component(policy_list, bucket, mmap))):
        continue

      sys.stdout.write('%6d: %8s [%6d: %8s] @' % (
          int(words[ALLOC_COUNT]) - int(words[FREE_COUNT]),
          words[COMMITTED],
          int(words[ALLOC_COUNT]) - int(words[FREE_COUNT]),
          words[COMMITTED]))
      for address in bucket.stacktrace:
        sys.stdout.write(' ' + address)
      sys.stdout.write('\n')

  def dump_for_pprof(self, policy_list, buckets, mapping_lines, component_name):
    """Converts the log file so it can be processed by pprof.

    Args:
        policy_list: A list containing Policy objects.  (Parsed policy data by
            parse_policy.)
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
        mapping_lines: A list of strings containing /proc/.../maps.
        component_name: A name of component for filtering.
    """
    sys.stdout.write('heap profile: ')
    com_committed, com_allocs = self.accumulate_size_for_pprof(
        self.mmap_stacktrace_lines, policy_list, buckets, component_name,
        True)
    add_committed, add_allocs = self.accumulate_size_for_pprof(
        self.malloc_stacktrace_lines, policy_list, buckets, component_name,
        False)
    com_committed += add_committed
    com_allocs += add_allocs

    sys.stdout.write('%6d: %8s [%6d: %8s] @ heapprofile\n' % (
        com_allocs, com_committed, com_allocs, com_committed))

    self.dump_stacktrace_lines_for_pprof(
        self.mmap_stacktrace_lines, policy_list, buckets, component_name,
        True)
    self.dump_stacktrace_lines_for_pprof(
        self.malloc_stacktrace_lines, policy_list, buckets, component_name,
        False)

    sys.stdout.write('MAPPED_LIBRARIES:\n')
    for l in mapping_lines:
      sys.stdout.write(l)

  @staticmethod
  def check_stacktrace_line(stacktrace_line, buckets):
    """Checks if a given stacktrace_line is valid as stacktrace.

    Args:
        stacktrace_line: A string to be checked.
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.

    Returns:
        True if the given stacktrace_line is valid.
    """
    words = stacktrace_line.split()
    if len(words) < BUCKET_ID + 1:
      return False
    if words[BUCKET_ID - 1] != '@':
      return False
    bucket = buckets.get(int(words[BUCKET_ID]))
    if bucket:
      for address in bucket.stacktrace:
        appeared_addresses.add(address)
    return True

  @staticmethod
  def skip_lines_while(line_number, max_line_number, skipping_condition):
    """Increments line_number until skipping_condition(line_number) is false.

    Returns:
        A pair of an integer indicating a line number after skipped, and a
        boolean value which is True if found a line which skipping_condition
        is False for.
    """
    while skipping_condition(line_number):
      line_number += 1
      if line_number >= max_line_number:
        return line_number, False
    return line_number, True

  def parse_stacktraces_while_valid(self, buckets, log_lines, line_number):
    """Parses stacktrace lines while the lines are valid.

    Args:
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
        log_lines: A list of lines to be parsed.
        line_number: An integer representing the starting line number in
            log_lines.

    Returns:
        A pair of a list of valid lines and an integer representing the last
        line number in log_lines.
    """
    (line_number, _) = self.skip_lines_while(
        line_number, len(log_lines),
        lambda n: not log_lines[n].split()[0].isdigit())
    stacktrace_lines_start = line_number
    (line_number, _) = self.skip_lines_while(
        line_number, len(log_lines),
        lambda n: self.check_stacktrace_line(log_lines[n], buckets))
    return (log_lines[stacktrace_lines_start:line_number], line_number)

  def parse_stacktraces(self, buckets, line_number):
    """Parses lines in self.log_lines as stacktrace.

    Valid stacktrace lines are stored into self.mmap_stacktrace_lines and
    self.malloc_stacktrace_lines.

    Args:
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
        line_number: An integer representing the starting line number in
            log_lines.

    Raises:
        ParsingException for invalid dump versions.
    """
    sys.stderr.write('  Version: %s\n' % self.log_version)

    if self.log_version in (DUMP_DEEP_3, DUMP_DEEP_4):
      (self.mmap_stacktrace_lines, line_number) = (
          self.parse_stacktraces_while_valid(
              buckets, self.log_lines, line_number))
      (line_number, _) = self.skip_lines_while(
          line_number, len(self.log_lines),
          lambda n: self.log_lines[n] != 'MALLOC_STACKTRACES:\n')
      (self.malloc_stacktrace_lines, line_number) = (
          self.parse_stacktraces_while_valid(
              buckets, self.log_lines, line_number))

    elif self.log_version == DUMP_DEEP_2:
      (self.mmap_stacktrace_lines, line_number) = (
          self.parse_stacktraces_while_valid(
              buckets, self.log_lines, line_number))
      (line_number, _) = self.skip_lines_while(
          line_number, len(self.log_lines),
          lambda n: self.log_lines[n] != 'MALLOC_STACKTRACES:\n')
      (self.malloc_stacktrace_lines, line_number) = (
          self.parse_stacktraces_while_valid(
              buckets, self.log_lines, line_number))
      self.malloc_stacktrace_lines.extend(self.mmap_stacktrace_lines)
      self.mmap_stacktrace_lines = []

    elif self.log_version == DUMP_DEEP_1:
      (self.malloc_stacktrace_lines, line_number) = (
          self.parse_stacktraces_while_valid(
              buckets, self.log_lines, line_number))

    else:
      raise ParsingException('invalid heap profile dump version: %s' % (
          self.log_version))

  def parse_global_stats(self):
    """Parses lines in self.log_lines as global stats."""
    (ln, _) = self.skip_lines_while(
        0, len(self.log_lines),
        lambda n: self.log_lines[n] != 'GLOBAL_STATS:\n')

    if self.log_version == DUMP_DEEP_4:
      global_stat_names = [
          'total', 'file-exec', 'file-nonexec', 'anonymous', 'stack', 'other',
          'nonprofiled-absent', 'nonprofiled-anonymous',
          'nonprofiled-file-exec', 'nonprofiled-file-nonexec',
          'nonprofiled-stack', 'nonprofiled-other',
          'profiled-mmap', 'profiled-malloc']
    else:
      global_stat_names = [
          'total', 'file', 'anonymous', 'other', 'mmap', 'tcmalloc']

    for prefix in global_stat_names:
      (ln, _) = self.skip_lines_while(
          ln, len(self.log_lines),
          lambda n: self.log_lines[n].split()[0] != prefix)
      words = self.log_lines[ln].split()
      self.counters[prefix + '_virtual'] = int(words[-2])
      self.counters[prefix + '_committed'] = int(words[-1])

  def parse_version(self):
    """Parses a version string in self.log_lines.

    Returns:
        A pair of (a string representing a version of the stacktrace dump,
        and an integer indicating a line number next to the version string).

    Raises:
        ParsingException for invalid dump versions.
    """
    version = ''

    # Skip until an identifiable line.
    headers = ('STACKTRACES:\n', 'MMAP_STACKTRACES:\n', 'heap profile: ')
    if not self.log_lines:
      raise ParsingException('Empty heap dump file.')
    (ln, found) = self.skip_lines_while(
        0, len(self.log_lines),
        lambda n: not self.log_lines[n].startswith(headers))
    if not found:
      raise ParsingException('Invalid heap dump file (no version header).')

    # Identify a version.
    if self.log_lines[ln].startswith('heap profile: '):
      version = self.log_lines[ln][13:].strip()
      if (version == DUMP_DEEP_2 or version == DUMP_DEEP_3 or
          version == DUMP_DEEP_4):
        (ln, _) = self.skip_lines_while(
            ln, len(self.log_lines),
            lambda n: self.log_lines[n] != 'MMAP_STACKTRACES:\n')
      else:
        raise ParsingException('invalid heap profile dump version: %s'
                               % version)
    elif self.log_lines[ln] == 'STACKTRACES:\n':
      version = DUMP_DEEP_1
    elif self.log_lines[ln] == 'MMAP_STACKTRACES:\n':
      version = DUMP_DEEP_2

    return (version, ln)

  def parse_log(self, buckets):
    self.log_version, ln = self.parse_version()
    self.parse_global_stats()
    self.parse_stacktraces(buckets, ln)

  @staticmethod
  def accumulate_size_for_policy(stacktrace_lines,
                                 policy_list, buckets, sizes, mmap):
    for l in stacktrace_lines:
      words = l.split()
      bucket = buckets.get(int(words[BUCKET_ID]))
      component_match = get_component(policy_list, bucket, mmap)
      sizes[component_match] += int(words[COMMITTED])

      if component_match.startswith('tc-'):
        sizes['tc-total-log'] += int(words[COMMITTED])
      elif component_match.startswith('mmap-'):
        sizes['mmap-total-log'] += int(words[COMMITTED])
      else:
        sizes['other-total-log'] += int(words[COMMITTED])

  def apply_policy(self, policy_list, buckets, first_log_time):
    """Aggregates the total memory size of each component.

    Iterate through all stacktraces and attribute them to one of the components
    based on the policy.  It is important to apply policy in right order.

    Args:
        policy_list: A list containing Policy objects.  (Parsed policy data by
            parse_policy.)
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
        first_log_time: An integer representing time when the first log is
            dumped.

    Returns:
        A dict mapping components and their corresponding sizes.
    """

    sys.stderr.write('apply policy:%s\n' % (self.log_path))
    sizes = dict((c, 0) for c in components)

    self.accumulate_size_for_policy(self.mmap_stacktrace_lines,
                                    policy_list, buckets, sizes, True)
    self.accumulate_size_for_policy(self.malloc_stacktrace_lines,
                                    policy_list, buckets, sizes, False)

    if self.log_version == DUMP_DEEP_4:
      mmap_prefix = 'profiled-mmap'
      malloc_prefix = 'profiled-malloc'
    else:
      mmap_prefix = 'mmap'
      malloc_prefix = 'tcmalloc'

    sizes['mmap-no-log'] = (
        self.counters['%s_committed' % mmap_prefix] - sizes['mmap-total-log'])
    sizes['mmap-total-record'] = self.counters['%s_committed' % mmap_prefix]
    sizes['mmap-total-record-vm'] = self.counters['%s_virtual' % mmap_prefix]

    sizes['tc-no-log'] = (
        self.counters['%s_committed' % malloc_prefix] - sizes['tc-total-log'])
    sizes['tc-total-record'] = self.counters['%s_committed' % malloc_prefix]
    sizes['tc-unused'] = (
        sizes['mmap-tcmalloc'] - self.counters['%s_committed' % malloc_prefix])
    sizes['tc-total'] = sizes['mmap-tcmalloc']

    for key, value in {
        'total': 'total_committed',
        'filemapped': 'file_committed',
        'file-exec': 'file-exec_committed',
        'file-nonexec': 'file-nonexec_committed',
        'anonymous': 'anonymous_committed',
        'stack': 'stack_committed',
        'other': 'other_committed',
        'nonprofiled-absent': 'nonprofiled-absent_committed',
        'nonprofiled-anonymous': 'nonprofiled-anonymous_committed',
        'nonprofiled-file-exec': 'nonprofiled-file-exec_committed',
        'nonprofiled-file-nonexec': 'nonprofiled-file-nonexec_committed',
        'nonprofiled-stack': 'nonprofiled-stack_committed',
        'nonprofiled-other': 'nonprofiled-other_committed',
        'total-vm': 'total_virtual',
        'filemapped-vm': 'file_virtual',
        'anonymous-vm': 'anonymous_virtual',
        'other-vm': 'other_virtual' }.iteritems():
      if key in sizes:
        sizes[key] = self.counters[value]

    if 'mustbezero' in sizes:
      removed = (
          '%s_committed' % mmap_prefix,
          'nonprofiled-absent_committed',
          'nonprofiled-anonymous_committed',
          'nonprofiled-file-exec_committed',
          'nonprofiled-file-nonexec_committed',
          'nonprofiled-stack_committed',
          'nonprofiled-other_committed')
      sizes['mustbezero'] = (
          self.counters['total_committed'] -
          sum(self.counters[i] for i in removed))
    if 'total-exclude-profiler' in sizes:
      sizes['total-exclude-profiler'] = (
          self.counters['total_committed'] - sizes['mmap-profiler'])
    if 'hour' in sizes:
      sizes['hour'] = (self.log_time - first_log_time) / 60.0 / 60.0
    if 'minute' in sizes:
      sizes['minute'] = (self.log_time - first_log_time) / 60.0
    if 'second' in sizes:
      sizes['second'] = self.log_time - first_log_time

    return sizes

  @staticmethod
  def accumulate_size_for_expand(stacktrace_lines, policy_list, buckets,
                                 component_name, depth, sizes, mmap):
    for line in stacktrace_lines:
      words = line.split()
      bucket = buckets.get(int(words[BUCKET_ID]))
      component_match = get_component(policy_list, bucket, mmap)
      if component_match == component_name:
        stacktrace_sequence = ''
        for address in bucket.stacktrace[0 : min(len(bucket.stacktrace),
                                                 1 + depth)]:
          stacktrace_sequence += address_symbol_dict[address] + ' '
        if not stacktrace_sequence in sizes:
          sizes[stacktrace_sequence] = 0
        sizes[stacktrace_sequence] += int(words[COMMITTED])

  def expand(self, policy_list, buckets, component_name, depth):
    """Prints all stacktraces in a given component of given depth.

    Args:
        policy_list: A list containing Policy objects.  (Parsed policy data by
            parse_policy.)
        buckets: A dict mapping bucket ids and their corresponding Bucket
            objects.
        component_name: A name of component for filtering.
        depth: An integer representing depth to be printed.
    """
    sizes = {}

    self.accumulate_size_for_expand(
        self.mmap_stacktrace_lines, policy_list, buckets, component_name,
        depth, sizes, True)
    self.accumulate_size_for_expand(
        self.malloc_stacktrace_lines, policy_list, buckets, component_name,
        depth, sizes, False)

    sorted_sizes_list = sorted(
        sizes.iteritems(), key=(lambda x: x[1]), reverse=True)
    total = 0
    for size_pair in sorted_sizes_list:
      sys.stdout.write('%10d %s\n' % (size_pair[1], size_pair[0]))
      total += size_pair[1]
    sys.stderr.write('total: %d\n' % (total))


def update_symbols(symbol_path, mapping_lines, chrome_path):
  """Updates address/symbol mapping on memory and in a .symbol cache file.

  It reads cached address/symbol mapping from a .symbol file if it exists.
  Then, it resolves unresolved addresses from a Chrome binary with pprof.
  Both mappings on memory and in a .symbol cache file are updated.

  Symbol files are formatted as follows:
    <Address> <Symbol>
    <Address> <Symbol>
    <Address> <Symbol>
    ...

  Args:
      symbol_path: A string representing a path for a .symbol file.
      mapping_lines: A list of strings containing /proc/.../maps.
      chrome_path: A string representing a path for a Chrome binary.
  """
  with open(symbol_path, mode='a+') as symbol_f:
    symbol_lines = symbol_f.readlines()
    if symbol_lines:
      for line in symbol_lines:
        items = line.split(None, 1)
        address_symbol_dict[items[0]] = items[1].rstrip()

    unresolved_addresses = sorted(
        a for a in appeared_addresses if a not in address_symbol_dict)

    if unresolved_addresses:
      with tempfile.NamedTemporaryFile(
          suffix='maps', prefix="dmprof", mode='w+') as pprof_in:
        with tempfile.NamedTemporaryFile(
            suffix='symbols', prefix="dmprof", mode='w+') as pprof_out:
          for line in mapping_lines:
            pprof_in.write(line)

          for address in unresolved_addresses:
            pprof_in.write(address + '\n')

          pprof_in.seek(0)

          p = subprocess.Popen(
              '%s --symbols %s' % (PPROF_PATH, chrome_path),
              shell=True, stdin=pprof_in, stdout=pprof_out)
          p.wait()

          pprof_out.seek(0)
          symbols = pprof_out.readlines()
          symbol_f.seek(0, 2)
          for address, symbol in zip(unresolved_addresses, symbols):
            stripped_symbol = symbol.strip()
            address_symbol_dict[address] = stripped_symbol
            symbol_f.write('%s %s\n' % (address, symbol.strip()))


def parse_policy(policy_path):
  """Parses policy file.

  A policy file contains component's names and their
  stacktrace pattern written in regular expression.
  Those patterns are matched against each symbols of
  each stacktraces in the order written in the policy file

  Args:
       policy_path: A path for a policy file.
  Returns:
       A list containing component's name and its regex object
  """
  with open(policy_path, mode='r') as policy_f:
    policy_lines = policy_f.readlines()

  policy_version = POLICY_DEEP_1
  if policy_lines[0].startswith('heap profile policy: '):
    policy_version = policy_lines[0][21:].strip()
    policy_lines.pop(0)
  policy_list = []

  if policy_version == POLICY_DEEP_2 or policy_version == POLICY_DEEP_1:
    sys.stderr.write('  heap profile policy version: %s\n' % policy_version)
    for line in policy_lines:
      if line[0] == '#':
        continue

      if policy_version == POLICY_DEEP_2:
        (name, allocation_type, pattern) = line.strip().split(None, 2)
        mmap = False
        if allocation_type == 'mmap':
          mmap = True
      elif policy_version == POLICY_DEEP_1:
        name = line.split()[0]
        pattern = line[len(name) : len(line)].strip()
        mmap = False

      if pattern != 'default':
        policy_list.append(Policy(name, mmap, pattern))
      if components.count(name) == 0:
        components.append(name)

  else:
    sys.stderr.write('  invalid heap profile policy version: %s\n' % (
        policy_version))

  return policy_list


def main():
  if (len(sys.argv) < 4) or (not (sys.argv[1] in ['--csv',
                                                  '--json',
                                                  '--expand',
                                                  '--list',
                                                  '--stacktrace',
                                                  '--pprof'])):
    sys.stderr.write("""Usage:
%s [options] <chrome-binary> <policy> <profile> [component-name] [depth]

Options:
  --csv                         Output result in csv format
  --json                        Output result in json format
  --stacktrace                  Convert raw address to symbol names
  --list                        Lists components and their sizes
  --expand                      Show all stacktraces in the specified component
                                of given depth with their sizes
  --pprof                       Format the profile file so it can be processed
                                by pprof

Examples:
  dmprof --csv Debug/chrome dmpolicy hprof.12345.0001.heap > result.csv
  dmprof --json Debug/chrome dmpolicy hprof.12345.0001.heap > result.json
  dmprof --list Debug/chrome dmpolicy hprof.12345.0012.heap
  dmprof --expand Debug/chrome dmpolicy hprof.12345.0012.heap tc-webkit 4
  dmprof --pprof Debug/chrome dmpolicy hprof.12345.0012.heap > for_pprof.txt
""" % (sys.argv[0]))
    sys.exit(1)

  action = sys.argv[1]
  chrome_path = sys.argv[2]
  policy_path = sys.argv[3]
  log_path = sys.argv[4]

  sys.stderr.write('parsing a policy file\n')
  policy_list = parse_policy(policy_path)

  p = re.compile('\.[0-9][0-9][0-9][0-9]\.heap')
  prefix = p.sub('', log_path)
  symbol_path = prefix + '.symbols'

  sys.stderr.write('parsing the maps file\n')
  maps_path = prefix + '.maps'
  with open(maps_path, 'r') as maps_f:
    maps_lines = maps_f.readlines()

  # Reading buckets
  sys.stderr.write('parsing the bucket file\n')
  buckets = {}
  bucket_count = 0
  n = 0
  while True:
    buckets_path = '%s.%04d.buckets' % (prefix, n)
    if not os.path.exists(buckets_path):
      if n > 10:
        break
      n += 1
      continue
    sys.stderr.write('reading buckets from %s\n' % (buckets_path))
    with open(buckets_path, 'r') as buckets_f:
      for l in buckets_f:
        words = l.split()
        buckets[int(words[0])] = Bucket(words[1:])
    n += 1

  log_path_list = [log_path]

  if action in ('--csv', '--json'):
    # search for the sequence of files
    n = int(log_path[len(log_path) - 9 : len(log_path) - 5])
    n += 1  # skip current file
    while True:
      p = '%s.%04d.heap' % (prefix, n)
      if os.path.exists(p):
        log_path_list.append(p)
      else:
        break
      n += 1

  logs = []
  for path in log_path_list:
    new_log = Log(path)
    sys.stderr.write('Parsing a dump: %s\n' % path)
    try:
      new_log.parse_log(buckets)
    except ParsingException:
      sys.stderr.write('  Ignored an invalid dump: %s\n' % path)
    else:
      logs.append(new_log)

  sys.stderr.write('getting symbols\n')
  update_symbols(symbol_path, maps_lines, chrome_path)

  # TODO(dmikurube): Many modes now.  Split them into separete functions.
  if action == '--stacktrace':
    logs[0].dump_stacktrace(buckets)

  elif action == '--csv':
    sys.stdout.write(','.join(components))
    sys.stdout.write('\n')

    for log in logs:
      component_sizes = log.apply_policy(policy_list, buckets, logs[0].log_time)
      s = []
      for c in components:
        if c in ('hour', 'minute', 'second'):
          s.append('%05.5f' % (component_sizes[c]))
        else:
          s.append('%05.5f' % (component_sizes[c] / 1024.0 / 1024.0))
      sys.stdout.write(','.join(s))
      sys.stdout.write('\n')

  elif action == '--json':
    json_base = {
      'version': 'JSON_DEEP_1',
      'legends': components,
      'snapshots': [],
    }
    for log in logs:
      component_sizes = log.apply_policy(policy_list, buckets, logs[0].log_time)
      component_sizes['log_path'] = log.log_path
      component_sizes['log_time'] = datetime.fromtimestamp(
          log.log_time).strftime('%Y-%m-%d %H:%M:%S')
      json_base['snapshots'].append(component_sizes)
    json.dump(json_base, sys.stdout, indent=2, sort_keys=True)

  elif action == '--list':
    component_sizes = logs[0].apply_policy(
        policy_list, buckets, logs[0].log_time)
    for c in components:
      if c in ['hour', 'minute', 'second']:
        sys.stdout.write('%30s %10.3f\n' % (c, component_sizes[c]))
      else:
        sys.stdout.write('%30s %10.3f\n' % (
            c, component_sizes[c] / 1024.0 / 1024.0))

  elif action == '--expand':
    component_name = sys.argv[5]
    depth = sys.argv[6]
    logs[0].expand(policy_list, buckets, component_name, int(depth))

  elif action == '--pprof':
    if len(sys.argv) > 5:
      logs[0].dump_for_pprof(policy_list, buckets, maps_lines, sys.argv[5])
    else:
      logs[0].dump_for_pprof(policy_list, buckets, maps_lines, None)


if __name__ == '__main__':
  sys.exit(main())