diff options
author | dmikurube@chromium.org <dmikurube@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2013-08-21 05:14:11 +0000 |
---|---|---|
committer | dmikurube@chromium.org <dmikurube@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2013-08-21 05:14:11 +0000 |
commit | 29a6f5ddca2e74d9990942480e62a1fee727e147 (patch) | |
tree | 65838df45b32a004693179b499090ca39f5e7795 /tools/deep_memory_profiler | |
parent | 2e919ddddd7312131fc2ef0b2b272e17e789d78f (diff) | |
download | chromium_src-29a6f5ddca2e74d9990942480e62a1fee727e147.zip chromium_src-29a6f5ddca2e74d9990942480e62a1fee727e147.tar.gz chromium_src-29a6f5ddca2e74d9990942480e62a1fee727e147.tar.bz2 |
An accumulater script to dump CSV or other summary from a result of 'dmprof cat'.
BUG=259206
NOTRY=True
R=peria@chromium.org
Review URL: https://codereview.chromium.org/22448004
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@218653 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools/deep_memory_profiler')
-rwxr-xr-x | tools/deep_memory_profiler/accumulate.py | 259 |
1 files changed, 259 insertions, 0 deletions
diff --git a/tools/deep_memory_profiler/accumulate.py b/tools/deep_memory_profiler/accumulate.py new file mode 100755 index 0000000..6998dc0 --- /dev/null +++ b/tools/deep_memory_profiler/accumulate.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# A script to accumulate values from the 'dmprof cat' command into CSV or else. +# +# Usage: +# ./accumulate.py -f <format> -t <template-name> < input.json > output +# +# <format> is one of "csv", "json", and "tree". If "csv" or "json" is given, +# accumulate.py dumps a similar file to "dmprof csv|json". If "tree" is given, +# accumulate.py dumps a human-readable breakdown tree. +# +# <template-name> is a label in templates.json. + +import datetime +import json +import logging +import optparse +import sys + +from lib.ordered_dict import OrderedDict + + +LOGGER = logging.getLogger('dmprof-accumulate') + + +def visit_in_template(template, snapshot, depth): + """Visits all categories via a given template. + + This function is not used. It's a sample function to traverse a template. + """ + world = template[0] + breakdown = template[1] + rules = template[2] + + for rule, _ in snapshot[world]['breakdown'][breakdown].iteritems(): + print (' ' * depth) + rule + if rule in rules: + visit_in_template(rules[rule], snapshot, depth + 1) + + +def accumulate(template, snapshot, units_dict, target_units): + """Accumulates units in a JSON |snapshot| with applying a given |template|. + + Args: + template: A template tree included in a dmprof cat JSON file. + snapshot: A snapshot in a dmprof cat JSON file. + units_dict: A dict of units in worlds. + target_units: A list of unit ids which are a target of this accumulation. + """ + world = template[0] + breakdown = template[1] + rules = template[2] + + remainder_units = target_units.copy() + category_tree = OrderedDict() + total = 0 + + for rule, match in snapshot[world]['breakdown'][breakdown].iteritems(): + if 'hidden' in match and match['hidden']: + continue + matched_units = set(match['units']).intersection(target_units) + subtotal = 0 + for unit_id in matched_units: + subtotal += units_dict[world][unit_id] + total += subtotal + remainder_units = remainder_units.difference(matched_units) + if rule not in rules: + # A category matched with |rule| is a leaf of the breakdown tree. + # It is NOT broken down more. + category_tree[rule] = subtotal + continue + + # A category matched with |rule| is broken down more. + subtemplate = rules[rule] + subworld = subtemplate[0] + subbreakdown = subtemplate[1] + + if subworld == world: + # Break down in the same world: consider units. + category_tree[rule], accounted_total, subremainder_units = accumulate( + subtemplate, snapshot, units_dict, matched_units) + subremainder_total = 0 + if subremainder_units: + for unit_id in subremainder_units: + subremainder_total += units_dict[world][unit_id] + category_tree[rule][None] = subremainder_total + if subtotal != accounted_total + subremainder_total: + print >> sys.stderr, ( + 'WARNING: Sum of %s:%s is different from %s by %d bytes.' % ( + subworld, subbreakdown, rule, + subtotal - (accounted_total + subremainder_total))) + else: + # Break down in a different world: consider only the total size. + category_tree[rule], accounted_total, _ = accumulate( + subtemplate, snapshot, units_dict, set(units_dict[subworld].keys())) + if subtotal >= accounted_total: + category_tree[rule][None] = subtotal - accounted_total + else: + print >> sys.stderr, ( + 'WARNING: Sum of %s:%s is larger than %s by %d bytes.' % ( + subworld, subbreakdown, rule, accounted_total - subtotal)) + print >> sys.stderr, ( + 'WARNING: Assuming remainder of %s is 0.' % rule) + category_tree[rule][None] = 0 + + return category_tree, total, remainder_units + + +def flatten(category_tree, header=''): + """Flattens a category tree into a flat list.""" + result = [] + for rule, sub in category_tree.iteritems(): + if not rule: + rule = 'remaining' + if header: + flattened_rule = header + '>' + rule + else: + flattened_rule = rule + if isinstance(sub, dict) or isinstance(sub, OrderedDict): + result.extend(flatten(sub, flattened_rule)) + else: + result.append((flattened_rule, sub)) + return result + + +def print_category_tree(category_tree, output, depth=0): + """Prints a category tree in a human-readable format.""" + for label in category_tree: + print >> output, (' ' * depth), + if (isinstance(category_tree[label], dict) or + isinstance(category_tree[label], OrderedDict)): + print >> output, '%s:' % label + print_category_tree(category_tree[label], output, depth + 1) + else: + print >> output, '%s: %d' % (label, category_tree[label]) + + +def flatten_all_category_trees(category_trees): + flattened_labels = set() + flattened_table = [] + for category_tree in category_trees: + flattened = OrderedDict() + for label, subtotal in flatten(category_tree): + flattened_labels.add(label) + flattened[label] = subtotal + flattened_table.append(flattened) + return flattened_labels, flattened_table + + +def output_csv(output, category_trees, data, first_time): + flattened_labels, flattened_table = flatten_all_category_trees(category_trees) + + sorted_flattened_labels = sorted(flattened_labels) + print >> output, ','.join(['second'] + sorted_flattened_labels) + for index, row in enumerate(flattened_table): + values = [str(data['snapshots'][index]['time'] - first_time)] + for label in sorted_flattened_labels: + if label in row: + values.append(str(row[label] / 1024.0 / 1024.0)) + else: + values.append('0') + print >> output, ','.join(values) + + +def output_json(output, category_trees, data, first_time, template_label): + flattened_labels, flattened_table = flatten_all_category_trees(category_trees) + + json_snapshots = [] + for index, row in enumerate(flattened_table): + row_with_meta = row.copy() + row_with_meta['second'] = data['snapshots'][index]['time'] - first_time + row_with_meta['dump_time'] = datetime.datetime.fromtimestamp( + data['snapshots'][index]['time']).strftime('%Y-%m-%d %H:%M:%S') + json_snapshots.append(row_with_meta) + json_root = { + 'version': 'JSON_DEEP_2', + 'policies': { + template_label: { + 'legends': sorted(flattened_labels), + 'snapshots': json_snapshots + } + } + } + json.dump(json_root, output, indent=2, sort_keys=True) + + +def output_tree(output, category_trees): + for index, category_tree in enumerate(category_trees): + print >> output, '< Snapshot #%d >' % index + print_category_tree(category_tree, output, 1) + print >> output, '' + + +def do_main(cat_input, output, template_label, output_format): + """Does the main work: accumulate for every snapshot and print a result.""" + if output_format not in ['csv', 'json', 'tree']: + raise NotImplementedError('The output format \"%s\" is not implemented.' % + output_format) + + data = json.loads(cat_input.read(), object_pairs_hook=OrderedDict) + + templates = data['templates'] + if not template_label: + template_label = data['default_template'] + if template_label not in templates: + LOGGER.error('A template \'%s\' is not found.' % template_label) + return + template = templates[template_label] + + category_trees = [] + first_time = None + + for snapshot in data['snapshots']: + if not first_time: + first_time = snapshot['time'] + + units = {} + for world_name in snapshot['worlds']: + world_units = {} + for unit_id, sizes in snapshot['worlds'][world_name]['units'].iteritems(): + world_units[int(unit_id)] = sizes[0] + units[world_name] = world_units + + category_tree, _, _ = accumulate( + template, snapshot['worlds'], units, set(units[template[0]].keys())) + category_trees.append(category_tree) + + if output_format == 'csv': + output_csv(output, category_trees, data, first_time) + elif output_format == 'json': + output_json(output, category_trees, data, first_time, template_label) + elif output_format == 'tree': + output_tree(output, category_trees) + + +def main(): + LOGGER.setLevel(logging.DEBUG) + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(message)s') + handler.setFormatter(formatter) + LOGGER.addHandler(handler) + + parser = optparse.OptionParser() + parser.add_option('-t', '--template', dest='template', + metavar='TEMPLATE', + help='Apply TEMPLATE to list up.') + parser.add_option('-f', '--format', dest='format', default='csv', + help='Specify the output format: csv, json or tree.') + + options, _ = parser.parse_args(sys.argv) + do_main(sys.stdin, sys.stdout, options.template, options.format) + + +if __name__ == '__main__': + sys.exit(main()) |