diff options
author | dmikurube@chromium.org <dmikurube@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-09-26 16:37:24 +0000 |
---|---|---|
committer | dmikurube@chromium.org <dmikurube@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-09-26 16:37:24 +0000 |
commit | ca790e60e3aa40b24391b4eacc2a741e1d102018 (patch) | |
tree | dee90431389cab0a42b84abef7b555478787f3f6 /tools/deep_memory_profiler | |
parent | 7709a81b235e422ca27b997164d391fad202d0e4 (diff) | |
download | chromium_src-ca790e60e3aa40b24391b4eacc2a741e1d102018.zip chromium_src-ca790e60e3aa40b24391b4eacc2a741e1d102018.tar.gz chromium_src-ca790e60e3aa40b24391b4eacc2a741e1d102018.tar.bz2 |
Classify memory usage by allocated type in Deep Memory Profiler.
BUG=123758
Review URL: https://chromiumcodereview.appspot.com/10825075
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@158812 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'tools/deep_memory_profiler')
-rwxr-xr-x | tools/deep_memory_profiler/dmprof | 169 | ||||
-rw-r--r-- | tools/deep_memory_profiler/policies.json | 4 | ||||
-rw-r--r-- | tools/deep_memory_profiler/policy.l0.json | 6 | ||||
-rw-r--r-- | tools/deep_memory_profiler/policy.l1.json | 6 | ||||
-rw-r--r-- | tools/deep_memory_profiler/policy.l2.json | 6 | ||||
-rw-r--r-- | tools/deep_memory_profiler/policy.t0.json | 198 |
6 files changed, 337 insertions, 52 deletions
diff --git a/tools/deep_memory_profiler/dmprof b/tools/deep_memory_profiler/dmprof index 77b1f31..c27e35b 100755 --- a/tools/deep_memory_profiler/dmprof +++ b/tools/deep_memory_profiler/dmprof @@ -22,8 +22,9 @@ FIND_RUNTIME_SYMBOLS_PATH = os.path.join( sys.path.append(FIND_RUNTIME_SYMBOLS_PATH) from find_runtime_symbols import find_runtime_symbols_list +from find_runtime_symbols import find_runtime_typeinfo_symbols_list +from find_runtime_symbols import RuntimeSymbolsInProcess from prepare_symbol_info import prepare_symbol_info -from static_symbols import StaticSymbols BUCKET_ID = 5 VIRTUAL = 0 @@ -81,6 +82,9 @@ POLICY_DEEP_2 = 'POLICY_DEEP_2' # POLICY_DEEP_3 is in JSON format. POLICY_DEEP_3 = 'POLICY_DEEP_3' +# POLICY_DEEP_3 contains typeinfo. +POLICY_DEEP_4 = 'POLICY_DEEP_4' + class EmptyDumpException(Exception): def __init__(self, value): @@ -119,27 +123,33 @@ class DelayedStaticSymbols(object): if keep: self.prepared_data_dir = prefix + '.pre' self.loaded_static_symbols = None + self.loaded_symbols_in_process = None def get(self): - if not self.loaded_static_symbols: + if not self.loaded_symbols_in_process: if not self.keep: self.prepared_data_dir = tempfile.mkdtemp() try: prepare_symbol_info(self.maps_path, self.prepared_data_dir) - self.loaded_static_symbols = StaticSymbols.load(self.prepared_data_dir) + self.loaded_symbols_in_process = RuntimeSymbolsInProcess.load( + self.prepared_data_dir) finally: if not self.keep: shutil.rmtree(self.prepared_data_dir) - return self.loaded_static_symbols + return self.loaded_symbols_in_process class Rule(object): """Represents one matching rule in a policy file.""" - def __init__(self, name, mmap, stacktrace_pattern): + def __init__(self, name, mmap, stacktrace_pattern, typeinfo_pattern=None): self.name = name self.mmap = mmap self.stacktrace_pattern = re.compile(stacktrace_pattern + r'\Z') + if typeinfo_pattern: + self.typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z') + else: + self.typeinfo_pattern = None class Policy(object): @@ -171,9 +181,14 @@ def get_component(rule_list, bucket, symbols): return bucket.component_cache stacktrace = ''.join(symbols[a] + ' ' for a in bucket.stacktrace).strip() + typeinfo = bucket.typeinfo_symbol + if typeinfo.startswith('0x'): + typeinfo = bucket.typename for rule in rule_list: - if bucket.mmap == rule.mmap and rule.stacktrace_pattern.match(stacktrace): + if (bucket.mmap == rule.mmap and + rule.stacktrace_pattern.match(stacktrace) and + (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))): bucket.component_cache = rule.name return rule.name @@ -183,9 +198,12 @@ def get_component(rule_list, bucket, symbols): class Bucket(object): """Represents a bucket, which is a unit of memory classification.""" - def __init__(self, stacktrace, mmap): + def __init__(self, stacktrace, mmap, typeinfo, typename): self.stacktrace = stacktrace self.mmap = mmap + self.typeinfo = typeinfo + self.typeinfo_symbol = typename + self.typename = typename self.component_cache = '' def clear_component_cache(self): @@ -563,13 +581,17 @@ class Dump(object): @staticmethod def accumulate_size_for_expand(stacktrace_lines, rule_list, buckets, - component_name, depth, sizes, symbols): + component_name, depth, sizes, symbols, + typeinfo_symbols): for line in stacktrace_lines: words = line.split() bucket = buckets.get(int(words[BUCKET_ID])) component_match = get_component(rule_list, bucket, symbols) if component_match == component_name: stacktrace_sequence = '' + if bucket.typeinfo: + stacktrace_sequence += '(type=%s)' % typeinfo_symbols[bucket.typeinfo] + stacktrace_sequence += ' (type.name=%s) ' % bucket.typename for address in bucket.stacktrace[0 : min(len(bucket.stacktrace), 1 + depth)]: stacktrace_sequence += symbols[address] + ' ' @@ -577,7 +599,9 @@ class Dump(object): sizes[stacktrace_sequence] = 0 sizes[stacktrace_sequence] += int(words[COMMITTED]) - def expand(self, rule_list, buckets, component_name, depth, symbols): + def expand( + self, rule_list, buckets, component_name, depth, symbols, + typeinfo_symbols): """Prints all stacktraces in a given component of given depth. Args: @@ -591,7 +615,7 @@ class Dump(object): self.accumulate_size_for_expand( self.stacktrace_lines, rule_list, buckets, component_name, - depth, sizes, symbols) + depth, sizes, symbols, typeinfo_symbols) sorted_sizes_list = sorted( sizes.iteritems(), key=(lambda x: x[1]), reverse=True) @@ -603,7 +627,8 @@ class Dump(object): def update_symbols( - symbol_path, delayed_static_symbols, appeared_addresses, symbols): + symbol_path, delayed_static_symbols, appeared_addresses, + parameter_find_runtime_symbols_list, symbols): """Updates address/symbol mapping on memory and in a .symbol cache file. It reads cached address/symbol mapping from a .symbol file if it exists. @@ -620,6 +645,7 @@ def update_symbols( symbol_path: A string representing a path for a .symbol file. delayed_static_symbols: A DelayedStaticSymbols object. appeared_addresses: A list of known addresses. + parameter_find_runtime_symbols_list: A function to find symbols. symbols: A dict mapping runtime addresses to symbol names. """ with open(symbol_path, mode='a+') as symbol_f: @@ -643,9 +669,12 @@ def update_symbols( else: sys.stderr.write(' %d addresses unresolved.\n' % len(unresolved_addresses)) - static_symbols = delayed_static_symbols.get() - symbol_list = find_runtime_symbols_list( - static_symbols, unresolved_addresses) + + sys.stderr.write(' Loading symbols\n') + symbols_in_process = delayed_static_symbols.get() + symbol_list = parameter_find_runtime_symbols_list( + symbols_in_process, unresolved_addresses) + sys.stderr.write(' Loaded\n') for address, symbol in zip(unresolved_addresses, symbol_list): if not symbol: @@ -727,8 +756,13 @@ def parse_policy_json(policy_path): rules = [] for rule in policy['rules']: - rules.append(Rule( - rule['name'], rule['allocator'] == 'mmap', rule['stacktrace'])) + if 'typeinfo' in rule: + rules.append(Rule( + rule['name'], rule['allocator'] == 'mmap', rule['stacktrace'], + rule['typeinfo'])) + else: + rules.append(Rule( + rule['name'], rule['allocator'] == 'mmap', rule['stacktrace'])) return Policy(rules, policy['version'], policy['components']) @@ -739,6 +773,7 @@ def find_prefix(path): def load_buckets(prefix): # Reading buckets sys.stderr.write('Loading bucket files.\n') + appeared_typeinfo_addresses = set() buckets = {} bucket_count = 0 n = 0 @@ -753,11 +788,26 @@ def load_buckets(prefix): with open(buckets_path, 'r') as buckets_f: for line in buckets_f: words = line.split() - stacktrace = [int(address, 16) for address in words[2:]] - buckets[int(words[0])] = Bucket(stacktrace, words[1] == 'mmap') + typeinfo = None + typename = '' + stacktrace_begin = 2 + for index, word in enumerate(words): + if index < 2: + continue + if word[0] == 't': + typeinfo = int(word[1:], 16) + appeared_typeinfo_addresses.add(typeinfo) + elif word[0] == 'n': + typename = word[1:] + else: + stacktrace_begin = index + break + stacktrace = [int(address, 16) for address in words[stacktrace_begin:]] + buckets[int(words[0])] = Bucket( + stacktrace, words[1] == 'mmap', typeinfo, typename) n += 1 - return buckets + return buckets, appeared_typeinfo_addresses def determine_dump_path_list(dump_path, prefix): @@ -812,13 +862,24 @@ def load_dumps(dump_path_list, buckets): def load_and_update_symbol_cache( - prefix, appeared_addresses, delayed_static_symbols): + prefix, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols): symbol_path = prefix + '.symbols' sys.stderr.write('Loading and updating symbol cache: "%s".\n' % symbol_path) symbols = {} update_symbols( - symbol_path, delayed_static_symbols, appeared_addresses, symbols) - return symbols + symbol_path, delayed_static_symbols, appeared_addresses, + find_runtime_symbols_list, symbols) + + typeinfo_symbol_path = prefix + '.tsymbols' + sys.stderr.write('Loading and updating typeinfo symbol cache: "%s".\n' % + typeinfo_symbol_path) + typeinfo_symbols = {} + update_symbols( + typeinfo_symbol_path, delayed_static_symbols, appeared_typeinfo_addresses, + find_runtime_typeinfo_symbols_list, typeinfo_symbols) + + return symbols, typeinfo_symbols def load_default_policies(): @@ -870,23 +931,37 @@ def load_policies(options_policy): def load_basic_files_with_multiple_dumps(dump_path, keep): prefix = find_prefix(dump_path) - buckets = load_buckets(prefix) + buckets, appeared_typeinfo_addresses = load_buckets(prefix) dumps, appeared_addresses = load_dumps( determine_dump_path_list(dump_path, prefix), buckets) delayed_static_symbols = DelayedStaticSymbols(prefix, keep) - symbols = load_and_update_symbol_cache( - prefix, appeared_addresses, delayed_static_symbols) - return buckets, dumps, appeared_addresses, delayed_static_symbols, symbols + symbols, typeinfo_symbols = load_and_update_symbol_cache( + prefix, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols) + for bucket in buckets: + if buckets[bucket].typeinfo != None: + buckets[bucket].typeinfo_symbol = typeinfo_symbols[ + buckets[bucket].typeinfo] + + return (buckets, dumps, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols, symbols, typeinfo_symbols) def load_basic_files_with_single_dump(dump_path, keep): prefix = find_prefix(dump_path) - buckets = load_buckets(prefix) + buckets, appeared_typeinfo_addresses = load_buckets(prefix) dump, appeared_addresses = load_dump(dump_path, buckets) delayed_static_symbols = DelayedStaticSymbols(prefix, keep) - symbols = load_and_update_symbol_cache( - prefix, appeared_addresses, delayed_static_symbols) - return buckets, dump, appeared_addresses, delayed_static_symbols, symbols + symbols, typeinfo_symbols = load_and_update_symbol_cache( + prefix, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols) + for bucket in buckets: + if buckets[bucket].typeinfo != None: + buckets[bucket].typeinfo_symbol = typeinfo_symbols[ + buckets[bucket].typeinfo] + + return (buckets, dump, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols, symbols, typeinfo_symbols) def do_stacktrace(sys_argv): @@ -901,8 +976,9 @@ def do_stacktrace(sys_argv): dump_path = args[1] - buckets, dump, appeared_addresses, delayed_static_symbols, symbols = ( - load_basic_files_with_single_dump(dump_path, options.keep)) + (buckets, dump, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols, symbols, typeinfo_symbols) = ( + load_basic_files_with_single_dump(dump_path, options.keep)) dump.print_stacktrace(buckets, symbols) @@ -923,8 +999,10 @@ def do_csv(sys_argv): dump_path = args[1] - buckets, dumps, appeared_addresses, delayed_static_symbols, symbols = ( - load_basic_files_with_multiple_dumps(dump_path, options.keep)) + (buckets, dumps, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols, symbols, typeinfo_symbols) = ( + load_basic_files_with_multiple_dumps(dump_path, options.keep)) + policies = load_policies(options.policy) max_components = 0 @@ -972,8 +1050,9 @@ def do_json(sys_argv): dump_path = args[1] - buckets, dumps, appeared_addresses, delayed_static_symbols, symbols = ( - load_basic_files_with_multiple_dumps(dump_path, options.keep)) + (buckets, dumps, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols, symbols, typeinfo_symbols) = ( + load_basic_files_with_multiple_dumps(dump_path, options.keep)) policies = load_policies(options.policy) json_base = { @@ -1020,8 +1099,9 @@ def do_list(sys_argv): dump_path = args[1] - buckets, dumps, appeared_addresses, delayed_static_symbols, symbols = ( - load_basic_files_with_multiple_dumps(dump_path, options.keep)) + (buckets, dumps, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols, symbols, typeinfo_symbols) = ( + load_basic_files_with_multiple_dumps(dump_path, options.keep)) policies = load_policies(options.policy) for policy in sorted(policies): @@ -1059,13 +1139,15 @@ def do_expand(sys_argv): component_name = args[3] depth = args[4] - buckets, dump, appeared_addresses, delayed_static_symbols, symbols = ( - load_basic_files_with_single_dump(dump_path, options.keep)) + (buckets, dump, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols, symbols, typeinfo_symbols) = ( + load_basic_files_with_single_dump(dump_path, options.keep)) policies = load_policies(target_policy) rule_list = policies[target_policy].rules - dump.expand(rule_list, buckets, component_name, int(depth), symbols) + dump.expand(rule_list, buckets, component_name, int(depth), symbols, + typeinfo_symbols) return 0 @@ -1086,8 +1168,9 @@ def do_pprof(sys_argv): target_policy = args[2] component = options.component - buckets, dump, appeared_addresses, delayed_static_symbols, symbols = ( - load_basic_files_with_single_dump(dump_path, options.keep)) + (buckets, dump, appeared_addresses, appeared_typeinfo_addresses, + delayed_static_symbols, symbols, typeinfo_symbols) = ( + load_basic_files_with_single_dump(dump_path, options.keep)) policies = load_policies(target_policy) rule_list = policies[target_policy].rules diff --git a/tools/deep_memory_profiler/policies.json b/tools/deep_memory_profiler/policies.json index 5535056..b6441cf 100644 --- a/tools/deep_memory_profiler/policies.json +++ b/tools/deep_memory_profiler/policies.json @@ -10,5 +10,9 @@ "l2": { "file": "policy.l2.json", "format": "json" + }, + "t0": { + "file": "policy.t0.json", + "format": "json" } }
\ No newline at end of file diff --git a/tools/deep_memory_profiler/policy.l0.json b/tools/deep_memory_profiler/policy.l0.json index 4c24c15..9d0373b 100644 --- a/tools/deep_memory_profiler/policy.l0.json +++ b/tools/deep_memory_profiler/policy.l0.json @@ -2,7 +2,7 @@ "components": [ "second", "mmap-profiler", - "mmap-allocated-type", + "mmap-type-profiler", "mmap-tcmalloc", "FROM_HERE_FOR_TOTAL", "mustbezero", @@ -46,8 +46,8 @@ "allocator": "mmap" }, { - "name": "mmap-allocated-type", - "stacktrace": ".*(AllocatedTypeMalloc).*", + "name": "mmap-type-profiler", + "stacktrace": ".*(TypeProfilerMalloc).*", "allocator": "mmap" }, { diff --git a/tools/deep_memory_profiler/policy.l1.json b/tools/deep_memory_profiler/policy.l1.json index f300b90..104a1be 100644 --- a/tools/deep_memory_profiler/policy.l1.json +++ b/tools/deep_memory_profiler/policy.l1.json @@ -2,7 +2,7 @@ "components": [ "second", "mmap-profiler", - "mmap-allocated-type", + "mmap-type-profiler", "mmap-tcmalloc", "FROM_HERE_FOR_TOTAL", "mustbezero", @@ -53,8 +53,8 @@ "allocator": "mmap" }, { - "name": "mmap-allocated-type", - "stacktrace": ".*(AllocatedTypeMalloc).*", + "name": "mmap-type-profiler", + "stacktrace": ".*(TypeProfilerMalloc).*", "allocator": "mmap" }, { diff --git a/tools/deep_memory_profiler/policy.l2.json b/tools/deep_memory_profiler/policy.l2.json index 6ffa60f..e4e4917 100644 --- a/tools/deep_memory_profiler/policy.l2.json +++ b/tools/deep_memory_profiler/policy.l2.json @@ -2,7 +2,7 @@ "components": [ "second", "mmap-profiler", - "mmap-allocated-type", + "mmap-type-profiler", "mmap-tcmalloc", "FROM_HERE_FOR_TOTAL", "mustbezero", @@ -76,8 +76,8 @@ "allocator": "mmap" }, { - "name": "mmap-allocated-type", - "stacktrace": ".*(AllocatedTypeMalloc).*", + "name": "mmap-type-profiler", + "stacktrace": ".*(TypeProfilerMalloc).*", "allocator": "mmap" }, { diff --git a/tools/deep_memory_profiler/policy.t0.json b/tools/deep_memory_profiler/policy.t0.json new file mode 100644 index 0000000..55ceba4 --- /dev/null +++ b/tools/deep_memory_profiler/policy.t0.json @@ -0,0 +1,198 @@ +{ + "components": [ + "second", + "mmap-profiler", + "mmap-type-profiler", + "mmap-tcmalloc", + "FROM_HERE_FOR_TOTAL", + "mustbezero", + "nonprofiled-absent", + "nonprofiled-anonymous", + "nonprofiled-file-exec", + "nonprofiled-file-nonexec", + "nonprofiled-stack", + "nonprofiled-other", + "no-bucket", + "mmap-v8", + "mmap-catch-all", + "tc-std-string", + "tc-WTF-String", + "tc-no-typeinfo-StringImpl", + "tc-Skia", + "tc-WebCore-Style", + "tc-no-typeinfo-other", + "tc-other", + "tc-unused", + "UNTIL_HERE_FOR_TOTAL", + "total-exclude-profiler", + "total", + "anonymous", + "file-exec", + "file-nonexec", + "stack", + "other", + "mmap-total-log", + "mmap-no-log", + "mmap-total-record", + "other-total-log", + "tc-total-log", + "tc-no-log", + "tc-total-record", + "tc-total" + ], + "rules": [ + { + "name": "second", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "mmap-profiler", + "stacktrace": ".*(ProfilerMalloc|MemoryRegionMap::).*", + "allocator": "mmap" + }, + { + "name": "mmap-type-profiler", + "stacktrace": ".*(TypeProfilerMalloc).*", + "allocator": "mmap" + }, + { + "name": "mmap-tcmalloc", + "stacktrace": ".*(DoAllocWithArena|SbrkSysAllocator::Alloc|MmapSysAllocator::Alloc|LowLevelAlloc::Alloc|LowLevelAlloc::AllocWithArena).*", + "allocator": "mmap" + }, + { + "name": "FROM_HERE_FOR_TOTAL", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "mustbezero", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "nonprofiled-absent", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "nonprofiled-anonymous", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "nonprofiled-file-exec", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "nonprofiled-file-nonexec", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "nonprofiled-stack", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "nonprofiled-other", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "mmap-v8", + "stacktrace": ".*v8::.*", + "allocator": "mmap" + }, + { + "name": "mmap-catch-all", + "stacktrace": ".*", + "allocator": "mmap" + }, + { + "name": "tc-std-string", + "stacktrace": ".*", + "typeinfo": "std::basic_string.*", + "allocator": "malloc" + }, + { + "name": "tc-WTF-String", + "stacktrace": ".*", + "typeinfo": "WTF::String.*", + "allocator": "malloc" + }, + { + "name": "tc-no-typeinfo-StringImpl", + "stacktrace": ".*WTF::StringImpl::getData16SlowCase.*", + "typeinfo": "no typeinfo", + "allocator": "malloc" + }, + { + "name": "tc-Skia", + "stacktrace": ".*", + "typeinfo": "(skia::|SkGlyph).*", + "allocator": "malloc" + }, + { + "name": "tc-WebCore-Style", + "stacktrace": ".*", + "typeinfo": "WebCore::Style.*", + "allocator": "malloc" + }, + { + "name": "tc-no-typeinfo-other", + "stacktrace": ".*", + "typeinfo": "no typeinfo", + "allocator": "malloc" + }, + { + "name": "tc-other", + "stacktrace": ".*", + "typeinfo": ".*", + "allocator": "malloc" + }, + { + "name": "UNTIL_HERE_FOR_TOTAL", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "total-exclude-profiler", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "total", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "anonymous", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "file-exec", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "file-nonexec", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "stack", + "stacktrace": "optional", + "allocator": "optional" + }, + { + "name": "other", + "stacktrace": "optional", + "allocator": "optional" + } + ], + "version": "POLICY_DEEP_4" +}
\ No newline at end of file |