diff options
Diffstat (limited to 'third_party/tcmalloc/vendor/src/heap-checker.cc')
| -rw-r--r-- | third_party/tcmalloc/vendor/src/heap-checker.cc | 2369 |
1 files changed, 2369 insertions, 0 deletions
diff --git a/third_party/tcmalloc/vendor/src/heap-checker.cc b/third_party/tcmalloc/vendor/src/heap-checker.cc new file mode 100644 index 0000000..59288e6 --- /dev/null +++ b/third_party/tcmalloc/vendor/src/heap-checker.cc @@ -0,0 +1,2369 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// All Rights Reserved. +// +// Author: Maxim Lifantsev +// + +#include "config.h" + +#include <fcntl.h> // for O_RDONLY (we use syscall to do actual reads) +#include <string.h> +#include <errno.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_MMAP +#include <sys/mman.h> +#endif +#ifdef HAVE_PTHREAD +#include <pthread.h> +#endif +#include <sys/stat.h> +#include <sys/types.h> +#include <time.h> +#include <assert.h> + +#ifdef HAVE_LINUX_PTRACE_H +#include <linux/ptrace.h> +#endif +#ifdef HAVE_SYS_SYSCALL_H +#include <sys/syscall.h> +#endif +#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__) +#include <wtypes.h> +#include <winbase.h> +#undef ERROR // windows defines these as macros, which can cause trouble +#undef max +#undef min +#endif + +#include <string> +#include <vector> +#include <map> +#include <set> +#include <algorithm> +#include <functional> + +#include <google/heap-checker.h> + +#include "base/basictypes.h" +#include "base/googleinit.h" +#include "base/logging.h" +#include <google/stacktrace.h> +#include "base/commandlineflags.h" +#include "base/elfcore.h" // for i386_regs +#include "base/thread_lister.h" +#include "heap-profile-table.h" +#include "base/low_level_alloc.h" +#include "malloc_hook-inl.h" +#include <google/malloc_hook.h> +#include <google/malloc_extension.h> +#include "maybe_threads.h" +#include "memory_region_map.h" +#include "base/spinlock.h" +#include "base/sysinfo.h" +#include "base/stl_allocator.h" + +using std::string; +using std::basic_string; +using std::pair; +using std::map; +using std::set; +using std::vector; +using std::swap; +using std::make_pair; +using std::min; +using std::max; +using std::less; +using std::char_traits; + +// This is the default if you don't link in -lprofiler +extern "C" { +ATTRIBUTE_WEAK PERFTOOLS_DLL_DECL bool ProfilingIsEnabledForAllThreads(); +bool ProfilingIsEnabledForAllThreads() { return false; } +} + +//---------------------------------------------------------------------- +// Flags that control heap-checking +//---------------------------------------------------------------------- + +DEFINE_string(heap_check, + EnvToString("HEAPCHECK", ""), + "The heap leak checking to be done over the whole executable: " + "\"minimal\", \"normal\", \"strict\", " + "\"draconian\", \"as-is\", and \"local\" " + " or the empty string are the supported choices. " + "(See HeapLeakChecker::InternalInitStart for details.)"); + +DEFINE_bool(heap_check_report, true, "Obsolete"); + +DEFINE_bool(heap_check_before_constructors, + true, + "deprecated; pretty much always true now"); + +DEFINE_bool(heap_check_after_destructors, + EnvToBool("HEAP_CHECK_AFTER_DESTRUCTORS", false), + "If overall heap check is to end after global destructors " + "or right after all REGISTER_HEAPCHECK_CLEANUP's"); + +DEFINE_bool(heap_check_strict_check, true, "Obsolete"); + +DEFINE_bool(heap_check_ignore_global_live, + EnvToBool("HEAP_CHECK_IGNORE_GLOBAL_LIVE", true), + "If overall heap check is to ignore heap objects reachable " + "from the global data"); + +DEFINE_bool(heap_check_identify_leaks, + EnvToBool("HEAP_CHECK_IDENTIFY_LEAKS", false), + "If heap check should generate the addresses of the leaked " + "objects in the memory leak profiles. This may be useful " + "in tracking down leaks where only a small fraction of " + "objects allocated at the same stack trace are leaked."); + +DEFINE_bool(heap_check_ignore_thread_live, + EnvToBool("HEAP_CHECK_IGNORE_THREAD_LIVE", true), + "If set to true, objects reachable from thread stacks " + "and registers are not reported as leaks"); + +DEFINE_bool(heap_check_test_pointer_alignment, + EnvToBool("HEAP_CHECK_TEST_POINTER_ALIGNMENT", false), + "Set to true to check if the found leak can be due to " + "use of unaligned pointers"); + +// A reasonable default to handle pointers inside of typical class objects: +// Too low and we won't be able to traverse pointers to normally-used +// nested objects and base parts of multiple-inherited objects. +// Too high and it will both slow down leak checking (FindInsideAlloc +// in HaveOnHeapLocked will get slower when there are large on-heap objects) +// and make it probabilistically more likely to miss leaks +// of large-sized objects. +static const int64 kHeapCheckMaxPointerOffset = 1024; +DEFINE_int64(heap_check_max_pointer_offset, + EnvToInt("HEAP_CHECK_MAX_POINTER_OFFSET", + kHeapCheckMaxPointerOffset), + "Largest pointer offset for which we traverse " + "pointers going inside of heap allocated objects. " + "Set to -1 to use the actual largest heap object size."); + +DEFINE_bool(heap_check_run_under_gdb, + EnvToBool("HEAP_CHECK_RUN_UNDER_GDB", false), + "If false, turns off heap-checking library when running under gdb " + "(normally, set to 'true' only when debugging the heap-checker)"); + +DEFINE_int32(heap_check_delay_seconds, 0, + "Number of seconds to delay on-exit heap checking." + " If you set this flag," + " you may also want to set exit_timeout_seconds in order to" + " avoid exit timeouts.\n" + "NOTE: This flag is to be used only to help diagnose issues" + " where it is suspected that the heap checker is reporting" + " false leaks that will disappear if the heap checker delays" + " its checks. Report any such issues to the heap-checker" + " maintainer(s)."); + +//---------------------------------------------------------------------- + +DEFINE_string(heap_profile_pprof, + EnvToString("PPROF_PATH", "pprof"), + "OBSOLETE; not used"); + +DEFINE_string(heap_check_dump_directory, + EnvToString("HEAP_CHECK_DUMP_DIRECTORY", "/tmp"), + "Directory to put heap-checker leak dump information"); + + +//---------------------------------------------------------------------- +// HeapLeakChecker global data +//---------------------------------------------------------------------- + +// Global lock for all the global data of this module. +static SpinLock heap_checker_lock(SpinLock::LINKER_INITIALIZED); + +//---------------------------------------------------------------------- + +// Heap profile prefix for leak checking profiles. +// Gets assigned once when leak checking is turned on, then never modified. +static const string* profile_name_prefix = NULL; + +// Whole-program heap leak checker. +// Gets assigned once when leak checking is turned on, +// then main_heap_checker is never deleted. +static HeapLeakChecker* main_heap_checker = NULL; + +// Whether we will use main_heap_checker to do a check at program exit +// automatically. In any case user can ask for more checks on main_heap_checker +// via GlobalChecker(). +static bool do_main_heap_check = false; + +// The heap profile we use to collect info about the heap. +// This is created in HeapLeakChecker::BeforeConstructorsLocked +// together with setting heap_checker_on (below) to true +// and registering our new/delete malloc hooks; +// similarly all are unset in HeapLeakChecker::TurnItselfOffLocked. +static HeapProfileTable* heap_profile = NULL; + +// If we are doing (or going to do) any kind of heap-checking. +static bool heap_checker_on = false; + +// pid of the process that does whole-program heap leak checking +static pid_t heap_checker_pid = 0; + +// If we did heap profiling during global constructors execution +static bool constructor_heap_profiling = false; + +// RAW_VLOG level we dump key INFO messages at. If you want to turn +// off these messages, set the environment variable PERFTOOLS_VERBOSE=-1. +static const int heap_checker_info_level = 0; + +//---------------------------------------------------------------------- + +// Alignment at which all pointers in memory are supposed to be located; +// use 1 if any alignment is ok. +// heap_check_test_pointer_alignment flag guides if we try the value of 1. +// The larger it can be, the lesser is the chance of missing real leaks. +static const size_t kPointerSourceAlignment = sizeof(void*); + +// Cancel our InitialMallocHook_* if present. +static void CancelInitialMallocHooks(); // defined below + +//---------------------------------------------------------------------- +// HeapLeakChecker's own memory allocator that is +// independent of the normal program allocator. +//---------------------------------------------------------------------- + +// Wrapper of LowLevelAlloc for STL_Allocator and direct use. +// We always access this class under held heap_checker_lock, +// this allows us to in particular protect the period when threads are stopped +// at random spots with ListAllProcessThreads by heap_checker_lock, +// w/o worrying about the lock in LowLevelAlloc::Arena. +// We rely on the fact that we use an own arena with an own lock here. +class HeapLeakChecker::Allocator { + public: + static void Init() { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + RAW_DCHECK(arena_ == NULL, ""); + arena_ = LowLevelAlloc::NewArena(0, LowLevelAlloc::DefaultArena()); + } + static void Shutdown() { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + if (!LowLevelAlloc::DeleteArena(arena_) || alloc_count_ != 0) { + RAW_LOG(FATAL, "Internal heap checker leak of %d objects", alloc_count_); + } + } + static int alloc_count() { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + return alloc_count_; + } + static void* Allocate(size_t n) { + RAW_DCHECK(arena_ && heap_checker_lock.IsHeld(), ""); + void* p = LowLevelAlloc::AllocWithArena(n, arena_); + if (p) alloc_count_ += 1; + return p; + } + static void Free(void* p) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + if (p) alloc_count_ -= 1; + LowLevelAlloc::Free(p); + } + // destruct, free, and make *p to be NULL + template<typename T> static void DeleteAndNull(T** p) { + (*p)->~T(); + Free(*p); + *p = NULL; + } + template<typename T> static void DeleteAndNullIfNot(T** p) { + if (*p != NULL) DeleteAndNull(p); + } + private: + static LowLevelAlloc::Arena* arena_; + static int alloc_count_; +}; + +LowLevelAlloc::Arena* HeapLeakChecker::Allocator::arena_ = NULL; +int HeapLeakChecker::Allocator::alloc_count_ = 0; + +//---------------------------------------------------------------------- +// HeapLeakChecker live object tracking components +//---------------------------------------------------------------------- + +// Cases of live object placement we distinguish +enum ObjectPlacement { + MUST_BE_ON_HEAP, // Must point to a live object of the matching size in the + // heap_profile map of the heap when we get to it + IGNORED_ON_HEAP, // Is a live (ignored) object on heap + MAYBE_LIVE, // Is a piece of writable memory from /proc/self/maps + IN_GLOBAL_DATA, // Is part of global data region of the executable + THREAD_DATA, // Part of a thread stack and a thread descriptor with TLS + THREAD_REGISTERS, // Values in registers of some thread +}; + +// Information about an allocated object +struct AllocObject { + const void* ptr; // the object + uintptr_t size; // its size + ObjectPlacement place; // where ptr points to + + AllocObject(const void* p, size_t s, ObjectPlacement l) + : ptr(p), size(s), place(l) { } +}; + +// All objects (memory ranges) ignored via HeapLeakChecker::IgnoreObject +// Key is the object's address; value is its size. +typedef map<uintptr_t, size_t, less<uintptr_t>, + STL_Allocator<pair<const uintptr_t, size_t>, + HeapLeakChecker::Allocator> + > IgnoredObjectsMap; +static IgnoredObjectsMap* ignored_objects = NULL; + +// All objects (memory ranges) that we consider to be the sources of pointers +// to live (not leaked) objects. +// At different times this holds (what can be reached from) global data regions +// and the objects we've been told to ignore. +// For any AllocObject::ptr "live_objects" is supposed to contain at most one +// record at any time. We maintain this by checking with the heap_profile map +// of the heap and removing the live heap objects we've handled from it. +// This vector is maintained as a stack and the frontier of reachable +// live heap objects in our flood traversal of them. +typedef vector<AllocObject, + STL_Allocator<AllocObject, HeapLeakChecker::Allocator> + > LiveObjectsStack; +static LiveObjectsStack* live_objects = NULL; + +// A special string type that uses my allocator +typedef basic_string<char, char_traits<char>, + STL_Allocator<char, HeapLeakChecker::Allocator> + > HCL_string; + +// A placeholder to fill-in the starting values for live_objects +// for each library so we can keep the library-name association for logging. +typedef map<HCL_string, LiveObjectsStack, less<HCL_string>, + STL_Allocator<pair<const HCL_string, LiveObjectsStack>, + HeapLeakChecker::Allocator> + > LibraryLiveObjectsStacks; +static LibraryLiveObjectsStacks* library_live_objects = NULL; + +// Value stored in the map of disabled address ranges; +// its key is the end of the address range. +// We'll ignore allocations with a return address in a disabled range +// if the address occurs at 'max_depth' or less in the stack trace. +struct HeapLeakChecker::RangeValue { + uintptr_t start_address; // the start of the range + int max_depth; // the maximal stack depth to disable at +}; +typedef map<uintptr_t, HeapLeakChecker::RangeValue, less<uintptr_t>, + STL_Allocator<pair<const uintptr_t, HeapLeakChecker::RangeValue>, + HeapLeakChecker::Allocator> + > DisabledRangeMap; +// The disabled program counter address ranges for profile dumping +// that are registered with HeapLeakChecker::DisableChecksFromToLocked. +static DisabledRangeMap* disabled_ranges = NULL; + +// Set of stack tops. +// These are used to consider live only appropriate chunks of the memory areas +// that are used for stacks (and maybe thread-specific data as well) +// so that we do not treat pointers from outdated stack frames as live. +typedef set<uintptr_t, less<uintptr_t>, + STL_Allocator<uintptr_t, HeapLeakChecker::Allocator> + > StackTopSet; +static StackTopSet* stack_tops = NULL; + +// A map of ranges of code addresses for the system libraries +// that can mmap/mremap/sbrk-allocate memory regions for stacks +// and thread-local storage that we want to consider as live global data. +// Maps from the end address to the start address. +typedef map<uintptr_t, uintptr_t, less<uintptr_t>, + STL_Allocator<pair<const uintptr_t, uintptr_t>, + HeapLeakChecker::Allocator> + > GlobalRegionCallerRangeMap; +static GlobalRegionCallerRangeMap* global_region_caller_ranges = NULL; + +// TODO(maxim): make our big data structs into own modules + +// Disabler is implemented by keeping track of a per-thread count +// of active Disabler objects. Any objects allocated while the +// count > 0 are not reported. + +#ifdef HAVE_TLS + +static __thread int thread_disable_counter +// The "inital exec" model is faster than the default TLS model, at +// the cost you can't dlopen this library. But dlopen on heap-checker +// doesn't work anyway -- it must run before main -- so this is a good +// trade-off. +# ifdef HAVE___ATTRIBUTE__ + __attribute__ ((tls_model ("initial-exec"))) +# endif + ; +inline int get_thread_disable_counter() { + return thread_disable_counter; +} +inline void set_thread_disable_counter(int value) { + thread_disable_counter = value; +} + +#else // #ifdef HAVE_TLS + +static pthread_key_t thread_disable_counter_key; +static int main_thread_counter; // storage for use before main() +static bool use_main_thread_counter = true; + +// TODO(csilvers): this is called from NewHook, in the middle of malloc(). +// If perftools_pthread_getspecific calls malloc, that will lead to an +// infinite loop. I don't know how to fix that, so I hope it never happens! +inline int get_thread_disable_counter() { + if (use_main_thread_counter) // means we're running really early + return main_thread_counter; + void* p = perftools_pthread_getspecific(thread_disable_counter_key); + return (intptr_t)p; // kinda evil: store the counter directly in the void* +} + +inline void set_thread_disable_counter(int value) { + if (use_main_thread_counter) { // means we're running really early + main_thread_counter = value; + return; + } + intptr_t pointer_sized_value = value; + // kinda evil: store the counter directly in the void* + void* p = (void*)pointer_sized_value; + // NOTE: this may call malloc, which will call NewHook which will call + // get_thread_disable_counter() which will call pthread_getspecific(). I + // don't know if anything bad can happen if we call getspecific() in the + // middle of a setspecific() call. It seems to work ok in practice... + perftools_pthread_setspecific(thread_disable_counter_key, p); +} + +// The idea here is that this initializer will run pretty late: after +// pthreads have been totally set up. At this point we can call +// pthreads routines, so we set those up. +class InitThreadDisableCounter { + public: + InitThreadDisableCounter() { + perftools_pthread_key_create(&thread_disable_counter_key, NULL); + // Set up the main thread's value, which we have a special variable for. + void* p = (void*)main_thread_counter; // store the counter directly + perftools_pthread_setspecific(thread_disable_counter_key, p); + use_main_thread_counter = false; + } +}; +InitThreadDisableCounter init_thread_disable_counter; + +#endif // #ifdef HAVE_TLS + +HeapLeakChecker::Disabler::Disabler() { + // It is faster to unconditionally increment the thread-local + // counter than to check whether or not heap-checking is on + // in a thread-safe manner. + int counter = get_thread_disable_counter(); + set_thread_disable_counter(counter + 1); + RAW_VLOG(1, "Increasing thread disable counter to %d", counter + 1); +} + +HeapLeakChecker::Disabler::~Disabler() { + int counter = get_thread_disable_counter(); + RAW_DCHECK(counter > 0, ""); + if (counter > 0) { + set_thread_disable_counter(counter - 1); + RAW_VLOG(1, "Decreasing thread disable counter to %d", counter); + } else { + RAW_VLOG(0, "Thread disable counter underflow : %d", counter); + } +} + +//---------------------------------------------------------------------- + +// The size of the largest heap object allocated so far. +static size_t max_heap_object_size = 0; +// The possible range of addresses that can point +// into one of the elements of heap_objects. +static uintptr_t min_heap_address = uintptr_t(-1LL); +static uintptr_t max_heap_address = 0; + +//---------------------------------------------------------------------- + +// Simple casting helpers for uintptr_t and void*: +template<typename T> +inline static const void* AsPtr(T addr) { + return reinterpret_cast<void*>(addr); +} +inline static uintptr_t AsInt(const void* ptr) { + return reinterpret_cast<uintptr_t>(ptr); +} + +//---------------------------------------------------------------------- + +// Our hooks for MallocHook +static void NewHook(const void* ptr, size_t size) { + if (ptr != NULL) { + const int counter = get_thread_disable_counter(); + const bool ignore = (counter > 0); + RAW_VLOG(7, "Recording Alloc: %p of %"PRIuS "; %d", ptr, size, + int(counter)); + { SpinLockHolder l(&heap_checker_lock); + if (size > max_heap_object_size) max_heap_object_size = size; + uintptr_t addr = AsInt(ptr); + if (addr < min_heap_address) min_heap_address = addr; + addr += size; + if (addr > max_heap_address) max_heap_address = addr; + if (heap_checker_on) { + heap_profile->RecordAlloc(ptr, size, 0); + if (ignore) { + heap_profile->MarkAsIgnored(ptr); + } + } + } + RAW_VLOG(8, "Alloc Recorded: %p of %"PRIuS"", ptr, size); + } +} + +static void DeleteHook(const void* ptr) { + if (ptr != NULL) { + RAW_VLOG(7, "Recording Free %p", ptr); + { SpinLockHolder l(&heap_checker_lock); + if (heap_checker_on) heap_profile->RecordFree(ptr); + } + RAW_VLOG(8, "Free Recorded: %p", ptr); + } +} + +//---------------------------------------------------------------------- + +enum StackDirection { + GROWS_TOWARDS_HIGH_ADDRESSES, + GROWS_TOWARDS_LOW_ADDRESSES, + UNKNOWN_DIRECTION +}; + +// Determine which way the stack grows: + +static StackDirection ATTRIBUTE_NOINLINE GetStackDirection( + const uintptr_t *const ptr) { + uintptr_t x; + if (&x < ptr) + return GROWS_TOWARDS_LOW_ADDRESSES; + if (ptr < &x) + return GROWS_TOWARDS_HIGH_ADDRESSES; + + RAW_CHECK(0, ""); // Couldn't determine the stack direction. + + return UNKNOWN_DIRECTION; +} + +// Direction of stack growth (will initialize via GetStackDirection()) +static StackDirection stack_direction = UNKNOWN_DIRECTION; + +// This routine is called for every thread stack we know about to register it. +static void RegisterStackLocked(const void* top_ptr) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + RAW_DCHECK(MemoryRegionMap::LockIsHeld(), ""); + RAW_VLOG(1, "Thread stack at %p", top_ptr); + uintptr_t top = AsInt(top_ptr); + stack_tops->insert(top); // add for later use + + // make sure stack_direction is initialized + if (stack_direction == UNKNOWN_DIRECTION) { + stack_direction = GetStackDirection(&top); + } + + // Find memory region with this stack + MemoryRegionMap::Region region; + if (MemoryRegionMap::FindAndMarkStackRegion(top, ®ion)) { + // Make the proper portion of the stack live: + if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) { + RAW_VLOG(2, "Live stack at %p of %"PRIuPTR" bytes", + top_ptr, region.end_addr - top); + live_objects->push_back(AllocObject(top_ptr, region.end_addr - top, + THREAD_DATA)); + } else { // GROWS_TOWARDS_HIGH_ADDRESSES + RAW_VLOG(2, "Live stack at %p of %"PRIuPTR" bytes", + AsPtr(region.start_addr), + top - region.start_addr); + live_objects->push_back(AllocObject(AsPtr(region.start_addr), + top - region.start_addr, + THREAD_DATA)); + } + // not in MemoryRegionMap, look in library_live_objects: + } else if (FLAGS_heap_check_ignore_global_live) { + for (LibraryLiveObjectsStacks::iterator lib = library_live_objects->begin(); + lib != library_live_objects->end(); ++lib) { + for (LiveObjectsStack::iterator span = lib->second.begin(); + span != lib->second.end(); ++span) { + uintptr_t start = AsInt(span->ptr); + uintptr_t end = start + span->size; + if (start <= top && top < end) { + RAW_VLOG(2, "Stack at %p is inside /proc/self/maps chunk %p..%p", + top_ptr, AsPtr(start), AsPtr(end)); + // Shrink start..end region by chopping away the memory regions in + // MemoryRegionMap that land in it to undo merging of regions + // in /proc/self/maps, so that we correctly identify what portion + // of start..end is actually the stack region. + uintptr_t stack_start = start; + uintptr_t stack_end = end; + // can optimize-away this loop, but it does not run often + RAW_DCHECK(MemoryRegionMap::LockIsHeld(), ""); + for (MemoryRegionMap::RegionIterator r = + MemoryRegionMap::BeginRegionLocked(); + r != MemoryRegionMap::EndRegionLocked(); ++r) { + if (top < r->start_addr && r->start_addr < stack_end) { + stack_end = r->start_addr; + } + if (stack_start < r->end_addr && r->end_addr <= top) { + stack_start = r->end_addr; + } + } + if (stack_start != start || stack_end != end) { + RAW_VLOG(2, "Stack at %p is actually inside memory chunk %p..%p", + top_ptr, AsPtr(stack_start), AsPtr(stack_end)); + } + // Make the proper portion of the stack live: + if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) { + RAW_VLOG(2, "Live stack at %p of %"PRIuPTR" bytes", + top_ptr, stack_end - top); + live_objects->push_back( + AllocObject(top_ptr, stack_end - top, THREAD_DATA)); + } else { // GROWS_TOWARDS_HIGH_ADDRESSES + RAW_VLOG(2, "Live stack at %p of %"PRIuPTR" bytes", + AsPtr(stack_start), top - stack_start); + live_objects->push_back( + AllocObject(AsPtr(stack_start), top - stack_start, THREAD_DATA)); + } + lib->second.erase(span); // kill the rest of the region + // Put the non-stack part(s) of the region back: + if (stack_start != start) { + lib->second.push_back(AllocObject(AsPtr(start), stack_start - start, + MAYBE_LIVE)); + } + if (stack_end != end) { + lib->second.push_back(AllocObject(AsPtr(stack_end), end - stack_end, + MAYBE_LIVE)); + } + return; + } + } + } + RAW_LOG(ERROR, "Memory region for stack at %p not found. " + "Will likely report false leak positives.", top_ptr); + } +} + +// Iterator for heap allocation map data to make ignored objects "live" +// (i.e., treated as roots for the mark-and-sweep phase) +static void MakeIgnoredObjectsLiveCallbackLocked( + const void* ptr, const HeapProfileTable::AllocInfo& info) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + if (info.ignored) { + live_objects->push_back(AllocObject(ptr, info.object_size, + MUST_BE_ON_HEAP)); + } +} + +// Iterator for heap allocation map data to make objects allocated from +// disabled regions of code to be live. +static void MakeDisabledLiveCallbackLocked( + const void* ptr, const HeapProfileTable::AllocInfo& info) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + bool stack_disable = false; + bool range_disable = false; + for (int depth = 0; depth < info.stack_depth; depth++) { + uintptr_t addr = AsInt(info.call_stack[depth]); + if (disabled_ranges) { + DisabledRangeMap::const_iterator iter + = disabled_ranges->upper_bound(addr); + if (iter != disabled_ranges->end()) { + RAW_DCHECK(iter->first > addr, ""); + if (iter->second.start_address < addr && + iter->second.max_depth > depth) { + range_disable = true; // in range; dropping + break; + } + } + } + } + if (stack_disable || range_disable) { + uintptr_t start_address = AsInt(ptr); + uintptr_t end_address = start_address + info.object_size; + StackTopSet::const_iterator iter + = stack_tops->lower_bound(start_address); + if (iter != stack_tops->end()) { + RAW_DCHECK(*iter >= start_address, ""); + if (*iter < end_address) { + // We do not disable (treat as live) whole allocated regions + // if they are used to hold thread call stacks + // (i.e. when we find a stack inside). + // The reason is that we'll treat as live the currently used + // stack portions anyway (see RegisterStackLocked), + // and the rest of the region where the stack lives can well + // contain outdated stack variables which are not live anymore, + // hence should not be treated as such. + RAW_VLOG(2, "Not %s-disabling %"PRIuS" bytes at %p" + ": have stack inside: %p", + (stack_disable ? "stack" : "range"), + info.object_size, ptr, AsPtr(*iter)); + return; + } + } + RAW_VLOG(2, "%s-disabling %"PRIuS" bytes at %p", + (stack_disable ? "Stack" : "Range"), info.object_size, ptr); + live_objects->push_back(AllocObject(ptr, info.object_size, + MUST_BE_ON_HEAP)); + } +} + +// This function takes some fields from a /proc/self/maps line: +// +// start_address start address of a memory region. +// end_address end address of a memory region +// permissions rwx + private/shared bit +// filename filename of the mapped file +// +// If the region is not writeable, then it cannot have any heap +// pointers in it, otherwise we record it as a candidate live region +// to get filtered later. +static void RecordGlobalDataLocked(uintptr_t start_address, + uintptr_t end_address, + const char* permissions, + const char* filename) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + // Ignore non-writeable regions. + if (strchr(permissions, 'w') == NULL) return; + if (filename == NULL || *filename == '\0') filename = "UNNAMED"; + RAW_VLOG(2, "Looking into %s: 0x%" PRIxPTR "..0x%" PRIxPTR, + filename, start_address, end_address); + (*library_live_objects)[filename]. + push_back(AllocObject(AsPtr(start_address), + end_address - start_address, + MAYBE_LIVE)); +} + +// See if 'library' from /proc/self/maps has base name 'library_base' +// i.e. contains it and has '.' or '-' after it. +static bool IsLibraryNamed(const char* library, const char* library_base) { + const char* p = strstr(library, library_base); + size_t sz = strlen(library_base); + return p != NULL && (p[sz] == '.' || p[sz] == '-'); +} + +// static +void HeapLeakChecker::DisableLibraryAllocsLocked(const char* library, + uintptr_t start_address, + uintptr_t end_address) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + int depth = 0; + // TODO(maxim): maybe this should be extended to also use objdump + // and pick the text portion of the library more precisely. + if (IsLibraryNamed(library, "/libpthread") || + // libpthread has a lot of small "system" leaks we don't care about. + // In particular it allocates memory to store data supplied via + // pthread_setspecific (which can be the only pointer to a heap object). + IsLibraryNamed(library, "/libdl") || + // library loaders leak some "system" heap that we don't care about + IsLibraryNamed(library, "/libcrypto") + // Sometimes libcrypto of OpenSSH is compiled with -fomit-frame-pointer + // (any library can be, of course, but this one often is because speed + // is so important for making crypto usable). We ignore all its + // allocations because we can't see the call stacks. We'd prefer + // to ignore allocations done in files/symbols that match + // "default_malloc_ex|default_realloc_ex" + // but that doesn't work when the end-result binary is stripped. + ) { + depth = 1; // only disable allocation calls directly from the library code + } else if (IsLibraryNamed(library, "/ld") + // library loader leaks some "system" heap + // (e.g. thread-local storage) that we don't care about + ) { + depth = 2; // disable allocation calls directly from the library code + // and at depth 2 from it. + // We need depth 2 here solely because of a libc bug that + // forces us to jump through __memalign_hook and MemalignOverride hoops + // in tcmalloc.cc. + // Those buggy __libc_memalign() calls are in ld-linux.so and happen for + // thread-local storage allocations that we want to ignore here. + // We go with the depth-2 hack as a workaround for this libc bug: + // otherwise we'd need to extend MallocHook interface + // so that correct stack depth adjustment can be propagated from + // the exceptional case of MemalignOverride. + // Using depth 2 here should not mask real leaks because ld-linux.so + // does not call user code. + } + if (depth) { + RAW_VLOG(1, "Disabling allocations from %s at depth %d:", library, depth); + DisableChecksFromToLocked(AsPtr(start_address), AsPtr(end_address), depth); + if (IsLibraryNamed(library, "/libpthread") || + IsLibraryNamed(library, "/libdl") || + IsLibraryNamed(library, "/ld")) { + RAW_VLOG(1, "Global memory regions made by %s will be live data", + library); + if (global_region_caller_ranges == NULL) { + global_region_caller_ranges = + new(Allocator::Allocate(sizeof(GlobalRegionCallerRangeMap))) + GlobalRegionCallerRangeMap; + } + global_region_caller_ranges + ->insert(make_pair(end_address, start_address)); + } + } +} + +// static +HeapLeakChecker::ProcMapsResult HeapLeakChecker::UseProcMapsLocked( + ProcMapsTask proc_maps_task) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + // Need to provide own scratch memory to ProcMapsIterator: + ProcMapsIterator::Buffer buffer; + ProcMapsIterator it(0, &buffer); + if (!it.Valid()) { + int errsv = errno; + RAW_LOG(ERROR, "Could not open /proc/self/maps: errno=%d. " + "Libraries will not be handled correctly.", errsv); + return CANT_OPEN_PROC_MAPS; + } + uint64 start_address, end_address, file_offset; + int64 inode; + char *permissions, *filename; + bool saw_shared_lib = false; + while (it.Next(&start_address, &end_address, &permissions, + &file_offset, &inode, &filename)) { + if (start_address >= end_address) { + // Warn if a line we can be interested in is ill-formed: + if (inode != 0) { + RAW_LOG(ERROR, "Errors reading /proc/self/maps. " + "Some global memory regions will not " + "be handled correctly."); + } + // Silently skip other ill-formed lines: some are possible + // probably due to the interplay of how /proc/self/maps is updated + // while we read it in chunks in ProcMapsIterator and + // do things in this loop. + continue; + } + // Determine if any shared libraries are present. + if (inode != 0 && strstr(filename, "lib") && strstr(filename, ".so")) { + saw_shared_lib = true; + } + switch (proc_maps_task) { + case DISABLE_LIBRARY_ALLOCS: + // All lines starting like + // "401dc000-4030f000 r??p 00132000 03:01 13991972 lib/bin" + // identify a data and code sections of a shared library or our binary + if (inode != 0 && strncmp(permissions, "r-xp", 4) == 0) { + DisableLibraryAllocsLocked(filename, start_address, end_address); + } + break; + case RECORD_GLOBAL_DATA: + RecordGlobalDataLocked(start_address, end_address, + permissions, filename); + break; + default: + RAW_CHECK(0, ""); + } + } + if (!saw_shared_lib) { + RAW_LOG(ERROR, "No shared libs detected. Will likely report false leak " + "positives for statically linked executables."); + return NO_SHARED_LIBS_IN_PROC_MAPS; + } + return PROC_MAPS_USED; +} + +// Total number and size of live objects dropped from the profile; +// (re)initialized in IgnoreAllLiveObjectsLocked. +static int64 live_objects_total; +static int64 live_bytes_total; + +// pid of the thread that is doing the current leak check +// (protected by our lock; IgnoreAllLiveObjectsLocked sets it) +static pid_t self_thread_pid = 0; + +// Status of our thread listing callback execution +// (protected by our lock; used from within IgnoreAllLiveObjectsLocked) +static enum { + CALLBACK_NOT_STARTED, + CALLBACK_STARTED, + CALLBACK_COMPLETED, +} thread_listing_status = CALLBACK_NOT_STARTED; + +// Ideally to avoid deadlocks this function should not result in any libc +// or other function calls that might need to lock a mutex: +// It is called when all threads of a process are stopped +// at arbitrary points thus potentially holding those locks. +// +// In practice we are calling some simple i/o and sprintf-type library functions +// for logging messages, but use only our own LowLevelAlloc::Arena allocator. +// +// This is known to be buggy: the library i/o function calls are able to cause +// deadlocks when they request a lock that a stopped thread happens to hold. +// This issue as far as we know have so far not resulted in any deadlocks +// in practice, so for now we are taking our chance that the deadlocks +// have insignificant frequency. +// +// If such deadlocks become a problem we should make the i/o calls +// into appropriately direct system calls (or eliminate them), +// in particular write() is not safe and vsnprintf() is potentially dangerous +// due to reliance on locale functions (these are called through RAW_LOG +// and in other ways). +// +/*static*/ int HeapLeakChecker::IgnoreLiveThreadsLocked(void* parameter, + int num_threads, + pid_t* thread_pids, + va_list /*ap*/) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + thread_listing_status = CALLBACK_STARTED; + RAW_VLOG(2, "Found %d threads (from pid %d)", num_threads, getpid()); + + if (FLAGS_heap_check_ignore_global_live) { + UseProcMapsLocked(RECORD_GLOBAL_DATA); + } + + // We put the registers from other threads here + // to make pointers stored in them live. + vector<void*, STL_Allocator<void*, Allocator> > thread_registers; + + int failures = 0; + for (int i = 0; i < num_threads; ++i) { + // the leak checking thread itself is handled + // specially via self_thread_stack, not here: + if (thread_pids[i] == self_thread_pid) continue; + RAW_VLOG(2, "Handling thread with pid %d", thread_pids[i]); +#if defined(HAVE_LINUX_PTRACE_H) && defined(HAVE_SYS_SYSCALL_H) && defined(DUMPER) + i386_regs thread_regs; +#define sys_ptrace(r, p, a, d) syscall(SYS_ptrace, (r), (p), (a), (d)) + // We use sys_ptrace to avoid thread locking + // because this is called from ListAllProcessThreads + // when all but this thread are suspended. + if (sys_ptrace(PTRACE_GETREGS, thread_pids[i], NULL, &thread_regs) == 0) { + // Need to use SP to get all the data from the very last stack frame: + COMPILE_ASSERT(sizeof(thread_regs.SP) == sizeof(void*), + SP_register_does_not_look_like_a_pointer); + RegisterStackLocked(reinterpret_cast<void*>(thread_regs.SP)); + // Make registers live (just in case PTRACE_ATTACH resulted in some + // register pointers still being in the registers and not on the stack): + for (void** p = reinterpret_cast<void**>(&thread_regs); + p < reinterpret_cast<void**>(&thread_regs + 1); ++p) { + RAW_VLOG(3, "Thread register %p", *p); + thread_registers.push_back(*p); + } + } else { + failures += 1; + } +#else + failures += 1; +#endif + } + // Use all the collected thread (stack) liveness sources: + IgnoreLiveObjectsLocked("threads stack data", ""); + if (thread_registers.size()) { + // Make thread registers be live heap data sources. + // we rely here on the fact that vector is in one memory chunk: + RAW_VLOG(2, "Live registers at %p of %"PRIuS" bytes", + &thread_registers[0], thread_registers.size() * sizeof(void*)); + live_objects->push_back(AllocObject(&thread_registers[0], + thread_registers.size() * sizeof(void*), + THREAD_REGISTERS)); + IgnoreLiveObjectsLocked("threads register data", ""); + } + // Do all other liveness walking while all threads are stopped: + IgnoreNonThreadLiveObjectsLocked(); + // Can now resume the threads: + ResumeAllProcessThreads(num_threads, thread_pids); + thread_listing_status = CALLBACK_COMPLETED; + return failures; +} + +// Stack top of the thread that is doing the current leak check +// (protected by our lock; IgnoreAllLiveObjectsLocked sets it) +static const void* self_thread_stack_top; + +// static +void HeapLeakChecker::IgnoreNonThreadLiveObjectsLocked() { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + RAW_DCHECK(MemoryRegionMap::LockIsHeld(), ""); + RAW_VLOG(2, "Handling self thread with pid %d", self_thread_pid); + // Register our own stack: + + // Important that all stack ranges (including the one here) + // are known before we start looking at them + // in MakeDisabledLiveCallbackLocked: + RegisterStackLocked(self_thread_stack_top); + IgnoreLiveObjectsLocked("stack data", ""); + + // Make objects we were told to ignore live: + if (ignored_objects) { + for (IgnoredObjectsMap::const_iterator object = ignored_objects->begin(); + object != ignored_objects->end(); ++object) { + const void* ptr = AsPtr(object->first); + RAW_VLOG(2, "Ignored live object at %p of %"PRIuS" bytes", + ptr, object->second); + live_objects-> + push_back(AllocObject(ptr, object->second, MUST_BE_ON_HEAP)); + // we do this liveness check for ignored_objects before doing any + // live heap walking to make sure it does not fail needlessly: + size_t object_size; + if (!(heap_profile->FindAlloc(ptr, &object_size) && + object->second == object_size)) { + RAW_LOG(FATAL, "Object at %p of %"PRIuS" bytes from an" + " IgnoreObject() has disappeared", ptr, object->second); + } + } + IgnoreLiveObjectsLocked("ignored objects", ""); + } + + // Treat objects that were allocated when a Disabler was live as + // roots. I.e., if X was allocated while a Disabler was active, + // and Y is reachable from X, arrange that neither X nor Y are + // treated as leaks. + heap_profile->IterateAllocs(MakeIgnoredObjectsLiveCallbackLocked); + IgnoreLiveObjectsLocked("disabled objects", ""); + + // Make code-address-disabled objects live and ignored: + // This in particular makes all thread-specific data live + // because the basic data structure to hold pointers to thread-specific data + // is allocated from libpthreads and we have range-disabled that + // library code with UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS); + // so now we declare all thread-specific data reachable from there as live. + heap_profile->IterateAllocs(MakeDisabledLiveCallbackLocked); + IgnoreLiveObjectsLocked("disabled code", ""); + + // Actually make global data live: + if (FLAGS_heap_check_ignore_global_live) { + bool have_null_region_callers = false; + for (LibraryLiveObjectsStacks::iterator l = library_live_objects->begin(); + l != library_live_objects->end(); ++l) { + RAW_CHECK(live_objects->empty(), ""); + // Process library_live_objects in l->second + // filtering them by MemoryRegionMap: + // It's safe to iterate over MemoryRegionMap + // w/o locks here as we are inside MemoryRegionMap::Lock(): + RAW_DCHECK(MemoryRegionMap::LockIsHeld(), ""); + // The only change to MemoryRegionMap possible in this loop + // is region addition as a result of allocating more memory + // for live_objects. This won't invalidate the RegionIterator + // or the intent of the loop. + // --see the comment by MemoryRegionMap::BeginRegionLocked(). + for (MemoryRegionMap::RegionIterator region = + MemoryRegionMap::BeginRegionLocked(); + region != MemoryRegionMap::EndRegionLocked(); ++region) { + // "region" from MemoryRegionMap is to be subtracted from + // (tentatively live) regions in l->second + // if it has a stack inside or it was allocated by + // a non-special caller (not one covered by a range + // in global_region_caller_ranges). + // This will in particular exclude all memory chunks used + // by the heap itself as well as what's been allocated with + // any allocator on top of mmap. + bool subtract = true; + if (!region->is_stack && global_region_caller_ranges) { + if (region->caller() == static_cast<uintptr_t>(NULL)) { + have_null_region_callers = true; + } else { + GlobalRegionCallerRangeMap::const_iterator iter + = global_region_caller_ranges->upper_bound(region->caller()); + if (iter != global_region_caller_ranges->end()) { + RAW_DCHECK(iter->first > region->caller(), ""); + if (iter->second < region->caller()) { // in special region + subtract = false; + } + } + } + } + if (subtract) { + // The loop puts the result of filtering l->second into live_objects: + for (LiveObjectsStack::const_iterator i = l->second.begin(); + i != l->second.end(); ++i) { + // subtract *region from *i + uintptr_t start = AsInt(i->ptr); + uintptr_t end = start + i->size; + if (region->start_addr <= start && end <= region->end_addr) { + // full deletion due to subsumption + } else if (start < region->start_addr && + region->end_addr < end) { // cutting-out split + live_objects->push_back(AllocObject(i->ptr, + region->start_addr - start, + IN_GLOBAL_DATA)); + live_objects->push_back(AllocObject(AsPtr(region->end_addr), + end - region->end_addr, + IN_GLOBAL_DATA)); + } else if (region->end_addr > start && + region->start_addr <= start) { // cut from start + live_objects->push_back(AllocObject(AsPtr(region->end_addr), + end - region->end_addr, + IN_GLOBAL_DATA)); + } else if (region->start_addr > start && + region->start_addr < end) { // cut from end + live_objects->push_back(AllocObject(i->ptr, + region->start_addr - start, + IN_GLOBAL_DATA)); + } else { // pass: no intersection + live_objects->push_back(AllocObject(i->ptr, i->size, + IN_GLOBAL_DATA)); + } + } + // Move live_objects back into l->second + // for filtering by the next region. + live_objects->swap(l->second); + live_objects->clear(); + } + } + // Now get and use live_objects from the final version of l->second: + if (VLOG_IS_ON(2)) { + for (LiveObjectsStack::const_iterator i = l->second.begin(); + i != l->second.end(); ++i) { + RAW_VLOG(2, "Library live region at %p of %"PRIuPTR" bytes", + i->ptr, i->size); + } + } + live_objects->swap(l->second); + IgnoreLiveObjectsLocked("in globals of\n ", l->first.c_str()); + } + if (have_null_region_callers) { + RAW_LOG(ERROR, "Have memory regions w/o callers: " + "might report false leaks"); + } + Allocator::DeleteAndNull(&library_live_objects); + } +} + +// Callback for ListAllProcessThreads in IgnoreAllLiveObjectsLocked below +// to test/verify that we have just the one main thread, in which case +// we can do everything in that main thread, +// so that CPU profiler can collect all its samples. +// Returns the number of threads in the process. +static int IsOneThread(void* parameter, int num_threads, + pid_t* thread_pids, va_list ap) { + if (num_threads != 1) { + RAW_LOG(WARNING, "Have threads: Won't CPU-profile the bulk of leak " + "checking work happening in IgnoreLiveThreadsLocked!"); + } + ResumeAllProcessThreads(num_threads, thread_pids); + return num_threads; +} + +// Dummy for IgnoreAllLiveObjectsLocked below. +// Making it global helps with compiler warnings. +static va_list dummy_ap; + +// static +void HeapLeakChecker::IgnoreAllLiveObjectsLocked(const void* self_stack_top) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + RAW_CHECK(live_objects == NULL, ""); + live_objects = new(Allocator::Allocate(sizeof(LiveObjectsStack))) + LiveObjectsStack; + stack_tops = new(Allocator::Allocate(sizeof(StackTopSet))) StackTopSet; + // reset the counts + live_objects_total = 0; + live_bytes_total = 0; + // Reduce max_heap_object_size to FLAGS_heap_check_max_pointer_offset + // for the time of leak check. + // FLAGS_heap_check_max_pointer_offset caps max_heap_object_size + // to manage reasonably low chances of random bytes + // appearing to be pointing into large actually leaked heap objects. + const size_t old_max_heap_object_size = max_heap_object_size; + max_heap_object_size = ( + FLAGS_heap_check_max_pointer_offset != -1 + ? min(size_t(FLAGS_heap_check_max_pointer_offset), max_heap_object_size) + : max_heap_object_size); + // Record global data as live: + if (FLAGS_heap_check_ignore_global_live) { + library_live_objects = + new(Allocator::Allocate(sizeof(LibraryLiveObjectsStacks))) + LibraryLiveObjectsStacks; + } + // Ignore all thread stacks: + thread_listing_status = CALLBACK_NOT_STARTED; + bool need_to_ignore_non_thread_objects = true; + self_thread_pid = getpid(); + self_thread_stack_top = self_stack_top; + if (FLAGS_heap_check_ignore_thread_live) { + // In case we are doing CPU profiling we'd like to do all the work + // in the main thread, not in the special thread created by + // ListAllProcessThreads, so that CPU profiler can collect all its samples. + // The machinery of ListAllProcessThreads conflicts with the CPU profiler + // by also relying on signals and ::sigaction. + // We can do this (run everything in the main thread) safely + // only if there's just the main thread itself in our process. + // This variable reflects these two conditions: + bool want_and_can_run_in_main_thread = + ProfilingIsEnabledForAllThreads() && + ListAllProcessThreads(NULL, IsOneThread) == 1; + // When the normal path of ListAllProcessThreads below is taken, + // we fully suspend the threads right here before any liveness checking + // and keep them suspended for the whole time of liveness checking + // inside of the IgnoreLiveThreadsLocked callback. + // (The threads can't (de)allocate due to lock on the delete hook but + // if not suspended they could still mess with the pointer + // graph while we walk it). + int r = want_and_can_run_in_main_thread + ? IgnoreLiveThreadsLocked(NULL, 1, &self_thread_pid, dummy_ap) + : ListAllProcessThreads(NULL, IgnoreLiveThreadsLocked); + need_to_ignore_non_thread_objects = r < 0; + if (r < 0) { + RAW_LOG(WARNING, "Thread finding failed with %d errno=%d", r, errno); + if (thread_listing_status == CALLBACK_COMPLETED) { + RAW_LOG(INFO, "Thread finding callback " + "finished ok; hopefully everything is fine"); + need_to_ignore_non_thread_objects = false; + } else if (thread_listing_status == CALLBACK_STARTED) { + RAW_LOG(FATAL, "Thread finding callback was " + "interrupted or crashed; can't fix this"); + } else { // CALLBACK_NOT_STARTED + RAW_LOG(ERROR, "Could not find thread stacks. " + "Will likely report false leak positives."); + } + } else if (r != 0) { + RAW_LOG(ERROR, "Thread stacks not found for %d threads. " + "Will likely report false leak positives.", r); + } else { + RAW_VLOG(2, "Thread stacks appear to be found for all threads"); + } + } else { + RAW_LOG(WARNING, "Not looking for thread stacks; " + "objects reachable only from there " + "will be reported as leaks"); + } + // Do all other live data ignoring here if we did not do it + // within thread listing callback with all threads stopped. + if (need_to_ignore_non_thread_objects) { + if (FLAGS_heap_check_ignore_global_live) { + UseProcMapsLocked(RECORD_GLOBAL_DATA); + } + IgnoreNonThreadLiveObjectsLocked(); + } + if (live_objects_total) { + RAW_VLOG(1, "Ignoring %"PRId64" reachable objects of %"PRId64" bytes", + live_objects_total, live_bytes_total); + } + // Free these: we made them here and heap_profile never saw them + Allocator::DeleteAndNull(&live_objects); + Allocator::DeleteAndNull(&stack_tops); + max_heap_object_size = old_max_heap_object_size; // reset this var +} + +// Alignment at which we should consider pointer positions +// in IgnoreLiveObjectsLocked. Use 1 if any alignment is ok. +static size_t pointer_source_alignment = kPointerSourceAlignment; +// Global lock for HeapLeakChecker::DoNoLeaks +// to protect pointer_source_alignment. +static SpinLock alignment_checker_lock(SpinLock::LINKER_INITIALIZED); + +// This function changes the live bits in the heap_profile-table's state: +// we only record the live objects to be skipped. +// +// When checking if a byte sequence points to a heap object we use +// HeapProfileTable::FindInsideAlloc to handle both pointers to +// the start and inside of heap-allocated objects. +// The "inside" case needs to be checked to support +// at least the following relatively common cases: +// - C++ arrays allocated with new FooClass[size] for classes +// with destructors have their size recorded in a sizeof(int) field +// before the place normal pointers point to. +// - basic_string<>-s for e.g. the C++ library of gcc 3.4 +// have the meta-info in basic_string<...>::_Rep recorded +// before the place normal pointers point to. +// - Multiple-inherited objects have their pointers when cast to +// different base classes pointing inside of the actually +// allocated object. +// - Sometimes reachability pointers point to member objects of heap objects, +// and then those member objects point to the full heap object. +// - Third party UnicodeString: it stores a 32-bit refcount +// (in both 32-bit and 64-bit binaries) as the first uint32 +// in the allocated memory and a normal pointer points at +// the second uint32 behind the refcount. +// By finding these additional objects here +// we slightly increase the chance to mistake random memory bytes +// for a pointer and miss a leak in a particular run of a binary. +// +/*static*/ void HeapLeakChecker::IgnoreLiveObjectsLocked(const char* name, + const char* name2) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + int64 live_object_count = 0; + int64 live_byte_count = 0; + while (!live_objects->empty()) { + const char* object = + reinterpret_cast<const char*>(live_objects->back().ptr); + size_t size = live_objects->back().size; + const ObjectPlacement place = live_objects->back().place; + live_objects->pop_back(); + if (place == MUST_BE_ON_HEAP && heap_profile->MarkAsLive(object)) { + live_object_count += 1; + live_byte_count += size; + } + RAW_VLOG(4, "Looking for heap pointers in %p of %"PRIuS" bytes", + object, size); + const char* const whole_object = object; + size_t const whole_size = size; + // Try interpretting any byte sequence in object,size as a heap pointer: + const size_t remainder = AsInt(object) % pointer_source_alignment; + if (remainder) { + object += pointer_source_alignment - remainder; + if (size >= pointer_source_alignment - remainder) { + size -= pointer_source_alignment - remainder; + } else { + size = 0; + } + } + if (size < sizeof(void*)) continue; + const char* const max_object = object + size - sizeof(void*); + while (object <= max_object) { + // potentially unaligned load: + const uintptr_t addr = *reinterpret_cast<const uintptr_t*>(object); + // Do fast check before the more expensive HaveOnHeapLocked lookup: + // this code runs for all memory words that are potentially pointers: + const bool can_be_on_heap = + // Order tests by the likelyhood of the test failing in 64/32 bit modes. + // Yes, this matters: we either lose 5..6% speed in 32 bit mode + // (which is already slower) or by a factor of 1.5..1.91 in 64 bit mode. + // After the alignment test got dropped the above performance figures + // must have changed; might need to revisit this. +#if defined(__x86_64__) + addr <= max_heap_address && // <= is for 0-sized object with max addr + min_heap_address <= addr; +#else + min_heap_address <= addr && + addr <= max_heap_address; // <= is for 0-sized object with max addr +#endif + if (can_be_on_heap) { + const void* ptr = reinterpret_cast<const void*>(addr); + // Too expensive (inner loop): manually uncomment when debugging: + // RAW_VLOG(8, "Trying pointer to %p at %p", ptr, object); + size_t object_size; + if (HaveOnHeapLocked(&ptr, &object_size) && + heap_profile->MarkAsLive(ptr)) { + // We take the (hopefully low) risk here of encountering by accident + // a byte sequence in memory that matches an address of + // a heap object which is in fact leaked. + // I.e. in very rare and probably not repeatable/lasting cases + // we might miss some real heap memory leaks. + RAW_VLOG(5, "Found pointer to %p of %"PRIuS" bytes at %p " + "inside %p of size %"PRIuS"", + ptr, object_size, object, whole_object, whole_size); + if (VLOG_IS_ON(6)) { + // log call stacks to help debug how come something is not a leak + HeapProfileTable::AllocInfo alloc; + bool r = heap_profile->FindAllocDetails(ptr, &alloc); + r = r; // suppress compiler warning in non-debug mode + RAW_DCHECK(r, ""); // sanity + RAW_LOG(INFO, "New live %p object's alloc stack:", ptr); + for (int i = 0; i < alloc.stack_depth; ++i) { + RAW_LOG(INFO, " @ %p", alloc.call_stack[i]); + } + } + live_object_count += 1; + live_byte_count += object_size; + live_objects->push_back(AllocObject(ptr, object_size, + IGNORED_ON_HEAP)); + } + } + object += pointer_source_alignment; + } + } + live_objects_total += live_object_count; + live_bytes_total += live_byte_count; + if (live_object_count) { + RAW_VLOG(1, "Removed %"PRId64" live heap objects of %"PRId64" bytes: %s%s", + live_object_count, live_byte_count, name, name2); + } +} + +//---------------------------------------------------------------------- +// HeapLeakChecker leak check disabling components +//---------------------------------------------------------------------- + +// static +void HeapLeakChecker::DisableChecksIn(const char* pattern) { + RAW_LOG(WARNING, "DisableChecksIn(%s) is ignored", pattern); +} + +// static +void HeapLeakChecker::IgnoreObject(const void* ptr) { + SpinLockHolder l(&heap_checker_lock); + if (!heap_checker_on) return; + size_t object_size; + if (!HaveOnHeapLocked(&ptr, &object_size)) { + RAW_LOG(ERROR, "No live heap object at %p to ignore", ptr); + } else { + RAW_VLOG(1, "Going to ignore live object at %p of %"PRIuS" bytes", + ptr, object_size); + if (ignored_objects == NULL) { + ignored_objects = new(Allocator::Allocate(sizeof(IgnoredObjectsMap))) + IgnoredObjectsMap; + } + if (!ignored_objects->insert(make_pair(AsInt(ptr), object_size)).second) { + RAW_LOG(FATAL, "Object at %p is already being ignored", ptr); + } + } +} + +// static +void HeapLeakChecker::UnIgnoreObject(const void* ptr) { + SpinLockHolder l(&heap_checker_lock); + if (!heap_checker_on) return; + size_t object_size; + if (!HaveOnHeapLocked(&ptr, &object_size)) { + RAW_LOG(FATAL, "No live heap object at %p to un-ignore", ptr); + } else { + bool found = false; + if (ignored_objects) { + IgnoredObjectsMap::iterator object = ignored_objects->find(AsInt(ptr)); + if (object != ignored_objects->end() && object_size == object->second) { + ignored_objects->erase(object); + found = true; + RAW_VLOG(1, "Now not going to ignore live object " + "at %p of %"PRIuS" bytes", ptr, object_size); + } + } + if (!found) RAW_LOG(FATAL, "Object at %p has not been ignored", ptr); + } +} + +//---------------------------------------------------------------------- +// HeapLeakChecker non-static functions +//---------------------------------------------------------------------- + +char* HeapLeakChecker::MakeProfileNameLocked() { + RAW_DCHECK(lock_->IsHeld(), ""); + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + const int len = profile_name_prefix->size() + strlen(name_) + 5 + + strlen(HeapProfileTable::kFileExt) + 1; + char* file_name = reinterpret_cast<char*>(Allocator::Allocate(len)); + snprintf(file_name, len, "%s.%s-end%s", + profile_name_prefix->c_str(), name_, + HeapProfileTable::kFileExt); + return file_name; +} + +void HeapLeakChecker::Create(const char *name, bool make_start_snapshot) { + SpinLockHolder l(lock_); + name_ = NULL; // checker is inactive + start_snapshot_ = NULL; + has_checked_ = false; + inuse_bytes_increase_ = 0; + inuse_allocs_increase_ = 0; + keep_profiles_ = false; + char* n = new char[strlen(name) + 1]; // do this before we lock + IgnoreObject(n); // otherwise it might be treated as live due to our stack + { // Heap activity in other threads is paused for this whole scope. + SpinLockHolder al(&alignment_checker_lock); + SpinLockHolder hl(&heap_checker_lock); + MemoryRegionMap::LockHolder ml; + if (heap_checker_on && profile_name_prefix != NULL) { + RAW_DCHECK(strchr(name, '/') == NULL, "must be a simple name"); + memcpy(n, name, strlen(name) + 1); + name_ = n; // checker is active + if (make_start_snapshot) { + start_snapshot_ = heap_profile->TakeSnapshot(); + } + + const HeapProfileTable::Stats& t = heap_profile->total(); + const size_t start_inuse_bytes = t.alloc_size - t.free_size; + const size_t start_inuse_allocs = t.allocs - t.frees; + RAW_VLOG(1, "Start check \"%s\" profile: %"PRIuS" bytes " + "in %"PRIuS" objects", + name_, start_inuse_bytes, start_inuse_allocs); + } else { + RAW_LOG(WARNING, "Heap checker is not active, " + "hence checker \"%s\" will do nothing!", name); + RAW_LOG(WARNING, "To activate set the HEAPCHECK environment variable.\n"); + } + } + if (name_ == NULL) { + UnIgnoreObject(n); + delete[] n; // must be done after we unlock + } +} + +HeapLeakChecker::HeapLeakChecker(const char *name) : lock_(new SpinLock) { + RAW_DCHECK(strcmp(name, "_main_") != 0, "_main_ is reserved"); + Create(name, true/*create start_snapshot_*/); +} + +HeapLeakChecker::HeapLeakChecker() : lock_(new SpinLock) { + if (FLAGS_heap_check_before_constructors) { + // We want to check for leaks of objects allocated during global + // constructors (i.e., objects allocated already). So we do not + // create a baseline snapshot and hence check for leaks of objects + // that may have already been created. + Create("_main_", false); + } else { + // We want to ignore leaks of objects allocated during global + // constructors (i.e., objects allocated already). So we snapshot + // the current heap contents and use them as a baseline that is + // not reported by the leak checker. + Create("_main_", true); + } +} + +ssize_t HeapLeakChecker::BytesLeaked() const { + SpinLockHolder l(lock_); + if (!has_checked_) { + RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call"); + } + return inuse_bytes_increase_; +} + +ssize_t HeapLeakChecker::ObjectsLeaked() const { + SpinLockHolder l(lock_); + if (!has_checked_) { + RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call"); + } + return inuse_allocs_increase_; +} + +// Save pid of main thread for using in naming dump files +static int32 main_thread_pid = getpid(); +#ifdef HAVE_PROGRAM_INVOCATION_NAME +extern char* program_invocation_name; +extern char* program_invocation_short_name; +static const char* invocation_name() { return program_invocation_short_name; } +static string invocation_path() { return program_invocation_name; } +#else +static const char* invocation_name() { return "<your binary>"; } +static string invocation_path() { return "<your binary>"; } +#endif + +// Prints commands that users can run to get more information +// about the reported leaks. +static void SuggestPprofCommand(const char* pprof_file_arg) { + // Extra help information to print for the user when the test is + // being run in a way where the straightforward pprof command will + // not suffice. + string extra_help; + + // Common header info to print for remote runs + const string remote_header = + "This program is being executed remotely and therefore the pprof\n" + "command printed above will not work. Either run this program\n" + "locally, or adjust the pprof command as follows to allow it to\n" + "work on your local machine:\n"; + + // Extra command for fetching remote data + string fetch_cmd; + + RAW_LOG(WARNING, + "\n\n" + "If the preceding stack traces are not enough to find " + "the leaks, try running THIS shell command:\n\n" + "%s%s %s \"%s\" --inuse_objects --lines --heapcheck " + " --edgefraction=1e-10 --nodefraction=1e-10 --gv\n" + "\n" + "%s" + "If you are still puzzled about why the leaks are " + "there, try rerunning this program with " + "HEAP_CHECK_TEST_POINTER_ALIGNMENT=1 and/or with " + "HEAP_CHECK_MAX_POINTER_OFFSET=-1\n" + "If the leak report occurs in a small fraction of runs, " + "try running with TCMALLOC_MAX_FREE_QUEUE_SIZE of few hundred MB " + "or with TCMALLOC_RECLAIM_MEMORY=false, " // only works for debugalloc + "it might help find leaks more repeatably\n", + fetch_cmd.c_str(), + "pprof", // works as long as pprof is on your path + invocation_path().c_str(), + pprof_file_arg, + extra_help.c_str() + ); +} + +bool HeapLeakChecker::DoNoLeaks(ShouldSymbolize should_symbolize) { + SpinLockHolder l(lock_); + // The locking also helps us keep the messages + // for the two checks close together. + SpinLockHolder al(&alignment_checker_lock); + + // thread-safe: protected by alignment_checker_lock + static bool have_disabled_hooks_for_symbolize = false; + // Once we've checked for leaks and symbolized the results once, it's + // not safe to do it again. This is because in order to symbolize + // safely, we had to disable all the malloc hooks here, so we no + // longer can be confident we've collected all the data we need. + if (have_disabled_hooks_for_symbolize) { + RAW_LOG(FATAL, "Must not call heap leak checker manually after " + " program-exit's automatic check."); + } + + HeapProfileTable::Snapshot* leaks = NULL; + char* pprof_file = NULL; + + { + // Heap activity in other threads is paused during this function + // (i.e. until we got all profile difference info). + SpinLockHolder l(&heap_checker_lock); + if (heap_checker_on == false) { + if (name_ != NULL) { // leak checking enabled when created the checker + RAW_LOG(WARNING, "Heap leak checker got turned off after checker " + "\"%s\" has been created, no leak check is being done for it!", + name_); + } + return true; + } + + // Keep track of number of internally allocated objects so we + // can detect leaks in the heap-leak-checket itself + const int initial_allocs = Allocator::alloc_count(); + + if (name_ == NULL) { + RAW_LOG(FATAL, "Heap leak checker must not be turned on " + "after construction of a HeapLeakChecker"); + } + + MemoryRegionMap::LockHolder ml; + int a_local_var; // Use our stack ptr to make stack data live: + + // Sanity check that nobody is messing with the hooks we need: + // Important to have it here: else we can misteriously SIGSEGV + // in IgnoreLiveObjectsLocked inside ListAllProcessThreads's callback + // by looking into a region that got unmapped w/o our knowledge. + MemoryRegionMap::CheckMallocHooks(); + if (MallocHook::GetNewHook() != NewHook || + MallocHook::GetDeleteHook() != DeleteHook) { + RAW_LOG(FATAL, "Had our new/delete MallocHook-s replaced. " + "Are you using another MallocHook client? " + "Use --heap_check=\"\" to avoid this conflict."); + } + + // Make the heap profile, other threads are locked out. + HeapProfileTable::Snapshot* base = + reinterpret_cast<HeapProfileTable::Snapshot*>(start_snapshot_); + IgnoreAllLiveObjectsLocked(&a_local_var); + leaks = heap_profile->NonLiveSnapshot(base); + + inuse_bytes_increase_ = static_cast<ssize_t>(leaks->total().alloc_size); + inuse_allocs_increase_ = static_cast<ssize_t>(leaks->total().allocs); + if (leaks->Empty()) { + heap_profile->ReleaseSnapshot(leaks); + leaks = NULL; + + // We can only check for internal leaks along the no-user-leak + // path since in the leak path we temporarily release + // heap_checker_lock and another thread can come in and disturb + // allocation counts. + if (Allocator::alloc_count() != initial_allocs) { + RAW_LOG(FATAL, "Internal HeapChecker leak of %d objects ; %d -> %d", + Allocator::alloc_count() - initial_allocs, + initial_allocs, Allocator::alloc_count()); + } + } else if (FLAGS_heap_check_test_pointer_alignment) { + // Try with reduced pointer aligment + pointer_source_alignment = 1; + IgnoreAllLiveObjectsLocked(&a_local_var); + HeapProfileTable::Snapshot* leaks_wo_align = + heap_profile->NonLiveSnapshot(base); + pointer_source_alignment = kPointerSourceAlignment; + if (leaks_wo_align->Empty()) { + RAW_LOG(WARNING, "Found no leaks without pointer alignment: " + "something might be placing pointers at " + "unaligned addresses! This needs to be fixed."); + } else { + RAW_LOG(INFO, "Found leaks without pointer alignment as well: " + "unaligned pointers must not be the cause of leaks."); + RAW_LOG(INFO, "--heap_check_test_pointer_alignment did not help " + "to diagnose the leaks."); + } + heap_profile->ReleaseSnapshot(leaks_wo_align); + } + + if (leaks != NULL) { + pprof_file = MakeProfileNameLocked(); + } + } + + has_checked_ = true; + if (leaks == NULL) { + if (FLAGS_heap_check_max_pointer_offset == -1) { + RAW_LOG(WARNING, + "Found no leaks without max_pointer_offset restriction: " + "it's possible that the default value of " + "heap_check_max_pointer_offset flag is too low. " + "Do you use pointers with larger than that offsets " + "pointing in the middle of heap-allocated objects?"); + } + const HeapProfileTable::Stats& stats = heap_profile->total(); + RAW_VLOG(heap_checker_info_level, + "No leaks found for check \"%s\" " + "(but no 100%% guarantee that there aren't any): " + "found %"PRId64" reachable heap objects of %"PRId64" bytes", + name_, + int64(stats.allocs - stats.frees), + int64(stats.alloc_size - stats.free_size)); + } else { + if (should_symbolize == SYMBOLIZE) { + // To turn addresses into symbols, we need to fork, which is a + // problem if both parent and child end up trying to call the + // same malloc-hooks we've set up, at the same time. To avoid + // trouble, we turn off the hooks before symbolizing. Note that + // this makes it unsafe to ever leak-report again! Luckily, we + // typically only want to report once in a program's run, at the + // very end. + if (MallocHook::GetNewHook() == NewHook) + MallocHook::SetNewHook(NULL); + if (MallocHook::GetDeleteHook() == DeleteHook) + MallocHook::SetDeleteHook(NULL); + MemoryRegionMap::Shutdown(); + // Make sure all the hooks really got unset: + RAW_CHECK(MallocHook::GetNewHook() == NULL, ""); + RAW_CHECK(MallocHook::GetDeleteHook() == NULL, ""); + RAW_CHECK(MallocHook::GetMmapHook() == NULL, ""); + RAW_CHECK(MallocHook::GetSbrkHook() == NULL, ""); + have_disabled_hooks_for_symbolize = true; + leaks->ReportLeaks(name_, pprof_file, true); // true = should_symbolize + } else { + leaks->ReportLeaks(name_, pprof_file, false); + } + if (FLAGS_heap_check_identify_leaks) { + leaks->ReportIndividualObjects(); + } + + SuggestPprofCommand(pprof_file); + + { + SpinLockHolder l(&heap_checker_lock); + heap_profile->ReleaseSnapshot(leaks); + Allocator::Free(pprof_file); + } + } + + return (leaks == NULL); +} + +HeapLeakChecker::~HeapLeakChecker() { + if (name_ != NULL) { // had leak checking enabled when created the checker + if (!has_checked_) { + RAW_LOG(FATAL, "Some *NoLeaks|SameHeap method" + " must be called on any created HeapLeakChecker"); + } + + // Deallocate any snapshot taken at start + if (start_snapshot_ != NULL) { + SpinLockHolder l(&heap_checker_lock); + heap_profile->ReleaseSnapshot( + reinterpret_cast<HeapProfileTable::Snapshot*>(start_snapshot_)); + } + + UnIgnoreObject(name_); + delete[] name_; + name_ = NULL; + } + delete lock_; +} + +//---------------------------------------------------------------------- +// HeapLeakChecker overall heap check components +//---------------------------------------------------------------------- + +// static +bool HeapLeakChecker::IsActive() { + SpinLockHolder l(&heap_checker_lock); + return heap_checker_on; +} + +vector<HeapCleaner::void_function>* HeapCleaner::heap_cleanups_ = NULL; + +// When a HeapCleaner object is intialized, add its function to the static list +// of cleaners to be run before leaks checking. +HeapCleaner::HeapCleaner(void_function f) { + if (heap_cleanups_ == NULL) + heap_cleanups_ = new vector<HeapCleaner::void_function>; + heap_cleanups_->push_back(f); +} + +// Run all of the cleanup functions and delete the vector. +void HeapCleaner::RunHeapCleanups() { + if (!heap_cleanups_) + return; + for (int i = 0; i < heap_cleanups_->size(); i++) { + void (*f)(void) = (*heap_cleanups_)[i]; + f(); + } + delete heap_cleanups_; + heap_cleanups_ = NULL; +} + +// Program exit heap cleanup registered with atexit(). +// Will not get executed when we crash on a signal. +// +/*static*/ void HeapLeakChecker::RunHeapCleanups() { + { SpinLockHolder l(&heap_checker_lock); + // can get here (via forks?) with other pids + if (heap_checker_pid != getpid()) return; + } + HeapCleaner::RunHeapCleanups(); + if (!FLAGS_heap_check_after_destructors) DoMainHeapCheck(); +} + +// defined below +static int GetCommandLineFrom(const char* file, char* cmdline, int size); + +static bool internal_init_start_has_run = false; + +// Called exactly once, before main() (but hopefully just before). +// This picks a good unique name for the dumped leak checking heap profiles. +// +// Because we crash when InternalInitStart is called more than once, +// it's fine that we hold heap_checker_lock only around pieces of +// this function: this is still enough for thread-safety w.r.t. other functions +// of this module. +// We can't hold heap_checker_lock throughout because it would deadlock +// on a memory allocation since our new/delete hooks can be on. +// +/*static*/ void HeapLeakChecker::InternalInitStart() { + { SpinLockHolder l(&heap_checker_lock); + RAW_CHECK(!internal_init_start_has_run, "Only one call is expected"); + internal_init_start_has_run = true; + + if (FLAGS_heap_check.empty()) { + // turns out we do not need checking in the end; can stop profiling + TurnItselfOffLocked(); + return; + } + } + + // Changing this to false can be useful when debugging heap-checker itself: + if (!FLAGS_heap_check_run_under_gdb) { + // See if heap checker should turn itself off because we are + // running under gdb (to avoid conflicts over ptrace-ing rights): + char name_buf[15+15]; + snprintf(name_buf, sizeof(name_buf), + "/proc/%d/cmdline", static_cast<int>(getppid())); + char cmdline[1024*8]; // /proc/*/cmdline is at most 4Kb anyway usually + int size = GetCommandLineFrom(name_buf, cmdline, sizeof(cmdline)-1); + cmdline[size] = '\0'; + // look for "gdb" in the executable's name: + const char* last = strrchr(cmdline, '/'); + if (last) last += 1; + else last = cmdline; + if (strncmp(last, "gdb", 3) == 0) { + RAW_LOG(WARNING, "We seem to be running under gdb; will turn itself off"); + SpinLockHolder l(&heap_checker_lock); + TurnItselfOffLocked(); + return; + } + } + + { SpinLockHolder l(&heap_checker_lock); + if (!constructor_heap_profiling) { + RAW_LOG(FATAL, "Can not start so late. You have to enable heap checking " + "with HEAPCHECK=<mode>."); + } + } + + // Set all flags + if (FLAGS_heap_check == "minimal") { + // The least we can check. + FLAGS_heap_check_before_constructors = false; // from after main + // (ignore more) + FLAGS_heap_check_after_destructors = false; // to after cleanup + // (most data is live) + FLAGS_heap_check_ignore_thread_live = true; // ignore all live + FLAGS_heap_check_ignore_global_live = true; // ignore all live + } else if (FLAGS_heap_check == "normal") { + // Faster than 'minimal' and not much stricter. + FLAGS_heap_check_before_constructors = true; // from no profile (fast) + FLAGS_heap_check_after_destructors = false; // to after cleanup + // (most data is live) + FLAGS_heap_check_ignore_thread_live = true; // ignore all live + FLAGS_heap_check_ignore_global_live = true; // ignore all live + } else if (FLAGS_heap_check == "strict") { + // A bit stricter than 'normal': global destructors must fully clean up + // after themselves if they are present. + FLAGS_heap_check_before_constructors = true; // from no profile (fast) + FLAGS_heap_check_after_destructors = true; // to after destructors + // (less data live) + FLAGS_heap_check_ignore_thread_live = true; // ignore all live + FLAGS_heap_check_ignore_global_live = true; // ignore all live + } else if (FLAGS_heap_check == "draconian") { + // Drop not very portable and not very exact live heap flooding. + FLAGS_heap_check_before_constructors = true; // from no profile (fast) + FLAGS_heap_check_after_destructors = true; // to after destructors + // (need them) + FLAGS_heap_check_ignore_thread_live = false; // no live flood (stricter) + FLAGS_heap_check_ignore_global_live = false; // no live flood (stricter) + } else if (FLAGS_heap_check == "as-is") { + // do nothing: use other flags as is + } else if (FLAGS_heap_check == "local") { + // do nothing + } else { + RAW_LOG(FATAL, "Unsupported heap_check flag: %s", + FLAGS_heap_check.c_str()); + } + { SpinLockHolder l(&heap_checker_lock); + RAW_DCHECK(heap_checker_pid == getpid(), ""); + heap_checker_on = true; + RAW_DCHECK(heap_profile, ""); + ProcMapsResult pm_result = UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS); + // might neeed to do this more than once + // if one later dynamically loads libraries that we want disabled + if (pm_result != PROC_MAPS_USED) { // can't function + TurnItselfOffLocked(); + return; + } + } + + // make a good place and name for heap profile leak dumps + string* profile_prefix = + new string(FLAGS_heap_check_dump_directory + "/" + invocation_name()); + + // Finalize prefix for dumping leak checking profiles. + const int32 our_pid = getpid(); // safest to call getpid() outside lock + { SpinLockHolder l(&heap_checker_lock); + // main_thread_pid might still be 0 if this function is being called before + // global constructors. In that case, our pid *is* the main pid. + if (main_thread_pid == 0) + main_thread_pid = our_pid; + } + char pid_buf[15]; + snprintf(pid_buf, sizeof(pid_buf), ".%d", main_thread_pid); + *profile_prefix += pid_buf; + { SpinLockHolder l(&heap_checker_lock); + RAW_DCHECK(profile_name_prefix == NULL, ""); + profile_name_prefix = profile_prefix; + } + + // Make sure new/delete hooks are installed properly + // and heap profiler is indeed able to keep track + // of the objects being allocated. + // We test this to make sure we are indeed checking for leaks. + char* test_str = new char[5]; + size_t size; + { SpinLockHolder l(&heap_checker_lock); + RAW_CHECK(heap_profile->FindAlloc(test_str, &size), + "our own new/delete not linked?"); + } + delete[] test_str; + { SpinLockHolder l(&heap_checker_lock); + // This check can fail when it should not if another thread allocates + // into this same spot right this moment, + // which is unlikely since this code runs in InitGoogle. + RAW_CHECK(!heap_profile->FindAlloc(test_str, &size), + "our own new/delete not linked?"); + } + // If we crash in the above code, it probably means that + // "nm <this_binary> | grep new" will show that tcmalloc's new/delete + // implementation did not get linked-in into this binary + // (i.e. nm will list __builtin_new and __builtin_vec_new as undefined). + // If this happens, it is a BUILD bug to be fixed. + + RAW_VLOG(heap_checker_info_level, + "WARNING: Perftools heap leak checker is active " + "-- Performance may suffer"); + + if (FLAGS_heap_check != "local") { + // Schedule registered heap cleanup + atexit(RunHeapCleanups); + HeapLeakChecker* main_hc = new HeapLeakChecker(); + SpinLockHolder l(&heap_checker_lock); + RAW_DCHECK(main_heap_checker == NULL, + "Repeated creation of main_heap_checker"); + main_heap_checker = main_hc; + do_main_heap_check = true; + } + + { SpinLockHolder l(&heap_checker_lock); + RAW_CHECK(heap_checker_on && constructor_heap_profiling, + "Leak checking is expected to be fully turned on now"); + } + + // For binaries built in debug mode, this will set release queue of + // debugallocation.cc to 100M to make it less likely for real leaks to + // be hidden due to reuse of heap memory object addresses. + // Running a test with --malloc_reclaim_memory=0 would help find leaks even + // better, but the test might run out of memory as a result. + // The scenario is that a heap object at address X is allocated and freed, + // but some other data-structure still retains a pointer to X. + // Then the same heap memory is used for another object, which is leaked, + // but the leak is not noticed due to the pointer to the original object at X. + // TODO(csilvers): support this in some manner. +#if 0 + SetCommandLineOptionWithMode("max_free_queue_size", "104857600", // 100M + SET_FLAG_IF_DEFAULT); +#endif +} + +// We want this to run early as well, but not so early as +// ::BeforeConstructors (we want flag assignments to have already +// happened, for instance). Initializer-registration does the trick. +REGISTER_MODULE_INITIALIZER(init_start, HeapLeakChecker::InternalInitStart()); + +// static +bool HeapLeakChecker::DoMainHeapCheck() { + if (FLAGS_heap_check_delay_seconds > 0) { + sleep(FLAGS_heap_check_delay_seconds); + } + { SpinLockHolder l(&heap_checker_lock); + if (!do_main_heap_check) return false; + RAW_DCHECK(heap_checker_pid == getpid(), ""); + do_main_heap_check = false; // will do it now; no need to do it more + } + + if (!NoGlobalLeaks()) { + if (FLAGS_heap_check_identify_leaks) { + RAW_LOG(FATAL, "Whole-program memory leaks found."); + } + RAW_LOG(ERROR, "Exiting with error code (instead of crashing) " + "because of whole-program memory leaks"); + _exit(1); // we don't want to call atexit() routines! + } + return true; +} + +// static +HeapLeakChecker* HeapLeakChecker::GlobalChecker() { + SpinLockHolder l(&heap_checker_lock); + return main_heap_checker; +} + +// static +bool HeapLeakChecker::NoGlobalLeaks() { + // we never delete or change main_heap_checker once it's set: + HeapLeakChecker* main_hc = GlobalChecker(); + if (main_hc) { + RAW_VLOG(1, "Checking for whole-program memory leaks"); + // The program is over, so it's safe to symbolize addresses (which + // requires a fork) because no serious work is expected to be done + // after this. Symbolizing is really useful -- knowing what + // function has a leak is better than knowing just an address -- + // and while we can only safely symbolize once in a program run, + // now is the time (after all, there's no "later" that would be better). + return main_hc->DoNoLeaks(SYMBOLIZE); + } + return true; +} + +// static +void HeapLeakChecker::CancelGlobalCheck() { + SpinLockHolder l(&heap_checker_lock); + if (do_main_heap_check) { + RAW_VLOG(heap_checker_info_level, + "Canceling the automatic at-exit whole-program memory leak check"); + do_main_heap_check = false; + } +} + +//---------------------------------------------------------------------- +// HeapLeakChecker global constructor/destructor ordering components +//---------------------------------------------------------------------- + +static bool in_initial_malloc_hook = false; + +#ifdef HAVE___ATTRIBUTE__ // we need __attribute__((weak)) for this to work +#define INSTALLED_INITIAL_MALLOC_HOOKS + +void HeapLeakChecker_BeforeConstructors(); // below + +// Helper for InitialMallocHook_* below +static inline void InitHeapLeakCheckerFromMallocHook() { + { SpinLockHolder l(&heap_checker_lock); + RAW_CHECK(!in_initial_malloc_hook, + "Something did not reset initial MallocHook-s"); + in_initial_malloc_hook = true; + } + // Initialize heap checker on the very first allocation/mmap/sbrk call: + HeapLeakChecker_BeforeConstructors(); + { SpinLockHolder l(&heap_checker_lock); + in_initial_malloc_hook = false; + } +} + +// These will owerwrite the weak definitions in malloc_hook.cc: + +// Important to have this to catch the first allocation call from the binary: +extern void InitialMallocHook_New(const void* ptr, size_t size) { + InitHeapLeakCheckerFromMallocHook(); + // record this first allocation as well (if we need to): + MallocHook::InvokeNewHook(ptr, size); +} + +// Important to have this to catch the first mmap call (say from tcmalloc): +extern void InitialMallocHook_MMap(const void* result, + const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset) { + InitHeapLeakCheckerFromMallocHook(); + // record this first mmap as well (if we need to): + MallocHook::InvokeMmapHook( + result, start, size, protection, flags, fd, offset); +} + +// Important to have this to catch the first sbrk call (say from tcmalloc): +extern void InitialMallocHook_Sbrk(const void* result, ptrdiff_t increment) { + InitHeapLeakCheckerFromMallocHook(); + // record this first sbrk as well (if we need to): + MallocHook::InvokeSbrkHook(result, increment); +} + +// static +void CancelInitialMallocHooks() { + if (MallocHook::GetNewHook() == InitialMallocHook_New) { + MallocHook::SetNewHook(NULL); + } + RAW_DCHECK(MallocHook::GetNewHook() == NULL, ""); + if (MallocHook::GetMmapHook() == InitialMallocHook_MMap) { + MallocHook::SetMmapHook(NULL); + } + RAW_DCHECK(MallocHook::GetMmapHook() == NULL, ""); + if (MallocHook::GetSbrkHook() == InitialMallocHook_Sbrk) { + MallocHook::SetSbrkHook(NULL); + } + RAW_DCHECK(MallocHook::GetSbrkHook() == NULL, ""); +} + +#else + +// static +void CancelInitialMallocHooks() {} + +#endif + +// static +void HeapLeakChecker::BeforeConstructorsLocked() { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + RAW_CHECK(!constructor_heap_profiling, + "BeforeConstructorsLocked called multiple times"); + CancelInitialMallocHooks(); + // Set hooks early to crash if 'new' gets called before we make heap_profile, + // and make sure no other hooks existed: + if (MallocHook::SetNewHook(NewHook) != NULL || + MallocHook::SetDeleteHook(DeleteHook) != NULL) { + RAW_LOG(FATAL, "Had other new/delete MallocHook-s set. " + "Somehow leak checker got activated " + "after something else have set up these hooks."); + } + constructor_heap_profiling = true; + MemoryRegionMap::Init(1); + // Set up MemoryRegionMap with (at least) one caller stack frame to record + // (important that it's done before HeapProfileTable creation below). + Allocator::Init(); + RAW_CHECK(heap_profile == NULL, ""); + heap_profile = new(Allocator::Allocate(sizeof(HeapProfileTable))) + HeapProfileTable(&Allocator::Allocate, &Allocator::Free); + RAW_VLOG(1, "Starting tracking the heap"); + heap_checker_on = true; +} + +// static +void HeapLeakChecker::TurnItselfOffLocked() { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + // Set FLAGS_heap_check to "", for users who test for it + if (!FLAGS_heap_check.empty()) // be a noop in the common case + FLAGS_heap_check.clear(); // because clear() could allocate memory + if (constructor_heap_profiling) { + RAW_CHECK(heap_checker_on, ""); + RAW_VLOG(heap_checker_info_level, "Turning perftools heap leak checking off"); + heap_checker_on = false; + // Unset our hooks checking they were the ones set: + if (MallocHook::SetNewHook(NULL) != NewHook || + MallocHook::SetDeleteHook(NULL) != DeleteHook) { + RAW_LOG(FATAL, "Had our new/delete MallocHook-s replaced. " + "Are you using another MallocHook client?"); + } + Allocator::DeleteAndNull(&heap_profile); + // free our optional global data: + Allocator::DeleteAndNullIfNot(&ignored_objects); + Allocator::DeleteAndNullIfNot(&disabled_ranges); + Allocator::DeleteAndNullIfNot(&global_region_caller_ranges); + Allocator::Shutdown(); + MemoryRegionMap::Shutdown(); + } + RAW_CHECK(!heap_checker_on, ""); +} + +// Read in the command line from 'file' into 'cmdline' and return the size read +// 'size' is the space available in 'cmdline'. +// We need this because we don't yet have argv/argc. +// CAVEAT: 'file' (some /proc/*/cmdline) usually contains the command line +// already truncated (to 4K on Linux). +// Arguments in cmdline will be '\0'-terminated, +// the first one will be the binary's name. +static int GetCommandLineFrom(const char* file, char* cmdline, int size) { + // This routine is only used to check if we're running under gdb, so + // it's ok if this #if fails and the routine is a no-op. + // + // This function is called before memory allocation hooks are set up + // so we must not have any memory allocations in it. We use syscall + // versions of open/read/close here because we don't trust the non-syscall + // versions: they might 'accidentally' cause a memory allocation. + // Here's a real-life problem scenario we had: + // 1) A program LD_PRELOADed a library called list_file_used.a + // 2) list_file_used intercepted open/read/close and called dlsym() + // 3) dlsym() called pthread_setspecific() which called malloc(). + // This malloced memory is 'hidden' from the heap-checker. By + // definition, this thread-local data is live, and everything it points + // to is live (not a memory leak) as well. But because this memory + // was hidden from the heap-checker, everything it points to was + // taken to be orphaned, and therefore, a memory leak. +#if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__) + // Use a win32 call to get the command line. + const char* command_line = ::GetCommandLine(); + strncpy(cmdline, command_line, size); + cmdline[size - 1] = '\0'; + return strlen(cmdline); +#elif defined(HAVE_SYS_SYSCALL_H) + int fd = syscall(SYS_open, file, O_RDONLY); + int result = 0; + if (fd >= 0) { + ssize_t r; + while ((r = syscall(SYS_read, fd, cmdline + result, size)) > 0) { + result += r; + size -= r; + } + syscall(SYS_close, fd); + } + return result; +#else + return 0; +#endif +} + +extern bool heap_leak_checker_bcad_variable; // in heap-checker-bcad.cc + +static bool has_called_before_constructors = false; + +void HeapLeakChecker_BeforeConstructors() { + SpinLockHolder l(&heap_checker_lock); + // We can be called from several places: the first mmap/sbrk/alloc call + // or the first global c-tor from heap-checker-bcad.cc: + // Do not re-execute initialization: + if (has_called_before_constructors) return; + has_called_before_constructors = true; + + heap_checker_pid = getpid(); // set it always + heap_leak_checker_bcad_variable = true; + // just to reference it, so that heap-checker-bcad.o is linked in + + // This function can be called *very* early, before the normal + // global-constructor that sets FLAGS_verbose. Set it manually now, + // so the RAW_LOG messages here are controllable. + const char* verbose_str = GetenvBeforeMain("PERFTOOLS_VERBOSE"); + if (verbose_str && atoi(verbose_str)) { // different than the default of 0? + FLAGS_verbose = atoi(verbose_str); + } + + bool need_heap_check = true; + // The user indicates a desire for heap-checking via the HEAPCHECK + // environment variable. If it's not set, there's no way to do + // heap-checking. + if (!GetenvBeforeMain("HEAPCHECK")) { + need_heap_check = false; + } +#ifdef HAVE_GETEUID + if (need_heap_check && getuid() != geteuid()) { + // heap-checker writes out files. Thus, for security reasons, we don't + // recognize the env. var. to turn on heap-checking if we're setuid. + RAW_LOG(WARNING, ("HeapChecker: ignoring HEAPCHECK because " + "program seems to be setuid\n")); + need_heap_check = false; + } +#endif + if (need_heap_check) { + HeapLeakChecker::BeforeConstructorsLocked(); + } else { // cancel our initial hooks + CancelInitialMallocHooks(); + } +} + +// This function is executed after all global object destructors run. +void HeapLeakChecker_AfterDestructors() { + { SpinLockHolder l(&heap_checker_lock); + // can get here (via forks?) with other pids + if (heap_checker_pid != getpid()) return; + } + if (FLAGS_heap_check_after_destructors) { + if (HeapLeakChecker::DoMainHeapCheck()) { + const struct timespec sleep_time = { 0, 500000000 }; // 500 ms + nanosleep(&sleep_time, NULL); + // Need this hack to wait for other pthreads to exit. + // Otherwise tcmalloc find errors + // on a free() call from pthreads. + } + } + SpinLockHolder l(&heap_checker_lock); + RAW_CHECK(!do_main_heap_check, "should have done it"); +} + +//---------------------------------------------------------------------- +// HeapLeakChecker disabling helpers +//---------------------------------------------------------------------- + +// These functions are at the end of the file to prevent their inlining: + +// static +void HeapLeakChecker::DisableChecksFromToLocked(const void* start_address, + const void* end_address, + int max_depth) { + RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + RAW_DCHECK(start_address < end_address, ""); + if (disabled_ranges == NULL) { + disabled_ranges = new(Allocator::Allocate(sizeof(DisabledRangeMap))) + DisabledRangeMap; + } + RangeValue value; + value.start_address = AsInt(start_address); + value.max_depth = max_depth; + if (disabled_ranges->insert(make_pair(AsInt(end_address), value)).second) { + RAW_VLOG(1, "Disabling leak checking in stack traces " + "under frame addresses between %p..%p", + start_address, end_address); + } else { // check that this is just a verbatim repetition + RangeValue const& val = disabled_ranges->find(AsInt(end_address))->second; + if (val.max_depth != value.max_depth || + val.start_address != value.start_address) { + RAW_LOG(FATAL, "Two DisableChecksToHereFrom calls conflict: " + "(%p, %p, %d) vs. (%p, %p, %d)", + AsPtr(val.start_address), end_address, val.max_depth, + start_address, end_address, max_depth); + } + } +} + +// static +inline bool HeapLeakChecker::HaveOnHeapLocked(const void** ptr, + size_t* object_size) { + // Commented-out because HaveOnHeapLocked is very performance-critical: + // RAW_DCHECK(heap_checker_lock.IsHeld(), ""); + const uintptr_t addr = AsInt(*ptr); + if (heap_profile->FindInsideAlloc( + *ptr, max_heap_object_size, ptr, object_size)) { + RAW_VLOG(7, "Got pointer into %p at +%"PRIuPTR" offset", + *ptr, addr - AsInt(*ptr)); + return true; + } + return false; +} + +// static +const void* HeapLeakChecker::GetAllocCaller(void* ptr) { + // this is used only in the unittest, so the heavy checks are fine + HeapProfileTable::AllocInfo info; + { SpinLockHolder l(&heap_checker_lock); + RAW_CHECK(heap_profile->FindAllocDetails(ptr, &info), ""); + } + RAW_CHECK(info.stack_depth >= 1, ""); + return info.call_stack[0]; +} |