diff options
Diffstat (limited to 'third_party/tcmalloc/chromium/src/malloc_hook.cc')
| -rw-r--r-- | third_party/tcmalloc/chromium/src/malloc_hook.cc | 588 |
1 files changed, 484 insertions, 104 deletions
diff --git a/third_party/tcmalloc/chromium/src/malloc_hook.cc b/third_party/tcmalloc/chromium/src/malloc_hook.cc index dfb34ce..f6af7d8 100644 --- a/third_party/tcmalloc/chromium/src/malloc_hook.cc +++ b/third_party/tcmalloc/chromium/src/malloc_hook.cc @@ -41,9 +41,15 @@ # undef mremap #endif +#include <stddef.h> +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif #include <algorithm> #include "base/basictypes.h" #include "base/logging.h" +#include "base/spinlock.h" +#include "maybe_threads.h" #include "malloc_hook-inl.h" #include <google/malloc_hook.h> @@ -66,48 +72,93 @@ using std::copy; -// Declarations of three default weak hook functions, that can be overridden by -// linking-in a strong definition (as heap-checker.cc does) -// -// These default hooks let some other library we link in -// to define strong versions of InitialMallocHook_New, InitialMallocHook_MMap, -// InitialMallocHook_PreMMap, InitialMallocHook_PreSbrk, and -// InitialMallocHook_Sbrk to have a chance to hook into the very first -// invocation of an allocation function call, mmap, or sbrk. +// Declaration of default weak initialization function, that can be overridden +// by linking-in a strong definition (as heap-checker.cc does). This is +// extern "C" so that it doesn't trigger gold's --detect-odr-violations warning, +// which only looks at C++ symbols. // -// These functions are declared here as weak, and defined later, rather than a -// more straightforward simple weak definition, as a workround for an icc -// compiler issue ((Intel reference 290819). This issue causes icc to resolve -// weak symbols too early, at compile rather than link time. By declaring it -// (weak) here, then defining it below after its use, we can avoid the problem. +// This function is declared here as weak, and defined later, rather than a more +// straightforward simple weak definition, as a workround for an icc compiler +// issue ((Intel reference 290819). This issue causes icc to resolve weak +// symbols too early, at compile rather than link time. By declaring it (weak) +// here, then defining it below after its use, we can avoid the problem. +extern "C" { +ATTRIBUTE_WEAK void MallocHook_InitAtFirstAllocation_HeapLeakChecker(); +} + +namespace { + +void RemoveInitialHooksAndCallInitializers(); // below. + +pthread_once_t once = PTHREAD_ONCE_INIT; + +// These hooks are installed in MallocHook as the only initial hooks. The first +// hook that is called will run RemoveInitialHooksAndCallInitializers (see the +// definition below) and then redispatch to any malloc hooks installed by +// RemoveInitialHooksAndCallInitializers. // -ATTRIBUTE_WEAK -extern void InitialMallocHook_New(const void* ptr, size_t size); +// Note(llib): there is a possibility of a race in the event that there are +// multiple threads running before the first allocation. This is pretty +// difficult to achieve, but if it is then multiple threads may concurrently do +// allocations. The first caller will call +// RemoveInitialHooksAndCallInitializers via one of the initial hooks. A +// concurrent allocation may, depending on timing either: +// * still have its initial malloc hook installed, run that and block on waiting +// for the first caller to finish its call to +// RemoveInitialHooksAndCallInitializers, and proceed normally. +// * occur some time during the RemoveInitialHooksAndCallInitializers call, at +// which point there could be no initial hooks and the subsequent hooks that +// are about to be set up by RemoveInitialHooksAndCallInitializers haven't +// been installed yet. I think the worst we can get is that some allocations +// will not get reported to some hooks set by the initializers called from +// RemoveInitialHooksAndCallInitializers. + +void InitialNewHook(const void* ptr, size_t size) { + perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); + MallocHook::InvokeNewHook(ptr, size); +} -ATTRIBUTE_WEAK -extern void InitialMallocHook_PreMMap(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); +void InitialPreMMapHook(const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset) { + perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); + MallocHook::InvokePreMmapHook(start, size, protection, flags, fd, offset); +} -ATTRIBUTE_WEAK -extern void InitialMallocHook_MMap(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); +void InitialPreSbrkHook(std::ptrdiff_t increment) { + perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers); + MallocHook::InvokePreSbrkHook(increment); +} -ATTRIBUTE_WEAK -extern void InitialMallocHook_PreSbrk(std::ptrdiff_t increment); +// This function is called at most once by one of the above initial malloc +// hooks. It removes all initial hooks and initializes all other clients that +// want to get control at the very first memory allocation. The initializers +// may assume that the initial malloc hooks have been removed. The initializers +// may set up malloc hooks and allocate memory. +void RemoveInitialHooksAndCallInitializers() { + RAW_CHECK(MallocHook::RemoveNewHook(&InitialNewHook), ""); + RAW_CHECK(MallocHook::RemovePreMmapHook(&InitialPreMMapHook), ""); + RAW_CHECK(MallocHook::RemovePreSbrkHook(&InitialPreSbrkHook), ""); + + // HeapLeakChecker is currently the only module that needs to get control on + // the first memory allocation, but one can add other modules by following the + // same weak/strong function pattern. + MallocHook_InitAtFirstAllocation_HeapLeakChecker(); +} + +} // namespace -ATTRIBUTE_WEAK -extern void InitialMallocHook_Sbrk(const void* result, std::ptrdiff_t increment); +// Weak default initialization function that must go after its use. +extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() { + // Do nothing. +} namespace base { namespace internal { + +// The code below is DEPRECATED. template<typename PtrT> PtrT AtomicPtr<PtrT>::Exchange(PtrT new_val) { base::subtle::MemoryBarrier(); // Release semantics. @@ -123,22 +174,139 @@ PtrT AtomicPtr<PtrT>::Exchange(PtrT new_val) { return old_val; } -AtomicPtr<MallocHook::NewHook> new_hook_ = { - reinterpret_cast<AtomicWord>(InitialMallocHook_New) }; +template<typename PtrT> +PtrT AtomicPtr<PtrT>::CompareAndSwap(PtrT old_val, PtrT new_val) { + base::subtle::MemoryBarrier(); // Release semantics. + PtrT retval = reinterpret_cast<PtrT>(static_cast<AtomicWord>( + base::subtle::NoBarrier_CompareAndSwap( + &data_, + reinterpret_cast<AtomicWord>(old_val), + reinterpret_cast<AtomicWord>(new_val)))); + base::subtle::MemoryBarrier(); // And acquire semantics. + return retval; +} + +AtomicPtr<MallocHook::NewHook> new_hook_ = { 0 }; AtomicPtr<MallocHook::DeleteHook> delete_hook_ = { 0 }; -AtomicPtr<MallocHook::PreMmapHook> premmap_hook_ = { - reinterpret_cast<AtomicWord>(InitialMallocHook_PreMMap) }; -AtomicPtr<MallocHook::MmapHook> mmap_hook_ = { - reinterpret_cast<AtomicWord>(InitialMallocHook_MMap) }; +AtomicPtr<MallocHook::PreMmapHook> premmap_hook_ = { 0 }; +AtomicPtr<MallocHook::MmapHook> mmap_hook_ = { 0 }; AtomicPtr<MallocHook::MunmapHook> munmap_hook_ = { 0 }; AtomicPtr<MallocHook::MremapHook> mremap_hook_ = { 0 }; -AtomicPtr<MallocHook::PreSbrkHook> presbrk_hook_ = { - reinterpret_cast<AtomicWord>(InitialMallocHook_PreSbrk) }; -AtomicPtr<MallocHook::SbrkHook> sbrk_hook_ = { - reinterpret_cast<AtomicWord>(InitialMallocHook_Sbrk) }; +AtomicPtr<MallocHook::PreSbrkHook> presbrk_hook_ = { 0 }; +AtomicPtr<MallocHook::SbrkHook> sbrk_hook_ = { 0 }; +// End of DEPRECATED code section. + +// This lock is shared between all implementations of HookList::Add & Remove. +// The potential for contention is very small. This needs to be a SpinLock and +// not a Mutex since it's possible for Mutex locking to allocate memory (e.g., +// per-thread allocation in debug builds), which could cause infinite recursion. +static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED); + +template <typename T> +bool HookList<T>::Add(T value_as_t) { + // Note: we need to check this _before_ reinterpret_cast, since + // reinterpret_cast may include random junk from memory. + if (value_as_t == 0) { + return false; + } + AtomicWord value = reinterpret_cast<const AtomicWord&>(value_as_t); + if (value == 0) { + // This should not actually happen, but just to be sure... + return false; + } + SpinLockHolder l(&hooklist_spinlock); + // Find the first slot in data that is 0. + int index = 0; + while ((index < kHookListMaxValues) && + (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) { + ++index; + } + if (index == kHookListMaxValues) { + return false; + } + AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end); + base::subtle::Release_Store(&priv_data[index], value); + if (prev_num_hooks <= index) { + base::subtle::Release_Store(&priv_end, index + 1); + } + return true; +} + +template <typename T> +bool HookList<T>::Remove(T value_as_t) { + if (value_as_t == 0) { + return false; + } + SpinLockHolder l(&hooklist_spinlock); + AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); + int index = 0; + // Note: we need to cast back to T since T may be smaller than AtomicWord. + while (index < hooks_end && value_as_t != reinterpret_cast<T>( + base::subtle::Acquire_Load(&priv_data[index]))) { + ++index; + } + if (index == hooks_end) { + return false; + } + base::subtle::Release_Store(&priv_data[index], 0); + if (hooks_end == index + 1) { + // Adjust hooks_end down to the lowest possible value. + hooks_end = index; + while ((hooks_end > 0) && + (base::subtle::Acquire_Load(&priv_data[hooks_end - 1]) == 0)) { + --hooks_end; + } + base::subtle::Release_Store(&priv_end, hooks_end); + } + return true; +} + +template <typename T> +int HookList<T>::Traverse(T* output_array, int n) const { + AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); + int actual_hooks_end = 0; + for (int i = 0; i < hooks_end && n > 0; ++i) { + AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]); + if (data != 0) { + *output_array++ = reinterpret_cast<const T&>(data); + ++actual_hooks_end; + --n; + } + } + return actual_hooks_end; +} + +// Initialize a HookList (optionally with the given initial_value in index 0). +#define INIT_HOOK_LIST { 0 } +#define INIT_HOOK_LIST_WITH_VALUE(initial_value) \ + { 1, { reinterpret_cast<AtomicWord>(initial_value) } } + +// Explicit instantiation for malloc_hook_test.cc. This ensures all the methods +// are instantiated. +template class HookList<MallocHook::NewHook>; + +HookList<MallocHook::NewHook> new_hooks_ = + INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook); +HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST; +HookList<MallocHook::PreMmapHook> premmap_hooks_ = + INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook); +HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST; +HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST; +HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST; +HookList<MallocHook::PreSbrkHook> presbrk_hooks_ = + INIT_HOOK_LIST_WITH_VALUE(InitialPreSbrkHook); +HookList<MallocHook::SbrkHook> sbrk_hooks_ = INIT_HOOK_LIST; + +// These lists contain either 0 or 1 hooks. +HookList<MallocHook::MmapReplacement> mmap_replacement_ = { 0 }; +HookList<MallocHook::MunmapReplacement> munmap_replacement_ = { 0 }; + +#undef INIT_HOOK_LIST_WITH_VALUE +#undef INIT_HOOK_LIST } } // namespace base::internal +// The code below is DEPRECATED. using base::internal::new_hook_; using base::internal::delete_hook_; using base::internal::premmap_hook_; @@ -147,109 +315,290 @@ using base::internal::munmap_hook_; using base::internal::mremap_hook_; using base::internal::presbrk_hook_; using base::internal::sbrk_hook_; - +// End of DEPRECATED code section. + +using base::internal::kHookListMaxValues; +using base::internal::new_hooks_; +using base::internal::delete_hooks_; +using base::internal::premmap_hooks_; +using base::internal::mmap_hooks_; +using base::internal::mmap_replacement_; +using base::internal::munmap_hooks_; +using base::internal::munmap_replacement_; +using base::internal::mremap_hooks_; +using base::internal::presbrk_hooks_; +using base::internal::sbrk_hooks_; // These are available as C bindings as well as C++, hence their // definition outside the MallocHook class. extern "C" +int MallocHook_AddNewHook(MallocHook_NewHook hook) { + RAW_VLOG(10, "AddNewHook(%p)", hook); + return new_hooks_.Add(hook); +} + +extern "C" +int MallocHook_RemoveNewHook(MallocHook_NewHook hook) { + RAW_VLOG(10, "RemoveNewHook(%p)", hook); + return new_hooks_.Remove(hook); +} + +extern "C" +int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook) { + RAW_VLOG(10, "AddDeleteHook(%p)", hook); + return delete_hooks_.Add(hook); +} + +extern "C" +int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook) { + RAW_VLOG(10, "RemoveDeleteHook(%p)", hook); + return delete_hooks_.Remove(hook); +} + +extern "C" +int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook) { + RAW_VLOG(10, "AddPreMmapHook(%p)", hook); + return premmap_hooks_.Add(hook); +} + +extern "C" +int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook) { + RAW_VLOG(10, "RemovePreMmapHook(%p)", hook); + return premmap_hooks_.Remove(hook); +} + +extern "C" +int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook) { + RAW_VLOG(10, "SetMmapReplacement(%p)", hook); + // NOTE this is a best effort CHECK. Concurrent sets could succeed since + // this test is outside of the Add spin lock. + RAW_CHECK(mmap_replacement_.empty(), "Only one MMapReplacement is allowed."); + return mmap_replacement_.Add(hook); +} + +extern "C" +int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook) { + RAW_VLOG(10, "RemoveMmapReplacement(%p)", hook); + return mmap_replacement_.Remove(hook); +} + +extern "C" +int MallocHook_AddMmapHook(MallocHook_MmapHook hook) { + RAW_VLOG(10, "AddMmapHook(%p)", hook); + return mmap_hooks_.Add(hook); +} + +extern "C" +int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook) { + RAW_VLOG(10, "RemoveMmapHook(%p)", hook); + return mmap_hooks_.Remove(hook); +} + +extern "C" +int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook) { + RAW_VLOG(10, "AddMunmapHook(%p)", hook); + return munmap_hooks_.Add(hook); +} + +extern "C" +int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook) { + RAW_VLOG(10, "RemoveMunmapHook(%p)", hook); + return munmap_hooks_.Remove(hook); +} + +extern "C" +int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook) { + RAW_VLOG(10, "SetMunmapReplacement(%p)", hook); + // NOTE this is a best effort CHECK. Concurrent sets could succeed since + // this test is outside of the Add spin lock. + RAW_CHECK(munmap_replacement_.empty(), + "Only one MunmapReplacement is allowed."); + return munmap_replacement_.Add(hook); +} + +extern "C" +int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook) { + RAW_VLOG(10, "RemoveMunmapReplacement(%p)", hook); + return munmap_replacement_.Remove(hook); +} + +extern "C" +int MallocHook_AddMremapHook(MallocHook_MremapHook hook) { + RAW_VLOG(10, "AddMremapHook(%p)", hook); + return mremap_hooks_.Add(hook); +} + +extern "C" +int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook) { + RAW_VLOG(10, "RemoveMremapHook(%p)", hook); + return mremap_hooks_.Remove(hook); +} + +extern "C" +int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook) { + RAW_VLOG(10, "AddPreSbrkHook(%p)", hook); + return presbrk_hooks_.Add(hook); +} + +extern "C" +int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook) { + RAW_VLOG(10, "RemovePreSbrkHook(%p)", hook); + return presbrk_hooks_.Remove(hook); +} + +extern "C" +int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook) { + RAW_VLOG(10, "AddSbrkHook(%p)", hook); + return sbrk_hooks_.Add(hook); +} + +extern "C" +int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook) { + RAW_VLOG(10, "RemoveSbrkHook(%p)", hook); + return sbrk_hooks_.Remove(hook); +} + +// The code below is DEPRECATED. +extern "C" MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook) { + RAW_VLOG(10, "SetNewHook(%p)", hook); return new_hook_.Exchange(hook); } extern "C" MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook) { + RAW_VLOG(10, "SetDeleteHook(%p)", hook); return delete_hook_.Exchange(hook); } extern "C" MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook) { + RAW_VLOG(10, "SetPreMmapHook(%p)", hook); return premmap_hook_.Exchange(hook); } extern "C" MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook) { + RAW_VLOG(10, "SetMmapHook(%p)", hook); return mmap_hook_.Exchange(hook); } extern "C" MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook) { + RAW_VLOG(10, "SetMunmapHook(%p)", hook); return munmap_hook_.Exchange(hook); } extern "C" MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook) { + RAW_VLOG(10, "SetMremapHook(%p)", hook); return mremap_hook_.Exchange(hook); } extern "C" MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook) { + RAW_VLOG(10, "SetPreSbrkHook(%p)", hook); return presbrk_hook_.Exchange(hook); } extern "C" MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook) { + RAW_VLOG(10, "SetSbrkHook(%p)", hook); return sbrk_hook_.Exchange(hook); } +// End of DEPRECATED code section. + +// Note: embedding the function calls inside the traversal of HookList would be +// very confusing, as it is legal for a hook to remove itself and add other +// hooks. Doing traversal first, and then calling the hooks ensures we only +// call the hooks registered at the start. +#define INVOKE_HOOKS(HookType, hook_list, args) do { \ + HookType hooks[kHookListMaxValues]; \ + int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \ + for (int i = 0; i < num_hooks; ++i) { \ + (*hooks[i])args; \ + } \ + } while (0) + +// There should only be one replacement. Return the result of the first +// one, or false if there is none. +#define INVOKE_REPLACEMENT(HookType, hook_list, args) do { \ + HookType hooks[kHookListMaxValues]; \ + int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \ + return (num_hooks > 0 && (*hooks[0])args); \ + } while (0) + + +void MallocHook::InvokeNewHookSlow(const void* p, size_t s) { + INVOKE_HOOKS(NewHook, new_hooks_, (p, s)); +} +void MallocHook::InvokeDeleteHookSlow(const void* p) { + INVOKE_HOOKS(DeleteHook, delete_hooks_, (p)); +} -// The definitions of weak default malloc hooks (New, MMap, and Sbrk) -// that self deinstall on their first call. This is entirely for -// efficiency: the default version of these functions will be called a -// maximum of one time. If these functions were a no-op instead, they'd -// be called every time, costing an extra function call per malloc. -// -// However, this 'delete self' isn't safe in general -- it's possible -// that this function will be called via a daisy chain. That is, -// someone else might do -// old_hook = MallocHook::SetNewHook(&myhook); -// void myhook(void* ptr, size_t size) { -// do_my_stuff(); -// old_hook(ptr, size); // daisy-chain the hooks -// } -// If old_hook is InitialMallocHook_New(), then this is broken code! -- -// after the first run it'll deregister not only InitialMallocHook_New() -// but also myhook. To protect against that, InitialMallocHook_New() -// makes sure it's the 'top-level' hook before doing the deregistration. -// This means the daisy-chain case will be less efficient because the -// hook will be called, and do an if check, for every new. Alas. -// TODO(csilvers): add support for removing a hook from the middle of a chain. - -void InitialMallocHook_New(const void* ptr, size_t size) { - if (MallocHook::GetNewHook() == &InitialMallocHook_New) - MallocHook::SetNewHook(NULL); -} - -void InitialMallocHook_PreMMap(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset) { - if (MallocHook::GetPreMmapHook() == &InitialMallocHook_PreMMap) - MallocHook::SetPreMmapHook(NULL); +void MallocHook::InvokePreMmapHookSlow(const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset) { + INVOKE_HOOKS(PreMmapHook, premmap_hooks_, (start, size, protection, flags, fd, + offset)); } -void InitialMallocHook_MMap(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset) { - if (MallocHook::GetMmapHook() == &InitialMallocHook_MMap) - MallocHook::SetMmapHook(NULL); +void MallocHook::InvokeMmapHookSlow(const void* result, + const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset) { + INVOKE_HOOKS(MmapHook, mmap_hooks_, (result, start, size, protection, flags, + fd, offset)); } -void InitialMallocHook_PreSbrk(std::ptrdiff_t increment) { - if (MallocHook::GetPreSbrkHook() == &InitialMallocHook_PreSbrk) - MallocHook::SetPreSbrkHook(NULL); +bool MallocHook::InvokeMmapReplacementSlow(const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset, + void** result) { + INVOKE_REPLACEMENT(MmapReplacement, mmap_replacement_, + (start, size, protection, flags, fd, offset, result)); +} + +void MallocHook::InvokeMunmapHookSlow(const void* p, size_t s) { + INVOKE_HOOKS(MunmapHook, munmap_hooks_, (p, s)); +} + +bool MallocHook::InvokeMunmapReplacementSlow(const void* p, + size_t s, + int* result) { + INVOKE_REPLACEMENT(MunmapReplacement, munmap_replacement_, (p, s, result)); +} + +void MallocHook::InvokeMremapHookSlow(const void* result, + const void* old_addr, + size_t old_size, + size_t new_size, + int flags, + const void* new_addr) { + INVOKE_HOOKS(MremapHook, mremap_hooks_, (result, old_addr, old_size, new_size, + flags, new_addr)); } -void InitialMallocHook_Sbrk(const void* result, std::ptrdiff_t increment) { - if (MallocHook::GetSbrkHook() == &InitialMallocHook_Sbrk) - MallocHook::SetSbrkHook(NULL); +void MallocHook::InvokePreSbrkHookSlow(std::ptrdiff_t increment) { + INVOKE_HOOKS(PreSbrkHook, presbrk_hooks_, (increment)); } +void MallocHook::InvokeSbrkHookSlow(const void* result, std::ptrdiff_t increment) { + INVOKE_HOOKS(SbrkHook, sbrk_hooks_, (result, increment)); +} + +#undef INVOKE_HOOKS + DEFINE_ATTRIBUTE_SECTION_VARS(google_malloc); DECLARE_ATTRIBUTE_SECTION_VARS(google_malloc); // actual functions are in debugallocation.cc or tcmalloc.cc @@ -398,7 +747,8 @@ static inline void* do_mmap64(void *start, size_t length, } result = (void *)syscall(SYS_mmap2, - start, length, prot, flags, fd, offset / pagesize); + start, length, prot, flags, fd, + (off_t) (offset / pagesize)); if (result != MAP_FAILED || errno != ENOSYS) goto out; // We don't have mmap2() after all - don't bother trying it in future @@ -456,7 +806,11 @@ extern "C" { extern "C" void* mmap64(void *start, size_t length, int prot, int flags, int fd, __off64_t offset) __THROW { MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); - void *result = do_mmap64(start, length, prot, flags, fd, offset); + void *result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = do_mmap64(start, length, prot, flags, fd, offset); + } MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); return result; } @@ -466,8 +820,12 @@ extern "C" void* mmap64(void *start, size_t length, int prot, int flags, extern "C" void* mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) __THROW { MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); - void *result = do_mmap64(start, length, prot, flags, fd, - static_cast<size_t>(offset)); // avoid sign extension + void *result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = do_mmap64(start, length, prot, flags, fd, + static_cast<size_t>(offset)); // avoid sign extension + } MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); return result; } @@ -476,7 +834,11 @@ extern "C" void* mmap(void *start, size_t length, int prot, int flags, extern "C" int munmap(void* start, size_t length) __THROW { MallocHook::InvokeMunmapHook(start, length); - return syscall(SYS_munmap, start, length); + int result; + if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { + result = syscall(SYS_munmap, start, length); + } + return result; } extern "C" void* mremap(void* old_addr, size_t old_size, size_t new_size, @@ -503,11 +865,20 @@ extern "C" void* sbrk(std::ptrdiff_t increment) __THROW { /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, int flags, int fd, off_t offset) { - return do_mmap64(start, length, prot, flags, fd, offset); + void* result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = do_mmap64(start, length, prot, flags, fd, offset); + } + return result; } /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { - return sys_munmap(start, length); + int result; + if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { + result = sys_munmap(start, length); + } + return result; } #else // defined(__linux) && @@ -515,11 +886,20 @@ extern "C" void* sbrk(std::ptrdiff_t increment) __THROW { /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, int flags, int fd, off_t offset) { - return mmap(start, length, prot, flags, fd, offset); + void* result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = mmap(start, length, prot, flags, fd, offset); + } + return result; } /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { - return munmap(start, length); + int result; + if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { + result = munmap(start, length); + } + return result; } #endif // defined(__linux) && |