diff options
| author | dmikurube@google.com <dmikurube@google.com@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-03-03 07:15:16 +0000 |
|---|---|---|
| committer | dmikurube@google.com <dmikurube@google.com@0039d316-1c4b-4281-b951-d872f2087c98> | 2012-03-03 07:15:16 +0000 |
| commit | d2a95102b1c460746ce64c3ad47442cf2bda0f5e (patch) | |
| tree | 0773b435ceddc44135c8f2475edc316eeabc0b48 /third_party/tcmalloc | |
| parent | 9cf1c2d4f2285402a744d30d3761d7284e5d367d (diff) | |
| download | chromium_src-d2a95102b1c460746ce64c3ad47442cf2bda0f5e.zip chromium_src-d2a95102b1c460746ce64c3ad47442cf2bda0f5e.tar.gz chromium_src-d2a95102b1c460746ce64c3ad47442cf2bda0f5e.tar.bz2 | |
Update the tcmalloc chromium branch to r144 (gperftools 2.0), and merge chromium-specific changes.
The major reason for us is to enable a fix for HEAP_PROFILE_MMAP.
This change merges tcmalloc r144 (gperftools 2.0 f.k.a. google-perftools) with the tcmalloc/chromium branch, which is the forked Chromium version of tcmalloc.
The change is created by 3-way merge from
1) the original google-perftools r109 ( = vendor base),
2) the original gperftools 2.0 r144 ( = branch), and
3) the chromium branch ( = another branch)
with the following additional changes :
* base/allocator/allocator.gyp is modified.
* Many [#include "third_party/tcmalloc/chromium/src/google/..."] are replaced with "gperftools/". (Many files in Chromium)
* gperftools/tcmalloc.h (formerly google/tcmalloc.h) is replaced with the original (generated) one.
* windows/gperftools/tcmalloc.h (formerly windows/google/tcmalloc.h) is replaced with the original (generated) one.
* malloc_hook-like functions are moved to libc_override*.h in gperftools 2.0. Some changes due to it.
* MALLOC_HOOK_MAYBE_VOLATILE is redefined using __MALLOC_HOOK_VOLATILE. (config.h, tcmalloc.cc and libc_override_glibc.h)
* The macro "CRASH(...)" is replaced with "Log(kCrash, __FILE__, __LINE__, ...)". (Many files)
* LARGE_PAGE-related parameters (which may affect performance?) are merged. (common.h)
* RAW_VLOG() calls are removed. (base/googleinit.h)
* sys_{mmap|munmap|mremap}(...) calls are tentatively replaced with syscall(SYS_{mmap|munmap|mremap}, ...). (malloc_hook_mmap_linux.h)
* tc_mallinfo is declared only when HAVE_STRUCT_MALLINFO is defined. (gperftools/tcmalloc.h)
* "libc_override_redefine.h" is not included in Windows. (libc_override.h)
* Chromium-original "sys_alloc" is not declared. (windows/port.cc)
* base/spinlock_win32-inl.h is reverted from r144 because 64-bit atomicops are not implemented on Windows. (base/atomicops-internals-windows.h)
The vendor branch is updated in another change.
BUG=114302
TEST=run all existing tests.
Review URL: https://chromiumcodereview.appspot.com/9311003
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@124832 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'third_party/tcmalloc')
136 files changed, 8616 insertions, 5304 deletions
diff --git a/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-gcc.h b/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-gcc.h deleted file mode 100644 index 423e993..0000000 --- a/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-gcc.h +++ /dev/null @@ -1,234 +0,0 @@ -/* Copyright (c) 2010, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Lei Zhang, Sasha Levitskiy - */ - -// This file is an internal atomic implementation, use base/atomicops.h instead. -// -// LinuxKernelCmpxchg and Barrier_AtomicIncrement are from Google Gears. - -#ifndef BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ -#define BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ - -#include <stdio.h> -#include "base/basictypes.h" // For COMPILE_ASSERT - -typedef int32_t Atomic32; - -namespace base { -namespace subtle { - -typedef int64_t Atomic64; - -// 0xffff0fc0 is the hard coded address of a function provided by -// the kernel which implements an atomic compare-exchange. On older -// ARM architecture revisions (pre-v6) this may be implemented using -// a syscall. This address is stable, and in active use (hard coded) -// by at least glibc-2.7 and the Android C library. -// pLinuxKernelCmpxchg has both acquire and release barrier sematincs. -typedef Atomic32 (*LinuxKernelCmpxchgFunc)(Atomic32 old_value, - Atomic32 new_value, - volatile Atomic32* ptr); -LinuxKernelCmpxchgFunc pLinuxKernelCmpxchg __attribute__((weak)) = - (LinuxKernelCmpxchgFunc) 0xffff0fc0; - -typedef void (*LinuxKernelMemoryBarrierFunc)(void); -LinuxKernelMemoryBarrierFunc pLinuxKernelMemoryBarrier __attribute__((weak)) = - (LinuxKernelMemoryBarrierFunc) 0xffff0fa0; - - -inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - Atomic32 prev_value = *ptr; - do { - if (!pLinuxKernelCmpxchg(old_value, new_value, - const_cast<Atomic32*>(ptr))) { - return old_value; - } - prev_value = *ptr; - } while (prev_value == old_value); - return prev_value; -} - -inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, - Atomic32 new_value) { - Atomic32 old_value; - do { - old_value = *ptr; - } while (pLinuxKernelCmpxchg(old_value, new_value, - const_cast<Atomic32*>(ptr))); - return old_value; -} - -inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - for (;;) { - // Atomic exchange the old value with an incremented one. - Atomic32 old_value = *ptr; - Atomic32 new_value = old_value + increment; - if (pLinuxKernelCmpxchg(old_value, new_value, - const_cast<Atomic32*>(ptr)) == 0) { - // The exchange took place as expected. - return new_value; - } - // Otherwise, *ptr changed mid-loop and we need to retry. - } -} - -inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, - Atomic32 increment) { - return Barrier_AtomicIncrement(ptr, increment); -} - -inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, - Atomic32 old_value, - Atomic32 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; -} - -inline void MemoryBarrier() { - pLinuxKernelMemoryBarrier(); -} - -inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { - *ptr = value; - MemoryBarrier(); -} - -inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { - MemoryBarrier(); - *ptr = value; -} - -inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { - return *ptr; -} - -inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { - Atomic32 value = *ptr; - MemoryBarrier(); - return value; -} - -inline Atomic32 Release_Load(volatile const Atomic32* ptr) { - MemoryBarrier(); - return *ptr; -} - - -// 64-bit versions are not implemented yet. - -inline void NotImplementedFatalError(const char *function_name) { - fprintf(stderr, "64-bit %s() not implemented on this platform\n", - function_name); - abort(); -} - -inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_CompareAndSwap"); - return 0; -} - -inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, - Atomic64 new_value) { - NotImplementedFatalError("NoBarrier_AtomicExchange"); - return 0; -} - -inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - NotImplementedFatalError("NoBarrier_AtomicIncrement"); - return 0; -} - -inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, - Atomic64 increment) { - NotImplementedFatalError("Barrier_AtomicIncrement"); - return 0; -} - -inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("NoBarrier_Store"); -} - -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NoBarrier_AtomicExchange(ptr, value); - // acts as a barrier in this implementation -} - -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Release_Store"); -} - -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("NoBarrier_Load"); - return 0; -} - -inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - Atomic64 value = NoBarrier_Load(ptr); - return value; -} - -inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - MemoryBarrier(); - return NoBarrier_Load(ptr); -} - -inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, - Atomic64 old_value, - Atomic64 new_value) { - return NoBarrier_CompareAndSwap(ptr, old_value, new_value); -} - -} // namespace base::subtle -} // namespace base - -#endif // BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ diff --git a/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-generic.h b/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-generic.h index 7882b0d..4acb76a 100644 --- a/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-generic.h +++ b/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-generic.h @@ -39,8 +39,7 @@ #include <stdio.h> #include <stdlib.h> -#include "base/macros.h" // For COMPILE_ASSERT -#include "base/port.h" // ATTRIBUTE_WEAK +#include "base/basictypes.h" typedef int32_t Atomic32; diff --git a/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h b/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h index ee09f32..8d5b9b5 100644 --- a/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h +++ b/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h @@ -42,6 +42,13 @@ #include <stdlib.h> #include "base/basictypes.h" // For COMPILE_ASSERT +// The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, +// only some variants support it. For simplicity, we only use exclusive +// 64-bit load/store in V7 or above. +#if defined(ARMV7) +# define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD +#endif + typedef int32_t Atomic32; namespace base { @@ -60,6 +67,10 @@ inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, "ldrex %1, [%3]\n" "mov %0, #0\n" "teq %1, %4\n" + // The following IT (if-then) instruction is needed for the subsequent + // conditional instruction STREXEQ when compiling in THUMB mode. + // In ARM mode, the compiler/assembler will not generate any code for it. + "it eq\n" "strexeq %0, %5, [%3]\n" : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) : "r" (ptr), "Ir" (old_value), "r" (new_value) @@ -164,7 +175,114 @@ inline Atomic32 Release_Load(volatile const Atomic32* ptr) { return *ptr; } -// 64-bit versions are not implemented yet. +// 64-bit versions are only available if LDREXD and STREXD instructions +// are available. +#ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD + +#define BASE_HAS_ATOMIC64 1 + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + Atomic64 oldval, res; + do { + __asm__ __volatile__( + "ldrexd %1, [%3]\n" + "mov %0, #0\n" + "teq %Q1, %Q4\n" + // The following IT (if-then) instructions are needed for the subsequent + // conditional instructions when compiling in THUMB mode. + // In ARM mode, the compiler/assembler will not generate any code for it. + "it eq\n" + "teqeq %R1, %R4\n" + "it eq\n" + "strexdeq %0, %5, [%3]\n" + : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) + : "r" (ptr), "Ir" (old_value), "r" (new_value) + : "cc"); + } while (res); + return oldval; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + int store_failed; + Atomic64 old; + __asm__ __volatile__( + "1:\n" + "ldrexd %1, [%2]\n" + "strexd %0, %3, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (store_failed), "=&r" (old) + : "r" (ptr), "r" (new_value) + : "cc", "memory"); + return old; +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + int store_failed; + Atomic64 res; + __asm__ __volatile__( + "1:\n" + "ldrexd %1, [%2]\n" + "adds %Q1, %Q1, %Q3\n" + "adc %R1, %R1, %R3\n" + "strexd %0, %1, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (store_failed), "=&r"(res) + : "r" (ptr), "r"(increment) + : "cc", "memory"); + return res; +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + int store_failed; + Atomic64 res; + __asm__ __volatile__( + "1:\n" + "ldrexd %1, [%2]\n" + "adds %Q1, %Q1, %Q3\n" + "adc %R1, %R1, %R3\n" + "dmb\n" + "strexd %0, %1, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (store_failed), "=&r"(res) + : "r" (ptr), "r"(increment) + : "cc", "memory"); + return res; +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + int store_failed; + Atomic64 dummy; + __asm__ __volatile__( + "1:\n" + // Dummy load to lock cache line. + "ldrexd %1, [%3]\n" + "strexd %0, %2, [%3]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (store_failed), "=&r"(dummy) + : "r"(value), "r" (ptr) + : "cc", "memory"); +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + Atomic64 res; + __asm__ __volatile__( + "ldrexd %0, [%1]\n" + "clrex\n" + : "=r" (res) + : "r"(ptr), "Q"(*ptr)); + return res; +} + +#else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD inline void NotImplementedFatalError(const char *function_name) { fprintf(stderr, "64-bit %s() not implemented on this platform\n", @@ -201,41 +319,47 @@ inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { NotImplementedFatalError("NoBarrier_Store"); } -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Acquire_Store64"); +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("NoBarrier_Load"); + return 0; } -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Release_Store"); +#endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + NoBarrier_Store(ptr, value); + MemoryBarrier(); } -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("NoBarrier_Load"); - return 0; +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + MemoryBarrier(); + NoBarrier_Store(ptr, value); } inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("Atomic64 Acquire_Load"); - return 0; + Atomic64 value = NoBarrier_Load(ptr); + MemoryBarrier(); + return value; } inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("Atomic64 Release_Load"); - return 0; + MemoryBarrier(); + return NoBarrier_Load(ptr); } inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, Atomic64 old_value, Atomic64 new_value) { - NotImplementedFatalError("Atomic64 Acquire_CompareAndSwap"); - return 0; + Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); + MemoryBarrier(); + return value; } inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, Atomic64 old_value, Atomic64 new_value) { - NotImplementedFatalError("Atomic64 Release_CompareAndSwap"); - return 0; + MemoryBarrier(); + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); } } // namespace subtle ends diff --git a/third_party/tcmalloc/chromium/src/base/atomicops-internals-windows.h b/third_party/tcmalloc/chromium/src/base/atomicops-internals-windows.h index 58782a17..bd42c82 100644 --- a/third_party/tcmalloc/chromium/src/base/atomicops-internals-windows.h +++ b/third_party/tcmalloc/chromium/src/base/atomicops-internals-windows.h @@ -55,28 +55,74 @@ typedef int64 Atomic64; // 32-bit low-level operations on any platform +extern "C" { +// We use windows intrinsics when we can (they seem to be supported +// well on MSVC 8.0 and above). Unfortunately, in some +// environments, <windows.h> and <intrin.h> have conflicting +// declarations of some other intrinsics, breaking compilation: +// http://connect.microsoft.com/VisualStudio/feedback/details/262047 +// Therefore, we simply declare the relevant intrinsics ourself. + // MinGW has a bug in the header files where it doesn't indicate the // first argument is volatile -- they're not up to date. See // http://readlist.com/lists/lists.sourceforge.net/mingw-users/0/3861.html // We have to const_cast away the volatile to avoid compiler warnings. // TODO(csilvers): remove this once MinGW has updated MinGW/include/winbase.h -#ifdef __MINGW32__ -inline LONG InterlockedCompareExchange(volatile LONG* ptr, - LONG newval, LONG oldval) { +#if defined(__MINGW32__) +inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, + LONG newval, LONG oldval) { return ::InterlockedCompareExchange(const_cast<LONG*>(ptr), newval, oldval); } -inline LONG InterlockedExchange(volatile LONG* ptr, LONG newval) { +inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { return ::InterlockedExchange(const_cast<LONG*>(ptr), newval); } -inline LONG InterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { +inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { return ::InterlockedExchangeAdd(const_cast<LONG*>(ptr), increment); } + +#elif _MSC_VER >= 1400 // intrinsics didn't work so well before MSVC 8.0 +// Unfortunately, in some environments, <windows.h> and <intrin.h> +// have conflicting declarations of some intrinsics, breaking +// compilation. So we declare the intrinsics we need ourselves. See +// http://connect.microsoft.com/VisualStudio/feedback/details/262047 +LONG _InterlockedCompareExchange(volatile LONG* ptr, LONG newval, LONG oldval); +#pragma intrinsic(_InterlockedCompareExchange) +inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, + LONG newval, LONG oldval) { + return _InterlockedCompareExchange(ptr, newval, oldval); +} + +LONG _InterlockedExchange(volatile LONG* ptr, LONG newval); +#pragma intrinsic(_InterlockedExchange) +inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { + return _InterlockedExchange(ptr, newval); +} + +LONG _InterlockedExchangeAdd(volatile LONG* ptr, LONG increment); +#pragma intrinsic(_InterlockedExchangeAdd) +inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { + return _InterlockedExchangeAdd(ptr, increment); +} + +#else +inline LONG FastInterlockedCompareExchange(volatile LONG* ptr, + LONG newval, LONG oldval) { + return ::InterlockedCompareExchange(ptr, newval, oldval); +} +inline LONG FastInterlockedExchange(volatile LONG* ptr, LONG newval) { + return ::InterlockedExchange(ptr, newval); +} +inline LONG FastInterlockedExchangeAdd(volatile LONG* ptr, LONG increment) { + return ::InterlockedExchangeAdd(ptr, increment); +} + #endif // ifdef __MINGW32__ +} // extern "C" inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, Atomic32 old_value, Atomic32 new_value) { - LONG result = InterlockedCompareExchange( + LONG result = FastInterlockedCompareExchange( reinterpret_cast<volatile LONG*>(ptr), static_cast<LONG>(new_value), static_cast<LONG>(old_value)); @@ -85,7 +131,7 @@ inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, Atomic32 new_value) { - LONG result = InterlockedExchange( + LONG result = FastInterlockedExchange( reinterpret_cast<volatile LONG*>(ptr), static_cast<LONG>(new_value)); return static_cast<Atomic32>(result); @@ -93,7 +139,7 @@ inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, Atomic32 increment) { - return InterlockedExchangeAdd( + return FastInterlockedExchangeAdd( reinterpret_cast<volatile LONG*>(ptr), static_cast<LONG>(increment)) + increment; } @@ -173,27 +219,68 @@ inline Atomic32 Release_Load(volatile const Atomic32* ptr) { COMPILE_ASSERT(sizeof(Atomic64) == sizeof(PVOID), atomic_word_is_atomic); -// Like for the __MINGW32__ case above, this works around a header -// error in mingw, where it's missing 'volatile'. -#ifdef __MINGW64__ -inline PVOID InterlockedCompareExchangePointer(volatile PVOID* ptr, - PVOID newval, PVOID oldval) { +// These are the intrinsics needed for 64-bit operations. Similar to the +// 32-bit case above. + +extern "C" { +#if defined(__MINGW64__) +inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, + PVOID newval, PVOID oldval) { return ::InterlockedCompareExchangePointer(const_cast<PVOID*>(ptr), newval, oldval); } -inline PVOID InterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { +inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { return ::InterlockedExchangePointer(const_cast<PVOID*>(ptr), newval); } -inline LONGLONG InterlockedExchangeAdd64(volatile LONGLONG* ptr, - LONGLONG increment) { +inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, + LONGLONG increment) { return ::InterlockedExchangeAdd64(const_cast<LONGLONG*>(ptr), increment); } + +#elif _MSC_VER >= 1400 // intrinsics didn't work so well before MSVC 8.0 +// Like above, we need to declare the intrinsics ourselves. +PVOID _InterlockedCompareExchangePointer(volatile PVOID* ptr, + PVOID newval, PVOID oldval); +#pragma intrinsic(_InterlockedCompareExchangePointer) +inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, + PVOID newval, PVOID oldval) { + return _InterlockedCompareExchangePointer(const_cast<PVOID*>(ptr), + newval, oldval); +} + +PVOID _InterlockedExchangePointer(volatile PVOID* ptr, PVOID newval); +#pragma intrinsic(_InterlockedExchangePointer) +inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { + return _InterlockedExchangePointer(const_cast<PVOID*>(ptr), newval); +} + +LONGLONG _InterlockedExchangeAdd64(volatile LONGLONG* ptr, LONGLONG increment); +#pragma intrinsic(_InterlockedExchangeAdd64) +inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, + LONGLONG increment) { + return _InterlockedExchangeAdd64(const_cast<LONGLONG*>(ptr), increment); +} + +#else +inline PVOID FastInterlockedCompareExchangePointer(volatile PVOID* ptr, + PVOID newval, PVOID oldval) { + return ::InterlockedCompareExchangePointer(ptr, newval, oldval); +} +inline PVOID FastInterlockedExchangePointer(volatile PVOID* ptr, PVOID newval) { + return ::InterlockedExchangePointer(ptr, newval); +} +inline LONGLONG FastInterlockedExchangeAdd64(volatile LONGLONG* ptr, + LONGLONG increment) { + return ::InterlockedExchangeAdd64(ptr, increment); +} + #endif // ifdef __MINGW64__ +} // extern "C" inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, Atomic64 old_value, Atomic64 new_value) { - PVOID result = InterlockedCompareExchangePointer( + PVOID result = FastInterlockedCompareExchangePointer( reinterpret_cast<volatile PVOID*>(ptr), reinterpret_cast<PVOID>(new_value), reinterpret_cast<PVOID>(old_value)); return reinterpret_cast<Atomic64>(result); @@ -201,7 +288,7 @@ inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, Atomic64 new_value) { - PVOID result = InterlockedExchangePointer( + PVOID result = FastInterlockedExchangePointer( reinterpret_cast<volatile PVOID*>(ptr), reinterpret_cast<PVOID>(new_value)); return reinterpret_cast<Atomic64>(result); @@ -209,7 +296,7 @@ inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, Atomic64 increment) { - return InterlockedExchangeAdd64( + return FastInterlockedExchangeAdd64( reinterpret_cast<volatile LONGLONG*>(ptr), static_cast<LONGLONG>(increment)) + increment; } @@ -258,7 +345,7 @@ inline Atomic64 Release_Load(volatile const Atomic64* ptr) { // 64-bit low-level operations on 32-bit platform // TODO(vchen): The GNU assembly below must be converted to MSVC inline -// assembly. Then the file should be renamed to ...-x86-mscv.h, probably. +// assembly. Then the file should be renamed to ...-x86-msvc.h, probably. inline void NotImplementedFatalError(const char *function_name) { fprintf(stderr, "64-bit %s() not implemented on this platform\n", diff --git a/third_party/tcmalloc/chromium/src/base/basictypes.h b/third_party/tcmalloc/chromium/src/base/basictypes.h index 0f21fca..75b7b5a 100644 --- a/third_party/tcmalloc/chromium/src/base/basictypes.h +++ b/third_party/tcmalloc/chromium/src/base/basictypes.h @@ -31,6 +31,7 @@ #define _BASICTYPES_H_ #include <config.h> +#include <string.h> // for memcpy() #ifdef HAVE_INTTYPES_H #include <inttypes.h> // gets us PRId64, etc #endif @@ -193,6 +194,28 @@ struct CompileAssert { (reinterpret_cast<char*>(&reinterpret_cast<strct*>(16)->field) - \ reinterpret_cast<char*>(16)) +// bit_cast<Dest,Source> implements the equivalent of +// "*reinterpret_cast<Dest*>(&source)". +// +// The reinterpret_cast method would produce undefined behavior +// according to ISO C++ specification section 3.10 -15 -. +// bit_cast<> calls memcpy() which is blessed by the standard, +// especially by the example in section 3.9. +// +// Fortunately memcpy() is very fast. In optimized mode, with a +// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline +// code with the minimal amount of data movement. On a 32-bit system, +// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8) +// compiles to two loads and two stores. + +template <class Dest, class Source> +inline Dest bit_cast(const Source& source) { + COMPILE_ASSERT(sizeof(Dest) == sizeof(Source), bitcasting_unequal_sizes); + Dest dest; + memcpy(&dest, &source, sizeof(dest)); + return dest; +} + #ifdef HAVE___ATTRIBUTE__ # define ATTRIBUTE_WEAK __attribute__((weak)) # define ATTRIBUTE_NOINLINE __attribute__((noinline)) @@ -309,8 +332,7 @@ class AssignAttributeStartEnd { #endif // HAVE___ATTRIBUTE__ and __ELF__ or __MACH__ #if defined(HAVE___ATTRIBUTE__) && (defined(__i386__) || defined(__x86_64__)) -# define CACHELINE_SIZE 64 -# define CACHELINE_ALIGNED __attribute__((aligned(CACHELINE_SIZE))) +# define CACHELINE_ALIGNED __attribute__((aligned(64))) #else # define CACHELINE_ALIGNED #endif // defined(HAVE___ATTRIBUTE__) && (__i386__ || __x86_64__) diff --git a/third_party/tcmalloc/chromium/src/base/cycleclock.h b/third_party/tcmalloc/chromium/src/base/cycleclock.h index 6d6822a..0ce1638 100644 --- a/third_party/tcmalloc/chromium/src/base/cycleclock.h +++ b/third_party/tcmalloc/chromium/src/base/cycleclock.h @@ -47,21 +47,30 @@ #include "base/basictypes.h" // make sure we get the def for int64 #include "base/arm_instruction_set_select.h" +// base/sysinfo.h is really big and we don't want to include it unless +// it is necessary. +#if defined(__arm__) || defined(__mips__) +# include "base/sysinfo.h" +#endif #if defined(__MACH__) && defined(__APPLE__) # include <mach/mach_time.h> #endif -// For MSVC, we want the __rdtsc intrinsic, declared in <intrin.h>. -// Unfortunately, in some environments, <windows.h> and <intrin.h> have -// conflicting declarations of some other intrinsics, breaking compilation. +// For MSVC, we want to use '_asm rdtsc' when possible (since it works +// with even ancient MSVC compilers), and when not possible the +// __rdtsc intrinsic, declared in <intrin.h>. Unfortunately, in some +// environments, <windows.h> and <intrin.h> have conflicting +// declarations of some other intrinsics, breaking compilation. // Therefore, we simply declare __rdtsc ourselves. See also // http://connect.microsoft.com/VisualStudio/feedback/details/262047 -#if defined(_MSC_VER) +#if defined(_MSC_VER) && !defined(_M_IX86) extern "C" uint64 __rdtsc(); #pragma intrinsic(__rdtsc) #endif +#if defined(ARMV3) || defined(__mips__) #ifdef HAVE_SYS_TIME_H #include <sys/time.h> #endif +#endif // NOTE: only i386 and x86_64 have been well tested. // PPC, sparc, alpha, and ia64 are based on @@ -108,6 +117,12 @@ struct CycleClock { int64 itc; asm("mov %0 = ar.itc" : "=r" (itc)); return itc; +#elif defined(_MSC_VER) && defined(_M_IX86) + // Older MSVC compilers (like 7.x) don't seem to support the + // __rdtsc intrinsic properly, so I prefer to use _asm instead + // when I know it will work. Otherwise, I'll use __rdtsc and hope + // the code is being compiled with a non-ancient compiler. + _asm rdtsc #elif defined(_MSC_VER) return __rdtsc(); #elif defined(ARMV3) @@ -116,11 +131,11 @@ struct CycleClock { uint32 pmuseren; uint32 pmcntenset; // Read the user mode perf monitor counter access permissions. - asm("mrc p15, 0, %0, c9, c14, 0" : "=r" (pmuseren)); + asm volatile ("mrc p15, 0, %0, c9, c14, 0" : "=r" (pmuseren)); if (pmuseren & 1) { // Allows reading perfmon counters for user mode code. - asm("mrc p15, 0, %0, c9, c12, 1" : "=r" (pmcntenset)); + asm volatile ("mrc p15, 0, %0, c9, c12, 1" : "=r" (pmcntenset)); if (pmcntenset & 0x80000000ul) { // Is it counting? - asm("mrc p15, 0, %0, c9, c13, 0" : "=r" (pmccntr)); + asm volatile ("mrc p15, 0, %0, c9, c13, 0" : "=r" (pmccntr)); // The counter is set up to count every 64th cycle return static_cast<int64>(pmccntr) * 64; // Should optimize to << 6 } @@ -128,7 +143,15 @@ struct CycleClock { #endif struct timeval tv; gettimeofday(&tv, NULL); - return static_cast<int64>(tv.tv_sec) * 1000000 + tv.tv_usec; + return static_cast<int64>((tv.tv_sec + tv.tv_usec * 0.000001) + * CyclesPerSecond()); +#elif defined(__mips__) + // mips apparently only allows rdtsc for superusers, so we fall + // back to gettimeofday. It's possible clock_gettime would be better. + struct timeval tv; + gettimeofday(&tv, NULL); + return static_cast<int64>((tv.tv_sec + tv.tv_usec * 0.000001) + * CyclesPerSecond()); #else // The soft failover to a generic implementation is automatic only for ARM. // For other platforms the developer is expected to make an attempt to create diff --git a/third_party/tcmalloc/chromium/src/base/dynamic_annotations.c b/third_party/tcmalloc/chromium/src/base/dynamic_annotations.c index 1005f90..c8b61be 100644 --- a/third_party/tcmalloc/chromium/src/base/dynamic_annotations.c +++ b/third_party/tcmalloc/chromium/src/base/dynamic_annotations.c @@ -50,10 +50,19 @@ # endif #endif +/* Compiler-based ThreadSanitizer defines + DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL = 1 + and provides its own definitions of the functions. */ + +#ifndef DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL +# define DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL 0 +#endif + /* Each function is empty and called (via a macro) only in debug mode. The arguments are captured by dynamic tools at runtime. */ -#if DYNAMIC_ANNOTATIONS_ENABLED == 1 +#if DYNAMIC_ANNOTATIONS_ENABLED == 1 \ + && DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 void AnnotateRWLockCreate(const char *file, int line, const volatile void *lock){} @@ -122,7 +131,10 @@ void AnnotateNoOp(const char *file, int line, const volatile void *arg){} void AnnotateFlushState(const char *file, int line){} -#endif /* DYNAMIC_ANNOTATIONS_ENABLED == 1 */ +#endif /* DYNAMIC_ANNOTATIONS_ENABLED == 1 + && DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 */ + +#if DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 static int GetRunningOnValgrind(void) { #ifdef RUNNING_ON_VALGRIND @@ -159,6 +171,8 @@ int RunningOnValgrind(void) { return local_running_on_valgrind; } +#endif /* DYNAMIC_ANNOTATIONS_EXTERNAL_IMPL == 0 */ + /* See the comments in dynamic_annotations.h */ double ValgrindSlowdown(void) { /* Same initialization hack as in RunningOnValgrind(). */ diff --git a/third_party/tcmalloc/chromium/src/base/dynamic_annotations.h b/third_party/tcmalloc/chromium/src/base/dynamic_annotations.h index 811bb5e..4669315 100644 --- a/third_party/tcmalloc/chromium/src/base/dynamic_annotations.h +++ b/third_party/tcmalloc/chromium/src/base/dynamic_annotations.h @@ -378,9 +378,14 @@ #define ANNOTALYSIS_STATIC_INLINE #define ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY ; +#define ANNOTALYSIS_IGNORE_READS_BEGIN +#define ANNOTALYSIS_IGNORE_READS_END +#define ANNOTALYSIS_IGNORE_WRITES_BEGIN +#define ANNOTALYSIS_IGNORE_WRITES_END +#define ANNOTALYSIS_UNPROTECTED_READ -#if defined(__GNUC__) && defined(__SUPPORT_TS_ANNOTATION__) \ - && (!defined(SWIG)) && defined(__SUPPORT_DYN_ANNOTATION__) +#if defined(__GNUC__) && (!defined(SWIG)) && (!defined(__clang__)) && \ + defined(__SUPPORT_TS_ANNOTATION__) && defined(__SUPPORT_DYN_ANNOTATION__) #if DYNAMIC_ANNOTATIONS_ENABLED == 0 #define ANNOTALYSIS_ONLY 1 @@ -389,22 +394,23 @@ #undef ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY #define ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY { (void)file; (void)line; } #endif -#define ANNOTALYSIS_IGNORE_READS_BEGIN __attribute__ ((ignore_reads_begin)) -#define ANNOTALYSIS_IGNORE_READS_END __attribute__ ((ignore_reads_end)) -#define ANNOTALYSIS_IGNORE_WRITES_BEGIN __attribute__ ((ignore_writes_begin)) -#define ANNOTALYSIS_IGNORE_WRITES_END __attribute__ ((ignore_writes_end)) -#define ANNOTALYSIS_UNPROTECTED_READ __attribute__ ((unprotected_read)) - -#else - -#define ANNOTALYSIS_IGNORE_READS_BEGIN -#define ANNOTALYSIS_IGNORE_READS_END -#define ANNOTALYSIS_IGNORE_WRITES_BEGIN -#define ANNOTALYSIS_IGNORE_WRITES_END -#define ANNOTALYSIS_UNPROTECTED_READ +/* Only emit attributes when annotalysis is enabled. */ +#if defined(__SUPPORT_TS_ANNOTATION__) && defined(__SUPPORT_DYN_ANNOTATION__) +#undef ANNOTALYSIS_IGNORE_READS_BEGIN +#define ANNOTALYSIS_IGNORE_READS_BEGIN __attribute__ ((ignore_reads_begin)) +#undef ANNOTALYSIS_IGNORE_READS_END +#define ANNOTALYSIS_IGNORE_READS_END __attribute__ ((ignore_reads_end)) +#undef ANNOTALYSIS_IGNORE_WRITES_BEGIN +#define ANNOTALYSIS_IGNORE_WRITES_BEGIN __attribute__ ((ignore_writes_begin)) +#undef ANNOTALYSIS_IGNORE_WRITES_END +#define ANNOTALYSIS_IGNORE_WRITES_END __attribute__ ((ignore_writes_end)) +#undef ANNOTALYSIS_UNPROTECTED_READ +#define ANNOTALYSIS_UNPROTECTED_READ __attribute__ ((unprotected_read)) #endif +#endif // defined(__GNUC__) && (!defined(SWIG)) && (!defined(__clang__)) + /* Use the macros above rather than using these functions directly. */ #ifdef __cplusplus extern "C" { @@ -604,7 +610,7 @@ double ValgrindSlowdown(void); #undef ANNOTATE_UNPROTECTED_READ template <class T> inline T ANNOTATE_UNPROTECTED_READ(const volatile T &x) - __attribute__ ((unprotected_read)) { + ANNOTALYSIS_UNPROTECTED_READ { ANNOTATE_IGNORE_READS_BEGIN(); T res = x; ANNOTATE_IGNORE_READS_END(); diff --git a/third_party/tcmalloc/chromium/src/base/elf_mem_image.cc b/third_party/tcmalloc/chromium/src/base/elf_mem_image.cc new file mode 100644 index 0000000..2949343 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/base/elf_mem_image.cc @@ -0,0 +1,433 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Paul Pluzhnikov +// +// Allow dynamic symbol lookup in an in-memory Elf image. +// + +#include "base/elf_mem_image.h" + +#ifdef HAVE_ELF_MEM_IMAGE // defined in elf_mem_image.h + +#include <stddef.h> // for size_t, ptrdiff_t +#include "base/logging.h" + +// From binutils/include/elf/common.h (this doesn't appear to be documented +// anywhere else). +// +// /* This flag appears in a Versym structure. It means that the symbol +// is hidden, and is only visible with an explicit version number. +// This is a GNU extension. */ +// #define VERSYM_HIDDEN 0x8000 +// +// /* This is the mask for the rest of the Versym information. */ +// #define VERSYM_VERSION 0x7fff + +#define VERSYM_VERSION 0x7fff + +namespace base { + +namespace { +template <int N> class ElfClass { + public: + static const int kElfClass = -1; + static int ElfBind(const ElfW(Sym) *) { + CHECK(false); // << "Unexpected word size"; + return 0; + } + static int ElfType(const ElfW(Sym) *) { + CHECK(false); // << "Unexpected word size"; + return 0; + } +}; + +template <> class ElfClass<32> { + public: + static const int kElfClass = ELFCLASS32; + static int ElfBind(const ElfW(Sym) *symbol) { + return ELF32_ST_BIND(symbol->st_info); + } + static int ElfType(const ElfW(Sym) *symbol) { + return ELF32_ST_TYPE(symbol->st_info); + } +}; + +template <> class ElfClass<64> { + public: + static const int kElfClass = ELFCLASS64; + static int ElfBind(const ElfW(Sym) *symbol) { + return ELF64_ST_BIND(symbol->st_info); + } + static int ElfType(const ElfW(Sym) *symbol) { + return ELF64_ST_TYPE(symbol->st_info); + } +}; + +typedef ElfClass<__WORDSIZE> CurrentElfClass; + +// Extract an element from one of the ELF tables, cast it to desired type. +// This is just a simple arithmetic and a glorified cast. +// Callers are responsible for bounds checking. +template <class T> +const T* GetTableElement(const ElfW(Ehdr) *ehdr, + ElfW(Off) table_offset, + ElfW(Word) element_size, + size_t index) { + return reinterpret_cast<const T*>(reinterpret_cast<const char *>(ehdr) + + table_offset + + index * element_size); +} +} // namespace + +const void *const ElfMemImage::kInvalidBase = + reinterpret_cast<const void *>(~0L); + +ElfMemImage::ElfMemImage(const void *base) { + CHECK(base != kInvalidBase); + Init(base); +} + +int ElfMemImage::GetNumSymbols() const { + if (!hash_) { + return 0; + } + // See http://www.caldera.com/developers/gabi/latest/ch5.dynamic.html#hash + return hash_[1]; +} + +const ElfW(Sym) *ElfMemImage::GetDynsym(int index) const { + CHECK_LT(index, GetNumSymbols()); + return dynsym_ + index; +} + +const ElfW(Versym) *ElfMemImage::GetVersym(int index) const { + CHECK_LT(index, GetNumSymbols()); + return versym_ + index; +} + +const ElfW(Phdr) *ElfMemImage::GetPhdr(int index) const { + CHECK_LT(index, ehdr_->e_phnum); + return GetTableElement<ElfW(Phdr)>(ehdr_, + ehdr_->e_phoff, + ehdr_->e_phentsize, + index); +} + +const char *ElfMemImage::GetDynstr(ElfW(Word) offset) const { + CHECK_LT(offset, strsize_); + return dynstr_ + offset; +} + +const void *ElfMemImage::GetSymAddr(const ElfW(Sym) *sym) const { + if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) { + // Symbol corresponds to "special" (e.g. SHN_ABS) section. + return reinterpret_cast<const void *>(sym->st_value); + } + CHECK_LT(link_base_, sym->st_value); + return GetTableElement<char>(ehdr_, 0, 1, sym->st_value) - link_base_; +} + +const ElfW(Verdef) *ElfMemImage::GetVerdef(int index) const { + CHECK_LE(index, verdefnum_); + const ElfW(Verdef) *version_definition = verdef_; + while (version_definition->vd_ndx < index && version_definition->vd_next) { + const char *const version_definition_as_char = + reinterpret_cast<const char *>(version_definition); + version_definition = + reinterpret_cast<const ElfW(Verdef) *>(version_definition_as_char + + version_definition->vd_next); + } + return version_definition->vd_ndx == index ? version_definition : NULL; +} + +const ElfW(Verdaux) *ElfMemImage::GetVerdefAux( + const ElfW(Verdef) *verdef) const { + return reinterpret_cast<const ElfW(Verdaux) *>(verdef+1); +} + +const char *ElfMemImage::GetVerstr(ElfW(Word) offset) const { + CHECK_LT(offset, strsize_); + return dynstr_ + offset; +} + +void ElfMemImage::Init(const void *base) { + ehdr_ = NULL; + dynsym_ = NULL; + dynstr_ = NULL; + versym_ = NULL; + verdef_ = NULL; + hash_ = NULL; + strsize_ = 0; + verdefnum_ = 0; + link_base_ = ~0L; // Sentinel: PT_LOAD .p_vaddr can't possibly be this. + if (!base) { + return; + } + const intptr_t base_as_uintptr_t = reinterpret_cast<uintptr_t>(base); + // Fake VDSO has low bit set. + const bool fake_vdso = ((base_as_uintptr_t & 1) != 0); + base = reinterpret_cast<const void *>(base_as_uintptr_t & ~1); + const char *const base_as_char = reinterpret_cast<const char *>(base); + if (base_as_char[EI_MAG0] != ELFMAG0 || base_as_char[EI_MAG1] != ELFMAG1 || + base_as_char[EI_MAG2] != ELFMAG2 || base_as_char[EI_MAG3] != ELFMAG3) { + RAW_DCHECK(false, "no ELF magic"); // at %p", base); + return; + } + int elf_class = base_as_char[EI_CLASS]; + if (elf_class != CurrentElfClass::kElfClass) { + DCHECK_EQ(elf_class, CurrentElfClass::kElfClass); + return; + } + switch (base_as_char[EI_DATA]) { + case ELFDATA2LSB: { + if (__LITTLE_ENDIAN != __BYTE_ORDER) { + DCHECK_EQ(__LITTLE_ENDIAN, __BYTE_ORDER); // << ": wrong byte order"; + return; + } + break; + } + case ELFDATA2MSB: { + if (__BIG_ENDIAN != __BYTE_ORDER) { + DCHECK_EQ(__BIG_ENDIAN, __BYTE_ORDER); // << ": wrong byte order"; + return; + } + break; + } + default: { + RAW_DCHECK(false, "unexpected data encoding"); // << base_as_char[EI_DATA]; + return; + } + } + + ehdr_ = reinterpret_cast<const ElfW(Ehdr) *>(base); + const ElfW(Phdr) *dynamic_program_header = NULL; + for (int i = 0; i < ehdr_->e_phnum; ++i) { + const ElfW(Phdr) *const program_header = GetPhdr(i); + switch (program_header->p_type) { + case PT_LOAD: + if (link_base_ == ~0L) { + link_base_ = program_header->p_vaddr; + } + break; + case PT_DYNAMIC: + dynamic_program_header = program_header; + break; + } + } + if (link_base_ == ~0L || !dynamic_program_header) { + RAW_DCHECK(~0L != link_base_, "no PT_LOADs in VDSO"); + RAW_DCHECK(dynamic_program_header, "no PT_DYNAMIC in VDSO"); + // Mark this image as not present. Can not recur infinitely. + Init(0); + return; + } + ptrdiff_t relocation = + base_as_char - reinterpret_cast<const char *>(link_base_); + ElfW(Dyn) *dynamic_entry = + reinterpret_cast<ElfW(Dyn) *>(dynamic_program_header->p_vaddr + + relocation); + for (; dynamic_entry->d_tag != DT_NULL; ++dynamic_entry) { + ElfW(Xword) value = dynamic_entry->d_un.d_val; + if (fake_vdso) { + // A complication: in the real VDSO, dynamic entries are not relocated + // (it wasn't loaded by a dynamic loader). But when testing with a + // "fake" dlopen()ed vdso library, the loader relocates some (but + // not all!) of them before we get here. + if (dynamic_entry->d_tag == DT_VERDEF) { + // The only dynamic entry (of the ones we care about) libc-2.3.6 + // loader doesn't relocate. + value += relocation; + } + } else { + // Real VDSO. Everything needs to be relocated. + value += relocation; + } + switch (dynamic_entry->d_tag) { + case DT_HASH: + hash_ = reinterpret_cast<ElfW(Word) *>(value); + break; + case DT_SYMTAB: + dynsym_ = reinterpret_cast<ElfW(Sym) *>(value); + break; + case DT_STRTAB: + dynstr_ = reinterpret_cast<const char *>(value); + break; + case DT_VERSYM: + versym_ = reinterpret_cast<ElfW(Versym) *>(value); + break; + case DT_VERDEF: + verdef_ = reinterpret_cast<ElfW(Verdef) *>(value); + break; + case DT_VERDEFNUM: + verdefnum_ = dynamic_entry->d_un.d_val; + break; + case DT_STRSZ: + strsize_ = dynamic_entry->d_un.d_val; + break; + default: + // Unrecognized entries explicitly ignored. + break; + } + } + if (!hash_ || !dynsym_ || !dynstr_ || !versym_ || + !verdef_ || !verdefnum_ || !strsize_) { + RAW_DCHECK(hash_, "invalid VDSO (no DT_HASH)"); + RAW_DCHECK(dynsym_, "invalid VDSO (no DT_SYMTAB)"); + RAW_DCHECK(dynstr_, "invalid VDSO (no DT_STRTAB)"); + RAW_DCHECK(versym_, "invalid VDSO (no DT_VERSYM)"); + RAW_DCHECK(verdef_, "invalid VDSO (no DT_VERDEF)"); + RAW_DCHECK(verdefnum_, "invalid VDSO (no DT_VERDEFNUM)"); + RAW_DCHECK(strsize_, "invalid VDSO (no DT_STRSZ)"); + // Mark this image as not present. Can not recur infinitely. + Init(0); + return; + } +} + +bool ElfMemImage::LookupSymbol(const char *name, + const char *version, + int type, + SymbolInfo *info) const { + for (SymbolIterator it = begin(); it != end(); ++it) { + if (strcmp(it->name, name) == 0 && strcmp(it->version, version) == 0 && + CurrentElfClass::ElfType(it->symbol) == type) { + if (info) { + *info = *it; + } + return true; + } + } + return false; +} + +bool ElfMemImage::LookupSymbolByAddress(const void *address, + SymbolInfo *info_out) const { + for (SymbolIterator it = begin(); it != end(); ++it) { + const char *const symbol_start = + reinterpret_cast<const char *>(it->address); + const char *const symbol_end = symbol_start + it->symbol->st_size; + if (symbol_start <= address && address < symbol_end) { + if (info_out) { + // Client wants to know details for that symbol (the usual case). + if (CurrentElfClass::ElfBind(it->symbol) == STB_GLOBAL) { + // Strong symbol; just return it. + *info_out = *it; + return true; + } else { + // Weak or local. Record it, but keep looking for a strong one. + *info_out = *it; + } + } else { + // Client only cares if there is an overlapping symbol. + return true; + } + } + } + return false; +} + +ElfMemImage::SymbolIterator::SymbolIterator(const void *const image, int index) + : index_(index), image_(image) { +} + +const ElfMemImage::SymbolInfo *ElfMemImage::SymbolIterator::operator->() const { + return &info_; +} + +const ElfMemImage::SymbolInfo& ElfMemImage::SymbolIterator::operator*() const { + return info_; +} + +bool ElfMemImage::SymbolIterator::operator==(const SymbolIterator &rhs) const { + return this->image_ == rhs.image_ && this->index_ == rhs.index_; +} + +bool ElfMemImage::SymbolIterator::operator!=(const SymbolIterator &rhs) const { + return !(*this == rhs); +} + +ElfMemImage::SymbolIterator &ElfMemImage::SymbolIterator::operator++() { + this->Update(1); + return *this; +} + +ElfMemImage::SymbolIterator ElfMemImage::begin() const { + SymbolIterator it(this, 0); + it.Update(0); + return it; +} + +ElfMemImage::SymbolIterator ElfMemImage::end() const { + return SymbolIterator(this, GetNumSymbols()); +} + +void ElfMemImage::SymbolIterator::Update(int increment) { + const ElfMemImage *image = reinterpret_cast<const ElfMemImage *>(image_); + CHECK(image->IsPresent() || increment == 0); + if (!image->IsPresent()) { + return; + } + index_ += increment; + if (index_ >= image->GetNumSymbols()) { + index_ = image->GetNumSymbols(); + return; + } + const ElfW(Sym) *symbol = image->GetDynsym(index_); + const ElfW(Versym) *version_symbol = image->GetVersym(index_); + CHECK(symbol && version_symbol); + const char *const symbol_name = image->GetDynstr(symbol->st_name); + const ElfW(Versym) version_index = version_symbol[0] & VERSYM_VERSION; + const ElfW(Verdef) *version_definition = NULL; + const char *version_name = ""; + if (symbol->st_shndx == SHN_UNDEF) { + // Undefined symbols reference DT_VERNEED, not DT_VERDEF, and + // version_index could well be greater than verdefnum_, so calling + // GetVerdef(version_index) may trigger assertion. + } else { + version_definition = image->GetVerdef(version_index); + } + if (version_definition) { + // I am expecting 1 or 2 auxiliary entries: 1 for the version itself, + // optional 2nd if the version has a parent. + CHECK_LE(1, version_definition->vd_cnt); + CHECK_LE(version_definition->vd_cnt, 2); + const ElfW(Verdaux) *version_aux = image->GetVerdefAux(version_definition); + version_name = image->GetVerstr(version_aux->vda_name); + } + info_.name = symbol_name; + info_.version = version_name; + info_.address = image->GetSymAddr(symbol); + info_.symbol = symbol; +} + +} // namespace base + +#endif // HAVE_ELF_MEM_IMAGE diff --git a/third_party/tcmalloc/chromium/src/base/elf_mem_image.h b/third_party/tcmalloc/chromium/src/base/elf_mem_image.h new file mode 100644 index 0000000..6f1f097 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/base/elf_mem_image.h @@ -0,0 +1,134 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Paul Pluzhnikov +// +// Allow dynamic symbol lookup for in-memory Elf images. + +#ifndef BASE_ELF_MEM_IMAGE_H_ +#define BASE_ELF_MEM_IMAGE_H_ + +#include <config.h> +#ifdef HAVE_FEATURES_H +#include <features.h> // for __GLIBC__ +#endif + +// Maybe one day we can rewrite this file not to require the elf +// symbol extensions in glibc, but for right now we need them. +#if defined(__ELF__) && defined(__GLIBC__) && !defined(__native_client__) + +#define HAVE_ELF_MEM_IMAGE 1 + +#include <stdlib.h> +#include <link.h> // for ElfW + +namespace base { + +// An in-memory ELF image (may not exist on disk). +class ElfMemImage { + public: + // Sentinel: there could never be an elf image at this address. + static const void *const kInvalidBase; + + // Information about a single vdso symbol. + // All pointers are into .dynsym, .dynstr, or .text of the VDSO. + // Do not free() them or modify through them. + struct SymbolInfo { + const char *name; // E.g. "__vdso_getcpu" + const char *version; // E.g. "LINUX_2.6", could be "" + // for unversioned symbol. + const void *address; // Relocated symbol address. + const ElfW(Sym) *symbol; // Symbol in the dynamic symbol table. + }; + + // Supports iteration over all dynamic symbols. + class SymbolIterator { + public: + friend class ElfMemImage; + const SymbolInfo *operator->() const; + const SymbolInfo &operator*() const; + SymbolIterator& operator++(); + bool operator!=(const SymbolIterator &rhs) const; + bool operator==(const SymbolIterator &rhs) const; + private: + SymbolIterator(const void *const image, int index); + void Update(int incr); + SymbolInfo info_; + int index_; + const void *const image_; + }; + + + explicit ElfMemImage(const void *base); + void Init(const void *base); + bool IsPresent() const { return ehdr_ != NULL; } + const ElfW(Phdr)* GetPhdr(int index) const; + const ElfW(Sym)* GetDynsym(int index) const; + const ElfW(Versym)* GetVersym(int index) const; + const ElfW(Verdef)* GetVerdef(int index) const; + const ElfW(Verdaux)* GetVerdefAux(const ElfW(Verdef) *verdef) const; + const char* GetDynstr(ElfW(Word) offset) const; + const void* GetSymAddr(const ElfW(Sym) *sym) const; + const char* GetVerstr(ElfW(Word) offset) const; + int GetNumSymbols() const; + + SymbolIterator begin() const; + SymbolIterator end() const; + + // Look up versioned dynamic symbol in the image. + // Returns false if image is not present, or doesn't contain given + // symbol/version/type combination. + // If info_out != NULL, additional details are filled in. + bool LookupSymbol(const char *name, const char *version, + int symbol_type, SymbolInfo *info_out) const; + + // Find info about symbol (if any) which overlaps given address. + // Returns true if symbol was found; false if image isn't present + // or doesn't have a symbol overlapping given address. + // If info_out != NULL, additional details are filled in. + bool LookupSymbolByAddress(const void *address, SymbolInfo *info_out) const; + + private: + const ElfW(Ehdr) *ehdr_; + const ElfW(Sym) *dynsym_; + const ElfW(Versym) *versym_; + const ElfW(Verdef) *verdef_; + const ElfW(Word) *hash_; + const char *dynstr_; + size_t strsize_; + size_t verdefnum_; + ElfW(Addr) link_base_; // Link-time base (p_vaddr of first PT_LOAD). +}; + +} // namespace base + +#endif // __ELF__ and __GLIBC__ and !__native_client__ + +#endif // BASE_ELF_MEM_IMAGE_H_ diff --git a/third_party/tcmalloc/chromium/src/base/googleinit.h b/third_party/tcmalloc/chromium/src/base/googleinit.h index 62ad84c..a48375d 100644 --- a/third_party/tcmalloc/chromium/src/base/googleinit.h +++ b/third_party/tcmalloc/chromium/src/base/googleinit.h @@ -33,19 +33,55 @@ #ifndef _GOOGLEINIT_H #define _GOOGLEINIT_H +#include "base/logging.h" + class GoogleInitializer { public: - typedef void (*void_function)(void); - GoogleInitializer(const char* name, void_function f) { - f(); + typedef void (*VoidFunction)(void); + GoogleInitializer(const char* name, VoidFunction ctor, VoidFunction dtor) + : name_(name), destructor_(dtor) { + // TODO(dmikurube): Re-enable the commented-out code. + // We commented out the following line, since Chromium does not have the + // proper includes to log using these macros. + // + // Commended-out code: + // RAW_VLOG(10, "<GoogleModuleObject> constructing: %s\n", name_); + // + // This googleinit.h is included from out of third_party/tcmalloc, such as + // net/tools/flip_server/balsa_headers.cc. + // "base/logging.h" (included above) indicates Chromium's base/logging.h + // when this googleinit.h is included from out of third_party/tcmalloc. + if (ctor) + ctor(); + } + ~GoogleInitializer() { + // TODO(dmikurube): Re-enable the commented-out code. + // The same as above. The following line is commented out in Chromium. + // + // Commended-out code: + // RAW_VLOG(10, "<GoogleModuleObject> destroying: %s\n", name_); + if (destructor_) + destructor_(); } + + private: + const char* const name_; + const VoidFunction destructor_; }; #define REGISTER_MODULE_INITIALIZER(name, body) \ namespace { \ static void google_init_module_##name () { body; } \ GoogleInitializer google_initializer_module_##name(#name, \ - google_init_module_##name); \ + google_init_module_##name, NULL); \ } +#define REGISTER_MODULE_DESTRUCTOR(name, body) \ + namespace { \ + static void google_destruct_module_##name () { body; } \ + GoogleInitializer google_destructor_module_##name(#name, \ + NULL, google_destruct_module_##name); \ + } + + #endif /* _GOOGLEINIT_H */ diff --git a/third_party/tcmalloc/chromium/src/base/linux_syscall_support.h b/third_party/tcmalloc/chromium/src/base/linux_syscall_support.h index f11f823..99dac9e 100644 --- a/third_party/tcmalloc/chromium/src/base/linux_syscall_support.h +++ b/third_party/tcmalloc/chromium/src/base/linux_syscall_support.h @@ -69,6 +69,63 @@ * This file defines a few internal symbols that all start with "LSS_". * Do not access these symbols from outside this file. They are not part * of the supported API. + * + * NOTE: This is a stripped down version of the official opensource + * version of linux_syscall_support.h, which lives at + * http://code.google.com/p/linux-syscall-support/ + * It includes only the syscalls that are used in perftools, plus a + * few extra. Here's the breakdown: + * 1) Perftools uses these: grep -rho 'sys_[a-z0-9_A-Z]* *(' src | sort -u + * sys__exit( + * sys_clone( + * sys_close( + * sys_fcntl( + * sys_fstat( + * sys_futex( + * sys_futex1( + * sys_getcpu( + * sys_getdents( + * sys_getppid( + * sys_gettid( + * sys_lseek( + * sys_mmap( + * sys_mremap( + * sys_munmap( + * sys_open( + * sys_pipe( + * sys_prctl( + * sys_ptrace( + * sys_ptrace_detach( + * sys_read( + * sys_sched_yield( + * sys_sigaction( + * sys_sigaltstack( + * sys_sigdelset( + * sys_sigfillset( + * sys_sigprocmask( + * sys_socket( + * sys_stat( + * sys_waitpid( + * 2) These are used as subroutines of the above: + * sys_getpid -- gettid + * sys_kill -- ptrace_detach + * sys_restore -- sigaction + * sys_restore_rt -- sigaction + * sys_socketcall -- socket + * sys_wait4 -- waitpid + * 3) I left these in even though they're not used. They either + * complement the above (write vs read) or are variants (rt_sigaction): + * sys_fstat64 + * sys_getdents64 + * sys_llseek + * sys_mmap2 + * sys_openat + * sys_rt_sigaction + * sys_rt_sigprocmask + * sys_sigaddset + * sys_sigemptyset + * sys_stat64 + * sys_write */ #ifndef SYS_LINUX_SYSCALL_SUPPORT_H #define SYS_LINUX_SYSCALL_SUPPORT_H @@ -76,7 +133,7 @@ /* We currently only support x86-32, x86-64, ARM, MIPS, and PPC on Linux. * Porting to other related platforms should not be difficult. */ -#if (defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__) || \ +#if (defined(__i386__) || defined(__x86_64__) || defined(__arm__) || \ defined(__mips__) || defined(__PPC__)) && defined(__linux) #ifndef SYS_CPLUSPLUS @@ -154,36 +211,6 @@ struct kernel_dirent { char d_name[256]; }; -/* include/linux/uio.h */ -struct kernel_iovec { - void *iov_base; - unsigned long iov_len; -}; - -/* include/linux/socket.h */ -struct kernel_msghdr { - void *msg_name; - int msg_namelen; - struct kernel_iovec*msg_iov; - unsigned long msg_iovlen; - void *msg_control; - unsigned long msg_controllen; - unsigned msg_flags; -}; - -/* include/asm-generic/poll.h */ -struct kernel_pollfd { - int fd; - short events; - short revents; -}; - -/* include/linux/resource.h */ -struct kernel_rlimit { - unsigned long rlim_cur; - unsigned long rlim_max; -}; - /* include/linux/time.h */ struct kernel_timespec { long tv_sec; @@ -217,7 +244,7 @@ struct kernel_rusage { }; struct siginfo; -#if defined(__i386__) || defined(__ARM_ARCH_3__) || defined(__PPC__) +#if defined(__i386__) || defined(__arm__) || defined(__PPC__) /* include/asm-{arm,i386,mips,ppc}/signal.h */ struct kernel_old_sigaction { @@ -274,12 +301,6 @@ struct kernel_sigaction { #endif }; -/* include/linux/socket.h */ -struct kernel_sockaddr { - unsigned short sa_family; - char sa_data[14]; -}; - /* include/asm-{arm,i386,mips,ppc}/stat.h */ #ifdef __mips__ #if _MIPS_SIM == _MIPS_SIM_ABI64 @@ -354,7 +375,7 @@ struct kernel_stat64 { #endif /* include/asm-{arm,i386,mips,x86_64,ppc}/stat.h */ -#if defined(__i386__) || defined(__ARM_ARCH_3__) +#if defined(__i386__) || defined(__arm__) struct kernel_stat { /* The kernel headers suggest that st_dev and st_rdev should be 32bit * quantities encoding 12bit major and 20bit minor numbers in an interleaved @@ -449,89 +470,15 @@ struct kernel_stat { }; #endif -/* include/asm-{arm,i386,mips,x86_64,ppc}/statfs.h */ -#ifdef __mips__ -#if _MIPS_SIM != _MIPS_SIM_ABI64 -struct kernel_statfs64 { - unsigned long f_type; - unsigned long f_bsize; - unsigned long f_frsize; - unsigned long __pad; - unsigned long long f_blocks; - unsigned long long f_bfree; - unsigned long long f_files; - unsigned long long f_ffree; - unsigned long long f_bavail; - struct { int val[2]; } f_fsid; - unsigned long f_namelen; - unsigned long f_spare[6]; -}; -#endif -#elif !defined(__x86_64__) -struct kernel_statfs64 { - unsigned long f_type; - unsigned long f_bsize; - unsigned long long f_blocks; - unsigned long long f_bfree; - unsigned long long f_bavail; - unsigned long long f_files; - unsigned long long f_ffree; - struct { int val[2]; } f_fsid; - unsigned long f_namelen; - unsigned long f_frsize; - unsigned long f_spare[5]; -}; -#endif - -/* include/asm-{arm,i386,mips,x86_64,ppc,generic}/statfs.h */ -#ifdef __mips__ -struct kernel_statfs { - long f_type; - long f_bsize; - long f_frsize; - long f_blocks; - long f_bfree; - long f_files; - long f_ffree; - long f_bavail; - struct { int val[2]; } f_fsid; - long f_namelen; - long f_spare[6]; -}; -#else -struct kernel_statfs { - /* x86_64 actually defines all these fields as signed, whereas all other */ - /* platforms define them as unsigned. Leaving them at unsigned should not */ - /* cause any problems. */ - unsigned long f_type; - unsigned long f_bsize; - unsigned long f_blocks; - unsigned long f_bfree; - unsigned long f_bavail; - unsigned long f_files; - unsigned long f_ffree; - struct { int val[2]; } f_fsid; - unsigned long f_namelen; - unsigned long f_frsize; - unsigned long f_spare[5]; -}; -#endif - /* Definitions missing from the standard header files */ #ifndef O_DIRECTORY -#if defined(__ARM_ARCH_3__) +#if defined(__arm__) #define O_DIRECTORY 0040000 #else #define O_DIRECTORY 0200000 #endif #endif -#ifndef NT_PRXFPREG -#define NT_PRXFPREG 0x46e62b7f -#endif -#ifndef PTRACE_GETFPXREGS -#define PTRACE_GETFPXREGS ((enum __ptrace_request)18) -#endif #ifndef PR_GET_DUMPABLE #define PR_GET_DUMPABLE 3 #endif @@ -553,46 +500,11 @@ struct kernel_statfs { #ifndef SA_RESTORER #define SA_RESTORER 0x04000000 #endif -#ifndef CPUCLOCK_PROF -#define CPUCLOCK_PROF 0 -#endif -#ifndef CPUCLOCK_VIRT -#define CPUCLOCK_VIRT 1 -#endif -#ifndef CPUCLOCK_SCHED -#define CPUCLOCK_SCHED 2 -#endif -#ifndef CPUCLOCK_PERTHREAD_MASK -#define CPUCLOCK_PERTHREAD_MASK 4 -#endif -#ifndef MAKE_PROCESS_CPUCLOCK -#define MAKE_PROCESS_CPUCLOCK(pid, clock) \ - ((~(int)(pid) << 3) | (int)(clock)) -#endif -#ifndef MAKE_THREAD_CPUCLOCK -#define MAKE_THREAD_CPUCLOCK(tid, clock) \ - ((~(int)(tid) << 3) | (int)((clock) | CPUCLOCK_PERTHREAD_MASK)) -#endif #if defined(__i386__) -#ifndef __NR_setresuid -#define __NR_setresuid 164 -#define __NR_setresgid 170 -#endif #ifndef __NR_rt_sigaction #define __NR_rt_sigaction 174 #define __NR_rt_sigprocmask 175 -#define __NR_rt_sigpending 176 -#define __NR_rt_sigsuspend 179 -#endif -#ifndef __NR_pread64 -#define __NR_pread64 180 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 181 -#endif -#ifndef __NR_ugetrlimit -#define __NR_ugetrlimit 191 #endif #ifndef __NR_stat64 #define __NR_stat64 195 @@ -600,110 +512,43 @@ struct kernel_statfs { #ifndef __NR_fstat64 #define __NR_fstat64 197 #endif -#ifndef __NR_setresuid32 -#define __NR_setresuid32 208 -#define __NR_setresgid32 210 -#endif -#ifndef __NR_setfsuid32 -#define __NR_setfsuid32 215 -#define __NR_setfsgid32 216 -#endif #ifndef __NR_getdents64 #define __NR_getdents64 220 #endif #ifndef __NR_gettid #define __NR_gettid 224 #endif -#ifndef __NR_readahead -#define __NR_readahead 225 -#endif -#ifndef __NR_setxattr -#define __NR_setxattr 226 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 227 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 229 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 230 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 232 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 233 -#endif #ifndef __NR_futex #define __NR_futex 240 #endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 241 -#define __NR_sched_getaffinity 242 -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 258 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 265 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 266 -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 268 -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 269 -#endif -#ifndef __NR_fadvise64_64 -#define __NR_fadvise64_64 272 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 289 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 290 -#endif #ifndef __NR_openat #define __NR_openat 295 #endif -#ifndef __NR_fstatat64 -#define __NR_fstatat64 300 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 301 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 317 -#endif #ifndef __NR_getcpu #define __NR_getcpu 318 #endif -#ifndef __NR_fallocate -#define __NR_fallocate 324 -#endif /* End of i386 definitions */ -#elif defined(__ARM_ARCH_3__) -#ifndef __NR_setresuid -#define __NR_setresuid (__NR_SYSCALL_BASE + 164) -#define __NR_setresgid (__NR_SYSCALL_BASE + 170) +#elif defined(__arm__) +#ifndef __syscall +#if defined(__thumb__) || defined(__ARM_EABI__) +#define __SYS_REG(name) register long __sysreg __asm__("r6") = __NR_##name; +#define __SYS_REG_LIST(regs...) [sysreg] "r" (__sysreg) , ##regs +#define __syscall(name) "swi\t0" +#define __syscall_safe(name) \ + "push {r7}\n" \ + "mov r7,%[sysreg]\n" \ + __syscall(name)"\n" \ + "pop {r7}" +#else +#define __SYS_REG(name) +#define __SYS_REG_LIST(regs...) regs +#define __syscall(name) "swi\t" __sys1(__NR_##name) "" +#define __syscall_safe(name) __syscall(name) +#endif #endif #ifndef __NR_rt_sigaction #define __NR_rt_sigaction (__NR_SYSCALL_BASE + 174) #define __NR_rt_sigprocmask (__NR_SYSCALL_BASE + 175) -#define __NR_rt_sigpending (__NR_SYSCALL_BASE + 176) -#define __NR_rt_sigsuspend (__NR_SYSCALL_BASE + 179) -#endif -#ifndef __NR_pread64 -#define __NR_pread64 (__NR_SYSCALL_BASE + 180) -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 (__NR_SYSCALL_BASE + 181) -#endif -#ifndef __NR_ugetrlimit -#define __NR_ugetrlimit (__NR_SYSCALL_BASE + 191) #endif #ifndef __NR_stat64 #define __NR_stat64 (__NR_SYSCALL_BASE + 195) @@ -711,172 +556,35 @@ struct kernel_statfs { #ifndef __NR_fstat64 #define __NR_fstat64 (__NR_SYSCALL_BASE + 197) #endif -#ifndef __NR_setresuid32 -#define __NR_setresuid32 (__NR_SYSCALL_BASE + 208) -#define __NR_setresgid32 (__NR_SYSCALL_BASE + 210) -#endif -#ifndef __NR_setfsuid32 -#define __NR_setfsuid32 (__NR_SYSCALL_BASE + 215) -#define __NR_setfsgid32 (__NR_SYSCALL_BASE + 216) -#endif #ifndef __NR_getdents64 #define __NR_getdents64 (__NR_SYSCALL_BASE + 217) #endif #ifndef __NR_gettid #define __NR_gettid (__NR_SYSCALL_BASE + 224) #endif -#ifndef __NR_readahead -#define __NR_readahead (__NR_SYSCALL_BASE + 225) -#endif -#ifndef __NR_setxattr -#define __NR_setxattr (__NR_SYSCALL_BASE + 226) -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr (__NR_SYSCALL_BASE + 227) -#endif -#ifndef __NR_getxattr -#define __NR_getxattr (__NR_SYSCALL_BASE + 229) -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr (__NR_SYSCALL_BASE + 230) -#endif -#ifndef __NR_listxattr -#define __NR_listxattr (__NR_SYSCALL_BASE + 232) -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr (__NR_SYSCALL_BASE + 233) -#endif #ifndef __NR_futex #define __NR_futex (__NR_SYSCALL_BASE + 240) #endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity (__NR_SYSCALL_BASE + 241) -#define __NR_sched_getaffinity (__NR_SYSCALL_BASE + 242) -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address (__NR_SYSCALL_BASE + 256) -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime (__NR_SYSCALL_BASE + 263) -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres (__NR_SYSCALL_BASE + 264) -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 (__NR_SYSCALL_BASE + 266) -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 (__NR_SYSCALL_BASE + 267) -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set (__NR_SYSCALL_BASE + 314) -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get (__NR_SYSCALL_BASE + 315) -#endif -#ifndef __NR_move_pages -#define __NR_move_pages (__NR_SYSCALL_BASE + 344) -#endif -#ifndef __NR_getcpu -#define __NR_getcpu (__NR_SYSCALL_BASE + 345) -#endif -/* End of ARM 3 definitions */ +/* End of ARM definitions */ #elif defined(__x86_64__) -#ifndef __NR_pread64 -#define __NR_pread64 17 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 18 -#endif -#ifndef __NR_setresuid -#define __NR_setresuid 117 -#define __NR_setresgid 119 -#endif #ifndef __NR_gettid #define __NR_gettid 186 #endif -#ifndef __NR_readahead -#define __NR_readahead 187 -#endif -#ifndef __NR_setxattr -#define __NR_setxattr 188 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 189 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 191 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 192 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 194 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 195 -#endif #ifndef __NR_futex #define __NR_futex 202 #endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 203 -#define __NR_sched_getaffinity 204 -#endif #ifndef __NR_getdents64 #define __NR_getdents64 217 #endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 218 -#endif -#ifndef __NR_fadvise64 -#define __NR_fadvise64 221 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 228 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 229 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 251 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 252 -#endif #ifndef __NR_openat #define __NR_openat 257 #endif -#ifndef __NR_newfstatat -#define __NR_newfstatat 262 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 263 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 279 -#endif -#ifndef __NR_fallocate -#define __NR_fallocate 285 -#endif /* End of x86-64 definitions */ #elif defined(__mips__) #if _MIPS_SIM == _MIPS_SIM_ABI32 -#ifndef __NR_setresuid -#define __NR_setresuid (__NR_Linux + 185) -#define __NR_setresgid (__NR_Linux + 190) -#endif #ifndef __NR_rt_sigaction #define __NR_rt_sigaction (__NR_Linux + 194) #define __NR_rt_sigprocmask (__NR_Linux + 195) -#define __NR_rt_sigpending (__NR_Linux + 196) -#define __NR_rt_sigsuspend (__NR_Linux + 199) -#endif -#ifndef __NR_pread64 -#define __NR_pread64 (__NR_Linux + 200) -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 (__NR_Linux + 201) #endif #ifndef __NR_stat64 #define __NR_stat64 (__NR_Linux + 213) @@ -890,245 +598,59 @@ struct kernel_statfs { #ifndef __NR_gettid #define __NR_gettid (__NR_Linux + 222) #endif -#ifndef __NR_readahead -#define __NR_readahead (__NR_Linux + 223) -#endif -#ifndef __NR_setxattr -#define __NR_setxattr (__NR_Linux + 224) -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr (__NR_Linux + 225) -#endif -#ifndef __NR_getxattr -#define __NR_getxattr (__NR_Linux + 227) -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr (__NR_Linux + 228) -#endif -#ifndef __NR_listxattr -#define __NR_listxattr (__NR_Linux + 230) -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr (__NR_Linux + 231) -#endif #ifndef __NR_futex #define __NR_futex (__NR_Linux + 238) #endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity (__NR_Linux + 239) -#define __NR_sched_getaffinity (__NR_Linux + 240) -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address (__NR_Linux + 252) -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 (__NR_Linux + 255) -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 (__NR_Linux + 256) -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime (__NR_Linux + 263) -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres (__NR_Linux + 264) -#endif #ifndef __NR_openat #define __NR_openat (__NR_Linux + 288) #endif #ifndef __NR_fstatat #define __NR_fstatat (__NR_Linux + 293) #endif -#ifndef __NR_unlinkat -#define __NR_unlinkat (__NR_Linux + 294) -#endif -#ifndef __NR_move_pages -#define __NR_move_pages (__NR_Linux + 308) -#endif #ifndef __NR_getcpu #define __NR_getcpu (__NR_Linux + 312) #endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set (__NR_Linux + 314) -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get (__NR_Linux + 315) -#endif /* End of MIPS (old 32bit API) definitions */ #elif _MIPS_SIM == _MIPS_SIM_ABI64 -#ifndef __NR_pread64 -#define __NR_pread64 (__NR_Linux + 16) -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 (__NR_Linux + 17) -#endif -#ifndef __NR_setresuid -#define __NR_setresuid (__NR_Linux + 115) -#define __NR_setresgid (__NR_Linux + 117) -#endif #ifndef __NR_gettid #define __NR_gettid (__NR_Linux + 178) #endif -#ifndef __NR_readahead -#define __NR_readahead (__NR_Linux + 179) -#endif -#ifndef __NR_setxattr -#define __NR_setxattr (__NR_Linux + 180) -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr (__NR_Linux + 181) -#endif -#ifndef __NR_getxattr -#define __NR_getxattr (__NR_Linux + 183) -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr (__NR_Linux + 184) -#endif -#ifndef __NR_listxattr -#define __NR_listxattr (__NR_Linux + 186) -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr (__NR_Linux + 187) -#endif #ifndef __NR_futex #define __NR_futex (__NR_Linux + 194) #endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity (__NR_Linux + 195) -#define __NR_sched_getaffinity (__NR_Linux + 196) -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address (__NR_Linux + 212) -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime (__NR_Linux + 222) -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres (__NR_Linux + 223) -#endif #ifndef __NR_openat #define __NR_openat (__NR_Linux + 247) #endif #ifndef __NR_fstatat #define __NR_fstatat (__NR_Linux + 252) #endif -#ifndef __NR_unlinkat -#define __NR_unlinkat (__NR_Linux + 253) -#endif -#ifndef __NR_move_pages -#define __NR_move_pages (__NR_Linux + 267) -#endif #ifndef __NR_getcpu #define __NR_getcpu (__NR_Linux + 271) #endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set (__NR_Linux + 273) -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get (__NR_Linux + 274) -#endif /* End of MIPS (64bit API) definitions */ #else -#ifndef __NR_setresuid -#define __NR_setresuid (__NR_Linux + 115) -#define __NR_setresgid (__NR_Linux + 117) -#endif #ifndef __NR_gettid #define __NR_gettid (__NR_Linux + 178) #endif -#ifndef __NR_readahead -#define __NR_readahead (__NR_Linux + 179) -#endif -#ifndef __NR_setxattr -#define __NR_setxattr (__NR_Linux + 180) -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr (__NR_Linux + 181) -#endif -#ifndef __NR_getxattr -#define __NR_getxattr (__NR_Linux + 183) -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr (__NR_Linux + 184) -#endif -#ifndef __NR_listxattr -#define __NR_listxattr (__NR_Linux + 186) -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr (__NR_Linux + 187) -#endif #ifndef __NR_futex #define __NR_futex (__NR_Linux + 194) #endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity (__NR_Linux + 195) -#define __NR_sched_getaffinity (__NR_Linux + 196) -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address (__NR_Linux + 213) -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 (__NR_Linux + 217) -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 (__NR_Linux + 218) -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime (__NR_Linux + 226) -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres (__NR_Linux + 227) -#endif #ifndef __NR_openat #define __NR_openat (__NR_Linux + 251) #endif #ifndef __NR_fstatat #define __NR_fstatat (__NR_Linux + 256) #endif -#ifndef __NR_unlinkat -#define __NR_unlinkat (__NR_Linux + 257) -#endif -#ifndef __NR_move_pages -#define __NR_move_pages (__NR_Linux + 271) -#endif #ifndef __NR_getcpu #define __NR_getcpu (__NR_Linux + 275) #endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set (__NR_Linux + 277) -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get (__NR_Linux + 278) -#endif /* End of MIPS (new 32bit API) definitions */ #endif /* End of MIPS definitions */ #elif defined(__PPC__) -#ifndef __NR_setfsuid -#define __NR_setfsuid 138 -#define __NR_setfsgid 139 -#endif -#ifndef __NR_setresuid -#define __NR_setresuid 164 -#define __NR_setresgid 169 -#endif #ifndef __NR_rt_sigaction #define __NR_rt_sigaction 173 #define __NR_rt_sigprocmask 174 -#define __NR_rt_sigpending 175 -#define __NR_rt_sigsuspend 178 -#endif -#ifndef __NR_pread64 -#define __NR_pread64 179 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 180 -#endif -#ifndef __NR_ugetrlimit -#define __NR_ugetrlimit 190 -#endif -#ifndef __NR_readahead -#define __NR_readahead 191 #endif #ifndef __NR_stat64 #define __NR_stat64 195 @@ -1142,67 +664,12 @@ struct kernel_statfs { #ifndef __NR_gettid #define __NR_gettid 207 #endif -#ifndef __NR_setxattr -#define __NR_setxattr 209 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 210 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 212 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 213 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 215 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 216 -#endif #ifndef __NR_futex #define __NR_futex 221 #endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 222 -#define __NR_sched_getaffinity 223 -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 232 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 246 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 247 -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 252 -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 253 -#endif -#ifndef __NR_fadvise64_64 -#define __NR_fadvise64_64 254 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 273 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 274 -#endif #ifndef __NR_openat #define __NR_openat 286 #endif -#ifndef __NR_fstatat64 -#define __NR_fstatat64 291 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 292 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 301 -#endif #ifndef __NR_getcpu #define __NR_getcpu 302 #endif @@ -1269,7 +736,7 @@ struct kernel_statfs { #endif #undef LSS_RETURN - #if (defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__)) + #if (defined(__i386__) || defined(__x86_64__) || defined(__arm__)) /* Failing system calls return a negative result in the range of * -1..-4095. These are "errno" values with the sign inverted. */ @@ -1307,6 +774,15 @@ struct kernel_statfs { } while (0) #endif #if defined(__i386__) + #if defined(NO_FRAME_POINTER) && (100 * __GNUC__ + __GNUC_MINOR__ >= 404) + /* This only works for GCC-4.4 and above -- the first version to use + .cfi directives for dwarf unwind info. */ + #define CFI_ADJUST_CFA_OFFSET(adjust) \ + ".cfi_adjust_cfa_offset " #adjust "\n" + #else + #define CFI_ADJUST_CFA_OFFSET(adjust) /**/ + #endif + /* In PIC mode (e.g. when building shared libraries), gcc for i386 * reserves ebx. Unfortunately, most distribution ship with implementations * of _syscallX() which clobber ebx. @@ -1319,11 +795,13 @@ struct kernel_statfs { #define LSS_BODY(type,args...) \ long __res; \ __asm__ __volatile__("push %%ebx\n" \ + CFI_ADJUST_CFA_OFFSET(4) \ "movl %2,%%ebx\n" \ "int $0x80\n" \ - "pop %%ebx" \ + "pop %%ebx\n" \ + CFI_ADJUST_CFA_OFFSET(-4) \ args \ - : "memory"); \ + : "esp", "memory"); \ LSS_RETURN(type,__res) #undef _syscall0 #define _syscall0(type,name) \ @@ -1380,7 +858,7 @@ struct kernel_statfs { : "i" (__NR_##name), "ri" ((long)(arg1)), \ "c" ((long)(arg2)), "d" ((long)(arg3)), \ "S" ((long)(arg4)), "D" ((long)(arg5)) \ - : "memory"); \ + : "esp", "memory"); \ LSS_RETURN(type,__res); \ } #undef _syscall6 @@ -1402,7 +880,7 @@ struct kernel_statfs { : "i" (__NR_##name), "0" ((long)(&__s)), \ "c" ((long)(arg2)), "d" ((long)(arg3)), \ "S" ((long)(arg4)), "D" ((long)(arg5)) \ - : "memory"); \ + : "esp", "memory"); \ LSS_RETURN(type,__res); \ } LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, @@ -1488,36 +966,10 @@ struct kernel_statfs { : "0"(-EINVAL), "i"(__NR_clone), "m"(fn), "m"(child_stack), "m"(flags), "m"(arg), "m"(parent_tidptr), "m"(newtls), "m"(child_tidptr) - : "memory", "ecx", "edx", "esi", "edi"); + : "esp", "memory", "ecx", "edx", "esi", "edi"); LSS_RETURN(int, __res); } - #define __NR__fadvise64_64 __NR_fadvise64_64 - LSS_INLINE _syscall6(int, _fadvise64_64, int, fd, - unsigned, offset_lo, unsigned, offset_hi, - unsigned, len_lo, unsigned, len_hi, - int, advice) - - LSS_INLINE int LSS_NAME(fadvise64)(int fd, loff_t offset, - loff_t len, int advice) { - return LSS_NAME(_fadvise64_64)(fd, - (unsigned)offset, (unsigned)(offset >>32), - (unsigned)len, (unsigned)(len >> 32), - advice); - } - - #define __NR__fallocate __NR_fallocate - LSS_INLINE _syscall6(int, _fallocate, int, fd, - int, mode, - unsigned, offset_lo, unsigned, offset_hi, - unsigned, len_lo, unsigned, len_hi) - - LSS_INLINE int LSS_NAME(fallocate)(int fd, int mode, - loff_t offset, loff_t len) { - union { loff_t off; unsigned w[2]; } o = { offset }, l = { len }; - return LSS_NAME(_fallocate)(fd, mode, o.w[0], o.w[1], l.w[0], l.w[1]); - } - LSS_INLINE void (*LSS_NAME(restore_rt)(void))(void) { /* On i386, the kernel does not know how to return from a signal * handler. Instead, it relies on user space to provide a @@ -1596,7 +1048,7 @@ struct kernel_statfs { __asm__ __volatile__("movq %5,%%r10; syscall" : \ "=a" (__res) : "0" (__NR_##name), \ "D" ((long)(arg1)), "S" ((long)(arg2)), "d" ((long)(arg3)), \ - "g" ((long)(arg4)) : "r10", "r11", "rcx", "memory"); \ + "r" ((long)(arg4)) : "r10", "r11", "rcx", "memory"); \ LSS_RETURN(type, __res); \ } #undef _syscall5 @@ -1608,7 +1060,7 @@ struct kernel_statfs { __asm__ __volatile__("movq %5,%%r10; movq %6,%%r8; syscall" : \ "=a" (__res) : "0" (__NR_##name), \ "D" ((long)(arg1)), "S" ((long)(arg2)), "d" ((long)(arg3)), \ - "g" ((long)(arg4)), "g" ((long)(arg5)) : \ + "r" ((long)(arg4)), "r" ((long)(arg5)) : \ "r8", "r10", "r11", "rcx", "memory"); \ LSS_RETURN(type, __res); \ } @@ -1622,7 +1074,7 @@ struct kernel_statfs { "syscall" : \ "=a" (__res) : "0" (__NR_##name), \ "D" ((long)(arg1)), "S" ((long)(arg2)), "d" ((long)(arg3)), \ - "g" ((long)(arg4)), "g" ((long)(arg5)), "g" ((long)(arg6)) : \ + "r" ((long)(arg4)), "r" ((long)(arg5)), "r" ((long)(arg6)) : \ "r8", "r9", "r10", "r11", "rcx", "memory"); \ LSS_RETURN(type, __res); \ } @@ -1631,8 +1083,6 @@ struct kernel_statfs { void *newtls, int *child_tidptr) { long __res; { - register void *__tls __asm__("r8") = newtls; - register int *__ctid __asm__("r10") = child_tidptr; __asm__ __volatile__(/* if (fn == NULL) * return -EINVAL; */ @@ -1665,6 +1115,8 @@ struct kernel_statfs { * %r10 = child_tidptr) */ "movq %2,%%rax\n" + "movq %9,%%r8\n" + "movq %10,%%r10\n" "syscall\n" /* if (%rax != 0) @@ -1695,13 +1147,11 @@ struct kernel_statfs { : "=a" (__res) : "0"(-EINVAL), "i"(__NR_clone), "i"(__NR_exit), "r"(fn), "S"(child_stack), "D"(flags), "r"(arg), - "d"(parent_tidptr), "r"(__tls), "r"(__ctid) - : "memory", "r11", "rcx"); + "d"(parent_tidptr), "g"(newtls), "g"(child_tidptr) + : "rsp", "memory", "r8", "r10", "r11", "rcx"); } LSS_RETURN(int, __res); } - LSS_INLINE _syscall4(int, fadvise64, int, fd, loff_t, offset, loff_t, len, - int, advice) LSS_INLINE void (*LSS_NAME(restore_rt)(void))(void) { /* On x86-64, the kernel does not know how to return from @@ -1721,7 +1171,7 @@ struct kernel_statfs { : "i" (__NR_rt_sigreturn)); return res; } - #elif defined(__ARM_ARCH_3__) + #elif defined(__arm__) /* Most definitions of _syscallX() neglect to mark "memory" as being * clobbered. This causes problems with compilers, that do a better job * at optimizing across __asm__ calls. @@ -1729,12 +1179,26 @@ struct kernel_statfs { */ #undef LSS_REG #define LSS_REG(r,a) register long __r##r __asm__("r"#r) = (long)a + + /* r0..r3 are scratch registers and not preserved across function + * calls. We need to first evaluate the first 4 syscall arguments + * and store them on stack. They must be loaded into r0..r3 after + * all function calls to avoid r0..r3 being clobbered. + */ + #undef LSS_SAVE_ARG + #define LSS_SAVE_ARG(r,a) long __tmp##r = (long)a + #undef LSS_LOAD_ARG + #define LSS_LOAD_ARG(r) register long __r##r __asm__("r"#r) = __tmp##r + #undef LSS_BODY - #define LSS_BODY(type,name,args...) \ + #define LSS_BODY(type, name, args...) \ register long __res_r0 __asm__("r0"); \ long __res; \ - __asm__ __volatile__ (__syscall(name) \ - : "=r"(__res_r0) : args : "lr", "memory"); \ + __SYS_REG(name) \ + __asm__ __volatile__ (__syscall_safe(name) \ + : "=r"(__res_r0) \ + : __SYS_REG_LIST(args) \ + : "lr", "memory"); \ __res = __res_r0; \ LSS_RETURN(type, __res) #undef _syscall0 @@ -1745,77 +1209,126 @@ struct kernel_statfs { #undef _syscall1 #define _syscall1(type, name, type1, arg1) \ type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(0, arg1); LSS_BODY(type, name, "r"(__r0)); \ + /* There is no need for using a volatile temp. */ \ + LSS_REG(0, arg1); \ + LSS_BODY(type, name, "r"(__r0)); \ } #undef _syscall2 #define _syscall2(type, name, type1, arg1, type2, arg2) \ type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); \ + LSS_SAVE_ARG(0, arg1); \ + LSS_SAVE_ARG(1, arg2); \ + LSS_LOAD_ARG(0); \ + LSS_LOAD_ARG(1); \ LSS_BODY(type, name, "r"(__r0), "r"(__r1)); \ } #undef _syscall3 #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ + LSS_SAVE_ARG(0, arg1); \ + LSS_SAVE_ARG(1, arg2); \ + LSS_SAVE_ARG(2, arg3); \ + LSS_LOAD_ARG(0); \ + LSS_LOAD_ARG(1); \ + LSS_LOAD_ARG(2); \ LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2)); \ } #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ + #define _syscall4(type, name, type1, arg1, type2, arg2, type3, arg3, \ + type4, arg4) \ type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); \ + LSS_SAVE_ARG(0, arg1); \ + LSS_SAVE_ARG(1, arg2); \ + LSS_SAVE_ARG(2, arg3); \ + LSS_SAVE_ARG(3, arg4); \ + LSS_LOAD_ARG(0); \ + LSS_LOAD_ARG(1); \ + LSS_LOAD_ARG(2); \ + LSS_LOAD_ARG(3); \ LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3)); \ } #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ + #define _syscall5(type, name, type1, arg1, type2, arg2, type3, arg3, \ + type4, arg4, type5, arg5) \ type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ type5 arg5) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); \ + LSS_SAVE_ARG(0, arg1); \ + LSS_SAVE_ARG(1, arg2); \ + LSS_SAVE_ARG(2, arg3); \ + LSS_SAVE_ARG(3, arg4); \ + LSS_REG(4, arg5); \ + LSS_LOAD_ARG(0); \ + LSS_LOAD_ARG(1); \ + LSS_LOAD_ARG(2); \ + LSS_LOAD_ARG(3); \ LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ "r"(__r4)); \ } #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ + #define _syscall6(type, name, type1, arg1, type2, arg2, type3, arg3, \ + type4, arg4, type5, arg5, type6, arg6) \ type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ type5 arg5, type6 arg6) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); LSS_REG(5, arg6); \ + LSS_SAVE_ARG(0, arg1); \ + LSS_SAVE_ARG(1, arg2); \ + LSS_SAVE_ARG(2, arg3); \ + LSS_SAVE_ARG(3, arg4); \ + LSS_REG(4, arg5); \ + LSS_REG(5, arg6); \ + LSS_LOAD_ARG(0); \ + LSS_LOAD_ARG(1); \ + LSS_LOAD_ARG(2); \ + LSS_LOAD_ARG(3); \ LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ "r"(__r4), "r"(__r5)); \ } LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, int flags, void *arg, int *parent_tidptr, void *newtls, int *child_tidptr) { - long __res; + register long __res __asm__("r5"); { - register int __flags __asm__("r0") = flags; - register void *__stack __asm__("r1") = child_stack; - register void *__ptid __asm__("r2") = parent_tidptr; - register void *__tls __asm__("r3") = newtls; - register int *__ctid __asm__("r4") = child_tidptr; - __asm__ __volatile__(/* if (fn == NULL || child_stack == NULL) - * return -EINVAL; - */ - "cmp %2,#0\n" - "cmpne %3,#0\n" - "moveq %0,%1\n" - "beq 1f\n" + if (fn == NULL || child_stack == NULL) { + __res = -EINVAL; + goto clone_exit; + } - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "str %5,[%3,#-4]!\n" - "str %2,[%3,#-4]!\n" + /* stash first 4 arguments on stack first because we can only load + * them after all function calls. + */ + int tmp_flags = flags; + int * tmp_stack = (int*) child_stack; + void * tmp_ptid = parent_tidptr; + void * tmp_tls = newtls; + + register int *__ctid __asm__("r4") = child_tidptr; - /* %r0 = syscall(%r0 = flags, + /* Push "arg" and "fn" onto the stack that will be + * used by the child. + */ + *(--tmp_stack) = (int) arg; + *(--tmp_stack) = (int) fn; + + /* We must load r0..r3 last after all possible function calls. */ + register int __flags __asm__("r0") = tmp_flags; + register void *__stack __asm__("r1") = tmp_stack; + register void *__ptid __asm__("r2") = tmp_ptid; + register void *__tls __asm__("r3") = tmp_tls; + + /* %r0 = syscall(%r0 = flags, + * %r1 = child_stack, + * %r2 = parent_tidptr, + * %r3 = newtls, + * %r4 = child_tidptr) + */ + __SYS_REG(clone) + __asm__ __volatile__(/* %r0 = syscall(%r0 = flags, * %r1 = child_stack, * %r2 = parent_tidptr, * %r3 = newtls, * %r4 = child_tidptr) */ + "push {r7}\n" + "mov r7,%1\n" __syscall(clone)"\n" /* if (%r0 != 0) @@ -1830,30 +1343,48 @@ struct kernel_statfs { "mov lr,pc\n" "ldr pc,[sp]\n" - /* Call _exit(%r0). + /* Call _exit(%r0), which never returns. We only + * need to set r7 for EABI syscall ABI but we do + * this always to simplify code sharing between + * old and new syscall ABIs. */ + "mov r7,%2\n" __syscall(exit)"\n" - "1:\n" + + /* Pop r7 from the stack only in the parent. + */ + "1: pop {r7}\n" : "=r" (__res) - : "i"(-EINVAL), - "r"(fn), "r"(__stack), "r"(__flags), "r"(arg), + : "r"(__sysreg), + "i"(__NR_exit), "r"(__stack), "r"(__flags), "r"(__ptid), "r"(__tls), "r"(__ctid) - : "lr", "memory"); + : "cc", "lr", "memory"); } + clone_exit: LSS_RETURN(int, __res); } #elif defined(__mips__) #undef LSS_REG #define LSS_REG(r,a) register unsigned long __r##r __asm__("$"#r) = \ (unsigned long)(a) + + #if _MIPS_SIM == _MIPS_SIM_ABI32 + // See http://sources.redhat.com/ml/libc-alpha/2004-10/msg00050.html + // or http://www.linux-mips.org/archives/linux-mips/2004-10/msg00142.html + #define MIPS_SYSCALL_CLOBBERS "$1", "$3", "$8", "$9", "$10", "$11", "$12",\ + "$13", "$14", "$15", "$24", "$25", "memory" + #else + #define MIPS_SYSCALL_CLOBBERS "$1", "$3", "$10", "$11", "$12", "$13", \ + "$14", "$15", "$24", "$25", "memory" + #endif + #undef LSS_BODY #define LSS_BODY(type,name,r7,...) \ register unsigned long __v0 __asm__("$2") = __NR_##name; \ __asm__ __volatile__ ("syscall\n" \ : "=&r"(__v0), r7 (__r7) \ : "0"(__v0), ##__VA_ARGS__ \ - : "$8", "$9", "$10", "$11", "$12", \ - "$13", "$14", "$15", "$24", "memory"); \ + : MIPS_SYSCALL_CLOBBERS); \ LSS_RETURN(type, __v0, __r7) #undef _syscall0 #define _syscall0(type, name) \ @@ -1911,8 +1442,7 @@ struct kernel_statfs { : "=&r"(__v0), "+r" (__r7) \ : "i" (__NR_##name), "r"(__r4), "r"(__r5), \ "r"(__r6), "m" ((unsigned long)arg5) \ - : "$8", "$9", "$10", "$11", "$12", \ - "$13", "$14", "$15", "$24", "memory"); \ + : MIPS_SYSCALL_CLOBBERS); \ LSS_RETURN(type, __v0, __r7); \ } #else @@ -1952,8 +1482,7 @@ struct kernel_statfs { : "i" (__NR_##name), "r"(__r4), "r"(__r5), \ "r"(__r6), "r" ((unsigned long)arg5), \ "r" ((unsigned long)arg6) \ - : "$8", "$9", "$10", "$11", "$12", \ - "$13", "$14", "$15", "$24", "memory"); \ + : MIPS_SYSCALL_CLOBBERS); \ LSS_RETURN(type, __v0, __r7); \ } #else @@ -2249,173 +1778,68 @@ struct kernel_statfs { #define __NR__exit __NR_exit #define __NR__gettid __NR_gettid #define __NR__mremap __NR_mremap - LSS_INLINE _syscall1(int, chdir, const char *,p) LSS_INLINE _syscall1(int, close, int, f) - LSS_INLINE _syscall2(int, clock_getres, int, c, - struct kernel_timespec*, t) - LSS_INLINE _syscall2(int, clock_gettime, int, c, - struct kernel_timespec*, t) - LSS_INLINE _syscall1(int, dup, int, f) - LSS_INLINE _syscall2(int, dup2, int, s, - int, d) - LSS_INLINE _syscall3(int, execve, const char*, f, - const char*const*,a,const char*const*, e) LSS_INLINE _syscall1(int, _exit, int, e) LSS_INLINE _syscall3(int, fcntl, int, f, int, c, long, a) - LSS_INLINE _syscall0(pid_t, fork) LSS_INLINE _syscall2(int, fstat, int, f, struct kernel_stat*, b) - LSS_INLINE _syscall2(int, fstatfs, int, f, - struct kernel_statfs*, b) LSS_INLINE _syscall4(int, futex, int*, a, int, o, int, v, struct kernel_timespec*, t) LSS_INLINE _syscall3(int, getdents, int, f, struct kernel_dirent*, d, int, c) +#ifdef __NR_getdents64 LSS_INLINE _syscall3(int, getdents64, int, f, struct kernel_dirent64*, d, int, c) - LSS_INLINE _syscall0(gid_t, getegid) - LSS_INLINE _syscall0(uid_t, geteuid) - LSS_INLINE _syscall0(pid_t, getpgrp) +#endif LSS_INLINE _syscall0(pid_t, getpid) LSS_INLINE _syscall0(pid_t, getppid) - LSS_INLINE _syscall2(int, getpriority, int, a, - int, b) - LSS_INLINE _syscall2(int, getrlimit, int, r, - struct kernel_rlimit*, l) - LSS_INLINE _syscall1(pid_t, getsid, pid_t, p) LSS_INLINE _syscall0(pid_t, _gettid) - LSS_INLINE _syscall5(int, setxattr, const char *,p, - const char *, n, const void *,v, - size_t, s, int, f) - LSS_INLINE _syscall5(int, lsetxattr, const char *,p, - const char *, n, const void *,v, - size_t, s, int, f) - LSS_INLINE _syscall4(ssize_t, getxattr, const char *,p, - const char *, n, void *, v, size_t, s) - LSS_INLINE _syscall4(ssize_t, lgetxattr, const char *,p, - const char *, n, void *, v, size_t, s) - LSS_INLINE _syscall3(ssize_t, listxattr, const char *,p, - char *, l, size_t, s) - LSS_INLINE _syscall3(ssize_t, llistxattr, const char *,p, - char *, l, size_t, s) - LSS_INLINE _syscall2(int, ioprio_get, int, which, - int, who) - LSS_INLINE _syscall3(int, ioprio_set, int, which, - int, who, int, ioprio) LSS_INLINE _syscall2(int, kill, pid_t, p, int, s) LSS_INLINE _syscall3(off_t, lseek, int, f, off_t, o, int, w) LSS_INLINE _syscall2(int, munmap, void*, s, size_t, l) - LSS_INLINE _syscall6(long, move_pages, pid_t, p, - unsigned long, n, void **,g, int *, d, - int *, s, int, f) LSS_INLINE _syscall5(void*, _mremap, void*, o, size_t, os, size_t, ns, unsigned long, f, void *, a) LSS_INLINE _syscall3(int, open, const char*, p, int, f, int, m) - LSS_INLINE _syscall3(int, poll, struct kernel_pollfd*, u, - unsigned int, n, int, t) LSS_INLINE _syscall2(int, prctl, int, o, long, a) LSS_INLINE _syscall4(long, ptrace, int, r, pid_t, p, void *, a, void *, d) LSS_INLINE _syscall3(ssize_t, read, int, f, void *, b, size_t, c) - LSS_INLINE _syscall3(int, readlink, const char*, p, - char*, b, size_t, s) LSS_INLINE _syscall4(int, rt_sigaction, int, s, const struct kernel_sigaction*, a, struct kernel_sigaction*, o, size_t, c) - LSS_INLINE _syscall2(int, rt_sigpending, struct kernel_sigset_t *, s, - size_t, c) LSS_INLINE _syscall4(int, rt_sigprocmask, int, h, const struct kernel_sigset_t*, s, struct kernel_sigset_t*, o, size_t, c); - LSS_INLINE _syscall2(int, rt_sigsuspend, - const struct kernel_sigset_t*, s, size_t, c); - LSS_INLINE _syscall3(int, sched_getaffinity,pid_t, p, - unsigned int, l, unsigned long *, m) - LSS_INLINE _syscall3(int, sched_setaffinity,pid_t, p, - unsigned int, l, unsigned long *, m) LSS_INLINE _syscall0(int, sched_yield) - LSS_INLINE _syscall1(long, set_tid_address, int *, t) - LSS_INLINE _syscall1(int, setfsgid, gid_t, g) - LSS_INLINE _syscall1(int, setfsuid, uid_t, u) - LSS_INLINE _syscall1(int, setuid, uid_t, u) - LSS_INLINE _syscall1(int, setgid, gid_t, g) - LSS_INLINE _syscall2(int, setpgid, pid_t, p, - pid_t, g) - LSS_INLINE _syscall3(int, setpriority, int, a, - int, b, int, p) - LSS_INLINE _syscall3(int, setresgid, gid_t, r, - gid_t, e, gid_t, s) - LSS_INLINE _syscall3(int, setresuid, uid_t, r, - uid_t, e, uid_t, s) - LSS_INLINE _syscall2(int, setrlimit, int, r, - const struct kernel_rlimit*, l) - LSS_INLINE _syscall0(pid_t, setsid) LSS_INLINE _syscall2(int, sigaltstack, const stack_t*, s, const stack_t*, o) LSS_INLINE _syscall2(int, stat, const char*, f, struct kernel_stat*, b) - LSS_INLINE _syscall2(int, statfs, const char*, f, - struct kernel_statfs*, b) LSS_INLINE _syscall3(ssize_t, write, int, f, const void *, b, size_t, c) - LSS_INLINE _syscall3(ssize_t, writev, int, f, - const struct kernel_iovec*, v, size_t, c) #if defined(__NR_getcpu) LSS_INLINE _syscall3(long, getcpu, unsigned *, cpu, unsigned *, node, void *, unused); #endif #if defined(__x86_64__) || \ (defined(__mips__) && _MIPS_SIM != _MIPS_SIM_ABI32) - LSS_INLINE _syscall3(int, recvmsg, int, s, - struct kernel_msghdr*, m, int, f) - LSS_INLINE _syscall3(int, sendmsg, int, s, - const struct kernel_msghdr*, m, int, f) - LSS_INLINE _syscall6(int, sendto, int, s, - const void*, m, size_t, l, - int, f, - const struct kernel_sockaddr*, a, int, t) - LSS_INLINE _syscall2(int, shutdown, int, s, - int, h) LSS_INLINE _syscall3(int, socket, int, d, int, t, int, p) - LSS_INLINE _syscall4(int, socketpair, int, d, - int, t, int, p, int*, s) #endif #if defined(__x86_64__) - LSS_INLINE _syscall4(int, fallocate, int, fd, int, mode, - loff_t, offset, loff_t, len) LSS_INLINE _syscall6(void*, mmap, void*, s, size_t, l, int, p, int, f, int, d, __off64_t, o) - LSS_INLINE _syscall4(int, newfstatat, int, d, - const char *, p, - struct kernel_stat*, b, int, f) - - LSS_INLINE int LSS_NAME(setfsgid32)(gid_t gid) { - return LSS_NAME(setfsgid)(gid); - } - - LSS_INLINE int LSS_NAME(setfsuid32)(uid_t uid) { - return LSS_NAME(setfsuid)(uid); - } - - LSS_INLINE int LSS_NAME(setresgid32)(gid_t rgid, gid_t egid, gid_t sgid) { - return LSS_NAME(setresgid)(rgid, egid, sgid); - } - - LSS_INLINE int LSS_NAME(setresuid32)(uid_t ruid, uid_t euid, uid_t suid) { - return LSS_NAME(setresuid)(ruid, euid, suid); - } LSS_INLINE int LSS_NAME(sigaction)(int signum, const struct kernel_sigaction *act, @@ -2437,114 +1861,35 @@ struct kernel_statfs { } } - LSS_INLINE int LSS_NAME(sigpending)(struct kernel_sigset_t *set) { - return LSS_NAME(rt_sigpending)(set, (KERNEL_NSIG+7)/8); - } - LSS_INLINE int LSS_NAME(sigprocmask)(int how, const struct kernel_sigset_t *set, struct kernel_sigset_t *oldset) { return LSS_NAME(rt_sigprocmask)(how, set, oldset, (KERNEL_NSIG+7)/8); } - - LSS_INLINE int LSS_NAME(sigsuspend)(const struct kernel_sigset_t *set) { - return LSS_NAME(rt_sigsuspend)(set, (KERNEL_NSIG+7)/8); - } #endif - #if defined(__x86_64__) || defined(__ARM_ARCH_3__) || \ + #if defined(__x86_64__) || \ + defined(__arm__) || \ (defined(__mips__) && _MIPS_SIM != _MIPS_SIM_ABI32) LSS_INLINE _syscall4(pid_t, wait4, pid_t, p, int*, s, int, o, - struct kernel_rusage*, r) - + struct kernel_rusage*, r) LSS_INLINE pid_t LSS_NAME(waitpid)(pid_t pid, int *status, int options){ return LSS_NAME(wait4)(pid, status, options, 0); } - #endif - #if defined(__i386__) || defined(__x86_64__) + #endif + #if defined(__i386__) || defined(__x86_64__) || defined(__arm__) LSS_INLINE _syscall4(int, openat, int, d, const char *, p, int, f, int, m) - LSS_INLINE _syscall3(int, unlinkat, int, d, const char *, p, int, f) - #endif - #if defined(__i386__) || defined(__ARM_ARCH_3__) - #define __NR__setfsgid32 __NR_setfsgid32 - #define __NR__setfsuid32 __NR_setfsuid32 - #define __NR__setresgid32 __NR_setresgid32 - #define __NR__setresuid32 __NR_setresuid32 - LSS_INLINE _syscall2(int, ugetrlimit, int, r, - struct kernel_rlimit*, l) - LSS_INLINE _syscall1(int, _setfsgid32, gid_t, f) - LSS_INLINE _syscall1(int, _setfsuid32, uid_t, f) - LSS_INLINE _syscall3(int, _setresgid32, gid_t, r, - gid_t, e, gid_t, s) - LSS_INLINE _syscall3(int, _setresuid32, uid_t, r, - uid_t, e, uid_t, s) - - LSS_INLINE int LSS_NAME(setfsgid32)(gid_t gid) { - int rc; - if ((rc = LSS_NAME(_setfsgid32)(gid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((unsigned int)gid & ~0xFFFFu) { - rc = EINVAL; - } else { - rc = LSS_NAME(setfsgid)(gid); - } - } - return rc; - } - - LSS_INLINE int LSS_NAME(setfsuid32)(uid_t uid) { - int rc; - if ((rc = LSS_NAME(_setfsuid32)(uid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((unsigned int)uid & ~0xFFFFu) { - rc = EINVAL; - } else { - rc = LSS_NAME(setfsuid)(uid); - } - } - return rc; - } - - LSS_INLINE int LSS_NAME(setresgid32)(gid_t rgid, gid_t egid, gid_t sgid) { - int rc; - if ((rc = LSS_NAME(_setresgid32)(rgid, egid, sgid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((unsigned int)rgid & ~0xFFFFu || - (unsigned int)egid & ~0xFFFFu || - (unsigned int)sgid & ~0xFFFFu) { - rc = EINVAL; - } else { - rc = LSS_NAME(setresgid)(rgid, egid, sgid); - } - } - return rc; - } - - LSS_INLINE int LSS_NAME(setresuid32)(uid_t ruid, uid_t euid, uid_t suid) { - int rc; - if ((rc = LSS_NAME(_setresuid32)(ruid, euid, suid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((unsigned int)ruid & ~0xFFFFu || - (unsigned int)euid & ~0xFFFFu || - (unsigned int)suid & ~0xFFFFu) { - rc = EINVAL; - } else { - rc = LSS_NAME(setresuid)(ruid, euid, suid); - } - } - return rc; - } #endif LSS_INLINE int LSS_NAME(sigemptyset)(struct kernel_sigset_t *set) { memset(&set->sig, 0, sizeof(set->sig)); return 0; } - + LSS_INLINE int LSS_NAME(sigfillset)(struct kernel_sigset_t *set) { memset(&set->sig, -1, sizeof(set->sig)); return 0; } - + LSS_INLINE int LSS_NAME(sigaddset)(struct kernel_sigset_t *set, int signum) { if (signum < 1 || signum > (int)(8*sizeof(set->sig))) { @@ -2556,7 +1901,7 @@ struct kernel_statfs { return 0; } } - + LSS_INLINE int LSS_NAME(sigdelset)(struct kernel_sigset_t *set, int signum) { if (signum < 1 || signum > (int)(8*sizeof(set->sig))) { @@ -2568,30 +1913,26 @@ struct kernel_statfs { return 0; } } - - LSS_INLINE int LSS_NAME(sigismember)(struct kernel_sigset_t *set, - int signum) { - if (signum < 1 || signum > (int)(8*sizeof(set->sig))) { - LSS_ERRNO = EINVAL; - return -1; - } else { - return !!(set->sig[(signum - 1)/(8*sizeof(set->sig[0]))] & - (1UL << ((signum - 1) % (8*sizeof(set->sig[0]))))); - } - } - #if defined(__i386__) || defined(__ARM_ARCH_3__) || \ + + #if defined(__i386__) || \ + defined(__arm__) || \ (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) || defined(__PPC__) #define __NR__sigaction __NR_sigaction - #define __NR__sigpending __NR_sigpending #define __NR__sigprocmask __NR_sigprocmask - #define __NR__sigsuspend __NR_sigsuspend - #define __NR__socketcall __NR_socketcall LSS_INLINE _syscall2(int, fstat64, int, f, struct kernel_stat64 *, b) LSS_INLINE _syscall5(int, _llseek, uint, fd, ulong, hi, ulong, lo, loff_t *, res, uint, wh) +#ifdef __PPC64__ + LSS_INLINE _syscall6(void*, mmap, void*, s, + size_t, l, int, p, + int, f, int, d, + off_t, o) +#else + #ifndef __ARM_EABI__ + /* Not available on ARM EABI Linux. */ LSS_INLINE _syscall1(void*, mmap, void*, a) -#ifndef __PPC64__ + #endif LSS_INLINE _syscall6(void*, mmap2, void*, s, size_t, l, int, p, int, f, int, d, @@ -2600,17 +1941,9 @@ struct kernel_statfs { LSS_INLINE _syscall3(int, _sigaction, int, s, const struct kernel_old_sigaction*, a, struct kernel_old_sigaction*, o) - LSS_INLINE _syscall1(int, _sigpending, unsigned long*, s) LSS_INLINE _syscall3(int, _sigprocmask, int, h, const unsigned long*, s, unsigned long*, o) - #ifdef __PPC__ - LSS_INLINE _syscall1(int, _sigsuspend, unsigned long, s) - #else - LSS_INLINE _syscall3(int, _sigsuspend, const void*, a, - int, b, - unsigned long, s) - #endif LSS_INLINE _syscall2(int, stat64, const char *, p, struct kernel_stat64 *, b) @@ -2676,17 +2009,6 @@ struct kernel_statfs { return rc; } - LSS_INLINE int LSS_NAME(sigpending)(struct kernel_sigset_t *set) { - int old_errno = LSS_ERRNO; - int rc = LSS_NAME(rt_sigpending)(set, (KERNEL_NSIG+7)/8); - if (rc < 0 && LSS_ERRNO == ENOSYS) { - LSS_ERRNO = old_errno; - LSS_NAME(sigemptyset)(set); - rc = LSS_NAME(_sigpending)(&set->sig[0]); - } - return rc; - } - LSS_INLINE int LSS_NAME(sigprocmask)(int how, const struct kernel_sigset_t *set, struct kernel_sigset_t *oldset) { @@ -2703,20 +2025,6 @@ struct kernel_statfs { } return rc; } - - LSS_INLINE int LSS_NAME(sigsuspend)(const struct kernel_sigset_t *set) { - int olderrno = LSS_ERRNO; - int rc = LSS_NAME(rt_sigsuspend)(set, (KERNEL_NSIG+7)/8); - if (rc < 0 && LSS_ERRNO == ENOSYS) { - LSS_ERRNO = olderrno; - rc = LSS_NAME(_sigsuspend)( - #ifndef __PPC__ - set, 0, - #endif - set->sig[0]); - } - return rc; - } #endif #if defined(__PPC__) #undef LSS_SC_LOADARGS_0 @@ -2773,90 +2081,31 @@ struct kernel_statfs { } \ LSS_RETURN(type, __sc_ret, __sc_err) - LSS_INLINE ssize_t LSS_NAME(recvmsg)(int s,struct kernel_msghdr *msg, - int flags){ - LSS_SC_BODY(3, ssize_t, 17, s, msg, flags); - } - - LSS_INLINE ssize_t LSS_NAME(sendmsg)(int s, - const struct kernel_msghdr *msg, - int flags) { - LSS_SC_BODY(3, ssize_t, 16, s, msg, flags); - } - - // TODO(csilvers): why is this ifdef'ed out? -#if 0 - LSS_INLINE ssize_t LSS_NAME(sendto)(int s, const void *buf, size_t len, - int flags, - const struct kernel_sockaddr *to, - unsigned int tolen) { - LSS_BODY(6, ssize_t, 11, s, buf, len, flags, to, tolen); - } -#endif - - LSS_INLINE int LSS_NAME(shutdown)(int s, int how) { - LSS_SC_BODY(2, int, 13, s, how); - } - LSS_INLINE int LSS_NAME(socket)(int domain, int type, int protocol) { LSS_SC_BODY(3, int, 1, domain, type, protocol); } - - LSS_INLINE int LSS_NAME(socketpair)(int d, int type, int protocol, - int sv[2]) { - LSS_SC_BODY(4, int, 8, d, type, protocol, sv); - } #endif - #if defined(__i386__) || defined(__ARM_ARCH_3__) || \ + #if defined(__i386__) || \ + (defined(__arm__) && !defined(__ARM_EABI__)) || \ (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) - #define __NR__socketcall __NR_socketcall - LSS_INLINE _syscall2(int, _socketcall, int, c, - va_list, a) - - LSS_INLINE int LSS_NAME(socketcall)(int op, ...) { - int rc; - va_list ap; - va_start(ap, op); - rc = LSS_NAME(_socketcall)(op, ap); - va_end(ap); - return rc; - } - - LSS_INLINE ssize_t LSS_NAME(recvmsg)(int s,struct kernel_msghdr *msg, - int flags){ - return (ssize_t)LSS_NAME(socketcall)(17, s, msg, flags); - } - LSS_INLINE ssize_t LSS_NAME(sendmsg)(int s, - const struct kernel_msghdr *msg, - int flags) { - return (ssize_t)LSS_NAME(socketcall)(16, s, msg, flags); - } - - LSS_INLINE ssize_t LSS_NAME(sendto)(int s, const void *buf, size_t len, - int flags, - const struct kernel_sockaddr *to, - unsigned int tolen) { - return (ssize_t)LSS_NAME(socketcall)(11, s, buf, len, flags, to, tolen); - } - - LSS_INLINE int LSS_NAME(shutdown)(int s, int how) { - return LSS_NAME(socketcall)(13, s, how); - } + /* See sys_socketcall in net/socket.c in kernel source. + * It de-multiplexes on its first arg and unpacks the arglist + * array in its second arg. + */ + LSS_INLINE _syscall2(long, socketcall, int, c, unsigned long*, a) LSS_INLINE int LSS_NAME(socket)(int domain, int type, int protocol) { - return LSS_NAME(socketcall)(1, domain, type, protocol); + unsigned long args[3] = { + (unsigned long) domain, + (unsigned long) type, + (unsigned long) protocol + }; + return LSS_NAME(socketcall)(1, args); } - - LSS_INLINE int LSS_NAME(socketpair)(int d, int type, int protocol, - int sv[2]) { - return LSS_NAME(socketcall)(8, d, type, protocol, sv); - } - #endif - #if defined(__i386__) || defined(__PPC__) - LSS_INLINE _syscall4(int, fstatat64, int, d, - const char *, p, - struct kernel_stat64 *, b, int, f) + #elif defined(__ARM_EABI__) + LSS_INLINE _syscall3(int, socket, int, d, + int, t, int, p) #endif #if defined(__i386__) || defined(__PPC__) || \ (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) @@ -2888,28 +2137,6 @@ struct kernel_statfs { #else LSS_INLINE _syscall1(int, pipe, int *, p) #endif - /* TODO(csilvers): see if ppc can/should support this as well */ - #if defined(__i386__) || defined(__ARM_ARCH_3__) || \ - (defined(__mips__) && _MIPS_SIM != _MIPS_SIM_ABI64) - #define __NR__statfs64 __NR_statfs64 - #define __NR__fstatfs64 __NR_fstatfs64 - LSS_INLINE _syscall3(int, _statfs64, const char*, p, - size_t, s,struct kernel_statfs64*, b) - LSS_INLINE _syscall3(int, _fstatfs64, int, f, - size_t, s,struct kernel_statfs64*, b) - LSS_INLINE int LSS_NAME(statfs64)(const char *p, - struct kernel_statfs64 *b) { - return LSS_NAME(_statfs64)(p, sizeof(*b), b); - } - LSS_INLINE int LSS_NAME(fstatfs64)(int f,struct kernel_statfs64 *b) { - return LSS_NAME(_fstatfs64)(f, sizeof(*b), b); - } - #endif - - LSS_INLINE int LSS_NAME(execv)(const char *path, const char *const argv[]) { - extern char **environ; - return LSS_NAME(execve)(path, argv, (const char *const *)environ); - } LSS_INLINE pid_t LSS_NAME(gettid)() { pid_t tid = LSS_NAME(_gettid)(); @@ -2946,72 +2173,6 @@ struct kernel_statfs { LSS_ERRNO = err; return rc; } - - LSS_INLINE int LSS_NAME(raise)(int sig) { - return LSS_NAME(kill)(LSS_NAME(getpid)(), sig); - } - - LSS_INLINE int LSS_NAME(setpgrp)() { - return LSS_NAME(setpgid)(0, 0); - } - - LSS_INLINE int LSS_NAME(sysconf)(int name) { - extern int __getpagesize(void); - switch (name) { - case _SC_OPEN_MAX: { - struct kernel_rlimit limit; - return LSS_NAME(getrlimit)(RLIMIT_NOFILE, &limit) < 0 - ? 8192 : limit.rlim_cur; - } - case _SC_PAGESIZE: - return __getpagesize(); - default: - errno = ENOSYS; - return -1; - } - } - #if defined(__x86_64__) || \ - (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI64) - LSS_INLINE _syscall4(ssize_t, pread64, int, f, - void *, b, size_t, c, - loff_t, o) - LSS_INLINE _syscall4(ssize_t, pwrite64, int, f, - const void *, b, size_t, c, - loff_t, o) - LSS_INLINE _syscall3(int, readahead, int, f, - loff_t, o, unsigned, c) - #else - #define __NR__pread64 __NR_pread64 - #define __NR__pwrite64 __NR_pwrite64 - #define __NR__readahead __NR_readahead - LSS_INLINE _syscall5(ssize_t, _pread64, int, f, - void *, b, size_t, c, unsigned, o1, - unsigned, o2) - LSS_INLINE _syscall5(ssize_t, _pwrite64, int, f, - const void *, b, size_t, c, unsigned, o1, - long, o2) - LSS_INLINE _syscall4(int, _readahead, int, f, - unsigned, o1, unsigned, o2, size_t, c); - /* We force 64bit-wide parameters onto the stack, then access each - * 32-bit component individually. This guarantees that we build the - * correct parameters independent of the native byte-order of the - * underlying architecture. - */ - LSS_INLINE ssize_t LSS_NAME(pread64)(int fd, void *buf, size_t count, - loff_t off) { - union { loff_t off; unsigned arg[2]; } o = { off }; - return LSS_NAME(_pread64)(fd, buf, count, o.arg[0], o.arg[1]); - } - LSS_INLINE ssize_t LSS_NAME(pwrite64)(int fd, const void *buf, - size_t count, loff_t off) { - union { loff_t off; unsigned arg[2]; } o = { off }; - return LSS_NAME(_pwrite64)(fd, buf, count, o.arg[0], o.arg[1]); - } - LSS_INLINE int LSS_NAME(readahead)(int fd, loff_t off, int len) { - union { loff_t off; unsigned arg[2]; } o = { off }; - return LSS_NAME(_readahead)(fd, o.arg[0], o.arg[1], len); - } - #endif #endif #if defined(__cplusplus) && !defined(SYS_CPLUSPLUS) diff --git a/third_party/tcmalloc/chromium/src/base/logging.h b/third_party/tcmalloc/chromium/src/base/logging.h index b24a030..d06d6a6 100644 --- a/third_party/tcmalloc/chromium/src/base/logging.h +++ b/third_party/tcmalloc/chromium/src/base/logging.h @@ -85,7 +85,7 @@ DECLARE_int32(verbose); if (!(condition)) { \ WRITE_TO_STDERR("Check failed: " #condition "\n", \ sizeof("Check failed: " #condition "\n")-1); \ - exit(1); \ + abort(); \ } \ } while (0) @@ -95,7 +95,7 @@ DECLARE_int32(verbose); if (!(condition)) { \ WRITE_TO_STDERR("Check failed: " #condition ": " message "\n", \ sizeof("Check failed: " #condition ": " message "\n")-1);\ - exit(1); \ + abort(); \ } \ } while (0) @@ -118,7 +118,7 @@ enum { DEBUG_MODE = 1 }; sizeof("Check failed: " #condition ": ")-1); \ WRITE_TO_STDERR(strerror(err_no), strlen(strerror(err_no))); \ WRITE_TO_STDERR("\n", sizeof("\n")-1); \ - exit(1); \ + abort(); \ } \ } while (0) @@ -135,7 +135,7 @@ enum { DEBUG_MODE = 1 }; do { \ if (!((val1) op (val2))) { \ fprintf(stderr, "Check failed: %s %s %s\n", #val1, #op, #val2); \ - exit(1); \ + abort(); \ } \ } while (0) diff --git a/third_party/tcmalloc/chromium/src/base/low_level_alloc.cc b/third_party/tcmalloc/chromium/src/base/low_level_alloc.cc index 532c594..c043cb6 100644 --- a/third_party/tcmalloc/chromium/src/base/low_level_alloc.cc +++ b/third_party/tcmalloc/chromium/src/base/low_level_alloc.cc @@ -38,7 +38,7 @@ #include "base/spinlock.h" #include "base/logging.h" #include "malloc_hook-inl.h" -#include <google/malloc_hook.h> +#include <gperftools/malloc_hook.h> #include <errno.h> #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -233,7 +233,7 @@ namespace { this->arena_->mu.Lock(); } ~ArenaLock() { RAW_CHECK(this->left_, "haven't left Arena region"); } - void Leave() UNLOCK_FUNCTION() { + void Leave() /*UNLOCK_FUNCTION()*/ { this->arena_->mu.Unlock(); #if 0 if (this->mask_valid_) { diff --git a/third_party/tcmalloc/chromium/src/base/spinlock_internal.cc b/third_party/tcmalloc/chromium/src/base/spinlock_internal.cc index b5b6ca4..b9fadde 100644 --- a/third_party/tcmalloc/chromium/src/base/spinlock_internal.cc +++ b/third_party/tcmalloc/chromium/src/base/spinlock_internal.cc @@ -42,6 +42,9 @@ #include "base/spinlock_internal.h" +// forward declaration for use by spinlock_*-inl.h +namespace base { namespace internal { static int SuggestedDelayNS(int loop); }} + #if defined(_WIN32) #include "base/spinlock_win32-inl.h" #elif defined(__linux__) @@ -73,5 +76,27 @@ int32 SpinLockWait(volatile Atomic32 *w, int n, return v; } +// Return a suggested delay in nanoseconds for iteration number "loop" +static int SuggestedDelayNS(int loop) { + // Weak pseudo-random number generator to get some spread between threads + // when many are spinning. + static base::subtle::Atomic64 rand; + uint64 r = base::subtle::NoBarrier_Load(&rand); + r = 0x5deece66dLL * r + 0xb; // numbers from nrand48() + base::subtle::NoBarrier_Store(&rand, r); + + r <<= 16; // 48-bit random number now in top 48-bits. + if (loop < 0 || loop > 32) { // limit loop to 0..32 + loop = 32; + } + // loop>>3 cannot exceed 4 because loop cannot exceed 32. + // Select top 20..24 bits of lower 48 bits, + // giving approximately 0ms to 16ms. + // Mean is exponential in loop for first 32 iterations, then 8ms. + // The futex path multiplies this by 16, since we expect explicit wakeups + // almost always on that path. + return r >> (44 - (loop >> 3)); +} + } // namespace internal } // namespace base diff --git a/third_party/tcmalloc/chromium/src/base/spinlock_linux-inl.h b/third_party/tcmalloc/chromium/src/base/spinlock_linux-inl.h index dc2c6ba..c190d52 100644 --- a/third_party/tcmalloc/chromium/src/base/spinlock_linux-inl.h +++ b/third_party/tcmalloc/chromium/src/base/spinlock_linux-inl.h @@ -31,6 +31,7 @@ * This file is a Linux-specific part of spinlock_internal.cc */ +#include <errno.h> #include <sched.h> #include <time.h> #include <limits.h> @@ -86,12 +87,12 @@ void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { struct timespec tm; tm.tv_sec = 0; if (have_futex) { - tm.tv_nsec = 1000000; // 1ms; really we're trying to sleep for one - // kernel clock tick + tm.tv_nsec = base::internal::SuggestedDelayNS(loop); } else { tm.tv_nsec = 2000001; // above 2ms so linux 2.4 doesn't spin } if (have_futex) { + tm.tv_nsec *= 16; // increase the delay; we expect explicit wakeups syscall(__NR_futex, reinterpret_cast<int *>(const_cast<Atomic32 *>(w)), FUTEX_WAIT | futex_private_flag, value, reinterpret_cast<struct kernel_timespec *>(&tm)); diff --git a/third_party/tcmalloc/chromium/src/base/spinlock_posix-inl.h b/third_party/tcmalloc/chromium/src/base/spinlock_posix-inl.h index d188ebd..e1d43b7 100644 --- a/third_party/tcmalloc/chromium/src/base/spinlock_posix-inl.h +++ b/third_party/tcmalloc/chromium/src/base/spinlock_posix-inl.h @@ -49,7 +49,7 @@ void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { } else { struct timespec tm; tm.tv_sec = 0; - tm.tv_nsec = 1000000; + tm.tv_nsec = base::internal::SuggestedDelayNS(loop); nanosleep(&tm, NULL); } errno = save_errno; diff --git a/third_party/tcmalloc/chromium/src/base/spinlock_win32-inl.h b/third_party/tcmalloc/chromium/src/base/spinlock_win32-inl.h index ee23541..9e77311 100644 --- a/third_party/tcmalloc/chromium/src/base/spinlock_win32-inl.h +++ b/third_party/tcmalloc/chromium/src/base/spinlock_win32-inl.h @@ -42,6 +42,13 @@ void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { } else if (loop == 1) { Sleep(0); } else { + // TODO(dmikurube): Re-enable the commented-out code. + // We commented out the following line and used the old code "Sleep(1)" + // since base/atomicops-internals-windows.h doesn't support 64-bit + // operations. + // + // Commended-out code: + // Sleep(base::internal::SuggestedDelayNS(loop) / 1000000); Sleep(1); } } diff --git a/third_party/tcmalloc/chromium/src/base/stl_allocator.h b/third_party/tcmalloc/chromium/src/base/stl_allocator.h index 22bd4ae..f7adb68 100644 --- a/third_party/tcmalloc/chromium/src/base/stl_allocator.h +++ b/third_party/tcmalloc/chromium/src/base/stl_allocator.h @@ -87,6 +87,7 @@ class STL_Allocator { size_type max_size() const { return size_t(-1) / sizeof(T); } void construct(pointer p, const T& val) { ::new(p) T(val); } + void construct(pointer p) { ::new(p) T(); } void destroy(pointer p) { p->~T(); } // There's no state, so these allocators are always equal diff --git a/third_party/tcmalloc/chromium/src/base/sysinfo.cc b/third_party/tcmalloc/chromium/src/base/sysinfo.cc index 285630e..3a1873e 100644 --- a/third_party/tcmalloc/chromium/src/base/sysinfo.cc +++ b/third_party/tcmalloc/chromium/src/base/sysinfo.cc @@ -86,12 +86,20 @@ // time, so prefer making the syscalls directly if we can. #ifdef HAVE_SYS_SYSCALL_H # include <sys/syscall.h> +#endif +#ifdef SYS_open // solaris 11, at least sometimes, only defines SYS_openat # define safeopen(filename, mode) syscall(SYS_open, filename, mode) -# define saferead(fd, buffer, size) syscall(SYS_read, fd, buffer, size) -# define safeclose(fd) syscall(SYS_close, fd) #else # define safeopen(filename, mode) open(filename, mode) +#endif +#ifdef SYS_read +# define saferead(fd, buffer, size) syscall(SYS_read, fd, buffer, size) +#else # define saferead(fd, buffer, size) read(fd, buffer, size) +#endif +#ifdef SYS_close +# define safeclose(fd) syscall(SYS_close, fd) +#else # define safeclose(fd) close(fd) #endif diff --git a/third_party/tcmalloc/chromium/src/base/vdso_support.cc b/third_party/tcmalloc/chromium/src/base/vdso_support.cc index 444be26..c98d9fa 100644 --- a/third_party/tcmalloc/chromium/src/base/vdso_support.cc +++ b/third_party/tcmalloc/chromium/src/base/vdso_support.cc @@ -43,8 +43,8 @@ #include <stddef.h> // for std::ptrdiff_t #include "base/atomicops.h" // for MemoryBarrier -#include "base/logging.h" #include "base/linux_syscall_support.h" +#include "base/logging.h" #include "base/dynamic_annotations.h" #include "base/basictypes.h" // for COMPILE_ASSERT @@ -54,285 +54,14 @@ using base::subtle::MemoryBarrier; #define AT_SYSINFO_EHDR 33 #endif -// From binutils/include/elf/common.h (this doesn't appear to be documented -// anywhere else). -// -// /* This flag appears in a Versym structure. It means that the symbol -// is hidden, and is only visible with an explicit version number. -// This is a GNU extension. */ -// #define VERSYM_HIDDEN 0x8000 -// -// /* This is the mask for the rest of the Versym information. */ -// #define VERSYM_VERSION 0x7fff - -#define VERSYM_VERSION 0x7fff - namespace base { -namespace { -template <int N> class ElfClass { - public: - static const int kElfClass = -1; - static int ElfBind(const ElfW(Sym) *) { - CHECK(false); // << "Unexpected word size"; - return 0; - } - static int ElfType(const ElfW(Sym) *) { - CHECK(false); // << "Unexpected word size"; - return 0; - } -}; - -template <> class ElfClass<32> { - public: - static const int kElfClass = ELFCLASS32; - static int ElfBind(const ElfW(Sym) *symbol) { - return ELF32_ST_BIND(symbol->st_info); - } - static int ElfType(const ElfW(Sym) *symbol) { - return ELF32_ST_TYPE(symbol->st_info); - } -}; - -template <> class ElfClass<64> { - public: - static const int kElfClass = ELFCLASS64; - static int ElfBind(const ElfW(Sym) *symbol) { - return ELF64_ST_BIND(symbol->st_info); - } - static int ElfType(const ElfW(Sym) *symbol) { - return ELF64_ST_TYPE(symbol->st_info); - } -}; - -typedef ElfClass<__WORDSIZE> CurrentElfClass; - -// Extract an element from one of the ELF tables, cast it to desired type. -// This is just a simple arithmetic and a glorified cast. -// Callers are responsible for bounds checking. -template <class T> -const T* GetTableElement(const ElfW(Ehdr) *ehdr, - ElfW(Off) table_offset, - ElfW(Word) element_size, - size_t index) { - return reinterpret_cast<const T*>(reinterpret_cast<const char *>(ehdr) - + table_offset - + index * element_size); -} -} // namespace - -const void *const VDSOSupport::kInvalidBase = - reinterpret_cast<const void *>(~0L); - -const void *VDSOSupport::vdso_base_ = kInvalidBase; +const void *VDSOSupport::vdso_base_ = ElfMemImage::kInvalidBase; VDSOSupport::GetCpuFn VDSOSupport::getcpu_fn_ = &InitAndGetCPU; - -VDSOSupport::ElfMemImage::ElfMemImage(const void *base) { - CHECK(base != kInvalidBase); - Init(base); -} - -int VDSOSupport::ElfMemImage::GetNumSymbols() const { - if (!hash_) { - return 0; - } - // See http://www.caldera.com/developers/gabi/latest/ch5.dynamic.html#hash - return hash_[1]; -} - -const ElfW(Sym) *VDSOSupport::ElfMemImage::GetDynsym(int index) const { - CHECK_LT(index, GetNumSymbols()); - return dynsym_ + index; -} - -const ElfW(Versym) *VDSOSupport::ElfMemImage::GetVersym(int index) const { - CHECK_LT(index, GetNumSymbols()); - return versym_ + index; -} - -const ElfW(Phdr) *VDSOSupport::ElfMemImage::GetPhdr(int index) const { - CHECK_LT(index, ehdr_->e_phnum); - return GetTableElement<ElfW(Phdr)>(ehdr_, - ehdr_->e_phoff, - ehdr_->e_phentsize, - index); -} - -const char *VDSOSupport::ElfMemImage::GetDynstr(ElfW(Word) offset) const { - CHECK_LT(offset, strsize_); - return dynstr_ + offset; -} - -const void *VDSOSupport::ElfMemImage::GetSymAddr(const ElfW(Sym) *sym) const { - if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) { - // Symbol corresponds to "special" (e.g. SHN_ABS) section. - return reinterpret_cast<const void *>(sym->st_value); - } - CHECK_LT(link_base_, sym->st_value); - return GetTableElement<char>(ehdr_, 0, 1, sym->st_value) - link_base_; -} - -const ElfW(Verdef) *VDSOSupport::ElfMemImage::GetVerdef(int index) const { - CHECK_LE(index, verdefnum_); - const ElfW(Verdef) *version_definition = verdef_; - while (version_definition->vd_ndx < index && version_definition->vd_next) { - const char *const version_definition_as_char = - reinterpret_cast<const char *>(version_definition); - version_definition = - reinterpret_cast<const ElfW(Verdef) *>(version_definition_as_char + - version_definition->vd_next); - } - return version_definition->vd_ndx == index ? version_definition : NULL; -} - -const ElfW(Verdaux) *VDSOSupport::ElfMemImage::GetVerdefAux( - const ElfW(Verdef) *verdef) const { - return reinterpret_cast<const ElfW(Verdaux) *>(verdef+1); -} - -const char *VDSOSupport::ElfMemImage::GetVerstr(ElfW(Word) offset) const { - CHECK_LT(offset, strsize_); - return dynstr_ + offset; -} - -void VDSOSupport::ElfMemImage::Init(const void *base) { - ehdr_ = NULL; - dynsym_ = NULL; - dynstr_ = NULL; - versym_ = NULL; - verdef_ = NULL; - hash_ = NULL; - strsize_ = 0; - verdefnum_ = 0; - link_base_ = ~0L; // Sentinel: PT_LOAD .p_vaddr can't possibly be this. - if (!base) { - return; - } - const intptr_t base_as_uintptr_t = reinterpret_cast<uintptr_t>(base); - // Fake VDSO has low bit set. - const bool fake_vdso = ((base_as_uintptr_t & 1) != 0); - base = reinterpret_cast<const void *>(base_as_uintptr_t & ~1); - const char *const base_as_char = reinterpret_cast<const char *>(base); - if (base_as_char[EI_MAG0] != ELFMAG0 || base_as_char[EI_MAG1] != ELFMAG1 || - base_as_char[EI_MAG2] != ELFMAG2 || base_as_char[EI_MAG3] != ELFMAG3) { - RAW_DCHECK(false, "no ELF magic"); // at %p", base); - return; - } - int elf_class = base_as_char[EI_CLASS]; - if (elf_class != CurrentElfClass::kElfClass) { - DCHECK_EQ(elf_class, CurrentElfClass::kElfClass); - return; - } - switch (base_as_char[EI_DATA]) { - case ELFDATA2LSB: { - if (__LITTLE_ENDIAN != __BYTE_ORDER) { - DCHECK_EQ(__LITTLE_ENDIAN, __BYTE_ORDER); // << ": wrong byte order"; - return; - } - break; - } - case ELFDATA2MSB: { - if (__BIG_ENDIAN != __BYTE_ORDER) { - DCHECK_EQ(__BIG_ENDIAN, __BYTE_ORDER); // << ": wrong byte order"; - return; - } - break; - } - default: { - RAW_DCHECK(false, "unexpected data encoding"); // << base_as_char[EI_DATA]; - return; - } - } - - ehdr_ = reinterpret_cast<const ElfW(Ehdr) *>(base); - const ElfW(Phdr) *dynamic_program_header = NULL; - for (int i = 0; i < ehdr_->e_phnum; ++i) { - const ElfW(Phdr) *const program_header = GetPhdr(i); - switch (program_header->p_type) { - case PT_LOAD: - if (link_base_ == ~0L) { - link_base_ = program_header->p_vaddr; - } - break; - case PT_DYNAMIC: - dynamic_program_header = program_header; - break; - } - } - if (link_base_ == ~0L || !dynamic_program_header) { - RAW_DCHECK(~0L != link_base_, "no PT_LOADs in VDSO"); - RAW_DCHECK(dynamic_program_header, "no PT_DYNAMIC in VDSO"); - // Mark this image as not present. Can not recur infinitely. - Init(0); - return; - } - std::ptrdiff_t relocation = - base_as_char - reinterpret_cast<const char *>(link_base_); - ElfW(Dyn) *dynamic_entry = - reinterpret_cast<ElfW(Dyn) *>(dynamic_program_header->p_vaddr + - relocation); - for (; dynamic_entry->d_tag != DT_NULL; ++dynamic_entry) { - ElfW(Xword) value = dynamic_entry->d_un.d_val; - if (fake_vdso) { - // A complication: in the real VDSO, dynamic entries are not relocated - // (it wasn't loaded by a dynamic loader). But when testing with a - // "fake" dlopen()ed vdso library, the loader relocates some (but - // not all!) of them before we get here. - if (dynamic_entry->d_tag == DT_VERDEF) { - // The only dynamic entry (of the ones we care about) libc-2.3.6 - // loader doesn't relocate. - value += relocation; - } - } else { - // Real VDSO. Everything needs to be relocated. - value += relocation; - } - switch (dynamic_entry->d_tag) { - case DT_HASH: - hash_ = reinterpret_cast<ElfW(Word) *>(value); - break; - case DT_SYMTAB: - dynsym_ = reinterpret_cast<ElfW(Sym) *>(value); - break; - case DT_STRTAB: - dynstr_ = reinterpret_cast<const char *>(value); - break; - case DT_VERSYM: - versym_ = reinterpret_cast<ElfW(Versym) *>(value); - break; - case DT_VERDEF: - verdef_ = reinterpret_cast<ElfW(Verdef) *>(value); - break; - case DT_VERDEFNUM: - verdefnum_ = dynamic_entry->d_un.d_val; - break; - case DT_STRSZ: - strsize_ = dynamic_entry->d_un.d_val; - break; - default: - // Unrecognized entries explicitly ignored. - break; - } - } - if (!hash_ || !dynsym_ || !dynstr_ || !versym_ || - !verdef_ || !verdefnum_ || !strsize_) { - RAW_DCHECK(hash_, "invalid VDSO (no DT_HASH)"); - RAW_DCHECK(dynsym_, "invalid VDSO (no DT_SYMTAB)"); - RAW_DCHECK(dynstr_, "invalid VDSO (no DT_STRTAB)"); - RAW_DCHECK(versym_, "invalid VDSO (no DT_VERSYM)"); - RAW_DCHECK(verdef_, "invalid VDSO (no DT_VERDEF)"); - RAW_DCHECK(verdefnum_, "invalid VDSO (no DT_VERDEFNUM)"); - RAW_DCHECK(strsize_, "invalid VDSO (no DT_STRSZ)"); - // Mark this image as not present. Can not recur infinitely. - Init(0); - return; - } -} - VDSOSupport::VDSOSupport() // If vdso_base_ is still set to kInvalidBase, we got here // before VDSOSupport::Init has been called. Call it now. - : image_(vdso_base_ == kInvalidBase ? Init() : vdso_base_) { + : image_(vdso_base_ == ElfMemImage::kInvalidBase ? Init() : vdso_base_) { } // NOTE: we can't use GoogleOnceInit() below, because we can be @@ -345,7 +74,7 @@ VDSOSupport::VDSOSupport() // Finally, even if there is a race here, it is harmless, because // the operation should be idempotent. const void *VDSOSupport::Init() { - if (vdso_base_ == kInvalidBase) { + if (vdso_base_ == ElfMemImage::kInvalidBase) { // Valgrind zaps AT_SYSINFO_EHDR and friends from the auxv[] // on stack, and so glibc works as if VDSO was not present. // But going directly to kernel via /proc/self/auxv below bypasses @@ -372,7 +101,7 @@ const void *VDSOSupport::Init() { } } close(fd); - if (vdso_base_ == kInvalidBase) { + if (vdso_base_ == ElfMemImage::kInvalidBase) { // Didn't find AT_SYSINFO_EHDR in auxv[]. vdso_base_ = NULL; } @@ -395,6 +124,7 @@ const void *VDSOSupport::Init() { } const void *VDSOSupport::SetBase(const void *base) { + CHECK(base != ElfMemImage::kInvalidBase); const void *old_base = vdso_base_; vdso_base_ = base; image_.Init(base); @@ -407,116 +137,12 @@ bool VDSOSupport::LookupSymbol(const char *name, const char *version, int type, SymbolInfo *info) const { - for (SymbolIterator it = begin(); it != end(); ++it) { - if (strcmp(it->name, name) == 0 && strcmp(it->version, version) == 0 && - CurrentElfClass::ElfType(it->symbol) == type) { - if (info) { - *info = *it; - } - return true; - } - } - return false; + return image_.LookupSymbol(name, version, type, info); } bool VDSOSupport::LookupSymbolByAddress(const void *address, SymbolInfo *info_out) const { - for (SymbolIterator it = begin(); it != end(); ++it) { - const char *const symbol_start = - reinterpret_cast<const char *>(it->address); - const char *const symbol_end = symbol_start + it->symbol->st_size; - if (symbol_start <= address && address < symbol_end) { - if (info_out) { - // Client wants to know details for that symbol (the usual case). - if (CurrentElfClass::ElfBind(it->symbol) == STB_GLOBAL) { - // Strong symbol; just return it. - *info_out = *it; - return true; - } else { - // Weak or local. Record it, but keep looking for a strong one. - *info_out = *it; - } - } else { - // Client only cares if there is an overlapping symbol. - return true; - } - } - } - return false; -} - -VDSOSupport::SymbolIterator::SymbolIterator(const void *const image, int index) - : index_(index), image_(image) { -} - -const VDSOSupport::SymbolInfo *VDSOSupport::SymbolIterator::operator->() const { - return &info_; -} - -const VDSOSupport::SymbolInfo& VDSOSupport::SymbolIterator::operator*() const { - return info_; -} - -bool VDSOSupport::SymbolIterator::operator==(const SymbolIterator &rhs) const { - return this->image_ == rhs.image_ && this->index_ == rhs.index_; -} - -bool VDSOSupport::SymbolIterator::operator!=(const SymbolIterator &rhs) const { - return !(*this == rhs); -} - -VDSOSupport::SymbolIterator &VDSOSupport::SymbolIterator::operator++() { - this->Update(1); - return *this; -} - -VDSOSupport::SymbolIterator VDSOSupport::begin() const { - SymbolIterator it(&image_, 0); - it.Update(0); - return it; -} - -VDSOSupport::SymbolIterator VDSOSupport::end() const { - return SymbolIterator(&image_, image_.GetNumSymbols()); -} - -void VDSOSupport::SymbolIterator::Update(int increment) { - const ElfMemImage *image = reinterpret_cast<const ElfMemImage *>(image_); - CHECK(image->IsPresent() || increment == 0); - if (!image->IsPresent()) { - return; - } - index_ += increment; - if (index_ >= image->GetNumSymbols()) { - index_ = image->GetNumSymbols(); - return; - } - const ElfW(Sym) *symbol = image->GetDynsym(index_); - const ElfW(Versym) *version_symbol = image->GetVersym(index_); - CHECK(symbol && version_symbol); - const char *const symbol_name = image->GetDynstr(symbol->st_name); - const ElfW(Versym) version_index = version_symbol[0] & VERSYM_VERSION; - const ElfW(Verdef) *version_definition = NULL; - const char *version_name = ""; - if (symbol->st_shndx == SHN_UNDEF) { - // Undefined symbols reference DT_VERNEED, not DT_VERDEF, and - // version_index could well be greater than verdefnum_, so calling - // GetVerdef(version_index) may trigger assertion. - } else { - version_definition = image->GetVerdef(version_index); - } - if (version_definition) { - // I am expecting 1 or 2 auxiliary entries: 1 for the version itself, - // optional 2nd if the version has a parent. - CHECK_LE(1, version_definition->vd_cnt); - CHECK_LE(version_definition->vd_cnt, 2); - const ElfW(Verdaux) *version_aux = image->GetVerdefAux(version_definition); - version_name = image->GetVerstr(version_aux->vda_name); - } - info_.name = symbol_name; - info_.version = version_name; - info_.address = image->GetSymAddr(symbol); - info_.symbol = symbol; + return image_.LookupSymbolByAddress(address, info_out); } // NOLINT on 'long' because this routine mimics kernel api. diff --git a/third_party/tcmalloc/chromium/src/base/vdso_support.h b/third_party/tcmalloc/chromium/src/base/vdso_support.h index 131646a..b97ab25 100644 --- a/third_party/tcmalloc/chromium/src/base/vdso_support.h +++ b/third_party/tcmalloc/chromium/src/base/vdso_support.h @@ -1,5 +1,34 @@ -// Copyright 2008 Google Inc. All Rights Reserved. -// Author: ppluzhnikov@google.com (Paul Pluzhnikov) +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Paul Pluzhnikov // // Allow dynamic symbol lookup in the kernel VDSO page. // @@ -27,19 +56,14 @@ #define BASE_VDSO_SUPPORT_H_ #include <config.h> -#ifdef HAVE_FEATURES_H -#include <features.h> // for __GLIBC__ -#endif #include "base/basictypes.h" +#include "base/elf_mem_image.h" -// Maybe one day we can rewrite this file not to require the elf -// symbol extensions in glibc, but for right now we need them. -#if defined(__ELF__) && defined(__GLIBC__) +#ifdef HAVE_ELF_MEM_IMAGE #define HAVE_VDSO_SUPPORT 1 #include <stdlib.h> // for NULL -#include <link.h> // for ElfW namespace base { @@ -47,45 +71,17 @@ namespace base { // use any memory allocation routines. class VDSOSupport { public: - // Sentinel: there could never be a VDSO at this address. - static const void *const kInvalidBase; - - // Information about a single vdso symbol. - // All pointers are into .dynsym, .dynstr, or .text of the VDSO. - // Do not free() them or modify through them. - struct SymbolInfo { - const char *name; // E.g. "__vdso_getcpu" - const char *version; // E.g. "LINUX_2.6", could be "" - // for unversioned symbol. - const void *address; // Relocated symbol address. - const ElfW(Sym) *symbol; // Symbol in the dynamic symbol table. - }; - - // Supports iteration over all dynamic symbols. - class SymbolIterator { - public: - friend class VDSOSupport; - const SymbolInfo *operator->() const; - const SymbolInfo &operator*() const; - SymbolIterator& operator++(); - bool operator!=(const SymbolIterator &rhs) const; - bool operator==(const SymbolIterator &rhs) const; - private: - SymbolIterator(const void *const image, int index); - void Update(int incr); - SymbolInfo info_; - int index_; - const void *const image_; - }; - VDSOSupport(); + typedef ElfMemImage::SymbolInfo SymbolInfo; + typedef ElfMemImage::SymbolIterator SymbolIterator; + // Answers whether we have a vdso at all. bool IsPresent() const { return image_.IsPresent(); } // Allow to iterate over all VDSO symbols. - SymbolIterator begin() const; - SymbolIterator end() const; + SymbolIterator begin() const { return image_.begin(); } + SymbolIterator end() const { return image_.end(); } // Look up versioned dynamic symbol in the kernel VDSO. // Returns false if VDSO is not present, or doesn't contain given @@ -111,33 +107,6 @@ class VDSOSupport { static const void *Init(); private: - // An in-memory ELF image (may not exist on disk). - class ElfMemImage { - public: - explicit ElfMemImage(const void *base); - void Init(const void *base); - bool IsPresent() const { return ehdr_ != NULL; } - const ElfW(Phdr)* GetPhdr(int index) const; - const ElfW(Sym)* GetDynsym(int index) const; - const ElfW(Versym)* GetVersym(int index) const; - const ElfW(Verdef)* GetVerdef(int index) const; - const ElfW(Verdaux)* GetVerdefAux(const ElfW(Verdef) *verdef) const; - const char* GetDynstr(ElfW(Word) offset) const; - const void* GetSymAddr(const ElfW(Sym) *sym) const; - const char* GetVerstr(ElfW(Word) offset) const; - int GetNumSymbols() const; - private: - const ElfW(Ehdr) *ehdr_; - const ElfW(Sym) *dynsym_; - const ElfW(Versym) *versym_; - const ElfW(Verdef) *verdef_; - const ElfW(Word) *hash_; - const char *dynstr_; - size_t strsize_; - size_t verdefnum_; - ElfW(Addr) link_base_; // Link-time base (p_vaddr of first PT_LOAD). - }; - // image_ represents VDSO ELF image in memory. // image_.ehdr_ == NULL implies there is no VDSO. ElfMemImage image_; @@ -181,6 +150,6 @@ class VDSOSupport { int GetCPU(); } // namespace base -#endif // __ELF__ and __GLIBC__ +#endif // HAVE_ELF_MEM_IMAGE #endif // BASE_VDSO_SUPPORT_H_ diff --git a/third_party/tcmalloc/chromium/src/central_freelist.cc b/third_party/tcmalloc/chromium/src/central_freelist.cc index fff1260..0f8a5c0 100644 --- a/third_party/tcmalloc/chromium/src/central_freelist.cc +++ b/third_party/tcmalloc/chromium/src/central_freelist.cc @@ -31,28 +31,51 @@ // Author: Sanjay Ghemawat <opensource@google.com> #include "config.h" +#include <algorithm> #include "central_freelist.h" #include "free_list.h" // for FL_Next, FL_Push, etc #include "internal_logging.h" // for ASSERT, MESSAGE #include "page_heap.h" // for PageHeap #include "static_vars.h" // for Static +using std::min; +using std::max; + namespace tcmalloc { void CentralFreeList::Init(size_t cl) { size_class_ = cl; tcmalloc::DLL_Init(&empty_); tcmalloc::DLL_Init(&nonempty_); + num_spans_ = 0; counter_ = 0; + max_cache_size_ = kMaxNumTransferEntries; #ifdef TCMALLOC_SMALL_BUT_SLOW // Disable the transfer cache for the small footprint case. cache_size_ = 0; #else cache_size_ = 16; #endif + if (cl > 0) { + // Limit the maximum size of the cache based on the size class. If this + // is not done, large size class objects will consume a lot of memory if + // they just sit in the transfer cache. + int32_t bytes = Static::sizemap()->ByteSizeForClass(cl); + int32_t objs_to_move = Static::sizemap()->num_objects_to_move(cl); + + ASSERT(objs_to_move > 0 && bytes > 0); + // Limit each size class cache to at most 1MB of objects or one entry, + // whichever is greater. Total transfer cache memory used across all + // size classes then can't be greater than approximately + // 1MB * kMaxNumTransferEntries. + // min and max are in parens to avoid macro-expansion on windows. + max_cache_size_ = (min)(max_cache_size_, + (max)(1, (1024 * 1024) / (bytes * objs_to_move))); + cache_size_ = (min)(cache_size_, max_cache_size_); + } used_slots_ = 0; - ASSERT(cache_size_ <= kNumTransferEntries); + ASSERT(cache_size_ <= max_cache_size_); } void CentralFreeList::ReleaseListToSpans(void* start) { @@ -109,6 +132,7 @@ void CentralFreeList::ReleaseToSpans(void* object) { counter_ -= ((span->length<<kPageShift) / Static::sizemap()->ByteSizeForClass(span->sizeclass)); tcmalloc::DLL_Remove(span); + --num_spans_; // Release central list lock while operating on pageheap lock_.Unlock(); @@ -142,7 +166,7 @@ bool CentralFreeList::MakeCacheSpace() { // Is there room in the cache? if (used_slots_ < cache_size_) return true; // Check if we can expand this cache? - if (cache_size_ == kNumTransferEntries) return false; + if (cache_size_ == max_cache_size_) return false; // Ok, we'll try to grab an entry from some other size class. if (EvictRandomSizeClass(size_class_, false) || EvictRandomSizeClass(size_class_, true)) { @@ -151,7 +175,7 @@ bool CentralFreeList::MakeCacheSpace() { // EvictRandomSizeClass (via ShrinkCache and the LockInverter), so the // cache_size may have changed. Therefore, check and verify that it is // still OK to increase the cache_size. - if (cache_size_ < kNumTransferEntries) { + if (cache_size_ < max_cache_size_) { cache_size_++; return true; } @@ -208,7 +232,7 @@ void CentralFreeList::InsertRange(void *start, void *end, int N) { MakeCacheSpace()) { int slot = used_slots_++; ASSERT(slot >=0); - ASSERT(slot < kNumTransferEntries); + ASSERT(slot < max_cache_size_); TCEntry *entry = &tc_slots_[slot]; entry->head = start; entry->tail = end; @@ -292,7 +316,8 @@ void CentralFreeList::Populate() { if (span) Static::pageheap()->RegisterSizeClass(span, size_class_); } if (span == NULL) { - MESSAGE("tcmalloc: allocation failed", npages << kPageShift); + Log(kLog, __FILE__, __LINE__, + "tcmalloc: allocation failed", npages << kPageShift); lock_.Lock(); return; } @@ -323,6 +348,7 @@ void CentralFreeList::Populate() { // Add span to list of non-empty spans lock_.Lock(); tcmalloc::DLL_Prepend(&nonempty_, span); + ++num_spans_; counter_ += num; } @@ -331,4 +357,16 @@ int CentralFreeList::tc_length() { return used_slots_ * Static::sizemap()->num_objects_to_move(size_class_); } +size_t CentralFreeList::OverheadBytes() { + SpinLockHolder h(&lock_); + if (size_class_ == 0) { // 0 holds the 0-sized allocations + return 0; + } + const size_t pages_per_span = Static::sizemap()->class_to_pages(size_class_); + const size_t object_size = Static::sizemap()->class_to_size(size_class_); + ASSERT(object_size > 0); + const size_t overhead_per_span = (pages_per_span * kPageSize) % object_size; + return num_spans_ * overhead_per_span; +} + } // namespace tcmalloc diff --git a/third_party/tcmalloc/chromium/src/central_freelist.h b/third_party/tcmalloc/chromium/src/central_freelist.h index e09210a..4fd5799 100644 --- a/third_party/tcmalloc/chromium/src/central_freelist.h +++ b/third_party/tcmalloc/chromium/src/central_freelist.h @@ -48,6 +48,11 @@ namespace tcmalloc { // Data kept per size-class in central cache. class CentralFreeList { public: + // A CentralFreeList may be used before its constructor runs. + // So we prevent lock_'s constructor from doing anything to the + // lock_ state. + CentralFreeList() : lock_(base::LINKER_INITIALIZED) { } + void Init(size_t cl); // These methods all do internal locking. @@ -68,6 +73,12 @@ class CentralFreeList { // Returns the number of free objects in the transfer cache. int tc_length(); + // Returns the memory overhead (internal fragmentation) attributable + // to the freelist. This is memory lost when the size of elements + // in a freelist doesn't exactly divide the page-size (an 8192-byte + // page full of 5-byte objects would have 2 bytes memory overhead). + size_t OverheadBytes(); + private: // TransferCache is used to cache transfers of // sizemap.num_objects_to_move(size_class) back and forth between @@ -77,16 +88,16 @@ class CentralFreeList { void *tail; // Tail of chain of objects. }; - // A central cache freelist can have anywhere from 0 to kNumTransferEntries - // slots to put link list chains into. To keep memory usage bounded the total - // number of TCEntries across size classes is fixed. Currently each size - // class is initially given one TCEntry which also means that the maximum any - // one class can have is kNumClasses. + // A central cache freelist can have anywhere from 0 to kMaxNumTransferEntries + // slots to put link list chains into. #ifdef TCMALLOC_SMALL_BUT_SLOW // For the small memory model, the transfer cache is not used. - static const int kNumTransferEntries = 0; + static const int kMaxNumTransferEntries = 0; #else - static const int kNumTransferEntries = kNumClasses; + // Starting point for the the maximum number of entries in the transfer cache. + // This actual maximum for a given size class may be lower than this + // maximum value. + static const int kMaxNumTransferEntries = 64; #endif // REQUIRES: lock_ is held @@ -145,12 +156,15 @@ class CentralFreeList { size_t size_class_; // My size class Span empty_; // Dummy header for list of empty spans Span nonempty_; // Dummy header for list of non-empty spans + size_t num_spans_; // Number of spans in empty_ plus nonempty_ size_t counter_; // Number of free objects in cache entry - // Here we reserve space for TCEntry cache slots. Since one size class can - // end up getting all the TCEntries quota in the system we just preallocate - // sufficient number of entries here. - TCEntry tc_slots_[kNumTransferEntries]; + // Here we reserve space for TCEntry cache slots. Space is preallocated + // for the largest possible number of entries than any one size class may + // accumulate. Not all size classes are allowed to accumulate + // kMaxNumTransferEntries, so there is some wasted space for those size + // classes. + TCEntry tc_slots_[kMaxNumTransferEntries]; // Number of currently used cached entries in tc_slots_. This variable is // updated under a lock but can be read without one. @@ -159,6 +173,8 @@ class CentralFreeList { // adaptive value that is increased if there is lots of traffic // on a given size class. int32_t cache_size_; + // Maximum size of the cache for a given size class. + int32_t max_cache_size_; }; // Pads each CentralCache object to multiple of 64 bytes. Since some diff --git a/third_party/tcmalloc/chromium/src/common.cc b/third_party/tcmalloc/chromium/src/common.cc index 69514b6..dd175f2 100644 --- a/third_party/tcmalloc/chromium/src/common.cc +++ b/third_party/tcmalloc/chromium/src/common.cc @@ -58,9 +58,9 @@ static inline int LgFloor(size_t n) { int AlignmentForSize(size_t size) { int alignment = kAlignment; - if (size >= 2048) { - // Cap alignment at 256 for large sizes. - alignment = 256; + if (size > kMaxSize) { + // Cap alignment at kPageSize for large sizes. + alignment = kPageSize; } else if (size >= 128) { // Space wasted due to alignment is at most 1/8, i.e., 12.5%. alignment = (1 << LgFloor(size)) / 8; @@ -69,6 +69,10 @@ int AlignmentForSize(size_t size) { // requirements for some SSE types. alignment = 16; } + // Maximum alignment allowed is page size alignment. + if (alignment > kPageSize) { + alignment = kPageSize; + } CHECK_CONDITION(size < 16 || alignment >= 16); CHECK_CONDITION((alignment & (alignment - 1)) == 0); return alignment; @@ -99,32 +103,35 @@ int SizeMap::NumMoveSize(size_t size) { void SizeMap::Init() { // Do some sanity checking on add_amount[]/shift_amount[]/class_array[] if (ClassIndex(0) < 0) { - CRASH("Invalid class index %d for size 0\n", ClassIndex(0)); + Log(kCrash, __FILE__, __LINE__, + "Invalid class index for size 0", ClassIndex(0)); } if (ClassIndex(kMaxSize) >= sizeof(class_array_)) { - CRASH("Invalid class index %d for kMaxSize\n", ClassIndex(kMaxSize)); + Log(kCrash, __FILE__, __LINE__, + "Invalid class index for kMaxSize", ClassIndex(kMaxSize)); } // Compute the size classes we want to use int sc = 1; // Next size class to assign int alignment = kAlignment; CHECK_CONDITION(kAlignment <= 16); - int last_lg = -1; for (size_t size = kMinClassSize; size <= kMaxSize; size += alignment) { - int lg = LgFloor(size); - if (lg > last_lg) { - // Increase alignment every so often to reduce number of size classes. - alignment = AlignmentForSize(size); - last_lg = lg; - } + alignment = AlignmentForSize(size); CHECK_CONDITION((size % alignment) == 0); - // Allocate enough pages so leftover is less than 1/8 of total. - // This bounds wasted space to at most 12.5%. - size_t psize = kPageSize; - while ((psize % size) > (psize >> 3)) { + int blocks_to_move = NumMoveSize(size) / 4; + size_t psize = 0; + do { psize += kPageSize; - } + // Allocate enough pages so leftover is less than 1/8 of total. + // This bounds wasted space to at most 12.5%. + while ((psize % size) > (psize >> 3)) { + psize += kPageSize; + } + // Continue to add pages until there are at least as many objects in + // the span as are needed when moving objects from the central + // freelists and spans to the thread caches. + } while ((psize / size) < (blocks_to_move)); const size_t my_pages = psize >> kPageShift; if (sc > 1 && my_pages == class_to_pages_[sc-1]) { @@ -146,8 +153,8 @@ void SizeMap::Init() { sc++; } if (sc != kNumClasses) { - CRASH("wrong number of size classes: found %d instead of %d\n", - sc, int(kNumClasses)); + Log(kCrash, __FILE__, __LINE__, + "wrong number of size classes: (found vs. expected )", sc, kNumClasses); } // Initialize the mapping arrays @@ -164,18 +171,17 @@ void SizeMap::Init() { for (size_t size = 0; size <= kMaxSize; size++) { const int sc = SizeClass(size); if (sc <= 0 || sc >= kNumClasses) { - CRASH("Bad size class %d for %" PRIuS "\n", sc, size); + Log(kCrash, __FILE__, __LINE__, + "Bad size class (class, size)", sc, size); } if (sc > 1 && size <= class_to_size_[sc-1]) { - CRASH("Allocating unnecessarily large class %d for %" PRIuS - "\n", sc, size); + Log(kCrash, __FILE__, __LINE__, + "Allocating unnecessarily large class (class, size)", sc, size); } const size_t s = class_to_size_[sc]; - if (size > s) { - CRASH("Bad size %" PRIuS " for %" PRIuS " (sc = %d)\n", s, size, sc); - } - if (s == 0) { - CRASH("Bad size %" PRIuS " for %" PRIuS " (sc = %d)\n", s, size, sc); + if (size > s || s == 0) { + Log(kCrash, __FILE__, __LINE__, + "Bad (class, size, requested)", sc, s, size); } } @@ -185,23 +191,6 @@ void SizeMap::Init() { } } -void SizeMap::Dump(TCMalloc_Printer* out) { - // Dump class sizes and maximum external wastage per size class - for (size_t cl = 1; cl < kNumClasses; ++cl) { - const int alloc_size = class_to_pages_[cl] << kPageShift; - const int alloc_objs = alloc_size / class_to_size_[cl]; - const int min_used = (class_to_size_[cl-1] + 1) * alloc_objs; - const int max_waste = alloc_size - min_used; - out->printf("SC %3d [ %8d .. %8d ] from %8d ; %2.0f%% maxwaste\n", - int(cl), - int(class_to_size_[cl-1] + 1), - int(class_to_size_[cl]), - int(class_to_pages_[cl] << kPageShift), - max_waste * 100.0 / alloc_size - ); - } -} - // Metadata allocator -- keeps stats about how many bytes allocated. static uint64_t metadata_system_bytes_ = 0; void* MetaDataAlloc(size_t bytes) { diff --git a/third_party/tcmalloc/chromium/src/common.h b/third_party/tcmalloc/chromium/src/common.h index 78cdc03..4f848fa 100644 --- a/third_party/tcmalloc/chromium/src/common.h +++ b/third_party/tcmalloc/chromium/src/common.h @@ -31,6 +31,7 @@ // Author: Sanjay Ghemawat <opensource@google.com> // // Common definitions for tcmalloc code. + #ifndef TCMALLOC_COMMON_H_ #define TCMALLOC_COMMON_H_ @@ -63,7 +64,8 @@ typedef uintptr_t Length; static const size_t kAlignment = 8; -// Constants dependant on tcmalloc configuration and archetecture. +// Constants dependant on tcmalloc configuration and archetecture. Chromium +// tunes these constants. // We need to guarantee the smallest class size is big enough to hold the // pointers that form the free list. static const size_t kNumFreeListPointers = @@ -75,16 +77,16 @@ static const size_t kSkippedClasses = (kAlignment < kMinClassSize ? 1 : 0); #if defined(TCMALLOC_LARGE_PAGES) static const size_t kPageShift = 15; -static const size_t kNumClasses = 95 - kSkippedClasses; -static const size_t kMaxThreadCacheSize = 4 << 20; +static const size_t kNumClasses = 78 - kSkippedClasses; #else -static const size_t kPageShift = 12; -static const size_t kNumClasses = 61 - kSkippedClasses; -static const size_t kMaxThreadCacheSize = 2 << 20; +static const size_t kPageShift = 13; +static const size_t kNumClasses = 86 - kSkippedClasses; #endif +static const size_t kMaxThreadCacheSize = 4 << 20; static const size_t kPageSize = 1 << kPageShift; -static const size_t kMaxSize = 8u * kPageSize; +// TODO(dmikurube): We Chromium may want to tune this kMaxSize. +static const size_t kMaxSize = 256 * 1024; // For all span-lengths < kMaxPages we keep an exact-size list. static const size_t kMaxPages = 1 << (20 - kPageShift); @@ -176,7 +178,7 @@ class SizeMap { // 32768 (32768 + 127 + (120<<7)) / 128 376 static const int kMaxSmallSize = 1024; static const size_t kClassArraySize = - (((1 << kPageShift) * 8u + 127 + (120 << 7)) >> 7) + 1; + ((kMaxSize + 127 + (120 << 7)) >> 7) + 1; unsigned char class_array_[kClassArraySize]; // Compute index of the class_array[] entry for a given size @@ -232,9 +234,6 @@ class SizeMap { inline int num_objects_to_move(size_t cl) { return num_objects_to_move_[cl]; } - - // Dump contents of the computed size map - void Dump(TCMalloc_Printer* out); }; // Allocates "bytes" worth of memory and returns it. Increments diff --git a/third_party/tcmalloc/chromium/src/config.h b/third_party/tcmalloc/chromium/src/config.h index 047b878..90e687f 100644 --- a/third_party/tcmalloc/chromium/src/config.h +++ b/third_party/tcmalloc/chromium/src/config.h @@ -6,11 +6,6 @@ #include "build/build_config.h" -#define TC_VERSION_MAJOR 1 -#define TC_VERSION_MINOR 4 -#define TC_VERSION_PATCH "" -#define TC_VERSION_STRING "google-perftools 1.4" - #if defined(OS_WIN) #include "third_party/tcmalloc/chromium/src/config_win.h" #elif defined(OS_LINUX) diff --git a/third_party/tcmalloc/chromium/src/config.h.in b/third_party/tcmalloc/chromium/src/config.h.in index 5ba784e..43f9f36 100644 --- a/third_party/tcmalloc/chromium/src/config.h.in +++ b/third_party/tcmalloc/chromium/src/config.h.in @@ -1,8 +1,8 @@ /* src/config.h.in. Generated from configure.ac by autoheader. */ -#ifndef GOOGLE_PERFTOOLS_CONFIG_H_ -#define GOOGLE_PERFTOOLS_CONFIG_H_ +#ifndef GPERFTOOLS_CONFIG_H_ +#define GPERFTOOLS_CONFIG_H_ /* Define to 1 if compiler supports __builtin_stack_pointer */ @@ -192,6 +192,9 @@ */ #undef LT_OBJDIR +/* Define to 'volatile' if __malloc_hook is declared volatile */ +#undef MALLOC_HOOK_MAYBE_VOLATILE + /* Define to 1 if your C compiler doesn't accept -c and -o together. */ #undef NO_MINUS_C_MINUS_O @@ -235,6 +238,12 @@ /* printf format code for printing a size_t and ssize_t */ #undef PRIxS +/* Mark the systems where we know it's bad if pthreads runs too + early before main (before threads are initialized, presumably). */ +#ifdef __FreeBSD__ +#define PTHREADS_CRASHES_IF_RUN_TOO_EARLY 1 +#endif + /* Define to necessary symbol if this constant uses a non-standard name on your system. */ #undef PTHREAD_CREATE_JOINABLE @@ -264,4 +273,5 @@ #include "windows/mingw.h" #endif -#endif /* #ifndef GOOGLE_PERFTOOLS_CONFIG_H_ */ +#endif /* #ifndef GPERFTOOLS_CONFIG_H_ */ + diff --git a/third_party/tcmalloc/chromium/src/debugallocation.cc b/third_party/tcmalloc/chromium/src/debugallocation.cc index 9de927a..70ec162 100644 --- a/third_party/tcmalloc/chromium/src/debugallocation.cc +++ b/third_party/tcmalloc/chromium/src/debugallocation.cc @@ -31,6 +31,13 @@ // Author: Urs Holzle <opensource@google.com> #include "config.h" +#include <errno.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#ifdef HAVE_INTTYPES_H +#include <inttypes.h> +#endif // We only need malloc.h for struct mallinfo. #ifdef HAVE_STRUCT_MALLINFO // Malloc can be in several places on older versions of OS X. @@ -42,34 +49,29 @@ # include <sys/malloc.h> # endif #endif +#ifdef HAVE_PTHREAD #include <pthread.h> -#include <stdio.h> -#ifdef HAVE_INTTYPES_H -#include <inttypes.h> #endif #include <stdarg.h> +#include <stdio.h> +#include <string.h> #ifdef HAVE_MMAP #include <sys/mman.h> #endif -#include <sys/types.h> #include <sys/stat.h> -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif +#include <sys/types.h> #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#include <errno.h> -#include <string.h> -#include <google/malloc_extension.h> -#include <google/malloc_hook.h> -#include <google/stacktrace.h> +#include <gperftools/malloc_extension.h> +#include <gperftools/malloc_hook.h> +#include <gperftools/stacktrace.h> +#include "addressmap-inl.h" #include "base/commandlineflags.h" #include "base/googleinit.h" #include "base/logging.h" #include "base/spinlock.h" -#include "addressmap-inl.h" #include "malloc_hook-inl.h" #include "symbolize.h" @@ -124,6 +126,13 @@ DEFINE_bool(symbolize_stacktrace, EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true), "Symbolize the stack trace when provided (on some error exits)"); +// If we are LD_PRELOAD-ed against a non-pthreads app, then +// pthread_once won't be defined. We declare it here, for that +// case (with weak linkage) which will cause the non-definition to +// resolve to NULL. We can then check for NULL or not in Instance. +extern "C" int pthread_once(pthread_once_t *, void (*)(void)) + ATTRIBUTE_WEAK; + // ========================================================================= // // A safe version of printf() that does not do any allocation and @@ -134,13 +143,13 @@ static void TracePrintf(int fd, const char *fmt, ...) // The do_* functions are defined in tcmalloc/tcmalloc.cc, // which is included before this file // when TCMALLOC_FOR_DEBUGALLOCATION is defined -#define BASE_MALLOC_NEW(size) cpp_alloc(size, false) +// TODO(csilvers): get rid of these now that we are tied to tcmalloc. +#define BASE_MALLOC_NEW do_malloc #define BASE_MALLOC do_malloc #define BASE_FREE do_free #define BASE_MALLOC_STATS do_malloc_stats #define BASE_MALLOPT do_mallopt #define BASE_MALLINFO do_mallinfo -#define BASE_MALLOC_SIZE(ptr) GetSizeWithCallback(ptr, &InvalidGetAllocatedSize) // ========================================================================= // @@ -169,6 +178,7 @@ class FreeQueue { } QueueEntry Pop() { + RAW_CHECK(q_back_ != q_front_, "Queue is empty"); const QueueEntry& ret = q_[q_back_]; q_back_ = (q_back_ + 1) % kFreeQueueSize; return ret; @@ -191,7 +201,7 @@ struct MallocBlockQueueEntry { MallocBlockQueueEntry() : block(NULL), size(0), num_deleter_pcs(0), deleter_threadid(0) {} MallocBlockQueueEntry(MallocBlock* b, size_t s) : block(b), size(s) { - if (FLAGS_max_free_queue_size != 0) { + if (FLAGS_max_free_queue_size != 0 && b != NULL) { // Adjust the number of frames to skip (4) if you change the // location of this call. num_deleter_pcs = @@ -266,7 +276,8 @@ class MallocBlock { // This array will be filled with 0xCD, for use with memcmp. static unsigned char kMagicDeletedBuffer[1024]; - static bool deleted_buffer_initialized_; + static pthread_once_t deleted_buffer_initialized_; + static bool deleted_buffer_initialized_no_pthreads_; private: // data layout @@ -561,14 +572,18 @@ class MallocBlock { static void ProcessFreeQueue(MallocBlock* b, size_t size, int max_free_queue_size) { - SpinLockHolder l(&free_queue_lock_); + // MallocBlockQueueEntry are about 144 in size, so we can only + // use a small array of them on the stack. + MallocBlockQueueEntry entries[4]; + int num_entries = 0; + MallocBlockQueueEntry new_entry(b, size); + free_queue_lock_.Lock(); if (free_queue_ == NULL) free_queue_ = new FreeQueue<MallocBlockQueueEntry>; RAW_CHECK(!free_queue_->Full(), "Free queue mustn't be full!"); if (b != NULL) { free_queue_size_ += size + sizeof(MallocBlockQueueEntry); - MallocBlockQueueEntry new_entry(b, size); free_queue_->Push(new_entry); } @@ -576,20 +591,46 @@ class MallocBlock { // max_free_queue_size, and the free queue has at least one free // space in it. while (free_queue_size_ > max_free_queue_size || free_queue_->Full()) { - MallocBlockQueueEntry cur = free_queue_->Pop(); - CheckForDanglingWrites(cur); - free_queue_size_ -= cur.size + sizeof(MallocBlockQueueEntry); - BASE_FREE(cur.block); + RAW_CHECK(num_entries < arraysize(entries), "entries array overflow"); + entries[num_entries] = free_queue_->Pop(); + free_queue_size_ -= + entries[num_entries].size + sizeof(MallocBlockQueueEntry); + num_entries++; + if (num_entries == arraysize(entries)) { + // The queue will not be full at this point, so it is ok to + // release the lock. The queue may still contain more than + // max_free_queue_size, but this is not a strict invariant. + free_queue_lock_.Unlock(); + for (int i = 0; i < num_entries; i++) { + CheckForDanglingWrites(entries[i]); + BASE_FREE(entries[i].block); + } + num_entries = 0; + free_queue_lock_.Lock(); + } } RAW_CHECK(free_queue_size_ >= 0, "Free queue size went negative!"); + free_queue_lock_.Unlock(); + for (int i = 0; i < num_entries; i++) { + CheckForDanglingWrites(entries[i]); + BASE_FREE(entries[i].block); + } + } + + static void InitDeletedBuffer() { + memset(kMagicDeletedBuffer, kMagicDeletedByte, sizeof(kMagicDeletedBuffer)); + deleted_buffer_initialized_no_pthreads_ = true; } static void CheckForDanglingWrites(const MallocBlockQueueEntry& queue_entry) { // Initialize the buffer if necessary. - if (!deleted_buffer_initialized_) { - // This is threadsafe. We hold free_queue_lock_. - memset(kMagicDeletedBuffer, 0xcd, sizeof(kMagicDeletedBuffer)); - deleted_buffer_initialized_ = true; + if (pthread_once) + pthread_once(&deleted_buffer_initialized_, &InitDeletedBuffer); + if (!deleted_buffer_initialized_no_pthreads_) { + // This will be the case on systems that don't link in pthreads, + // including on FreeBSD where pthread_once has a non-zero address + // (but doesn't do anything) even when pthreads isn't linked in. + InitDeletedBuffer(); } const unsigned char* p = @@ -625,9 +666,9 @@ class MallocBlock { // lines we'll output: if (size_of_buffer <= 1024) { for (int i = 0; i < size_of_buffer; ++i) { - if (buffer[i] != 0xcd) { - RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0xcd).", - i, buffer[i]); + if (buffer[i] != kMagicDeletedByte) { + RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0x%02x).", + i, buffer[i], kMagicDeletedByte); } } } else { @@ -671,8 +712,10 @@ class MallocBlock { RAW_LOG(FATAL, "Memory was written to after being freed. MallocBlock: %p, user " "ptr: %p, size: %zd. If you can't find the source of the error, " - "try using valgrind or purify, or study the output of the " - "deleter's stack printed above.", b, b->data_addr(), size); + "try using ASan (http://code.google.com/p/address-sanitizer/), " + "Valgrind, or Purify, or study the " + "output of the deleter's stack printed above.", + b, b->data_addr(), size); } static MallocBlock* FromRawPointer(void* p) { @@ -699,7 +742,7 @@ class MallocBlock { return FromRawPointer(const_cast<void*>(p)); } - void Check(int type) { + void Check(int type) const { alloc_map_lock_.Lock(); CheckLocked(type); alloc_map_lock_.Unlock(); @@ -777,7 +820,8 @@ size_t MallocBlock::free_queue_size_ = 0; SpinLock MallocBlock::free_queue_lock_(SpinLock::LINKER_INITIALIZED); unsigned char MallocBlock::kMagicDeletedBuffer[1024]; -bool MallocBlock::deleted_buffer_initialized_ = false; +pthread_once_t MallocBlock::deleted_buffer_initialized_ = PTHREAD_ONCE_INIT; +bool MallocBlock::deleted_buffer_initialized_no_pthreads_ = false; const char* const MallocBlock::kAllocName[] = { "malloc", @@ -971,17 +1015,17 @@ class DebugMallocImplementation : public TCMallocImplementation { return result; } - virtual bool VerifyNewMemory(void* p) { + virtual bool VerifyNewMemory(const void* p) { if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType); return true; } - virtual bool VerifyArrayNewMemory(void* p) { + virtual bool VerifyArrayNewMemory(const void* p) { if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType); return true; } - virtual bool VerifyMallocMemory(void* p) { + virtual bool VerifyMallocMemory(const void* p) { if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType); return true; } @@ -995,14 +1039,25 @@ class DebugMallocImplementation : public TCMallocImplementation { return MallocBlock::MemoryStats(blocks, total, histogram); } - virtual size_t GetAllocatedSize(void* p) { + virtual size_t GetEstimatedAllocatedSize(size_t size) { + return size; + } + + virtual size_t GetAllocatedSize(const void* p) { if (p) { + RAW_CHECK(GetOwnership(p) != MallocExtension::kNotOwned, + "ptr not allocated by tcmalloc"); return MallocBlock::FromRawPointer(p)->data_size(); } return 0; } - virtual size_t GetEstimatedAllocatedSize(size_t size) { - return size; + + virtual MallocExtension::Ownership GetOwnership(const void* p) { + if (p) { + const MallocBlock* mb = MallocBlock::FromRawPointer(p); + return TCMallocImplementation::GetOwnership(mb); + } + return MallocExtension::kNotOwned; // nobody owns NULL } virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { @@ -1024,22 +1079,28 @@ static DebugMallocImplementation debug_malloc_implementation; REGISTER_MODULE_INITIALIZER(debugallocation, { // Either we or valgrind will control memory management. We - // register our extension if we're the winner. - if (RunningOnValgrind()) { - // Let Valgrind uses its own malloc (so don't register our extension). - } else { + // register our extension if we're the winner. Otherwise let + // Valgrind use its own malloc (so don't register our extension). + if (!RunningOnValgrind()) { MallocExtension::Register(&debug_malloc_implementation); + } +}); + +REGISTER_MODULE_DESTRUCTOR(debugallocation, { + if (!RunningOnValgrind()) { // When the program exits, check all blocks still in the free // queue for corruption. - atexit(DanglingWriteChecker); + DanglingWriteChecker(); } }); // ========================================================================= // // This is mostly the same a cpp_alloc in tcmalloc.cc. -// TODO(csilvers): write a wrapper for new-handler so we don't have to -// copy this code so much. +// TODO(csilvers): change Allocate() above to call cpp_alloc, so we +// don't have to reproduce the logic here. To make tc_new_mode work +// properly, I think we'll need to separate out the logic of throwing +// from the logic of calling the new-handler. inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) { for (;;) { void* p = DebugAllocate(size, new_type); @@ -1355,29 +1416,5 @@ extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { #endif extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { - if (!ptr) { - return 0; - } - MallocBlock* mb = MallocBlock::FromRawPointer(ptr); - // This is just to make sure we actually own mb (and ptr). We don't - // use the actual value, just the 'exception' it raises on error. - (void)BASE_MALLOC_SIZE(mb); - return mb->data_size(); -} - -// Override __libc_memalign in libc on linux boxes. -// They have a bug in libc that causes them (very rarely) to allocate -// with __libc_memalign() yet deallocate with free(). -// This function is an exception to the rule of calling MallocHook method -// from the stack frame of the allocation function; -// heap-checker handles this special case explicitly. -static void *MemalignOverride(size_t align, size_t size, const void *caller) - __THROW ATTRIBUTE_SECTION(google_malloc); - -static void *MemalignOverride(size_t align, size_t size, const void *caller) - __THROW { - void *p = do_debug_memalign_or_debug_cpp_memalign(align, size); - MallocHook::InvokeNewHook(p, size); - return p; + return MallocExtension::instance()->GetAllocatedSize(ptr); } -void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride; diff --git a/third_party/tcmalloc/chromium/src/free_list.cc b/third_party/tcmalloc/chromium/src/free_list.cc index 842f391..b4176da 100644 --- a/third_party/tcmalloc/chromium/src/free_list.cc +++ b/third_party/tcmalloc/chromium/src/free_list.cc @@ -64,16 +64,18 @@ #if defined(TCMALLOC_USE_DOUBLYLINKED_FREELIST) +using tcmalloc::kCrash; + // TODO(jar): We should use C++ rather than a macro here. #define MEMORY_CHECK(v1, v2) \ - if (v1 != v2) CRASH("Memory corruption detected.\n") + if (v1 != v2) Log(kCrash, __FILE__, __LINE__, "Memory corruption detected.\n") namespace { void EnsureNonLoop(void* node, void* next) { // We only have time to do minimal checking. We don't traverse the list, but // only look for an immediate loop (cycle back to ourself). if (node != next) return; - CRASH("Circular loop in list detected: %p\n", next); + Log(kCrash, __FILE__, __LINE__, "Circular loop in list detected: %p\n", next); } // Returns value of the |previous| pointer w/out running a sanity diff --git a/third_party/tcmalloc/chromium/src/free_list.h b/third_party/tcmalloc/chromium/src/free_list.h index e1a88af..5b4cacf 100644 --- a/third_party/tcmalloc/chromium/src/free_list.h +++ b/third_party/tcmalloc/chromium/src/free_list.h @@ -40,7 +40,7 @@ #define TCMALLOC_FREE_LIST_H_ #include <stddef.h> -#include "internal_logging.h" // For CRASH() macro. +#include "internal_logging.h" #include "linked_list.h" // Remove to enable singly linked lists (the default for open source tcmalloc). @@ -77,7 +77,7 @@ inline void FL_Push(void **list, void *element) { SLL_Push(list,element); return; } - CRASH("Double Free of %p detected", element); + Log(kCrash, __FILE__, __LINE__, "Double Free of %p detected", element); } inline void *FL_Pop(void **list) { diff --git a/third_party/tcmalloc/chromium/src/google/heap-checker.h b/third_party/tcmalloc/chromium/src/google/heap-checker.h index f46f353..8aa5ea4 100644 --- a/third_party/tcmalloc/chromium/src/google/heap-checker.h +++ b/third_party/tcmalloc/chromium/src/google/heap-checker.h @@ -27,392 +27,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// --- -// Author: Maxim Lifantsev (with design ideas by Sanjay Ghemawat) -// -// -// Module for detecing heap (memory) leaks. -// -// For full(er) information, see doc/heap_checker.html -// -// This module can be linked into programs with -// no slowdown caused by this unless you activate the leak-checker: -// -// 1. Set the environment variable HEAPCHEK to _type_ before -// running the program. -// -// _type_ is usually "normal" but can also be "minimal", "strict", or -// "draconian". (See the html file for other options, like 'local'.) -// -// After that, just run your binary. If the heap-checker detects -// a memory leak at program-exit, it will print instructions on how -// to track down the leak. - -#ifndef BASE_HEAP_CHECKER_H_ -#define BASE_HEAP_CHECKER_H_ - -#include <sys/types.h> // for size_t -// I can't #include config.h in this public API file, but I should -// really use configure (and make malloc_extension.h a .in file) to -// figure out if the system has stdint.h or not. But I'm lazy, so -// for now I'm assuming it's a problem only with MSVC. -#ifndef _MSC_VER -#include <stdint.h> // for uintptr_t -#endif -#include <stdarg.h> // for va_list -#include <vector> - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - - -// The class is thread-safe with respect to all the provided static methods, -// as well as HeapLeakChecker objects: they can be accessed by multiple threads. -class PERFTOOLS_DLL_DECL HeapLeakChecker { - public: - - // ----------------------------------------------------------------------- // - // Static functions for working with (whole-program) leak checking. - - // If heap leak checking is currently active in some mode - // e.g. if leak checking was started (and is still active now) - // due to HEAPCHECK=... defined in the environment. - // The return value reflects iff HeapLeakChecker objects manually - // constructed right now will be doing leak checking or nothing. - // Note that we can go from active to inactive state during InitGoogle() - // if FLAGS_heap_check gets set to "" by some code before/during InitGoogle(). - static bool IsActive(); - - // Return pointer to the whole-program checker if it has been created - // and NULL otherwise. - // Once GlobalChecker() returns non-NULL that object will not disappear and - // will be returned by all later GlobalChecker calls. - // This is mainly to access BytesLeaked() and ObjectsLeaked() (see below) - // for the whole-program checker after one calls NoGlobalLeaks() - // or similar and gets false. - static HeapLeakChecker* GlobalChecker(); - - // Do whole-program leak check now (if it was activated for this binary); - // return false only if it was activated and has failed. - // The mode of the check is controlled by the command-line flags. - // This method can be called repeatedly. - // Things like GlobalChecker()->SameHeap() can also be called explicitly - // to do the desired flavor of the check. - static bool NoGlobalLeaks(); - - // If whole-program checker if active, - // cancel its automatic execution after main() exits. - // This requires that some leak check (e.g. NoGlobalLeaks()) - // has been called at least once on the whole-program checker. - static void CancelGlobalCheck(); - - // ----------------------------------------------------------------------- // - // Non-static functions for starting and doing leak checking. - - // Start checking and name the leak check performed. - // The name is used in naming dumped profiles - // and needs to be unique only within your binary. - // It must also be a string that can be a part of a file name, - // in particular not contain path expressions. - explicit HeapLeakChecker(const char *name); - - // Destructor (verifies that some *NoLeaks or *SameHeap method - // has been called at least once). - ~HeapLeakChecker(); - - // These used to be different but are all the same now: they return - // true iff all memory allocated since this HeapLeakChecker object - // was constructor is still reachable from global state. - // - // Because we fork to convert addresses to symbol-names, and forking - // is not thread-safe, and we may be called in a threaded context, - // we do not try to symbolize addresses when called manually. - bool NoLeaks() { return DoNoLeaks(DO_NOT_SYMBOLIZE); } - - // These forms are obsolete; use NoLeaks() instead. - // TODO(csilvers): mark as DEPRECATED. - bool QuickNoLeaks() { return NoLeaks(); } - bool BriefNoLeaks() { return NoLeaks(); } - bool SameHeap() { return NoLeaks(); } - bool QuickSameHeap() { return NoLeaks(); } - bool BriefSameHeap() { return NoLeaks(); } - - // Detailed information about the number of leaked bytes and objects - // (both of these can be negative as well). - // These are available only after a *SameHeap or *NoLeaks - // method has been called. - // Note that it's possible for both of these to be zero - // while SameHeap() or NoLeaks() returned false in case - // of a heap state change that is significant - // but preserves the byte and object counts. - ssize_t BytesLeaked() const; - ssize_t ObjectsLeaked() const; - - // ----------------------------------------------------------------------- // - // Static helpers to make us ignore certain leaks. - - // Scoped helper class. Should be allocated on the stack inside a - // block of code. Any heap allocations done in the code block - // covered by the scoped object (including in nested function calls - // done by the code block) will not be reported as leaks. This is - // the recommended replacement for the GetDisableChecksStart() and - // DisableChecksToHereFrom() routines below. - // - // Example: - // void Foo() { - // HeapLeakChecker::Disabler disabler; - // ... code that allocates objects whose leaks should be ignored ... - // } - // - // REQUIRES: Destructor runs in same thread as constructor - class Disabler { - public: - Disabler(); - ~Disabler(); - private: - Disabler(const Disabler&); // disallow copy - void operator=(const Disabler&); // and assign - }; - - // Ignore an object located at 'ptr' (can go at the start or into the object) - // as well as all heap objects (transitively) referenced from it - // for the purposes of heap leak checking. - // If 'ptr' does not point to an active allocated object - // at the time of this call, it is ignored; - // but if it does, the object must not get deleted from the heap later on; - // it must also be not already ignored at the time of this call. - // - // See also HiddenPointer, below, if you need to prevent a pointer from - // being traversed by the heap checker but do not wish to transitively - // whitelist objects referenced through it. - static void IgnoreObject(const void* ptr); - - // Undo what an earlier IgnoreObject() call promised and asked to do. - // At the time of this call 'ptr' must point at or inside of an active - // allocated object which was previously registered with IgnoreObject(). - static void UnIgnoreObject(const void* ptr); - - // ----------------------------------------------------------------------- // - // Initialization; to be called from main() only. - - // Full starting of recommended whole-program checking. - static void InternalInitStart(); - - // ----------------------------------------------------------------------- // - // Internal types defined in .cc - - class Allocator; - struct RangeValue; - - private: - - // ----------------------------------------------------------------------- // - // Various helpers - - // Create the name of the heap profile file. - // Should be deleted via Allocator::Free(). - char* MakeProfileNameLocked(); - - // Helper for constructors - void Create(const char *name, bool make_start_snapshot); - - enum ShouldSymbolize { SYMBOLIZE, DO_NOT_SYMBOLIZE }; - - // Helper for *NoLeaks and *SameHeap - bool DoNoLeaks(ShouldSymbolize should_symbolize); - - // These used to be public, but they are now deprecated. - // Will remove entirely when all internal uses are fixed. - // In the meantime, use friendship so the unittest can still test them. - static void* GetDisableChecksStart(); - static void DisableChecksToHereFrom(const void* start_address); - static void DisableChecksIn(const char* pattern); - friend void RangeDisabledLeaks(); - friend void NamedTwoDisabledLeaks(); - friend void* RunNamedDisabledLeaks(void*); - friend void TestHeapLeakCheckerNamedDisabling(); - friend int main(int, char**); - - - // Helper for DisableChecksIn - static void DisableChecksInLocked(const char* pattern); - - // Disable checks based on stack trace entry at a depth <= - // max_depth. Used to hide allocations done inside some special - // libraries. - static void DisableChecksFromToLocked(const void* start_address, - const void* end_address, - int max_depth); - - // Helper for DoNoLeaks to ignore all objects reachable from all live data - static void IgnoreAllLiveObjectsLocked(const void* self_stack_top); - - // Callback we pass to ListAllProcessThreads (see thread_lister.h) - // that is invoked when all threads of our process are found and stopped. - // The call back does the things needed to ignore live data reachable from - // thread stacks and registers for all our threads - // as well as do other global-live-data ignoring - // (via IgnoreNonThreadLiveObjectsLocked) - // during the quiet state of all threads being stopped. - // For the argument meaning see the comment by ListAllProcessThreads. - // Here we only use num_threads and thread_pids, that ListAllProcessThreads - // fills for us with the number and pids of all the threads of our process - // it found and attached to. - static int IgnoreLiveThreadsLocked(void* parameter, - int num_threads, - pid_t* thread_pids, - va_list ap); - - // Helper for IgnoreAllLiveObjectsLocked and IgnoreLiveThreadsLocked - // that we prefer to execute from IgnoreLiveThreadsLocked - // while all threads are stopped. - // This helper does live object discovery and ignoring - // for all objects that are reachable from everything - // not related to thread stacks and registers. - static void IgnoreNonThreadLiveObjectsLocked(); - - // Helper for IgnoreNonThreadLiveObjectsLocked and IgnoreLiveThreadsLocked - // to discover and ignore all heap objects - // reachable from currently considered live objects - // (live_objects static global variable in out .cc file). - // "name", "name2" are two strings that we print one after another - // in a debug message to describe what kind of live object sources - // are being used. - static void IgnoreLiveObjectsLocked(const char* name, const char* name2); - - // Runs REGISTER_HEAPCHECK_CLEANUP cleanups and potentially - // calls DoMainHeapCheck - static void RunHeapCleanups(); - - // Do the overall whole-program heap leak check if needed; - // returns true when did the leak check. - static bool DoMainHeapCheck(); - - // Type of task for UseProcMapsLocked - enum ProcMapsTask { - RECORD_GLOBAL_DATA, - DISABLE_LIBRARY_ALLOCS - }; - - // Success/Error Return codes for UseProcMapsLocked. - enum ProcMapsResult { - PROC_MAPS_USED, - CANT_OPEN_PROC_MAPS, - NO_SHARED_LIBS_IN_PROC_MAPS - }; - - // Read /proc/self/maps, parse it, and do the 'proc_maps_task' for each line. - static ProcMapsResult UseProcMapsLocked(ProcMapsTask proc_maps_task); - - // A ProcMapsTask to disable allocations from 'library' - // that is mapped to [start_address..end_address) - // (only if library is a certain system library). - static void DisableLibraryAllocsLocked(const char* library, - uintptr_t start_address, - uintptr_t end_address); - - // Return true iff "*ptr" points to a heap object - // ("*ptr" can point at the start or inside of a heap object - // so that this works e.g. for pointers to C++ arrays, C++ strings, - // multiple-inherited objects, or pointers to members). - // We also fill *object_size for this object then - // and we move "*ptr" to point to the very start of the heap object. - static inline bool HaveOnHeapLocked(const void** ptr, size_t* object_size); - - // Helper to shutdown heap leak checker when it's not needed - // or can't function properly. - static void TurnItselfOffLocked(); - - // Internally-used c-tor to start whole-executable checking. - HeapLeakChecker(); - - // ----------------------------------------------------------------------- // - // Friends and externally accessed helpers. - - // Helper for VerifyHeapProfileTableStackGet in the unittest - // to get the recorded allocation caller for ptr, - // which must be a heap object. - static const void* GetAllocCaller(void* ptr); - friend void VerifyHeapProfileTableStackGet(); - - // This gets to execute before constructors for all global objects - static void BeforeConstructorsLocked(); - friend void HeapLeakChecker_BeforeConstructors(); - - // This gets to execute after destructors for all global objects - friend void HeapLeakChecker_AfterDestructors(); - - // ----------------------------------------------------------------------- // - // Member data. - - class SpinLock* lock_; // to make HeapLeakChecker objects thread-safe - const char* name_; // our remembered name (we own it) - // NULL means this leak checker is a noop - - // Snapshot taken when the checker was created. May be NULL - // for the global heap checker object. We use void* instead of - // HeapProfileTable::Snapshot* to avoid including heap-profile-table.h. - void* start_snapshot_; - - bool has_checked_; // if we have done the leak check, so these are ready: - ssize_t inuse_bytes_increase_; // bytes-in-use increase for this checker - ssize_t inuse_allocs_increase_; // allocations-in-use increase - // for this checker - bool keep_profiles_; // iff we should keep the heap profiles we've made - - // ----------------------------------------------------------------------- // - - // Disallow "evil" constructors. - HeapLeakChecker(const HeapLeakChecker&); - void operator=(const HeapLeakChecker&); -}; - - -// Holds a pointer that will not be traversed by the heap checker. -// Contrast with HeapLeakChecker::IgnoreObject(o), in which o and -// all objects reachable from o are ignored by the heap checker. -template <class T> -class HiddenPointer { - public: - explicit HiddenPointer(T* t) - : masked_t_(reinterpret_cast<uintptr_t>(t) ^ kHideMask) { - } - // Returns unhidden pointer. Be careful where you save the result. - T* get() const { return reinterpret_cast<T*>(masked_t_ ^ kHideMask); } - - private: - // Arbitrary value, but not such that xor'ing with it is likely - // to map one valid pointer to another valid pointer: - static const uintptr_t kHideMask = - static_cast<uintptr_t>(0xF03A5F7BF03A5F7Bll); - uintptr_t masked_t_; -}; - -// A class that exists solely to run its destructor. This class should not be -// used directly, but instead by the REGISTER_HEAPCHECK_CLEANUP macro below. -class PERFTOOLS_DLL_DECL HeapCleaner { - public: - typedef void (*void_function)(void); - HeapCleaner(void_function f); - static void RunHeapCleanups(); - private: - static std::vector<void_function>* heap_cleanups_; -}; - -// A macro to declare module heap check cleanup tasks -// (they run only if we are doing heap leak checking.) -// 'body' should be the cleanup code to run. 'name' doesn't matter, -// but must be unique amongst all REGISTER_HEAPCHECK_CLEANUP calls. -#define REGISTER_HEAPCHECK_CLEANUP(name, body) \ - namespace { \ - void heapcheck_cleanup_##name() { body; } \ - static HeapCleaner heapcheck_cleaner_##name(&heapcheck_cleanup_##name); \ - } - -#endif // BASE_HEAP_CHECKER_H_ +/* The code has moved to gperftools/. Use that include-directory for + * new code. + */ +#include <gperftools/heap-checker.h> diff --git a/third_party/tcmalloc/chromium/src/google/heap-profiler.h b/third_party/tcmalloc/chromium/src/google/heap-profiler.h index 57cb97a..be43959 100644 --- a/third_party/tcmalloc/chromium/src/google/heap-profiler.h +++ b/third_party/tcmalloc/chromium/src/google/heap-profiler.h @@ -26,79 +26,9 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - * - * Module for heap-profiling. - * - * For full(er) information, see doc/heapprofile.html - * - * This module can be linked into your program with - * no slowdown caused by this unless you activate the profiler - * using one of the following methods: - * - * 1. Before starting the program, set the environment variable - * "HEAPPROFILE" to be the name of the file to which the profile - * data should be written. - * - * 2. Programmatically, start and stop the profiler using the - * routines "HeapProfilerStart(filename)" and "HeapProfilerStop()". - * */ -#ifndef BASE_HEAP_PROFILER_H_ -#define BASE_HEAP_PROFILER_H_ - -#include <stddef.h> - -/* Annoying stuff for windows; makes sure clients can import these functions */ -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -/* All this code should be usable from within C apps. */ -#ifdef __cplusplus -extern "C" { -#endif - -/* Start profiling and arrange to write profile data to file names - * of the form: "prefix.0000", "prefix.0001", ... - */ -PERFTOOLS_DLL_DECL void HeapProfilerStart(const char* prefix); - -/* Returns non-zero if we are currently profiling the heap. (Returns - * an int rather than a bool so it's usable from C.) This is true - * between calls to HeapProfilerStart() and HeapProfilerStop(), and - * also if the program has been run with HEAPPROFILER, or some other - * way to turn on whole-program profiling. - */ -int IsHeapProfilerRunning(); - -/* Stop heap profiling. Can be restarted again with HeapProfilerStart(), - * but the currently accumulated profiling information will be cleared. - */ -PERFTOOLS_DLL_DECL void HeapProfilerStop(); - -/* Dump a profile now - can be used for dumping at a hopefully - * quiescent state in your program, in order to more easily track down - * memory leaks. Will include the reason in the logged message +/* The code has moved to gperftools/. Use that include-directory for + * new code. */ -PERFTOOLS_DLL_DECL void HeapProfilerDump(const char *reason); - -/* Generate current heap profiling information. - * Returns an empty string when heap profiling is not active. - * The returned pointer is a '\0'-terminated string allocated using malloc() - * and should be free()-ed as soon as the caller does not need it anymore. - */ -PERFTOOLS_DLL_DECL char* GetHeapProfile(); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* BASE_HEAP_PROFILER_H_ */ +#include <gperftools/heap-profiler.h> diff --git a/third_party/tcmalloc/chromium/src/google/malloc_extension.h b/third_party/tcmalloc/chromium/src/google/malloc_extension.h index 0e15c04..55150e5 100644 --- a/third_party/tcmalloc/chromium/src/google/malloc_extension.h +++ b/third_party/tcmalloc/chromium/src/google/malloc_extension.h @@ -27,359 +27,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// --- -// Author: Sanjay Ghemawat <opensource@google.com> -// -// Extra extensions exported by some malloc implementations. These -// extensions are accessed through a virtual base class so an -// application can link against a malloc that does not implement these -// extensions, and it will get default versions that do nothing. -// -// NOTE FOR C USERS: If you wish to use this functionality from within -// a C program, see malloc_extension_c.h. - -#ifndef BASE_MALLOC_EXTENSION_H_ -#define BASE_MALLOC_EXTENSION_H_ - -#include <stddef.h> -// I can't #include config.h in this public API file, but I should -// really use configure (and make malloc_extension.h a .in file) to -// figure out if the system has stdint.h or not. But I'm lazy, so -// for now I'm assuming it's a problem only with MSVC. -#ifndef _MSC_VER -#include <stdint.h> -#endif -#include <string> -#include <vector> - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -static const int kMallocHistogramSize = 64; - -// One day, we could support other types of writers (perhaps for C?) -typedef std::string MallocExtensionWriter; - -namespace base { -struct MallocRange; -} - -// Interface to a pluggable system allocator. -class SysAllocator { - public: - SysAllocator() { - } - virtual ~SysAllocator(); - - // Allocates "size"-byte of memory from system aligned with "alignment". - // Returns NULL if failed. Otherwise, the returned pointer p up to and - // including (p + actual_size -1) have been allocated. - virtual void* Alloc(size_t size, size_t *actual_size, size_t alignment) = 0; - - // Notification that command-line flags have been initialized. - virtual void FlagsInitialized() = 0; -}; - -// The default implementations of the following routines do nothing. -// All implementations should be thread-safe; the current one -// (TCMallocImplementation) is. -class PERFTOOLS_DLL_DECL MallocExtension { - public: - virtual ~MallocExtension(); - - // Call this very early in the program execution -- say, in a global - // constructor -- to set up parameters and state needed by all - // instrumented malloc implemenatations. One example: this routine - // sets environemnt variables to tell STL to use libc's malloc() - // instead of doing its own memory management. This is safe to call - // multiple times, as long as each time is before threads start up. - static void Initialize(); - - // See "verify_memory.h" to see what these routines do - virtual bool VerifyAllMemory(); - virtual bool VerifyNewMemory(void* p); - virtual bool VerifyArrayNewMemory(void* p); - virtual bool VerifyMallocMemory(void* p); - virtual bool MallocMemoryStats(int* blocks, size_t* total, - int histogram[kMallocHistogramSize]); - - // Get a human readable description of the current state of the malloc - // data structures. The state is stored as a null-terminated string - // in a prefix of "buffer[0,buffer_length-1]". - // REQUIRES: buffer_length > 0. - virtual void GetStats(char* buffer, int buffer_length); - - // Outputs to "writer" a sample of live objects and the stack traces - // that allocated these objects. The format of the returned output - // is equivalent to the output of the heap profiler and can - // therefore be passed to "pprof". This function is equivalent to - // ReadStackTraces. The main difference is that this function returns - // serialized data appropriately formatted for use by the pprof tool. - // NOTE: by default, tcmalloc does not do any heap sampling, and this - // function will always return an empty sample. To get useful - // data from GetHeapSample, you must also set the environment - // variable TCMALLOC_SAMPLE_PARAMETER to a value such as 524288. - virtual void GetHeapSample(MallocExtensionWriter* writer); - - // Outputs to "writer" the stack traces that caused growth in the - // address space size. The format of the returned output is - // equivalent to the output of the heap profiler and can therefore - // be passed to "pprof". This function is equivalent to - // ReadHeapGrowthStackTraces. The main difference is that this function - // returns serialized data appropriately formatted for use by the - // pprof tool. (This does not depend on, or require, - // TCMALLOC_SAMPLE_PARAMETER.) - virtual void GetHeapGrowthStacks(MallocExtensionWriter* writer); - - // Invokes func(arg, range) for every controlled memory - // range. *range is filled in with information about the range. - // - // This is a best-effort interface useful only for performance - // analysis. The implementation may not call func at all. - typedef void (RangeFunction)(void*, const base::MallocRange*); - virtual void Ranges(void* arg, RangeFunction func); - - // ------------------------------------------------------------------- - // Control operations for getting and setting malloc implementation - // specific parameters. Some currently useful properties: - // - // generic - // ------- - // "generic.current_allocated_bytes" - // Number of bytes currently allocated by application - // This property is not writable. - // - // "generic.heap_size" - // Number of bytes in the heap == - // current_allocated_bytes + - // fragmentation + - // freed memory regions - // This property is not writable. - // - // tcmalloc - // -------- - // "tcmalloc.max_total_thread_cache_bytes" - // Upper limit on total number of bytes stored across all - // per-thread caches. Default: 16MB. - // - // "tcmalloc.current_total_thread_cache_bytes" - // Number of bytes used across all thread caches. - // This property is not writable. - // - // "tcmalloc.pageheap_free_bytes" - // Number of bytes in free, mapped pages in page heap. These - // bytes can be used to fulfill allocation requests. They - // always count towards virtual memory usage, and unless the - // underlying memory is swapped out by the OS, they also count - // towards physical memory usage. This property is not writable. - // - // "tcmalloc.pageheap_unmapped_bytes" - // Number of bytes in free, unmapped pages in page heap. - // These are bytes that have been released back to the OS, - // possibly by one of the MallocExtension "Release" calls. - // They can be used to fulfill allocation requests, but - // typically incur a page fault. They always count towards - // virtual memory usage, and depending on the OS, typically - // do not count towards physical memory usage. This property - // is not writable. - // ------------------------------------------------------------------- - - // Get the named "property"'s value. Returns true if the property - // is known. Returns false if the property is not a valid property - // name for the current malloc implementation. - // REQUIRES: property != NULL; value != NULL - virtual bool GetNumericProperty(const char* property, size_t* value); - - // Set the named "property"'s value. Returns true if the property - // is known and writable. Returns false if the property is not a - // valid property name for the current malloc implementation, or - // is not writable. - // REQUIRES: property != NULL - virtual bool SetNumericProperty(const char* property, size_t value); - - // Mark the current thread as "idle". This routine may optionally - // be called by threads as a hint to the malloc implementation that - // any thread-specific resources should be released. Note: this may - // be an expensive routine, so it should not be called too often. - // - // Also, if the code that calls this routine will go to sleep for - // a while, it should take care to not allocate anything between - // the call to this routine and the beginning of the sleep. - // - // Most malloc implementations ignore this routine. - virtual void MarkThreadIdle(); - - // Mark the current thread as "busy". This routine should be - // called after MarkThreadIdle() if the thread will now do more - // work. If this method is not called, performance may suffer. - // - // Most malloc implementations ignore this routine. - virtual void MarkThreadBusy(); - - // Gets the system allocator used by the malloc extension instance. Returns - // NULL for malloc implementations that do not support pluggable system - // allocators. - virtual SysAllocator* GetSystemAllocator(); - - // Sets the system allocator to the specified. - // - // Users could register their own system allocators for malloc implementation - // that supports pluggable system allocators, such as TCMalloc, by doing: - // alloc = new MyOwnSysAllocator(); - // MallocExtension::instance()->SetSystemAllocator(alloc); - // It's up to users whether to fall back (recommended) to the default - // system allocator (use GetSystemAllocator() above) or not. The caller is - // responsible to any necessary locking. - // See tcmalloc/system-alloc.h for the interface and - // tcmalloc/memfs_malloc.cc for the examples. - // - // It's a no-op for malloc implementations that do not support pluggable - // system allocators. - virtual void SetSystemAllocator(SysAllocator *a); - - // Try to release num_bytes of free memory back to the operating - // system for reuse. Use this extension with caution -- to get this - // memory back may require faulting pages back in by the OS, and - // that may be slow. (Currently only implemented in tcmalloc.) - virtual void ReleaseToSystem(size_t num_bytes); - - // Same as ReleaseToSystem() but release as much memory as possible. - virtual void ReleaseFreeMemory(); - - // Sets the rate at which we release unused memory to the system. - // Zero means we never release memory back to the system. Increase - // this flag to return memory faster; decrease it to return memory - // slower. Reasonable rates are in the range [0,10]. (Currently - // only implemented in tcmalloc). - virtual void SetMemoryReleaseRate(double rate); - - // Gets the release rate. Returns a value < 0 if unknown. - virtual double GetMemoryReleaseRate(); - - // Returns the estimated number of bytes that will be allocated for - // a request of "size" bytes. This is an estimate: an allocation of - // SIZE bytes may reserve more bytes, but will never reserve less. - // (Currently only implemented in tcmalloc, other implementations - // always return SIZE.) - // This is equivalent to malloc_good_size() in OS X. - virtual size_t GetEstimatedAllocatedSize(size_t size); - - // Returns the actual number N of bytes reserved by tcmalloc for the - // pointer p. The client is allowed to use the range of bytes - // [p, p+N) in any way it wishes (i.e. N is the "usable size" of this - // allocation). This number may be equal to or greater than the number - // of bytes requested when p was allocated. - // p must have been allocated by this malloc implementation, - // must not be an interior pointer -- that is, must be exactly - // the pointer returned to by malloc() et al., not some offset - // from that -- and should not have been freed yet. p may be NULL. - // (Currently only implemented in tcmalloc; other implementations - // will return 0.) - // This is equivalent to malloc_size() in OS X, malloc_usable_size() - // in glibc, and _msize() for windows. - virtual size_t GetAllocatedSize(void* p); - - // The current malloc implementation. Always non-NULL. - static MallocExtension* instance(); - - // Change the malloc implementation. Typically called by the - // malloc implementation during initialization. - static void Register(MallocExtension* implementation); - - // Returns detailed information about malloc's freelists. For each list, - // return a FreeListInfo: - struct FreeListInfo { - size_t min_object_size; - size_t max_object_size; - size_t total_bytes_free; - const char* type; - }; - // Each item in the vector refers to a different freelist. The lists - // are identified by the range of allocations that objects in the - // list can satisfy ([min_object_size, max_object_size]) and the - // type of freelist (see below). The current size of the list is - // returned in total_bytes_free (which count against a processes - // resident and virtual size). - // - // Currently supported types are: - // - // "tcmalloc.page{_unmapped}" - tcmalloc's page heap. An entry for each size - // class in the page heap is returned. Bytes in "page_unmapped" - // are no longer backed by physical memory and do not count against - // the resident size of a process. - // - // "tcmalloc.large{_unmapped}" - tcmalloc's list of objects larger - // than the largest page heap size class. Only one "large" - // entry is returned. There is no upper-bound on the size - // of objects in the large free list; this call returns - // kint64max for max_object_size. Bytes in - // "large_unmapped" are no longer backed by physical memory - // and do not count against the resident size of a process. - // - // "tcmalloc.central" - tcmalloc's central free-list. One entry per - // size-class is returned. Never unmapped. - // - // "debug.free_queue" - free objects queued by the debug allocator - // and not returned to tcmalloc. - // - // "tcmalloc.thread" - tcmalloc's per-thread caches. Never unmapped. - virtual void GetFreeListSizes(std::vector<FreeListInfo>* v); - - // Get a list of stack traces of sampled allocation points. Returns - // a pointer to a "new[]-ed" result array, and stores the sample - // period in "sample_period". - // - // The state is stored as a sequence of adjacent entries - // in the returned array. Each entry has the following form: - // uintptr_t count; // Number of objects with following trace - // uintptr_t size; // Total size of objects with following trace - // uintptr_t depth; // Number of PC values in stack trace - // void* stack[depth]; // PC values that form the stack trace - // - // The list of entries is terminated by a "count" of 0. - // - // It is the responsibility of the caller to "delete[]" the returned array. - // - // May return NULL to indicate no results. - // - // This is an internal extension. Callers should use the more - // convenient "GetHeapSample(string*)" method defined above. - virtual void** ReadStackTraces(int* sample_period); - - // Like ReadStackTraces(), but returns stack traces that caused growth - // in the address space size. - virtual void** ReadHeapGrowthStackTraces(); -}; - -namespace base { - -// Information passed per range. More fields may be added later. -struct MallocRange { - enum Type { - INUSE, // Application is using this range - FREE, // Range is currently free - UNMAPPED, // Backing physical memory has been returned to the OS - UNKNOWN, - // More enum values may be added in the future - }; - - uintptr_t address; // Address of range - size_t length; // Byte length of range - Type type; // Type of this range - double fraction; // Fraction of range that is being used (0 if !INUSE) - - // Perhaps add the following: - // - stack trace if this range was sampled - // - heap growth stack trace if applicable to this range - // - age when allocated (for inuse) or freed (if not in use) -}; - -} // namespace base - -#endif // BASE_MALLOC_EXTENSION_H_ +/* The code has moved to gperftools/. Use that include-directory for + * new code. + */ +#include <gperftools/malloc_extension.h> diff --git a/third_party/tcmalloc/chromium/src/google/malloc_extension_c.h b/third_party/tcmalloc/chromium/src/google/malloc_extension_c.h index fcaa8cd..87d727b 100644 --- a/third_party/tcmalloc/chromium/src/google/malloc_extension_c.h +++ b/third_party/tcmalloc/chromium/src/google/malloc_extension_c.h @@ -26,62 +26,9 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * -- - * Author: Craig Silverstein - * - * C shims for the C++ malloc_extension.h. See malloc_extension.h for - * details. Note these C shims always work on - * MallocExtension::instance(); it is not possible to have more than - * one MallocExtension object in C applications. */ -#ifndef _MALLOC_EXTENSION_C_H_ -#define _MALLOC_EXTENSION_C_H_ - -#include <stddef.h> -#include <sys/types.h> - -/* Annoying stuff for windows -- makes sure clients can import these fns */ -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define kMallocExtensionHistogramSize 64 - -PERFTOOLS_DLL_DECL int MallocExtension_VerifyAllMemory(void); -PERFTOOLS_DLL_DECL int MallocExtension_VerifyNewMemory(void* p); -PERFTOOLS_DLL_DECL int MallocExtension_VerifyArrayNewMemory(void* p); -PERFTOOLS_DLL_DECL int MallocExtension_VerifyMallocMemory(void* p); -PERFTOOLS_DLL_DECL int MallocExtension_MallocMemoryStats(int* blocks, size_t* total, - int histogram[kMallocExtensionHistogramSize]); -PERFTOOLS_DLL_DECL void MallocExtension_GetStats(char* buffer, int buffer_length); - -/* TODO(csilvers): write a C version of these routines, that perhaps - * takes a function ptr and a void *. +/* The code has moved to gperftools/. Use that include-directory for + * new code. */ -/* void MallocExtension_GetHeapSample(string* result); */ -/* void MallocExtension_GetHeapGrowthStacks(string* result); */ - -PERFTOOLS_DLL_DECL int MallocExtension_GetNumericProperty(const char* property, size_t* value); -PERFTOOLS_DLL_DECL int MallocExtension_SetNumericProperty(const char* property, size_t value); -PERFTOOLS_DLL_DECL void MallocExtension_MarkThreadIdle(void); -PERFTOOLS_DLL_DECL void MallocExtension_MarkThreadBusy(void); -PERFTOOLS_DLL_DECL void MallocExtension_ReleaseToSystem(size_t num_bytes); -PERFTOOLS_DLL_DECL void MallocExtension_ReleaseFreeMemory(void); -PERFTOOLS_DLL_DECL size_t MallocExtension_GetEstimatedAllocatedSize(size_t size); -PERFTOOLS_DLL_DECL size_t MallocExtension_GetAllocatedSize(void* p); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* _MALLOC_EXTENSION_C_H_ */ +#include <gperftools/malloc_extension_c.h> diff --git a/third_party/tcmalloc/chromium/src/google/malloc_hook.h b/third_party/tcmalloc/chromium/src/google/malloc_hook.h index f5575f1..e5b8e7c 100644 --- a/third_party/tcmalloc/chromium/src/google/malloc_hook.h +++ b/third_party/tcmalloc/chromium/src/google/malloc_hook.h @@ -27,319 +27,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// --- -// Author: Sanjay Ghemawat -// -// Some of our malloc implementations can invoke the following hooks whenever -// memory is allocated or deallocated. MallocHook is thread-safe, and things -// you do before calling AddFooHook(MyHook) are visible to any resulting calls -// to MyHook. Hooks must be thread-safe. If you write: -// -// CHECK(MallocHook::AddNewHook(&MyNewHook)); -// -// MyNewHook will be invoked in subsequent calls in the current thread, but -// there are no guarantees on when it might be invoked in other threads. -// -// There are a limited number of slots available for each hook type. Add*Hook -// will return false if there are no slots available. Remove*Hook will return -// false if the given hook was not already installed. -// -// The order in which individual hooks are called in Invoke*Hook is undefined. -// -// It is safe for a hook to remove itself within Invoke*Hook and add other -// hooks. Any hooks added inside a hook invocation (for the same hook type) -// will not be invoked for the current invocation. -// -// One important user of these hooks is the heap profiler. -// -// CAVEAT: If you add new MallocHook::Invoke* calls then those calls must be -// directly in the code of the (de)allocation function that is provided to the -// user and that function must have an ATTRIBUTE_SECTION(malloc_hook) attribute. -// -// Note: the Invoke*Hook() functions are defined in malloc_hook-inl.h. If you -// need to invoke a hook (which you shouldn't unless you're part of tcmalloc), -// be sure to #include malloc_hook-inl.h in addition to malloc_hook.h. -// -// NOTE FOR C USERS: If you want to use malloc_hook functionality from -// a C program, #include malloc_hook_c.h instead of this file. - -#ifndef _MALLOC_HOOK_H_ -#define _MALLOC_HOOK_H_ - -#include <stddef.h> -#include <sys/types.h> -extern "C" { -#include <google/malloc_hook_c.h> // a C version of the malloc_hook interface -} - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -// Note: malloc_hook_c.h defines MallocHook_*Hook and -// MallocHook_{Add,Remove}*Hook. The version of these inside the MallocHook -// class are defined in terms of the malloc_hook_c version. See malloc_hook_c.h -// for details of these types/functions. - -class PERFTOOLS_DLL_DECL MallocHook { - public: - // The NewHook is invoked whenever an object is allocated. - // It may be passed NULL if the allocator returned NULL. - typedef MallocHook_NewHook NewHook; - inline static bool AddNewHook(NewHook hook) { - return MallocHook_AddNewHook(hook); - } - inline static bool RemoveNewHook(NewHook hook) { - return MallocHook_RemoveNewHook(hook); - } - inline static void InvokeNewHook(const void* p, size_t s); - - // The DeleteHook is invoked whenever an object is deallocated. - // It may be passed NULL if the caller is trying to delete NULL. - typedef MallocHook_DeleteHook DeleteHook; - inline static bool AddDeleteHook(DeleteHook hook) { - return MallocHook_AddDeleteHook(hook); - } - inline static bool RemoveDeleteHook(DeleteHook hook) { - return MallocHook_RemoveDeleteHook(hook); - } - inline static void InvokeDeleteHook(const void* p); - - // The PreMmapHook is invoked with mmap or mmap64 arguments just - // before the call is actually made. Such a hook may be useful - // in memory limited contexts, to catch allocations that will exceed - // a memory limit, and take outside actions to increase that limit. - typedef MallocHook_PreMmapHook PreMmapHook; - inline static bool AddPreMmapHook(PreMmapHook hook) { - return MallocHook_AddPreMmapHook(hook); - } - inline static bool RemovePreMmapHook(PreMmapHook hook) { - return MallocHook_RemovePreMmapHook(hook); - } - inline static void InvokePreMmapHook(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); - - // The MmapReplacement is invoked after the PreMmapHook but before - // the call is actually made. The MmapReplacement should return true - // if it handled the call, or false if it is still necessary to - // call mmap/mmap64. - // This should be used only by experts, and users must be be - // extremely careful to avoid recursive calls to mmap. The replacement - // should be async signal safe. - // Only one MmapReplacement is supported. After setting an MmapReplacement - // you must call RemoveMmapReplacement before calling SetMmapReplacement - // again. - typedef MallocHook_MmapReplacement MmapReplacement; - inline static bool SetMmapReplacement(MmapReplacement hook) { - return MallocHook_SetMmapReplacement(hook); - } - inline static bool RemoveMmapReplacement(MmapReplacement hook) { - return MallocHook_RemoveMmapReplacement(hook); - } - inline static bool InvokeMmapReplacement(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset, - void** result); - - - // The MmapHook is invoked whenever a region of memory is mapped. - // It may be passed MAP_FAILED if the mmap failed. - typedef MallocHook_MmapHook MmapHook; - inline static bool AddMmapHook(MmapHook hook) { - return MallocHook_AddMmapHook(hook); - } - inline static bool RemoveMmapHook(MmapHook hook) { - return MallocHook_RemoveMmapHook(hook); - } - inline static void InvokeMmapHook(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); - - // The MunmapReplacement is invoked with munmap arguments just before - // the call is actually made. The MunmapReplacement should return true - // if it handled the call, or false if it is still necessary to - // call munmap. - // This should be used only by experts. The replacement should be - // async signal safe. - // Only one MunmapReplacement is supported. After setting an - // MunmapReplacement you must call RemoveMunmapReplacement before - // calling SetMunmapReplacement again. - typedef MallocHook_MunmapReplacement MunmapReplacement; - inline static bool SetMunmapReplacement(MunmapReplacement hook) { - return MallocHook_SetMunmapReplacement(hook); - } - inline static bool RemoveMunmapReplacement(MunmapReplacement hook) { - return MallocHook_RemoveMunmapReplacement(hook); - } - inline static bool InvokeMunmapReplacement(const void* p, - size_t size, - int* result); - - // The MunmapHook is invoked whenever a region of memory is unmapped. - typedef MallocHook_MunmapHook MunmapHook; - inline static bool AddMunmapHook(MunmapHook hook) { - return MallocHook_AddMunmapHook(hook); - } - inline static bool RemoveMunmapHook(MunmapHook hook) { - return MallocHook_RemoveMunmapHook(hook); - } - inline static void InvokeMunmapHook(const void* p, size_t size); - - // The MremapHook is invoked whenever a region of memory is remapped. - typedef MallocHook_MremapHook MremapHook; - inline static bool AddMremapHook(MremapHook hook) { - return MallocHook_AddMremapHook(hook); - } - inline static bool RemoveMremapHook(MremapHook hook) { - return MallocHook_RemoveMremapHook(hook); - } - inline static void InvokeMremapHook(const void* result, - const void* old_addr, - size_t old_size, - size_t new_size, - int flags, - const void* new_addr); - - // The PreSbrkHook is invoked just before sbrk is called -- except when - // the increment is 0. This is because sbrk(0) is often called - // to get the top of the memory stack, and is not actually a - // memory-allocation call. It may be useful in memory-limited contexts, - // to catch allocations that will exceed the limit and take outside - // actions to increase such a limit. - typedef MallocHook_PreSbrkHook PreSbrkHook; - inline static bool AddPreSbrkHook(PreSbrkHook hook) { - return MallocHook_AddPreSbrkHook(hook); - } - inline static bool RemovePreSbrkHook(PreSbrkHook hook) { - return MallocHook_RemovePreSbrkHook(hook); - } - inline static void InvokePreSbrkHook(std::ptrdiff_t increment); - - // The SbrkHook is invoked whenever sbrk is called -- except when - // the increment is 0. This is because sbrk(0) is often called - // to get the top of the memory stack, and is not actually a - // memory-allocation call. - typedef MallocHook_SbrkHook SbrkHook; - inline static bool AddSbrkHook(SbrkHook hook) { - return MallocHook_AddSbrkHook(hook); - } - inline static bool RemoveSbrkHook(SbrkHook hook) { - return MallocHook_RemoveSbrkHook(hook); - } - inline static void InvokeSbrkHook(const void* result, std::ptrdiff_t increment); - - // Get the current stack trace. Try to skip all routines up to and - // and including the caller of MallocHook::Invoke*. - // Use "skip_count" (similarly to GetStackTrace from stacktrace.h) - // as a hint about how many routines to skip if better information - // is not available. - inline static int GetCallerStackTrace(void** result, int max_depth, - int skip_count) { - return MallocHook_GetCallerStackTrace(result, max_depth, skip_count); - } - - // Unhooked versions of mmap() and munmap(). These should be used - // only by experts, since they bypass heapchecking, etc. - // Note: These do not run hooks, but they still use the MmapReplacement - // and MunmapReplacement. - static void* UnhookedMMap(void *start, size_t length, int prot, int flags, - int fd, off_t offset); - static int UnhookedMUnmap(void *start, size_t length); - - // The following are DEPRECATED. - inline static NewHook GetNewHook(); - inline static NewHook SetNewHook(NewHook hook) { - return MallocHook_SetNewHook(hook); - } - - inline static DeleteHook GetDeleteHook(); - inline static DeleteHook SetDeleteHook(DeleteHook hook) { - return MallocHook_SetDeleteHook(hook); - } - - inline static PreMmapHook GetPreMmapHook(); - inline static PreMmapHook SetPreMmapHook(PreMmapHook hook) { - return MallocHook_SetPreMmapHook(hook); - } - - inline static MmapHook GetMmapHook(); - inline static MmapHook SetMmapHook(MmapHook hook) { - return MallocHook_SetMmapHook(hook); - } - - inline static MunmapHook GetMunmapHook(); - inline static MunmapHook SetMunmapHook(MunmapHook hook) { - return MallocHook_SetMunmapHook(hook); - } - - inline static MremapHook GetMremapHook(); - inline static MremapHook SetMremapHook(MremapHook hook) { - return MallocHook_SetMremapHook(hook); - } - - inline static PreSbrkHook GetPreSbrkHook(); - inline static PreSbrkHook SetPreSbrkHook(PreSbrkHook hook) { - return MallocHook_SetPreSbrkHook(hook); - } - - inline static SbrkHook GetSbrkHook(); - inline static SbrkHook SetSbrkHook(SbrkHook hook) { - return MallocHook_SetSbrkHook(hook); - } - // End of DEPRECATED methods. - - private: - // Slow path versions of Invoke*Hook. - static void InvokeNewHookSlow(const void* p, size_t s); - static void InvokeDeleteHookSlow(const void* p); - static void InvokePreMmapHookSlow(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); - static void InvokeMmapHookSlow(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); - static bool InvokeMmapReplacementSlow(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset, - void** result); - static void InvokeMunmapHookSlow(const void* p, size_t size); - static bool InvokeMunmapReplacementSlow(const void* p, - size_t size, - int* result); - static void InvokeMremapHookSlow(const void* result, - const void* old_addr, - size_t old_size, - size_t new_size, - int flags, - const void* new_addr); - static void InvokePreSbrkHookSlow(std::ptrdiff_t increment); - static void InvokeSbrkHookSlow(const void* result, std::ptrdiff_t increment); -}; - -#endif /* _MALLOC_HOOK_H_ */ +/* The code has moved to gperftools/. Use that include-directory for + * new code. + */ +#include <gperftools/malloc_hook.h> diff --git a/third_party/tcmalloc/chromium/src/google/malloc_hook_c.h b/third_party/tcmalloc/chromium/src/google/malloc_hook_c.h index b8478c1..e3ac0a4 100644 --- a/third_party/tcmalloc/chromium/src/google/malloc_hook_c.h +++ b/third_party/tcmalloc/chromium/src/google/malloc_hook_c.h @@ -26,148 +26,9 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * -- - * Author: Craig Silverstein - * - * C shims for the C++ malloc_hook.h. See malloc_hook.h for details - * on how to use these. */ -#ifndef _MALLOC_HOOK_C_H_ -#define _MALLOC_HOOK_C_H_ - -#include <stddef.h> -#include <sys/types.h> - -/* Annoying stuff for windows; makes sure clients can import these functions */ -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/* Get the current stack trace. Try to skip all routines up to and - * and including the caller of MallocHook::Invoke*. - * Use "skip_count" (similarly to GetStackTrace from stacktrace.h) - * as a hint about how many routines to skip if better information - * is not available. - */ -PERFTOOLS_DLL_DECL -int MallocHook_GetCallerStackTrace(void** result, int max_depth, - int skip_count); - -/* The MallocHook_{Add,Remove}*Hook functions return 1 on success and 0 on - * failure. +/* The code has moved to gperftools/. Use that include-directory for + * new code. */ - -typedef void (*MallocHook_NewHook)(const void* ptr, size_t size); -PERFTOOLS_DLL_DECL -int MallocHook_AddNewHook(MallocHook_NewHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveNewHook(MallocHook_NewHook hook); - -typedef void (*MallocHook_DeleteHook)(const void* ptr); -PERFTOOLS_DLL_DECL -int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook); - -typedef void (*MallocHook_PreMmapHook)(const void *start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); -PERFTOOLS_DLL_DECL -int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook); - -typedef void (*MallocHook_MmapHook)(const void* result, - const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset); -PERFTOOLS_DLL_DECL -int MallocHook_AddMmapHook(MallocHook_MmapHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook); - -typedef int (*MallocHook_MmapReplacement)(const void* start, - size_t size, - int protection, - int flags, - int fd, - off_t offset, - void** result); -int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook); -int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook); - -typedef void (*MallocHook_MunmapHook)(const void* ptr, size_t size); -PERFTOOLS_DLL_DECL -int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook); - -typedef int (*MallocHook_MunmapReplacement)(const void* ptr, - size_t size, - int* result); -int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook); -int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook); - -typedef void (*MallocHook_MremapHook)(const void* result, - const void* old_addr, - size_t old_size, - size_t new_size, - int flags, - const void* new_addr); -PERFTOOLS_DLL_DECL -int MallocHook_AddMremapHook(MallocHook_MremapHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook); - -typedef void (*MallocHook_PreSbrkHook)(std::ptrdiff_t increment); -PERFTOOLS_DLL_DECL -int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook); - -typedef void (*MallocHook_SbrkHook)(const void* result, std::ptrdiff_t increment); -PERFTOOLS_DLL_DECL -int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook); -PERFTOOLS_DLL_DECL -int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook); - -/* The following are DEPRECATED. */ -PERFTOOLS_DLL_DECL -MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook); -PERFTOOLS_DLL_DECL -MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook); -PERFTOOLS_DLL_DECL -MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook); -PERFTOOLS_DLL_DECL -MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook); -PERFTOOLS_DLL_DECL -MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook); -PERFTOOLS_DLL_DECL -MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook); -PERFTOOLS_DLL_DECL -MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook); -PERFTOOLS_DLL_DECL -MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook); -/* End of DEPRECATED functions. */ - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* _MALLOC_HOOK_C_H_ */ +#include <gperftools/malloc_hook_c.h> diff --git a/third_party/tcmalloc/chromium/src/google/profiler.h b/third_party/tcmalloc/chromium/src/google/profiler.h index 07323e4..67a89c1 100644 --- a/third_party/tcmalloc/chromium/src/google/profiler.h +++ b/third_party/tcmalloc/chromium/src/google/profiler.h @@ -26,141 +26,9 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat - * - * Module for CPU profiling based on periodic pc-sampling. - * - * For full(er) information, see doc/cpuprofile.html - * - * This module is linked into your program with - * no slowdown caused by this unless you activate the profiler - * using one of the following methods: - * - * 1. Before starting the program, set the environment variable - * "PROFILE" to be the name of the file to which the profile - * data should be written. - * - * 2. Programmatically, start and stop the profiler using the - * routines "ProfilerStart(filename)" and "ProfilerStop()". - * - * - * (Note: if using linux 2.4 or earlier, only the main thread may be - * profiled.) - * - * Use pprof to view the resulting profile output. - * % pprof <path_to_executable> <profile_file_name> - * % pprof --gv <path_to_executable> <profile_file_name> - * - * These functions are thread-safe. */ -#ifndef BASE_PROFILER_H_ -#define BASE_PROFILER_H_ - -#include <time.h> /* For time_t */ - -/* Annoying stuff for windows; makes sure clients can import these functions */ -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -/* All this code should be usable from within C apps. */ -#ifdef __cplusplus -extern "C" { -#endif - -/* Profiler options, for use with ProfilerStartWithOptions. To use: - * - * struct ProfilerOptions options; - * memset(&options, 0, sizeof options); - * - * then fill in fields as needed. - * - * This structure is intended to be usable from C code, so no constructor - * is provided to initialize it. (Use memset as described above). - */ -struct ProfilerOptions { - /* Filter function and argument. - * - * If filter_in_thread is not NULL, when a profiling tick is delivered - * the profiler will call: - * - * (*filter_in_thread)(filter_in_thread_arg) - * - * If it returns nonzero, the sample will be included in the profile. - * Note that filter_in_thread runs in a signal handler, so must be - * async-signal-safe. - * - * A typical use would be to set up filter results for each thread - * in the system before starting the profiler, then to make - * filter_in_thread be a very simple function which retrieves those - * results in an async-signal-safe way. Retrieval could be done - * using thread-specific data, or using a shared data structure that - * supports async-signal-safe lookups. - */ - int (*filter_in_thread)(void *arg); - void *filter_in_thread_arg; -}; - -/* Start profiling and write profile info into fname. - * - * This is equivalent to calling ProfilerStartWithOptions(fname, NULL). +/* The code has moved to gperftools/. Use that include-directory for + * new code. */ -PERFTOOLS_DLL_DECL int ProfilerStart(const char* fname); - -/* Start profiling and write profile into fname. - * - * The profiler is configured using the options given by 'options'. - * Options which are not specified are given default values. - * - * 'options' may be NULL, in which case all are given default values. - * - * Returns nonzero if profiling was started sucessfully, or zero else. - */ -PERFTOOLS_DLL_DECL int ProfilerStartWithOptions( - const char *fname, const struct ProfilerOptions *options); - -/* Stop profiling. Can be started again with ProfilerStart(), but - * the currently accumulated profiling data will be cleared. - */ -PERFTOOLS_DLL_DECL void ProfilerStop(); - -/* Flush any currently buffered profiling state to the profile file. - * Has no effect if the profiler has not been started. - */ -PERFTOOLS_DLL_DECL void ProfilerFlush(); - - -/* DEPRECATED: these functions were used to enable/disable profiling - * in the current thread, but no longer do anything. - */ -PERFTOOLS_DLL_DECL void ProfilerEnable(); -PERFTOOLS_DLL_DECL void ProfilerDisable(); - -/* Returns nonzero if profile is currently enabled, zero if it's not. */ -PERFTOOLS_DLL_DECL int ProfilingIsEnabledForAllThreads(); - -/* Routine for registering new threads with the profiler. - */ -PERFTOOLS_DLL_DECL void ProfilerRegisterThread(); - -/* Stores state about profiler's current status into "*state". */ -struct ProfilerState { - int enabled; /* Is profiling currently enabled? */ - time_t start_time; /* If enabled, when was profiling started? */ - char profile_name[1024]; /* Name of profile file being written, or '\0' */ - int samples_gathered; /* Number of samples gathered so far (or 0) */ -}; -PERFTOOLS_DLL_DECL void ProfilerGetCurrentState(struct ProfilerState* state); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif /* BASE_PROFILER_H_ */ +#include <gperftools/profiler.h> diff --git a/third_party/tcmalloc/chromium/src/google/stacktrace.h b/third_party/tcmalloc/chromium/src/google/stacktrace.h index fd186d6..eb761ca 100644 --- a/third_party/tcmalloc/chromium/src/google/stacktrace.h +++ b/third_party/tcmalloc/chromium/src/google/stacktrace.h @@ -27,90 +27,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// --- -// Author: Sanjay Ghemawat -// -// Routines to extract the current stack trace. These functions are -// thread-safe. - -#ifndef GOOGLE_STACKTRACE_H_ -#define GOOGLE_STACKTRACE_H_ - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - - -// Skips the most recent "skip_count" stack frames (also skips the -// frame generated for the "GetStackFrames" routine itself), and then -// records the pc values for up to the next "max_depth" frames in -// "result", and the corresponding stack frame sizes in "sizes". -// Returns the number of values recorded in "result"/"sizes". -// -// Example: -// main() { foo(); } -// foo() { bar(); } -// bar() { -// void* result[10]; -// int sizes[10]; -// int depth = GetStackFrames(result, sizes, 10, 1); -// } -// -// The GetStackFrames call will skip the frame for "bar". It will -// return 2 and will produce pc values that map to the following -// procedures: -// result[0] foo -// result[1] main -// (Actually, there may be a few more entries after "main" to account for -// startup procedures.) -// And corresponding stack frame sizes will also be recorded: -// sizes[0] 16 -// sizes[1] 16 -// (Stack frame sizes of 16 above are just for illustration purposes.) -// Stack frame sizes of 0 or less indicate that those frame sizes couldn't -// be identified. -// -// This routine may return fewer stack frame entries than are -// available. Also note that "result" and "sizes" must both be non-NULL. -extern PERFTOOLS_DLL_DECL int GetStackFrames(void** result, int* sizes, int max_depth, - int skip_count); - -// Same as above, but to be used from a signal handler. The "uc" parameter -// should be the pointer to ucontext_t which was passed as the 3rd parameter -// to sa_sigaction signal handler. It may help the unwinder to get a -// better stack trace under certain conditions. The "uc" may safely be NULL. -extern PERFTOOLS_DLL_DECL int GetStackFramesWithContext(void** result, int* sizes, int max_depth, - int skip_count, const void *uc); - -// This is similar to the GetStackFrames routine, except that it returns -// the stack trace only, and not the stack frame sizes as well. -// Example: -// main() { foo(); } -// foo() { bar(); } -// bar() { -// void* result[10]; -// int depth = GetStackTrace(result, 10, 1); -// } -// -// This produces: -// result[0] foo -// result[1] main -// .... ... -// -// "result" must not be NULL. -extern PERFTOOLS_DLL_DECL int GetStackTrace(void** result, int max_depth, - int skip_count); - -// Same as above, but to be used from a signal handler. The "uc" parameter -// should be the pointer to ucontext_t which was passed as the 3rd parameter -// to sa_sigaction signal handler. It may help the unwinder to get a -// better stack trace under certain conditions. The "uc" may safely be NULL. -extern PERFTOOLS_DLL_DECL int GetStackTraceWithContext(void** result, int max_depth, - int skip_count, const void *uc); - -#endif /* GOOGLE_STACKTRACE_H_ */ +/* The code has moved to gperftools/. Use that include-directory for + * new code. + */ +#include <gperftools/stacktrace.h> diff --git a/third_party/tcmalloc/chromium/src/google/tcmalloc.h b/third_party/tcmalloc/chromium/src/google/tcmalloc.h index 3b0fe7c..c7db631 100644 --- a/third_party/tcmalloc/chromium/src/google/tcmalloc.h +++ b/third_party/tcmalloc/chromium/src/google/tcmalloc.h @@ -1,69 +1,34 @@ -// Copyright (c) 2007, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* Copyright (c) 2003, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ -// --- -// Author: Craig Silverstein <opensource@google.com> -// -// Some obscure memory-allocation routines may not be declared on all -// systems. In those cases, we'll just declare them ourselves. -// This file is meant to be used only internally, for unittests. - -#include <config.h> - -#ifndef _XOPEN_SOURCE -# define _XOPEN_SOURCE 600 // for posix_memalign -#endif -#include <stdlib.h> // for posix_memalign -// FreeBSD has malloc.h, but complains if you use it -#if defined(HAVE_MALLOC_H) && !defined(__FreeBSD__) -#include <malloc.h> // for memalign, valloc, pvalloc -#endif - -// __THROW is defined in glibc systems. It means, counter-intuitively, -// "This function will never throw an exception." It's an optional -// optimization tool, but we may need to use it to match glibc prototypes. -#ifndef __THROW // I guess we're not on a glibc system -# define __THROW // __THROW is just an optimization, so ok to make it "" -#endif - -#if !HAVE_CFREE_SYMBOL -extern "C" void cfree(void* ptr) __THROW; -#endif -#if !HAVE_POSIX_MEMALIGN_SYMBOL -extern "C" int posix_memalign(void** ptr, size_t align, size_t size) __THROW; -#endif -#if !HAVE_MEMALIGN_SYMBOL -extern "C" void* memalign(size_t __alignment, size_t __size) __THROW; -#endif -#if !HAVE_VALLOC_SYMBOL -extern "C" void* valloc(size_t __size) __THROW; -#endif -#if !HAVE_PVALLOC_SYMBOL -extern "C" void* pvalloc(size_t __size) __THROW; -#endif +/* The code has moved to gperftools/. Use that include-directory for + * new code. + */ +#include <gperftools/tcmalloc.h> diff --git a/third_party/tcmalloc/chromium/src/gperftools/heap-checker.h b/third_party/tcmalloc/chromium/src/gperftools/heap-checker.h new file mode 100644 index 0000000..32ed10a --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/heap-checker.h @@ -0,0 +1,424 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Maxim Lifantsev (with design ideas by Sanjay Ghemawat) +// +// +// Module for detecing heap (memory) leaks. +// +// For full(er) information, see doc/heap_checker.html +// +// This module can be linked into programs with +// no slowdown caused by this unless you activate the leak-checker: +// +// 1. Set the environment variable HEAPCHEK to _type_ before +// running the program. +// +// _type_ is usually "normal" but can also be "minimal", "strict", or +// "draconian". (See the html file for other options, like 'local'.) +// +// After that, just run your binary. If the heap-checker detects +// a memory leak at program-exit, it will print instructions on how +// to track down the leak. + +#ifndef BASE_HEAP_CHECKER_H_ +#define BASE_HEAP_CHECKER_H_ + +#include <sys/types.h> // for size_t +// I can't #include config.h in this public API file, but I should +// really use configure (and make malloc_extension.h a .in file) to +// figure out if the system has stdint.h or not. But I'm lazy, so +// for now I'm assuming it's a problem only with MSVC. +#ifndef _MSC_VER +#include <stdint.h> // for uintptr_t +#endif +#include <stdarg.h> // for va_list +#include <vector> + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + + +// The class is thread-safe with respect to all the provided static methods, +// as well as HeapLeakChecker objects: they can be accessed by multiple threads. +class PERFTOOLS_DLL_DECL HeapLeakChecker { + public: + + // ----------------------------------------------------------------------- // + // Static functions for working with (whole-program) leak checking. + + // If heap leak checking is currently active in some mode + // e.g. if leak checking was started (and is still active now) + // due to HEAPCHECK=... defined in the environment. + // The return value reflects iff HeapLeakChecker objects manually + // constructed right now will be doing leak checking or nothing. + // Note that we can go from active to inactive state during InitGoogle() + // if FLAGS_heap_check gets set to "" by some code before/during InitGoogle(). + static bool IsActive(); + + // Return pointer to the whole-program checker if it has been created + // and NULL otherwise. + // Once GlobalChecker() returns non-NULL that object will not disappear and + // will be returned by all later GlobalChecker calls. + // This is mainly to access BytesLeaked() and ObjectsLeaked() (see below) + // for the whole-program checker after one calls NoGlobalLeaks() + // or similar and gets false. + static HeapLeakChecker* GlobalChecker(); + + // Do whole-program leak check now (if it was activated for this binary); + // return false only if it was activated and has failed. + // The mode of the check is controlled by the command-line flags. + // This method can be called repeatedly. + // Things like GlobalChecker()->SameHeap() can also be called explicitly + // to do the desired flavor of the check. + static bool NoGlobalLeaks(); + + // If whole-program checker if active, + // cancel its automatic execution after main() exits. + // This requires that some leak check (e.g. NoGlobalLeaks()) + // has been called at least once on the whole-program checker. + static void CancelGlobalCheck(); + + // ----------------------------------------------------------------------- // + // Non-static functions for starting and doing leak checking. + + // Start checking and name the leak check performed. + // The name is used in naming dumped profiles + // and needs to be unique only within your binary. + // It must also be a string that can be a part of a file name, + // in particular not contain path expressions. + explicit HeapLeakChecker(const char *name); + + // Destructor (verifies that some *NoLeaks or *SameHeap method + // has been called at least once). + ~HeapLeakChecker(); + + // These used to be different but are all the same now: they return + // true iff all memory allocated since this HeapLeakChecker object + // was constructor is still reachable from global state. + // + // Because we fork to convert addresses to symbol-names, and forking + // is not thread-safe, and we may be called in a threaded context, + // we do not try to symbolize addresses when called manually. + bool NoLeaks() { return DoNoLeaks(DO_NOT_SYMBOLIZE); } + + // These forms are obsolete; use NoLeaks() instead. + // TODO(csilvers): mark as DEPRECATED. + bool QuickNoLeaks() { return NoLeaks(); } + bool BriefNoLeaks() { return NoLeaks(); } + bool SameHeap() { return NoLeaks(); } + bool QuickSameHeap() { return NoLeaks(); } + bool BriefSameHeap() { return NoLeaks(); } + + // Detailed information about the number of leaked bytes and objects + // (both of these can be negative as well). + // These are available only after a *SameHeap or *NoLeaks + // method has been called. + // Note that it's possible for both of these to be zero + // while SameHeap() or NoLeaks() returned false in case + // of a heap state change that is significant + // but preserves the byte and object counts. + ssize_t BytesLeaked() const; + ssize_t ObjectsLeaked() const; + + // ----------------------------------------------------------------------- // + // Static helpers to make us ignore certain leaks. + + // Scoped helper class. Should be allocated on the stack inside a + // block of code. Any heap allocations done in the code block + // covered by the scoped object (including in nested function calls + // done by the code block) will not be reported as leaks. This is + // the recommended replacement for the GetDisableChecksStart() and + // DisableChecksToHereFrom() routines below. + // + // Example: + // void Foo() { + // HeapLeakChecker::Disabler disabler; + // ... code that allocates objects whose leaks should be ignored ... + // } + // + // REQUIRES: Destructor runs in same thread as constructor + class Disabler { + public: + Disabler(); + ~Disabler(); + private: + Disabler(const Disabler&); // disallow copy + void operator=(const Disabler&); // and assign + }; + + // Ignore an object located at 'ptr' (can go at the start or into the object) + // as well as all heap objects (transitively) referenced from it for the + // purposes of heap leak checking. Returns 'ptr' so that one can write + // static T* obj = IgnoreObject(new T(...)); + // + // If 'ptr' does not point to an active allocated object at the time of this + // call, it is ignored; but if it does, the object must not get deleted from + // the heap later on. + // + // See also HiddenPointer, below, if you need to prevent a pointer from + // being traversed by the heap checker but do not wish to transitively + // whitelist objects referenced through it. + template <typename T> + static T* IgnoreObject(T* ptr) { + DoIgnoreObject(static_cast<const void*>(const_cast<const T*>(ptr))); + return ptr; + } + + // Undo what an earlier IgnoreObject() call promised and asked to do. + // At the time of this call 'ptr' must point at or inside of an active + // allocated object which was previously registered with IgnoreObject(). + static void UnIgnoreObject(const void* ptr); + + // ----------------------------------------------------------------------- // + // Internal types defined in .cc + + class Allocator; + struct RangeValue; + + private: + + // ----------------------------------------------------------------------- // + // Various helpers + + // Create the name of the heap profile file. + // Should be deleted via Allocator::Free(). + char* MakeProfileNameLocked(); + + // Helper for constructors + void Create(const char *name, bool make_start_snapshot); + + enum ShouldSymbolize { SYMBOLIZE, DO_NOT_SYMBOLIZE }; + + // Helper for *NoLeaks and *SameHeap + bool DoNoLeaks(ShouldSymbolize should_symbolize); + + // Helper for NoGlobalLeaks, also called by the global destructor. + static bool NoGlobalLeaksMaybeSymbolize(ShouldSymbolize should_symbolize); + + // These used to be public, but they are now deprecated. + // Will remove entirely when all internal uses are fixed. + // In the meantime, use friendship so the unittest can still test them. + static void* GetDisableChecksStart(); + static void DisableChecksToHereFrom(const void* start_address); + static void DisableChecksIn(const char* pattern); + friend void RangeDisabledLeaks(); + friend void NamedTwoDisabledLeaks(); + friend void* RunNamedDisabledLeaks(void*); + friend void TestHeapLeakCheckerNamedDisabling(); + // TODO(csilvers): remove this one, at least + friend int main(int, char**); + + + // Actually implements IgnoreObject(). + static void DoIgnoreObject(const void* ptr); + + // Disable checks based on stack trace entry at a depth <= + // max_depth. Used to hide allocations done inside some special + // libraries. + static void DisableChecksFromToLocked(const void* start_address, + const void* end_address, + int max_depth); + + // Helper for DoNoLeaks to ignore all objects reachable from all live data + static void IgnoreAllLiveObjectsLocked(const void* self_stack_top); + + // Callback we pass to ListAllProcessThreads (see thread_lister.h) + // that is invoked when all threads of our process are found and stopped. + // The call back does the things needed to ignore live data reachable from + // thread stacks and registers for all our threads + // as well as do other global-live-data ignoring + // (via IgnoreNonThreadLiveObjectsLocked) + // during the quiet state of all threads being stopped. + // For the argument meaning see the comment by ListAllProcessThreads. + // Here we only use num_threads and thread_pids, that ListAllProcessThreads + // fills for us with the number and pids of all the threads of our process + // it found and attached to. + static int IgnoreLiveThreadsLocked(void* parameter, + int num_threads, + pid_t* thread_pids, + va_list ap); + + // Helper for IgnoreAllLiveObjectsLocked and IgnoreLiveThreadsLocked + // that we prefer to execute from IgnoreLiveThreadsLocked + // while all threads are stopped. + // This helper does live object discovery and ignoring + // for all objects that are reachable from everything + // not related to thread stacks and registers. + static void IgnoreNonThreadLiveObjectsLocked(); + + // Helper for IgnoreNonThreadLiveObjectsLocked and IgnoreLiveThreadsLocked + // to discover and ignore all heap objects + // reachable from currently considered live objects + // (live_objects static global variable in out .cc file). + // "name", "name2" are two strings that we print one after another + // in a debug message to describe what kind of live object sources + // are being used. + static void IgnoreLiveObjectsLocked(const char* name, const char* name2); + + // Do the overall whole-program heap leak check if needed; + // returns true when did the leak check. + static bool DoMainHeapCheck(); + + // Type of task for UseProcMapsLocked + enum ProcMapsTask { + RECORD_GLOBAL_DATA, + DISABLE_LIBRARY_ALLOCS + }; + + // Success/Error Return codes for UseProcMapsLocked. + enum ProcMapsResult { + PROC_MAPS_USED, + CANT_OPEN_PROC_MAPS, + NO_SHARED_LIBS_IN_PROC_MAPS + }; + + // Read /proc/self/maps, parse it, and do the 'proc_maps_task' for each line. + static ProcMapsResult UseProcMapsLocked(ProcMapsTask proc_maps_task); + + // A ProcMapsTask to disable allocations from 'library' + // that is mapped to [start_address..end_address) + // (only if library is a certain system library). + static void DisableLibraryAllocsLocked(const char* library, + uintptr_t start_address, + uintptr_t end_address); + + // Return true iff "*ptr" points to a heap object + // ("*ptr" can point at the start or inside of a heap object + // so that this works e.g. for pointers to C++ arrays, C++ strings, + // multiple-inherited objects, or pointers to members). + // We also fill *object_size for this object then + // and we move "*ptr" to point to the very start of the heap object. + static inline bool HaveOnHeapLocked(const void** ptr, size_t* object_size); + + // Helper to shutdown heap leak checker when it's not needed + // or can't function properly. + static void TurnItselfOffLocked(); + + // Internally-used c-tor to start whole-executable checking. + HeapLeakChecker(); + + // ----------------------------------------------------------------------- // + // Friends and externally accessed helpers. + + // Helper for VerifyHeapProfileTableStackGet in the unittest + // to get the recorded allocation caller for ptr, + // which must be a heap object. + static const void* GetAllocCaller(void* ptr); + friend void VerifyHeapProfileTableStackGet(); + + // This gets to execute before constructors for all global objects + static void BeforeConstructorsLocked(); + friend void HeapLeakChecker_BeforeConstructors(); + + // This gets to execute after destructors for all global objects + friend void HeapLeakChecker_AfterDestructors(); + + // Full starting of recommended whole-program checking. + friend void HeapLeakChecker_InternalInitStart(); + + // Runs REGISTER_HEAPCHECK_CLEANUP cleanups and potentially + // calls DoMainHeapCheck + friend void HeapLeakChecker_RunHeapCleanups(); + + // ----------------------------------------------------------------------- // + // Member data. + + class SpinLock* lock_; // to make HeapLeakChecker objects thread-safe + const char* name_; // our remembered name (we own it) + // NULL means this leak checker is a noop + + // Snapshot taken when the checker was created. May be NULL + // for the global heap checker object. We use void* instead of + // HeapProfileTable::Snapshot* to avoid including heap-profile-table.h. + void* start_snapshot_; + + bool has_checked_; // if we have done the leak check, so these are ready: + ssize_t inuse_bytes_increase_; // bytes-in-use increase for this checker + ssize_t inuse_allocs_increase_; // allocations-in-use increase + // for this checker + bool keep_profiles_; // iff we should keep the heap profiles we've made + + // ----------------------------------------------------------------------- // + + // Disallow "evil" constructors. + HeapLeakChecker(const HeapLeakChecker&); + void operator=(const HeapLeakChecker&); +}; + + +// Holds a pointer that will not be traversed by the heap checker. +// Contrast with HeapLeakChecker::IgnoreObject(o), in which o and +// all objects reachable from o are ignored by the heap checker. +template <class T> +class HiddenPointer { + public: + explicit HiddenPointer(T* t) + : masked_t_(reinterpret_cast<uintptr_t>(t) ^ kHideMask) { + } + // Returns unhidden pointer. Be careful where you save the result. + T* get() const { return reinterpret_cast<T*>(masked_t_ ^ kHideMask); } + + private: + // Arbitrary value, but not such that xor'ing with it is likely + // to map one valid pointer to another valid pointer: + static const uintptr_t kHideMask = + static_cast<uintptr_t>(0xF03A5F7BF03A5F7Bll); + uintptr_t masked_t_; +}; + +// A class that exists solely to run its destructor. This class should not be +// used directly, but instead by the REGISTER_HEAPCHECK_CLEANUP macro below. +class PERFTOOLS_DLL_DECL HeapCleaner { + public: + typedef void (*void_function)(void); + HeapCleaner(void_function f); + static void RunHeapCleanups(); + private: + static std::vector<void_function>* heap_cleanups_; +}; + +// A macro to declare module heap check cleanup tasks +// (they run only if we are doing heap leak checking.) +// 'body' should be the cleanup code to run. 'name' doesn't matter, +// but must be unique amongst all REGISTER_HEAPCHECK_CLEANUP calls. +#define REGISTER_HEAPCHECK_CLEANUP(name, body) \ + namespace { \ + void heapcheck_cleanup_##name() { body; } \ + static HeapCleaner heapcheck_cleaner_##name(&heapcheck_cleanup_##name); \ + } + +#endif // BASE_HEAP_CHECKER_H_ diff --git a/third_party/tcmalloc/chromium/src/gperftools/heap-profiler.h b/third_party/tcmalloc/chromium/src/gperftools/heap-profiler.h new file mode 100644 index 0000000..57cb97a --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/heap-profiler.h @@ -0,0 +1,104 @@ +/* Copyright (c) 2005, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Sanjay Ghemawat + * + * Module for heap-profiling. + * + * For full(er) information, see doc/heapprofile.html + * + * This module can be linked into your program with + * no slowdown caused by this unless you activate the profiler + * using one of the following methods: + * + * 1. Before starting the program, set the environment variable + * "HEAPPROFILE" to be the name of the file to which the profile + * data should be written. + * + * 2. Programmatically, start and stop the profiler using the + * routines "HeapProfilerStart(filename)" and "HeapProfilerStop()". + * + */ + +#ifndef BASE_HEAP_PROFILER_H_ +#define BASE_HEAP_PROFILER_H_ + +#include <stddef.h> + +/* Annoying stuff for windows; makes sure clients can import these functions */ +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + +/* All this code should be usable from within C apps. */ +#ifdef __cplusplus +extern "C" { +#endif + +/* Start profiling and arrange to write profile data to file names + * of the form: "prefix.0000", "prefix.0001", ... + */ +PERFTOOLS_DLL_DECL void HeapProfilerStart(const char* prefix); + +/* Returns non-zero if we are currently profiling the heap. (Returns + * an int rather than a bool so it's usable from C.) This is true + * between calls to HeapProfilerStart() and HeapProfilerStop(), and + * also if the program has been run with HEAPPROFILER, or some other + * way to turn on whole-program profiling. + */ +int IsHeapProfilerRunning(); + +/* Stop heap profiling. Can be restarted again with HeapProfilerStart(), + * but the currently accumulated profiling information will be cleared. + */ +PERFTOOLS_DLL_DECL void HeapProfilerStop(); + +/* Dump a profile now - can be used for dumping at a hopefully + * quiescent state in your program, in order to more easily track down + * memory leaks. Will include the reason in the logged message + */ +PERFTOOLS_DLL_DECL void HeapProfilerDump(const char *reason); + +/* Generate current heap profiling information. + * Returns an empty string when heap profiling is not active. + * The returned pointer is a '\0'-terminated string allocated using malloc() + * and should be free()-ed as soon as the caller does not need it anymore. + */ +PERFTOOLS_DLL_DECL char* GetHeapProfile(); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* BASE_HEAP_PROFILER_H_ */ diff --git a/third_party/tcmalloc/chromium/src/gperftools/malloc_extension.h b/third_party/tcmalloc/chromium/src/gperftools/malloc_extension.h new file mode 100644 index 0000000..5bee019 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/malloc_extension.h @@ -0,0 +1,400 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Sanjay Ghemawat <opensource@google.com> +// +// Extra extensions exported by some malloc implementations. These +// extensions are accessed through a virtual base class so an +// application can link against a malloc that does not implement these +// extensions, and it will get default versions that do nothing. +// +// NOTE FOR C USERS: If you wish to use this functionality from within +// a C program, see malloc_extension_c.h. + +#ifndef BASE_MALLOC_EXTENSION_H_ +#define BASE_MALLOC_EXTENSION_H_ + +#include <stddef.h> +// I can't #include config.h in this public API file, but I should +// really use configure (and make malloc_extension.h a .in file) to +// figure out if the system has stdint.h or not. But I'm lazy, so +// for now I'm assuming it's a problem only with MSVC. +#ifndef _MSC_VER +#include <stdint.h> +#endif +#include <string> +#include <vector> + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + +static const int kMallocHistogramSize = 64; + +// One day, we could support other types of writers (perhaps for C?) +typedef std::string MallocExtensionWriter; + +namespace base { +struct MallocRange; +} + +// Interface to a pluggable system allocator. +class SysAllocator { + public: + SysAllocator() { + } + virtual ~SysAllocator(); + + // Allocates "size"-byte of memory from system aligned with "alignment". + // Returns NULL if failed. Otherwise, the returned pointer p up to and + // including (p + actual_size -1) have been allocated. + virtual void* Alloc(size_t size, size_t *actual_size, size_t alignment) = 0; +}; + +// The default implementations of the following routines do nothing. +// All implementations should be thread-safe; the current one +// (TCMallocImplementation) is. +class PERFTOOLS_DLL_DECL MallocExtension { + public: + virtual ~MallocExtension(); + + // Call this very early in the program execution -- say, in a global + // constructor -- to set up parameters and state needed by all + // instrumented malloc implemenatations. One example: this routine + // sets environemnt variables to tell STL to use libc's malloc() + // instead of doing its own memory management. This is safe to call + // multiple times, as long as each time is before threads start up. + static void Initialize(); + + // See "verify_memory.h" to see what these routines do + virtual bool VerifyAllMemory(); + virtual bool VerifyNewMemory(const void* p); + virtual bool VerifyArrayNewMemory(const void* p); + virtual bool VerifyMallocMemory(const void* p); + virtual bool MallocMemoryStats(int* blocks, size_t* total, + int histogram[kMallocHistogramSize]); + + // Get a human readable description of the current state of the malloc + // data structures. The state is stored as a null-terminated string + // in a prefix of "buffer[0,buffer_length-1]". + // REQUIRES: buffer_length > 0. + virtual void GetStats(char* buffer, int buffer_length); + + // Outputs to "writer" a sample of live objects and the stack traces + // that allocated these objects. The format of the returned output + // is equivalent to the output of the heap profiler and can + // therefore be passed to "pprof". This function is equivalent to + // ReadStackTraces. The main difference is that this function returns + // serialized data appropriately formatted for use by the pprof tool. + // NOTE: by default, tcmalloc does not do any heap sampling, and this + // function will always return an empty sample. To get useful + // data from GetHeapSample, you must also set the environment + // variable TCMALLOC_SAMPLE_PARAMETER to a value such as 524288. + virtual void GetHeapSample(MallocExtensionWriter* writer); + + // Outputs to "writer" the stack traces that caused growth in the + // address space size. The format of the returned output is + // equivalent to the output of the heap profiler and can therefore + // be passed to "pprof". This function is equivalent to + // ReadHeapGrowthStackTraces. The main difference is that this function + // returns serialized data appropriately formatted for use by the + // pprof tool. (This does not depend on, or require, + // TCMALLOC_SAMPLE_PARAMETER.) + virtual void GetHeapGrowthStacks(MallocExtensionWriter* writer); + + // Invokes func(arg, range) for every controlled memory + // range. *range is filled in with information about the range. + // + // This is a best-effort interface useful only for performance + // analysis. The implementation may not call func at all. + typedef void (RangeFunction)(void*, const base::MallocRange*); + virtual void Ranges(void* arg, RangeFunction func); + + // ------------------------------------------------------------------- + // Control operations for getting and setting malloc implementation + // specific parameters. Some currently useful properties: + // + // generic + // ------- + // "generic.current_allocated_bytes" + // Number of bytes currently allocated by application + // This property is not writable. + // + // "generic.heap_size" + // Number of bytes in the heap == + // current_allocated_bytes + + // fragmentation + + // freed memory regions + // This property is not writable. + // + // tcmalloc + // -------- + // "tcmalloc.max_total_thread_cache_bytes" + // Upper limit on total number of bytes stored across all + // per-thread caches. Default: 16MB. + // + // "tcmalloc.current_total_thread_cache_bytes" + // Number of bytes used across all thread caches. + // This property is not writable. + // + // "tcmalloc.pageheap_free_bytes" + // Number of bytes in free, mapped pages in page heap. These + // bytes can be used to fulfill allocation requests. They + // always count towards virtual memory usage, and unless the + // underlying memory is swapped out by the OS, they also count + // towards physical memory usage. This property is not writable. + // + // "tcmalloc.pageheap_unmapped_bytes" + // Number of bytes in free, unmapped pages in page heap. + // These are bytes that have been released back to the OS, + // possibly by one of the MallocExtension "Release" calls. + // They can be used to fulfill allocation requests, but + // typically incur a page fault. They always count towards + // virtual memory usage, and depending on the OS, typically + // do not count towards physical memory usage. This property + // is not writable. + // ------------------------------------------------------------------- + + // Get the named "property"'s value. Returns true if the property + // is known. Returns false if the property is not a valid property + // name for the current malloc implementation. + // REQUIRES: property != NULL; value != NULL + virtual bool GetNumericProperty(const char* property, size_t* value); + + // Set the named "property"'s value. Returns true if the property + // is known and writable. Returns false if the property is not a + // valid property name for the current malloc implementation, or + // is not writable. + // REQUIRES: property != NULL + virtual bool SetNumericProperty(const char* property, size_t value); + + // Mark the current thread as "idle". This routine may optionally + // be called by threads as a hint to the malloc implementation that + // any thread-specific resources should be released. Note: this may + // be an expensive routine, so it should not be called too often. + // + // Also, if the code that calls this routine will go to sleep for + // a while, it should take care to not allocate anything between + // the call to this routine and the beginning of the sleep. + // + // Most malloc implementations ignore this routine. + virtual void MarkThreadIdle(); + + // Mark the current thread as "busy". This routine should be + // called after MarkThreadIdle() if the thread will now do more + // work. If this method is not called, performance may suffer. + // + // Most malloc implementations ignore this routine. + virtual void MarkThreadBusy(); + + // Gets the system allocator used by the malloc extension instance. Returns + // NULL for malloc implementations that do not support pluggable system + // allocators. + virtual SysAllocator* GetSystemAllocator(); + + // Sets the system allocator to the specified. + // + // Users could register their own system allocators for malloc implementation + // that supports pluggable system allocators, such as TCMalloc, by doing: + // alloc = new MyOwnSysAllocator(); + // MallocExtension::instance()->SetSystemAllocator(alloc); + // It's up to users whether to fall back (recommended) to the default + // system allocator (use GetSystemAllocator() above) or not. The caller is + // responsible to any necessary locking. + // See tcmalloc/system-alloc.h for the interface and + // tcmalloc/memfs_malloc.cc for the examples. + // + // It's a no-op for malloc implementations that do not support pluggable + // system allocators. + virtual void SetSystemAllocator(SysAllocator *a); + + // Try to release num_bytes of free memory back to the operating + // system for reuse. Use this extension with caution -- to get this + // memory back may require faulting pages back in by the OS, and + // that may be slow. (Currently only implemented in tcmalloc.) + virtual void ReleaseToSystem(size_t num_bytes); + + // Same as ReleaseToSystem() but release as much memory as possible. + virtual void ReleaseFreeMemory(); + + // Sets the rate at which we release unused memory to the system. + // Zero means we never release memory back to the system. Increase + // this flag to return memory faster; decrease it to return memory + // slower. Reasonable rates are in the range [0,10]. (Currently + // only implemented in tcmalloc). + virtual void SetMemoryReleaseRate(double rate); + + // Gets the release rate. Returns a value < 0 if unknown. + virtual double GetMemoryReleaseRate(); + + // Returns the estimated number of bytes that will be allocated for + // a request of "size" bytes. This is an estimate: an allocation of + // SIZE bytes may reserve more bytes, but will never reserve less. + // (Currently only implemented in tcmalloc, other implementations + // always return SIZE.) + // This is equivalent to malloc_good_size() in OS X. + virtual size_t GetEstimatedAllocatedSize(size_t size); + + // Returns the actual number N of bytes reserved by tcmalloc for the + // pointer p. The client is allowed to use the range of bytes + // [p, p+N) in any way it wishes (i.e. N is the "usable size" of this + // allocation). This number may be equal to or greater than the number + // of bytes requested when p was allocated. + // p must have been allocated by this malloc implementation, + // must not be an interior pointer -- that is, must be exactly + // the pointer returned to by malloc() et al., not some offset + // from that -- and should not have been freed yet. p may be NULL. + // (Currently only implemented in tcmalloc; other implementations + // will return 0.) + // This is equivalent to malloc_size() in OS X, malloc_usable_size() + // in glibc, and _msize() for windows. + virtual size_t GetAllocatedSize(const void* p); + + // Returns kOwned if this malloc implementation allocated the memory + // pointed to by p, or kNotOwned if some other malloc implementation + // allocated it or p is NULL. May also return kUnknownOwnership if + // the malloc implementation does not keep track of ownership. + // REQUIRES: p must be a value returned from a previous call to + // malloc(), calloc(), realloc(), memalign(), posix_memalign(), + // valloc(), pvalloc(), new, or new[], and must refer to memory that + // is currently allocated (so, for instance, you should not pass in + // a pointer after having called free() on it). + enum Ownership { + // NOTE: Enum values MUST be kept in sync with the version in + // malloc_extension_c.h + kUnknownOwnership = 0, + kOwned, + kNotOwned + }; + virtual Ownership GetOwnership(const void* p); + + // The current malloc implementation. Always non-NULL. + static MallocExtension* instance(); + + // Change the malloc implementation. Typically called by the + // malloc implementation during initialization. + static void Register(MallocExtension* implementation); + + // Returns detailed information about malloc's freelists. For each list, + // return a FreeListInfo: + struct FreeListInfo { + size_t min_object_size; + size_t max_object_size; + size_t total_bytes_free; + const char* type; + }; + // Each item in the vector refers to a different freelist. The lists + // are identified by the range of allocations that objects in the + // list can satisfy ([min_object_size, max_object_size]) and the + // type of freelist (see below). The current size of the list is + // returned in total_bytes_free (which count against a processes + // resident and virtual size). + // + // Currently supported types are: + // + // "tcmalloc.page{_unmapped}" - tcmalloc's page heap. An entry for each size + // class in the page heap is returned. Bytes in "page_unmapped" + // are no longer backed by physical memory and do not count against + // the resident size of a process. + // + // "tcmalloc.large{_unmapped}" - tcmalloc's list of objects larger + // than the largest page heap size class. Only one "large" + // entry is returned. There is no upper-bound on the size + // of objects in the large free list; this call returns + // kint64max for max_object_size. Bytes in + // "large_unmapped" are no longer backed by physical memory + // and do not count against the resident size of a process. + // + // "tcmalloc.central" - tcmalloc's central free-list. One entry per + // size-class is returned. Never unmapped. + // + // "debug.free_queue" - free objects queued by the debug allocator + // and not returned to tcmalloc. + // + // "tcmalloc.thread" - tcmalloc's per-thread caches. Never unmapped. + virtual void GetFreeListSizes(std::vector<FreeListInfo>* v); + + // Get a list of stack traces of sampled allocation points. Returns + // a pointer to a "new[]-ed" result array, and stores the sample + // period in "sample_period". + // + // The state is stored as a sequence of adjacent entries + // in the returned array. Each entry has the following form: + // uintptr_t count; // Number of objects with following trace + // uintptr_t size; // Total size of objects with following trace + // uintptr_t depth; // Number of PC values in stack trace + // void* stack[depth]; // PC values that form the stack trace + // + // The list of entries is terminated by a "count" of 0. + // + // It is the responsibility of the caller to "delete[]" the returned array. + // + // May return NULL to indicate no results. + // + // This is an internal extension. Callers should use the more + // convenient "GetHeapSample(string*)" method defined above. + virtual void** ReadStackTraces(int* sample_period); + + // Like ReadStackTraces(), but returns stack traces that caused growth + // in the address space size. + virtual void** ReadHeapGrowthStackTraces(); +}; + +namespace base { + +// Information passed per range. More fields may be added later. +struct MallocRange { + enum Type { + INUSE, // Application is using this range + FREE, // Range is currently free + UNMAPPED, // Backing physical memory has been returned to the OS + UNKNOWN, + // More enum values may be added in the future + }; + + uintptr_t address; // Address of range + size_t length; // Byte length of range + Type type; // Type of this range + double fraction; // Fraction of range that is being used (0 if !INUSE) + + // Perhaps add the following: + // - stack trace if this range was sampled + // - heap growth stack trace if applicable to this range + // - age when allocated (for inuse) or freed (if not in use) +}; + +} // namespace base + +#endif // BASE_MALLOC_EXTENSION_H_ diff --git a/third_party/tcmalloc/chromium/src/gperftools/malloc_extension_c.h b/third_party/tcmalloc/chromium/src/gperftools/malloc_extension_c.h new file mode 100644 index 0000000..72a0a7c --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/malloc_extension_c.h @@ -0,0 +1,99 @@ +/* Copyright (c) 2008, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * -- + * Author: Craig Silverstein + * + * C shims for the C++ malloc_extension.h. See malloc_extension.h for + * details. Note these C shims always work on + * MallocExtension::instance(); it is not possible to have more than + * one MallocExtension object in C applications. + */ + +#ifndef _MALLOC_EXTENSION_C_H_ +#define _MALLOC_EXTENSION_C_H_ + +#include <stddef.h> +#include <sys/types.h> + +/* Annoying stuff for windows -- makes sure clients can import these fns */ +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define kMallocExtensionHistogramSize 64 + +PERFTOOLS_DLL_DECL int MallocExtension_VerifyAllMemory(void); +PERFTOOLS_DLL_DECL int MallocExtension_VerifyNewMemory(const void* p); +PERFTOOLS_DLL_DECL int MallocExtension_VerifyArrayNewMemory(const void* p); +PERFTOOLS_DLL_DECL int MallocExtension_VerifyMallocMemory(const void* p); +PERFTOOLS_DLL_DECL int MallocExtension_MallocMemoryStats(int* blocks, size_t* total, + int histogram[kMallocExtensionHistogramSize]); +PERFTOOLS_DLL_DECL void MallocExtension_GetStats(char* buffer, int buffer_length); + +/* TODO(csilvers): write a C version of these routines, that perhaps + * takes a function ptr and a void *. + */ +/* void MallocExtension_GetHeapSample(string* result); */ +/* void MallocExtension_GetHeapGrowthStacks(string* result); */ + +PERFTOOLS_DLL_DECL int MallocExtension_GetNumericProperty(const char* property, size_t* value); +PERFTOOLS_DLL_DECL int MallocExtension_SetNumericProperty(const char* property, size_t value); +PERFTOOLS_DLL_DECL void MallocExtension_MarkThreadIdle(void); +PERFTOOLS_DLL_DECL void MallocExtension_MarkThreadBusy(void); +PERFTOOLS_DLL_DECL void MallocExtension_ReleaseToSystem(size_t num_bytes); +PERFTOOLS_DLL_DECL void MallocExtension_ReleaseFreeMemory(void); +PERFTOOLS_DLL_DECL size_t MallocExtension_GetEstimatedAllocatedSize(size_t size); +PERFTOOLS_DLL_DECL size_t MallocExtension_GetAllocatedSize(const void* p); + +/* + * NOTE: These enum values MUST be kept in sync with the version in + * malloc_extension.h + */ +typedef enum { + MallocExtension_kUnknownOwnership = 0, + MallocExtension_kOwned, + MallocExtension_kNotOwned +} MallocExtension_Ownership; + +PERFTOOLS_DLL_DECL MallocExtension_Ownership MallocExtension_GetOwnership(const void* p); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* _MALLOC_EXTENSION_C_H_ */ diff --git a/third_party/tcmalloc/chromium/src/gperftools/malloc_hook.h b/third_party/tcmalloc/chromium/src/gperftools/malloc_hook.h new file mode 100644 index 0000000..673a3f3 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/malloc_hook.h @@ -0,0 +1,358 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Sanjay Ghemawat +// +// Some of our malloc implementations can invoke the following hooks whenever +// memory is allocated or deallocated. MallocHook is thread-safe, and things +// you do before calling AddFooHook(MyHook) are visible to any resulting calls +// to MyHook. Hooks must be thread-safe. If you write: +// +// CHECK(MallocHook::AddNewHook(&MyNewHook)); +// +// MyNewHook will be invoked in subsequent calls in the current thread, but +// there are no guarantees on when it might be invoked in other threads. +// +// There are a limited number of slots available for each hook type. Add*Hook +// will return false if there are no slots available. Remove*Hook will return +// false if the given hook was not already installed. +// +// The order in which individual hooks are called in Invoke*Hook is undefined. +// +// It is safe for a hook to remove itself within Invoke*Hook and add other +// hooks. Any hooks added inside a hook invocation (for the same hook type) +// will not be invoked for the current invocation. +// +// One important user of these hooks is the heap profiler. +// +// CAVEAT: If you add new MallocHook::Invoke* calls then those calls must be +// directly in the code of the (de)allocation function that is provided to the +// user and that function must have an ATTRIBUTE_SECTION(malloc_hook) attribute. +// +// Note: the Invoke*Hook() functions are defined in malloc_hook-inl.h. If you +// need to invoke a hook (which you shouldn't unless you're part of tcmalloc), +// be sure to #include malloc_hook-inl.h in addition to malloc_hook.h. +// +// NOTE FOR C USERS: If you want to use malloc_hook functionality from +// a C program, #include malloc_hook_c.h instead of this file. + +#ifndef _MALLOC_HOOK_H_ +#define _MALLOC_HOOK_H_ + +#include <stddef.h> +#include <sys/types.h> +extern "C" { +#include <gperftools/malloc_hook_c.h> // a C version of the malloc_hook interface +} + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + +// The C++ methods below call the C version (MallocHook_*), and thus +// convert between an int and a bool. Windows complains about this +// (a "performance warning") which we don't care about, so we suppress. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4800) +#endif + +// Note: malloc_hook_c.h defines MallocHook_*Hook and +// MallocHook_{Add,Remove}*Hook. The version of these inside the MallocHook +// class are defined in terms of the malloc_hook_c version. See malloc_hook_c.h +// for details of these types/functions. + +class PERFTOOLS_DLL_DECL MallocHook { + public: + // The NewHook is invoked whenever an object is allocated. + // It may be passed NULL if the allocator returned NULL. + typedef MallocHook_NewHook NewHook; + inline static bool AddNewHook(NewHook hook) { + return MallocHook_AddNewHook(hook); + } + inline static bool RemoveNewHook(NewHook hook) { + return MallocHook_RemoveNewHook(hook); + } + inline static void InvokeNewHook(const void* p, size_t s); + + // The DeleteHook is invoked whenever an object is deallocated. + // It may be passed NULL if the caller is trying to delete NULL. + typedef MallocHook_DeleteHook DeleteHook; + inline static bool AddDeleteHook(DeleteHook hook) { + return MallocHook_AddDeleteHook(hook); + } + inline static bool RemoveDeleteHook(DeleteHook hook) { + return MallocHook_RemoveDeleteHook(hook); + } + inline static void InvokeDeleteHook(const void* p); + + // The PreMmapHook is invoked with mmap or mmap64 arguments just + // before the call is actually made. Such a hook may be useful + // in memory limited contexts, to catch allocations that will exceed + // a memory limit, and take outside actions to increase that limit. + typedef MallocHook_PreMmapHook PreMmapHook; + inline static bool AddPreMmapHook(PreMmapHook hook) { + return MallocHook_AddPreMmapHook(hook); + } + inline static bool RemovePreMmapHook(PreMmapHook hook) { + return MallocHook_RemovePreMmapHook(hook); + } + inline static void InvokePreMmapHook(const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset); + + // The MmapReplacement is invoked after the PreMmapHook but before + // the call is actually made. The MmapReplacement should return true + // if it handled the call, or false if it is still necessary to + // call mmap/mmap64. + // This should be used only by experts, and users must be be + // extremely careful to avoid recursive calls to mmap. The replacement + // should be async signal safe. + // Only one MmapReplacement is supported. After setting an MmapReplacement + // you must call RemoveMmapReplacement before calling SetMmapReplacement + // again. + typedef MallocHook_MmapReplacement MmapReplacement; + inline static bool SetMmapReplacement(MmapReplacement hook) { + return MallocHook_SetMmapReplacement(hook); + } + inline static bool RemoveMmapReplacement(MmapReplacement hook) { + return MallocHook_RemoveMmapReplacement(hook); + } + inline static bool InvokeMmapReplacement(const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset, + void** result); + + + // The MmapHook is invoked whenever a region of memory is mapped. + // It may be passed MAP_FAILED if the mmap failed. + typedef MallocHook_MmapHook MmapHook; + inline static bool AddMmapHook(MmapHook hook) { + return MallocHook_AddMmapHook(hook); + } + inline static bool RemoveMmapHook(MmapHook hook) { + return MallocHook_RemoveMmapHook(hook); + } + inline static void InvokeMmapHook(const void* result, + const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset); + + // The MunmapReplacement is invoked with munmap arguments just before + // the call is actually made. The MunmapReplacement should return true + // if it handled the call, or false if it is still necessary to + // call munmap. + // This should be used only by experts. The replacement should be + // async signal safe. + // Only one MunmapReplacement is supported. After setting an + // MunmapReplacement you must call RemoveMunmapReplacement before + // calling SetMunmapReplacement again. + typedef MallocHook_MunmapReplacement MunmapReplacement; + inline static bool SetMunmapReplacement(MunmapReplacement hook) { + return MallocHook_SetMunmapReplacement(hook); + } + inline static bool RemoveMunmapReplacement(MunmapReplacement hook) { + return MallocHook_RemoveMunmapReplacement(hook); + } + inline static bool InvokeMunmapReplacement(const void* p, + size_t size, + int* result); + + // The MunmapHook is invoked whenever a region of memory is unmapped. + typedef MallocHook_MunmapHook MunmapHook; + inline static bool AddMunmapHook(MunmapHook hook) { + return MallocHook_AddMunmapHook(hook); + } + inline static bool RemoveMunmapHook(MunmapHook hook) { + return MallocHook_RemoveMunmapHook(hook); + } + inline static void InvokeMunmapHook(const void* p, size_t size); + + // The MremapHook is invoked whenever a region of memory is remapped. + typedef MallocHook_MremapHook MremapHook; + inline static bool AddMremapHook(MremapHook hook) { + return MallocHook_AddMremapHook(hook); + } + inline static bool RemoveMremapHook(MremapHook hook) { + return MallocHook_RemoveMremapHook(hook); + } + inline static void InvokeMremapHook(const void* result, + const void* old_addr, + size_t old_size, + size_t new_size, + int flags, + const void* new_addr); + + // The PreSbrkHook is invoked just before sbrk is called -- except when + // the increment is 0. This is because sbrk(0) is often called + // to get the top of the memory stack, and is not actually a + // memory-allocation call. It may be useful in memory-limited contexts, + // to catch allocations that will exceed the limit and take outside + // actions to increase such a limit. + typedef MallocHook_PreSbrkHook PreSbrkHook; + inline static bool AddPreSbrkHook(PreSbrkHook hook) { + return MallocHook_AddPreSbrkHook(hook); + } + inline static bool RemovePreSbrkHook(PreSbrkHook hook) { + return MallocHook_RemovePreSbrkHook(hook); + } + inline static void InvokePreSbrkHook(std::ptrdiff_t increment); + + // The SbrkHook is invoked whenever sbrk is called -- except when + // the increment is 0. This is because sbrk(0) is often called + // to get the top of the memory stack, and is not actually a + // memory-allocation call. + typedef MallocHook_SbrkHook SbrkHook; + inline static bool AddSbrkHook(SbrkHook hook) { + return MallocHook_AddSbrkHook(hook); + } + inline static bool RemoveSbrkHook(SbrkHook hook) { + return MallocHook_RemoveSbrkHook(hook); + } + inline static void InvokeSbrkHook(const void* result, std::ptrdiff_t increment); + + // Get the current stack trace. Try to skip all routines up to and + // and including the caller of MallocHook::Invoke*. + // Use "skip_count" (similarly to GetStackTrace from stacktrace.h) + // as a hint about how many routines to skip if better information + // is not available. + inline static int GetCallerStackTrace(void** result, int max_depth, + int skip_count) { + return MallocHook_GetCallerStackTrace(result, max_depth, skip_count); + } + + // Unhooked versions of mmap() and munmap(). These should be used + // only by experts, since they bypass heapchecking, etc. + // Note: These do not run hooks, but they still use the MmapReplacement + // and MunmapReplacement. + static void* UnhookedMMap(void *start, size_t length, int prot, int flags, + int fd, off_t offset); + static int UnhookedMUnmap(void *start, size_t length); + + // The following are DEPRECATED. + inline static NewHook GetNewHook(); + inline static NewHook SetNewHook(NewHook hook) { + return MallocHook_SetNewHook(hook); + } + + inline static DeleteHook GetDeleteHook(); + inline static DeleteHook SetDeleteHook(DeleteHook hook) { + return MallocHook_SetDeleteHook(hook); + } + + inline static PreMmapHook GetPreMmapHook(); + inline static PreMmapHook SetPreMmapHook(PreMmapHook hook) { + return MallocHook_SetPreMmapHook(hook); + } + + inline static MmapHook GetMmapHook(); + inline static MmapHook SetMmapHook(MmapHook hook) { + return MallocHook_SetMmapHook(hook); + } + + inline static MunmapHook GetMunmapHook(); + inline static MunmapHook SetMunmapHook(MunmapHook hook) { + return MallocHook_SetMunmapHook(hook); + } + + inline static MremapHook GetMremapHook(); + inline static MremapHook SetMremapHook(MremapHook hook) { + return MallocHook_SetMremapHook(hook); + } + + inline static PreSbrkHook GetPreSbrkHook(); + inline static PreSbrkHook SetPreSbrkHook(PreSbrkHook hook) { + return MallocHook_SetPreSbrkHook(hook); + } + + inline static SbrkHook GetSbrkHook(); + inline static SbrkHook SetSbrkHook(SbrkHook hook) { + return MallocHook_SetSbrkHook(hook); + } + // End of DEPRECATED methods. + + private: + // Slow path versions of Invoke*Hook. + static void InvokeNewHookSlow(const void* p, size_t s); + static void InvokeDeleteHookSlow(const void* p); + static void InvokePreMmapHookSlow(const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset); + static void InvokeMmapHookSlow(const void* result, + const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset); + static bool InvokeMmapReplacementSlow(const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset, + void** result); + static void InvokeMunmapHookSlow(const void* p, size_t size); + static bool InvokeMunmapReplacementSlow(const void* p, + size_t size, + int* result); + static void InvokeMremapHookSlow(const void* result, + const void* old_addr, + size_t old_size, + size_t new_size, + int flags, + const void* new_addr); + static void InvokePreSbrkHookSlow(std::ptrdiff_t increment); + static void InvokeSbrkHookSlow(const void* result, std::ptrdiff_t increment); +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + +#endif /* _MALLOC_HOOK_H_ */ diff --git a/third_party/tcmalloc/chromium/src/gperftools/malloc_hook_c.h b/third_party/tcmalloc/chromium/src/gperftools/malloc_hook_c.h new file mode 100644 index 0000000..b8478c1 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/malloc_hook_c.h @@ -0,0 +1,173 @@ +/* Copyright (c) 2008, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * -- + * Author: Craig Silverstein + * + * C shims for the C++ malloc_hook.h. See malloc_hook.h for details + * on how to use these. + */ + +#ifndef _MALLOC_HOOK_C_H_ +#define _MALLOC_HOOK_C_H_ + +#include <stddef.h> +#include <sys/types.h> + +/* Annoying stuff for windows; makes sure clients can import these functions */ +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Get the current stack trace. Try to skip all routines up to and + * and including the caller of MallocHook::Invoke*. + * Use "skip_count" (similarly to GetStackTrace from stacktrace.h) + * as a hint about how many routines to skip if better information + * is not available. + */ +PERFTOOLS_DLL_DECL +int MallocHook_GetCallerStackTrace(void** result, int max_depth, + int skip_count); + +/* The MallocHook_{Add,Remove}*Hook functions return 1 on success and 0 on + * failure. + */ + +typedef void (*MallocHook_NewHook)(const void* ptr, size_t size); +PERFTOOLS_DLL_DECL +int MallocHook_AddNewHook(MallocHook_NewHook hook); +PERFTOOLS_DLL_DECL +int MallocHook_RemoveNewHook(MallocHook_NewHook hook); + +typedef void (*MallocHook_DeleteHook)(const void* ptr); +PERFTOOLS_DLL_DECL +int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook); +PERFTOOLS_DLL_DECL +int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook); + +typedef void (*MallocHook_PreMmapHook)(const void *start, + size_t size, + int protection, + int flags, + int fd, + off_t offset); +PERFTOOLS_DLL_DECL +int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook); +PERFTOOLS_DLL_DECL +int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook); + +typedef void (*MallocHook_MmapHook)(const void* result, + const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset); +PERFTOOLS_DLL_DECL +int MallocHook_AddMmapHook(MallocHook_MmapHook hook); +PERFTOOLS_DLL_DECL +int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook); + +typedef int (*MallocHook_MmapReplacement)(const void* start, + size_t size, + int protection, + int flags, + int fd, + off_t offset, + void** result); +int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook); +int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook); + +typedef void (*MallocHook_MunmapHook)(const void* ptr, size_t size); +PERFTOOLS_DLL_DECL +int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook); +PERFTOOLS_DLL_DECL +int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook); + +typedef int (*MallocHook_MunmapReplacement)(const void* ptr, + size_t size, + int* result); +int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook); +int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook); + +typedef void (*MallocHook_MremapHook)(const void* result, + const void* old_addr, + size_t old_size, + size_t new_size, + int flags, + const void* new_addr); +PERFTOOLS_DLL_DECL +int MallocHook_AddMremapHook(MallocHook_MremapHook hook); +PERFTOOLS_DLL_DECL +int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook); + +typedef void (*MallocHook_PreSbrkHook)(std::ptrdiff_t increment); +PERFTOOLS_DLL_DECL +int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook); +PERFTOOLS_DLL_DECL +int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook); + +typedef void (*MallocHook_SbrkHook)(const void* result, std::ptrdiff_t increment); +PERFTOOLS_DLL_DECL +int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook); +PERFTOOLS_DLL_DECL +int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook); + +/* The following are DEPRECATED. */ +PERFTOOLS_DLL_DECL +MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook); +PERFTOOLS_DLL_DECL +MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook); +PERFTOOLS_DLL_DECL +MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook); +PERFTOOLS_DLL_DECL +MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook); +PERFTOOLS_DLL_DECL +MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook); +PERFTOOLS_DLL_DECL +MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook); +PERFTOOLS_DLL_DECL +MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook); +PERFTOOLS_DLL_DECL +MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook); +/* End of DEPRECATED functions. */ + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* _MALLOC_HOOK_C_H_ */ diff --git a/third_party/tcmalloc/chromium/src/gperftools/profiler.h b/third_party/tcmalloc/chromium/src/gperftools/profiler.h new file mode 100644 index 0000000..07323e4 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/profiler.h @@ -0,0 +1,166 @@ +/* Copyright (c) 2005, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Sanjay Ghemawat + * + * Module for CPU profiling based on periodic pc-sampling. + * + * For full(er) information, see doc/cpuprofile.html + * + * This module is linked into your program with + * no slowdown caused by this unless you activate the profiler + * using one of the following methods: + * + * 1. Before starting the program, set the environment variable + * "PROFILE" to be the name of the file to which the profile + * data should be written. + * + * 2. Programmatically, start and stop the profiler using the + * routines "ProfilerStart(filename)" and "ProfilerStop()". + * + * + * (Note: if using linux 2.4 or earlier, only the main thread may be + * profiled.) + * + * Use pprof to view the resulting profile output. + * % pprof <path_to_executable> <profile_file_name> + * % pprof --gv <path_to_executable> <profile_file_name> + * + * These functions are thread-safe. + */ + +#ifndef BASE_PROFILER_H_ +#define BASE_PROFILER_H_ + +#include <time.h> /* For time_t */ + +/* Annoying stuff for windows; makes sure clients can import these functions */ +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + +/* All this code should be usable from within C apps. */ +#ifdef __cplusplus +extern "C" { +#endif + +/* Profiler options, for use with ProfilerStartWithOptions. To use: + * + * struct ProfilerOptions options; + * memset(&options, 0, sizeof options); + * + * then fill in fields as needed. + * + * This structure is intended to be usable from C code, so no constructor + * is provided to initialize it. (Use memset as described above). + */ +struct ProfilerOptions { + /* Filter function and argument. + * + * If filter_in_thread is not NULL, when a profiling tick is delivered + * the profiler will call: + * + * (*filter_in_thread)(filter_in_thread_arg) + * + * If it returns nonzero, the sample will be included in the profile. + * Note that filter_in_thread runs in a signal handler, so must be + * async-signal-safe. + * + * A typical use would be to set up filter results for each thread + * in the system before starting the profiler, then to make + * filter_in_thread be a very simple function which retrieves those + * results in an async-signal-safe way. Retrieval could be done + * using thread-specific data, or using a shared data structure that + * supports async-signal-safe lookups. + */ + int (*filter_in_thread)(void *arg); + void *filter_in_thread_arg; +}; + +/* Start profiling and write profile info into fname. + * + * This is equivalent to calling ProfilerStartWithOptions(fname, NULL). + */ +PERFTOOLS_DLL_DECL int ProfilerStart(const char* fname); + +/* Start profiling and write profile into fname. + * + * The profiler is configured using the options given by 'options'. + * Options which are not specified are given default values. + * + * 'options' may be NULL, in which case all are given default values. + * + * Returns nonzero if profiling was started sucessfully, or zero else. + */ +PERFTOOLS_DLL_DECL int ProfilerStartWithOptions( + const char *fname, const struct ProfilerOptions *options); + +/* Stop profiling. Can be started again with ProfilerStart(), but + * the currently accumulated profiling data will be cleared. + */ +PERFTOOLS_DLL_DECL void ProfilerStop(); + +/* Flush any currently buffered profiling state to the profile file. + * Has no effect if the profiler has not been started. + */ +PERFTOOLS_DLL_DECL void ProfilerFlush(); + + +/* DEPRECATED: these functions were used to enable/disable profiling + * in the current thread, but no longer do anything. + */ +PERFTOOLS_DLL_DECL void ProfilerEnable(); +PERFTOOLS_DLL_DECL void ProfilerDisable(); + +/* Returns nonzero if profile is currently enabled, zero if it's not. */ +PERFTOOLS_DLL_DECL int ProfilingIsEnabledForAllThreads(); + +/* Routine for registering new threads with the profiler. + */ +PERFTOOLS_DLL_DECL void ProfilerRegisterThread(); + +/* Stores state about profiler's current status into "*state". */ +struct ProfilerState { + int enabled; /* Is profiling currently enabled? */ + time_t start_time; /* If enabled, when was profiling started? */ + char profile_name[1024]; /* Name of profile file being written, or '\0' */ + int samples_gathered; /* Number of samples gathered so far (or 0) */ +}; +PERFTOOLS_DLL_DECL void ProfilerGetCurrentState(struct ProfilerState* state); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif /* BASE_PROFILER_H_ */ diff --git a/third_party/tcmalloc/chromium/src/gperftools/stacktrace.h b/third_party/tcmalloc/chromium/src/gperftools/stacktrace.h new file mode 100644 index 0000000..fd186d6 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/stacktrace.h @@ -0,0 +1,116 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Sanjay Ghemawat +// +// Routines to extract the current stack trace. These functions are +// thread-safe. + +#ifndef GOOGLE_STACKTRACE_H_ +#define GOOGLE_STACKTRACE_H_ + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + + +// Skips the most recent "skip_count" stack frames (also skips the +// frame generated for the "GetStackFrames" routine itself), and then +// records the pc values for up to the next "max_depth" frames in +// "result", and the corresponding stack frame sizes in "sizes". +// Returns the number of values recorded in "result"/"sizes". +// +// Example: +// main() { foo(); } +// foo() { bar(); } +// bar() { +// void* result[10]; +// int sizes[10]; +// int depth = GetStackFrames(result, sizes, 10, 1); +// } +// +// The GetStackFrames call will skip the frame for "bar". It will +// return 2 and will produce pc values that map to the following +// procedures: +// result[0] foo +// result[1] main +// (Actually, there may be a few more entries after "main" to account for +// startup procedures.) +// And corresponding stack frame sizes will also be recorded: +// sizes[0] 16 +// sizes[1] 16 +// (Stack frame sizes of 16 above are just for illustration purposes.) +// Stack frame sizes of 0 or less indicate that those frame sizes couldn't +// be identified. +// +// This routine may return fewer stack frame entries than are +// available. Also note that "result" and "sizes" must both be non-NULL. +extern PERFTOOLS_DLL_DECL int GetStackFrames(void** result, int* sizes, int max_depth, + int skip_count); + +// Same as above, but to be used from a signal handler. The "uc" parameter +// should be the pointer to ucontext_t which was passed as the 3rd parameter +// to sa_sigaction signal handler. It may help the unwinder to get a +// better stack trace under certain conditions. The "uc" may safely be NULL. +extern PERFTOOLS_DLL_DECL int GetStackFramesWithContext(void** result, int* sizes, int max_depth, + int skip_count, const void *uc); + +// This is similar to the GetStackFrames routine, except that it returns +// the stack trace only, and not the stack frame sizes as well. +// Example: +// main() { foo(); } +// foo() { bar(); } +// bar() { +// void* result[10]; +// int depth = GetStackTrace(result, 10, 1); +// } +// +// This produces: +// result[0] foo +// result[1] main +// .... ... +// +// "result" must not be NULL. +extern PERFTOOLS_DLL_DECL int GetStackTrace(void** result, int max_depth, + int skip_count); + +// Same as above, but to be used from a signal handler. The "uc" parameter +// should be the pointer to ucontext_t which was passed as the 3rd parameter +// to sa_sigaction signal handler. It may help the unwinder to get a +// better stack trace under certain conditions. The "uc" may safely be NULL. +extern PERFTOOLS_DLL_DECL int GetStackTraceWithContext(void** result, int max_depth, + int skip_count, const void *uc); + +#endif /* GOOGLE_STACKTRACE_H_ */ diff --git a/third_party/tcmalloc/chromium/src/gperftools/tcmalloc.h b/third_party/tcmalloc/chromium/src/gperftools/tcmalloc.h new file mode 100644 index 0000000..7c88917 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/gperftools/tcmalloc.h @@ -0,0 +1,123 @@ +/* Copyright (c) 2003, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Sanjay Ghemawat <opensource@google.com> + * .h file by Craig Silverstein <opensource@google.com> + */ + +#ifndef TCMALLOC_TCMALLOC_H_ +#define TCMALLOC_TCMALLOC_H_ + +#include <stddef.h> // for size_t +#ifdef HAVE_SYS_CDEFS_H +#include <sys/cdefs.h> // where glibc defines __THROW +#endif + +// __THROW is defined in glibc systems. It means, counter-intuitively, +// "This function will never throw an exception." It's an optional +// optimization tool, but we may need to use it to match glibc prototypes. +#ifndef __THROW /* I guess we're not on a glibc system */ +# define __THROW /* __THROW is just an optimization, so ok to make it "" */ +#endif + +// Define the version number so folks can check against it +#define TC_VERSION_MAJOR 2 +#define TC_VERSION_MINOR 0 +#define TC_VERSION_PATCH "" +#define TC_VERSION_STRING "gperftools 2.0" + +#include <stdlib.h> // for struct mallinfo, if it's defined + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + +#ifdef __cplusplus +namespace std { +struct nothrow_t; +} + +extern "C" { +#endif + // Returns a human-readable version string. If major, minor, + // and/or patch are not NULL, they are set to the major version, + // minor version, and patch-code (a string, usually ""). + PERFTOOLS_DLL_DECL const char* tc_version(int* major, int* minor, + const char** patch) __THROW; + + PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW; + PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW; + PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW; + PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW; + PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW; + + PERFTOOLS_DLL_DECL void* tc_memalign(size_t __alignment, + size_t __size) __THROW; + PERFTOOLS_DLL_DECL int tc_posix_memalign(void** ptr, + size_t align, size_t size) __THROW; + PERFTOOLS_DLL_DECL void* tc_valloc(size_t __size) __THROW; + PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t __size) __THROW; + + PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW; + PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW; +#ifdef HAVE_STRUCT_MALLINFO + PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW; +#endif + + // This is an alias for MallocExtension::instance()->GetAllocatedSize(). + // It is equivalent to + // OS X: malloc_size() + // glibc: malloc_usable_size() + // Windows: _msize() + PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW; + +#ifdef __cplusplus + PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) __THROW; + PERFTOOLS_DLL_DECL void* tc_new(size_t size); + PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, + const std::nothrow_t&) __THROW; + PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW; + PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, + const std::nothrow_t&) __THROW; + PERFTOOLS_DLL_DECL void* tc_newarray(size_t size); + PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, + const std::nothrow_t&) __THROW; + PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW; + PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, + const std::nothrow_t&) __THROW; +} +#endif + +#endif // #ifndef TCMALLOC_TCMALLOC_H_ diff --git a/third_party/tcmalloc/chromium/src/google/tcmalloc.h.in b/third_party/tcmalloc/chromium/src/gperftools/tcmalloc.h.in index c887559..dbca6ec 100644 --- a/third_party/tcmalloc/chromium/src/google/tcmalloc.h.in +++ b/third_party/tcmalloc/chromium/src/gperftools/tcmalloc.h.in @@ -51,7 +51,7 @@ #define TC_VERSION_MAJOR @TC_VERSION_MAJOR@ #define TC_VERSION_MINOR @TC_VERSION_MINOR@ #define TC_VERSION_PATCH "@TC_VERSION_PATCH@" -#define TC_VERSION_STRING "google-perftools @TC_VERSION_MAJOR@.@TC_VERSION_MINOR@@TC_VERSION_PATCH@" +#define TC_VERSION_STRING "gperftools @TC_VERSION_MAJOR@.@TC_VERSION_MINOR@@TC_VERSION_PATCH@" #include <stdlib.h> // for struct mallinfo, if it's defined diff --git a/third_party/tcmalloc/chromium/src/heap-checker-bcad.cc b/third_party/tcmalloc/chromium/src/heap-checker-bcad.cc index 82a3109..7ed6942 100644 --- a/third_party/tcmalloc/chromium/src/heap-checker-bcad.cc +++ b/third_party/tcmalloc/chromium/src/heap-checker-bcad.cc @@ -45,7 +45,7 @@ // the allocated object is reachable from global data and hence "live"). #include <stdlib.h> // for abort() -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> // A dummy variable to refer from heap-checker.cc. This is to make // sure this file is not optimized out by the linker. diff --git a/third_party/tcmalloc/chromium/src/heap-checker.cc b/third_party/tcmalloc/chromium/src/heap-checker.cc index 3f5765e..c53646b 100644 --- a/third_party/tcmalloc/chromium/src/heap-checker.cc +++ b/third_party/tcmalloc/chromium/src/heap-checker.cc @@ -52,7 +52,7 @@ #include <time.h> #include <assert.h> -#ifdef HAVE_LINUX_PTRACE_H +#if defined(HAVE_LINUX_PTRACE_H) #include <linux/ptrace.h> #endif #ifdef HAVE_SYS_SYSCALL_H @@ -73,20 +73,20 @@ #include <algorithm> #include <functional> -#include <google/heap-checker.h> +#include <gperftools/heap-checker.h> #include "base/basictypes.h" #include "base/googleinit.h" #include "base/logging.h" -#include <google/stacktrace.h> +#include <gperftools/stacktrace.h> #include "base/commandlineflags.h" #include "base/elfcore.h" // for i386_regs #include "base/thread_lister.h" #include "heap-profile-table.h" #include "base/low_level_alloc.h" #include "malloc_hook-inl.h" -#include <google/malloc_hook.h> -#include <google/malloc_extension.h> +#include <gperftools/malloc_hook.h> +#include <gperftools/malloc_extension.h> #include "maybe_threads.h" #include "memory_region_map.h" #include "base/spinlock.h" @@ -144,7 +144,7 @@ DEFINE_string(heap_check, "\"minimal\", \"normal\", \"strict\", " "\"draconian\", \"as-is\", and \"local\" " " or the empty string are the supported choices. " - "(See HeapLeakChecker::InternalInitStart for details.)"); + "(See HeapLeakChecker_InternalInitStart for details.)"); DEFINE_bool(heap_check_report, true, "Obsolete"); @@ -282,10 +282,6 @@ static bool constructor_heap_profiling = false; static const int heap_checker_info_level = 0; //---------------------------------------------------------------------- -// Cancel our InitialMallocHook_* if present. -static void CancelInitialMallocHooks(); // defined below - -//---------------------------------------------------------------------- // HeapLeakChecker's own memory allocator that is // independent of the normal program allocator. //---------------------------------------------------------------------- @@ -553,6 +549,23 @@ inline static uintptr_t AsInt(const void* ptr) { //---------------------------------------------------------------------- +// We've seen reports that strstr causes heap-checker crashes in some +// libc's (?): +// http://code.google.com/p/gperftools/issues/detail?id=263 +// It's simple enough to use our own. This is not in time-critical code. +static const char* hc_strstr(const char* s1, const char* s2) { + const size_t len = strlen(s2); + RAW_CHECK(len > 0, "Unexpected empty string passed to strstr()"); + for (const char* p = strchr(s1, *s2); p != NULL; p = strchr(p+1, *s2)) { + if (strncmp(p, s2, len) == 0) { + return p; + } + } + return NULL; +} + +//---------------------------------------------------------------------- + // Our hooks for MallocHook static void NewHook(const void* ptr, size_t size) { if (ptr != NULL) { @@ -560,6 +573,11 @@ static void NewHook(const void* ptr, size_t size) { const bool ignore = (counter > 0); RAW_VLOG(16, "Recording Alloc: %p of %"PRIuS "; %d", ptr, size, int(counter)); + + // Fetch the caller's stack trace before acquiring heap_checker_lock. + void* stack[HeapProfileTable::kMaxStackDepth]; + int depth = HeapProfileTable::GetCallerStackTrace(0, stack); + { SpinLockHolder l(&heap_checker_lock); if (size > max_heap_object_size) max_heap_object_size = size; uintptr_t addr = AsInt(ptr); @@ -567,7 +585,7 @@ static void NewHook(const void* ptr, size_t size) { addr += size; if (addr > max_heap_address) max_heap_address = addr; if (heap_checker_on) { - heap_profile->RecordAlloc(ptr, size, 0); + heap_profile->RecordAlloc(ptr, size, depth, stack); if (ignore) { heap_profile->MarkAsIgnored(ptr); } @@ -770,6 +788,8 @@ static void MakeDisabledLiveCallbackLocked( } } +static const char kUnnamedProcSelfMapEntry[] = "UNNAMED"; + // This function takes some fields from a /proc/self/maps line: // // start_address start address of a memory region. @@ -787,7 +807,9 @@ static void RecordGlobalDataLocked(uintptr_t start_address, RAW_DCHECK(heap_checker_lock.IsHeld(), ""); // Ignore non-writeable regions. if (strchr(permissions, 'w') == NULL) return; - if (filename == NULL || *filename == '\0') filename = "UNNAMED"; + if (filename == NULL || *filename == '\0') { + filename = kUnnamedProcSelfMapEntry; + } RAW_VLOG(11, "Looking into %s: 0x%" PRIxPTR "..0x%" PRIxPTR, filename, start_address, end_address); (*library_live_objects)[filename]. @@ -799,7 +821,7 @@ static void RecordGlobalDataLocked(uintptr_t start_address, // See if 'library' from /proc/self/maps has base name 'library_base' // i.e. contains it and has '.' or '-' after it. static bool IsLibraryNamed(const char* library, const char* library_base) { - const char* p = strstr(library, library_base); + const char* p = hc_strstr(library, library_base); size_t sz = strlen(library_base); return p != NULL && (p[sz] == '.' || p[sz] == '-'); } @@ -911,11 +933,11 @@ HeapLeakChecker::ProcMapsResult HeapLeakChecker::UseProcMapsLocked( if (inode != 0) { saw_nonzero_inode = true; } - if ((strstr(filename, "lib") && strstr(filename, ".so")) || - strstr(filename, ".dll") || + if ((hc_strstr(filename, "lib") && hc_strstr(filename, ".so")) || + hc_strstr(filename, ".dll") || // not all .dylib filenames start with lib. .dylib is big enough // that we are unlikely to get false matches just checking that. - strstr(filename, ".dylib") || strstr(filename, ".bundle")) { + hc_strstr(filename, ".dylib") || hc_strstr(filename, ".bundle")) { saw_shared_lib = true; if (inode != 0) { saw_shared_lib_with_nonzero_inode = true; @@ -1391,6 +1413,31 @@ static SpinLock alignment_checker_lock(SpinLock::LINKER_INITIALIZED); } } if (size < sizeof(void*)) continue; + +#ifdef NO_FRAME_POINTER + // Frame pointer omission requires us to use libunwind, which uses direct + // mmap and munmap system calls, and that needs special handling. + if (name2 == kUnnamedProcSelfMapEntry) { + static const uintptr_t page_mask = ~(getpagesize() - 1); + const uintptr_t addr = reinterpret_cast<uintptr_t>(object); + if ((addr & page_mask) == 0 && (size & page_mask) == 0) { + // This is an object we slurped from /proc/self/maps. + // It may or may not be readable at this point. + // + // In case all the above conditions made a mistake, and the object is + // not related to libunwind, we also verify that it's not readable + // before ignoring it. + if (msync(const_cast<char*>(object), size, MS_ASYNC) != 0) { + // Skip unreadable object, so we don't crash trying to sweep it. + RAW_VLOG(0, "Ignoring inaccessible object [%p, %p) " + "(msync error %d (%s))", + object, object + size, errno, strerror(errno)); + continue; + } + } + } +#endif + const char* const max_object = object + size - sizeof(void*); while (object <= max_object) { // potentially unaligned load: @@ -1463,7 +1510,7 @@ void HeapLeakChecker::DisableChecksIn(const char* pattern) { } // static -void HeapLeakChecker::IgnoreObject(const void* ptr) { +void HeapLeakChecker::DoIgnoreObject(const void* ptr) { SpinLockHolder l(&heap_checker_lock); if (!heap_checker_on) return; size_t object_size; @@ -1477,7 +1524,7 @@ void HeapLeakChecker::IgnoreObject(const void* ptr) { IgnoredObjectsMap; } if (!ignored_objects->insert(make_pair(AsInt(ptr), object_size)).second) { - RAW_LOG(FATAL, "Object at %p is already being ignored", ptr); + RAW_LOG(WARNING, "Object at %p is already being ignored", ptr); } } } @@ -1867,16 +1914,18 @@ void HeapCleaner::RunHeapCleanups() { heap_cleanups_ = NULL; } -// Program exit heap cleanup registered with atexit(). +// Program exit heap cleanup registered as a module object destructor. // Will not get executed when we crash on a signal. // -/*static*/ void HeapLeakChecker::RunHeapCleanups() { +void HeapLeakChecker_RunHeapCleanups() { + if (FLAGS_heap_check == "local") // don't check heap in this mode + return; { SpinLockHolder l(&heap_checker_lock); // can get here (via forks?) with other pids if (heap_checker_pid != getpid()) return; } HeapCleaner::RunHeapCleanups(); - if (!FLAGS_heap_check_after_destructors) DoMainHeapCheck(); + if (!FLAGS_heap_check_after_destructors) HeapLeakChecker::DoMainHeapCheck(); } static bool internal_init_start_has_run = false; @@ -1891,21 +1940,26 @@ static bool internal_init_start_has_run = false; // We can't hold heap_checker_lock throughout because it would deadlock // on a memory allocation since our new/delete hooks can be on. // -/*static*/ void HeapLeakChecker::InternalInitStart() { +void HeapLeakChecker_InternalInitStart() { { SpinLockHolder l(&heap_checker_lock); RAW_CHECK(!internal_init_start_has_run, "Heap-check constructor called twice. Perhaps you both linked" " in the heap checker, and also used LD_PRELOAD to load it?"); internal_init_start_has_run = true; +#ifdef ADDRESS_SANITIZER + // AddressSanitizer's custom malloc conflicts with HeapChecker. + FLAGS_heap_check = ""; +#endif + if (FLAGS_heap_check.empty()) { // turns out we do not need checking in the end; can stop profiling - TurnItselfOffLocked(); + HeapLeakChecker::TurnItselfOffLocked(); return; } else if (RunningOnValgrind()) { // There is no point in trying -- we'll just fail. RAW_LOG(WARNING, "Can't run under Valgrind; will turn itself off"); - TurnItselfOffLocked(); + HeapLeakChecker::TurnItselfOffLocked(); return; } } @@ -1914,7 +1968,7 @@ static bool internal_init_start_has_run = false; if (!FLAGS_heap_check_run_under_gdb && IsDebuggerAttached()) { RAW_LOG(WARNING, "Someone is ptrace()ing us; will turn itself off"); SpinLockHolder l(&heap_checker_lock); - TurnItselfOffLocked(); + HeapLeakChecker::TurnItselfOffLocked(); return; } @@ -1965,15 +2019,25 @@ static bool internal_init_start_has_run = false; RAW_LOG(FATAL, "Unsupported heap_check flag: %s", FLAGS_heap_check.c_str()); } + // FreeBSD doesn't seem to honor atexit execution order: + // http://code.google.com/p/gperftools/issues/detail?id=375 + // Since heap-checking before destructors depends on atexit running + // at the right time, on FreeBSD we always check after, even in the + // less strict modes. This just means FreeBSD is always a bit + // stricter in its checking than other OSes. +#ifdef __FreeBSD__ + FLAGS_heap_check_after_destructors = true; +#endif + { SpinLockHolder l(&heap_checker_lock); RAW_DCHECK(heap_checker_pid == getpid(), ""); heap_checker_on = true; RAW_DCHECK(heap_profile, ""); - ProcMapsResult pm_result = UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS); + HeapLeakChecker::ProcMapsResult pm_result = HeapLeakChecker::UseProcMapsLocked(HeapLeakChecker::DISABLE_LIBRARY_ALLOCS); // might neeed to do this more than once // if one later dynamically loads libraries that we want disabled - if (pm_result != PROC_MAPS_USED) { // can't function - TurnItselfOffLocked(); + if (pm_result != HeapLeakChecker::PROC_MAPS_USED) { // can't function + HeapLeakChecker::TurnItselfOffLocked(); return; } } @@ -2027,8 +2091,6 @@ static bool internal_init_start_has_run = false; "-- Performance may suffer"); if (FLAGS_heap_check != "local") { - // Schedule registered heap cleanup - atexit(RunHeapCleanups); HeapLeakChecker* main_hc = new HeapLeakChecker(); SpinLockHolder l(&heap_checker_lock); RAW_DCHECK(main_heap_checker == NULL, @@ -2061,7 +2123,20 @@ static bool internal_init_start_has_run = false; // We want this to run early as well, but not so early as // ::BeforeConstructors (we want flag assignments to have already // happened, for instance). Initializer-registration does the trick. -REGISTER_MODULE_INITIALIZER(init_start, HeapLeakChecker::InternalInitStart()); +REGISTER_MODULE_INITIALIZER(init_start, HeapLeakChecker_InternalInitStart()); +REGISTER_MODULE_DESTRUCTOR(init_start, HeapLeakChecker_RunHeapCleanups()); + +// static +bool HeapLeakChecker::NoGlobalLeaksMaybeSymbolize( + ShouldSymbolize should_symbolize) { + // we never delete or change main_heap_checker once it's set: + HeapLeakChecker* main_hc = GlobalChecker(); + if (main_hc) { + RAW_VLOG(10, "Checking for whole-program memory leaks"); + return main_hc->DoNoLeaks(should_symbolize); + } + return true; +} // static bool HeapLeakChecker::DoMainHeapCheck() { @@ -2074,7 +2149,13 @@ bool HeapLeakChecker::DoMainHeapCheck() { do_main_heap_check = false; // will do it now; no need to do it more } - if (!NoGlobalLeaks()) { + // The program is over, so it's safe to symbolize addresses (which + // requires a fork) because no serious work is expected to be done + // after this. Symbolizing is really useful -- knowing what + // function has a leak is better than knowing just an address -- + // and while we can only safely symbolize once in a program run, + // now is the time (after all, there's no "later" that would be better). + if (!NoGlobalLeaksMaybeSymbolize(SYMBOLIZE)) { if (FLAGS_heap_check_identify_leaks) { RAW_LOG(FATAL, "Whole-program memory leaks found."); } @@ -2094,19 +2175,8 @@ HeapLeakChecker* HeapLeakChecker::GlobalChecker() { // static bool HeapLeakChecker::NoGlobalLeaks() { - // we never delete or change main_heap_checker once it's set: - HeapLeakChecker* main_hc = GlobalChecker(); - if (main_hc) { - RAW_VLOG(10, "Checking for whole-program memory leaks"); - // The program is over, so it's safe to symbolize addresses (which - // requires a fork) because no serious work is expected to be done - // after this. Symbolizing is really useful -- knowing what - // function has a leak is better than knowing just an address -- - // and while we can only safely symbolize once in a program run, - // now is the time (after all, there's no "later" that would be better). - return main_hc->DoNoLeaks(SYMBOLIZE); - } - return true; + // symbolizing requires a fork, which isn't safe to do in general. + return NoGlobalLeaksMaybeSymbolize(DO_NOT_SYMBOLIZE); } // static @@ -2124,6 +2194,10 @@ void HeapLeakChecker::BeforeConstructorsLocked() { RAW_DCHECK(heap_checker_lock.IsHeld(), ""); RAW_CHECK(!constructor_heap_profiling, "BeforeConstructorsLocked called multiple times"); +#ifdef ADDRESS_SANITIZER + // AddressSanitizer's custom malloc conflicts with HeapChecker. + return; +#endif // Set hooks early to crash if 'new' gets called before we make heap_profile, // and make sure no other hooks existed: RAW_CHECK(MallocHook::AddNewHook(&NewHook), ""); diff --git a/third_party/tcmalloc/chromium/src/heap-profile-table.cc b/third_party/tcmalloc/chromium/src/heap-profile-table.cc index a1da388..1ef1858 100644 --- a/third_party/tcmalloc/chromium/src/heap-profile-table.cc +++ b/third_party/tcmalloc/chromium/src/heap-profile-table.cc @@ -61,8 +61,9 @@ #include "base/logging.h" #include "raw_printer.h" #include "symbolize.h" -#include <google/stacktrace.h> -#include <google/malloc_hook.h> +#include <gperftools/stacktrace.h> +#include <gperftools/malloc_hook.h> +#include "memory_region_map.h" #include "base/commandlineflags.h" #include "base/logging.h" // for the RawFD I/O commands #include "base/sysinfo.h" @@ -98,7 +99,8 @@ const char HeapProfileTable::kFileExt[] = ".heap"; //---------------------------------------------------------------------- -static const int kHashTableSize = 179999; // Size for table_. +// Size for alloc_table_ and mmap_table_. +static const int kHashTableSize = 179999; /*static*/ const int HeapProfileTable::kMaxStackDepth; //---------------------------------------------------------------------- @@ -122,38 +124,60 @@ static bool ByAllocatedSpace(HeapProfileTable::Stats* a, HeapProfileTable::HeapProfileTable(Allocator alloc, DeAllocator dealloc) : alloc_(alloc), dealloc_(dealloc) { - // Make the table - const int table_bytes = kHashTableSize * sizeof(*table_); - table_ = reinterpret_cast<Bucket**>(alloc_(table_bytes)); - memset(table_, 0, table_bytes); - // Make allocation map - allocation_ = - new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_); - // init the rest: + // Initialize the overall profile stats. memset(&total_, 0, sizeof(total_)); - num_buckets_ = 0; + + // Make the malloc table. + const int alloc_table_bytes = kHashTableSize * sizeof(*alloc_table_); + alloc_table_ = reinterpret_cast<Bucket**>(alloc_(alloc_table_bytes)); + memset(alloc_table_, 0, alloc_table_bytes); + num_alloc_buckets_ = 0; + + // Initialize the mmap table. + mmap_table_ = NULL; + num_available_mmap_buckets_ = 0; + + // Make malloc and mmap allocation maps. + alloc_address_map_ = + new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_); + mmap_address_map_ = NULL; } HeapProfileTable::~HeapProfileTable() { - // free allocation map - allocation_->~AllocationMap(); - dealloc_(allocation_); - allocation_ = NULL; - // free hash table - for (int b = 0; b < kHashTableSize; b++) { - for (Bucket* x = table_[b]; x != 0; /**/) { - Bucket* b = x; - x = x->next; - dealloc_(b->stack); - dealloc_(b); + DeallocateBucketTable(alloc_table_); + alloc_table_ = NULL; + DeallocateBucketTable(mmap_table_); + mmap_table_ = NULL; + DeallocateAllocationMap(alloc_address_map_); + alloc_address_map_ = NULL; + DeallocateAllocationMap(mmap_address_map_); + mmap_address_map_ = NULL; +} + +void HeapProfileTable::DeallocateAllocationMap(AllocationMap* allocation) { + if (allocation != NULL) { + alloc_address_map_->~AllocationMap(); + dealloc_(allocation); + } +} + +void HeapProfileTable::DeallocateBucketTable(Bucket** table) { + if (table != NULL) { + for (int b = 0; b < kHashTableSize; b++) { + for (Bucket* x = table[b]; x != 0; /**/) { + Bucket* b = x; + x = x->next; + dealloc_(b->stack); + dealloc_(b); + } } + dealloc_(table); } - dealloc_(table_); - table_ = NULL; } -HeapProfileTable::Bucket* HeapProfileTable::GetBucket(int depth, - const void* const key[]) { +HeapProfileTable::Bucket* HeapProfileTable::GetBucket( + int depth, const void* const key[], Bucket** table, + int* bucket_count) { // Make hash-value uintptr_t h = 0; for (int i = 0; i < depth; i++) { @@ -166,7 +190,7 @@ HeapProfileTable::Bucket* HeapProfileTable::GetBucket(int depth, // Lookup stack trace in table unsigned int buck = ((unsigned int) h) % kHashTableSize; - for (Bucket* b = table_[buck]; b != 0; b = b->next) { + for (Bucket* b = table[buck]; b != 0; b = b->next) { if ((b->hash == h) && (b->depth == depth) && equal(key, key + depth, b->stack)) { @@ -183,24 +207,25 @@ HeapProfileTable::Bucket* HeapProfileTable::GetBucket(int depth, b->hash = h; b->depth = depth; b->stack = kcopy; - b->next = table_[buck]; - table_[buck] = b; - num_buckets_++; + b->next = table[buck]; + table[buck] = b; + if (bucket_count != NULL) { + ++(*bucket_count); + } return b; } -void HeapProfileTable::RecordAlloc(const void* ptr, size_t bytes, - int skip_count) { - void* key[kMaxStackDepth]; - int depth = MallocHook::GetCallerStackTrace( - key, kMaxStackDepth, kStripFrames + skip_count + 1); - RecordAllocWithStack(ptr, bytes, depth, key); +int HeapProfileTable::GetCallerStackTrace( + int skip_count, void* stack[kMaxStackDepth]) { + return MallocHook::GetCallerStackTrace( + stack, kMaxStackDepth, kStripFrames + skip_count + 1); } -void HeapProfileTable::RecordAllocWithStack( +void HeapProfileTable::RecordAlloc( const void* ptr, size_t bytes, int stack_depth, const void* const call_stack[]) { - Bucket* b = GetBucket(stack_depth, call_stack); + Bucket* b = GetBucket(stack_depth, call_stack, alloc_table_, + &num_alloc_buckets_); b->allocs++; b->alloc_size += bytes; total_.allocs++; @@ -209,12 +234,12 @@ void HeapProfileTable::RecordAllocWithStack( AllocValue v; v.set_bucket(b); // also did set_live(false); set_ignore(false) v.bytes = bytes; - allocation_->Insert(ptr, v); + alloc_address_map_->Insert(ptr, v); } void HeapProfileTable::RecordFree(const void* ptr) { AllocValue v; - if (allocation_->FindAndRemove(ptr, &v)) { + if (alloc_address_map_->FindAndRemove(ptr, &v)) { Bucket* b = v.bucket(); b->frees++; b->free_size += v.bytes; @@ -224,14 +249,14 @@ void HeapProfileTable::RecordFree(const void* ptr) { } bool HeapProfileTable::FindAlloc(const void* ptr, size_t* object_size) const { - const AllocValue* alloc_value = allocation_->Find(ptr); + const AllocValue* alloc_value = alloc_address_map_->Find(ptr); if (alloc_value != NULL) *object_size = alloc_value->bytes; return alloc_value != NULL; } bool HeapProfileTable::FindAllocDetails(const void* ptr, AllocInfo* info) const { - const AllocValue* alloc_value = allocation_->Find(ptr); + const AllocValue* alloc_value = alloc_address_map_->Find(ptr); if (alloc_value != NULL) { info->object_size = alloc_value->bytes; info->call_stack = alloc_value->bucket()->stack; @@ -245,13 +270,13 @@ bool HeapProfileTable::FindInsideAlloc(const void* ptr, const void** object_ptr, size_t* object_size) const { const AllocValue* alloc_value = - allocation_->FindInside(&AllocValueSize, max_size, ptr, object_ptr); + alloc_address_map_->FindInside(&AllocValueSize, max_size, ptr, object_ptr); if (alloc_value != NULL) *object_size = alloc_value->bytes; return alloc_value != NULL; } bool HeapProfileTable::MarkAsLive(const void* ptr) { - AllocValue* alloc = allocation_->FindMutable(ptr); + AllocValue* alloc = alloc_address_map_->FindMutable(ptr); if (alloc && !alloc->live()) { alloc->set_live(true); return true; @@ -260,7 +285,7 @@ bool HeapProfileTable::MarkAsLive(const void* ptr) { } void HeapProfileTable::MarkAsIgnored(const void* ptr) { - AllocValue* alloc = allocation_->FindMutable(ptr); + AllocValue* alloc = alloc_address_map_->FindMutable(ptr); if (alloc) { alloc->set_ignore(true); } @@ -301,27 +326,81 @@ int HeapProfileTable::UnparseBucket(const Bucket& b, HeapProfileTable::Bucket** HeapProfileTable::MakeSortedBucketList() const { - Bucket** list = - reinterpret_cast<Bucket**>(alloc_(sizeof(Bucket) * num_buckets_)); + Bucket** list = reinterpret_cast<Bucket**>(alloc_(sizeof(Bucket) * + (num_alloc_buckets_ + num_available_mmap_buckets_))); + + RAW_DCHECK(mmap_table_ != NULL || num_available_mmap_buckets_ == 0, ""); int n = 0; + for (int b = 0; b < kHashTableSize; b++) { - for (Bucket* x = table_[b]; x != 0; x = x->next) { + for (Bucket* x = alloc_table_[b]; x != 0; x = x->next) { list[n++] = x; } } - RAW_DCHECK(n == num_buckets_, ""); + RAW_DCHECK(n == num_alloc_buckets_, ""); + + if (mmap_table_ != NULL) { + for (int b = 0; b < kHashTableSize; b++) { + for (Bucket* x = mmap_table_[b]; x != 0; x = x->next) { + list[n++] = x; + } + } + } + RAW_DCHECK(n == num_alloc_buckets_ + num_available_mmap_buckets_, ""); - sort(list, list + num_buckets_, ByAllocatedSpace); + sort(list, list + num_alloc_buckets_ + num_available_mmap_buckets_, + ByAllocatedSpace); return list; } +void HeapProfileTable::RefreshMMapData() { + // Make the table + static const int mmap_table_bytes = kHashTableSize * sizeof(*mmap_table_); + if (mmap_table_ == NULL) { + mmap_table_ = reinterpret_cast<Bucket**>(alloc_(mmap_table_bytes)); + memset(mmap_table_, 0, mmap_table_bytes); + } + num_available_mmap_buckets_ = 0; + + ClearMMapData(); + mmap_address_map_ = + new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_); + + MemoryRegionMap::LockHolder l; + for (MemoryRegionMap::RegionIterator r = + MemoryRegionMap::BeginRegionLocked(); + r != MemoryRegionMap::EndRegionLocked(); ++r) { + Bucket* b = + GetBucket(r->call_stack_depth, r->call_stack, mmap_table_, NULL); + if (b->alloc_size == 0) { + num_available_mmap_buckets_ += 1; + } + b->allocs += 1; + b->alloc_size += r->end_addr - r->start_addr; + + AllocValue v; + v.set_bucket(b); + v.bytes = r->end_addr - r->start_addr; + mmap_address_map_->Insert(reinterpret_cast<const void*>(r->start_addr), v); + } +} + +void HeapProfileTable::ClearMMapData() { + if (mmap_address_map_ != NULL) { + mmap_address_map_->Iterate(ZeroBucketCountsIterator, this); + mmap_address_map_->~AllocationMap(); + dealloc_(mmap_address_map_); + mmap_address_map_ = NULL; + } +} + void HeapProfileTable::IterateOrderedAllocContexts( AllocContextIterator callback) const { Bucket** list = MakeSortedBucketList(); AllocContextInfo info; - for (int i = 0; i < num_buckets_; ++i) { + for (int i = 0; i < num_alloc_buckets_; ++i) { *static_cast<Stats*>(&info) = *static_cast<Stats*>(list[i]); info.stack_depth = list[i]->depth; info.call_stack = list[i]->stack; @@ -353,9 +432,14 @@ int HeapProfileTable::FillOrderedProfile(char buf[], int size) const { memset(&stats, 0, sizeof(stats)); int bucket_length = snprintf(buf, size, "%s", kProfileHeader); if (bucket_length < 0 || bucket_length >= size) return 0; - bucket_length = UnparseBucket(total_, buf, bucket_length, size, + Bucket total_with_mmap(total_); + if (mmap_table_ != NULL) { + total_with_mmap.alloc_size += MemoryRegionMap::MapSize(); + total_with_mmap.free_size += MemoryRegionMap::UnmapSize(); + } + bucket_length = UnparseBucket(total_with_mmap, buf, bucket_length, size, " heapprofile", &stats); - for (int i = 0; i < num_buckets_; i++) { + for (int i = 0; i < num_alloc_buckets_; i++) { bucket_length = UnparseBucket(*list[i], buf, bucket_length, size, "", &stats); } @@ -390,6 +474,17 @@ void HeapProfileTable::DumpNonLiveIterator(const void* ptr, AllocValue* v, RawWrite(args.fd, buf, len); } +inline void HeapProfileTable::ZeroBucketCountsIterator( + const void* ptr, AllocValue* v, HeapProfileTable* heap_profile) { + Bucket* b = v->bucket(); + if (b != NULL) { + b->allocs = 0; + b->alloc_size = 0; + b->free_size = 0; + b->frees = 0; + } +} + // Callback from NonLiveSnapshot; adds entry to arg->dest // if not the entry is not live and is not present in arg->base. void HeapProfileTable::AddIfNonLive(const void* ptr, AllocValue* v, @@ -457,7 +552,7 @@ void HeapProfileTable::CleanupOldProfiles(const char* prefix) { HeapProfileTable::Snapshot* HeapProfileTable::TakeSnapshot() { Snapshot* s = new (alloc_(sizeof(Snapshot))) Snapshot(alloc_, dealloc_); - allocation_->Iterate(AddToSnapshot, s); + alloc_address_map_->Iterate(AddToSnapshot, s); return s; } @@ -482,7 +577,7 @@ HeapProfileTable::Snapshot* HeapProfileTable::NonLiveSnapshot( AddNonLiveArgs args; args.dest = s; args.base = base; - allocation_->Iterate<AddNonLiveArgs*>(AddIfNonLive, &args); + alloc_address_map_->Iterate<AddNonLiveArgs*>(AddIfNonLive, &args); RAW_VLOG(2, "NonLiveSnapshot output: %d %d\n", int(s->total_.allocs - s->total_.frees), int(s->total_.alloc_size - s->total_.free_size)); diff --git a/third_party/tcmalloc/chromium/src/heap-profile-table.h b/third_party/tcmalloc/chromium/src/heap-profile-table.h index c9bee15..abd3184 100644 --- a/third_party/tcmalloc/chromium/src/heap-profile-table.h +++ b/third_party/tcmalloc/chromium/src/heap-profile-table.h @@ -97,15 +97,20 @@ class HeapProfileTable { HeapProfileTable(Allocator alloc, DeAllocator dealloc); ~HeapProfileTable(); - // Record an allocation at 'ptr' of 'bytes' bytes. - // skip_count gives the number of stack frames between this call - // and the memory allocation function that was asked to do the allocation. - void RecordAlloc(const void* ptr, size_t bytes, int skip_count); + // Collect the stack trace for the function that asked to do the + // allocation for passing to RecordAlloc() below. + // + // The stack trace is stored in 'stack'. The stack depth is returned. + // + // 'skip_count' gives the number of stack frames between this call + // and the memory allocation function. + static int GetCallerStackTrace(int skip_count, void* stack[kMaxStackDepth]); - // Direct version of RecordAlloc when the caller stack to use - // is already known: call_stack of depth stack_depth. - void RecordAllocWithStack(const void* ptr, size_t bytes, - int stack_depth, const void* const call_stack[]); + // Record an allocation at 'ptr' of 'bytes' bytes. 'stack_depth' + // and 'call_stack' identifying the function that requested the + // allocation. They can be generated using GetCallerStackTrace() above. + void RecordAlloc(const void* ptr, size_t bytes, + int stack_depth, const void* const call_stack[]); // Record the deallocation of memory at 'ptr'. void RecordFree(const void* ptr); @@ -133,7 +138,8 @@ class HeapProfileTable { // are skipped in heap checking reports. void MarkAsIgnored(const void* ptr); - // Return current total (de)allocation statistics. + // Return current total (de)allocation statistics. It doesn't contain + // mmap'ed regions. const Stats& total() const { return total_; } // Allocation data iteration callback: gets passed object pointer and @@ -143,7 +149,7 @@ class HeapProfileTable { // Iterate over the allocation profile data calling "callback" // for every allocation. void IterateAllocs(AllocIterator callback) const { - allocation_->Iterate(MapArgsAllocIterator, callback); + alloc_address_map_->Iterate(MapArgsAllocIterator, callback); } // Allocation context profile data iteration callback @@ -181,6 +187,16 @@ class HeapProfileTable { // Caller must call ReleaseSnapshot() on result when no longer needed. Snapshot* NonLiveSnapshot(Snapshot* base); + // Refresh the internal mmap information from MemoryRegionMap. Results of + // FillOrderedProfile and IterateOrderedAllocContexts will contain mmap'ed + // memory regions as at calling RefreshMMapData. + void RefreshMMapData(); + + // Clear the internal mmap information. Results of FillOrderedProfile and + // IterateOrderedAllocContexts won't contain mmap'ed memory regions after + // calling ClearMMapData. + void ClearMMapData(); + private: // data types ---------------------------- @@ -258,9 +274,18 @@ class HeapProfileTable { const char* extra, Stats* profile_stats); - // Get the bucket for the caller stack trace 'key' of depth 'depth' - // creating the bucket if needed. - Bucket* GetBucket(int depth, const void* const key[]); + // Deallocate a given allocation map. + void DeallocateAllocationMap(AllocationMap* allocation); + + // Deallocate a given bucket table. + void DeallocateBucketTable(Bucket** table); + + // Get the bucket for the caller stack trace 'key' of depth 'depth' from a + // bucket hash map 'table' creating the bucket if needed. '*bucket_count' + // is incremented both when 'bucket_count' is not NULL and when a new + // bucket object is created. + Bucket* GetBucket(int depth, const void* const key[], Bucket** table, + int* bucket_count); // Helper for IterateAllocs to do callback signature conversion // from AllocationMap::Iterate to AllocIterator. @@ -280,9 +305,14 @@ class HeapProfileTable { inline static void DumpNonLiveIterator(const void* ptr, AllocValue* v, const DumpArgs& args); + // Helper for filling size variables in buckets by zero. + inline static void ZeroBucketCountsIterator( + const void* ptr, AllocValue* v, HeapProfileTable* heap_profile); + // Helper for IterateOrderedAllocContexts and FillOrderedProfile. - // Creates a sorted list of Buckets whose length is num_buckets_. - // The caller is responsible for dellocating the returned list. + // Creates a sorted list of Buckets whose length is num_alloc_buckets_ + + // num_avaliable_mmap_buckets_. + // The caller is responsible for deallocating the returned list. Bucket** MakeSortedBucketList() const; // Helper for TakeSnapshot. Saves object to snapshot. @@ -314,17 +344,25 @@ class HeapProfileTable { // Overall profile stats; we use only the Stats part, // but make it a Bucket to pass to UnparseBucket. + // It doesn't contain mmap'ed regions. Bucket total_; - // Bucket hash table. + // Bucket hash table for malloc. // We hand-craft one instead of using one of the pre-written // ones because we do not want to use malloc when operating on the table. // It is only few lines of code, so no big deal. - Bucket** table_; - int num_buckets_; - - // Map of all currently allocated objects we know about. - AllocationMap* allocation_; + Bucket** alloc_table_; + int num_alloc_buckets_; + + // Bucket hash table for mmap. + // This table is filled with the information from MemoryRegionMap by calling + // RefreshMMapData. + Bucket** mmap_table_; + int num_available_mmap_buckets_; + + // Map of all currently allocated objects and mapped regions we know about. + AllocationMap* alloc_address_map_; + AllocationMap* mmap_address_map_; DISALLOW_COPY_AND_ASSIGN(HeapProfileTable); }; diff --git a/third_party/tcmalloc/chromium/src/heap-profiler.cc b/third_party/tcmalloc/chromium/src/heap-profiler.cc index 1c88fd6..166afde 100644 --- a/third_party/tcmalloc/chromium/src/heap-profiler.cc +++ b/third_party/tcmalloc/chromium/src/heap-profiler.cc @@ -55,7 +55,7 @@ #include <algorithm> #include <string> -#include <google/heap-profiler.h> +#include <gperftools/heap-profiler.h> #include "base/logging.h" #include "base/basictypes.h" // for PRId64, among other things @@ -63,8 +63,8 @@ #include "base/commandlineflags.h" #include "malloc_hook-inl.h" #include "tcmalloc_guard.h" -#include <google/malloc_hook.h> -#include <google/malloc_extension.h> +#include <gperftools/malloc_hook.h> +#include <gperftools/malloc_extension.h> #include "base/spinlock.h" #include "base/low_level_alloc.h" #include "base/sysinfo.h" // for GetUniquePathFromEnv() @@ -184,29 +184,6 @@ static HeapProfileTable* heap_profile = NULL; // the heap profile table // Profile generation //---------------------------------------------------------------------- -enum AddOrRemove { ADD, REMOVE }; - -// Add or remove all MMap-allocated regions to/from *heap_profile. -// Assumes heap_lock is held. -static void AddRemoveMMapDataLocked(AddOrRemove mode) { - RAW_DCHECK(heap_lock.IsHeld(), ""); - if (!FLAGS_mmap_profile || !is_on) return; - // MemoryRegionMap maintained all the data we need for all - // mmap-like allocations, so we just use it here: - MemoryRegionMap::LockHolder l; - for (MemoryRegionMap::RegionIterator r = MemoryRegionMap::BeginRegionLocked(); - r != MemoryRegionMap::EndRegionLocked(); ++r) { - if (mode == ADD) { - heap_profile->RecordAllocWithStack( - reinterpret_cast<const void*>(r->start_addr), - r->end_addr - r->start_addr, - r->call_stack_depth, r->call_stack); - } else { - heap_profile->RecordFree(reinterpret_cast<void*>(r->start_addr)); - } - } -} - // Input must be a buffer of size at least 1MB. static char* DoGetHeapProfileLocked(char* buf, int buflen) { // We used to be smarter about estimating the required memory and @@ -217,16 +194,13 @@ static char* DoGetHeapProfileLocked(char* buf, int buflen) { RAW_DCHECK(heap_lock.IsHeld(), ""); int bytes_written = 0; if (is_on) { - HeapProfileTable::Stats const stats = heap_profile->total(); - (void)stats; // avoid an unused-variable warning in non-debug mode. - AddRemoveMMapDataLocked(ADD); + if (FLAGS_mmap_profile) { + heap_profile->RefreshMMapData(); + } bytes_written = heap_profile->FillOrderedProfile(buf, buflen - 1); - // FillOrderedProfile should not reduce the set of active mmap-ed regions, - // hence MemoryRegionMap will let us remove everything we've added above: - AddRemoveMMapDataLocked(REMOVE); - RAW_DCHECK(stats.Equivalent(heap_profile->total()), ""); - // if this fails, we somehow removed by AddRemoveMMapDataLocked - // more than we have added. + if (FLAGS_mmap_profile) { + heap_profile->ClearMMapData(); + } } buf[bytes_written] = '\0'; RAW_DCHECK(bytes_written == strlen(buf), ""); @@ -341,9 +315,12 @@ static void MaybeDumpProfileLocked() { // Record an allocation in the profile. static void RecordAlloc(const void* ptr, size_t bytes, int skip_count) { + // Take the stack trace outside the critical section. + void* stack[HeapProfileTable::kMaxStackDepth]; + int depth = HeapProfileTable::GetCallerStackTrace(skip_count + 1, stack); SpinLockHolder l(&heap_lock); if (is_on) { - heap_profile->RecordAlloc(ptr, bytes, skip_count + 1); + heap_profile->RecordAlloc(ptr, bytes, depth, stack); MaybeDumpProfileLocked(); } } diff --git a/third_party/tcmalloc/chromium/src/internal_logging.cc b/third_party/tcmalloc/chromium/src/internal_logging.cc index 4c90190..2189d84 100644 --- a/third_party/tcmalloc/chromium/src/internal_logging.cc +++ b/third_party/tcmalloc/chromium/src/internal_logging.cc @@ -40,65 +40,139 @@ #include <unistd.h> // for write() #endif -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> #include "base/logging.h" // for perftools_vsnprintf #include "base/spinlock.h" // for SpinLockHolder, SpinLock static const int kLogBufSize = 800; -void TCMalloc_MESSAGE(const char* filename, - int line_number, - const char* format, ...) { - char buf[kLogBufSize]; - const int n = snprintf(buf, sizeof(buf), "%s:%d] ", filename, line_number); - if (n < kLogBufSize) { - va_list ap; - va_start(ap, format); - perftools_vsnprintf(buf + n, kLogBufSize - n, format, ap); - va_end(ap); - } - write(STDERR_FILENO, buf, strlen(buf)); -} - +// Variables for storing crash output. Allocated statically since we +// may not be able to heap-allocate while crashing. +static SpinLock crash_lock(base::LINKER_INITIALIZED); +static bool crashed = false; static const int kStatsBufferSize = 16 << 10; static char stats_buffer[kStatsBufferSize] = { 0 }; -static void TCMalloc_CRASH_internal(bool dump_stats, - const char* filename, - int line_number, - const char* format, va_list ap) { - char buf[kLogBufSize]; - const int n = snprintf(buf, sizeof(buf), "%s:%d] ", filename, line_number); - if (n < kLogBufSize) { - perftools_vsnprintf(buf + n, kLogBufSize - n, format, ap); +namespace tcmalloc { + +static void WriteMessage(const char* msg, int length) { + write(STDERR_FILENO, msg, length); +} + +void (*log_message_writer)(const char* msg, int length) = WriteMessage; + + +class Logger { + public: + bool Add(const LogItem& item); + bool AddStr(const char* str, int n); + bool AddNum(uint64_t num, int base); // base must be 10 or 16. + + static const int kBufSize = 200; + char* p_; + char* end_; + char buf_[kBufSize]; +}; + +void Log(LogMode mode, const char* filename, int line, + LogItem a, LogItem b, LogItem c, LogItem d) { + Logger state; + state.p_ = state.buf_; + state.end_ = state.buf_ + sizeof(state.buf_); + state.AddStr(filename, strlen(filename)) + && state.AddStr(":", 1) + && state.AddNum(line, 10) + && state.AddStr("]", 1) + && state.Add(a) + && state.Add(b) + && state.Add(c) + && state.Add(d); + + // Teminate with newline + if (state.p_ >= state.end_) { + state.p_ = state.end_ - 1; + } + *state.p_ = '\n'; + state.p_++; + + int msglen = state.p_ - state.buf_; + if (mode == kLog) { + (*log_message_writer)(state.buf_, msglen); + return; } - write(STDERR_FILENO, buf, strlen(buf)); - if (dump_stats) { + + bool first_crash = false; + { + SpinLockHolder l(&crash_lock); + if (!crashed) { + crashed = true; + first_crash = true; + } + } + + (*log_message_writer)(state.buf_, msglen); + if (first_crash && mode == kCrashWithStats) { MallocExtension::instance()->GetStats(stats_buffer, kStatsBufferSize); - write(STDERR_FILENO, stats_buffer, strlen(stats_buffer)); + (*log_message_writer)(stats_buffer, strlen(stats_buffer)); } abort(); } -void TCMalloc_CRASH(bool dump_stats, - const char* filename, - int line_number, - const char* format, ...) { - va_list ap; - va_start(ap, format); - TCMalloc_CRASH_internal(dump_stats, filename, line_number, format, ap); - va_end(ap); +bool Logger::Add(const LogItem& item) { + // Separate items with spaces + if (p_ < end_) { + *p_ = ' '; + p_++; + } + + switch (item.tag_) { + case LogItem::kStr: + return AddStr(item.u_.str, strlen(item.u_.str)); + case LogItem::kUnsigned: + return AddNum(item.u_.unum, 10); + case LogItem::kSigned: + if (item.u_.snum < 0) { + // The cast to uint64_t is intentionally before the negation + // so that we do not attempt to negate -2^63. + return AddStr("-", 1) + && AddNum(- static_cast<uint64_t>(item.u_.snum), 10); + } else { + return AddNum(static_cast<uint64_t>(item.u_.snum), 10); + } + case LogItem::kPtr: + return AddStr("0x", 2) + && AddNum(reinterpret_cast<uintptr_t>(item.u_.ptr), 16); + default: + return false; + } } +bool Logger::AddStr(const char* str, int n) { + if (end_ - p_ < n) { + return false; + } else { + memcpy(p_, str, n); + p_ += n; + return true; + } +} -void TCMalloc_CrashReporter::PrintfAndDie(const char* format, ...) { - va_list ap; - va_start(ap, format); - TCMalloc_CRASH_internal(dump_stats_, file_, line_, format, ap); - va_end(ap); +bool Logger::AddNum(uint64_t num, int base) { + static const char kDigits[] = "0123456789abcdef"; + char space[22]; // more than enough for 2^64 in smallest supported base (10) + char* end = space + sizeof(space); + char* pos = end; + do { + pos--; + *pos = kDigits[num % base]; + num /= base; + } while (num > 0 && pos > space); + return AddStr(pos, end - pos); } +} // end tcmalloc namespace + void TCMalloc_Printer::printf(const char* format, ...) { if (left_ > 0) { va_list ap; diff --git a/third_party/tcmalloc/chromium/src/internal_logging.h b/third_party/tcmalloc/chromium/src/internal_logging.h index ce4a516..0267034 100644 --- a/third_party/tcmalloc/chromium/src/internal_logging.h +++ b/third_party/tcmalloc/chromium/src/internal_logging.h @@ -37,77 +37,79 @@ #include <config.h> #include <stddef.h> // for size_t +#if defined HAVE_STDINT_H +#include <stdint.h> +#elif defined HAVE_INTTYPES_H +#include <inttypes.h> +#else +#include <sys/types.h> +#endif //------------------------------------------------------------------- // Utility routines //------------------------------------------------------------------- -// Safe debugging routine: we write directly to the stderr file +// Safe logging helper: we write directly to the stderr file // descriptor and avoid FILE buffering because that may invoke -// malloc() -extern void TCMalloc_MESSAGE(const char* filename, - int line_number, - const char* format, ...) -#ifdef HAVE___ATTRIBUTE__ - __attribute__ ((__format__ (__printf__, 3, 4))) -#endif -; +// malloc(). +// +// Example: +// Log(kLog, __FILE__, __LINE__, "error", bytes); + +namespace tcmalloc { +enum LogMode { + kLog, // Just print the message + kCrash, // Print the message and crash + kCrashWithStats // Print the message, some stats, and crash +}; -// Right now, the only non-fatal messages we want to report are when -// an allocation fails (we'll return NULL eventually, but sometimes -// want more prominent notice to help debug). message should be -// a literal string with no %<whatever> format directives. -#ifdef TCMALLOC_WARNINGS -#define MESSAGE(message, num_bytes) \ - TCMalloc_MESSAGE(__FILE__, __LINE__, message " (%"PRIuS" bytes)\n", \ - static_cast<size_t>(num_bytes)) -#else -#define MESSAGE(message, num_bytes) -#endif +class Logger; -// Dumps the specified message and then calls abort(). If -// "dump_stats" is specified, the first call will also dump the -// tcmalloc stats. -extern void TCMalloc_CRASH(bool dump_stats, - const char* filename, - int line_number, - const char* format, ...) -#ifdef HAVE___ATTRIBUTE__ - __attribute__ ((__format__ (__printf__, 4, 5))) -#endif -; - -// This is a class that makes using the macro easier. With this class, -// CRASH("%d", i) expands to TCMalloc_CrashReporter.PrintfAndDie("%d", i). -class PERFTOOLS_DLL_DECL TCMalloc_CrashReporter { +// A LogItem holds any of the argument types that can be passed to Log() +class LogItem { public: - TCMalloc_CrashReporter(bool dump_stats, const char* file, int line) - : dump_stats_(dump_stats), file_(file), line_(line) { - } - void PrintfAndDie(const char* format, ...) -#ifdef HAVE___ATTRIBUTE__ - __attribute__ ((__format__ (__printf__, 2, 3))) // 2,3 due to "this" -#endif -; - + LogItem() : tag_(kEnd) { } + LogItem(const char* v) : tag_(kStr) { u_.str = v; } + LogItem(int v) : tag_(kSigned) { u_.snum = v; } + LogItem(long v) : tag_(kSigned) { u_.snum = v; } + LogItem(long long v) : tag_(kSigned) { u_.snum = v; } + LogItem(unsigned int v) : tag_(kUnsigned) { u_.unum = v; } + LogItem(unsigned long v) : tag_(kUnsigned) { u_.unum = v; } + LogItem(unsigned long long v) : tag_(kUnsigned) { u_.unum = v; } + LogItem(const void* v) : tag_(kPtr) { u_.ptr = v; } private: - bool dump_stats_; - const char* file_; - int line_; + friend class Logger; + enum Tag { + kStr, + kSigned, + kUnsigned, + kPtr, + kEnd + }; + Tag tag_; + union { + const char* str; + const void* ptr; + int64_t snum; + uint64_t unum; + } u_; }; -#define CRASH \ - TCMalloc_CrashReporter(false, __FILE__, __LINE__).PrintfAndDie +extern PERFTOOLS_DLL_DECL void Log(LogMode mode, const char* filename, int line, + LogItem a, LogItem b = LogItem(), + LogItem c = LogItem(), LogItem d = LogItem()); + +// Tests can override this function to collect logging messages. +extern PERFTOOLS_DLL_DECL void (*log_message_writer)(const char* msg, int length); -#define CRASH_WITH_STATS \ - TCMalloc_CrashReporter(true, __FILE__, __LINE__).PrintfAndDie +} // end tcmalloc namespace // Like assert(), but executed even in NDEBUG mode #undef CHECK_CONDITION #define CHECK_CONDITION(cond) \ do { \ if (!(cond)) { \ - CRASH("assertion failed: %s\n", #cond); \ + ::tcmalloc::Log(::tcmalloc::kCrash, __FILE__, __LINE__, #cond); \ } \ } while (0) diff --git a/third_party/tcmalloc/chromium/src/libc_override.h b/third_party/tcmalloc/chromium/src/libc_override.h new file mode 100644 index 0000000..089084a --- /dev/null +++ b/third_party/tcmalloc/chromium/src/libc_override.h @@ -0,0 +1,94 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein <opensource@google.com> +// +// This .h file imports the code that causes tcmalloc to override libc +// versions of malloc/free/new/delete/etc. That is, it provides the +// logic that makes it so calls to malloc(10) go through tcmalloc, +// rather than the default (libc) malloc. +// +// This file also provides a method: ReplaceSystemAlloc(), that every +// libc_override_*.h file it #includes is required to provide. This +// is called when first setting up tcmalloc -- that is, when a global +// constructor in tcmalloc.cc is executed -- to do any initialization +// work that may be required for this OS. (Note we cannot entirely +// control when tcmalloc is initialized, and the system may do some +// mallocs and frees before this routine is called.) It may be a +// noop. +// +// Every libc has its own way of doing this, and sometimes the compiler +// matters too, so we have a different file for each libc, and often +// for different compilers and OS's. + +#ifndef TCMALLOC_LIBC_OVERRIDE_INL_H_ +#define TCMALLOC_LIBC_OVERRIDE_INL_H_ + +#include <config.h> +#ifdef HAVE_FEATURES_H +#include <features.h> // for __GLIBC__ +#endif +#include <gperftools/tcmalloc.h> + +static void ReplaceSystemAlloc(); // defined in the .h files below + +// For windows, there are two ways to get tcmalloc. If we're +// patching, then src/windows/patch_function.cc will do the necessary +// overriding here. Otherwise, we doing the 'redefine' trick, where +// we remove malloc/new/etc from mscvcrt.dll, and just need to define +// them now. +#if defined(_WIN32) && defined(WIN32_DO_PATCHING) +void PatchWindowsFunctions(); // in src/windows/patch_function.cc +static void ReplaceSystemAlloc() { PatchWindowsFunctions(); } + +#elif defined(_WIN32) && !defined(WIN32_DO_PATCHING) +// "libc_override_redefine.h" is included in the original gperftools. But, +// we define allocator functions in Chromium's base/allocator/allocator_shim.cc +// on Windows. We don't include libc_override_redefine.h here. +// ReplaceSystemAlloc() is defined here instead. +static void ReplaceSystemAlloc() { } + +#elif defined(__APPLE__) +#include "libc_override_osx.h" + +#elif defined(__GLIBC__) +#include "libc_override_glibc.h" + +// Not all gcc systems necessarily support weak symbols, but all the +// ones I know of do, so for now just assume they all do. +#elif defined(__GNUC__) +#include "libc_override_gcc_and_weak.h" + +#else +#error Need to add support for your libc/OS here + +#endif + +#endif // TCMALLOC_LIBC_OVERRIDE_INL_H_ diff --git a/third_party/tcmalloc/chromium/src/libc_override_gcc_and_weak.h b/third_party/tcmalloc/chromium/src/libc_override_gcc_and_weak.h new file mode 100644 index 0000000..070ebf7 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/libc_override_gcc_and_weak.h @@ -0,0 +1,99 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein <opensource@google.com> +// +// Used to override malloc routines on systems that define the +// memory allocation routines to be weak symbols in their libc +// (almost all unix-based systems are like this), on gcc, which +// suppports the 'alias' attribute. + +#ifndef TCMALLOC_LIBC_OVERRIDE_GCC_AND_WEAK_INL_H_ +#define TCMALLOC_LIBC_OVERRIDE_GCC_AND_WEAK_INL_H_ + +#ifdef HAVE_SYS_CDEFS_H +#include <sys/cdefs.h> // for __THROW +#endif +#include <gperftools/tcmalloc.h> + +#ifndef __THROW // I guess we're not on a glibc-like system +# define __THROW // __THROW is just an optimization, so ok to make it "" +#endif + +#ifndef __GNUC__ +# error libc_override_gcc_and_weak.h is for gcc distributions only. +#endif + +#define ALIAS(tc_fn) __attribute__ ((alias (#tc_fn))) + +void* operator new(size_t size) throw (std::bad_alloc) + ALIAS(tc_new); +void operator delete(void* p) __THROW + ALIAS(tc_delete); +void* operator new[](size_t size) throw (std::bad_alloc) + ALIAS(tc_newarray); +void operator delete[](void* p) __THROW + ALIAS(tc_deletearray); +void* operator new(size_t size, const std::nothrow_t& nt) __THROW + ALIAS(tc_new_nothrow); +void* operator new[](size_t size, const std::nothrow_t& nt) __THROW + ALIAS(tc_newarray_nothrow); +void operator delete(void* p, const std::nothrow_t& nt) __THROW + ALIAS(tc_delete_nothrow); +void operator delete[](void* p, const std::nothrow_t& nt) __THROW + ALIAS(tc_deletearray_nothrow); + +extern "C" { + void* malloc(size_t size) __THROW ALIAS(tc_malloc); + void free(void* ptr) __THROW ALIAS(tc_free); + void* realloc(void* ptr, size_t size) __THROW ALIAS(tc_realloc); + void* calloc(size_t n, size_t size) __THROW ALIAS(tc_calloc); + void cfree(void* ptr) __THROW ALIAS(tc_cfree); + void* memalign(size_t align, size_t s) __THROW ALIAS(tc_memalign); + void* valloc(size_t size) __THROW ALIAS(tc_valloc); + void* pvalloc(size_t size) __THROW ALIAS(tc_pvalloc); + int posix_memalign(void** r, size_t a, size_t s) __THROW + ALIAS(tc_posix_memalign); + void malloc_stats(void) __THROW ALIAS(tc_malloc_stats); + int mallopt(int cmd, int value) __THROW ALIAS(tc_mallopt); +#ifdef HAVE_STRUCT_MALLINFO + struct mallinfo mallinfo(void) __THROW ALIAS(tc_mallinfo); +#endif + size_t malloc_size(void* p) __THROW ALIAS(tc_malloc_size); + size_t malloc_usable_size(void* p) __THROW ALIAS(tc_malloc_size); +} // extern "C" + +#undef ALIAS + +// No need to do anything at tcmalloc-registration time: we do it all +// via overriding weak symbols (at link time). +static void ReplaceSystemAlloc() { } + +#endif // TCMALLOC_LIBC_OVERRIDE_GCC_AND_WEAK_INL_H_ diff --git a/third_party/tcmalloc/chromium/src/libc_override_glibc.h b/third_party/tcmalloc/chromium/src/libc_override_glibc.h new file mode 100644 index 0000000..7cdbe97 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/libc_override_glibc.h @@ -0,0 +1,159 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein <opensource@google.com> +// +// Used to override malloc routines on systems that are using glibc. + +#ifndef TCMALLOC_LIBC_OVERRIDE_GLIBC_INL_H_ +#define TCMALLOC_LIBC_OVERRIDE_GLIBC_INL_H_ + +// MALLOC_HOOK_MAYBE_VOLATILE is defined at config.h in the original gperftools. +// Chromium does this check with the macro __MALLOC_HOOK_VOLATILE. +// GLibc 2.14+ requires the hook functions be declared volatile, based on the +// value of the define __MALLOC_HOOK_VOLATILE. For compatibility with +// older/non-GLibc implementations, provide an empty definition. +#if !defined(__MALLOC_HOOK_VOLATILE) +#define MALLOC_HOOK_MAYBE_VOLATILE /**/ +#else +#define MALLOC_HOOK_MAYBE_VOLATILE __MALLOC_HOOK_VOLATILE +#endif + +#include <config.h> +#include <features.h> // for __GLIBC__ +#ifdef HAVE_SYS_CDEFS_H +#include <sys/cdefs.h> // for __THROW +#endif +#include <gperftools/tcmalloc.h> + +#ifndef __GLIBC__ +# error libc_override_glibc.h is for glibc distributions only. +#endif + +// In glibc, the memory-allocation methods are weak symbols, so we can +// just override them with our own. If we're using gcc, we can use +// __attribute__((alias)) to do the overriding easily (exception: +// Mach-O, which doesn't support aliases). Otherwise we have to use a +// function call. +#if !defined(__GNUC__) || defined(__MACH__) + +// This also defines ReplaceSystemAlloc(). +# include "libc_override_redefine.h" // defines functions malloc()/etc + +#else // #if !defined(__GNUC__) || defined(__MACH__) + +// If we get here, we're a gcc system, so do all the overriding we do +// with gcc. This does the overriding of all the 'normal' memory +// allocation. This also defines ReplaceSystemAlloc(). +# include "libc_override_gcc_and_weak.h" + +// We also have to do some glibc-specific overriding. Some library +// routines on RedHat 9 allocate memory using malloc() and free it +// using __libc_free() (or vice-versa). Since we provide our own +// implementations of malloc/free, we need to make sure that the +// __libc_XXX variants (defined as part of glibc) also point to the +// same implementations. Since it only matters for redhat, we +// do it inside the gcc #ifdef, since redhat uses gcc. +// TODO(csilvers): only do this if we detect we're an old enough glibc? + +#define ALIAS(tc_fn) __attribute__ ((alias (#tc_fn))) +extern "C" { + void* __libc_malloc(size_t size) ALIAS(tc_malloc); + void __libc_free(void* ptr) ALIAS(tc_free); + void* __libc_realloc(void* ptr, size_t size) ALIAS(tc_realloc); + void* __libc_calloc(size_t n, size_t size) ALIAS(tc_calloc); + void __libc_cfree(void* ptr) ALIAS(tc_cfree); + void* __libc_memalign(size_t align, size_t s) ALIAS(tc_memalign); + void* __libc_valloc(size_t size) ALIAS(tc_valloc); + void* __libc_pvalloc(size_t size) ALIAS(tc_pvalloc); + int __posix_memalign(void** r, size_t a, size_t s) ALIAS(tc_posix_memalign); +} // extern "C" +#undef ALIAS + +#endif // #if defined(__GNUC__) && !defined(__MACH__) + + +// We also have to hook libc malloc. While our work with weak symbols +// should make sure libc malloc is never called in most situations, it +// can be worked around by shared libraries with the DEEPBIND +// environment variable set. The below hooks libc to call our malloc +// routines even in that situation. In other situations, this hook +// should never be called. +extern "C" { +static void* glibc_override_malloc(size_t size, const void *caller) { + return tc_malloc(size); +} +static void* glibc_override_realloc(void *ptr, size_t size, + const void *caller) { + return tc_realloc(ptr, size); +} +static void glibc_override_free(void *ptr, const void *caller) { + tc_free(ptr); +} +static void* glibc_override_memalign(size_t align, size_t size, + const void *caller) { + return tc_memalign(align, size); +} + +// We should be using __malloc_initialize_hook here, like the #if 0 +// code below. (See http://swoolley.org/man.cgi/3/malloc_hook.) +// However, this causes weird linker errors with programs that link +// with -static, so instead we just assign the vars directly at +// static-constructor time. That should serve the same effect of +// making sure the hooks are set before the first malloc call the +// program makes. +#if 0 +#include <malloc.h> // for __malloc_hook, etc. +void glibc_override_malloc_init_hook(void) { + __malloc_hook = glibc_override_malloc; + __realloc_hook = glibc_override_realloc; + __free_hook = glibc_override_free; + __memalign_hook = glibc_override_memalign; +} + +void (* MALLOC_HOOK_MAYBE_VOLATILE __malloc_initialize_hook)(void) + = &glibc_override_malloc_init_hook; +#endif + +void* (* MALLOC_HOOK_MAYBE_VOLATILE __malloc_hook)(size_t, const void*) + = &glibc_override_malloc; +void* (* MALLOC_HOOK_MAYBE_VOLATILE __realloc_hook)(void*, size_t, const void*) + = &glibc_override_realloc; +void (* MALLOC_HOOK_MAYBE_VOLATILE __free_hook)(void*, const void*) + = &glibc_override_free; +void* (* MALLOC_HOOK_MAYBE_VOLATILE __memalign_hook)(size_t,size_t, const void*) + = &glibc_override_memalign; + +} // extern "C" + +// No need to write ReplaceSystemAlloc(); one of the #includes above +// did it for us. + +#endif // TCMALLOC_LIBC_OVERRIDE_GLIBC_INL_H_ diff --git a/third_party/tcmalloc/chromium/src/libc_override_osx.h b/third_party/tcmalloc/chromium/src/libc_override_osx.h new file mode 100644 index 0000000..78a0ef2 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/libc_override_osx.h @@ -0,0 +1,275 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein <opensource@google.com> +// +// Used to override malloc routines on OS X systems. We use the +// malloc-zone functionality built into OS X to register our malloc +// routine. +// +// 1) We used to use the normal 'override weak libc malloc/etc' +// technique for OS X. This is not optimal because mach does not +// support the 'alias' attribute, so we had to have forwarding +// functions. It also does not work very well with OS X shared +// libraries (dylibs) -- in general, the shared libs don't use +// tcmalloc unless run with the DYLD_FORCE_FLAT_NAMESPACE envvar. +// +// 2) Another approach would be to use an interposition array: +// static const interpose_t interposers[] __attribute__((section("__DATA, __interpose"))) = { +// { (void *)tc_malloc, (void *)malloc }, +// { (void *)tc_free, (void *)free }, +// }; +// This requires the user to set the DYLD_INSERT_LIBRARIES envvar, so +// is not much better. +// +// 3) Registering a new malloc zone avoids all these issues: +// http://www.opensource.apple.com/source/Libc/Libc-583/include/malloc/malloc.h +// http://www.opensource.apple.com/source/Libc/Libc-583/gen/malloc.c +// If we make tcmalloc the default malloc zone (undocumented but +// possible) then all new allocs use it, even those in shared +// libraries. Allocs done before tcmalloc was installed, or in libs +// that aren't using tcmalloc for some reason, will correctly go +// through the malloc-zone interface when free-ing, and will pick up +// the libc free rather than tcmalloc free. So it should "never" +// cause a crash (famous last words). +// +// 4) The routines one must define for one's own malloc have changed +// between OS X versions. This requires some hoops on our part, but +// is only really annoying when it comes to posix_memalign. The right +// behavior there depends on what OS version tcmalloc was compiled on, +// but also what OS version the program is running on. For now, we +// punt and don't implement our own posix_memalign. Apps that really +// care can use tc_posix_memalign directly. + +#ifndef TCMALLOC_LIBC_OVERRIDE_OSX_INL_H_ +#define TCMALLOC_LIBC_OVERRIDE_OSX_INL_H_ + +#include <config.h> +#ifdef HAVE_FEATURES_H +#include <features.h> +#endif +#include <gperftools/tcmalloc.h> + +#if !defined(__APPLE__) +# error libc_override_glibc-osx.h is for OS X distributions only. +#endif + +#include <AvailabilityMacros.h> +#include <malloc/malloc.h> + +// from AvailabilityMacros.h +#if defined(MAC_OS_X_VERSION_10_6) && \ + MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 +extern "C" { + // This function is only available on 10.6 (and later) but the + // LibSystem headers do not use AvailabilityMacros.h to handle weak + // importing automatically. This prototype is a copy of the one in + // <malloc/malloc.h> with the WEAK_IMPORT_ATTRBIUTE added. + extern malloc_zone_t *malloc_default_purgeable_zone(void) + WEAK_IMPORT_ATTRIBUTE; +} +#endif + +// We need to provide wrappers around all the libc functions. +namespace { +size_t mz_size(malloc_zone_t* zone, const void* ptr) { + if (MallocExtension::instance()->GetOwnership(ptr) != MallocExtension::kOwned) + return 0; // malloc_zone semantics: return 0 if we don't own the memory + + // TODO(csilvers): change this method to take a const void*, one day. + return MallocExtension::instance()->GetAllocatedSize(const_cast<void*>(ptr)); +} + +void* mz_malloc(malloc_zone_t* zone, size_t size) { + return tc_malloc(size); +} + +void* mz_calloc(malloc_zone_t* zone, size_t num_items, size_t size) { + return tc_calloc(num_items, size); +} + +void* mz_valloc(malloc_zone_t* zone, size_t size) { + return tc_valloc(size); +} + +void mz_free(malloc_zone_t* zone, void* ptr) { + return tc_free(ptr); +} + +void* mz_realloc(malloc_zone_t* zone, void* ptr, size_t size) { + return tc_realloc(ptr, size); +} + +void* mz_memalign(malloc_zone_t* zone, size_t align, size_t size) { + return tc_memalign(align, size); +} + +void mz_destroy(malloc_zone_t* zone) { + // A no-op -- we will not be destroyed! +} + +// malloc_introspection callbacks. I'm not clear on what all of these do. +kern_return_t mi_enumerator(task_t task, void *, + unsigned type_mask, vm_address_t zone_address, + memory_reader_t reader, + vm_range_recorder_t recorder) { + // Should enumerate all the pointers we have. Seems like a lot of work. + return KERN_FAILURE; +} + +size_t mi_good_size(malloc_zone_t *zone, size_t size) { + // I think it's always safe to return size, but we maybe could do better. + return size; +} + +boolean_t mi_check(malloc_zone_t *zone) { + return MallocExtension::instance()->VerifyAllMemory(); +} + +void mi_print(malloc_zone_t *zone, boolean_t verbose) { + int bufsize = 8192; + if (verbose) + bufsize = 102400; // I picked this size arbitrarily + char* buffer = new char[bufsize]; + MallocExtension::instance()->GetStats(buffer, bufsize); + fprintf(stdout, "%s", buffer); + delete[] buffer; +} + +void mi_log(malloc_zone_t *zone, void *address) { + // I don't think we support anything like this +} + +void mi_force_lock(malloc_zone_t *zone) { + // Hopefully unneeded by us! +} + +void mi_force_unlock(malloc_zone_t *zone) { + // Hopefully unneeded by us! +} + +void mi_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) { + // TODO(csilvers): figure out how to fill these out + stats->blocks_in_use = 0; + stats->size_in_use = 0; + stats->max_size_in_use = 0; + stats->size_allocated = 0; +} + +boolean_t mi_zone_locked(malloc_zone_t *zone) { + return false; // Hopefully unneeded by us! +} + +} // unnamed namespace + +// OS X doesn't have pvalloc, cfree, malloc_statc, etc, so we can just +// define our own. :-) OS X supplies posix_memalign in some versions +// but not others, either strongly or weakly linked, in a way that's +// difficult enough to code to correctly, that I just don't try to +// support either memalign() or posix_memalign(). If you need them +// and are willing to code to tcmalloc, you can use tc_posix_memalign(). +extern "C" { + void cfree(void* p) { tc_cfree(p); } + void* pvalloc(size_t s) { return tc_pvalloc(s); } + void malloc_stats(void) { tc_malloc_stats(); } + int mallopt(int cmd, int v) { return tc_mallopt(cmd, v); } + // No struct mallinfo on OS X, so don't define mallinfo(). + // An alias for malloc_size(), which OS X defines. + size_t malloc_usable_size(void* p) { return tc_malloc_size(p); } +} // extern "C" + +static void ReplaceSystemAlloc() { + static malloc_introspection_t tcmalloc_introspection; + memset(&tcmalloc_introspection, 0, sizeof(tcmalloc_introspection)); + + tcmalloc_introspection.enumerator = &mi_enumerator; + tcmalloc_introspection.good_size = &mi_good_size; + tcmalloc_introspection.check = &mi_check; + tcmalloc_introspection.print = &mi_print; + tcmalloc_introspection.log = &mi_log; + tcmalloc_introspection.force_lock = &mi_force_lock; + tcmalloc_introspection.force_unlock = &mi_force_unlock; + + static malloc_zone_t tcmalloc_zone; + memset(&tcmalloc_zone, 0, sizeof(malloc_zone_t)); + + // Start with a version 4 zone which is used for OS X 10.4 and 10.5. + tcmalloc_zone.version = 4; + tcmalloc_zone.zone_name = "tcmalloc"; + tcmalloc_zone.size = &mz_size; + tcmalloc_zone.malloc = &mz_malloc; + tcmalloc_zone.calloc = &mz_calloc; + tcmalloc_zone.valloc = &mz_valloc; + tcmalloc_zone.free = &mz_free; + tcmalloc_zone.realloc = &mz_realloc; + tcmalloc_zone.destroy = &mz_destroy; + tcmalloc_zone.batch_malloc = NULL; + tcmalloc_zone.batch_free = NULL; + tcmalloc_zone.introspect = &tcmalloc_introspection; + + // from AvailabilityMacros.h +#if defined(MAC_OS_X_VERSION_10_6) && \ + MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 + // Switch to version 6 on OSX 10.6 to support memalign. + tcmalloc_zone.version = 6; + tcmalloc_zone.free_definite_size = NULL; + tcmalloc_zone.memalign = &mz_memalign; + tcmalloc_introspection.zone_locked = &mi_zone_locked; + + // Request the default purgable zone to force its creation. The + // current default zone is registered with the purgable zone for + // doing tiny and small allocs. Sadly, it assumes that the default + // zone is the szone implementation from OS X and will crash if it + // isn't. By creating the zone now, this will be true and changing + // the default zone won't cause a problem. This only needs to + // happen when actually running on OS X 10.6 and higher (note the + // ifdef above only checks if we were *compiled* with 10.6 or + // higher; at runtime we have to check if this symbol is defined.) + if (malloc_default_purgeable_zone) { + malloc_default_purgeable_zone(); + } +#endif + + // Register the tcmalloc zone. At this point, it will not be the + // default zone. + malloc_zone_register(&tcmalloc_zone); + + // Unregister and reregister the default zone. Unregistering swaps + // the specified zone with the last one registered which for the + // default zone makes the more recently registered zone the default + // zone. The default zone is then re-registered to ensure that + // allocations made from it earlier will be handled correctly. + // Things are not guaranteed to work that way, but it's how they work now. + malloc_zone_t *default_zone = malloc_default_zone(); + malloc_zone_unregister(default_zone); + malloc_zone_register(default_zone); +} + +#endif // TCMALLOC_LIBC_OVERRIDE_OSX_INL_H_ diff --git a/third_party/tcmalloc/chromium/src/libc_override_redefine.h b/third_party/tcmalloc/chromium/src/libc_override_redefine.h new file mode 100644 index 0000000..d8d999c --- /dev/null +++ b/third_party/tcmalloc/chromium/src/libc_override_redefine.h @@ -0,0 +1,93 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein <opensource@google.com> +// +// Used on systems that don't have their own definition of +// malloc/new/etc. (Typically this will be a windows msvcrt.dll that +// has been edited to remove the definitions.) We can just define our +// own as normal functions. +// +// This should also work on systems were all the malloc routines are +// defined as weak symbols, and there's no support for aliasing. + +#ifndef TCMALLOC_LIBC_OVERRIDE_REDEFINE_H_ +#define TCMALLOC_LIBC_OVERRIDE_REDEFINE_H_ + +#ifdef HAVE_SYS_CDEFS_H +#include <sys/cdefs.h> // for __THROW +#endif + +#ifndef __THROW // I guess we're not on a glibc-like system +# define __THROW // __THROW is just an optimization, so ok to make it "" +#endif + +void* operator new(size_t size) { return tc_new(size); } +void operator delete(void* p) __THROW { tc_delete(p); } +void* operator new[](size_t size) { return tc_newarray(size); } +void operator delete[](void* p) __THROW { tc_deletearray(p); } +void* operator new(size_t size, const std::nothrow_t& nt) __THROW { + return tc_new_nothrow(size, nt); +} +void* operator new[](size_t size, const std::nothrow_t& nt) __THROW { + return tc_newarray_nothrow(size, nt); +} +void operator delete(void* ptr, const std::nothrow_t& nt) __THROW { + return tc_delete_nothrow(ptr, nt); +} +void operator delete[](void* ptr, const std::nothrow_t& nt) __THROW { + return tc_deletearray_nothrow(ptr, nt); +} +extern "C" { + void* malloc(size_t s) __THROW { return tc_malloc(s); } + void free(void* p) __THROW { tc_free(p); } + void* realloc(void* p, size_t s) __THROW { return tc_realloc(p, s); } + void* calloc(size_t n, size_t s) __THROW { return tc_calloc(n, s); } + void cfree(void* p) __THROW { tc_cfree(p); } + void* memalign(size_t a, size_t s) __THROW { return tc_memalign(a, s); } + void* valloc(size_t s) __THROW { return tc_valloc(s); } + void* pvalloc(size_t s) __THROW { return tc_pvalloc(s); } + int posix_memalign(void** r, size_t a, size_t s) __THROW { + return tc_posix_memalign(r, a, s); + } + void malloc_stats(void) __THROW { tc_malloc_stats(); } + int mallopt(int cmd, int v) __THROW { return tc_mallopt(cmd, v); } +#ifdef HAVE_STRUCT_MALLINFO + struct mallinfo mallinfo(void) __THROW { return tc_mallinfo(); } +#endif + size_t malloc_size(void* p) __THROW { return tc_malloc_size(p); } + size_t malloc_usable_size(void* p) __THROW { return tc_malloc_size(p); } +} // extern "C" + +// No need to do anything at tcmalloc-registration time: we do it all +// via overriding weak symbols (at link time). +static void ReplaceSystemAlloc() { } + +#endif // TCMALLOC_LIBC_OVERRIDE_REDEFINE_H_ diff --git a/third_party/tcmalloc/chromium/src/malloc_extension.cc b/third_party/tcmalloc/chromium/src/malloc_extension.cc index e9a0da7..2d6497f 100644 --- a/third_party/tcmalloc/chromium/src/malloc_extension.cc +++ b/third_party/tcmalloc/chromium/src/malloc_extension.cc @@ -45,9 +45,10 @@ #include "base/dynamic_annotations.h" #include "base/sysinfo.h" // for FillProcSelfMaps #ifndef NO_HEAP_CHECK -#include "google/heap-checker.h" +#include "gperftools/heap-checker.h" #endif -#include "google/malloc_extension.h" +#include "gperftools/malloc_extension.h" +#include "gperftools/malloc_extension_c.h" #include "maybe_threads.h" using STL_NAMESPACE::string; @@ -107,9 +108,9 @@ SysAllocator::~SysAllocator() {} // Default implementation -- does nothing MallocExtension::~MallocExtension() { } bool MallocExtension::VerifyAllMemory() { return true; } -bool MallocExtension::VerifyNewMemory(void* p) { return true; } -bool MallocExtension::VerifyArrayNewMemory(void* p) { return true; } -bool MallocExtension::VerifyMallocMemory(void* p) { return true; } +bool MallocExtension::VerifyNewMemory(const void* p) { return true; } +bool MallocExtension::VerifyArrayNewMemory(const void* p) { return true; } +bool MallocExtension::VerifyMallocMemory(const void* p) { return true; } bool MallocExtension::GetNumericProperty(const char* property, size_t* value) { return false; @@ -176,10 +177,15 @@ size_t MallocExtension::GetEstimatedAllocatedSize(size_t size) { return size; } -size_t MallocExtension::GetAllocatedSize(void* p) { +size_t MallocExtension::GetAllocatedSize(const void* p) { + assert(GetOwnership(p) != kNotOwned); return 0; } +MallocExtension::Ownership MallocExtension::GetOwnership(const void* p) { + return kUnknownOwnership; +} + void MallocExtension::GetFreeListSizes( vector<MallocExtension::FreeListInfo>* v) { v->clear(); @@ -238,11 +244,11 @@ void PrintCountAndSize(MallocExtensionWriter* writer, uintptr_t count, uintptr_t size) { char buf[100]; snprintf(buf, sizeof(buf), - "%6lld: %8lld [%6lld: %8lld] @", - static_cast<long long>(count), - static_cast<long long>(size), - static_cast<long long>(count), - static_cast<long long>(size)); + "%6"PRIu64": %8"PRIu64" [%6"PRIu64": %8"PRIu64"] @", + static_cast<uint64>(count), + static_cast<uint64>(size), + static_cast<uint64>(count), + static_cast<uint64>(size)); writer->append(buf, strlen(buf)); } @@ -337,9 +343,9 @@ void MallocExtension::Ranges(void* arg, RangeFunction func) { } C_SHIM(VerifyAllMemory, int, (void), ()); -C_SHIM(VerifyNewMemory, int, (void* p), (p)); -C_SHIM(VerifyArrayNewMemory, int, (void* p), (p)); -C_SHIM(VerifyMallocMemory, int, (void* p), (p)); +C_SHIM(VerifyNewMemory, int, (const void* p), (p)); +C_SHIM(VerifyArrayNewMemory, int, (const void* p), (p)); +C_SHIM(VerifyMallocMemory, int, (const void* p), (p)); C_SHIM(MallocMemoryStats, int, (int* blocks, size_t* total, int histogram[kMallocHistogramSize]), (blocks, total, histogram)); @@ -356,4 +362,11 @@ C_SHIM(MarkThreadBusy, void, (void), ()); C_SHIM(ReleaseFreeMemory, void, (void), ()); C_SHIM(ReleaseToSystem, void, (size_t num_bytes), (num_bytes)); C_SHIM(GetEstimatedAllocatedSize, size_t, (size_t size), (size)); -C_SHIM(GetAllocatedSize, size_t, (void* p), (p)); +C_SHIM(GetAllocatedSize, size_t, (const void* p), (p)); + +// Can't use the shim here because of the need to translate the enums. +extern "C" +MallocExtension_Ownership MallocExtension_GetOwnership(const void* p) { + return static_cast<MallocExtension_Ownership>( + MallocExtension::instance()->GetOwnership(p)); +} diff --git a/third_party/tcmalloc/chromium/src/malloc_hook-inl.h b/third_party/tcmalloc/chromium/src/malloc_hook-inl.h index e7bfd61..7c783bf 100644 --- a/third_party/tcmalloc/chromium/src/malloc_hook-inl.h +++ b/third_party/tcmalloc/chromium/src/malloc_hook-inl.h @@ -41,7 +41,7 @@ #include <sys/types.h> #include "base/atomicops.h" #include "base/basictypes.h" -#include <google/malloc_hook.h> +#include <gperftools/malloc_hook.h> namespace base { namespace internal { @@ -104,7 +104,7 @@ static const int kHookListMaxValues = 7; // HookList: a class that provides synchronized insertions and removals and // lockless traversal. Most of the implementation is in malloc_hook.cc. template <typename T> -struct HookList { +struct PERFTOOLS_DLL_DECL HookList { COMPILE_ASSERT(sizeof(T) <= sizeof(AtomicWord), T_should_fit_in_AtomicWord); // Adds value to the list. Note that duplicates are allowed. Thread-safe and diff --git a/third_party/tcmalloc/chromium/src/malloc_hook.cc b/third_party/tcmalloc/chromium/src/malloc_hook.cc index f6af7d8..5d6b336 100644 --- a/third_party/tcmalloc/chromium/src/malloc_hook.cc +++ b/third_party/tcmalloc/chromium/src/malloc_hook.cc @@ -46,12 +46,11 @@ #include <stdint.h> #endif #include <algorithm> -#include "base/basictypes.h" #include "base/logging.h" #include "base/spinlock.h" #include "maybe_threads.h" #include "malloc_hook-inl.h" -#include <google/malloc_hook.h> +#include <gperftools/malloc_hook.h> // This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if // you're porting to a system where you really can't get a stacktrace. @@ -59,7 +58,7 @@ // We use #define so code compiles even if you #include stacktrace.h somehow. # define GetStackTrace(stack, depth, skip) (0) #else -# include <google/stacktrace.h> +# include <gperftools/stacktrace.h> #endif // __THROW is defined in glibc systems. It means, counter-intuitively, @@ -204,14 +203,8 @@ static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED); template <typename T> bool HookList<T>::Add(T value_as_t) { - // Note: we need to check this _before_ reinterpret_cast, since - // reinterpret_cast may include random junk from memory. - if (value_as_t == 0) { - return false; - } - AtomicWord value = reinterpret_cast<const AtomicWord&>(value_as_t); + AtomicWord value = bit_cast<AtomicWord>(value_as_t); if (value == 0) { - // This should not actually happen, but just to be sure... return false; } SpinLockHolder l(&hooklist_spinlock); @@ -240,8 +233,7 @@ bool HookList<T>::Remove(T value_as_t) { SpinLockHolder l(&hooklist_spinlock); AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end); int index = 0; - // Note: we need to cast back to T since T may be smaller than AtomicWord. - while (index < hooks_end && value_as_t != reinterpret_cast<T>( + while (index < hooks_end && value_as_t != bit_cast<T>( base::subtle::Acquire_Load(&priv_data[index]))) { ++index; } @@ -268,7 +260,7 @@ int HookList<T>::Traverse(T* output_array, int n) const { for (int i = 0; i < hooks_end && n > 0; ++i) { AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]); if (data != 0) { - *output_array++ = reinterpret_cast<const T&>(data); + *output_array++ = bit_cast<T>(data); ++actual_hooks_end; --n; } @@ -283,7 +275,7 @@ int HookList<T>::Traverse(T* output_array, int n) const { // Explicit instantiation for malloc_hook_test.cc. This ensures all the methods // are instantiated. -template class HookList<MallocHook::NewHook>; +template struct HookList<MallocHook::NewHook>; HookList<MallocHook::NewHook> new_hooks_ = INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook); @@ -698,191 +690,17 @@ extern "C" int MallocHook_GetCallerStackTrace(void** result, int max_depth, #endif } -// On Linux/x86, we override mmap/munmap/mremap/sbrk -// and provide support for calling the related hooks. -// -// We define mmap() and mmap64(), which somewhat reimplements libc's mmap -// syscall stubs. Unfortunately libc only exports the stubs via weak symbols -// (which we're overriding with our mmap64() and mmap() wrappers) so we can't -// just call through to them. - - -#if defined(__linux) && \ - (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) -#include <unistd.h> -#include <syscall.h> -#include <sys/mman.h> -#include <errno.h> -#include "base/linux_syscall_support.h" - -// The x86-32 case and the x86-64 case differ: -// 32b has a mmap2() syscall, 64b does not. -// 64b and 32b have different calling conventions for mmap(). -#if defined(__x86_64__) || defined(__PPC64__) - -static inline void* do_mmap64(void *start, size_t length, - int prot, int flags, - int fd, __off64_t offset) __THROW { - return (void *)syscall(SYS_mmap, start, length, prot, flags, fd, offset); -} - -#elif defined(__i386__) || defined(__PPC__) - -static inline void* do_mmap64(void *start, size_t length, - int prot, int flags, - int fd, __off64_t offset) __THROW { - void *result; - - // Try mmap2() unless it's not supported - static bool have_mmap2 = true; - if (have_mmap2) { - static int pagesize = 0; - if (!pagesize) pagesize = getpagesize(); - - // Check that the offset is page aligned - if (offset & (pagesize - 1)) { - result = MAP_FAILED; - errno = EINVAL; - goto out; - } - - result = (void *)syscall(SYS_mmap2, - start, length, prot, flags, fd, - (off_t) (offset / pagesize)); - if (result != MAP_FAILED || errno != ENOSYS) goto out; - - // We don't have mmap2() after all - don't bother trying it in future - have_mmap2 = false; - } - - if (((off_t)offset) != offset) { - // If we're trying to map a 64-bit offset, fail now since we don't - // have 64-bit mmap() support. - result = MAP_FAILED; - errno = EINVAL; - goto out; - } - - { - // Fall back to old 32-bit offset mmap() call - // Old syscall interface cannot handle six args, so pass in an array - int32 args[6] = { (int32) start, length, prot, flags, fd, (off_t) offset }; - result = (void *)syscall(SYS_mmap, args); - } - out: - return result; -} - -# endif // defined(__x86_64__) - -// We use do_mmap64 abstraction to put MallocHook::InvokeMmapHook -// calls right into mmap and mmap64, so that the stack frames in the caller's -// stack are at the same offsets for all the calls of memory allocating -// functions. - -// Put all callers of MallocHook::Invoke* in this module into -// malloc_hook section, -// so that MallocHook::GetCallerStackTrace can function accurately: - -// Make sure mmap doesn't get #define'd away by <sys/mman.h> -#undef mmap - -extern "C" { - void* mmap64(void *start, size_t length, int prot, int flags, - int fd, __off64_t offset ) __THROW - ATTRIBUTE_SECTION(malloc_hook); - void* mmap(void *start, size_t length,int prot, int flags, - int fd, off_t offset) __THROW - ATTRIBUTE_SECTION(malloc_hook); - int munmap(void* start, size_t length) __THROW - ATTRIBUTE_SECTION(malloc_hook); - void* mremap(void* old_addr, size_t old_size, size_t new_size, - int flags, ...) __THROW - ATTRIBUTE_SECTION(malloc_hook); - void* sbrk(std::ptrdiff_t increment) __THROW - ATTRIBUTE_SECTION(malloc_hook); -} - -extern "C" void* mmap64(void *start, size_t length, int prot, int flags, - int fd, __off64_t offset) __THROW { - MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); - void *result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = do_mmap64(start, length, prot, flags, fd, offset); - } - MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); - return result; -} - -#if !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) +// On systems where we know how, we override mmap/munmap/mremap/sbrk +// to provide support for calling the related hooks (in addition, +// of course, to doing what these functions normally do). -extern "C" void* mmap(void *start, size_t length, int prot, int flags, - int fd, off_t offset) __THROW { - MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); - void *result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = do_mmap64(start, length, prot, flags, fd, - static_cast<size_t>(offset)); // avoid sign extension - } - MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); - return result; -} +#if defined(__linux) +# include "malloc_hook_mmap_linux.h" -#endif // !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) +#elif defined(__FreeBSD__) +# include "malloc_hook_mmap_freebsd.h" -extern "C" int munmap(void* start, size_t length) __THROW { - MallocHook::InvokeMunmapHook(start, length); - int result; - if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { - result = syscall(SYS_munmap, start, length); - } - return result; -} - -extern "C" void* mremap(void* old_addr, size_t old_size, size_t new_size, - int flags, ...) __THROW { - va_list ap; - va_start(ap, flags); - void *new_address = va_arg(ap, void *); - va_end(ap); - void* result = sys_mremap(old_addr, old_size, new_size, flags, new_address); - MallocHook::InvokeMremapHook(result, old_addr, old_size, new_size, flags, - new_address); - return result; -} - -// libc's version: -extern "C" void* __sbrk(std::ptrdiff_t increment); - -extern "C" void* sbrk(std::ptrdiff_t increment) __THROW { - MallocHook::InvokePreSbrkHook(increment); - void *result = __sbrk(increment); - MallocHook::InvokeSbrkHook(result, increment); - return result; -} - -/*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, - int flags, int fd, off_t offset) { - void* result; - if (!MallocHook::InvokeMmapReplacement( - start, length, prot, flags, fd, offset, &result)) { - result = do_mmap64(start, length, prot, flags, fd, offset); - } - return result; -} - -/*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { - int result; - if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { - result = sys_munmap(start, length); - } - return result; -} - -#else // defined(__linux) && - // (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) +#else /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, int flags, int fd, off_t offset) { @@ -902,5 +720,4 @@ extern "C" void* sbrk(std::ptrdiff_t increment) __THROW { return result; } -#endif // defined(__linux) && - // (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) +#endif diff --git a/third_party/tcmalloc/chromium/src/malloc_hook_mmap_freebsd.h b/third_party/tcmalloc/chromium/src/malloc_hook_mmap_freebsd.h new file mode 100644 index 0000000..dae868c --- /dev/null +++ b/third_party/tcmalloc/chromium/src/malloc_hook_mmap_freebsd.h @@ -0,0 +1,165 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Override mmap/munmap/mremap/sbrk to provide support for calling the +// related hooks (in addition, of course, to doing what these +// functions normally do). + +#ifndef __FreeBSD__ +# error Should only be including malloc_hook_mmap_freebsd.h on FreeBSD systems. +#endif + +#include <unistd.h> +#include <sys/syscall.h> +#include <sys/mman.h> +#include <errno.h> + +// Make sure mmap doesn't get #define'd away by <sys/mman.h> +#undef mmap + +// According to the FreeBSD documentation, use syscall if you do not +// need 64-bit alignment otherwise use __syscall. Indeed, syscall +// doesn't work correctly in most situations on 64-bit. It's return +// type is 'int' so for things like SYS_mmap, it actually truncates +// the returned address to 32-bits. +#if defined(__amd64__) || defined(__x86_64__) +# define MALLOC_HOOK_SYSCALL __syscall +#else +# define MALLOC_HOOK_SYSCALL syscall +#endif + + +extern "C" { + void* mmap(void *start, size_t length,int prot, int flags, + int fd, off_t offset) __THROW + ATTRIBUTE_SECTION(malloc_hook); + int munmap(void* start, size_t length) __THROW + ATTRIBUTE_SECTION(malloc_hook); + void* sbrk(intptr_t increment) __THROW + ATTRIBUTE_SECTION(malloc_hook); +} + +static inline void* do_mmap(void *start, size_t length, + int prot, int flags, + int fd, off_t offset) __THROW { + return (void *)MALLOC_HOOK_SYSCALL(SYS_mmap, + start, length, prot, flags, fd, offset); +} + +static inline void* do_sbrk(intptr_t increment) { + void* curbrk = 0; + +#if defined(__x86_64__) || defined(__amd64__) +# ifdef PIC + __asm__ __volatile__( + "movq .curbrk@GOTPCREL(%%rip), %%rdx;" + "movq (%%rdx), %%rax;" + "movq %%rax, %0;" + : "=r" (curbrk) + :: "%rdx", "%rax"); +# else + __asm__ __volatile__( + "movq .curbrk(%%rip), %%rax;" + "movq %%rax, %0;" + : "=r" (curbrk) + :: "%rax"); +# endif +#else + __asm__ __volatile__( + "movl .curbrk, %%eax;" + "movl %%eax, %0;" + : "=r" (curbrk) + :: "%eax"); +#endif + + if (increment == 0) { + return curbrk; + } + + char* prevbrk = static_cast<char*>(curbrk); + void* newbrk = prevbrk + increment; + + if (brk(newbrk) == -1) { + return reinterpret_cast<void*>(static_cast<intptr_t>(-1)); + } + + return prevbrk; +} + + +extern "C" void* mmap(void *start, size_t length, int prot, int flags, + int fd, off_t offset) __THROW { + MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); + void *result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = do_mmap(start, length, prot, flags, fd, + static_cast<size_t>(offset)); // avoid sign extension + } + MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); + return result; +} + +extern "C" int munmap(void* start, size_t length) __THROW { + MallocHook::InvokeMunmapHook(start, length); + int result; + if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { + result = MALLOC_HOOK_SYSCALL(SYS_munmap, start, length); + } + + return result; +} + +extern "C" void* sbrk(intptr_t increment) __THROW { + MallocHook::InvokePreSbrkHook(increment); + void *result = do_sbrk(increment); + MallocHook::InvokeSbrkHook(result, increment); + return result; +} + +/*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, + int flags, int fd, off_t offset) { + void* result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = do_mmap(start, length, prot, flags, fd, offset); + } + + return result; +} + +/*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { + int result; + if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { + result = MALLOC_HOOK_SYSCALL(SYS_munmap, start, length); + } + return result; +} + +#undef MALLOC_HOOK_SYSCALL diff --git a/third_party/tcmalloc/chromium/src/malloc_hook_mmap_linux.h b/third_party/tcmalloc/chromium/src/malloc_hook_mmap_linux.h new file mode 100644 index 0000000..054b91b --- /dev/null +++ b/third_party/tcmalloc/chromium/src/malloc_hook_mmap_linux.h @@ -0,0 +1,241 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Sanjay Ghemawat <opensource@google.com> + +// We define mmap() and mmap64(), which somewhat reimplements libc's mmap +// syscall stubs. Unfortunately libc only exports the stubs via weak symbols +// (which we're overriding with our mmap64() and mmap() wrappers) so we can't +// just call through to them. + +#ifndef __linux +# error Should only be including malloc_hook_mmap_linux.h on linux systems. +#endif + +#include <unistd.h> +#include <syscall.h> +#include <sys/mman.h> +#include <errno.h> +#include "base/linux_syscall_support.h" + +// The x86-32 case and the x86-64 case differ: +// 32b has a mmap2() syscall, 64b does not. +// 64b and 32b have different calling conventions for mmap(). + +// I test for 64-bit first so I don't have to do things like +// '#if (defined(__mips__) && !defined(__MIPS64__))' as a mips32 check. +#if defined(__x86_64__) || defined(__PPC64__) || (defined(_MIPS_SIM) && _MIPS_SIM == _ABI64) + +static inline void* do_mmap64(void *start, size_t length, + int prot, int flags, + int fd, __off64_t offset) __THROW { + // The original gperftools uses sys_mmap() here. But, it is not allowed by + // Chromium's sandbox. + return (void *)syscall(SYS_mmap, start, length, prot, flags, fd, offset); +} + +#define MALLOC_HOOK_HAVE_DO_MMAP64 1 + +#elif defined(__i386__) || defined(__PPC__) || defined(__mips__) || \ + defined(__arm__) + +static inline void* do_mmap64(void *start, size_t length, + int prot, int flags, + int fd, __off64_t offset) __THROW { + void *result; + + // Try mmap2() unless it's not supported + static bool have_mmap2 = true; + if (have_mmap2) { + static int pagesize = 0; + if (!pagesize) pagesize = getpagesize(); + + // Check that the offset is page aligned + if (offset & (pagesize - 1)) { + result = MAP_FAILED; + errno = EINVAL; + goto out; + } + + result = (void *)syscall(SYS_mmap2, + start, length, prot, flags, fd, + (off_t) (offset / pagesize)); + if (result != MAP_FAILED || errno != ENOSYS) goto out; + + // We don't have mmap2() after all - don't bother trying it in future + have_mmap2 = false; + } + + if (((off_t)offset) != offset) { + // If we're trying to map a 64-bit offset, fail now since we don't + // have 64-bit mmap() support. + result = MAP_FAILED; + errno = EINVAL; + goto out; + } + +#ifdef __NR_mmap + { + // Fall back to old 32-bit offset mmap() call + // Old syscall interface cannot handle six args, so pass in an array + int32 args[6] = { (int32) start, (int32) length, prot, flags, fd, + (off_t) offset }; + result = (void *)syscall(SYS_mmap, args); + } +#else + // Some Linux ports like ARM EABI Linux has no mmap, just mmap2. + result = MAP_FAILED; +#endif + + out: + return result; +} + +#define MALLOC_HOOK_HAVE_DO_MMAP64 1 + +#endif // #if defined(__x86_64__) + + +#ifdef MALLOC_HOOK_HAVE_DO_MMAP64 + +// We use do_mmap64 abstraction to put MallocHook::InvokeMmapHook +// calls right into mmap and mmap64, so that the stack frames in the caller's +// stack are at the same offsets for all the calls of memory allocating +// functions. + +// Put all callers of MallocHook::Invoke* in this module into +// malloc_hook section, +// so that MallocHook::GetCallerStackTrace can function accurately: + +// Make sure mmap doesn't get #define'd away by <sys/mman.h> +# undef mmap + +extern "C" { + void* mmap64(void *start, size_t length, int prot, int flags, + int fd, __off64_t offset ) __THROW + ATTRIBUTE_SECTION(malloc_hook); + void* mmap(void *start, size_t length,int prot, int flags, + int fd, off_t offset) __THROW + ATTRIBUTE_SECTION(malloc_hook); + int munmap(void* start, size_t length) __THROW + ATTRIBUTE_SECTION(malloc_hook); + void* mremap(void* old_addr, size_t old_size, size_t new_size, + int flags, ...) __THROW + ATTRIBUTE_SECTION(malloc_hook); + void* sbrk(ptrdiff_t increment) __THROW + ATTRIBUTE_SECTION(malloc_hook); +} + +extern "C" void* mmap64(void *start, size_t length, int prot, int flags, + int fd, __off64_t offset) __THROW { + MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); + void *result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = do_mmap64(start, length, prot, flags, fd, offset); + } + MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); + return result; +} + +# if !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) + +extern "C" void* mmap(void *start, size_t length, int prot, int flags, + int fd, off_t offset) __THROW { + MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset); + void *result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = do_mmap64(start, length, prot, flags, fd, + static_cast<size_t>(offset)); // avoid sign extension + } + MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset); + return result; +} + +# endif // !defined(__USE_FILE_OFFSET64) || !defined(__REDIRECT_NTH) + +extern "C" int munmap(void* start, size_t length) __THROW { + MallocHook::InvokeMunmapHook(start, length); + int result; + if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { + // The original gperftools uses sys_munmap() here. But, it is not allowed + // by Chromium's sandbox. + result = syscall(SYS_munmap, start, length); + } + return result; +} + +extern "C" void* mremap(void* old_addr, size_t old_size, size_t new_size, + int flags, ...) __THROW { + va_list ap; + va_start(ap, flags); + void *new_address = va_arg(ap, void *); + va_end(ap); + // The original gperftools uses sys_mremap() here. But, it is not allowed by + // Chromium's sandbox. + void* result = (void *)syscall( + SYS_mremap, old_addr, old_size, new_size, flags, new_address); + MallocHook::InvokeMremapHook(result, old_addr, old_size, new_size, flags, + new_address); + return result; +} + +// libc's version: +extern "C" void* __sbrk(ptrdiff_t increment); + +extern "C" void* sbrk(ptrdiff_t increment) __THROW { + MallocHook::InvokePreSbrkHook(increment); + void *result = __sbrk(increment); + MallocHook::InvokeSbrkHook(result, increment); + return result; +} + +/*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot, + int flags, int fd, off_t offset) { + void* result; + if (!MallocHook::InvokeMmapReplacement( + start, length, prot, flags, fd, offset, &result)) { + result = do_mmap64(start, length, prot, flags, fd, offset); + } + return result; +} + +/*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) { + int result; + if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) { + result = syscall(SYS_munmap, start, length); + } + return result; +} + +#undef MALLOC_HOOK_HAVE_DO_MMAP64 + +#endif // #ifdef MALLOC_HOOK_HAVE_DO_MMAP64 diff --git a/third_party/tcmalloc/chromium/src/maybe_threads.cc b/third_party/tcmalloc/chromium/src/maybe_threads.cc index cda1d63..15c8a23 100644 --- a/third_party/tcmalloc/chromium/src/maybe_threads.cc +++ b/third_party/tcmalloc/chromium/src/maybe_threads.cc @@ -39,6 +39,7 @@ #include "config.h" #include <assert.h> #include <string.h> // for memcmp +#include <stdio.h> // for __isthreaded on FreeBSD // We don't actually need strings. But including this header seems to // stop the compiler trying to short-circuit our pthreads existence // tests and claiming that the address of a function is always @@ -98,9 +99,28 @@ int perftools_pthread_setspecific(pthread_key_t key, void *val) { } } + static pthread_once_t pthread_once_init = PTHREAD_ONCE_INIT; int perftools_pthread_once(pthread_once_t *ctl, void (*init_routine) (void)) { +#ifdef __FreeBSD__ + // On __FreeBSD__, calling pthread_once on a system that is not + // linked with -pthread is silently a noop. :-( Luckily, we have a + // workaround: FreeBSD exposes __isthreaded in <stdio.h>, which is + // set to 1 when the first thread is spawned. So on those systems, + // we can use our own separate pthreads-once mechanism, which is + // used until __isthreaded is 1 (which will never be true if the app + // is not linked with -pthread). + static bool pthread_once_ran_before_threads = false; + if (pthread_once_ran_before_threads) { + return 0; + } + if (!__isthreaded) { + init_routine(); + pthread_once_ran_before_threads = true; + return 0; + } +#endif if (pthread_once) { return pthread_once(ctl, init_routine); } else { diff --git a/third_party/tcmalloc/chromium/src/memfs_malloc.cc b/third_party/tcmalloc/chromium/src/memfs_malloc.cc index 3fb55a4..b59f6d9 100644 --- a/third_party/tcmalloc/chromium/src/memfs_malloc.cc +++ b/third_party/tcmalloc/chromium/src/memfs_malloc.cc @@ -54,13 +54,16 @@ #include <new> // for operator new #include <string> -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> #include "base/basictypes.h" #include "base/googleinit.h" #include "base/sysinfo.h" -#include "system-alloc.h" #include "internal_logging.h" +// TODO(sanjay): Move the code below into the tcmalloc namespace +using tcmalloc::kLog; +using tcmalloc::kCrash; +using tcmalloc::Log; using std::string; DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), @@ -86,7 +89,7 @@ DEFINE_bool(memfs_malloc_map_private, class HugetlbSysAllocator: public SysAllocator { public: explicit HugetlbSysAllocator(SysAllocator* fallback) - : failed_(true), // Unusable until FlagsInitialized() is called + : failed_(true), // To disable allocator until Initialize() is called. big_page_size_(0), hugetlb_fd_(-1), hugetlb_base_(0), @@ -94,10 +97,10 @@ public: } void* Alloc(size_t size, size_t *actual_size, size_t alignment); - - void FlagsInitialized(); + bool Initialize(); bool failed_; // Whether failed to allocate memory. + private: void* AllocInternal(size_t size, size_t *actual_size, size_t alignment); @@ -136,11 +139,11 @@ void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size, if (result != NULL) { return result; } - TCMalloc_MESSAGE(__FILE__, __LINE__, - "HugetlbSysAllocator: failed_=%d allocated=%"PRId64"\n", - failed_, static_cast<int64_t>(hugetlb_base_)); + Log(kLog, __FILE__, __LINE__, + "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_); if (FLAGS_memfs_malloc_abort_on_fail) { - CRASH("memfs_malloc_abort_on_fail is set\n"); + Log(kCrash, __FILE__, __LINE__, + "memfs_malloc_abort_on_fail is set"); } return fallback_->Alloc(size, actual_size, alignment); } @@ -158,13 +161,12 @@ void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, if (limit > 0 && hugetlb_base_ + size + extra > limit) { // Disable the allocator when there's less than one page left. if (limit - hugetlb_base_ < big_page_size_) { - TCMalloc_MESSAGE(__FILE__, __LINE__, "reached memfs_malloc_limit_mb\n"); + Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb"); failed_ = true; } else { - TCMalloc_MESSAGE(__FILE__, __LINE__, "alloc size=%"PRIuS - " too large while %"PRId64" bytes remain\n", - size, static_cast<int64_t>(limit - hugetlb_base_)); + Log(kLog, __FILE__, __LINE__, + "alloc too large (size, bytes left)", size, limit-hugetlb_base_); } return NULL; } @@ -173,8 +175,8 @@ void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, // hugetlbfs returns EINVAL for ftruncate. int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); if (ret != 0 && errno != EINVAL) { - TCMalloc_MESSAGE(__FILE__, __LINE__, "ftruncate failed: %s\n", - strerror(errno)); + Log(kLog, __FILE__, __LINE__, + "ftruncate failed", strerror(errno)); failed_ = true; return NULL; } @@ -189,8 +191,8 @@ void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, hugetlb_fd_, hugetlb_base_); if (result == reinterpret_cast<void*>(MAP_FAILED)) { if (!FLAGS_memfs_malloc_ignore_mmap_fail) { - TCMalloc_MESSAGE(__FILE__, __LINE__, "mmap of size %"PRIuS" failed: %s\n", - size + extra, strerror(errno)); + Log(kLog, __FILE__, __LINE__, + "mmap failed (size, error)", size + extra, strerror(errno)); failed_ = true; } return NULL; @@ -212,49 +214,54 @@ void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, return reinterpret_cast<void*>(ptr); } -void HugetlbSysAllocator::FlagsInitialized() { - if (FLAGS_memfs_malloc_path.length()) { - char path[PATH_MAX]; - int rc = snprintf(path, sizeof(path), "%s.XXXXXX", - FLAGS_memfs_malloc_path.c_str()); - if (rc < 0 || rc >= sizeof(path)) { - CRASH("XX fatal: memfs_malloc_path too long\n"); - } - - int hugetlb_fd = mkstemp(path); - if (hugetlb_fd == -1) { - TCMalloc_MESSAGE(__FILE__, __LINE__, - "warning: unable to create memfs_malloc_path %s: %s\n", - path, strerror(errno)); - return; - } +bool HugetlbSysAllocator::Initialize() { + char path[PATH_MAX]; + const int pathlen = FLAGS_memfs_malloc_path.size(); + if (pathlen + 8 > sizeof(path)) { + Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long"); + return false; + } + memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen); + memcpy(path + pathlen, ".XXXXXX", 8); // Also copies terminating \0 - // Cleanup memory on process exit - if (unlink(path) == -1) { - CRASH("fatal: error unlinking memfs_malloc_path %s: %s\n", - path, strerror(errno)); - } + int hugetlb_fd = mkstemp(path); + if (hugetlb_fd == -1) { + Log(kLog, __FILE__, __LINE__, + "warning: unable to create memfs_malloc_path", + path, strerror(errno)); + return false; + } - // Use fstatfs to figure out the default page size for memfs - struct statfs sfs; - if (fstatfs(hugetlb_fd, &sfs) == -1) { - CRASH("fatal: error fstatfs of memfs_malloc_path: %s\n", - strerror(errno)); - } - int64 page_size = sfs.f_bsize; + // Cleanup memory on process exit + if (unlink(path) == -1) { + Log(kCrash, __FILE__, __LINE__, + "fatal: error unlinking memfs_malloc_path", path, strerror(errno)); + return false; + } - hugetlb_fd_ = hugetlb_fd; - big_page_size_ = page_size; - failed_ = false; + // Use fstatfs to figure out the default page size for memfs + struct statfs sfs; + if (fstatfs(hugetlb_fd, &sfs) == -1) { + Log(kCrash, __FILE__, __LINE__, + "fatal: error fstatfs of memfs_malloc_path", strerror(errno)); + return false; } -} + int64 page_size = sfs.f_bsize; -static void InitSystemAllocator() { - SysAllocator *alloc = MallocExtension::instance()->GetSystemAllocator(); - HugetlbSysAllocator *hugetlb = new (hugetlb_space) HugetlbSysAllocator(alloc); - MallocExtension::instance()->SetSystemAllocator(hugetlb); + hugetlb_fd_ = hugetlb_fd; + big_page_size_ = page_size; + failed_ = false; + return true; } -REGISTER_MODULE_INITIALIZER(memfs_malloc, { InitSystemAllocator(); }); +REGISTER_MODULE_INITIALIZER(memfs_malloc, { + if (FLAGS_memfs_malloc_path.length()) { + SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator(); + HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc); + if (hp->Initialize()) { + MallocExtension::instance()->SetSystemAllocator(hp); + } + } +}); #endif /* ifdef __linux */ diff --git a/third_party/tcmalloc/chromium/src/memory_region_map.cc b/third_party/tcmalloc/chromium/src/memory_region_map.cc index 31c3bc2..85095bf 100644 --- a/third_party/tcmalloc/chromium/src/memory_region_map.cc +++ b/third_party/tcmalloc/chromium/src/memory_region_map.cc @@ -122,8 +122,8 @@ #include "base/low_level_alloc.h" #include "malloc_hook-inl.h" -#include <google/stacktrace.h> -#include <google/malloc_hook.h> +#include <gperftools/stacktrace.h> +#include <gperftools/malloc_hook.h> // MREMAP_FIXED is a linux extension. How it's used in this file, // setting it to 0 is equivalent to saying, "This feature isn't @@ -145,6 +145,8 @@ SpinLock MemoryRegionMap::owner_lock_( // ACQUIRED_AFTER(lock_) SpinLock::LINKER_INITIALIZED); int MemoryRegionMap::recursion_count_ = 0; // GUARDED_BY(owner_lock_) pthread_t MemoryRegionMap::lock_owner_tid_; // GUARDED_BY(owner_lock_) +int64 MemoryRegionMap::map_size_ = 0; +int64 MemoryRegionMap::unmap_size_ = 0; // ========================================================================= // @@ -462,6 +464,7 @@ void MemoryRegionMap::RecordRegionAddition(const void* start, size_t size) { reinterpret_cast<void*>(region.caller())); // Note: none of the above allocates memory. Lock(); // recursively lock + map_size_ += size; InsertRegionLocked(region); // This will (eventually) allocate storage for and copy over the stack data // from region.call_stack_data_ that is pointed by region.call_stack(). @@ -573,6 +576,7 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) { reinterpret_cast<void*>(end_addr), regions_->size()); if (VLOG_IS_ON(12)) LogAllLocked(); + unmap_size_ += size; Unlock(); } @@ -582,8 +586,8 @@ void MemoryRegionMap::MmapHook(const void* result, int fd, off_t offset) { // TODO(maxim): replace all 0x%"PRIxS" by %p when RAW_VLOG uses a safe // snprintf reimplementation that does not malloc to pretty-print NULL - RAW_VLOG(10, "MMap = 0x%"PRIxPTR" of %"PRIuS" at %llu " - "prot %d flags %d fd %d offs %lld", + RAW_VLOG(10, "MMap = 0x%"PRIxPTR" of %"PRIuS" at %"PRIu64" " + "prot %d flags %d fd %d offs %"PRId64, reinterpret_cast<uintptr_t>(result), size, reinterpret_cast<uint64>(start), prot, flags, fd, static_cast<int64>(offset)); diff --git a/third_party/tcmalloc/chromium/src/memory_region_map.h b/third_party/tcmalloc/chromium/src/memory_region_map.h index 09561ce..8c13fbc 100644 --- a/third_party/tcmalloc/chromium/src/memory_region_map.h +++ b/third_party/tcmalloc/chromium/src/memory_region_map.h @@ -252,6 +252,10 @@ class MemoryRegionMap { static RegionIterator BeginRegionLocked(); static RegionIterator EndRegionLocked(); + // Return the accumulated sizes of mapped and unmapped regions. + static int64 MapSize() { return map_size_; } + static int64 UnmapSize() { return unmap_size_; } + // Effectively private type from our .cc ================================= // public to let us declare global objects: union RegionSetRep; @@ -286,6 +290,11 @@ class MemoryRegionMap { // The thread id of the thread that's inside the recursive lock. static pthread_t lock_owner_tid_; + // Total size of all mapped pages so far + static int64 map_size_; + // Total size of all unmapped pages so far + static int64 unmap_size_; + // helpers ================================================================== // Helper for FindRegion and FindAndMarkStackRegion: diff --git a/third_party/tcmalloc/chromium/src/page_heap.cc b/third_party/tcmalloc/chromium/src/page_heap.cc index 83ff892..402dc1f 100644 --- a/third_party/tcmalloc/chromium/src/page_heap.cc +++ b/third_party/tcmalloc/chromium/src/page_heap.cc @@ -34,7 +34,7 @@ #ifdef HAVE_INTTYPES_H #include <inttypes.h> // for PRIuPTR #endif -#include <google/malloc_extension.h> // for MallocRange, etc +#include <gperftools/malloc_extension.h> // for MallocRange, etc #include "base/basictypes.h" #include "base/commandlineflags.h" #include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc @@ -403,103 +403,25 @@ void PageHeap::RegisterSizeClass(Span* span, size_t sc) { } } -static double MiB(uint64_t bytes) { - return bytes / 1048576.0; -} - -static double PagesToMiB(uint64_t pages) { - return (pages << kPageShift) / 1048576.0; -} - -void PageHeap::GetClassSizes(int64 class_sizes_normal[kMaxPages], - int64 class_sizes_returned[kMaxPages], - int64* normal_pages_in_spans, - int64* returned_pages_in_spans) { - +void PageHeap::GetSmallSpanStats(SmallSpanStats* result) { for (int s = 0; s < kMaxPages; s++) { - if (class_sizes_normal != NULL) { - class_sizes_normal[s] = DLL_Length(&free_[s].normal); - } - if (class_sizes_returned != NULL) { - class_sizes_returned[s] = DLL_Length(&free_[s].returned); - } - } - - if (normal_pages_in_spans != NULL) { - *normal_pages_in_spans = 0; - for (Span* s = large_.normal.next; s != &large_.normal; s = s->next) { - *normal_pages_in_spans += s->length;; - } - } - - if (returned_pages_in_spans != NULL) { - *returned_pages_in_spans = 0; - for (Span* s = large_.returned.next; s != &large_.returned; s = s->next) { - *returned_pages_in_spans += s->length; - } + result->normal_length[s] = DLL_Length(&free_[s].normal); + result->returned_length[s] = DLL_Length(&free_[s].returned); } } -void PageHeap::Dump(TCMalloc_Printer* out) { - int nonempty_sizes = 0; - for (int s = 0; s < kMaxPages; s++) { - if (!DLL_IsEmpty(&free_[s].normal) || !DLL_IsEmpty(&free_[s].returned)) { - nonempty_sizes++; - } - } - out->printf("------------------------------------------------\n"); - out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n", - nonempty_sizes, MiB(stats_.free_bytes), - MiB(stats_.unmapped_bytes)); - out->printf("------------------------------------------------\n"); - uint64_t total_normal = 0; - uint64_t total_returned = 0; - for (int s = 0; s < kMaxPages; s++) { - const int n_length = DLL_Length(&free_[s].normal); - const int r_length = DLL_Length(&free_[s].returned); - if (n_length + r_length > 0) { - uint64_t n_pages = s * n_length; - uint64_t r_pages = s * r_length; - total_normal += n_pages; - total_returned += r_pages; - out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum" - "; unmapped: %6.1f MiB; %6.1f MiB cum\n", - s, - (n_length + r_length), - PagesToMiB(n_pages + r_pages), - PagesToMiB(total_normal + total_returned), - PagesToMiB(r_pages), - PagesToMiB(total_returned)); - } - } - - uint64_t n_pages = 0; - uint64_t r_pages = 0; - int n_spans = 0; - int r_spans = 0; - out->printf("Normal large spans:\n"); +void PageHeap::GetLargeSpanStats(LargeSpanStats* result) { + result->spans = 0; + result->normal_pages = 0; + result->returned_pages = 0; for (Span* s = large_.normal.next; s != &large_.normal; s = s->next) { - out->printf(" [ %6" PRIuPTR " pages ] %6.1f MiB\n", - s->length, PagesToMiB(s->length)); - n_pages += s->length; - n_spans++; + result->normal_pages += s->length;; + result->spans++; } - out->printf("Unmapped large spans:\n"); for (Span* s = large_.returned.next; s != &large_.returned; s = s->next) { - out->printf(" [ %6" PRIuPTR " pages ] %6.1f MiB\n", - s->length, PagesToMiB(s->length)); - r_pages += s->length; - r_spans++; + result->returned_pages += s->length; + result->spans++; } - total_normal += n_pages; - total_returned += r_pages; - out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum" - "; unmapped: %6.1f MiB; %6.1f MiB cum\n", - (n_spans + r_spans), - PagesToMiB(n_pages + r_pages), - PagesToMiB(total_normal + total_returned), - PagesToMiB(r_pages), - PagesToMiB(total_returned)); } bool PageHeap::GetNextRange(PageID start, base::MallocRange* r) { diff --git a/third_party/tcmalloc/chromium/src/page_heap.h b/third_party/tcmalloc/chromium/src/page_heap.h index ab7a541..9376a66 100644 --- a/third_party/tcmalloc/chromium/src/page_heap.h +++ b/third_party/tcmalloc/chromium/src/page_heap.h @@ -38,7 +38,7 @@ #ifdef HAVE_STDINT_H #include <stdint.h> // for uint64_t, int64_t, uint16_t #endif -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> #include "base/basictypes.h" #include "common.h" #include "packed-cache-inl.h" @@ -61,10 +61,9 @@ // We use #define so code compiles even if you #include stacktrace.h somehow. # define GetStackTrace(stack, depth, skip) (0) #else -# include <google/stacktrace.h> +# include <gperftools/stacktrace.h> #endif -class TCMalloc_Printer; namespace base { struct MallocRange; } @@ -145,9 +144,6 @@ class PERFTOOLS_DLL_DECL PageHeap { return reinterpret_cast<Span*>(pagemap_.get(p)); } - // Dump state to stderr - void Dump(TCMalloc_Printer* out); - // If this page heap is managing a range with starting page # >= start, // store info about the range in *r and return true. Else return false. bool GetNextRange(PageID start, base::MallocRange* r); @@ -162,10 +158,22 @@ class PERFTOOLS_DLL_DECL PageHeap { }; inline Stats stats() const { return stats_; } - void GetClassSizes(int64 class_sizes_normal[kMaxPages], - int64 class_sizes_returned[kMaxPages], - int64* normal_pages_in_spans, - int64* returned_pages_in_spans); + + struct SmallSpanStats { + // For each free list of small spans, the length (in spans) of the + // normal and returned free lists for that size. + int64 normal_length[kMaxPages]; + int64 returned_length[kMaxPages]; + }; + void GetSmallSpanStats(SmallSpanStats* result); + + // Stats for free large spans (i.e., spans with more than kMaxPages pages). + struct LargeSpanStats { + int64 spans; // Number of such spans + int64 normal_pages; // Combined page length of normal large spans + int64 returned_pages; // Combined page length of unmapped spans + }; + void GetLargeSpanStats(LargeSpanStats* result); bool Check(); // Like Check() but does some more comprehensive checking. diff --git a/third_party/tcmalloc/chromium/src/page_heap_allocator.h b/third_party/tcmalloc/chromium/src/page_heap_allocator.h index eee1590..3595b95 100644 --- a/third_party/tcmalloc/chromium/src/page_heap_allocator.h +++ b/third_party/tcmalloc/chromium/src/page_heap_allocator.h @@ -37,7 +37,7 @@ #include "common.h" // for MetaDataAlloc #include "free_list.h" // for FL_Push/FL_Pop -#include "internal_logging.h" // for ASSERT, CRASH +#include "internal_logging.h" // for ASSERT #include "system-alloc.h" // for TCMalloc_SystemAddGuard namespace tcmalloc { @@ -71,9 +71,10 @@ class PageHeapAllocator { // suitably aligned memory. free_area_ = reinterpret_cast<char*>(MetaDataAlloc(kAllocIncrement)); if (free_area_ == NULL) { - CRASH("FATAL ERROR: Out of memory trying to allocate internal " - "tcmalloc data (%d bytes, object-size %d)\n", - kAllocIncrement, static_cast<int>(sizeof(T))); + Log(kCrash, __FILE__, __LINE__, + "FATAL ERROR: Out of memory trying to allocate internal " + "tcmalloc data (bytes, object-size)", + kAllocIncrement, sizeof(T)); } // This guard page protects the metadata from being corrupted by a @@ -85,9 +86,10 @@ class PageHeapAllocator { free_area_ += guard_size; free_avail_ = kAllocIncrement - guard_size; if (free_avail_ < sizeof(T)) { - CRASH("FATAL ERROR: Insufficient memory to guard internal tcmalloc " - "data (%d bytes, object-size %d, guard-size %d)\n", - kAllocIncrement, static_cast<int>(sizeof(T)), guard_size); + Log(kCrash, __FILE__, __LINE__, + "FATAL ERROR: Insufficient memory to guard internal tcmalloc " + "data (%d bytes, object-size %d, guard-size %d)\n", + kAllocIncrement, static_cast<int>(sizeof(T)), guard_size); } } result = free_area_; diff --git a/third_party/tcmalloc/chromium/src/pprof b/third_party/tcmalloc/chromium/src/pprof index 03bafa4..727eb43 100755 --- a/third_party/tcmalloc/chromium/src/pprof +++ b/third_party/tcmalloc/chromium/src/pprof @@ -72,7 +72,7 @@ use strict; use warnings; use Getopt::Long; -my $PPROF_VERSION = "1.7"; +my $PPROF_VERSION = "2.0"; # These are the object tools we use which can come from a # user-specified location using --tools, from the PPROF_TOOLS @@ -87,13 +87,14 @@ my %obj_tool_map = ( #"addr2line_pdb" => "addr2line-pdb", # ditto #"otool" => "otool", # equivalent of objdump on OS X ); -my $DOT = "dot"; # leave non-absolute, since it may be in /usr/local -my $GV = "gv"; -my $EVINCE = "evince"; # could also be xpdf or perhaps acroread -my $KCACHEGRIND = "kcachegrind"; -my $PS2PDF = "ps2pdf"; +# NOTE: these are lists, so you can put in commandline flags if you want. +my @DOT = ("dot"); # leave non-absolute, since it may be in /usr/local +my @GV = ("gv"); +my @EVINCE = ("evince"); # could also be xpdf or perhaps acroread +my @KCACHEGRIND = ("kcachegrind"); +my @PS2PDF = ("ps2pdf"); # These are used for dynamic profiles -my $URL_FETCHER = "curl -s"; +my @URL_FETCHER = ("curl", "-s"); # These are the web pages that servers need to support for dynamic profiles my $HEAP_PAGE = "/pprof/heap"; @@ -104,7 +105,10 @@ my $GROWTH_PAGE = "/pprof/growth"; my $CONTENTION_PAGE = "/pprof/contention"; my $WALL_PAGE = "/pprof/wall(?:\\?.*)?"; # accepts options like namefilter my $FILTEREDPROFILE_PAGE = "/pprof/filteredprofile(?:\\?.*)?"; -my $CENSUSPROFILE_PAGE = "/pprof/censusprofile"; # must support "?seconds=#" +my $CENSUSPROFILE_PAGE = "/pprof/censusprofile(?:\\?.*)?"; # must support cgi-param + # "?seconds=#", + # "?tags_regexp=#" and + # "?type=#". my $SYMBOL_PAGE = "/pprof/symbol"; # must support symbol lookup via POST my $PROGRAM_NAME_PAGE = "/pprof/cmdline"; @@ -156,7 +160,8 @@ pprof [options] <profile> The /<service> can be $HEAP_PAGE, $PROFILE_PAGE, /pprof/pmuprofile, $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, $CENSUSPROFILE_PAGE, or /pprof/filteredprofile. - For instance: "pprof http://myserver.com:80$HEAP_PAGE". + For instance: + pprof http://myserver.com:80$HEAP_PAGE If /<service> is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). pprof --symbols <program> Maps addresses to symbol names. In this mode, stdin should be a @@ -167,7 +172,7 @@ pprof --symbols <program> For more help with querying remote servers, including how to add the necessary server-side support code, see this filename (or one like it): - /usr/doc/google-perftools-$PPROF_VERSION/pprof_remote_servers.html + /usr/doc/gperftools-$PPROF_VERSION/pprof_remote_servers.html Options: --cum Sort by cumulative data @@ -265,7 +270,7 @@ EOF sub version_string { return <<EOF -pprof (part of google-perftools $PPROF_VERSION) +pprof (part of gperftools $PPROF_VERSION) Copyright 1998-2007 Google Inc. @@ -497,11 +502,13 @@ sub Init() { @main::pfile_args = (); # Remote profiling without a binary (using $SYMBOL_PAGE instead) - if (IsProfileURL($ARGV[0])) { - $main::use_symbol_page = 1; - } elsif (IsSymbolizedProfileFile($ARGV[0])) { - $main::use_symbolized_profile = 1; - $main::prog = $UNKNOWN_BINARY; # will be set later from the profile file + if (@ARGV > 0) { + if (IsProfileURL($ARGV[0])) { + $main::use_symbol_page = 1; + } elsif (IsSymbolizedProfileFile($ARGV[0])) { + $main::use_symbolized_profile = 1; + $main::prog = $UNKNOWN_BINARY; # will be set later from the profile file + } } if ($main::use_symbol_page || $main::use_symbolized_profile) { @@ -545,7 +552,7 @@ sub Init() { ConfigureObjTools($main::prog) } - # Break the opt_list_prefix into the prefix_list array + # Break the opt_lib_prefix into the prefix_list array @prefix_list = split (',', $main::opt_lib_prefix); # Remove trailing / from the prefixes, in the list to prevent @@ -643,7 +650,7 @@ sub Main() { if ($main::opt_disasm) { PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm); } elsif ($main::opt_list) { - PrintListing($libs, $flat, $cumulative, $main::opt_list); + PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); } elsif ($main::opt_text) { # Make sure the output is empty when have nothing to report # (only matters when --heapcheck is given but we must be @@ -661,7 +668,7 @@ sub Main() { if ($main::opt_gv) { RunGV(TempName($main::next_tmpfile, "ps"), ""); } elsif ($main::opt_evince) { - RunEvince(TempName($main::next_tmpfile, "pdf"), ""); + RunEvince(TempName($main::next_tmpfile, "pdf"), ""); } elsif ($main::opt_web) { my $tmp = TempName($main::next_tmpfile, "svg"); RunWeb($tmp); @@ -710,24 +717,25 @@ sub ReadlineMightFail { sub RunGV { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - if (!system("$GV --version >$dev_null 2>&1")) { + if (!system(ShellEscape(@GV, "--version") . " >$dev_null 2>&1")) { # Options using double dash are supported by this gv version. # Also, turn on noantialias to better handle bug in gv for # postscript files with large dimensions. # TODO: Maybe we should not pass the --noantialias flag # if the gv version is known to work properly without the flag. - system("$GV --scale=$main::opt_scale --noantialias " . $fname . $bg); + system(ShellEscape(@GV, "--scale=$main::opt_scale", "--noantialias", $fname) + . $bg); } else { # Old gv version - only supports options that use single dash. - print STDERR "$GV -scale $main::opt_scale\n"; - system("$GV -scale $main::opt_scale " . $fname . $bg); + print STDERR ShellEscape(@GV, "-scale", $main::opt_scale) . "\n"; + system(ShellEscape(@GV, "-scale", "$main::opt_scale", $fname) . $bg); } } sub RunEvince { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - system("$EVINCE " . $fname . $bg); + system(ShellEscape(@EVINCE, $fname) . $bg); } sub RunWeb { @@ -761,8 +769,8 @@ sub RunWeb { sub RunKcachegrind { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - print STDERR "Starting '$KCACHEGRIND " . $fname . $bg . "'\n"; - system("$KCACHEGRIND " . $fname . $bg); + print STDERR "Starting '@KCACHEGRIND " . $fname . $bg . "'\n"; + system(ShellEscape(@KCACHEGRIND, $fname) . $bg); } @@ -839,7 +847,7 @@ sub InteractiveCommand { my $ignore; ($routine, $ignore) = ParseInteractiveArgs($3); - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles @@ -866,21 +874,22 @@ sub InteractiveCommand { return 1; } - if (m/^\s*list\s*(.+)/) { + if (m/^\s*(web)?list\s*(.+)/) { + my $html = (defined($1) && ($1 eq "web")); $main::opt_list = 1; my $routine; my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($1); + ($routine, $ignore) = ParseInteractiveArgs($2); - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles my $flat = FlatProfile($reduced); my $cumulative = CumulativeProfile($reduced); - PrintListing($libs, $flat, $cumulative, $routine); + PrintListing($total, $libs, $flat, $cumulative, $routine, $html); return 1; } if (m/^\s*disasm\s*(.+)/) { @@ -891,7 +900,7 @@ sub InteractiveCommand { ($routine, $ignore) = ParseInteractiveArgs($1); # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles @@ -918,7 +927,8 @@ sub InteractiveCommand { ($focus, $ignore) = ParseInteractiveArgs($2); # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, $focus, $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, + $focus, $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles @@ -946,6 +956,7 @@ sub InteractiveCommand { sub ProcessProfile { + my $total_count = shift; my $orig_profile = shift; my $symbols = shift; my $focus = shift; @@ -953,7 +964,6 @@ sub ProcessProfile { # Process current profile to account for various settings my $profile = $orig_profile; - my $total_count = TotalProfile($profile); printf("Total: %s %s\n", Unparse($total_count), Units()); if ($focus ne '') { $profile = FocusProfile($symbols, $profile, $focus); @@ -1000,6 +1010,11 @@ Commands: list [routine_regexp] [-ignore1] [-ignore2] Show source listing of routines whose names match "routine_regexp" + weblist [routine_regexp] [-ignore1] [-ignore2] + Displays a source listing of routines whose names match "routine_regexp" + in a web browser. You can click on source lines to view the + corresponding disassembly. + top [--cum] [-ignore1] [-ignore2] top20 [--cum] [-ignore1] [-ignore2] top37 [--cum] [-ignore1] [-ignore2] @@ -1024,8 +1039,8 @@ parameters will be ignored. Further pprof details are available at this location (or one similar): - /usr/doc/google-perftools-$PPROF_VERSION/cpu_profiler.html - /usr/doc/google-perftools-$PPROF_VERSION/heap_profiler.html + /usr/doc/gperftools-$PPROF_VERSION/cpu_profiler.html + /usr/doc/gperftools-$PPROF_VERSION/heap_profiler.html ENDOFHELP } @@ -1175,7 +1190,29 @@ sub PrintText { $sym); } $lines++; - last if ($line_limit >= 0 && $lines > $line_limit); + last if ($line_limit >= 0 && $lines >= $line_limit); + } +} + +# Callgrind format has a compression for repeated function and file +# names. You show the name the first time, and just use its number +# subsequently. This can cut down the file to about a third or a +# quarter of its uncompressed size. $key and $val are the key/value +# pair that would normally be printed by callgrind; $map is a map from +# value to number. +sub CompressedCGName { + my($key, $val, $map) = @_; + my $idx = $map->{$val}; + # For very short keys, providing an index hurts rather than helps. + if (length($val) <= 3) { + return "$key=$val\n"; + } elsif (defined($idx)) { + return "$key=($idx)\n"; + } else { + # scalar(keys $map) gives the number of items in the map. + $idx = scalar(keys(%{$map})) + 1; + $map->{$val} = $idx; + return "$key=($idx) $val\n"; } } @@ -1183,13 +1220,16 @@ sub PrintText { sub PrintCallgrind { my $calls = shift; my $filename; + my %filename_to_index_map; + my %fnname_to_index_map; + if ($main::opt_interactive) { $filename = shift; print STDERR "Writing callgrind file to '$filename'.\n" } else { $filename = "&STDOUT"; } - open(CG, ">".$filename ); + open(CG, ">$filename"); printf CG ("events: Hits\n\n"); foreach my $call ( map { $_->[0] } sort { $a->[1] cmp $b ->[1] || @@ -1203,11 +1243,14 @@ sub PrintCallgrind { $callee_file, $callee_line, $callee_function ) = ( $1, $2, $3, $5, $6, $7 ); - - printf CG ("fl=$caller_file\nfn=$caller_function\n"); + # TODO(csilvers): for better compression, collect all the + # caller/callee_files and functions first, before printing + # anything, and only compress those referenced more than once. + printf CG CompressedCGName("fl", $caller_file, \%filename_to_index_map); + printf CG CompressedCGName("fn", $caller_function, \%fnname_to_index_map); if (defined $6) { - printf CG ("cfl=$callee_file\n"); - printf CG ("cfn=$callee_function\n"); + printf CG CompressedCGName("cfl", $callee_file, \%filename_to_index_map); + printf CG CompressedCGName("cfn", $callee_function, \%fnname_to_index_map); printf CG ("calls=$count $callee_line\n"); } printf CG ("$caller_line $count\n\n"); @@ -1256,10 +1299,10 @@ sub Disassemble { my $end_addr = shift; my $objdump = $obj_tool_map{"objdump"}; - my $cmd = sprintf("$objdump -C -d -l --no-show-raw-insn " . - "--start-address=0x$start_addr " . - "--stop-address=0x$end_addr $prog"); - open(OBJDUMP, "$cmd |") || error("$objdump: $!\n"); + my $cmd = ShellEscape($objdump, "-C", "-d", "-l", "--no-show-raw-insn", + "--start-address=0x$start_addr", + "--stop-address=0x$end_addr", $prog); + open(OBJDUMP, "$cmd |") || error("$cmd: $!\n"); my @result = (); my $filename = ""; my $linenumber = -1; @@ -1322,13 +1365,33 @@ sub ByName { return ShortFunctionName($a) cmp ShortFunctionName($b); } -# Print source-listing for all all routines that match $main::opt_list +# Print source-listing for all all routines that match $list_opts sub PrintListing { + my $total = shift; my $libs = shift; my $flat = shift; my $cumulative = shift; my $list_opts = shift; + my $html = shift; + + my $output = \*STDOUT; + my $fname = ""; + + if ($html) { + # Arrange to write the output to a temporary file + $fname = TempName($main::next_tmpfile, "html"); + $main::next_tmpfile++; + if (!open(TEMP, ">$fname")) { + print STDERR "$fname: $!\n"; + return; + } + $output = \*TEMP; + print $output HtmlListingHeader(); + printf $output ("<div class=\"legend\">%s<br>Total: %s %s</div>\n", + $main::prog, Unparse($total), Units()); + } + my $listed = 0; foreach my $lib (@{$libs}) { my $symbol_table = GetProcedureBoundaries($lib->[0], $list_opts); my $offset = AddressSub($lib->[1], $lib->[3]); @@ -1340,15 +1403,113 @@ sub PrintListing { my $addr = AddressAdd($start_addr, $offset); for (my $i = 0; $i < $length; $i++) { if (defined($cumulative->{$addr})) { - PrintSource($lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr); + $listed += PrintSource( + $lib->[0], $offset, + $routine, $flat, $cumulative, + $start_addr, $end_addr, + $html, + $output); last; } $addr = AddressInc($addr); } } } + + if ($html) { + if ($listed > 0) { + print $output HtmlListingFooter(); + close($output); + RunWeb($fname); + } else { + close($output); + unlink($fname); + } + } +} + +sub HtmlListingHeader { + return <<'EOF'; +<DOCTYPE html> +<html> +<head> +<title>Pprof listing</title> +<style type="text/css"> +body { + font-family: sans-serif; +} +h1 { + font-size: 1.5em; + margin-bottom: 4px; +} +.legend { + font-size: 1.25em; +} +.line { + color: #aaaaaa; +} +.nop { + color: #aaaaaa; +} +.unimportant { + color: #cccccc; +} +.disasmloc { + color: #000000; +} +.deadsrc { + cursor: pointer; +} +.deadsrc:hover { + background-color: #eeeeee; +} +.livesrc { + color: #0000ff; + cursor: pointer; +} +.livesrc:hover { + background-color: #eeeeee; +} +.asm { + color: #008800; + display: none; +} +</style> +<script type="text/javascript"> +function pprof_toggle_asm(e) { + var target; + if (!e) e = window.event; + if (e.target) target = e.target; + else if (e.srcElement) target = e.srcElement; + + if (target) { + var asm = target.nextSibling; + if (asm && asm.className == "asm") { + asm.style.display = (asm.style.display == "block" ? "" : "block"); + e.preventDefault(); + return false; + } + } +} +</script> +</head> +<body> +EOF +} + +sub HtmlListingFooter { + return <<'EOF'; +</body> +</html> +EOF +} + +sub HtmlEscape { + my $text = shift; + $text =~ s/&/&/g; + $text =~ s/</</g; + $text =~ s/>/>/g; + return $text; } # Returns the indentation of the line, if it has any non-whitespace @@ -1362,6 +1523,45 @@ sub Indentation { } } +# If the symbol table contains inlining info, Disassemble() may tag an +# instruction with a location inside an inlined function. But for +# source listings, we prefer to use the location in the function we +# are listing. So use MapToSymbols() to fetch full location +# information for each instruction and then pick out the first +# location from a location list (location list contains callers before +# callees in case of inlining). +# +# After this routine has run, each entry in $instructions contains: +# [0] start address +# [1] filename for function we are listing +# [2] line number for function we are listing +# [3] disassembly +# [4] limit address +# [5] most specific filename (may be different from [1] due to inlining) +# [6] most specific line number (may be different from [2] due to inlining) +sub GetTopLevelLineNumbers { + my ($lib, $offset, $instructions) = @_; + my $pcs = []; + for (my $i = 0; $i <= $#{$instructions}; $i++) { + push(@{$pcs}, $instructions->[$i]->[0]); + } + my $symbols = {}; + MapToSymbols($lib, $offset, $pcs, $symbols); + for (my $i = 0; $i <= $#{$instructions}; $i++) { + my $e = $instructions->[$i]; + push(@{$e}, $e->[1]); + push(@{$e}, $e->[2]); + my $addr = $e->[0]; + my $sym = $symbols->{$addr}; + if (defined($sym)) { + if ($#{$sym} >= 2 && $sym->[1] =~ m/^(.*):(\d+)$/) { + $e->[1] = $1; # File name + $e->[2] = $2; # Line number + } + } + } +} + # Print source-listing for one routine sub PrintSource { my $prog = shift; @@ -1371,9 +1571,12 @@ sub PrintSource { my $cumulative = shift; my $start_addr = shift; my $end_addr = shift; + my $html = shift; + my $output = shift; # Disassemble all instructions (just to get line numbers) my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); + GetTopLevelLineNumbers($prog, $offset, \@instructions); # Hack 1: assume that the first source file encountered in the # disassembly contains the routine @@ -1386,7 +1589,7 @@ sub PrintSource { } if (!defined($filename)) { print STDERR "no filename found in $routine\n"; - return; + return 0; } # Hack 2: assume that the largest line number from $filename is the @@ -1419,7 +1622,7 @@ sub PrintSource { { if (!open(FILE, "<$filename")) { print STDERR "$filename: $!\n"; - return; + return 0; } my $l = 0; my $first_indentation = -1; @@ -1447,12 +1650,24 @@ sub PrintSource { # Assign all samples to the range $firstline,$lastline, # Hack 4: If an instruction does not occur in the range, its samples # are moved to the next instruction that occurs in the range. - my $samples1 = {}; - my $samples2 = {}; - my $running1 = 0; # Unassigned flat counts - my $running2 = 0; # Unassigned cumulative counts - my $total1 = 0; # Total flat counts - my $total2 = 0; # Total cumulative counts + my $samples1 = {}; # Map from line number to flat count + my $samples2 = {}; # Map from line number to cumulative count + my $running1 = 0; # Unassigned flat counts + my $running2 = 0; # Unassigned cumulative counts + my $total1 = 0; # Total flat counts + my $total2 = 0; # Total cumulative counts + my %disasm = (); # Map from line number to disassembly + my $running_disasm = ""; # Unassigned disassembly + my $skip_marker = "---\n"; + if ($html) { + $skip_marker = ""; + for (my $l = $firstline; $l <= $lastline; $l++) { + $disasm{$l} = ""; + } + } + my $last_dis_filename = ''; + my $last_dis_linenum = -1; + my $last_touched_line = -1; # To detect gaps in disassembly for a line foreach my $e (@instructions) { # Add up counts for all address that fall inside this instruction my $c1 = 0; @@ -1461,6 +1676,38 @@ sub PrintSource { $c1 += GetEntry($flat, $a); $c2 += GetEntry($cumulative, $a); } + + if ($html) { + my $dis = sprintf(" %6s %6s \t\t%8s: %s ", + HtmlPrintNumber($c1), + HtmlPrintNumber($c2), + UnparseAddress($offset, $e->[0]), + CleanDisassembly($e->[3])); + + # Append the most specific source line associated with this instruction + if (length($dis) < 80) { $dis .= (' ' x (80 - length($dis))) }; + $dis = HtmlEscape($dis); + my $f = $e->[5]; + my $l = $e->[6]; + if ($f ne $last_dis_filename) { + $dis .= sprintf("<span class=disasmloc>%s:%d</span>", + HtmlEscape(CleanFileName($f)), $l); + } elsif ($l ne $last_dis_linenum) { + # De-emphasize the unchanged file name portion + $dis .= sprintf("<span class=unimportant>%s</span>" . + "<span class=disasmloc>:%d</span>", + HtmlEscape(CleanFileName($f)), $l); + } else { + # De-emphasize the entire location + $dis .= sprintf("<span class=unimportant>%s:%d</span>", + HtmlEscape(CleanFileName($f)), $l); + } + $last_dis_filename = $f; + $last_dis_linenum = $l; + $running_disasm .= $dis; + $running_disasm .= "\n"; + } + $running1 += $c1; $running2 += $c2; $total1 += $c1; @@ -1475,23 +1722,49 @@ sub PrintSource { AddEntry($samples2, $line, $running2); $running1 = 0; $running2 = 0; + if ($html) { + if ($line != $last_touched_line && $disasm{$line} ne '') { + $disasm{$line} .= "\n"; + } + $disasm{$line} .= $running_disasm; + $running_disasm = ''; + $last_touched_line = $line; + } } } # Assign any leftover samples to $lastline AddEntry($samples1, $lastline, $running1); AddEntry($samples2, $lastline, $running2); - - printf("ROUTINE ====================== %s in %s\n" . - "%6s %6s Total %s (flat / cumulative)\n", - ShortFunctionName($routine), - $filename, - Units(), - Unparse($total1), - Unparse($total2)); + if ($html) { + if ($lastline != $last_touched_line && $disasm{$lastline} ne '') { + $disasm{$lastline} .= "\n"; + } + $disasm{$lastline} .= $running_disasm; + } + + if ($html) { + printf $output ( + "<h1>%s</h1>%s\n<pre onClick=\"pprof_toggle_asm()\">\n" . + "Total:%6s %6s (flat / cumulative %s)\n", + HtmlEscape(ShortFunctionName($routine)), + HtmlEscape(CleanFileName($filename)), + Unparse($total1), + Unparse($total2), + Units()); + } else { + printf $output ( + "ROUTINE ====================== %s in %s\n" . + "%6s %6s Total %s (flat / cumulative)\n", + ShortFunctionName($routine), + CleanFileName($filename), + Unparse($total1), + Unparse($total2), + Units()); + } if (!open(FILE, "<$filename")) { print STDERR "$filename: $!\n"; - return; + return 0; } my $l = 0; while (<FILE>) { @@ -1501,16 +1774,47 @@ sub PrintSource { (($l <= $oldlastline + 5) || ($l <= $lastline))) { chop; my $text = $_; - if ($l == $firstline) { printf("---\n"); } - printf("%6s %6s %4d: %s\n", - UnparseAlt(GetEntry($samples1, $l)), - UnparseAlt(GetEntry($samples2, $l)), - $l, - $text); - if ($l == $lastline) { printf("---\n"); } + if ($l == $firstline) { print $output $skip_marker; } + my $n1 = GetEntry($samples1, $l); + my $n2 = GetEntry($samples2, $l); + if ($html) { + # Emit a span that has one of the following classes: + # livesrc -- has samples + # deadsrc -- has disassembly, but with no samples + # nop -- has no matching disasembly + # Also emit an optional span containing disassembly. + my $dis = $disasm{$l}; + my $asm = ""; + if (defined($dis) && $dis ne '') { + $asm = "<span class=\"asm\">" . $dis . "</span>"; + } + my $source_class = (($n1 + $n2 > 0) + ? "livesrc" + : (($asm ne "") ? "deadsrc" : "nop")); + printf $output ( + "<span class=\"line\">%5d</span> " . + "<span class=\"%s\">%6s %6s %s</span>%s\n", + $l, $source_class, + HtmlPrintNumber($n1), + HtmlPrintNumber($n2), + HtmlEscape($text), + $asm); + } else { + printf $output( + "%6s %6s %4d: %s\n", + UnparseAlt($n1), + UnparseAlt($n2), + $l, + $text); + } + if ($l == $lastline) { print $output $skip_marker; } }; } close(FILE); + if ($html) { + print $output "</pre>\n"; + } + return 1; } # Return the source line for the specified file/linenumber. @@ -1653,21 +1957,11 @@ sub PrintDisassembledFunction { # Print disassembly for (my $x = $first_inst; $x <= $last_inst; $x++) { my $e = $instructions[$x]; - my $address = $e->[0]; - $address = AddressSub($address, $offset); # Make relative to section - $address =~ s/^0x//; - $address =~ s/^0*//; - - # Trim symbols - my $d = $e->[3]; - while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) - while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments - printf("%6s %6s %8s: %6s\n", UnparseAlt($flat_count[$x]), UnparseAlt($cum_count[$x]), - $address, - $d); + UnparseAddress($offset, $e->[0]), + CleanDisassembly($e->[3])); } } } @@ -1713,19 +2007,24 @@ sub PrintDot { # Open DOT output file my $output; + my $escaped_dot = ShellEscape(@DOT); + my $escaped_ps2pdf = ShellEscape(@PS2PDF); if ($main::opt_gv) { - $output = "| $DOT -Tps2 >" . TempName($main::next_tmpfile, "ps"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "ps")); + $output = "| $escaped_dot -Tps2 >$escaped_outfile"; } elsif ($main::opt_evince) { - $output = "| $DOT -Tps2 | $PS2PDF - " . TempName($main::next_tmpfile, "pdf"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "pdf")); + $output = "| $escaped_dot -Tps2 | $escaped_ps2pdf - $escaped_outfile"; } elsif ($main::opt_ps) { - $output = "| $DOT -Tps2"; + $output = "| $escaped_dot -Tps2"; } elsif ($main::opt_pdf) { - $output = "| $DOT -Tps2 | $PS2PDF - -"; + $output = "| $escaped_dot -Tps2 | $escaped_ps2pdf - -"; } elsif ($main::opt_web || $main::opt_svg) { # We need to post-process the SVG, so write to a temporary file always. - $output = "| $DOT -Tsvg >" . TempName($main::next_tmpfile, "svg"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "svg")); + $output = "| $escaped_dot -Tsvg >$escaped_outfile"; } elsif ($main::opt_gif) { - $output = "| $DOT -Tgif"; + $output = "| $escaped_dot -Tgif"; } else { $output = ">&STDOUT"; } @@ -1806,10 +2105,12 @@ sub PrintDot { # Get edges and counts per edge my %edge = (); my $n; + my $fullname_to_shortname_map = {}; + FillFullnameToShortnameMap($symbols, $fullname_to_shortname_map); foreach my $k (keys(%{$raw})) { # TODO: omit low %age edges $n = $raw->{$k}; - my @translated = TranslateStack($symbols, $k); + my @translated = TranslateStack($symbols, $fullname_to_shortname_map, $k); for (my $i = 1; $i <= $#translated; $i++) { my $src = $translated[$i]; my $dst = $translated[$i-1]; @@ -2193,6 +2494,50 @@ function handleMouseUp(evt) { EOF } +# Provides a map from fullname to shortname for cases where the +# shortname is ambiguous. The symlist has both the fullname and +# shortname for all symbols, which is usually fine, but sometimes -- +# such as overloaded functions -- two different fullnames can map to +# the same shortname. In that case, we use the address of the +# function to disambiguate the two. This function fills in a map that +# maps fullnames to modified shortnames in such cases. If a fullname +# is not present in the map, the 'normal' shortname provided by the +# symlist is the appropriate one to use. +sub FillFullnameToShortnameMap { + my $symbols = shift; + my $fullname_to_shortname_map = shift; + my $shortnames_seen_once = {}; + my $shortnames_seen_more_than_once = {}; + + foreach my $symlist (values(%{$symbols})) { + # TODO(csilvers): deal with inlined symbols too. + my $shortname = $symlist->[0]; + my $fullname = $symlist->[2]; + if ($fullname !~ /<[0-9a-fA-F]+>$/) { # fullname doesn't end in an address + next; # the only collisions we care about are when addresses differ + } + if (defined($shortnames_seen_once->{$shortname}) && + $shortnames_seen_once->{$shortname} ne $fullname) { + $shortnames_seen_more_than_once->{$shortname} = 1; + } else { + $shortnames_seen_once->{$shortname} = $fullname; + } + } + + foreach my $symlist (values(%{$symbols})) { + my $shortname = $symlist->[0]; + my $fullname = $symlist->[2]; + # TODO(csilvers): take in a list of addresses we care about, and only + # store in the map if $symlist->[1] is in that list. Saves space. + next if defined($fullname_to_shortname_map->{$fullname}); + if (defined($shortnames_seen_more_than_once->{$shortname})) { + if ($fullname =~ /<0*([^>]*)>$/) { # fullname has address at end of it + $fullname_to_shortname_map->{$fullname} = "$shortname\@$1"; + } + } + } +} + # Return a small number that identifies the argument. # Multiple calls with the same argument will return the same number. # Calls with different arguments will return different numbers. @@ -2209,6 +2554,7 @@ sub ShortIdFor { # Translate a stack of addresses into a stack of symbols sub TranslateStack { my $symbols = shift; + my $fullname_to_shortname_map = shift; my $k = shift; my @addrs = split(/\n/, $k); @@ -2240,6 +2586,9 @@ sub TranslateStack { my $func = $symlist->[$j-2]; my $fileline = $symlist->[$j-1]; my $fullfunc = $symlist->[$j]; + if (defined($fullname_to_shortname_map->{$fullfunc})) { + $func = $fullname_to_shortname_map->{$fullfunc}; + } if ($j > 2) { $func = "$func (inline)"; } @@ -2326,6 +2675,16 @@ sub UnparseAlt { } } +# Alternate pretty-printed form: 0 maps to "" +sub HtmlPrintNumber { + my $num = shift; + if ($num == 0) { + return ""; + } else { + return Unparse($num); + } +} + # Return output units sub Units { if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { @@ -2482,6 +2841,13 @@ sub RemoveUninterestingFrames { '__builtin_vec_new', 'operator new', 'operator new[]', + # The entry to our memory-allocation routines on OS X + 'malloc_zone_malloc', + 'malloc_zone_calloc', + 'malloc_zone_valloc', + 'malloc_zone_realloc', + 'malloc_zone_memalign', + 'malloc_zone_free', # These mark the beginning/end of our custom sections '__start_google_malloc', '__stop_google_malloc', @@ -2573,9 +2939,11 @@ sub ReduceProfile { my $symbols = shift; my $profile = shift; my $result = {}; + my $fullname_to_shortname_map = {}; + FillFullnameToShortnameMap($symbols, $fullname_to_shortname_map); foreach my $k (keys(%{$profile})) { my $count = $profile->{$k}; - my @translated = TranslateStack($symbols, $k); + my @translated = TranslateStack($symbols, $fullname_to_shortname_map, $k); my @path = (); my %seen = (); $seen{''} = 1; # So that empty keys are skipped @@ -2782,7 +3150,8 @@ sub AddEntries { sub CheckSymbolPage { my $url = SymbolPageURL(); - open(SYMBOL, "$URL_FETCHER '$url' |"); + my $command = ShellEscape(@URL_FETCHER, $url); + open(SYMBOL, "$command |") or error($command); my $line = <SYMBOL>; $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines close(SYMBOL); @@ -2839,7 +3208,7 @@ sub SymbolPageURL { sub FetchProgramName() { my ($host, $baseURL, $path) = ParseProfileURL($main::pfile_args[0]); my $url = "$baseURL$PROGRAM_NAME_PAGE"; - my $command_line = "$URL_FETCHER '$url'"; + my $command_line = ShellEscape(@URL_FETCHER, $url); open(CMDLINE, "$command_line |") or error($command_line); my $cmdline = <CMDLINE>; $cmdline =~ s/\r//g; # turn windows-looking lines into unix-looking lines @@ -2856,7 +3225,7 @@ sub FetchProgramName() { # curl. Redirection happens on borg hosts. sub ResolveRedirectionForCurl { my $url = shift; - my $command_line = "$URL_FETCHER --head '$url'"; + my $command_line = ShellEscape(@URL_FETCHER, "--head", $url); open(CMDLINE, "$command_line |") or error($command_line); while (<CMDLINE>) { s/\r//g; # turn windows-looking lines into unix-looking lines @@ -2868,18 +3237,18 @@ sub ResolveRedirectionForCurl { return $url; } -# Add a timeout flat to URL_FETCHER +# Add a timeout flat to URL_FETCHER. Returns a new list. sub AddFetchTimeout { - my $fetcher = shift; my $timeout = shift; + my @fetcher = shift; if (defined($timeout)) { - if ($fetcher =~ m/\bcurl -s/) { - $fetcher .= sprintf(" --max-time %d", $timeout); - } elsif ($fetcher =~ m/\brpcget\b/) { - $fetcher .= sprintf(" --deadline=%d", $timeout); + if (join(" ", @fetcher) =~ m/\bcurl -s/) { + push(@fetcher, "--max-time", sprintf("%d", $timeout)); + } elsif (join(" ", @fetcher) =~ m/\brpcget\b/) { + push(@fetcher, sprintf("--deadline=%d", $timeout)); } } - return $fetcher; + return @fetcher; } # Reads a symbol map from the file handle name given as $1, returning @@ -2939,15 +3308,17 @@ sub FetchSymbols { my $url = SymbolPageURL(); my $command_line; - if ($URL_FETCHER =~ m/\bcurl -s/) { + if (join(" ", @URL_FETCHER) =~ m/\bcurl -s/) { $url = ResolveRedirectionForCurl($url); - $command_line = "$URL_FETCHER -d '\@$main::tmpfile_sym' '$url'"; + $command_line = ShellEscape(@URL_FETCHER, "-d", "\@$main::tmpfile_sym", + $url); } else { - $command_line = "$URL_FETCHER --post '$url' < '$main::tmpfile_sym'"; + $command_line = (ShellEscape(@URL_FETCHER, "--post", $url) + . " < " . ShellEscape($main::tmpfile_sym)); } # We use c++filt in case $SYMBOL_PAGE gives us mangled symbols. - my $cppfilt = $obj_tool_map{"c++filt"}; - open(SYMBOL, "$command_line | $cppfilt |") or error($command_line); + my $escaped_cppfilt = ShellEscape($obj_tool_map{"c++filt"}); + open(SYMBOL, "$command_line | $escaped_cppfilt |") or error($command_line); $symbol_map = ReadSymbols(*SYMBOL{IO}); close(SYMBOL); } @@ -2963,8 +3334,8 @@ sub FetchSymbols { my $shortpc = $pc; $shortpc =~ s/^0*//; # Each line may have a list of names, which includes the function - # and also other functions it has inlined. They are separated - # (in PrintSymbolizedFile), by --, which is illegal in function names. + # and also other functions it has inlined. They are separated (in + # PrintSymbolizedProfile), by --, which is illegal in function names. my $fullnames; if (defined($symbol_map->{$shortpc})) { $fullnames = $symbol_map->{$shortpc}; @@ -3042,8 +3413,8 @@ sub FetchDynamicProfile { return $real_profile; } - my $fetcher = AddFetchTimeout($URL_FETCHER, $fetch_timeout); - my $cmd = "$fetcher '$url' > '$tmp_profile'"; + my @fetcher = AddFetchTimeout($fetch_timeout, @URL_FETCHER); + my $cmd = ShellEscape(@fetcher, $url) . " > " . ShellEscape($tmp_profile); if ($path =~ m/$PROFILE_PAGE|$PMUPROFILE_PAGE|$CENSUSPROFILE_PAGE/){ print STDERR "Gathering CPU profile from $url for $main::opt_seconds seconds to\n ${real_profile}\n"; if ($encourage_patience) { @@ -3054,7 +3425,7 @@ sub FetchDynamicProfile { } (system($cmd) == 0) || error("Failed to get profile: $cmd: $!\n"); - (system("mv $tmp_profile $real_profile") == 0) || error("Unable to rename profile\n"); + (system("mv", $tmp_profile, $real_profile) == 0) || error("Unable to rename profile\n"); print STDERR "Wrote profile to $real_profile\n"; $main::collected_profile = $real_profile; return $main::collected_profile; @@ -3168,7 +3539,7 @@ BEGIN { my $has_q = 0; eval { $has_q = pack("Q", "1") ? 1 : 1; }; if (!$has_q) { - $self->{perl_is_64bit} = 0; + $self->{perl_is_64bit} = 0; } read($self->{file}, $str, 8); if (substr($str, 4, 4) eq chr(0)x4) { @@ -3204,17 +3575,17 @@ BEGIN { # TODO(csilvers): if this is a 32-bit perl, the math below # could end up in a too-large int, which perl will promote # to a double, losing necessary precision. Deal with that. - # Right now, we just die. - my ($lo, $hi) = ($b32_values[$i], $b32_values[$i+1]); + # Right now, we just die. + my ($lo, $hi) = ($b32_values[$i], $b32_values[$i+1]); if ($self->{unpack_code} eq 'N') { # big-endian - ($lo, $hi) = ($hi, $lo); - } - my $value = $lo + $hi * (2**32); - if (!$self->{perl_is_64bit} && # check value is exactly represented - (($value % (2**32)) != $lo || int($value / (2**32)) != $hi)) { - ::error("Need a 64-bit perl to process this 64-bit profile.\n"); - } - push(@b64_values, $value); + ($lo, $hi) = ($hi, $lo); + } + my $value = $lo + $hi * (2**32); + if (!$self->{perl_is_64bit} && # check value is exactly represented + (($value % (2**32)) != $lo || int($value / (2**32)) != $hi)) { + ::error("Need a 64-bit perl to process this 64-bit profile.\n"); + } + push(@b64_values, $value); } @$slots = @b64_values; } @@ -3342,7 +3713,7 @@ sub ReadProfile { if (!$main::use_symbolized_profile) { # we have both a binary and symbolized profiles, abort error("FATAL ERROR: Symbolized profile\n $fname\ncannot be used with " . - "a binary arg. Try again without passing\n $prog\n"); + "a binary arg. Try again without passing\n $prog\n"); } # Read the symbol section of the symbolized profile file. $symbols = ReadSymbols(*PROFILE{IO}); @@ -3643,18 +4014,18 @@ sub ReadHeapProfile { # The sampling frequency is the rate of a Poisson process. # This means that the probability of sampling an allocation of # size X with sampling rate Y is 1 - exp(-X/Y) - if ($n1 != 0) { - my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n1 *= $scale_factor; - $s1 *= $scale_factor; - } - if ($n2 != 0) { - my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n2 *= $scale_factor; - $s2 *= $scale_factor; - } + if ($n1 != 0) { + my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n1 *= $scale_factor; + $s1 *= $scale_factor; + } + if ($n2 != 0) { + my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n2 *= $scale_factor; + $s2 *= $scale_factor; + } } else { # Remote-heap version 1 my $ratio; @@ -3778,19 +4149,19 @@ sub ReadSynchProfile { return $r; } -# Given a hex value in the form "0x1abcd" return "0001abcd" or -# "000000000001abcd", depending on the current address length. -# There's probably a more idiomatic (or faster) way to do this... +# Given a hex value in the form "0x1abcd" or "1abcd", return either +# "0001abcd" or "000000000001abcd", depending on the current (global) +# address length. sub HexExtend { my $addr = shift; - $addr =~ s/^0x//; - - if (length $addr > $address_length) { - printf STDERR "Warning: address $addr is longer than address length $address_length\n"; + $addr =~ s/^(0x)?0*//; + my $zeros_needed = $address_length - length($addr); + if ($zeros_needed < 0) { + printf STDERR "Warning: address $addr is longer than address length $address_length\n"; + return $addr; } - - return substr("000000000000000".$addr, -$address_length); + return ("0" x $zeros_needed) . $addr; } ##### Symbol extraction ##### @@ -3841,9 +4212,8 @@ sub ParseTextSectionHeaderFromObjdump { my $file_offset; # Get objdump output from the library file to figure out how to # map between mapped addresses and addresses in the library. - my $objdump = $obj_tool_map{"objdump"}; - open(OBJDUMP, "$objdump -h $lib |") - || error("$objdump $lib: $!\n"); + my $cmd = ShellEscape($obj_tool_map{"objdump"}, "-h", $lib); + open(OBJDUMP, "$cmd |") || error("$cmd: $!\n"); while (<OBJDUMP>) { s/\r//g; # turn windows-looking lines into unix-looking lines # Idx Name Size VMA LMA File off Algn @@ -3881,9 +4251,8 @@ sub ParseTextSectionHeaderFromOtool { my $file_offset = undef; # Get otool output from the library file to figure out how to # map between mapped addresses and addresses in the library. - my $otool = $obj_tool_map{"otool"}; - open(OTOOL, "$otool -l $lib |") - || error("$otool $lib: $!\n"); + my $command = ShellEscape($obj_tool_map{"otool"}, "-l", $lib); + open(OTOOL, "$command |") || error("$command: $!\n"); my $cmd = ""; my $sectname = ""; my $segname = ""; @@ -4225,18 +4594,18 @@ sub ExtractSymbols { my ($start_pc_index, $finish_pc_index); # Find smallest finish_pc_index such that $finish < $pc[$finish_pc_index]. for ($finish_pc_index = $#pcs + 1; $finish_pc_index > 0; - $finish_pc_index--) { + $finish_pc_index--) { last if $pcs[$finish_pc_index - 1] le $finish; } # Find smallest start_pc_index such that $start <= $pc[$start_pc_index]. for ($start_pc_index = $finish_pc_index; $start_pc_index > 0; - $start_pc_index--) { + $start_pc_index--) { last if $pcs[$start_pc_index - 1] lt $start; } # This keeps PC values higher than $pc[$finish_pc_index] in @pcs, # in case there are overlaps in libraries and the main binary. @{$contained} = splice(@pcs, $start_pc_index, - $finish_pc_index - $start_pc_index); + $finish_pc_index - $start_pc_index); # Map to symbols MapToSymbols($libname, AddressSub($start, $offset), $contained, $symbols); } @@ -4258,15 +4627,15 @@ sub MapToSymbols { # Figure out the addr2line command to use my $addr2line = $obj_tool_map{"addr2line"}; - my $cmd = "$addr2line -f -C -e $image"; + my $cmd = ShellEscape($addr2line, "-f", "-C", "-e", $image); if (exists $obj_tool_map{"addr2line_pdb"}) { $addr2line = $obj_tool_map{"addr2line_pdb"}; - $cmd = "$addr2line --demangle -f -C -e $image"; + $cmd = ShellEscape($addr2line, "--demangle", "-f", "-C", "-e", $image); } # If "addr2line" isn't installed on the system at all, just use # nm to get what info we can (function names, but not line numbers). - if (system("$addr2line --help >$dev_null 2>&1") != 0) { + if (system(ShellEscape($addr2line, "--help") . " >$dev_null 2>&1") != 0) { MapSymbolsWithNM($image, $offset, $pclist, $symbols); return; } @@ -4280,7 +4649,6 @@ sub MapToSymbols { $sep_address = undef; # May be filled in by MapSymbolsWithNM() my $nm_symbols = {}; MapSymbolsWithNM($image, $offset, $pclist, $nm_symbols); - # TODO(csilvers): only add '-i' if addr2line supports it. if (defined($sep_address)) { # Only add " -i" to addr2line if the binary supports it. # addr2line --help returns 0, but not if it sees an unknown flag first. @@ -4306,13 +4674,14 @@ sub MapToSymbols { close(ADDRESSES); if ($debug) { print("----\n"); - system("cat $main::tmpfile_sym"); + system("cat", $main::tmpfile_sym); print("----\n"); - system("$cmd <$main::tmpfile_sym"); + system("$cmd < " . ShellEscape($main::tmpfile_sym)); print("----\n"); } - open(SYMBOLS, "$cmd <$main::tmpfile_sym |") || error("$cmd: $!\n"); + open(SYMBOLS, "$cmd <" . ShellEscape($main::tmpfile_sym) . " |") + || error("$cmd: $!\n"); my $count = 0; # Index in pclist while (<SYMBOLS>) { # Read fullfunction and filelineinfo from next pair of lines @@ -4332,15 +4701,29 @@ sub MapToSymbols { my $pcstr = $pclist->[$count]; my $function = ShortFunctionName($fullfunction); - if ($fullfunction eq '??') { - # See if nm found a symbol - my $nms = $nm_symbols->{$pcstr}; - if (defined($nms)) { + my $nms = $nm_symbols->{$pcstr}; + if (defined($nms)) { + if ($fullfunction eq '??') { + # nm found a symbol for us. $function = $nms->[0]; $fullfunction = $nms->[2]; + } else { + # MapSymbolsWithNM tags each routine with its starting address, + # useful in case the image has multiple occurrences of this + # routine. (It uses a syntax that resembles template paramters, + # that are automatically stripped out by ShortFunctionName().) + # addr2line does not provide the same information. So we check + # if nm disambiguated our symbol, and if so take the annotated + # (nm) version of the routine-name. TODO(csilvers): this won't + # catch overloaded, inlined symbols, which nm doesn't see. + # Better would be to do a check similar to nm's, in this fn. + if ($nms->[2] =~ m/^\Q$function\E/) { # sanity check it's the right fn + $function = $nms->[0]; + $fullfunction = $nms->[2]; + } } } - + # Prepend to accumulated symbols for pcstr # (so that caller comes before callee) my $sym = $symbols->{$pcstr}; @@ -4351,7 +4734,7 @@ sub MapToSymbols { unshift(@{$sym}, $function, $filelinenum, $fullfunction); if ($debug) { printf STDERR ("%s => [%s]\n", $pcstr, join(" ", @{$sym})); } if (!defined($sep_address)) { - # Inlining is off, se this entry ends immediately + # Inlining is off, so this entry ends immediately $count++; } } @@ -4414,6 +4797,31 @@ sub ShortFunctionName { return $function; } +# Trim overly long symbols found in disassembler output +sub CleanDisassembly { + my $d = shift; + while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) + while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments + return $d; +} + +# Clean file name for display +sub CleanFileName { + my ($f) = @_; + $f =~ s|^/proc/self/cwd/||; + $f =~ s|^\./||; + return $f; +} + +# Make address relative to section and clean up for display +sub UnparseAddress { + my ($offset, $address) = @_; + $address = AddressSub($address, $offset); + $address =~ s/^0x//; + $address =~ s/^0*//; + return $address; +} + ##### Miscellaneous ##### # Find the right versions of the above object tools to use. The @@ -4433,7 +4841,9 @@ sub ConfigureObjTools { my $file_type = undef; if (-e "/usr/bin/file") { # Follow symlinks (at least for systems where "file" supports that). - $file_type = `/usr/bin/file -L $prog_file 2>$dev_null || /usr/bin/file $prog_file`; + my $escaped_prog_file = ShellEscape($prog_file); + $file_type = `/usr/bin/file -L $escaped_prog_file 2>$dev_null || + /usr/bin/file $escaped_prog_file`; } elsif ($^O == "MSWin32") { $file_type = "MS Windows"; } else { @@ -4515,6 +4925,19 @@ sub ConfigureTool { return $path; } +sub ShellEscape { + my @escaped_words = (); + foreach my $word (@_) { + my $escaped_word = $word; + if ($word =~ m![^a-zA-Z0-9/.,_=-]!) { # check for anything not in whitelist + $escaped_word =~ s/'/'\\''/; + $escaped_word = "'$escaped_word'"; + } + push(@escaped_words, $escaped_word); + } + return join(" ", @escaped_words); +} + sub cleanup { unlink($main::tmpfile_sym); unlink(keys %main::tempnames); @@ -4552,11 +4975,11 @@ sub error { # names match "$regexp" and returns them in a hashtable mapping from # procedure name to a two-element vector of [start address, end address] sub GetProcedureBoundariesViaNm { - my $nm_command = shift; + my $escaped_nm_command = shift; # shell-escaped my $regexp = shift; my $symbol_table = {}; - open(NM, "$nm_command |") || error("$nm_command: $!\n"); + open(NM, "$escaped_nm_command |") || error("$escaped_nm_command: $!\n"); my $last_start = "0"; my $routine = ""; while (<NM>) { @@ -4634,6 +5057,21 @@ sub GetProcedureBoundaries { my $image = shift; my $regexp = shift; + # If $image doesn't start with /, then put ./ in front of it. This works + # around an obnoxious bug in our probing of nm -f behavior. + # "nm -f $image" is supposed to fail on GNU nm, but if: + # + # a. $image starts with [BbSsPp] (for example, bin/foo/bar), AND + # b. you have a.out in your current directory (a not uncommon occurence) + # + # then "nm -f $image" succeeds because -f only looks at the first letter of + # the argument, which looks valid because it's [BbSsPp], and then since + # there's no image provided, it looks for a.out and finds it. + # + # This regex makes sure that $image starts with . or /, forcing the -f + # parsing to fail since . and / are not valid formats. + $image =~ s#^[^/]#./$&#; + # For libc libraries, the copy in /usr/lib/debug contains debugging symbols my $debugging = DebuggingLibrary($image); if ($debugging) { @@ -4651,28 +5089,29 @@ sub GetProcedureBoundaries { # --demangle and -f. my $demangle_flag = ""; my $cppfilt_flag = ""; - if (system("$nm --demangle $image >$dev_null 2>&1") == 0) { + my $to_devnull = ">$dev_null 2>&1"; + if (system(ShellEscape($nm, "--demangle", "image") . $to_devnull) == 0) { # In this mode, we do "nm --demangle <foo>" $demangle_flag = "--demangle"; $cppfilt_flag = ""; - } elsif (system("$cppfilt $image >$dev_null 2>&1") == 0) { + } elsif (system(ShellEscape($cppfilt, $image) . $to_devnull) == 0) { # In this mode, we do "nm <foo> | c++filt" - $cppfilt_flag = " | $cppfilt"; + $cppfilt_flag = " | " . ShellEscape($cppfilt); }; my $flatten_flag = ""; - if (system("$nm -f $image >$dev_null 2>&1") == 0) { + if (system(ShellEscape($nm, "-f", $image) . $to_devnull) == 0) { $flatten_flag = "-f"; } # Finally, in the case $imagie isn't a debug library, we try again with # -D to at least get *exported* symbols. If we can't use --demangle, # we use c++filt instead, if it exists on this system. - my @nm_commands = ("$nm -n $flatten_flag $demangle_flag" . - " $image 2>$dev_null $cppfilt_flag", - "$nm -D -n $flatten_flag $demangle_flag" . - " $image 2>$dev_null $cppfilt_flag", + my @nm_commands = (ShellEscape($nm, "-n", $flatten_flag, $demangle_flag, + $image) . " 2>$dev_null $cppfilt_flag", + ShellEscape($nm, "-D", "-n", $flatten_flag, $demangle_flag, + $image) . " 2>$dev_null $cppfilt_flag", # 6nm is for Go binaries - "6nm $image 2>$dev_null | sort", + ShellEscape("6nm", "$image") . " 2>$dev_null | sort", ); # If the executable is an MS Windows PDB-format executable, we'll @@ -4680,8 +5119,9 @@ sub GetProcedureBoundaries { # want to use both unix nm and windows-specific nm_pdb, since # PDB-format executables can apparently include dwarf .o files. if (exists $obj_tool_map{"nm_pdb"}) { - my $nm_pdb = $obj_tool_map{"nm_pdb"}; - push(@nm_commands, "$nm_pdb --demangle $image 2>$dev_null"); + push(@nm_commands, + ShellEscape($obj_tool_map{"nm_pdb"}, "--demangle", $image) + . " 2>$dev_null"); } foreach my $nm_command (@nm_commands) { diff --git a/third_party/tcmalloc/chromium/src/profile-handler.cc b/third_party/tcmalloc/chromium/src/profile-handler.cc index 1946f7c..20e5cca 100644 --- a/third_party/tcmalloc/chromium/src/profile-handler.cc +++ b/third_party/tcmalloc/chromium/src/profile-handler.cc @@ -46,6 +46,7 @@ #include <string> #include "base/dynamic_annotations.h" +#include "base/googleinit.h" #include "base/logging.h" #include "base/spinlock.h" #include "maybe_threads.h" @@ -139,6 +140,9 @@ class ProfileHandler { // Counts the number of callbacks registered. int32 callback_count_ GUARDED_BY(control_lock_); + // Is profiling allowed at all? + bool allowed_; + // Whether or not the threading system provides interval timers that are // shared by all threads in a process. enum { @@ -199,6 +203,10 @@ class ProfileHandler { // Disables (ignores) the timer interrupt signal. void DisableHandler() EXCLUSIVE_LOCKS_REQUIRED(control_lock_); + // Returns true if the handler is not being used by something else. + // This checks the kernel's signal handler table. + bool IsSignalHandlerAvailable(); + // SIGPROF/SIGALRM handler. Iterate over and call all the registered callbacks. static void SignalHandler(int sig, siginfo_t* sinfo, void* ucontext); @@ -239,6 +247,7 @@ ProfileHandler* ProfileHandler::Instance() { ProfileHandler::ProfileHandler() : interrupts_(0), callback_count_(0), + allowed_(true), timer_sharing_(TIMERS_UNTOUCHED) { SpinLockHolder cl(&control_lock_); @@ -255,6 +264,19 @@ ProfileHandler::ProfileHandler() frequency_ = kDefaultFrequency; } + if (!allowed_) { + return; + } + + // If something else is using the signal handler, + // assume it has priority over us and stop. + if (!IsSignalHandlerAvailable()) { + RAW_LOG(INFO, "Disabling profiler because %s handler is already in use.", + timer_type_ == ITIMER_REAL ? "SIGALRM" : "SIGPROF"); + allowed_ = false; + return; + } + // Ignore signals until we decide to turn profiling on. (Paranoia; // should already be ignored.) DisableHandler(); @@ -267,6 +289,10 @@ ProfileHandler::~ProfileHandler() { void ProfileHandler::RegisterThread() { SpinLockHolder cl(&control_lock_); + if (!allowed_) { + return; + } + // We try to detect whether timers are being shared by setting a // timer in the first call to this function, then checking whether // it's set in the second call. @@ -312,6 +338,7 @@ void ProfileHandler::RegisterThread() { ProfileHandlerToken* ProfileHandler::RegisterCallback( ProfileHandlerCallback callback, void* callback_arg) { + ProfileHandlerToken* token = new ProfileHandlerToken(callback, callback_arg); SpinLockHolder cl(&control_lock_); @@ -386,9 +413,13 @@ void ProfileHandler::GetState(ProfileHandlerState* state) { } state->frequency = frequency_; state->callback_count = callback_count_; + state->allowed = allowed_; } void ProfileHandler::StartTimer() { + if (!allowed_) { + return; + } struct itimerval timer; timer.it_interval.tv_sec = 0; timer.it_interval.tv_usec = 1000000 / frequency_; @@ -397,12 +428,18 @@ void ProfileHandler::StartTimer() { } void ProfileHandler::StopTimer() { + if (!allowed_) { + return; + } struct itimerval timer; memset(&timer, 0, sizeof timer); setitimer(timer_type_, &timer, 0); } bool ProfileHandler::IsTimerRunning() { + if (!allowed_) { + return false; + } struct itimerval current_timer; RAW_CHECK(0 == getitimer(timer_type_, ¤t_timer), "getitimer"); return (current_timer.it_value.tv_sec != 0 || @@ -410,6 +447,9 @@ bool ProfileHandler::IsTimerRunning() { } void ProfileHandler::EnableHandler() { + if (!allowed_) { + return; + } struct sigaction sa; sa.sa_sigaction = SignalHandler; sa.sa_flags = SA_RESTART | SA_SIGINFO; @@ -419,6 +459,9 @@ void ProfileHandler::EnableHandler() { } void ProfileHandler::DisableHandler() { + if (!allowed_) { + return; + } struct sigaction sa; sa.sa_handler = SIG_IGN; sa.sa_flags = SA_RESTART; @@ -427,14 +470,33 @@ void ProfileHandler::DisableHandler() { RAW_CHECK(sigaction(signal_number, &sa, NULL) == 0, "sigprof (disable)"); } +bool ProfileHandler::IsSignalHandlerAvailable() { + struct sigaction sa; + const int signal_number = (timer_type_ == ITIMER_PROF ? SIGPROF : SIGALRM); + RAW_CHECK(sigaction(signal_number, NULL, &sa) == 0, "is-signal-handler avail"); + + // We only take over the handler if the current one is unset. + // It must be SIG_IGN or SIG_DFL, not some other function. + // SIG_IGN must be allowed because when profiling is allowed but + // not actively in use, this code keeps the handler set to SIG_IGN. + // That setting will be inherited across fork+exec. In order for + // any child to be able to use profiling, SIG_IGN must be treated + // as available. + return sa.sa_handler == SIG_IGN || sa.sa_handler == SIG_DFL; +} + void ProfileHandler::SignalHandler(int sig, siginfo_t* sinfo, void* ucontext) { int saved_errno = errno; - RAW_CHECK(instance_ != NULL, "ProfileHandler is not initialized"); + // At this moment, instance_ must be initialized because the handler is + // enabled in RegisterThread or RegisterCallback only after + // ProfileHandler::Instance runs. + ProfileHandler* instance = ANNOTATE_UNPROTECTED_READ(instance_); + RAW_CHECK(instance != NULL, "ProfileHandler is not initialized"); { - SpinLockHolder sl(&instance_->signal_lock_); - ++instance_->interrupts_; - for (CallbackIterator it = instance_->callbacks_.begin(); - it != instance_->callbacks_.end(); + SpinLockHolder sl(&instance->signal_lock_); + ++instance->interrupts_; + for (CallbackIterator it = instance->callbacks_.begin(); + it != instance->callbacks_.end(); ++it) { (*it)->callback(sig, sinfo, ucontext, (*it)->callback_arg); } @@ -442,20 +504,9 @@ void ProfileHandler::SignalHandler(int sig, siginfo_t* sinfo, void* ucontext) { errno = saved_errno; } -// The sole purpose of this class is to initialize the ProfileHandler singleton -// when the global static objects are created. Note that the main thread will -// be registered at this time. -class ProfileHandlerInitializer { - public: - ProfileHandlerInitializer() { - ProfileHandler::Instance()->RegisterThread(); - } - - private: - DISALLOW_COPY_AND_ASSIGN(ProfileHandlerInitializer); -}; -// ProfileHandlerInitializer singleton -static ProfileHandlerInitializer profile_handler_initializer; +// This module initializer registers the main thread, so it must be +// executed in the context of the main thread. +REGISTER_MODULE_INITIALIZER(profile_main, ProfileHandlerRegisterThread()); extern "C" void ProfileHandlerRegisterThread() { ProfileHandler::Instance()->RegisterThread(); diff --git a/third_party/tcmalloc/chromium/src/profile-handler.h b/third_party/tcmalloc/chromium/src/profile-handler.h index 1cbe253..4b078ec 100644 --- a/third_party/tcmalloc/chromium/src/profile-handler.h +++ b/third_party/tcmalloc/chromium/src/profile-handler.h @@ -137,6 +137,7 @@ struct ProfileHandlerState { int32 frequency; /* Profiling frequency */ int32 callback_count; /* Number of callbacks registered */ int64 interrupts; /* Number of interrupts received */ + bool allowed; /* Profiling is allowed */ }; void ProfileHandlerGetState(struct ProfileHandlerState* state); diff --git a/third_party/tcmalloc/chromium/src/profiler.cc b/third_party/tcmalloc/chromium/src/profiler.cc index 38fbb93..dfb6aab 100644 --- a/third_party/tcmalloc/chromium/src/profiler.cc +++ b/third_party/tcmalloc/chromium/src/profiler.cc @@ -55,8 +55,8 @@ typedef int ucontext_t; // just to quiet the compiler, mostly #endif #include <sys/time.h> #include <string> -#include <google/profiler.h> -#include <google/stacktrace.h> +#include <gperftools/profiler.h> +#include <gperftools/stacktrace.h> #include "base/commandlineflags.h" #include "base/logging.h" #include "base/googleinit.h" diff --git a/third_party/tcmalloc/chromium/src/span.cc b/third_party/tcmalloc/chromium/src/span.cc index 426a6bd..7600945 100644 --- a/third_party/tcmalloc/chromium/src/span.cc +++ b/third_party/tcmalloc/chromium/src/span.cc @@ -89,16 +89,6 @@ int DLL_Length(const Span* list) { return result; } -#if 0 // This isn't used. If that changes, rewrite to use TCMalloc_Printer. -void DLL_Print(const char* label, const Span* list) { - MESSAGE("%-10s %p:", label, list); - for (const Span* s = list->next; s != list; s = s->next) { - MESSAGE(" <%p,%"PRIuPTR",%"PRIuPTR">", s, s->start, s->length); - } - MESSAGE("%s\n", ""); // %s is to get around a compiler error. -} -#endif - void DLL_Prepend(Span* list, Span* span) { ASSERT(span->next == NULL); ASSERT(span->prev == NULL); diff --git a/third_party/tcmalloc/chromium/src/span.h b/third_party/tcmalloc/chromium/src/span.h index ab9a796..08db629 100644 --- a/third_party/tcmalloc/chromium/src/span.h +++ b/third_party/tcmalloc/chromium/src/span.h @@ -96,11 +96,6 @@ void DLL_Prepend(Span* list, Span* span); // Return the length of the linked list. O(n) int DLL_Length(const Span* list); -// Print the contents of the list to stderr. -#if 0 // This isn't used. -void DLL_Print(const char* label, const Span* list); -#endif - } // namespace tcmalloc #endif // TCMALLOC_SPAN_H_ diff --git a/third_party/tcmalloc/chromium/src/stack_trace_table.cc b/third_party/tcmalloc/chromium/src/stack_trace_table.cc index faeca6b..d258a4f 100644 --- a/third_party/tcmalloc/chromium/src/stack_trace_table.cc +++ b/third_party/tcmalloc/chromium/src/stack_trace_table.cc @@ -35,7 +35,7 @@ #include <string.h> // for NULL, memset #include "base/spinlock.h" // for SpinLockHolder #include "common.h" // for StackTrace -#include "internal_logging.h" // for MESSAGE, ASSERT +#include "internal_logging.h" // for ASSERT, Log #include "page_heap_allocator.h" // for PageHeapAllocator #include "static_vars.h" // for Static @@ -93,7 +93,8 @@ void StackTraceTable::AddTrace(const StackTrace& t) { bucket_total_++; b = Static::bucket_allocator()->New(); if (b == NULL) { - MESSAGE("tcmalloc: could not allocate bucket", sizeof(*b)); + Log(kLog, __FILE__, __LINE__, + "tcmalloc: could not allocate bucket", sizeof(*b)); error_ = true; } else { b->hash = h; @@ -114,8 +115,9 @@ void** StackTraceTable::ReadStackTracesAndClear() { const int out_len = bucket_total_ * 3 + depth_total_ + 1; void** out = new void*[out_len]; if (out == NULL) { - MESSAGE("tcmalloc: allocation failed for stack traces\n", - out_len * sizeof(*out)); + Log(kLog, __FILE__, __LINE__, + "tcmalloc: allocation failed for stack traces", + out_len * sizeof(*out)); return NULL; } diff --git a/third_party/tcmalloc/chromium/src/stacktrace.cc b/third_party/tcmalloc/chromium/src/stacktrace.cc index 68cb865..d96b4d3 100644 --- a/third_party/tcmalloc/chromium/src/stacktrace.cc +++ b/third_party/tcmalloc/chromium/src/stacktrace.cc @@ -53,7 +53,7 @@ // Some code may do that. #include <config.h> -#include <google/stacktrace.h> +#include <gperftools/stacktrace.h> #include "stacktrace_config.h" #if defined(STACKTRACE_INL_HEADER) @@ -99,11 +99,12 @@ #elif 0 // This is for the benefit of code analysis tools that may have // trouble with the computed #include above. -# include "base/stacktrace_x86-inl.h" -# include "base/stacktrace_libunwind-inl.h" -# include "base/stacktrace_generic-inl.h" -# include "base/stacktrace_powerpc-inl.h" -# include "base/stacktrace_win32-inl.h" +# include "stacktrace_x86-inl.h" +# include "stacktrace_libunwind-inl.h" +# include "stacktrace_generic-inl.h" +# include "stacktrace_powerpc-inl.h" +# include "stacktrace_win32-inl.h" +# include "stacktrace_arm-inl.h" #else # error Cannot calculate stack trace: will need to write for your environment #endif diff --git a/third_party/tcmalloc/chromium/src/stacktrace_arm-inl.h b/third_party/tcmalloc/chromium/src/stacktrace_arm-inl.h new file mode 100644 index 0000000..5ee1bf9 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/stacktrace_arm-inl.h @@ -0,0 +1,145 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Doug Kwan +// This is inspired by Craig Silverstein's PowerPC stacktrace code. +// + +#ifndef BASE_STACKTRACE_ARM_INL_H_ +#define BASE_STACKTRACE_ARM_INL_H_ +// Note: this file is included into stacktrace.cc more than once. +// Anything that should only be defined once should be here: + +#include <stdint.h> // for uintptr_t +#include "base/basictypes.h" // for NULL +#include <gperftools/stacktrace.h> + +// WARNING: +// This only works if all your code is in either ARM or THUMB mode. With +// interworking, the frame pointer of the caller can either be in r11 (ARM +// mode) or r7 (THUMB mode). A callee only saves the frame pointer of its +// mode in a fixed location on its stack frame. If the caller is a different +// mode, there is no easy way to find the frame pointer. It can either be +// still in the designated register or saved on stack along with other callee +// saved registers. + +// Given a pointer to a stack frame, locate and return the calling +// stackframe, or return NULL if no stackframe can be found. Perform sanity +// checks (the strictness of which is controlled by the boolean parameter +// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. +template<bool STRICT_UNWINDING> +static void **NextStackFrame(void **old_sp) { + void **new_sp = (void**) old_sp[-1]; + + // Check that the transition from frame pointer old_sp to frame + // pointer new_sp isn't clearly bogus + if (STRICT_UNWINDING) { + // With the stack growing downwards, older stack frame must be + // at a greater address that the current one. + if (new_sp <= old_sp) return NULL; + // Assume stack frames larger than 100,000 bytes are bogus. + if ((uintptr_t)new_sp - (uintptr_t)old_sp > 100000) return NULL; + } else { + // In the non-strict mode, allow discontiguous stack frames. + // (alternate-signal-stacks for example). + if (new_sp == old_sp) return NULL; + // And allow frames upto about 1MB. + if ((new_sp > old_sp) + && ((uintptr_t)new_sp - (uintptr_t)old_sp > 1000000)) return NULL; + } + if ((uintptr_t)new_sp & (sizeof(void *) - 1)) return NULL; + return new_sp; +} + +// This ensures that GetStackTrace stes up the Link Register properly. +#ifdef __GNUC__ +void StacktraceArmDummyFunction() __attribute__((noinline)); +void StacktraceArmDummyFunction() { __asm__ volatile(""); } +#else +# error StacktraceArmDummyFunction() needs to be ported to this platform. +#endif +#endif // BASE_STACKTRACE_ARM_INL_H_ + +// Note: this part of the file is included several times. +// Do not put globals below. + +// The following 4 functions are generated from the code below: +// GetStack{Trace,Frames}() +// GetStack{Trace,Frames}WithContext() +// +// These functions take the following args: +// void** result: the stack-trace, as an array +// int* sizes: the size of each stack frame, as an array +// (GetStackFrames* only) +// int max_depth: the size of the result (and sizes) array(s) +// int skip_count: how many stack pointers to skip before storing in result +// void* ucp: a ucontext_t* (GetStack{Trace,Frames}WithContext only) +int GET_STACK_TRACE_OR_FRAMES { +#ifdef __GNUC__ + void **sp = reinterpret_cast<void**>(__builtin_frame_address(0)); +#else +# error reading stack point not yet supported on this platform. +#endif + + // On ARM, the return address is stored in the link register (r14). + // This is not saved on the stack frame of a leaf function. To + // simplify code that reads return addresses, we call a dummy + // function so that the return address of this function is also + // stored in the stack frame. This works at least for gcc. + StacktraceArmDummyFunction(); + + int n = 0; + while (sp && n < max_depth) { + // The GetStackFrames routine is called when we are in some + // informational context (the failure signal handler for example). + // Use the non-strict unwinding rules to produce a stack trace + // that is as complete as possible (even if it contains a few bogus + // entries in some rare cases). + void **next_sp = NextStackFrame<IS_STACK_FRAMES == 0>(sp); + + if (skip_count > 0) { + skip_count--; + } else { + result[n] = *sp; + +#if IS_STACK_FRAMES + if (next_sp > sp) { + sizes[n] = (uintptr_t)next_sp - (uintptr_t)sp; + } else { + // A frame-size of 0 is used to indicate unknown frame size. + sizes[n] = 0; + } +#endif + n++; + } + sp = next_sp; + } + return n; +} diff --git a/third_party/tcmalloc/chromium/src/stacktrace_config.h b/third_party/tcmalloc/chromium/src/stacktrace_config.h index 18f16ab..72d108a 100644 --- a/third_party/tcmalloc/chromium/src/stacktrace_config.h +++ b/third_party/tcmalloc/chromium/src/stacktrace_config.h @@ -68,6 +68,14 @@ # define STACKTRACE_INL_HEADER "stacktrace_generic-inl.h" # endif +// The ARM case +#elif defined(__arm__) && __GNUC__ >= 2 +# if !defined(NO_FRAME_POINTER) +# define STACKTRACE_INL_HEADER "stacktrace_arm-inl.h" +# else +# error stacktrace without frame pointer is not supported on ARM +# endif + // The Windows case -- probably cygwin and mingw will use one of the // x86-includes above, but if not, we can fall back to windows intrinsics. #elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__) diff --git a/third_party/tcmalloc/chromium/src/stacktrace_generic-inl.h b/third_party/tcmalloc/chromium/src/stacktrace_generic-inl.h index 0e72ee7..5a526e2 100644 --- a/third_party/tcmalloc/chromium/src/stacktrace_generic-inl.h +++ b/third_party/tcmalloc/chromium/src/stacktrace_generic-inl.h @@ -42,7 +42,7 @@ #include <execinfo.h> #include <string.h> -#include "google/stacktrace.h" +#include "gperftools/stacktrace.h" #endif // BASE_STACKTRACE_GENERIC_INL_H_ // Note: this part of the file is included several times. diff --git a/third_party/tcmalloc/chromium/src/stacktrace_libunwind-inl.h b/third_party/tcmalloc/chromium/src/stacktrace_libunwind-inl.h index a1d5249..82b0cfe 100644 --- a/third_party/tcmalloc/chromium/src/stacktrace_libunwind-inl.h +++ b/third_party/tcmalloc/chromium/src/stacktrace_libunwind-inl.h @@ -45,7 +45,7 @@ extern "C" { #include <string.h> // for memset() #include <libunwind.h> } -#include "google/stacktrace.h" +#include "gperftools/stacktrace.h" #include "base/logging.h" // Sometimes, we can try to get a stack trace from within a stack diff --git a/third_party/tcmalloc/chromium/src/stacktrace_powerpc-inl.h b/third_party/tcmalloc/chromium/src/stacktrace_powerpc-inl.h index 9a07eea..acf2884 100644 --- a/third_party/tcmalloc/chromium/src/stacktrace_powerpc-inl.h +++ b/third_party/tcmalloc/chromium/src/stacktrace_powerpc-inl.h @@ -43,7 +43,7 @@ #include <stdint.h> // for uintptr_t #include <stdlib.h> // for NULL -#include <google/stacktrace.h> +#include <gperftools/stacktrace.h> // Given a pointer to a stack frame, locate and return the calling // stackframe, or return NULL if no stackframe can be found. Perform sanity @@ -126,16 +126,12 @@ int GET_STACK_TRACE_OR_FRAMES { int n = 0; while (sp && n < max_depth) { -#if IS_STACK_FRAMES // The GetStackFrames routine is called when we are in some // informational context (the failure signal handler for example). // Use the non-strict unwinding rules to produce a stack trace - // that is as complete as possible (even if it contains a few bogus - // entries in some rare cases). - void **next_sp = NextStackFrame<false>(sp); -#else - void **next_sp = NextStackFrame<true>(sp); -#endif + // that is as complete as possible (even if it contains a few + // bogus entries in some rare cases). + void **next_sp = NextStackFrame<!IS_STACK_FRAMES>(sp); if (skip_count > 0) { skip_count--; @@ -145,20 +141,20 @@ int GET_STACK_TRACE_OR_FRAMES { // linux ppc64), it's in sp[2]. For SYSV (used by linux ppc), // it's in sp[1]. #if defined(_CALL_AIX) || defined(_CALL_DARWIN) - result[n++] = *(sp+2); + result[n] = *(sp+2); #elif defined(_CALL_SYSV) - result[n++] = *(sp+1); + result[n] = *(sp+1); #elif defined(__APPLE__) || (defined(__linux) && defined(__PPC64__)) // This check is in case the compiler doesn't define _CALL_AIX/etc. - result[n++] = *(sp+2); + result[n] = *(sp+2); #elif defined(__linux) // This check is in case the compiler doesn't define _CALL_SYSV. - result[n++] = *(sp+1); + result[n] = *(sp+1); #else #error Need to specify the PPC ABI for your archiecture. #endif -#if IS_STACK_FRAME +#if IS_STACK_FRAMES if (next_sp > sp) { sizes[n] = (uintptr_t)next_sp - (uintptr_t)sp; } else { @@ -166,6 +162,7 @@ int GET_STACK_TRACE_OR_FRAMES { sizes[n] = 0; } #endif + n++; } sp = next_sp; } diff --git a/third_party/tcmalloc/chromium/src/stacktrace_win32-inl.h b/third_party/tcmalloc/chromium/src/stacktrace_win32-inl.h index 4119c04..04458a5 100644 --- a/third_party/tcmalloc/chromium/src/stacktrace_win32-inl.h +++ b/third_party/tcmalloc/chromium/src/stacktrace_win32-inl.h @@ -47,7 +47,7 @@ // server. // // This code is inspired by a patch from David Vitek: -// http://code.google.com/p/google-perftools/issues/detail?id=83 +// http://code.google.com/p/gperftools/issues/detail?id=83 #ifndef BASE_STACKTRACE_WIN32_INL_H_ #define BASE_STACKTRACE_WIN32_INL_H_ diff --git a/third_party/tcmalloc/chromium/src/stacktrace_with_context.cc b/third_party/tcmalloc/chromium/src/stacktrace_with_context.cc index ed7bfe3..036d984 100644 --- a/third_party/tcmalloc/chromium/src/stacktrace_with_context.cc +++ b/third_party/tcmalloc/chromium/src/stacktrace_with_context.cc @@ -42,7 +42,7 @@ // ATTRIBUTE_NOINLINE. #include <config.h> -#include <google/stacktrace.h> +#include <gperftools/stacktrace.h> #include "stacktrace_config.h" #include "base/basictypes.h" diff --git a/third_party/tcmalloc/chromium/src/stacktrace_x86-inl.h b/third_party/tcmalloc/chromium/src/stacktrace_x86-inl.h index 0f8c4de..abbe0a9 100644 --- a/third_party/tcmalloc/chromium/src/stacktrace_x86-inl.h +++ b/third_party/tcmalloc/chromium/src/stacktrace_x86-inl.h @@ -64,7 +64,7 @@ typedef ucontext ucontext_t; #include "base/vdso_support.h" #endif -#include "google/stacktrace.h" +#include "gperftools/stacktrace.h" #if defined(KEEP_SHADOW_STACKS) #include "linux_shadow_stacks.h" #endif // KEEP_SHADOW_STACKS @@ -241,9 +241,14 @@ static void **NextStackFrame(void **old_sp, const void *uc) { // In the non-strict mode, allow discontiguous stack frames. // (alternate-signal-stacks for example). if (new_sp == old_sp) return NULL; - // And allow frames upto about 1MB. - if ((new_sp > old_sp) - && ((uintptr_t)new_sp - (uintptr_t)old_sp > 1000000)) return NULL; + if (new_sp > old_sp) { + // And allow frames upto about 1MB. + const uintptr_t delta = (uintptr_t)new_sp - (uintptr_t)old_sp; + const uintptr_t acceptable_delta = 1000000; + if (delta > acceptable_delta) { + return NULL; + } + } } if ((uintptr_t)new_sp & (sizeof(void *) - 1)) return NULL; #ifdef __i386__ diff --git a/third_party/tcmalloc/chromium/src/stacktrace_x86_64-inl.h b/third_party/tcmalloc/chromium/src/stacktrace_x86_64-inl.h deleted file mode 100644 index 767ef9b..0000000 --- a/third_party/tcmalloc/chromium/src/stacktrace_x86_64-inl.h +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// Author: Arun Sharma -// -// Produce stack trace using libgcc - -extern "C" { -#include <stdlib.h> // for NULL -#include <unwind.h> // ABI defined unwinder -#include <string.h> // for memset -} -#include "google/stacktrace.h" - -typedef struct { - void **result; - int max_depth; - int skip_count; - int count; -} trace_arg_t; - - -// Workaround for the malloc() in _Unwind_Backtrace() issue. -static _Unwind_Reason_Code nop_backtrace(struct _Unwind_Context *uc, void *opq) { - return _URC_NO_REASON; -} - - -// This code is not considered ready to run until -// static initializers run so that we are guaranteed -// that any malloc-related initialization is done. -static bool ready_to_run = false; -class StackTraceInit { - public: - StackTraceInit() { - // Extra call to force initialization - _Unwind_Backtrace(nop_backtrace, NULL); - ready_to_run = true; - } -}; - -static StackTraceInit module_initializer; // Force initialization - -static _Unwind_Reason_Code GetOneFrame(struct _Unwind_Context *uc, void *opq) { - trace_arg_t *targ = (trace_arg_t *) opq; - - if (targ->skip_count > 0) { - targ->skip_count--; - } else { - targ->result[targ->count++] = (void *) _Unwind_GetIP(uc); - } - - if (targ->count == targ->max_depth) - return _URC_END_OF_STACK; - - return _URC_NO_REASON; -} - -// If you change this function, also change GetStackFrames below. -int GetStackTrace(void** result, int max_depth, int skip_count) { - if (!ready_to_run) - return 0; - - trace_arg_t targ; - - skip_count += 1; // Do not include the "GetStackTrace" frame - - targ.result = result; - targ.max_depth = max_depth; - targ.skip_count = skip_count; - targ.count = 0; - - _Unwind_Backtrace(GetOneFrame, &targ); - - return targ.count; -} - -// If you change this function, also change GetStackTrace above: -// -// This GetStackFrames routine shares a lot of code with GetStackTrace -// above. This code could have been refactored into a common routine, -// and then both GetStackTrace/GetStackFrames could call that routine. -// There are two problems with that: -// -// (1) The performance of the refactored-code suffers substantially - the -// refactored needs to be able to record the stack trace when called -// from GetStackTrace, and both the stack trace and stack frame sizes, -// when called from GetStackFrames - this introduces enough new -// conditionals that GetStackTrace performance can degrade by as much -// as 50%. -// -// (2) Whether the refactored routine gets inlined into GetStackTrace and -// GetStackFrames depends on the compiler, and we can't guarantee the -// behavior either-way, even with "__attribute__ ((always_inline))" -// or "__attribute__ ((noinline))". But we need this guarantee or the -// frame counts may be off by one. -// -// Both (1) and (2) can be addressed without this code duplication, by -// clever use of template functions, and by defining GetStackTrace and -// GetStackFrames as macros that expand to these template functions. -// However, this approach comes with its own set of problems - namely, -// macros and preprocessor trouble - for example, if GetStackTrace -// and/or GetStackFrames is ever defined as a member functions in some -// class, we are in trouble. -int GetStackFrames(void** pcs, int* sizes, int max_depth, int skip_count) { - if (!ready_to_run) - return 0; - - trace_arg_t targ; - - skip_count += 1; // Do not include the "GetStackFrames" frame - - targ.result = pcs; - targ.max_depth = max_depth; - targ.skip_count = skip_count; - targ.count = 0; - - _Unwind_Backtrace(GetOneFrame, &targ); - - // No implementation for finding out the stack frame sizes yet. - memset(sizes, 0, sizeof(*sizes) * targ.count); - - return targ.count; -} diff --git a/third_party/tcmalloc/chromium/src/static_vars.cc b/third_party/tcmalloc/chromium/src/static_vars.cc index 2ca132e..6fc852a 100644 --- a/third_party/tcmalloc/chromium/src/static_vars.cc +++ b/third_party/tcmalloc/chromium/src/static_vars.cc @@ -34,6 +34,7 @@ #include <stddef.h> // for NULL #include <new> // for operator new #include "internal_logging.h" // for CHECK_CONDITION +#include "common.h" #include "sampler.h" // for Sampler namespace tcmalloc { @@ -46,7 +47,7 @@ PageHeapAllocator<StackTrace> Static::stacktrace_allocator_; Span Static::sampled_objects_; PageHeapAllocator<StackTraceTable::Bucket> Static::bucket_allocator_; StackTrace* Static::growth_stacks_ = NULL; -char Static::pageheap_memory_[sizeof(PageHeap)]; +PageHeap* Static::pageheap_ = NULL; void Static::InitStaticVars() { sizemap_.Init(); @@ -60,7 +61,11 @@ void Static::InitStaticVars() { for (int i = 0; i < kNumClasses; ++i) { central_cache_[i].Init(i); } - new ((void*)pageheap_memory_) PageHeap; + // It's important to have PageHeap allocated, not in static storage, + // so that HeapLeakChecker does not consider all the byte patterns stored + // in is caches as pointers that are sources of heap object liveness, + // which leads to it missing some memory leaks. + pageheap_ = new (MetaDataAlloc(sizeof(PageHeap))) PageHeap; DLL_Init(&sampled_objects_); Sampler::InitStatics(); } diff --git a/third_party/tcmalloc/chromium/src/static_vars.h b/third_party/tcmalloc/chromium/src/static_vars.h index b21fe2b..185a1d4 100644 --- a/third_party/tcmalloc/chromium/src/static_vars.h +++ b/third_party/tcmalloc/chromium/src/static_vars.h @@ -65,9 +65,7 @@ class Static { // must be protected by pageheap_lock. // Page-level allocator. - static PageHeap* pageheap() { - return reinterpret_cast<PageHeap*>(pageheap_memory_); - } + static PageHeap* pageheap() { return pageheap_; } static PageHeapAllocator<Span>* span_allocator() { return &span_allocator_; } @@ -105,10 +103,7 @@ class Static { // is stored in trace->stack[kMaxStackDepth-1]. static StackTrace* growth_stacks_; - // PageHeap uses a constructor for initialization. Like the members above, - // we can't depend on initialization order, so pageheap is new'd - // into this buffer. - static char pageheap_memory_[sizeof(PageHeap)]; + static PageHeap* pageheap_; }; } // namespace tcmalloc diff --git a/third_party/tcmalloc/chromium/src/symbolize.cc b/third_party/tcmalloc/chromium/src/symbolize.cc index ff45e3e..d90c4b8 100644 --- a/third_party/tcmalloc/chromium/src/symbolize.cc +++ b/third_party/tcmalloc/chromium/src/symbolize.cc @@ -48,8 +48,16 @@ #ifdef HAVE_POLL_H #include <poll.h> #endif +#ifdef __MACH__ +#include <mach-o/dyld.h> // for GetProgramInvocationName() +#include <limits.h> // for PATH_MAX +#endif +#if defined(__CYGWIN__) || defined(__CYGWIN32__) +#include <io.h> // for get_osfhandle() +#endif #include <string> #include "base/commandlineflags.h" +#include "base/logging.h" #include "base/sysinfo.h" using std::string; @@ -65,6 +73,36 @@ DEFINE_string(symbolize_pprof, // a more-permanent copy that won't ever get destroyed. static string* g_pprof_path = new string(FLAGS_symbolize_pprof); +// Returns NULL if we're on an OS where we can't get the invocation name. +// Using a static var is ok because we're not called from a thread. +static char* GetProgramInvocationName() { +#if defined(HAVE_PROGRAM_INVOCATION_NAME) + extern char* program_invocation_name; // gcc provides this + return program_invocation_name; +#elif defined(__MACH__) + // We don't want to allocate memory for this since we may be + // calculating it when memory is corrupted. + static char program_invocation_name[PATH_MAX]; + if (program_invocation_name[0] == '\0') { // first time calculating + uint32_t length = sizeof(program_invocation_name); + if (_NSGetExecutablePath(program_invocation_name, &length)) + return NULL; + } + return program_invocation_name; +#else + return NULL; // figure out a way to get argv[0] +#endif +} + +// Prints an error message when you can't run Symbolize(). +static void PrintError(const char* reason) { + RAW_LOG(ERROR, + "*** WARNING: Cannot convert addresses to symbols in output below.\n" + "*** Reason: %s\n" + "*** If you cannot fix this, try running pprof directly.\n", + reason); +} + void SymbolTable::Add(const void* addr) { symbolization_table_[addr] = ""; } @@ -79,14 +117,24 @@ const char* SymbolTable::GetSymbol(const void* addr) { // Note that the forking/etc is not thread-safe or re-entrant. That's // ok for the purpose we need -- reporting leaks detected by heap-checker // -- but be careful if you decide to use this routine for other purposes. +// Returns number of symbols read on error. If can't symbolize, returns 0 +// and emits an error message about why. int SymbolTable::Symbolize() { #if !defined(HAVE_UNISTD_H) || !defined(HAVE_SYS_SOCKET_H) || !defined(HAVE_SYS_WAIT_H) + PrintError("Perftools does not know how to call a sub-process on this O/S"); return 0; -#elif !defined(HAVE_PROGRAM_INVOCATION_NAME) - return 0; // TODO(csilvers): get argv[0] somehow #else + const char* argv0 = GetProgramInvocationName(); + if (argv0 == NULL) { // can't call symbolize if we can't figure out our name + PrintError("Cannot figure out the name of this executable (argv0)"); + return 0; + } + if (access(g_pprof_path->c_str(), R_OK) != 0) { + PrintError("Cannot find 'pprof' (is PPROF_PATH set correctly?)"); + return 0; + } + // All this work is to do two-way communication. ugh. - extern char* program_invocation_name; // gcc provides this int *child_in = NULL; // file descriptors int *child_out = NULL; // for now, we don't worry about child_err int child_fds[5][2]; // socketpair may be called up to five times below @@ -102,6 +150,7 @@ int SymbolTable::Symbolize() { for (int j = 0; j < i; j++) { close(child_fds[j][0]); close(child_fds[j][1]); + PrintError("Cannot create a socket pair"); return 0; } } else { @@ -127,6 +176,7 @@ int SymbolTable::Symbolize() { close(child_in[1]); close(child_out[0]); close(child_out[1]); + PrintError("Unknown error calling fork()"); return 0; } case 0: { // child @@ -142,13 +192,15 @@ int SymbolTable::Symbolize() { unsetenv("HEAPCHECK"); unsetenv("PERFTOOLS_VERBOSE"); execlp(g_pprof_path->c_str(), g_pprof_path->c_str(), - "--symbols", program_invocation_name, NULL); + "--symbols", argv0, NULL); _exit(3); // if execvp fails, it's bad news for us } default: { // parent close(child_in[0]); // child uses the 0's, parent uses the 1's close(child_out[0]); // child uses the 0's, parent uses the 1's #ifdef HAVE_POLL_H + // Waiting for 1ms seems to give the OS time to notice any errors. + poll(0, 0, 1); // For maximum safety, we check to make sure the execlp // succeeded before trying to write. (Otherwise we'll get a // SIGPIPE.) For systems without poll.h, we'll just skip this @@ -156,10 +208,17 @@ int SymbolTable::Symbolize() { struct pollfd pfd = { child_in[1], POLLOUT, 0 }; if (!poll(&pfd, 1, 0) || !(pfd.revents & POLLOUT) || (pfd.revents & (POLLHUP|POLLERR))) { + PrintError("Cannot run 'pprof' (is PPROF_PATH set correctly?)"); return 0; } #endif +#if defined(__CYGWIN__) || defined(__CYGWIN32__) + // On cygwin, DumpProcSelfMaps() takes a HANDLE, not an fd. Convert. + const HANDLE symbols_handle = (HANDLE) get_osfhandle(child_in[1]); + DumpProcSelfMaps(symbols_handle); +#else DumpProcSelfMaps(child_in[1]); // what pprof expects on stdin +#endif // Allocate 24 bytes = ("0x" + 8 bytes + "\n" + overhead) for each // address to feed to pprof. @@ -185,6 +244,7 @@ int SymbolTable::Symbolize() { kSymbolBufferSize - total_bytes_read); if (bytes_read < 0) { close(child_out[1]); + PrintError("Cannot read data from pprof"); return 0; } else if (bytes_read == 0) { close(child_out[1]); @@ -214,6 +274,7 @@ int SymbolTable::Symbolize() { return num_symbols; } } + PrintError("Unkown error (should never occur!)"); return 0; // shouldn't be reachable #endif } diff --git a/third_party/tcmalloc/chromium/src/system-alloc.cc b/third_party/tcmalloc/chromium/src/system-alloc.cc index 03fc934..511990c 100644 --- a/third_party/tcmalloc/chromium/src/system-alloc.cc +++ b/third_party/tcmalloc/chromium/src/system-alloc.cc @@ -48,7 +48,7 @@ #include <unistd.h> // for sbrk, getpagesize, off_t #endif #include <new> // for operator new -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> #include "base/basictypes.h" #include "base/commandlineflags.h" #include "base/spinlock.h" // for SpinLockHolder, SpinLock, etc @@ -61,6 +61,13 @@ # define MAP_ANONYMOUS MAP_ANON #endif +// MADV_FREE is specifically designed for use by malloc(), but only +// FreeBSD supports it; in linux we fall back to the somewhat inferior +// MADV_DONTNEED. +#if !defined(MADV_FREE) && defined(MADV_DONTNEED) +# define MADV_FREE MADV_DONTNEED +#endif + // Solaris has a bug where it doesn't declare madvise() for C++. // http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 #if defined(__sun) && defined(__SVR4) @@ -76,6 +83,10 @@ static const bool kDebugMode = false; static const bool kDebugMode = true; #endif +// TODO(sanjay): Move the code below into the tcmalloc namespace +using tcmalloc::kLog; +using tcmalloc::Log; + // Anonymous namespace to avoid name conflicts on "CheckAddressBits". namespace { @@ -103,9 +114,11 @@ union MemoryAligner { static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); +#if defined(HAVE_MMAP) || defined(MADV_FREE) #ifdef HAVE_GETPAGESIZE static size_t pagesize = 0; #endif +#endif // The current system allocator SysAllocator* sys_alloc = NULL; @@ -132,7 +145,6 @@ public: SbrkSysAllocator() : SysAllocator() { } void* Alloc(size_t size, size_t *actual_size, size_t alignment); - void FlagsInitialized() {} }; static char sbrk_space[sizeof(SbrkSysAllocator)]; @@ -141,7 +153,6 @@ public: MmapSysAllocator() : SysAllocator() { } void* Alloc(size_t size, size_t *actual_size, size_t alignment); - void FlagsInitialized() {} }; static char mmap_space[sizeof(MmapSysAllocator)]; @@ -150,7 +161,6 @@ public: DevMemSysAllocator() : SysAllocator() { } void* Alloc(size_t size, size_t *actual_size, size_t alignment); - void FlagsInitialized() {} }; class DefaultSysAllocator : public SysAllocator { @@ -171,7 +181,6 @@ class DefaultSysAllocator : public SysAllocator { } } void* Alloc(size_t size, size_t *actual_size, size_t alignment); - void FlagsInitialized() {} private: static const int kMaxAllocators = 2; @@ -420,7 +429,6 @@ void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size, if (result != NULL) { return result; } - TCMalloc_MESSAGE(__FILE__, __LINE__, "%s failed.\n", names_[i]); failed_[i] = true; } } @@ -499,10 +507,10 @@ size_t TCMalloc_SystemAddGuard(void* start, size_t size) { } void TCMalloc_SystemRelease(void* start, size_t length) { -#ifdef MADV_DONTNEED +#ifdef MADV_FREE if (FLAGS_malloc_devmem_start) { - // It's not safe to use MADV_DONTNEED if we've been mapping - // /dev/mem for heap memory + // It's not safe to use MADV_FREE/MADV_DONTNEED if we've been + // mapping /dev/mem for heap memory. return; } if (pagesize == 0) pagesize = getpagesize(); @@ -526,7 +534,7 @@ void TCMalloc_SystemRelease(void* start, size_t length) { // Note -- ignoring most return codes, because if this fails it // doesn't matter... while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, - MADV_DONTNEED) == -1 && + MADV_FREE) == -1 && errno == EAGAIN) { // NOP } diff --git a/third_party/tcmalloc/chromium/src/tcmalloc.cc b/third_party/tcmalloc/chromium/src/tcmalloc.cc index 9381aaf..d7084c9 100644 --- a/third_party/tcmalloc/chromium/src/tcmalloc.cc +++ b/third_party/tcmalloc/chromium/src/tcmalloc.cc @@ -87,15 +87,12 @@ // goes from about 1100 ns to about 300 ns. #include "config.h" -#include <google/tcmalloc.h> +#include <gperftools/tcmalloc.h> #include <errno.h> // for ENOMEM, EINVAL, errno #ifdef HAVE_SYS_CDEFS_H #include <sys/cdefs.h> // for __THROW #endif -#ifdef HAVE_FEATURES_H -#include <features.h> // for __GLIBC__ -#endif #if defined HAVE_STDINT_H #include <stdint.h> #elif defined HAVE_INTTYPES_H @@ -114,8 +111,8 @@ #include <new> // for nothrow_t (ptr only), etc #include <vector> // for vector -#include <google/malloc_extension.h> -#include <google/malloc_hook.h> // for MallocHook +#include <gperftools/malloc_extension.h> +#include <gperftools/malloc_hook.h> // for MallocHook #include "base/basictypes.h" // for int64 #include "base/commandlineflags.h" // for RegisterFlagValidator, etc #include "base/dynamic_annotations.h" // for RunningOnValgrind @@ -150,17 +147,29 @@ # define WIN32_DO_PATCHING 1 #endif -// GLibc 2.14+ requires the hook functions be declared volatile, based on the -// value of the define __MALLOC_HOOK_VOLATILE. For compatibility with -// older/non-GLibc implementations, provide an empty definition. -#if !defined(__MALLOC_HOOK_VOLATILE) -#define __MALLOC_HOOK_VOLATILE -#endif +// Some windows file somewhere (at least on cygwin) #define's small (!) +// For instance, <windows.h> appears to have "#define small char". +#undef small using STL_NAMESPACE::max; using STL_NAMESPACE::numeric_limits; using STL_NAMESPACE::vector; + +#include "libc_override.h" + +// __THROW is defined in glibc (via <sys/cdefs.h>). It means, +// counter-intuitively, "This function will never throw an exception." +// It's an optional optimization tool, but we may need to use it to +// match glibc prototypes. +#ifndef __THROW // I guess we're not on a glibc system +# define __THROW // __THROW is just an optimization, so ok to make it "" +#endif + using tcmalloc::AlignmentForSize; +using tcmalloc::kLog; +using tcmalloc::kCrash; +using tcmalloc::kCrashWithStats; +using tcmalloc::Log; using tcmalloc::PageHeap; using tcmalloc::PageHeapAllocator; using tcmalloc::SizeMap; @@ -169,13 +178,6 @@ using tcmalloc::StackTrace; using tcmalloc::Static; using tcmalloc::ThreadCache; -// __THROW is defined in glibc systems. It means, counter-intuitively, -// "This function will never throw an exception." It's an optional -// optimization tool, but we may need to use it to match glibc prototypes. -#ifndef __THROW // I guess we're not on a glibc system -# define __THROW // __THROW is just an optimization, so ok to make it "" -#endif - // ---- Double free debug declarations static size_t ExcludeSpaceForMark(size_t size); static void AddRoomForMark(size_t* size); @@ -280,160 +282,6 @@ extern "C" { ATTRIBUTE_SECTION(google_malloc); } // extern "C" -// Override the libc functions to prefer our own instead. This comes -// first so code in tcmalloc.cc can use the overridden versions. One -// exception: in windows, by default, we patch our code into these -// functions (via src/windows/patch_function.cc) rather than override -// them. In that case, we don't want to do this overriding here. -#if !defined(WIN32_DO_PATCHING) - -// TODO(mbelshe): Turn off TCMalloc's symbols for libc. We do that -// elsewhere. -#ifndef _WIN32 - -#if defined(__GNUC__) && !defined(__MACH__) - // Potentially faster variants that use the gcc alias extension. - // FreeBSD does support aliases, but apparently not correctly. :-( - // NOTE: we make many of these symbols weak, but do so in the makefile - // (via objcopy -W) and not here. That ends up being more portable. -# define ALIAS(x) __attribute__ ((alias (x))) -void* operator new(size_t size) throw (std::bad_alloc) ALIAS("tc_new"); -void operator delete(void* p) __THROW ALIAS("tc_delete"); -void* operator new[](size_t size) throw (std::bad_alloc) ALIAS("tc_newarray"); -void operator delete[](void* p) __THROW ALIAS("tc_deletearray"); -void* operator new(size_t size, const std::nothrow_t&) __THROW - ALIAS("tc_new_nothrow"); -void* operator new[](size_t size, const std::nothrow_t&) __THROW - ALIAS("tc_newarray_nothrow"); -void operator delete(void* size, const std::nothrow_t&) __THROW - ALIAS("tc_delete_nothrow"); -void operator delete[](void* size, const std::nothrow_t&) __THROW - ALIAS("tc_deletearray_nothrow"); -extern "C" { - void* malloc(size_t size) __THROW ALIAS("tc_malloc"); - void free(void* ptr) __THROW ALIAS("tc_free"); - void* realloc(void* ptr, size_t size) __THROW ALIAS("tc_realloc"); - void* calloc(size_t n, size_t size) __THROW ALIAS("tc_calloc"); - void cfree(void* ptr) __THROW ALIAS("tc_cfree"); - void* memalign(size_t align, size_t s) __THROW ALIAS("tc_memalign"); - void* valloc(size_t size) __THROW ALIAS("tc_valloc"); - void* pvalloc(size_t size) __THROW ALIAS("tc_pvalloc"); - int posix_memalign(void** r, size_t a, size_t s) __THROW - ALIAS("tc_posix_memalign"); - void malloc_stats(void) __THROW ALIAS("tc_malloc_stats"); - int mallopt(int cmd, int value) __THROW ALIAS("tc_mallopt"); -#ifdef HAVE_STRUCT_MALLINFO - struct mallinfo mallinfo(void) __THROW ALIAS("tc_mallinfo"); -#endif - size_t malloc_size(void* p) __THROW ALIAS("tc_malloc_size"); - size_t malloc_usable_size(void* p) __THROW ALIAS("tc_malloc_size"); -} // extern "C" -#else // #if defined(__GNUC__) && !defined(__MACH__) -// Portable wrappers -void* operator new(size_t size) { return tc_new(size); } -void operator delete(void* p) __THROW { tc_delete(p); } -void* operator new[](size_t size) { return tc_newarray(size); } -void operator delete[](void* p) __THROW { tc_deletearray(p); } -void* operator new(size_t size, const std::nothrow_t& nt) __THROW { - return tc_new_nothrow(size, nt); -} -void* operator new[](size_t size, const std::nothrow_t& nt) __THROW { - return tc_newarray_nothrow(size, nt); -} -void operator delete(void* ptr, const std::nothrow_t& nt) __THROW { - return tc_delete_nothrow(ptr, nt); -} -void operator delete[](void* ptr, const std::nothrow_t& nt) __THROW { - return tc_deletearray_nothrow(ptr, nt); -} -extern "C" { - void* malloc(size_t s) __THROW { return tc_malloc(s); } - void free(void* p) __THROW { tc_free(p); } - void* realloc(void* p, size_t s) __THROW { return tc_realloc(p, s); } - void* calloc(size_t n, size_t s) __THROW { return tc_calloc(n, s); } - void cfree(void* p) __THROW { tc_cfree(p); } - void* memalign(size_t a, size_t s) __THROW { return tc_memalign(a, s); } - void* valloc(size_t s) __THROW { return tc_valloc(s); } - void* pvalloc(size_t s) __THROW { return tc_pvalloc(s); } - int posix_memalign(void** r, size_t a, size_t s) __THROW { - return tc_posix_memalign(r, a, s); - } - void malloc_stats(void) __THROW { tc_malloc_stats(); } - int mallopt(int cmd, int v) __THROW { return tc_mallopt(cmd, v); } -#ifdef HAVE_STRUCT_MALLINFO - struct mallinfo mallinfo(void) __THROW { return tc_mallinfo(); } -#endif - size_t malloc_size(void* p) __THROW { return tc_malloc_size(p); } - size_t malloc_usable_size(void* p) __THROW { return tc_malloc_size(p); } -} // extern "C" -#endif // #if defined(__GNUC__) - -// Some library routines on RedHat 9 allocate memory using malloc() -// and free it using __libc_free() (or vice-versa). Since we provide -// our own implementations of malloc/free, we need to make sure that -// the __libc_XXX variants (defined as part of glibc) also point to -// the same implementations. -#ifdef __GLIBC__ // only glibc defines __libc_* -extern "C" { -#ifdef ALIAS - void* __libc_malloc(size_t size) ALIAS("tc_malloc"); - void __libc_free(void* ptr) ALIAS("tc_free"); - void* __libc_realloc(void* ptr, size_t size) ALIAS("tc_realloc"); - void* __libc_calloc(size_t n, size_t size) ALIAS("tc_calloc"); - void __libc_cfree(void* ptr) ALIAS("tc_cfree"); - void* __libc_memalign(size_t align, size_t s) ALIAS("tc_memalign"); - void* __libc_valloc(size_t size) ALIAS("tc_valloc"); - void* __libc_pvalloc(size_t size) ALIAS("tc_pvalloc"); - int __posix_memalign(void** r, size_t a, size_t s) ALIAS("tc_posix_memalign"); -#else // #ifdef ALIAS - void* __libc_malloc(size_t size) { return malloc(size); } - void __libc_free(void* ptr) { free(ptr); } - void* __libc_realloc(void* ptr, size_t size) { return realloc(ptr, size); } - void* __libc_calloc(size_t n, size_t size) { return calloc(n, size); } - void __libc_cfree(void* ptr) { cfree(ptr); } - void* __libc_memalign(size_t align, size_t s) { return memalign(align, s); } - void* __libc_valloc(size_t size) { return valloc(size); } - void* __libc_pvalloc(size_t size) { return pvalloc(size); } - int __posix_memalign(void** r, size_t a, size_t s) { - return posix_memalign(r, a, s); - } -#endif // #ifdef ALIAS -} // extern "C" -#endif // ifdef __GLIBC__ - -#if defined(__GLIBC__) && defined(HAVE_MALLOC_H) -// If we're using glibc, then override glibc malloc hooks to make sure that even -// if calls fall through to ptmalloc (due to dlopen() with RTLD_DEEPBIND or what -// not), ptmalloc will use TCMalloc. - -static void* tc_ptmalloc_malloc_hook(size_t size, const void* caller) { - return tc_malloc(size); -} - -void* (*__MALLOC_HOOK_VOLATILE __malloc_hook)( - size_t size, const void* caller) = tc_ptmalloc_malloc_hook; - -static void* tc_ptmalloc_realloc_hook( - void* ptr, size_t size, const void* caller) { - return tc_realloc(ptr, size); -} - -void* (*__MALLOC_HOOK_VOLATILE __realloc_hook)( - void* ptr, size_t size, const void* caller) = tc_ptmalloc_realloc_hook; - -static void tc_ptmalloc_free_hook(void* ptr, const void* caller) { - tc_free(ptr); -} - -void (*__MALLOC_HOOK_VOLATILE __free_hook)(void* ptr, const void* caller) = tc_ptmalloc_free_hook; - -#endif - -#endif // #ifndef _WIN32 -#undef ALIAS - -#endif // #ifndef(WIN32_DO_PATCHING) - // ----------------------- IMPLEMENTATION ------------------------------- @@ -446,16 +294,18 @@ static int tc_new_mode = 0; // See tc_set_new_mode(). // required) kind of exception handling for these routines. namespace { void InvalidFree(void* ptr) { - CRASH("Attempt to free invalid pointer: %p\n", ptr); + Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr); } -size_t InvalidGetSizeForRealloc(void* old_ptr) { - CRASH("Attempt to realloc invalid pointer: %p\n", old_ptr); +size_t InvalidGetSizeForRealloc(const void* old_ptr) { + Log(kCrash, __FILE__, __LINE__, + "Attempt to realloc invalid pointer", old_ptr); return 0; } -size_t InvalidGetAllocatedSize(void* ptr) { - CRASH("Attempt to get the size of an invalid pointer: %p\n", ptr); +size_t InvalidGetAllocatedSize(const void* ptr) { + Log(kCrash, __FILE__, __LINE__, + "Attempt to get the size of an invalid pointer", ptr); return 0; } } // unnamed namespace @@ -470,15 +320,18 @@ struct TCMallocStats { }; // Get stats into "r". Also get per-size-class counts if class_count != NULL -static void ExtractStats(TCMallocStats* r, uint64_t* class_count) { +static void ExtractStats(TCMallocStats* r, uint64_t* class_count, + PageHeap::SmallSpanStats* small_spans, + PageHeap::LargeSpanStats* large_spans) { r->central_bytes = 0; r->transfer_bytes = 0; for (int cl = 0; cl < kNumClasses; ++cl) { const int length = Static::central_cache()[cl].length(); const int tc_length = Static::central_cache()[cl].tc_length(); + const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes(); const size_t size = static_cast<uint64_t>( Static::sizemap()->ByteSizeForClass(cl)); - r->central_bytes += (size * length); + r->central_bytes += (size * length) + cache_overhead; r->transfer_bytes += (size * tc_length); if (class_count) class_count[cl] = length + tc_length; } @@ -490,14 +343,30 @@ static void ExtractStats(TCMallocStats* r, uint64_t* class_count) { ThreadCache::GetThreadStats(&r->thread_bytes, class_count); r->metadata_bytes = tcmalloc::metadata_system_bytes(); r->pageheap = Static::pageheap()->stats(); + if (small_spans != NULL) { + Static::pageheap()->GetSmallSpanStats(small_spans); + } + if (large_spans != NULL) { + Static::pageheap()->GetLargeSpanStats(large_spans); + } } } +static double PagesToMiB(uint64_t pages) { + return (pages << kPageShift) / 1048576.0; +} + // WRITE stats to "out" static void DumpStats(TCMalloc_Printer* out, int level) { TCMallocStats stats; uint64_t class_count[kNumClasses]; - ExtractStats(&stats, (level >= 2 ? class_count : NULL)); + PageHeap::SmallSpanStats small; + PageHeap::LargeSpanStats large; + if (level >= 2) { + ExtractStats(&stats, class_count, &small, &large); + } else { + ExtractStats(&stats, NULL, NULL, NULL); + } static const double MiB = 1048576.0; @@ -581,8 +450,48 @@ static void DumpStats(TCMalloc_Printer* out, int level) { } } - SpinLockHolder h(Static::pageheap_lock()); - Static::pageheap()->Dump(out); + // append page heap info + int nonempty_sizes = 0; + for (int s = 0; s < kMaxPages; s++) { + if (small.normal_length[s] + small.returned_length[s] > 0) { + nonempty_sizes++; + } + } + out->printf("------------------------------------------------\n"); + out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n", + nonempty_sizes, stats.pageheap.free_bytes / MiB, + stats.pageheap.unmapped_bytes / MiB); + out->printf("------------------------------------------------\n"); + uint64_t total_normal = 0; + uint64_t total_returned = 0; + for (int s = 0; s < kMaxPages; s++) { + const int n_length = small.normal_length[s]; + const int r_length = small.returned_length[s]; + if (n_length + r_length > 0) { + uint64_t n_pages = s * n_length; + uint64_t r_pages = s * r_length; + total_normal += n_pages; + total_returned += r_pages; + out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum" + "; unmapped: %6.1f MiB; %6.1f MiB cum\n", + s, + (n_length + r_length), + PagesToMiB(n_pages + r_pages), + PagesToMiB(total_normal + total_returned), + PagesToMiB(r_pages), + PagesToMiB(total_returned)); + } + } + + total_normal += large.normal_pages; + total_returned += large.returned_pages; + out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum" + "; unmapped: %6.1f MiB; %6.1f MiB cum\n", + static_cast<unsigned int>(large.spans), + PagesToMiB(large.normal_pages + large.returned_pages), + PagesToMiB(total_normal + total_returned), + PagesToMiB(large.returned_pages), + PagesToMiB(total_returned)); } } @@ -612,8 +521,9 @@ static void** DumpHeapGrowthStackTraces() { void** result = new void*[needed_slots]; if (result == NULL) { - MESSAGE("tcmalloc: allocation failed for stack trace slots", - needed_slots * sizeof(*result)); + Log(kLog, __FILE__, __LINE__, + "tcmalloc: allocation failed for stack trace slots", + needed_slots * sizeof(*result)); return NULL; } @@ -739,7 +649,7 @@ class TCMallocImplementation : public MallocExtension { if (strcmp(name, "generic.current_allocated_bytes") == 0) { TCMallocStats stats; - ExtractStats(&stats, NULL); + ExtractStats(&stats, NULL, NULL, NULL); *value = stats.pageheap.system_bytes - stats.thread_bytes - stats.central_bytes @@ -751,7 +661,7 @@ class TCMallocImplementation : public MallocExtension { if (strcmp(name, "generic.heap_size") == 0) { TCMallocStats stats; - ExtractStats(&stats, NULL); + ExtractStats(&stats, NULL, NULL, NULL); *value = stats.pageheap.system_bytes; return true; } @@ -785,7 +695,7 @@ class TCMallocImplementation : public MallocExtension { if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) { TCMallocStats stats; - ExtractStats(&stats, NULL); + ExtractStats(&stats, NULL, NULL, NULL); *value = stats.thread_bytes; return true; } @@ -866,7 +776,26 @@ class TCMallocImplementation : public MallocExtension { // This just calls GetSizeWithCallback, but because that's in an // unnamed namespace, we need to move the definition below it in the // file. - virtual size_t GetAllocatedSize(void* ptr); + virtual size_t GetAllocatedSize(const void* ptr); + + // This duplicates some of the logic in GetSizeWithCallback, but is + // faster. This is important on OS X, where this function is called + // on every allocation operation. + virtual Ownership GetOwnership(const void* ptr) { + const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; + // The rest of tcmalloc assumes that all allocated pointers use at + // most kAddressBits bits. If ptr doesn't, then it definitely + // wasn't alloacted by tcmalloc. + if ((p >> (kAddressBits - kPageShift)) > 0) { + return kNotOwned; + } + size_t cl = Static::pageheap()->GetSizeClassIfCached(p); + if (cl != 0) { + return kOwned; + } + const Span *span = Static::pageheap()->GetDescriptor(p); + return span ? kOwned : kNotOwned; + } virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { static const char* kCentralCacheType = "tcmalloc.central"; @@ -921,42 +850,39 @@ class TCMallocImplementation : public MallocExtension { } // append page heap info - int64 page_count_normal[kMaxPages]; - int64 page_count_returned[kMaxPages]; - int64 span_count_normal; - int64 span_count_returned; + PageHeap::SmallSpanStats small; + PageHeap::LargeSpanStats large; { SpinLockHolder h(Static::pageheap_lock()); - Static::pageheap()->GetClassSizes(page_count_normal, - page_count_returned, - &span_count_normal, - &span_count_returned); + Static::pageheap()->GetSmallSpanStats(&small); + Static::pageheap()->GetLargeSpanStats(&large); } - // spans: mapped + // large spans: mapped MallocExtension::FreeListInfo span_info; span_info.type = kLargeSpanType; span_info.max_object_size = (numeric_limits<size_t>::max)(); span_info.min_object_size = kMaxPages << kPageShift; - span_info.total_bytes_free = span_count_normal << kPageShift; + span_info.total_bytes_free = large.normal_pages << kPageShift; v->push_back(span_info); - // spans: unmapped + // large spans: unmapped span_info.type = kLargeUnmappedSpanType; - span_info.total_bytes_free = span_count_returned << kPageShift; + span_info.total_bytes_free = large.returned_pages << kPageShift; v->push_back(span_info); + // small spans for (int s = 1; s < kMaxPages; s++) { MallocExtension::FreeListInfo i; i.max_object_size = (s << kPageShift); i.min_object_size = ((s - 1) << kPageShift); i.type = kPageHeapType; - i.total_bytes_free = (s << kPageShift) * page_count_normal[s]; + i.total_bytes_free = (s << kPageShift) * small.normal_length[s]; v->push_back(i); i.type = kPageHeapUnmappedType; - i.total_bytes_free = (s << kPageShift) * page_count_returned[s]; + i.total_bytes_free = (s << kPageShift) * small.returned_length[s]; v->push_back(i); } } @@ -981,10 +907,7 @@ TCMallocGuard::TCMallocGuard() { // Check whether the kernel also supports TLS (needs to happen at runtime) tcmalloc::CheckIfKernelSupportsTLS(); #endif -#ifdef WIN32_DO_PATCHING - // patch the windows VirtualAlloc, etc. - PatchWindowsFunctions(); // defined in windows/patch_functions.cc -#endif + ReplaceSystemAlloc(); // defined in libc_override_*.h tc_free(tc_malloc(1)); ThreadCache::InitTSD(); tc_free(tc_malloc(1)); @@ -1081,8 +1004,8 @@ static void ReportLargeAlloc(Length num_pages, void* result) { static const int N = 1000; char buffer[N]; TCMalloc_Printer printer(buffer, N); - printer.printf("tcmalloc: large alloc %llu bytes == %p @ ", - static_cast<unsigned long long>(num_pages) << kPageShift, + printer.printf("tcmalloc: large alloc %"PRIu64" bytes == %p @ ", + static_cast<uint64>(num_pages) << kPageShift, result); for (int i = 0; i < stack.depth; i++) { printer.printf(" %p", stack.stack[i]); @@ -1171,7 +1094,7 @@ inline void* do_malloc(size_t size) { ret = DoSampledAllocation(size); MarkAllocatedRegion(ret); } else { - // The common case, and also the simplest. This just pops the + // The common case, and also the simplest. This just pops the // size-appropriate freelist, after replenishing it if it's empty. ret = CheckedMallocResult(heap->Allocate(size, cl)); } @@ -1204,7 +1127,15 @@ static inline ThreadCache* GetCacheIfPresent() { // It is used primarily by windows code which wants a specialized callback. inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) { if (ptr == NULL) return; - ASSERT(Static::pageheap() != NULL); // Should not call free() before malloc() + if (Static::pageheap() == NULL) { + // We called free() before malloc(). This can occur if the + // (system) malloc() is called before tcmalloc is loaded, and then + // free() is called after tcmalloc is loaded (and tc_free has + // replaced free), but before the global constructor has run that + // sets up the tcmalloc data structures. + (*invalid_free_fn)(ptr); // Decide how to handle the bad free request + return; + } const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; Span* span = NULL; size_t cl = Static::pageheap()->GetSizeClassIfCached(p); @@ -1257,8 +1188,10 @@ inline void do_free(void* ptr) { return do_free_with_callback(ptr, &InvalidFree); } -inline size_t GetSizeWithCallback(void* ptr, - size_t (*invalid_getsize_fn)(void*)) { +// NOTE: some logic here is duplicated in GetOwnership (above), for +// speed. If you change this function, look at that one too. +inline size_t GetSizeWithCallback(const void* ptr, + size_t (*invalid_getsize_fn)(const void*)) { if (ptr == NULL) return 0; const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; @@ -1266,7 +1199,7 @@ inline size_t GetSizeWithCallback(void* ptr, if (cl != 0) { return Static::sizemap()->ByteSizeForClass(cl); } else { - Span *span = Static::pageheap()->GetDescriptor(p); + const Span *span = Static::pageheap()->GetDescriptor(p); if (span == NULL) { // means we do not own this memory return (*invalid_getsize_fn)(ptr); } else if (span->sizeclass != 0) { @@ -1283,7 +1216,7 @@ inline size_t GetSizeWithCallback(void* ptr, inline void* do_realloc_with_callback( void* old_ptr, size_t new_size, void (*invalid_free_fn)(void*), - size_t (*invalid_get_size_fn)(void*)) { + size_t (*invalid_get_size_fn)(const void*)) { AddRoomForMark(&new_size); // Get the size of the old entry const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn); @@ -1429,7 +1362,7 @@ inline int do_mallopt(int cmd, int value) { #ifdef HAVE_STRUCT_MALLINFO inline struct mallinfo do_mallinfo() { TCMallocStats stats; - ExtractStats(&stats, NULL); + ExtractStats(&stats, NULL, NULL, NULL); // Just some of the fields are filled in. struct mallinfo info; @@ -1553,7 +1486,9 @@ void* cpp_memalign(size_t align, size_t size) { } // end unnamed namespace // As promised, the definition of this function, declared above. -size_t TCMallocImplementation::GetAllocatedSize(void* ptr) { +size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) { + ASSERT(TCMallocImplementation::GetOwnership(ptr) + != TCMallocImplementation::kNotOwned); return ExcludeSpaceForMark( GetSizeWithCallback(ptr, &InvalidGetAllocatedSize)); } @@ -1752,27 +1687,9 @@ extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { #endif extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { - return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); + return MallocExtension::instance()->GetAllocatedSize(ptr); } - -// Override __libc_memalign in libc on linux boxes specially. -// They have a bug in libc that causes them to (very rarely) allocate -// with __libc_memalign() yet deallocate with free() and the -// definitions above don't catch it. -// This function is an exception to the rule of calling MallocHook method -// from the stack frame of the allocation function; -// heap-checker handles this special case explicitly. -static void *MemalignOverride(size_t align, size_t size, const void *caller) - __THROW ATTRIBUTE_SECTION(google_malloc); - -static void *MemalignOverride(size_t align, size_t size, const void *caller) - __THROW { - void* result = do_memalign_or_cpp_memalign(align, size); - MallocHook::InvokeNewHook(result, size); - return result; -} -void *(*__MALLOC_HOOK_VOLATILE __memalign_hook)(size_t, size_t, const void *) = MemalignOverride; #endif // TCMALLOC_USING_DEBUGALLOCATION // ---Double free() debugging implementation ----------------------------------- @@ -1823,7 +1740,7 @@ static void DieFromDoubleFree() { *p += 1; // Segv. } -static size_t DieFromBadFreePointer(void* unused) { +static size_t DieFromBadFreePointer(const void* unused) { char* p = NULL; p += 2; *p += 2; // Segv. diff --git a/third_party/tcmalloc/chromium/src/tests/current_allocated_bytes_test.cc b/third_party/tcmalloc/chromium/src/tests/current_allocated_bytes_test.cc index 8188e7b..e05ec18 100644 --- a/third_party/tcmalloc/chromium/src/tests/current_allocated_bytes_test.cc +++ b/third_party/tcmalloc/chromium/src/tests/current_allocated_bytes_test.cc @@ -42,7 +42,7 @@ #include "config_for_unittests.h" #include <stdlib.h> #include <stdio.h> -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> #include "base/logging.h" const char kCurrent[] = "generic.current_allocated_bytes"; diff --git a/third_party/tcmalloc/chromium/src/tests/debugallocation_test.cc b/third_party/tcmalloc/chromium/src/tests/debugallocation_test.cc index 07b8604..56ae30e 100644 --- a/third_party/tcmalloc/chromium/src/tests/debugallocation_test.cc +++ b/third_party/tcmalloc/chromium/src/tests/debugallocation_test.cc @@ -33,7 +33,7 @@ #include <stdio.h> #include <stdlib.h> #include <vector> -#include "google/malloc_extension.h" +#include "gperftools/malloc_extension.h" #include "base/logging.h" using std::vector; @@ -75,7 +75,10 @@ static int test_counter = 0; // incremented every time the macro is called // This flag won't be compiled in in opt mode. DECLARE_int32(max_free_queue_size); -// Test match as well as mismatch rules: +// Test match as well as mismatch rules. But do not test on OS X; on +// OS X the OS converts new/new[] to malloc before it gets to us, so +// we are unable to catch these mismatch errors. +#ifndef __APPLE__ TEST(DebugAllocationTest, DeallocMismatch) { // malloc can be matched only by free // new can be matched only by delete and delete(nothrow) @@ -132,6 +135,7 @@ TEST(DebugAllocationTest, DeallocMismatch) { ::operator delete[](y, std::nothrow); } } +#endif // #ifdef OS_MACOSX TEST(DebugAllocationTest, DoubleFree) { int* pint = new int; diff --git a/third_party/tcmalloc/chromium/src/tests/frag_unittest.cc b/third_party/tcmalloc/chromium/src/tests/frag_unittest.cc index 5ba02bd..1242770 100644 --- a/third_party/tcmalloc/chromium/src/tests/frag_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/frag_unittest.cc @@ -45,7 +45,7 @@ #include <vector> #include "base/logging.h" #include "common.h" -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> using std::vector; diff --git a/third_party/tcmalloc/chromium/src/tests/heap-checker-death_unittest.sh b/third_party/tcmalloc/chromium/src/tests/heap-checker-death_unittest.sh index 4a83fc2..ab4a666 100644 --- a/third_party/tcmalloc/chromium/src/tests/heap-checker-death_unittest.sh +++ b/third_party/tcmalloc/chromium/src/tests/heap-checker-death_unittest.sh @@ -84,11 +84,11 @@ Test() { output="$TMPDIR/output" ALARM $timeout env "$@" $EXE > "$output" 2>&1 actual_ec=$? - ec_ok=$(expr "$actual_ec" : "$expected_ec$" >/dev/null || echo false) - matches_ok=$(test -z "$expected_regexp" || \ - grep -q "$expected_regexp" "$output" || echo false) - negmatches_ok=$(test -z "$unexpected_regexp" || \ - ! grep -q "$unexpected_regexp" "$output" || echo false) + ec_ok=`expr "$actual_ec" : "$expected_ec$" >/dev/null || echo false` + matches_ok=`test -z "$expected_regexp" || \ + grep "$expected_regexp" "$output" >/dev/null 2>&1 || echo false` + negmatches_ok=`test -z "$unexpected_regexp" || \ + ! grep "$unexpected_regexp" "$output" >/dev/null 2>&1 || echo false` if $ec_ok && $matches_ok && $negmatches_ok; then echo "PASS" return 0 # 0: success diff --git a/third_party/tcmalloc/chromium/src/tests/heap-checker_unittest.cc b/third_party/tcmalloc/chromium/src/tests/heap-checker_unittest.cc index 404c9f1..ab326c9 100644 --- a/third_party/tcmalloc/chromium/src/tests/heap-checker_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/heap-checker_unittest.cc @@ -101,10 +101,10 @@ #include "base/logging.h" #include "base/commandlineflags.h" #include "base/thread_lister.h" -#include <google/heap-checker.h> +#include <gperftools/heap-checker.h> #include "memory_region_map.h" -#include <google/malloc_extension.h> -#include <google/stacktrace.h> +#include <gperftools/malloc_extension.h> +#include <gperftools/stacktrace.h> // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old // form of the name instead. @@ -1381,7 +1381,13 @@ int main(int argc, char** argv) { RunHidden(NewCallback(MakeALeak, &arr)); Use(&arr); LogHidden("Leaking", arr); - if (FLAGS_test_cancel_global_check) HeapLeakChecker::CancelGlobalCheck(); + if (FLAGS_test_cancel_global_check) { + HeapLeakChecker::CancelGlobalCheck(); + } else { + // Verify we can call NoGlobalLeaks repeatedly without deadlocking + HeapLeakChecker::NoGlobalLeaks(); + HeapLeakChecker::NoGlobalLeaks(); + } return Pass(); // whole-program leak-check should (with very high probability) // catch the leak of arr (10 * sizeof(int) bytes) @@ -1396,7 +1402,13 @@ int main(int argc, char** argv) { Use(&arr2); LogHidden("Loop leaking", arr1); LogHidden("Loop leaking", arr2); - if (FLAGS_test_cancel_global_check) HeapLeakChecker::CancelGlobalCheck(); + if (FLAGS_test_cancel_global_check) { + HeapLeakChecker::CancelGlobalCheck(); + } else { + // Verify we can call NoGlobalLeaks repeatedly without deadlocking + HeapLeakChecker::NoGlobalLeaks(); + HeapLeakChecker::NoGlobalLeaks(); + } return Pass(); // whole-program leak-check should (with very high probability) // catch the leak of arr1 and arr2 (4 * sizeof(void*) bytes) diff --git a/third_party/tcmalloc/chromium/src/tests/heap-profiler_unittest.cc b/third_party/tcmalloc/chromium/src/tests/heap-profiler_unittest.cc index e5dedee..5fd8bb7 100644 --- a/third_party/tcmalloc/chromium/src/tests/heap-profiler_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/heap-profiler_unittest.cc @@ -49,7 +49,7 @@ #include <string> #include "base/basictypes.h" #include "base/logging.h" -#include <google/heap-profiler.h> +#include <gperftools/heap-profiler.h> using std::string; diff --git a/third_party/tcmalloc/chromium/src/tests/low_level_alloc_unittest.cc b/third_party/tcmalloc/chromium/src/tests/low_level_alloc_unittest.cc index 4228e12..0e5a48a 100644 --- a/third_party/tcmalloc/chromium/src/tests/low_level_alloc_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/low_level_alloc_unittest.cc @@ -34,7 +34,7 @@ #include <map> #include "base/low_level_alloc.h" #include "base/logging.h" -#include <google/malloc_hook.h> +#include <gperftools/malloc_hook.h> using std::map; diff --git a/third_party/tcmalloc/chromium/src/tests/malloc_extension_c_test.c b/third_party/tcmalloc/chromium/src/tests/malloc_extension_c_test.c index e384b76..af0e0c1 100644 --- a/third_party/tcmalloc/chromium/src/tests/malloc_extension_c_test.c +++ b/third_party/tcmalloc/chromium/src/tests/malloc_extension_c_test.c @@ -40,8 +40,8 @@ #include <stdio.h> #include <stdlib.h> #include <stddef.h> /* for size_t */ -#include <google/malloc_extension_c.h> -#include <google/malloc_hook_c.h> +#include <gperftools/malloc_extension_c.h> +#include <gperftools/malloc_hook_c.h> #define FAIL(msg) do { \ fprintf(stderr, "FATAL ERROR: %s\n", msg); \ @@ -126,6 +126,14 @@ void TestMallocExtension(void) { if (MallocExtension_GetAllocatedSize(x) < 10) { FAIL("GetEstimatedAllocatedSize returned a bad value (too small)"); } + if (MallocExtension_GetOwnership(x) != MallocExtension_kOwned) { + FAIL("DidAllocatePtr returned a bad value (kNotOwned)"); + } + /* TODO(csilvers): this relies on undocumented behavior that + GetOwnership works on stack-allocated variables. Use a better test. */ + if (MallocExtension_GetOwnership(hist) != MallocExtension_kNotOwned) { + FAIL("DidAllocatePtr returned a bad value (kOwned)"); + } free(x); } diff --git a/third_party/tcmalloc/chromium/src/tests/malloc_extension_test.cc b/third_party/tcmalloc/chromium/src/tests/malloc_extension_test.cc index 0bd85ad..58fef7e 100644 --- a/third_party/tcmalloc/chromium/src/tests/malloc_extension_test.cc +++ b/third_party/tcmalloc/chromium/src/tests/malloc_extension_test.cc @@ -36,8 +36,8 @@ #include <stdio.h> #include <sys/types.h> #include "base/logging.h" -#include <google/malloc_extension.h> -#include <google/malloc_extension_c.h> +#include <gperftools/malloc_extension.h> +#include <gperftools/malloc_extension_c.h> using STL_NAMESPACE::vector; @@ -55,6 +55,14 @@ int main(int argc, char** argv) { ASSERT_TRUE(MallocExtension::instance()->VerifyAllMemory()); ASSERT_TRUE(MallocExtension_VerifyAllMemory()); + ASSERT_EQ(MallocExtension::kOwned, + MallocExtension::instance()->GetOwnership(a)); + // TODO(csilvers): this relies on undocumented behavior that + // GetOwnership works on stack-allocated variables. Use a better test. + ASSERT_EQ(MallocExtension::kNotOwned, + MallocExtension::instance()->GetOwnership(&cxx_bytes_used)); + ASSERT_EQ(MallocExtension::kNotOwned, + MallocExtension::instance()->GetOwnership(NULL)); ASSERT_GE(MallocExtension::instance()->GetAllocatedSize(a), 1000); // This is just a sanity check. If we allocated too much, tcmalloc is broken ASSERT_LE(MallocExtension::instance()->GetAllocatedSize(a), 5000); @@ -68,12 +76,24 @@ int main(int argc, char** argv) { } // Check the c-shim version too. + ASSERT_EQ(MallocExtension_kOwned, MallocExtension_GetOwnership(a)); + ASSERT_EQ(MallocExtension_kNotOwned, + MallocExtension_GetOwnership(&cxx_bytes_used)); + ASSERT_EQ(MallocExtension_kNotOwned, MallocExtension_GetOwnership(NULL)); ASSERT_GE(MallocExtension_GetAllocatedSize(a), 1000); ASSERT_LE(MallocExtension_GetAllocatedSize(a), 5000); ASSERT_GE(MallocExtension_GetEstimatedAllocatedSize(1000), 1000); free(a); + // Verify that the .cc file and .h file have the same enum values. + ASSERT_EQ(static_cast<int>(MallocExtension::kUnknownOwnership), + static_cast<int>(MallocExtension_kUnknownOwnership)); + ASSERT_EQ(static_cast<int>(MallocExtension::kOwned), + static_cast<int>(MallocExtension_kOwned)); + ASSERT_EQ(static_cast<int>(MallocExtension::kNotOwned), + static_cast<int>(MallocExtension_kNotOwned)); + printf("DONE\n"); return 0; } diff --git a/third_party/tcmalloc/chromium/src/tests/malloc_hook_test.cc b/third_party/tcmalloc/chromium/src/tests/malloc_hook_test.cc index dc65b68..cbf526a 100644 --- a/third_party/tcmalloc/chromium/src/tests/malloc_hook_test.cc +++ b/third_party/tcmalloc/chromium/src/tests/malloc_hook_test.cc @@ -36,16 +36,25 @@ #ifdef HAVE_MMAP #include <sys/mman.h> #endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> // for sleep() +#endif #include <algorithm> #include <string> #include <vector> -#include <google/malloc_hook.h> +#include <gperftools/malloc_hook.h> #include "malloc_hook-inl.h" #include "base/logging.h" -#include "base/spinlock.h" +#include "base/simple_mutex.h" #include "base/sysinfo.h" #include "tests/testutil.h" +// On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old +// form of the name instead. +#ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS MAP_ANON +#endif + namespace { using std::string; @@ -72,6 +81,15 @@ static int RUN_ALL_TESTS() { return 0; } +void Sleep(int seconds) { +#ifdef _MSC_VER + _sleep(seconds * 1000); // Windows's _sleep takes milliseconds argument +#else + sleep(seconds); +#endif +} + +using std::min; using base::internal::kHookListMaxValues; // Since HookList is a template and is defined in malloc_hook.cc, we can only @@ -81,7 +99,7 @@ typedef base::internal::HookList<MallocHook::NewHook> TestHookList; int TestHookList_Traverse(const TestHookList& list, int* output_array, int n) { MallocHook::NewHook values_as_hooks[kHookListMaxValues]; - int result = list.Traverse(values_as_hooks, std::min(n, kHookListMaxValues)); + int result = list.Traverse(values_as_hooks, min(n, kHookListMaxValues)); for (int i = 0; i < result; ++i) { output_array[i] = reinterpret_cast<const int&>(values_as_hooks[i]); } @@ -229,12 +247,12 @@ void MultithreadedTestThread(TestHookList* list, int shift, static volatile int num_threads_remaining; static TestHookList list = INIT_HOOK_LIST(69); -static SpinLock threadcount_lock; +static Mutex threadcount_lock; void MultithreadedTestThreadRunner(int thread_num) { // Wait for all threads to start running. { - SpinLockHolder h(&threadcount_lock); + MutexLock ml(&threadcount_lock); assert(num_threads_remaining > 0); --num_threads_remaining; @@ -242,7 +260,7 @@ void MultithreadedTestThreadRunner(int thread_num) { // go simple and busy-wait. while (num_threads_remaining > 0) { threadcount_lock.Unlock(); - SleepForMilliseconds(100); + Sleep(1); threadcount_lock.Lock(); } } @@ -271,7 +289,10 @@ TEST(HookListTest, MultithreadedTest) { EXPECT_EQ(0, list.priv_end); } -#ifdef HAVE_MMAP +// We only do mmap-hooking on (some) linux systems. +#if defined(HAVE_MMAP) && defined(__linux) && \ + (defined(__i386__) || defined(__x86_64__) || defined(__PPC__)) + int mmap_calls = 0; int mmap_matching_calls = 0; int munmap_calls = 0; @@ -336,7 +357,7 @@ TEST(MallocMookTest, MmapReplacements) { // whoever owns that memory now. // EXPECT_DEATH(*ptr = 'a', "SIGSEGV"); } -#endif // #ifdef HAVE_MMAN +#endif // #ifdef HAVE_MMAP && linux && ... } // namespace diff --git a/third_party/tcmalloc/chromium/src/tests/markidle_unittest.cc b/third_party/tcmalloc/chromium/src/tests/markidle_unittest.cc index ac9971f..2f150ab 100644 --- a/third_party/tcmalloc/chromium/src/tests/markidle_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/markidle_unittest.cc @@ -35,7 +35,7 @@ #include "config_for_unittests.h" #include "base/logging.h" -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> #include "tests/testutil.h" // for RunThread() // Helper routine to do lots of allocations diff --git a/third_party/tcmalloc/chromium/src/tests/profile-handler_unittest.cc b/third_party/tcmalloc/chromium/src/tests/profile-handler_unittest.cc index 84e035c..98cfe6d 100644 --- a/third_party/tcmalloc/chromium/src/tests/profile-handler_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/profile-handler_unittest.cc @@ -3,6 +3,13 @@ // Chris Demetriou (cgd@google.com) // // This file contains the unit tests for profile-handler.h interface. +// +// It is linked into three separate unit tests: +// profile-handler_unittest tests basic functionality +// profile-handler_disable_test tests that the profiler +// is disabled with --install_signal_handlers=false +// profile-handler_conflict_test tests that the profiler +// is disabled when a SIGPROF handler is registered before InitGoogle. #include "config.h" #include "profile-handler.h" @@ -17,6 +24,16 @@ // Some helpful macros for the test class #define TEST_F(cls, fn) void cls :: fn() +// Do we expect the profiler to be enabled? +DEFINE_bool(test_profiler_enabled, true, + "expect profiler to be enabled during tests"); + +// Should we look at the kernel signal handler settings during the test? +// Not if we're in conflict_test, because we can't distinguish its nop +// handler from the real one. +DEFINE_bool(test_profiler_signal_handler, true, + "check profiler signal handler during tests"); + namespace { // TODO(csilvers): error-checking on the pthreads routines @@ -278,17 +295,24 @@ class ProfileHandlerTest { // Check the callback count. EXPECT_GT(GetCallbackCount(), 0); // Check that the profile timer is enabled. - EXPECT_TRUE(IsTimerEnabled()); + EXPECT_EQ(FLAGS_test_profiler_enabled, IsTimerEnabled()); // Check that the signal handler is enabled. - EXPECT_TRUE(IsSignalEnabled()); + if (FLAGS_test_profiler_signal_handler) { + EXPECT_EQ(FLAGS_test_profiler_enabled, IsSignalEnabled()); + } uint64 interrupts_before = GetInterruptCount(); // Sleep for a bit and check that tick counter is making progress. int old_tick_count = tick_counter; Delay(kSleepInterval); int new_tick_count = tick_counter; - EXPECT_GT(new_tick_count, old_tick_count); uint64 interrupts_after = GetInterruptCount(); - EXPECT_GT(interrupts_after, interrupts_before); + if (FLAGS_test_profiler_enabled) { + EXPECT_GT(new_tick_count, old_tick_count); + EXPECT_GT(interrupts_after, interrupts_before); + } else { + EXPECT_EQ(new_tick_count, old_tick_count); + EXPECT_EQ(interrupts_after, interrupts_before); + } } // Verifies that a callback is not receiving profile ticks. @@ -300,7 +324,9 @@ class ProfileHandlerTest { EXPECT_EQ(old_tick_count, new_tick_count); // If no callbacks, signal handler and shared timer should be disabled. if (GetCallbackCount() == 0) { - EXPECT_FALSE(IsSignalEnabled()); + if (FLAGS_test_profiler_signal_handler) { + EXPECT_FALSE(IsSignalEnabled()); + } if (timer_separate_) { EXPECT_TRUE(IsTimerEnabled()); } else { @@ -313,7 +339,9 @@ class ProfileHandlerTest { // timer, if shared, is disabled. Expects the worker to be running. void VerifyDisabled() { // Check that the signal handler is disabled. - EXPECT_FALSE(IsSignalEnabled()); + if (FLAGS_test_profiler_signal_handler) { + EXPECT_FALSE(IsSignalEnabled()); + } // Check that the callback count is 0. EXPECT_EQ(0, GetCallbackCount()); // Check that the timer is disabled if shared, enabled otherwise. @@ -465,8 +493,10 @@ TEST_F(ProfileHandlerTest, RegisterCallbackBeforeThread) { // correctly enabled. RegisterThread(); EXPECT_EQ(1, GetCallbackCount()); - EXPECT_TRUE(IsTimerEnabled()); - EXPECT_TRUE(IsSignalEnabled()); + EXPECT_EQ(FLAGS_test_profiler_enabled, IsTimerEnabled()); + if (FLAGS_test_profiler_signal_handler) { + EXPECT_EQ(FLAGS_test_profiler_enabled, IsSignalEnabled()); + } } } // namespace diff --git a/third_party/tcmalloc/chromium/src/tests/profiler_unittest.cc b/third_party/tcmalloc/chromium/src/tests/profiler_unittest.cc index 19371b7..399891b 100644 --- a/third_party/tcmalloc/chromium/src/tests/profiler_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/profiler_unittest.cc @@ -41,7 +41,7 @@ #include <unistd.h> // for fork() #endif #include <sys/wait.h> // for wait() -#include "google/profiler.h" +#include "gperftools/profiler.h" #include "base/simple_mutex.h" #include "tests/testutil.h" @@ -61,7 +61,7 @@ static void test_other_thread() { for (i = 0; i < g_iters; ++i ) { result ^= i; } - snprintf(b, sizeof(b), "%d", result); // get some libc action + snprintf(b, sizeof(b), "other: %d", result); // get some libc action } #endif } @@ -74,7 +74,7 @@ static void test_main_thread() { for (i = 0; i < g_iters; ++i ) { result ^= i; } - snprintf(b, sizeof(b), "%d", result); // get some libc action + snprintf(b, sizeof(b), "same: %d", result); // get some libc action } } diff --git a/third_party/tcmalloc/chromium/src/tests/sampler_test.cc b/third_party/tcmalloc/chromium/src/tests/sampler_test.cc index 31c87cd..c55d5dc 100644 --- a/third_party/tcmalloc/chromium/src/tests/sampler_test.cc +++ b/third_party/tcmalloc/chromium/src/tests/sampler_test.cc @@ -357,7 +357,7 @@ bool CheckMean(size_t mean, int num_samples) { } double empirical_mean = total / static_cast<double>(num_samples); double expected_sd = mean / pow(num_samples * 1.0, 0.5); - return(abs(mean-empirical_mean) < expected_sd * kSigmas); + return(fabs(mean-empirical_mean) < expected_sd * kSigmas); } // Prints a sequence so you can look at the distribution @@ -409,8 +409,8 @@ TEST(Sampler, LargeAndSmallAllocs_CombinedTest) { size_small, kSamplingInterval); LOG(INFO) << StringPrintf("large_allocs_sds = %f\n", large_allocs_sds); LOG(INFO) << StringPrintf("small_allocs_sds = %f\n", small_allocs_sds); - CHECK_LE(abs(large_allocs_sds), kSigmas); - CHECK_LE(abs(small_allocs_sds), kSigmas); + CHECK_LE(fabs(large_allocs_sds), kSigmas); + CHECK_LE(fabs(small_allocs_sds), kSigmas); } // Tests whether the mean is about right over 1000 samples diff --git a/third_party/tcmalloc/chromium/src/tests/sampling_test.cc b/third_party/tcmalloc/chromium/src/tests/sampling_test.cc index c1bd693..8132475 100644 --- a/third_party/tcmalloc/chromium/src/tests/sampling_test.cc +++ b/third_party/tcmalloc/chromium/src/tests/sampling_test.cc @@ -41,7 +41,7 @@ #include <stdlib.h> #include <string> #include "base/logging.h" -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> using std::string; diff --git a/third_party/tcmalloc/chromium/src/tests/simple_compat_test.cc b/third_party/tcmalloc/chromium/src/tests/simple_compat_test.cc new file mode 100644 index 0000000..824cfcf --- /dev/null +++ b/third_party/tcmalloc/chromium/src/tests/simple_compat_test.cc @@ -0,0 +1,67 @@ +// Copyright (c) 2012, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// This just verifies that we can compile code that #includes stuff +// via the backwards-compatibility 'google/' #include-dir. It does +// not include config.h on purpose, to better simulate a perftools +// client. + +#include <stddef.h> +#include <stdio.h> +#include <google/heap-checker.h> +#include <google/heap-profiler.h> +#include <google/malloc_extension.h> +#include <google/malloc_extension_c.h> +#include <google/malloc_hook.h> +#include <google/malloc_hook_c.h> +#include <google/profiler.h> +#include <google/stacktrace.h> +#include <google/tcmalloc.h> + +// We don't link in -lprofiler for this test, so be sure not to make +// any function calls that require the cpu-profiler code. The +// heap-profiler is ok. + +HeapLeakChecker::Disabler* heap_checker_h; +void (*heap_profiler_h)(const char*) = &HeapProfilerStart; +MallocExtension::Ownership malloc_extension_h; +MallocExtension_Ownership malloc_extension_c_h; +MallocHook::NewHook* malloc_hook_h; +MallocHook_NewHook* malloc_hook_c_h; +ProfilerOptions* profiler_h; +int (*stacktrace_h)(void**, int, int) = &GetStackTrace; +void* (*tcmalloc_h)(size_t) = &tc_new; + +int main(int argc, char** argv) { + printf("PASS\n"); + return 0; +} diff --git a/third_party/tcmalloc/chromium/src/tests/stacktrace_unittest.cc b/third_party/tcmalloc/chromium/src/tests/stacktrace_unittest.cc index 69e20ba..3c9f735 100644 --- a/third_party/tcmalloc/chromium/src/tests/stacktrace_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/stacktrace_unittest.cc @@ -35,7 +35,7 @@ #include <stdlib.h> #include "base/commandlineflags.h" #include "base/logging.h" -#include <google/stacktrace.h> +#include <gperftools/stacktrace.h> namespace { diff --git a/third_party/tcmalloc/chromium/src/tests/system-alloc_unittest.cc b/third_party/tcmalloc/chromium/src/tests/system-alloc_unittest.cc index c006425..f0259a1 100644 --- a/third_party/tcmalloc/chromium/src/tests/system-alloc_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/system-alloc_unittest.cc @@ -40,8 +40,9 @@ #endif #include <sys/types.h> #include <algorithm> +#include <limits> #include "base/logging.h" // for Check_GEImpl, Check_LTImpl, etc -#include <google/malloc_extension.h> // for MallocExtension::instance +#include <gperftools/malloc_extension.h> // for MallocExtension::instance #include "common.h" // for kAddressBits class ArraySysAllocator : public SysAllocator { @@ -83,7 +84,6 @@ public: void DumpStats() { } - void FlagsInitialized() {} private: static const int kArraySize = 8 * 1024 * 1024; @@ -115,14 +115,30 @@ TEST(AddressBits, CpuVirtualBits) { const int kPointerBits = 8 * sizeof(void*); const int kImplementedVirtualBits = NumImplementedVirtualBits(); - CHECK_GE(kAddressBits, min(kImplementedVirtualBits, kPointerBits)); + CHECK_GE(kAddressBits, std::min(kImplementedVirtualBits, kPointerBits)); } #endif static void TestBasicRetryFailTest() { // Check with the allocator still works after a failed allocation. - void* p = malloc(1ULL << 50); // Asking for 1P ram - CHECK(p == NULL); + // + // There is no way to call malloc and guarantee it will fail. malloc takes a + // size_t parameter and the C++ standard does not constrain the size of + // size_t. For example, consider an implementation where size_t is 32 bits + // and pointers are 64 bits. + // + // It is likely, though, that sizeof(size_t) == sizeof(void*). In that case, + // the first allocation here might succeed but the second allocation must + // fail. + // + // If the second allocation succeeds, you will have to rewrite or + // disable this test. + // The weird parens are to avoid macro-expansion of 'max' on windows. + const size_t kHugeSize = (std::numeric_limits<size_t>::max)() / 2; + void* p1 = malloc(kHugeSize); + void* p2 = malloc(kHugeSize); + CHECK(p2 == NULL); + if (p1 != NULL) free(p1); char* q = new char[1024]; CHECK(q != NULL); diff --git a/third_party/tcmalloc/chromium/src/tests/tcmalloc_unittest.cc b/third_party/tcmalloc/chromium/src/tests/tcmalloc_unittest.cc index b430460..cfdc79c 100644 --- a/third_party/tcmalloc/chromium/src/tests/tcmalloc_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/tcmalloc_unittest.cc @@ -88,24 +88,57 @@ #include <new> #include "base/logging.h" #include "base/simple_mutex.h" -#include "google/malloc_hook.h" -#include "google/malloc_extension.h" -#include "google/tcmalloc.h" +#include "gperftools/malloc_hook.h" +#include "gperftools/malloc_extension.h" +#include "gperftools/tcmalloc.h" #include "thread_cache.h" #include "tests/testutil.h" // Windows doesn't define pvalloc and a few other obsolete unix // functions; nor does it define posix_memalign (which is not obsolete). -#if defined(_MSC_VER) || defined(__MINGW32__) +#if defined(_WIN32) # define cfree free // don't bother to try to test these obsolete fns # define valloc malloc # define pvalloc malloc // I'd like to map posix_memalign to _aligned_malloc, but _aligned_malloc // must be paired with _aligned_free (not normal free), which is too // invasive a change to how we allocate memory here. So just bail -# include <errno.h> -# define memalign(alignment, size) malloc(size) -# define posix_memalign(pptr, align, size) ((*(pptr)=malloc(size)) ? 0 : ENOMEM) +static bool kOSSupportsMemalign = false; +static inline void* Memalign(size_t align, size_t size) { + //LOG(FATAL) << "memalign not supported on windows"; + exit(1); + return NULL; +} +static inline int PosixMemalign(void** ptr, size_t align, size_t size) { + //LOG(FATAL) << "posix_memalign not supported on windows"; + exit(1); + return -1; +} + +// OS X defines posix_memalign in some OS versions but not others; +// it's confusing enough to check that it's easiest to just not to test. +#elif defined(__APPLE__) +static bool kOSSupportsMemalign = false; +static inline void* Memalign(size_t align, size_t size) { + //LOG(FATAL) << "memalign not supported on OS X"; + exit(1); + return NULL; +} +static inline int PosixMemalign(void** ptr, size_t align, size_t size) { + //LOG(FATAL) << "posix_memalign not supported on OS X"; + exit(1); + return -1; +} + +#else +static bool kOSSupportsMemalign = true; +static inline void* Memalign(size_t align, size_t size) { + return memalign(align, size); +} +static inline int PosixMemalign(void** ptr, size_t align, size_t size) { + return posix_memalign(ptr, align, size); +} + #endif // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old @@ -149,7 +182,12 @@ static const size_t kMaxSize = ~static_cast<size_t>(0); static const size_t kMaxSignedSize = ((size_t(1) << (kSizeBits-1)) - 1); static const size_t kNotTooBig = 100000; -static const size_t kTooBig = kMaxSize; +// We want an allocation that is definitely more than main memory. OS +// X has special logic to discard very big allocs before even passing +// the request along to the user-defined memory allocator; we're not +// interested in testing their logic, so we have to make sure we're +// not *too* big. +static const size_t kTooBig = kMaxSize - 100000; static int news_handled = 0; @@ -246,16 +284,18 @@ int TestHarness::PickType() { class AllocatorState : public TestHarness { public: - explicit AllocatorState(int seed) : TestHarness(seed) { - CHECK_GE(FLAGS_memalign_max_fraction, 0); - CHECK_LE(FLAGS_memalign_max_fraction, 1); - CHECK_GE(FLAGS_memalign_min_fraction, 0); - CHECK_LE(FLAGS_memalign_min_fraction, 1); - double delta = FLAGS_memalign_max_fraction - FLAGS_memalign_min_fraction; - CHECK_GE(delta, 0); - memalign_fraction_ = (Uniform(10000)/10000.0 * delta + - FLAGS_memalign_min_fraction); - //fprintf(LOGSTREAM, "memalign fraction: %f\n", memalign_fraction_); + explicit AllocatorState(int seed) : TestHarness(seed), memalign_fraction_(0) { + if (kOSSupportsMemalign) { + CHECK_GE(FLAGS_memalign_max_fraction, 0); + CHECK_LE(FLAGS_memalign_max_fraction, 1); + CHECK_GE(FLAGS_memalign_min_fraction, 0); + CHECK_LE(FLAGS_memalign_min_fraction, 1); + double delta = FLAGS_memalign_max_fraction - FLAGS_memalign_min_fraction; + CHECK_GE(delta, 0); + memalign_fraction_ = (Uniform(10000)/10000.0 * delta + + FLAGS_memalign_min_fraction); + //fprintf(LOGSTREAM, "memalign fraction: %f\n", memalign_fraction_); + } } virtual ~AllocatorState() {} @@ -269,7 +309,7 @@ class AllocatorState : public TestHarness { (size < sizeof(intptr_t) || alignment < FLAGS_memalign_max_alignment_ratio * size)) { void *result = reinterpret_cast<void*>(static_cast<intptr_t>(0x1234)); - int err = posix_memalign(&result, alignment, size); + int err = PosixMemalign(&result, alignment, size); if (err != 0) { CHECK_EQ(err, ENOMEM); } @@ -891,44 +931,42 @@ static void OnNoMemory() { static void TestSetNewMode() { int old_mode = tc_set_new_mode(1); - // DebugAllocation will try to catch huge allocations. We need to avoid this - // by requesting a smaller malloc block, that still can't be satisfied. - const size_t kHugeRequest = kTooBig - 1024; - - g_old_handler = std::set_new_handler(&OnNoMemory); - g_no_memory = false; - void* ret = malloc(kHugeRequest); - EXPECT_EQ(NULL, ret); - EXPECT_TRUE(g_no_memory); - g_old_handler = std::set_new_handler(&OnNoMemory); g_no_memory = false; - ret = calloc(1, kHugeRequest); + void* ret = malloc(kTooBig); EXPECT_EQ(NULL, ret); EXPECT_TRUE(g_no_memory); g_old_handler = std::set_new_handler(&OnNoMemory); g_no_memory = false; - ret = realloc(NULL, kHugeRequest); + ret = calloc(1, kTooBig); EXPECT_EQ(NULL, ret); EXPECT_TRUE(g_no_memory); - // Not really important, but must be small enough such that kAlignment + - // kHugeRequest does not overflow. - const int kAlignment = 1 << 5; - g_old_handler = std::set_new_handler(&OnNoMemory); g_no_memory = false; - ret = memalign(kAlignment, kHugeRequest); + ret = realloc(NULL, kTooBig); EXPECT_EQ(NULL, ret); EXPECT_TRUE(g_no_memory); - g_old_handler = std::set_new_handler(&OnNoMemory); - g_no_memory = false; - EXPECT_EQ(ENOMEM, - posix_memalign(&ret, kAlignment, kHugeRequest)); - EXPECT_EQ(NULL, ret); - EXPECT_TRUE(g_no_memory); + if (kOSSupportsMemalign) { + // Not really important, but must be small enough such that + // kAlignment + kTooBig does not overflow. + const int kAlignment = 1 << 5; + + g_old_handler = std::set_new_handler(&OnNoMemory); + g_no_memory = false; + ret = Memalign(kAlignment, kTooBig); + EXPECT_EQ(NULL, ret); + EXPECT_TRUE(g_no_memory); + + g_old_handler = std::set_new_handler(&OnNoMemory); + g_no_memory = false; + EXPECT_EQ(ENOMEM, + PosixMemalign(&ret, kAlignment, kTooBig)); + EXPECT_EQ(NULL, ret); + EXPECT_TRUE(g_no_memory); + } tc_set_new_mode(old_mode); } @@ -1032,21 +1070,24 @@ static int RunAllTests(int argc, char** argv) { cfree(p1); // synonym for free VerifyDeleteHookWasCalled(); - CHECK_EQ(posix_memalign(&p1, sizeof(p1), 40), 0); - CHECK(p1 != NULL); - VerifyNewHookWasCalled(); - free(p1); - VerifyDeleteHookWasCalled(); - - p1 = memalign(sizeof(p1) * 2, 50); - CHECK(p1 != NULL); - VerifyNewHookWasCalled(); - free(p1); - VerifyDeleteHookWasCalled(); + if (kOSSupportsMemalign) { + CHECK_EQ(PosixMemalign(&p1, sizeof(p1), 40), 0); + CHECK(p1 != NULL); + VerifyNewHookWasCalled(); + free(p1); + VerifyDeleteHookWasCalled(); + + p1 = Memalign(sizeof(p1) * 2, 50); + CHECK(p1 != NULL); + VerifyNewHookWasCalled(); + free(p1); + VerifyDeleteHookWasCalled(); + } // Windows has _aligned_malloc. Let's test that that's captured too. #if (defined(_MSC_VER) || defined(__MINGW32__)) && !defined(PERFTOOLS_NO_ALIGNED_MALLOC) p1 = _aligned_malloc(sizeof(p1) * 2, 64); + CHECK(p1 != NULL); VerifyNewHookWasCalled(); _aligned_free(p1); VerifyDeleteHookWasCalled(); diff --git a/third_party/tcmalloc/chromium/src/tests/thread_dealloc_unittest.cc b/third_party/tcmalloc/chromium/src/tests/thread_dealloc_unittest.cc index 32923e1..e6fd9b3 100644 --- a/third_party/tcmalloc/chromium/src/tests/thread_dealloc_unittest.cc +++ b/third_party/tcmalloc/chromium/src/tests/thread_dealloc_unittest.cc @@ -38,7 +38,7 @@ #include <unistd.h> // for sleep() #endif #include "base/logging.h" -#include <google/malloc_extension.h> +#include <gperftools/malloc_extension.h> #include "tests/testutil.h" // for RunThread() // Size/number of objects to allocate per thread (1 MB per thread) diff --git a/third_party/tcmalloc/chromium/src/thread_cache.cc b/third_party/tcmalloc/chromium/src/thread_cache.cc index 1c189f3..765786ff 100644 --- a/third_party/tcmalloc/chromium/src/thread_cache.cc +++ b/third_party/tcmalloc/chromium/src/thread_cache.cc @@ -32,6 +32,7 @@ #include <config.h> #include "thread_cache.h" +#include <errno.h> #include <string.h> // for memcpy #include <algorithm> // for max, min #include "base/commandlineflags.h" // for SpinLockHolder @@ -46,8 +47,9 @@ DEFINE_int64(tcmalloc_max_total_thread_cache_bytes, EnvToInt64("TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES", kDefaultOverallThreadCacheSize), "Bound on the total amount of bytes allocated to " - "thread caches. This bound is not strict, so it is possible " - "for the cache to go over this bound in certain circumstances. "); + "thread caches. This bound is not strict, so it is possible " + "for the cache to go over this bound in certain circumstances. " + "Maximum value of this flag is capped to 1 GB."); namespace tcmalloc { @@ -72,7 +74,11 @@ pthread_key_t ThreadCache::heap_key_; #if defined(HAVE_TLS) bool kernel_supports_tls = false; // be conservative -# if !HAVE_DECL_UNAME // if too old for uname, probably too old for TLS +# if defined(_WIN32) // windows has supported TLS since winnt, I think. + void CheckIfKernelSupportsTLS() { + kernel_supports_tls = true; + } +# elif !HAVE_DECL_UNAME // if too old for uname, probably too old for TLS void CheckIfKernelSupportsTLS() { kernel_supports_tls = false; } @@ -80,8 +86,9 @@ bool kernel_supports_tls = false; // be conservative # include <sys/utsname.h> // DECL_UNAME checked for <sys/utsname.h> too void CheckIfKernelSupportsTLS() { struct utsname buf; - if (uname(&buf) != 0) { // should be impossible - MESSAGE("uname failed assuming no TLS support (errno=%d)\n", errno); + if (uname(&buf) < 0) { // should be impossible + Log(kLog, __FILE__, __LINE__, + "uname failed assuming no TLS support (errno)", errno); kernel_supports_tls = false; } else if (strcasecmp(buf.sysname, "linux") == 0) { // The linux case: the first kernel to support TLS was 2.6.0 @@ -93,6 +100,10 @@ bool kernel_supports_tls = false; // be conservative kernel_supports_tls = false; else kernel_supports_tls = true; + } else if (strcasecmp(buf.sysname, "CYGWIN_NT-6.1-WOW64") == 0) { + // In my testing, this version of cygwin, at least, would hang + // when using TLS. + kernel_supports_tls = false; } else { // some other kernel, we'll be optimisitic kernel_supports_tls = true; } @@ -259,10 +270,6 @@ void ThreadCache::Scavenge() { } IncreaseCacheLimit(); - -// int64 finish = CycleClock::Now(); -// CycleTimer ct; -// MESSAGE("GC: %.0f ns\n", ct.CyclesToUsec(finish-start)*1000.0); } void ThreadCache::IncreaseCacheLimit() { @@ -322,6 +329,18 @@ void ThreadCache::InitTSD() { ASSERT(!tsd_inited_); perftools_pthread_key_create(&heap_key_, DestroyThreadCache); tsd_inited_ = true; + +#ifdef PTHREADS_CRASHES_IF_RUN_TOO_EARLY + // We may have used a fake pthread_t for the main thread. Fix it. + pthread_t zero; + memset(&zero, 0, sizeof(zero)); + SpinLockHolder h(Static::pageheap_lock()); + for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { + if (h->tid_ == zero) { + h->tid_ = pthread_self(); + } + } +#endif } ThreadCache* ThreadCache::CreateCacheIfNecessary() { @@ -329,17 +348,23 @@ ThreadCache* ThreadCache::CreateCacheIfNecessary() { ThreadCache* heap = NULL; { SpinLockHolder h(Static::pageheap_lock()); - // On very old libc's, this call may crash if it happens too - // early. No libc using NPTL should be affected. If there - // is a crash here, we could use code (on linux, at least) - // to detect NPTL vs LinuxThreads: - // http://www.redhat.com/archives/phil-list/2003-April/msg00038.html - // If we detect not-NPTL, we could execute the old code from - // http://google-perftools.googlecode.com/svn/tags/google-perftools-1.7/src/thread_cache.cc - // that avoids calling pthread_self too early. The problem with - // that code is it caused a race condition when tcmalloc is linked - // in statically and other libraries spawn threads before main. + // On some old glibc's, and on freebsd's libc (as of freebsd 8.1), + // calling pthread routines (even pthread_self) too early could + // cause a segfault. Since we can call pthreads quite early, we + // have to protect against that in such situations by making a + // 'fake' pthread. This is not ideal since it doesn't work well + // when linking tcmalloc statically with apps that create threads + // before main, so we only do it if we have to. +#ifdef PTHREADS_CRASHES_IF_RUN_TOO_EARLY + pthread_t me; + if (!tsd_inited_) { + memset(&me, 0, sizeof(me)); + } else { + me = pthread_self(); + } +#else const pthread_t me = pthread_self(); +#endif // This may be a recursive malloc call from pthread_setspecific() // In that case, the heap for this thread has already been created @@ -462,30 +487,6 @@ void ThreadCache::RecomputePerThreadCacheSize() { } unclaimed_cache_space_ = overall_thread_cache_size_ - claimed; per_thread_cache_size_ = space; - // TCMalloc_MESSAGE(__FILE__, __LINE__, "Threads %d => cache size %8d\n", n, int(space)); -} - -void ThreadCache::Print(TCMalloc_Printer* out) const { - for (int cl = 0; cl < kNumClasses; ++cl) { - out->printf(" %5" PRIuS " : %4" PRIuS " len; %4d lo; %4"PRIuS - " max; %4"PRIuS" overages;\n", - Static::sizemap()->ByteSizeForClass(cl), - list_[cl].length(), - list_[cl].lowwatermark(), - list_[cl].max_length(), - list_[cl].length_overages()); - } -} - -void ThreadCache::PrintThreads(TCMalloc_Printer* out) { - size_t actual_limit = 0; - for (ThreadCache* h = thread_heaps_; h != NULL; h = h->next_) { - h->Print(out); - actual_limit += h->max_size_; - } - out->printf("ThreadCache overall: %"PRIuS ", unclaimed: %"PRIuS - ", actual: %"PRIuS"\n", - overall_thread_cache_size_, unclaimed_cache_space_, actual_limit); } void ThreadCache::GetThreadStats(uint64_t* total_bytes, uint64_t* class_count) { diff --git a/third_party/tcmalloc/chromium/src/thread_cache.h b/third_party/tcmalloc/chromium/src/thread_cache.h index d631f45..5294029 100644 --- a/third_party/tcmalloc/chromium/src/thread_cache.h +++ b/third_party/tcmalloc/chromium/src/thread_cache.h @@ -88,7 +88,6 @@ class ThreadCache { void Deallocate(void* ptr, size_t size_class); void Scavenge(); - void Print(TCMalloc_Printer* out) const; int GetSamplePeriod(); @@ -125,10 +124,6 @@ class ThreadCache { // REQUIRES: Static::pageheap_lock is held. static void GetThreadStats(uint64_t* total_bytes, uint64_t* class_count); - // Write debugging statistics to 'out'. - // REQUIRES: Static::pageheap_lock is held. - static void PrintThreads(TCMalloc_Printer* out); - // Sets the total thread cache size to new_size, recomputing the // individual thread cache sizes as necessary. // REQUIRES: Static::pageheap lock is held. @@ -214,6 +209,11 @@ class ThreadCache { return FL_Pop(&list_); } + void* Next() { + if (list_ == NULL) return NULL; + return FL_Next(list_); + } + void PushRange(int N, void *start, void *end) { FL_PushRange(&list_, start, end); length_ += N; @@ -366,6 +366,12 @@ inline void ThreadCache::Deallocate(void* ptr, size_t cl) { FreeList* list = &list_[cl]; size_ += Static::sizemap()->ByteSizeForClass(cl); ssize_t size_headroom = max_size_ - size_ - 1; + + // This catches back-to-back frees of allocs in the same size + // class. A more comprehensive (and expensive) test would be to walk + // the entire freelist. But this might be enough to find some bugs. + ASSERT(ptr != list->Next()); + list->Push(ptr); ssize_t list_headroom = static_cast<ssize_t>(list->max_length()) - list->length(); diff --git a/third_party/tcmalloc/chromium/src/windows/auto_testing_hook.h b/third_party/tcmalloc/chromium/src/windows/auto_testing_hook.h new file mode 100644 index 0000000..5a04797 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/windows/auto_testing_hook.h @@ -0,0 +1,155 @@ +// Copyright (c) 2010 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Utility for using SideStep with unit tests. + +#ifndef CEEE_TESTING_SIDESTEP_AUTO_TESTING_HOOK_H_ +#define CEEE_TESTING_SIDESTEP_AUTO_TESTING_HOOK_H_ + +#include "base/basictypes.h" +#include "base/logging.h" +#include "preamble_patcher.h" + +#define SIDESTEP_CHK(x) CHECK(x) +#define SIDESTEP_EXPECT_TRUE(x) SIDESTEP_CHK(x) + +namespace sidestep { + +// Same trick as common/scope_cleanup.h ScopeGuardImplBase +class AutoTestingHookBase { + public: + virtual ~AutoTestingHookBase() {} +}; + +// This is the typedef you normally use for the class, e.g. +// +// AutoTestingHook hook = MakeTestingHook(TargetFunc, HookTargetFunc); +// +// The 'hook' variable will then be destroyed when it goes out of scope. +// +// NOTE: You must not hold this type as a member of another class. Its +// destructor will not get called. +typedef const AutoTestingHookBase& AutoTestingHook; + +// This is the class you must use when holding a hook as a member of another +// class, e.g. +// +// public: +// AutoTestingHookHolder holder_; +// MyClass() : my_hook_holder(MakeTestingHookHolder(Target, Hook)) {} +class AutoTestingHookHolder { + public: + explicit AutoTestingHookHolder(AutoTestingHookBase* hook) : hook_(hook) {} + ~AutoTestingHookHolder() { delete hook_; } + private: + AutoTestingHookHolder() {} // disallow + AutoTestingHookBase* hook_; +}; + +// This class helps patch a function, then unpatch it when the object exits +// scope, and also maintains the pointer to the original function stub. +// +// To enable use of the class without having to explicitly provide the +// type of the function pointers (and instead only providing it +// implicitly) we use the same trick as ScopeGuard (see +// common/scope_cleanup.h) uses, so to create a hook you use the MakeHook +// function rather than a constructor. +// +// NOTE: This function is only safe for e.g. unit tests and _not_ for +// production code. See PreamblePatcher class for details. +template <typename T> +class AutoTestingHookImpl : public AutoTestingHookBase { + public: + static AutoTestingHookImpl<T> MakeTestingHook(T target_function, + T replacement_function, + bool do_it) { + return AutoTestingHookImpl<T>(target_function, replacement_function, do_it); + } + + static AutoTestingHookImpl<T>* MakeTestingHookHolder(T target_function, + T replacement_function, + bool do_it) { + return new AutoTestingHookImpl<T>(target_function, + replacement_function, do_it); + } + + ~AutoTestingHookImpl() { + if (did_it_) { + SIDESTEP_CHK(SIDESTEP_SUCCESS == PreamblePatcher::Unpatch( + (void*)target_function_, (void*)replacement_function_, + (void*)original_function_)); + } + } + + // Returns a pointer to the original function. To use this method you will + // have to explicitly create an AutoTestingHookImpl of the specific + // function pointer type (i.e. not use the AutoTestingHook typedef). + T original_function() { + return original_function_; + } + + private: + AutoTestingHookImpl(T target_function, T replacement_function, bool do_it) + : target_function_(target_function), + original_function_(NULL), + replacement_function_(replacement_function), + did_it_(do_it) { + if (do_it) { + SIDESTEP_CHK(SIDESTEP_SUCCESS == PreamblePatcher::Patch(target_function, + replacement_function, + &original_function_)); + } + } + + T target_function_; // always valid + T original_function_; // always valid + T replacement_function_; // always valid + bool did_it_; // Remember if we did it or not... +}; + +template <typename T> +inline AutoTestingHookImpl<T> MakeTestingHook(T target, + T replacement, + bool do_it) { + return AutoTestingHookImpl<T>::MakeTestingHook(target, replacement, do_it); +} + +template <typename T> +inline AutoTestingHookImpl<T> MakeTestingHook(T target, T replacement) { + return AutoTestingHookImpl<T>::MakeTestingHook(target, replacement, true); +} + +template <typename T> +inline AutoTestingHookImpl<T>* MakeTestingHookHolder(T target, T replacement) { + return AutoTestingHookImpl<T>::MakeTestingHookHolder(target, replacement, + true); +} + +}; // namespace sidestep + +#endif // CEEE_TESTING_SIDESTEP_AUTO_TESTING_HOOK_H_ diff --git a/third_party/tcmalloc/chromium/src/windows/config.h b/third_party/tcmalloc/chromium/src/windows/config.h index 1d93c4f..9d61884 100644 --- a/third_party/tcmalloc/chromium/src/windows/config.h +++ b/third_party/tcmalloc/chromium/src/windows/config.h @@ -94,6 +94,9 @@ /* Define to 1 if you have the <malloc.h> header file. */ #define HAVE_MALLOC_H 1 +/* Define to 1 if you have the <malloc/malloc.h> header file. */ +#undef HAVE_MALLOC_MALLOC_H + /* Define to 1 if you have the <memory.h> header file. */ #undef HAVE_MEMORY_H @@ -136,6 +139,12 @@ /* Define to 1 if the system has the type `struct mallinfo'. */ #undef HAVE_STRUCT_MALLINFO +/* Define to 1 if you have the <sys/cdefs.h> header file. */ +#undef HAVE_SYS_CDEFS_H + +/* Define to 1 if you have the <sys/malloc.h> header file. */ +#undef HAVE_SYS_MALLOC_H + /* Define to 1 if you have the <sys/param.h> header file. */ #undef HAVE_SYS_PARAM_H @@ -197,29 +206,32 @@ */ #undef LT_OBJDIR +/* Define to 'volatile' if __malloc_hook is declared volatile */ +#undef MALLOC_HOOK_MAYBE_VOLATILE + /* Define to 1 if your C compiler doesn't accept -c and -o together. */ #undef NO_MINUS_C_MINUS_O /* Name of package */ -#define PACKAGE "google-perftools" +#define PACKAGE "gperftools" /* Define to the address where bug reports for this package should be sent. */ #define PACKAGE_BUGREPORT "opensource@google.com" /* Define to the full name of this package. */ -#define PACKAGE_NAME "google-perftools" +#define PACKAGE_NAME "gperftools" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "google-perftools 1.7" +#define PACKAGE_STRING "gperftools 2.0" /* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "google-perftools" +#define PACKAGE_TARNAME "gperftools" /* Define to the home page for this package. */ #undef PACKAGE_URL /* Define to the version of this package. */ -#define PACKAGE_VERSION "1.7" +#define PACKAGE_VERSION "2.0" /* How to access the PC from a struct ucontext */ #undef PC_FROM_UCONTEXT @@ -244,6 +256,12 @@ /* printf format code for printing a size_t and ssize_t */ #define PRIxS "Ix" +/* Mark the systems where we know it's bad if pthreads runs too + early before main (before threads are initialized, presumably). */ +#ifdef __FreeBSD__ +#define PTHREADS_CRASHES_IF_RUN_TOO_EARLY 1 +#endif + /* Define to necessary symbol if this constant uses a non-standard name on your system. */ #undef PTHREAD_CREATE_JOINABLE diff --git a/third_party/tcmalloc/chromium/src/windows/google/tcmalloc.h b/third_party/tcmalloc/chromium/src/windows/google/tcmalloc.h index f6c17f5..c7db631 100644 --- a/third_party/tcmalloc/chromium/src/windows/google/tcmalloc.h +++ b/third_party/tcmalloc/chromium/src/windows/google/tcmalloc.h @@ -26,91 +26,9 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Sanjay Ghemawat <opensource@google.com> - * .h file by Craig Silverstein <opensource@google.com> */ -#ifndef TCMALLOC_TCMALLOC_H_ -#define TCMALLOC_TCMALLOC_H_ - -// __THROW is defined in glibc systems. It means, counter-intuitively, -// "This function will never throw an exception." It's an optional -// optimization tool, but we may need to use it to match glibc prototypes. -#ifndef __THROW /* I guess we're not on a glibc system */ -# define __THROW /* __THROW is just an optimization, so ok to make it "" */ -#endif - -// Define the version number so folks can check against it -#define TC_VERSION_MAJOR 1 -#define TC_VERSION_MINOR 7 -#define TC_VERSION_PATCH "" -#define TC_VERSION_STRING "google-perftools 1.7" - -#include <stdlib.h> // for struct mallinfo, if it's defined - -// Annoying stuff for windows -- makes sure clients can import these functions -#ifndef PERFTOOLS_DLL_DECL -# ifdef _WIN32 -# define PERFTOOLS_DLL_DECL __declspec(dllimport) -# else -# define PERFTOOLS_DLL_DECL -# endif -#endif - -#ifdef __cplusplus -#include <new> // for std::nothrow_t - -extern "C" { -#endif - // Returns a human-readable version string. If major, minor, - // and/or patch are not NULL, they are set to the major version, - // minor version, and patch-code (a string, usually ""). - PERFTOOLS_DLL_DECL const char* tc_version(int* major, int* minor, - const char** patch) __THROW; - - PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW; - PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW; - PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW; - PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW; - PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW; - - PERFTOOLS_DLL_DECL void* tc_memalign(size_t __alignment, - size_t __size) __THROW; - PERFTOOLS_DLL_DECL int tc_posix_memalign(void** ptr, - size_t align, size_t size) __THROW; - PERFTOOLS_DLL_DECL void* tc_valloc(size_t __size) __THROW; - PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t __size) __THROW; - - PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW; - PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW; -#if 0 - PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW; -#endif - - // This is an alias for MallocExtension::instance()->GetAllocatedSize(). - // It is equivalent to - // OS X: malloc_size() - // glibc: malloc_usable_size() - // Windows: _msize() - PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW; - -#ifdef __cplusplus - PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) __THROW; - PERFTOOLS_DLL_DECL void* tc_new(size_t size); - PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, - const std::nothrow_t&) __THROW; - PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW; - PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, - const std::nothrow_t&) __THROW; - PERFTOOLS_DLL_DECL void* tc_newarray(size_t size); - PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, - const std::nothrow_t&) __THROW; - PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW; - PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, - const std::nothrow_t&) __THROW; -} -#endif - -#endif // #ifndef TCMALLOC_TCMALLOC_H_ +/* The code has moved to gperftools/. Use that include-directory for + * new code. + */ +#include <gperftools/tcmalloc.h> diff --git a/third_party/tcmalloc/chromium/src/windows/gperftools/tcmalloc.h b/third_party/tcmalloc/chromium/src/windows/gperftools/tcmalloc.h new file mode 100644 index 0000000..db32c53 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/windows/gperftools/tcmalloc.h @@ -0,0 +1,123 @@ +/* Copyright (c) 2003, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Sanjay Ghemawat <opensource@google.com> + * .h.in file by Craig Silverstein <opensource@google.com> + */ + +#ifndef TCMALLOC_TCMALLOC_H_ +#define TCMALLOC_TCMALLOC_H_ + +#include <stddef.h> // for size_t +#ifdef HAVE_SYS_CDEFS_H +#include <sys/cdefs.h> // where glibc defines __THROW +#endif + +// __THROW is defined in glibc systems. It means, counter-intuitively, +// "This function will never throw an exception." It's an optional +// optimization tool, but we may need to use it to match glibc prototypes. +#ifndef __THROW /* I guess we're not on a glibc system */ +# define __THROW /* __THROW is just an optimization, so ok to make it "" */ +#endif + +// Define the version number so folks can check against it +#define TC_VERSION_MAJOR 2 +#define TC_VERSION_MINOR 0 +#define TC_VERSION_PATCH "" +#define TC_VERSION_STRING "gperftools 2.0" + +#include <stdlib.h> // for struct mallinfo, if it's defined + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef PERFTOOLS_DLL_DECL +# ifdef _WIN32 +# define PERFTOOLS_DLL_DECL __declspec(dllimport) +# else +# define PERFTOOLS_DLL_DECL +# endif +#endif + +#ifdef __cplusplus +namespace std { +struct nothrow_t; +} + +extern "C" { +#endif + // Returns a human-readable version string. If major, minor, + // and/or patch are not NULL, they are set to the major version, + // minor version, and patch-code (a string, usually ""). + PERFTOOLS_DLL_DECL const char* tc_version(int* major, int* minor, + const char** patch) __THROW; + + PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW; + PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW; + PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW; + PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW; + PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW; + + PERFTOOLS_DLL_DECL void* tc_memalign(size_t __alignment, + size_t __size) __THROW; + PERFTOOLS_DLL_DECL int tc_posix_memalign(void** ptr, + size_t align, size_t size) __THROW; + PERFTOOLS_DLL_DECL void* tc_valloc(size_t __size) __THROW; + PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t __size) __THROW; + + PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW; + PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW; +#if 0 + PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW; +#endif + + // This is an alias for MallocExtension::instance()->GetAllocatedSize(). + // It is equivalent to + // OS X: malloc_size() + // glibc: malloc_usable_size() + // Windows: _msize() + PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW; + +#ifdef __cplusplus + PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) __THROW; + PERFTOOLS_DLL_DECL void* tc_new(size_t size); + PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, + const std::nothrow_t&) __THROW; + PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW; + PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, + const std::nothrow_t&) __THROW; + PERFTOOLS_DLL_DECL void* tc_newarray(size_t size); + PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, + const std::nothrow_t&) __THROW; + PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW; + PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, + const std::nothrow_t&) __THROW; +} +#endif + +#endif // #ifndef TCMALLOC_TCMALLOC_H_ diff --git a/third_party/tcmalloc/chromium/src/windows/google/tcmalloc.h.in b/third_party/tcmalloc/chromium/src/windows/gperftools/tcmalloc.h.in index a031b35..d09ec953 100644 --- a/third_party/tcmalloc/chromium/src/windows/google/tcmalloc.h.in +++ b/third_party/tcmalloc/chromium/src/windows/gperftools/tcmalloc.h.in @@ -35,6 +35,11 @@ #ifndef TCMALLOC_TCMALLOC_H_ #define TCMALLOC_TCMALLOC_H_ +#include <stddef.h> // for size_t +#ifdef HAVE_SYS_CDEFS_H +#include <sys/cdefs.h> // where glibc defines __THROW +#endif + // __THROW is defined in glibc systems. It means, counter-intuitively, // "This function will never throw an exception." It's an optional // optimization tool, but we may need to use it to match glibc prototypes. @@ -46,7 +51,7 @@ #define TC_VERSION_MAJOR @TC_VERSION_MAJOR@ #define TC_VERSION_MINOR @TC_VERSION_MINOR@ #define TC_VERSION_PATCH "@TC_VERSION_PATCH@" -#define TC_VERSION_STRING "google-perftools @TC_VERSION_MAJOR@.@TC_VERSION_MINOR@@TC_VERSION_PATCH@" +#define TC_VERSION_STRING "gperftools @TC_VERSION_MAJOR@.@TC_VERSION_MINOR@@TC_VERSION_PATCH@" #include <stdlib.h> // for struct mallinfo, if it's defined @@ -60,7 +65,9 @@ #endif #ifdef __cplusplus -#include <new> // for std::nothrow_t +namespace std { +struct nothrow_t; +} extern "C" { #endif diff --git a/third_party/tcmalloc/chromium/src/windows/ia32_opcode_map.cc b/third_party/tcmalloc/chromium/src/windows/ia32_opcode_map.cc index c9ec18b..ba6a79e 100644 --- a/third_party/tcmalloc/chromium/src/windows/ia32_opcode_map.cc +++ b/third_party/tcmalloc/chromium/src/windows/ia32_opcode_map.cc @@ -111,6 +111,25 @@ const Opcode s_first_opcode_byte[] = { /* 0x3D */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "cmp", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0x3E */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0x3F */ { 0, IT_GENERIC, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, "aas", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, +#ifdef _M_X64 + /* REX Prefixes in 64-bit mode. */ + /* 0x40 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x41 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x42 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x43 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x44 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x45 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x46 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x47 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x48 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x49 */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4A */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4B */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4C */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4D */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4E */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0x4F */ { 0, IT_PREFIX, AM_NOT_USED, AM_NOT_USED, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, +#else /* 0x40 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0x41 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0x42 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "inc", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, @@ -127,6 +146,7 @@ const Opcode s_first_opcode_byte[] = { /* 0x4D */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0x4E */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0x4F */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "dec", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, +#endif /* 0x50 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0x51 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0x52 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_NOT_USED, AM_NOT_USED, "push", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, @@ -231,6 +251,16 @@ const Opcode s_first_opcode_byte[] = { /* 0xB5 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0xB6 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0xB7 */ { 0, IT_GENERIC, AM_REGISTER | OT_B, AM_I | OT_B, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, +#ifdef _M_X64 + /* 0xB8 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V | IOS_64, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xB9 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V | IOS_64, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBA */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V | IOS_64, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBB */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V | IOS_64, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBC */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V | IOS_64, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBD */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V | IOS_64, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBE */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V | IOS_64, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, + /* 0xBF */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V | IOS_64, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, +#else /* 0xB8 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0xB9 */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0xBA */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, @@ -239,6 +269,7 @@ const Opcode s_first_opcode_byte[] = { /* 0xBD */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0xBE */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0xBF */ { 0, IT_GENERIC, AM_REGISTER | OT_V, AM_I | OT_V, AM_NOT_USED, "mov", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, +#endif /* 0xC0 */ { 6, IT_REFERENCE, AM_E | OT_B, AM_I | OT_B, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0xC1 */ { 7, IT_REFERENCE, AM_E | OT_V, AM_I | OT_B, AM_NOT_USED, 0, false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, /* 0xC2 */ { 0, IT_RETURN, AM_I | OT_W, AM_NOT_USED, AM_NOT_USED, "ret", false, /* F2h */ { 0 }, /* F3h */ { 0 }, /* 66h */ { 0 } }, diff --git a/third_party/tcmalloc/chromium/src/windows/mingw.h b/third_party/tcmalloc/chromium/src/windows/mingw.h index 747b285..2aa5eb3 100644 --- a/third_party/tcmalloc/chromium/src/windows/mingw.h +++ b/third_party/tcmalloc/chromium/src/windows/mingw.h @@ -53,8 +53,6 @@ # define _WIN32_WINNT 0x0501 #endif -#include "windows/port.h" - #define HAVE_SNPRINTF 1 // Some mingw distributions have a pthreads wrapper, but it doesn't @@ -62,6 +60,8 @@ // pretend the pthreads wrapper doesn't exist, even when it does. #undef HAVE_PTHREAD +#include "windows/port.h" + #endif /* __MINGW32__ */ #endif /* GOOGLE_PERFTOOLS_WINDOWS_MINGW_H_ */ diff --git a/third_party/tcmalloc/chromium/src/windows/mini_disassembler.cc b/third_party/tcmalloc/chromium/src/windows/mini_disassembler.cc index 30bdcc1..9e336ba 100644 --- a/third_party/tcmalloc/chromium/src/windows/mini_disassembler.cc +++ b/third_party/tcmalloc/chromium/src/windows/mini_disassembler.cc @@ -100,6 +100,12 @@ InstructionType MiniDisassembler::Disassemble( void MiniDisassembler::Initialize() { operand_is_32_bits_ = operand_default_is_32_bits_; address_is_32_bits_ = address_default_is_32_bits_; +#ifdef _M_X64 + operand_default_support_64_bits_ = true; +#else + operand_default_support_64_bits_ = false; +#endif + operand_is_64_bits_ = false; operand_bytes_ = 0; have_modrm_ = false; should_decode_modrm_ = false; @@ -129,6 +135,8 @@ InstructionType MiniDisassembler::ProcessPrefixes(unsigned char* start_byte, got_f3_prefix_ = true; else if (0x66 == (*start_byte)) got_66_prefix_ = true; + else if (operand_default_support_64_bits_ && (*start_byte) & 0x48) + operand_is_64_bits_ = true; instruction_type = opcode.type_; size ++; @@ -314,8 +322,12 @@ bool MiniDisassembler::ProcessOperand(int flag_operand) { // floating point succeeded = false; break; - case OT_V: // Word or doubleword, depending on operand-size attribute. - if (operand_is_32_bits_) + case OT_V: // Word, doubleword or quadword, depending on operand-size + // attribute. + if (operand_is_64_bits_ && flag_operand & AM_I && + flag_operand & IOS_64) + operand_bytes_ += OS_QUAD_WORD; + else if (operand_is_32_bits_) operand_bytes_ += OS_DOUBLE_WORD; else operand_bytes_ += OS_WORD; diff --git a/third_party/tcmalloc/chromium/src/windows/mini_disassembler.h b/third_party/tcmalloc/chromium/src/windows/mini_disassembler.h index e676232..52daa5d8 100644 --- a/third_party/tcmalloc/chromium/src/windows/mini_disassembler.h +++ b/third_party/tcmalloc/chromium/src/windows/mini_disassembler.h @@ -36,6 +36,7 @@ #ifndef GOOGLE_PERFTOOLS_MINI_DISASSEMBLER_H_ #define GOOGLE_PERFTOOLS_MINI_DISASSEMBLER_H_ +#include "config.h" #include <windows.h> #include "mini_disassembler_types.h" @@ -74,7 +75,7 @@ namespace sidestep { // IA-32 Intel® Architecture Software Developer’s Manual Volume 2: // Instruction Set Reference for information about operand decoding // etc. -class MiniDisassembler { +class PERFTOOLS_DLL_DECL MiniDisassembler { public: // Creates a new instance and sets defaults. @@ -165,6 +166,12 @@ class MiniDisassembler { // Default address size is 32 bits if true, 16 bits if false. bool address_default_is_32_bits_; + // Determines if 64 bit operands are supported (x64). + bool operand_default_support_64_bits_; + + // Current operand size is 64 bits if true, 32 bits if false. + bool operand_is_64_bits_; + // Huge big opcode table based on the IA-32 manual, defined // in Ia32OpcodeMap.cc static const OpcodeTable s_ia32_opcode_map_[]; diff --git a/third_party/tcmalloc/chromium/src/windows/mini_disassembler_types.h b/third_party/tcmalloc/chromium/src/windows/mini_disassembler_types.h index 7f8e997..83dee8b 100644 --- a/third_party/tcmalloc/chromium/src/windows/mini_disassembler_types.h +++ b/third_party/tcmalloc/chromium/src/windows/mini_disassembler_types.h @@ -143,6 +143,16 @@ enum OperandType { OT_ADDRESS_MODE_M = 0x80000000 }; +// Flag that indicates if an immediate operand is 64-bits. +// +// The Intel 64 and IA-32 Architecture Software Developer's Manual currently +// defines MOV as the only instruction supporting a 64-bit immediate operand. +enum ImmediateOperandSize { + IOS_MASK = 0x0000F000, + IOS_DEFAULT = 0x0, + IOS_64 = 0x00001000 +}; + // Everything that's in an Opcode (see below) except the three // alternative opcode structs for different prefixes. struct SpecificOpcode { @@ -154,8 +164,8 @@ struct SpecificOpcode { InstructionType type_; // Description of the type of the dest, src and aux operands, - // put together from an enOperandType flag and an enAddressingMethod - // flag. + // put together from enOperandType, enAddressingMethod and + // enImmediateOperandSize flags. int flag_dest_; int flag_source_; int flag_aux_; diff --git a/third_party/tcmalloc/chromium/src/windows/patch_functions.cc b/third_party/tcmalloc/chromium/src/windows/patch_functions.cc index f837e7a..7a7e6ad 100644 --- a/third_party/tcmalloc/chromium/src/windows/patch_functions.cc +++ b/third_party/tcmalloc/chromium/src/windows/patch_functions.cc @@ -91,7 +91,7 @@ #include <vector> #include <base/logging.h> #include "base/spinlock.h" -#include "google/malloc_hook.h" +#include "gperftools/malloc_hook.h" #include "malloc_hook-inl.h" #include "preamble_patcher.h" @@ -181,6 +181,8 @@ class LibcInfo { kNewNothrow, kNewArrayNothrow, kDeleteNothrow, kDeleteArrayNothrow, // These are windows-only functions from malloc.h k_Msize, k_Expand, + // A MS CRT "internal" function, implemented using _calloc_impl + k_CallocCrt, kNumFunctions }; @@ -404,7 +406,7 @@ const char* const LibcInfo::function_name_[] = { NULL, // kMangledNewArrayNothrow, NULL, // kMangledDeleteNothrow, NULL, // kMangledDeleteArrayNothrow, - "_msize", "_expand", + "_msize", "_expand", "_calloc_crt", }; // For mingw, I can't patch the new/delete here, because the @@ -435,6 +437,7 @@ const GenericFnPtr LibcInfo::static_fn_[] = { #endif (GenericFnPtr)&::_msize, (GenericFnPtr)&::_expand, + (GenericFnPtr)&::calloc, }; template<int T> GenericFnPtr LibcInfoWithPatchFunctions<T>::origstub_fn_[] = { @@ -457,6 +460,7 @@ const GenericFnPtr LibcInfoWithPatchFunctions<T>::perftools_fn_[] = { (GenericFnPtr)&Perftools_deletearray_nothrow, (GenericFnPtr)&Perftools__msize, (GenericFnPtr)&Perftools__expand, + (GenericFnPtr)&Perftools_calloc, }; /*static*/ WindowsInfo::FunctionInfo WindowsInfo::function_info_[] = { @@ -822,7 +826,7 @@ void* LibcInfoWithPatchFunctions<T>::Perftools_realloc( return do_realloc_with_callback( old_ptr, new_size, (void (*)(void*))origstub_fn_[kFree], - (size_t (*)(void*))origstub_fn_[k_Msize]); + (size_t (*)(const void*))origstub_fn_[k_Msize]); } template<int T> @@ -900,7 +904,7 @@ void LibcInfoWithPatchFunctions<T>::Perftools_deletearray_nothrow( template<int T> size_t LibcInfoWithPatchFunctions<T>::Perftools__msize(void* ptr) __THROW { - return GetSizeWithCallback(ptr, (size_t (*)(void*))origstub_fn_[k_Msize]); + return GetSizeWithCallback(ptr, (size_t (*)(const void*))origstub_fn_[k_Msize]); } // We need to define this because internal windows functions like to diff --git a/third_party/tcmalloc/chromium/src/windows/port.cc b/third_party/tcmalloc/chromium/src/windows/port.cc index ec01fd6..0205912 100644 --- a/third_party/tcmalloc/chromium/src/windows/port.cc +++ b/third_party/tcmalloc/chromium/src/windows/port.cc @@ -149,8 +149,8 @@ static void NTAPI on_tls_callback(HINSTANCE h, DWORD dwReason, PVOID pv) { #ifdef _MSC_VER -// extern "C" suppresses C++ name mangling so we know the symbol names for the -// linker /INCLUDE:symbol pragmas above. +// extern "C" suppresses C++ name mangling so we know the symbol names +// for the linker /INCLUDE:symbol pragmas above. extern "C" { // This tells the linker to run these functions. #pragma data_seg(push, old_seg) @@ -219,10 +219,6 @@ extern "C" int perftools_pthread_once(pthread_once_t *once_control, // ----------------------------------------------------------------------- // These functions replace system-alloc.cc -// The current system allocator. Because we don't link with system-alloc.cc, -// we need to define our own. -SysAllocator* sys_alloc = NULL; - // This is mostly like MmapSysAllocator::Alloc, except it does these weird // munmap's in the middle of the page, which is forbidden in windows. extern void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, @@ -338,6 +334,9 @@ void DumpSystemAllocatorStats(TCMalloc_Printer* printer) { // We don't dump stats on windows, right now } +// The current system allocator +SysAllocator* sys_alloc = NULL; + // ----------------------------------------------------------------------- // These functions rework existing functions of the same name in the diff --git a/third_party/tcmalloc/chromium/src/windows/port.h b/third_party/tcmalloc/chromium/src/windows/port.h index 0faba01..e9a0206 100644 --- a/third_party/tcmalloc/chromium/src/windows/port.h +++ b/third_party/tcmalloc/chromium/src/windows/port.h @@ -65,14 +65,15 @@ /* * 4018: signed/unsigned mismatch is common (and ok for signed_i < unsigned_i) - * 4244: otherwise we get problems when substracting two size_t's to an int + * 4244: otherwise we get problems when subtracting two size_t's to an int * 4288: VC++7 gets confused when a var is defined in a loop and then after it * 4267: too many false positives for "conversion gives possible data loss" * 4290: it's ok windows ignores the "throw" directive * 4996: Yes, we're ok using "unsafe" functions like vsnprintf and getenv() + * 4146: internal_logging.cc intentionally negates an unsigned value */ #ifdef _MSC_VER -#pragma warning(disable:4018 4244 4288 4267 4290 4996) +#pragma warning(disable:4018 4244 4288 4267 4290 4996 4146) #endif #ifndef __cplusplus @@ -165,6 +166,10 @@ EXTERN_C int perftools_pthread_once(pthread_once_t *once_control, #endif /* __cplusplus */ #endif /* HAVE_PTHREAD */ +inline void sched_yield(void) { + Sleep(0); +} + /* * __declspec(thread) isn't usable in a dll opened via LoadLibrary(). * But it doesn't work to LoadLibrary() us anyway, because of all the @@ -260,7 +265,7 @@ class SpinLockHolder { // Acquires a spinlock for as long as the scope lasts #define MAP_PRIVATE MEM_COMMIT #define MAP_SHARED MEM_RESERVE /* value of this #define is 100% arbitrary */ -#if __STDC__ +#if __STDC__ && !defined(__MINGW32__) typedef _off_t off_t; #endif @@ -372,7 +377,6 @@ inline char *getcwd(char *buf, size_t size) { inline int mkdir(const char *pathname, int) { return _mkdir(pathname); } -#endif inline FILE *popen(const char *command, const char *type) { return _popen(command, type); @@ -380,13 +384,14 @@ inline FILE *popen(const char *command, const char *type) { inline int pclose(FILE *stream) { return _pclose(stream); } +#endif EXTERN_C PERFTOOLS_DLL_DECL void WriteToStderr(const char* buf, int len); /* ----------------------------------- SYSTEM/PROCESS */ typedef int pid_t; -#if __STDC__ +#if __STDC__ && !defined(__MINGW32__) inline pid_t getpid(void) { return _getpid(); } #endif inline pid_t getppid(void) { return 0; } @@ -410,10 +415,14 @@ inline unsigned int sleep(unsigned int seconds) { return 0; } +// mingw64 seems to define timespec (though mingw.org mingw doesn't), +// protected by the _TIMESPEC_DEFINED macro. +#ifndef _TIMESPEC_DEFINED struct timespec { int tv_sec; int tv_nsec; }; +#endif inline int nanosleep(const struct timespec *req, struct timespec *rem) { Sleep(req->tv_sec * 1000 + req->tv_nsec / 1000000); diff --git a/third_party/tcmalloc/chromium/src/windows/preamble_patcher.cc b/third_party/tcmalloc/chromium/src/windows/preamble_patcher.cc index 78a4763..b27a95b 100644 --- a/third_party/tcmalloc/chromium/src/windows/preamble_patcher.cc +++ b/third_party/tcmalloc/chromium/src/windows/preamble_patcher.cc @@ -29,6 +29,7 @@ * * --- * Author: Joi Sigurdsson + * Author: Scott Francis * * Implementation of PreamblePatcher */ @@ -46,18 +47,42 @@ #define ASM_JMP32ABS_0 0xFF #define ASM_JMP32ABS_1 0x25 #define ASM_JMP8REL 0xEB +#define ASM_JCC32REL_0 0x0F +#define ASM_JCC32REL_1_MASK 0x80 +#define ASM_NOP 0x90 +// X64 opcodes +#define ASM_REXW 0x48 +#define ASM_MOVRAX_IMM 0xB8 +#define ASM_JMP 0xFF +#define ASM_JMP_RAX 0xE0 namespace sidestep { +PreamblePatcher::PreamblePage* PreamblePatcher::preamble_pages_ = NULL; +long PreamblePatcher::granularity_ = 0; +long PreamblePatcher::pagesize_ = 0; +bool PreamblePatcher::initialized_ = false; + +static const unsigned int kPreamblePageMagic = 0x4347414D; // "MAGC" + // Handle a special case that we see with functions that point into an // IAT table (including functions linked statically into the // application): these function already starts with ASM_JMP32*. For // instance, malloc() might be implemented as a JMP to __malloc(). // This function follows the initial JMPs for us, until we get to the // place where the actual code is defined. If we get to STOP_BEFORE, -// we return the address before stop_before. +// we return the address before stop_before. The stop_before_trampoline +// flag is used in 64-bit mode. If true, we will return the address +// before a trampoline is detected. Trampolines are defined as: +// +// nop +// mov rax, <replacement_function> +// jmp rax +// +// See PreamblePatcher::RawPatchWithStub for more information. void* PreamblePatcher::ResolveTargetImpl(unsigned char* target, - unsigned char* stop_before) { + unsigned char* stop_before, + bool stop_before_trampoline) { if (target == NULL) return NULL; while (1) { @@ -81,15 +106,26 @@ void* PreamblePatcher::ResolveTargetImpl(unsigned char* target, // Visual studio seems to sometimes do it this way instead of the // previous way. Not sure what the rules are, but it was happening // with operator new in some binaries. - void **new_target_v; - SIDESTEP_ASSERT(sizeof(new_target) == 4); - memcpy(&new_target_v, reinterpret_cast<void*>(target + 2), 4); + void** new_target_v; + if (kIs64BitBinary) { + // In 64-bit mode JMPs are RIP-relative, not absolute + int target_offset; + memcpy(reinterpret_cast<void*>(&target_offset), + reinterpret_cast<void*>(target + 2), 4); + new_target_v = reinterpret_cast<void**>(target + target_offset + 6); + } else { + SIDESTEP_ASSERT(sizeof(new_target) == 4); + memcpy(&new_target_v, reinterpret_cast<void*>(target + 2), 4); + } new_target = reinterpret_cast<unsigned char*>(*new_target_v); } else { break; } if (new_target == stop_before) break; + if (stop_before_trampoline && *new_target == ASM_NOP + && new_target[1] == ASM_REXW && new_target[2] == ASM_MOVRAX_IMM) + break; target = new_target; } return target; @@ -103,7 +139,7 @@ class DeleteUnsignedCharArray { ~DeleteUnsignedCharArray() { if (array_) { - delete [] array_; + PreamblePatcher::FreePreambleBlock(array_); } } @@ -191,9 +227,23 @@ SideStepError PreamblePatcher::RawPatch(void* target_function, return SIDESTEP_INVALID_PARAMETER; } - // @see MAX_PREAMBLE_STUB_SIZE for an explanation of how we arrives at - // this size - unsigned char* preamble_stub = new unsigned char[MAX_PREAMBLE_STUB_SIZE]; + BOOL succeeded = FALSE; + + // First, deal with a special case that we see with functions that + // point into an IAT table (including functions linked statically + // into the application): these function already starts with + // ASM_JMP32REL. For instance, malloc() might be implemented as a + // JMP to __malloc(). In that case, we replace the destination of + // the JMP (__malloc), rather than the JMP itself (malloc). This + // way we get the correct behavior no matter how malloc gets called. + void* new_target = ResolveTarget(target_function); + if (new_target != target_function) { + target_function = new_target; + } + + // In 64-bit mode, preamble_stub must be within 2GB of target function + // so that if target contains a jump, we can translate it. + unsigned char* preamble_stub = AllocPreambleBlockNear(target_function); if (!preamble_stub) { SIDESTEP_ASSERT(false && "Unable to allocate preamble-stub."); return SIDESTEP_INSUFFICIENT_BUFFER; @@ -202,19 +252,6 @@ SideStepError PreamblePatcher::RawPatch(void* target_function, // Frees the array at end of scope. DeleteUnsignedCharArray guard_preamble_stub(preamble_stub); - // Change the protection of the newly allocated preamble stub to - // PAGE_EXECUTE_READWRITE. This is required to work with DEP (Data - // Execution Prevention) which will cause an exception if code is executed - // from a page on which you do not have read access. - DWORD old_stub_protect = 0; - BOOL succeeded = ::VirtualProtect(preamble_stub, MAX_PREAMBLE_STUB_SIZE, - PAGE_EXECUTE_READWRITE, &old_stub_protect); - if (!succeeded) { - SIDESTEP_ASSERT(false && - "Failed to make page preamble stub read-write-execute."); - return SIDESTEP_ACCESS_DENIED; - } - SideStepError error_code = RawPatchWithStubAndProtections( target_function, replacement_function, preamble_stub, MAX_PREAMBLE_STUB_SIZE, NULL); @@ -260,23 +297,6 @@ SideStepError PreamblePatcher::Unpatch(void* target_function, return SIDESTEP_INVALID_PARAMETER; } - // We disassemble the preamble of the _stub_ to see how many bytes we - // originally copied to the stub. - MiniDisassembler disassembler; - unsigned int preamble_bytes = 0; - while (preamble_bytes < 5) { - InstructionType instruction_type = - disassembler.Disassemble( - reinterpret_cast<unsigned char*>(original_function_stub) + - preamble_bytes, - preamble_bytes); - if (IT_GENERIC != instruction_type) { - SIDESTEP_ASSERT(false && - "Should only have generic instructions in stub!!"); - return SIDESTEP_UNSUPPORTED_INSTRUCTION; - } - } - // Before unpatching, target_function should be a JMP to // replacement_function. If it's not, then either it's an error, or // we're falling into the case where the original instruction was a @@ -286,7 +306,8 @@ SideStepError PreamblePatcher::Unpatch(void* target_function, unsigned char* target = reinterpret_cast<unsigned char*>(target_function); target = reinterpret_cast<unsigned char*>( ResolveTargetImpl( - target, reinterpret_cast<unsigned char*>(replacement_function))); + target, reinterpret_cast<unsigned char*>(replacement_function), + true)); // We should end at the function we patched. When we patch, we insert // a ASM_JMP32REL instruction, so look for that as a sanity check. if (target[0] != ASM_JMP32REL) { @@ -295,11 +316,13 @@ SideStepError PreamblePatcher::Unpatch(void* target_function, return SIDESTEP_INVALID_PARAMETER; } + const unsigned int kRequiredTargetPatchBytes = 5; + // We need to be able to write to a process-local copy of the first - // MAX_PREAMBLE_STUB_SIZE bytes of target_function + // kRequiredTargetPatchBytes bytes of target_function DWORD old_target_function_protect = 0; - BOOL succeeded = ::VirtualProtect(reinterpret_cast<void*>(target_function), - MAX_PREAMBLE_STUB_SIZE, + BOOL succeeded = ::VirtualProtect(reinterpret_cast<void*>(target), + kRequiredTargetPatchBytes, PAGE_EXECUTE_READWRITE, &old_target_function_protect); if (!succeeded) { @@ -308,20 +331,67 @@ SideStepError PreamblePatcher::Unpatch(void* target_function, return SIDESTEP_ACCESS_DENIED; } - // Replace the first few bytes of the original function with the bytes we - // previously moved to the preamble stub. - memcpy(reinterpret_cast<void*>(target), - original_function_stub, preamble_bytes); + unsigned char* preamble_stub = reinterpret_cast<unsigned char*>( + original_function_stub); - // Stub is now useless so delete it. - // [csilvers: Commented out for perftools because it causes big problems - // when we're unpatching malloc. We just let this live on as a leak.] - //delete [] reinterpret_cast<unsigned char*>(original_function_stub); + // Disassemble the preamble of stub and copy the bytes back to target. + // If we've done any conditional jumps in the preamble we need to convert + // them back to the orignal REL8 jumps in the target. + MiniDisassembler disassembler; + unsigned int preamble_bytes = 0; + unsigned int target_bytes = 0; + while (target_bytes < kRequiredTargetPatchBytes) { + unsigned int cur_bytes = 0; + InstructionType instruction_type = + disassembler.Disassemble(preamble_stub + preamble_bytes, cur_bytes); + if (IT_JUMP == instruction_type) { + unsigned int jump_bytes = 0; + SideStepError jump_ret = SIDESTEP_JUMP_INSTRUCTION; + if (IsNearConditionalJump(preamble_stub + preamble_bytes, cur_bytes) || + IsNearRelativeJump(preamble_stub + preamble_bytes, cur_bytes) || + IsNearAbsoluteCall(preamble_stub + preamble_bytes, cur_bytes) || + IsNearRelativeCall(preamble_stub + preamble_bytes, cur_bytes)) { + jump_ret = PatchNearJumpOrCall(preamble_stub + preamble_bytes, + cur_bytes, target + target_bytes, + &jump_bytes, MAX_PREAMBLE_STUB_SIZE); + } + if (jump_ret == SIDESTEP_JUMP_INSTRUCTION) { + SIDESTEP_ASSERT(false && + "Found unsupported jump instruction in stub!!"); + return SIDESTEP_UNSUPPORTED_INSTRUCTION; + } + target_bytes += jump_bytes; + } else if (IT_GENERIC == instruction_type) { + if (IsMovWithDisplacement(preamble_stub + preamble_bytes, cur_bytes)) { + unsigned int mov_bytes = 0; + if (PatchMovWithDisplacement(preamble_stub + preamble_bytes, cur_bytes, + target + target_bytes, &mov_bytes, + MAX_PREAMBLE_STUB_SIZE) + != SIDESTEP_SUCCESS) { + SIDESTEP_ASSERT(false && + "Found unsupported generic instruction in stub!!"); + return SIDESTEP_UNSUPPORTED_INSTRUCTION; + } + } else { + memcpy(reinterpret_cast<void*>(target + target_bytes), + reinterpret_cast<void*>(reinterpret_cast<unsigned char*>( + original_function_stub) + preamble_bytes), cur_bytes); + target_bytes += cur_bytes; + } + } else { + SIDESTEP_ASSERT(false && + "Found unsupported instruction in stub!!"); + return SIDESTEP_UNSUPPORTED_INSTRUCTION; + } + preamble_bytes += cur_bytes; + } - // Restore the protection of the first MAX_PREAMBLE_STUB_SIZE bytes of + FreePreambleBlock(reinterpret_cast<unsigned char*>(original_function_stub)); + + // Restore the protection of the first kRequiredTargetPatchBytes bytes of // target to what they were before we started goofing around. succeeded = ::VirtualProtect(reinterpret_cast<void*>(target), - MAX_PREAMBLE_STUB_SIZE, + kRequiredTargetPatchBytes, old_target_function_protect, &old_target_function_protect); @@ -341,4 +411,274 @@ SideStepError PreamblePatcher::Unpatch(void* target_function, return SIDESTEP_SUCCESS; } +void PreamblePatcher::Initialize() { + if (!initialized_) { + SYSTEM_INFO si = { 0 }; + ::GetSystemInfo(&si); + granularity_ = si.dwAllocationGranularity; + pagesize_ = si.dwPageSize; + initialized_ = true; + } +} + +unsigned char* PreamblePatcher::AllocPreambleBlockNear(void* target) { + PreamblePage* preamble_page = preamble_pages_; + while (preamble_page != NULL) { + if (preamble_page->free_ != NULL) { + __int64 val = reinterpret_cast<__int64>(preamble_page) - + reinterpret_cast<__int64>(target); + if ((val > 0 && val + pagesize_ <= INT_MAX) || + (val < 0 && val >= INT_MIN)) { + break; + } + } + preamble_page = preamble_page->next_; + } + + // The free_ member of the page is used to store the next available block + // of memory to use or NULL if there are no chunks available, in which case + // we'll allocate a new page. + if (preamble_page == NULL || preamble_page->free_ == NULL) { + // Create a new preamble page and initialize the free list + preamble_page = reinterpret_cast<PreamblePage*>(AllocPageNear(target)); + SIDESTEP_ASSERT(preamble_page != NULL && "Could not allocate page!"); + void** pp = &preamble_page->free_; + unsigned char* ptr = reinterpret_cast<unsigned char*>(preamble_page) + + MAX_PREAMBLE_STUB_SIZE; + unsigned char* limit = reinterpret_cast<unsigned char*>(preamble_page) + + pagesize_; + while (ptr < limit) { + *pp = ptr; + pp = reinterpret_cast<void**>(ptr); + ptr += MAX_PREAMBLE_STUB_SIZE; + } + *pp = NULL; + // Insert the new page into the list + preamble_page->magic_ = kPreamblePageMagic; + preamble_page->next_ = preamble_pages_; + preamble_pages_ = preamble_page; + } + unsigned char* ret = reinterpret_cast<unsigned char*>(preamble_page->free_); + preamble_page->free_ = *(reinterpret_cast<void**>(preamble_page->free_)); + return ret; +} + +void PreamblePatcher::FreePreambleBlock(unsigned char* block) { + SIDESTEP_ASSERT(block != NULL); + SIDESTEP_ASSERT(granularity_ != 0); + uintptr_t ptr = reinterpret_cast<uintptr_t>(block); + ptr -= ptr & (granularity_ - 1); + PreamblePage* preamble_page = reinterpret_cast<PreamblePage*>(ptr); + SIDESTEP_ASSERT(preamble_page->magic_ == kPreamblePageMagic); + *(reinterpret_cast<void**>(block)) = preamble_page->free_; + preamble_page->free_ = block; +} + +void* PreamblePatcher::AllocPageNear(void* target) { + MEMORY_BASIC_INFORMATION mbi = { 0 }; + if (!::VirtualQuery(target, &mbi, sizeof(mbi))) { + SIDESTEP_ASSERT(false && "VirtualQuery failed on target address"); + return 0; + } + if (initialized_ == false) { + PreamblePatcher::Initialize(); + SIDESTEP_ASSERT(initialized_); + } + void* pv = NULL; + unsigned char* allocation_base = reinterpret_cast<unsigned char*>( + mbi.AllocationBase); + __int64 i = 1; + bool high_target = reinterpret_cast<__int64>(target) > UINT_MAX; + while (pv == NULL) { + __int64 val = reinterpret_cast<__int64>(allocation_base) - + (i * granularity_); + if (high_target && + reinterpret_cast<__int64>(target) - val > INT_MAX) { + // We're further than 2GB from the target + break; + } else if (val <= NULL) { + // Less than 0 + break; + } + pv = ::VirtualAlloc(reinterpret_cast<void*>(allocation_base - + (i++ * granularity_)), + pagesize_, MEM_COMMIT | MEM_RESERVE, + PAGE_EXECUTE_READWRITE); + } + + // We couldn't allocate low, try to allocate high + if (pv == NULL) { + i = 1; + // Round up to the next multiple of page granularity + allocation_base = reinterpret_cast<unsigned char*>( + (reinterpret_cast<__int64>(target) & + (~(granularity_ - 1))) + granularity_); + while (pv == NULL) { + __int64 val = reinterpret_cast<__int64>(allocation_base) + + (i * granularity_) - reinterpret_cast<__int64>(target); + if (val > INT_MAX || val < 0) { + // We're too far or we overflowed + break; + } + pv = ::VirtualAlloc(reinterpret_cast<void*>(allocation_base + + (i++ * granularity_)), + pagesize_, MEM_COMMIT | MEM_RESERVE, + PAGE_EXECUTE_READWRITE); + } + } + return pv; +} + +bool PreamblePatcher::IsShortConditionalJump( + unsigned char* target, + unsigned int instruction_size) { + return (*(target) & 0x70) == 0x70 && instruction_size == 2; +} + +bool PreamblePatcher::IsNearConditionalJump( + unsigned char* target, + unsigned int instruction_size) { + return *(target) == 0xf && (*(target + 1) & 0x80) == 0x80 && + instruction_size == 6; +} + +bool PreamblePatcher::IsNearRelativeJump( + unsigned char* target, + unsigned int instruction_size) { + return *(target) == 0xe9 && instruction_size == 5; +} + +bool PreamblePatcher::IsNearAbsoluteCall( + unsigned char* target, + unsigned int instruction_size) { + return *(target) == 0xff && (*(target + 1) & 0x10) == 0x10 && + instruction_size == 6; +} + +bool PreamblePatcher::IsNearRelativeCall( + unsigned char* target, + unsigned int instruction_size) { + return *(target) == 0xe8 && instruction_size == 5; +} + +bool PreamblePatcher::IsMovWithDisplacement( + unsigned char* target, + unsigned int instruction_size) { + // In this case, the ModRM byte's mod field will be 0 and r/m will be 101b (5) + return instruction_size == 7 && *target == 0x48 && *(target + 1) == 0x8b && + (*(target + 2) >> 6) == 0 && (*(target + 2) & 0x7) == 5; +} + +SideStepError PreamblePatcher::PatchShortConditionalJump( + unsigned char* source, + unsigned int instruction_size, + unsigned char* target, + unsigned int* target_bytes, + unsigned int target_size) { + unsigned char* original_jump_dest = (source + 2) + source[1]; + unsigned char* stub_jump_from = target + 6; + __int64 fixup_jump_offset = original_jump_dest - stub_jump_from; + if (fixup_jump_offset > INT_MAX || fixup_jump_offset < INT_MIN) { + SIDESTEP_ASSERT(false && + "Unable to fix up short jump because target" + " is too far away."); + return SIDESTEP_JUMP_INSTRUCTION; + } + + *target_bytes = 6; + if (target_size > *target_bytes) { + // Convert the short jump to a near jump. + // + // 0f 8x xx xx xx xx = Jcc rel32off + unsigned short jmpcode = ((0x80 | (source[0] & 0xf)) << 8) | 0x0f; + memcpy(reinterpret_cast<void*>(target), + reinterpret_cast<void*>(&jmpcode), 2); + memcpy(reinterpret_cast<void*>(target + 2), + reinterpret_cast<void*>(&fixup_jump_offset), 4); + } + + return SIDESTEP_SUCCESS; +} + +SideStepError PreamblePatcher::PatchNearJumpOrCall( + unsigned char* source, + unsigned int instruction_size, + unsigned char* target, + unsigned int* target_bytes, + unsigned int target_size) { + SIDESTEP_ASSERT(instruction_size == 5 || instruction_size == 6); + unsigned int jmp_offset_in_instruction = instruction_size == 5 ? 1 : 2; + unsigned char* original_jump_dest = reinterpret_cast<unsigned char *>( + reinterpret_cast<__int64>(source + instruction_size) + + *(reinterpret_cast<int*>(source + jmp_offset_in_instruction))); + unsigned char* stub_jump_from = target + instruction_size; + __int64 fixup_jump_offset = original_jump_dest - stub_jump_from; + if (fixup_jump_offset > INT_MAX || fixup_jump_offset < INT_MIN) { + SIDESTEP_ASSERT(false && + "Unable to fix up near jump because target" + " is too far away."); + return SIDESTEP_JUMP_INSTRUCTION; + } + + if ((fixup_jump_offset < SCHAR_MAX && fixup_jump_offset > SCHAR_MIN)) { + *target_bytes = 2; + if (target_size > *target_bytes) { + // If the new offset is in range, use a short jump instead of a near jump. + if (source[0] == ASM_JCC32REL_0 && + (source[1] & ASM_JCC32REL_1_MASK) == ASM_JCC32REL_1_MASK) { + unsigned short jmpcode = (static_cast<unsigned char>( + fixup_jump_offset) << 8) | (0x70 | (source[1] & 0xf)); + memcpy(reinterpret_cast<void*>(target), + reinterpret_cast<void*>(&jmpcode), + 2); + } else { + target[0] = ASM_JMP8REL; + target[1] = static_cast<unsigned char>(fixup_jump_offset); + } + } + } else { + *target_bytes = instruction_size; + if (target_size > *target_bytes) { + memcpy(reinterpret_cast<void*>(target), + reinterpret_cast<void*>(source), + jmp_offset_in_instruction); + memcpy(reinterpret_cast<void*>(target + jmp_offset_in_instruction), + reinterpret_cast<void*>(&fixup_jump_offset), + 4); + } + } + + return SIDESTEP_SUCCESS; +} + +SideStepError PreamblePatcher::PatchMovWithDisplacement( + unsigned char* source, + unsigned int instruction_size, + unsigned char* target, + unsigned int* target_bytes, + unsigned int target_size) { + SIDESTEP_ASSERT(instruction_size == 7); + const int mov_offset_in_instruction = 3; // 0x48 0x8b 0x0d <offset> + unsigned char* original_mov_dest = reinterpret_cast<unsigned char*>( + reinterpret_cast<__int64>(source + instruction_size) + + *(reinterpret_cast<int*>(source + mov_offset_in_instruction))); + unsigned char* stub_mov_from = target + instruction_size; + __int64 fixup_mov_offset = original_mov_dest - stub_mov_from; + if (fixup_mov_offset > INT_MAX || fixup_mov_offset < INT_MIN) { + SIDESTEP_ASSERT(false && + "Unable to fix up near MOV because target is too far away."); + return SIDESTEP_UNEXPECTED; + } + *target_bytes = instruction_size; + if (target_size > *target_bytes) { + memcpy(reinterpret_cast<void*>(target), + reinterpret_cast<void*>(source), + mov_offset_in_instruction); + memcpy(reinterpret_cast<void*>(target + mov_offset_in_instruction), + reinterpret_cast<void*>(&fixup_mov_offset), + 4); + } + return SIDESTEP_SUCCESS; +} + }; // namespace sidestep diff --git a/third_party/tcmalloc/chromium/src/windows/preamble_patcher.h b/third_party/tcmalloc/chromium/src/windows/preamble_patcher.h index 0028e4e..4fdb7d0 100644 --- a/third_party/tcmalloc/chromium/src/windows/preamble_patcher.h +++ b/third_party/tcmalloc/chromium/src/windows/preamble_patcher.h @@ -29,6 +29,7 @@ * * --- * Author: Joi Sigurdsson + * Author: Scott Francis * * Definition of PreamblePatcher */ @@ -36,6 +37,7 @@ #ifndef GOOGLE_PERFTOOLS_PREAMBLE_PATCHER_H_ #define GOOGLE_PERFTOOLS_PREAMBLE_PATCHER_H_ +#include "config.h" #include <windows.h> // compatibility shim @@ -47,7 +49,25 @@ // bytes of the function. Considering the worst case scenario, we need 4 // bytes + the max instruction size + 5 more bytes for our jump back to // the original code. With that in mind, 32 is a good number :) +#ifdef _M_X64 +// In 64-bit mode we may need more room. In 64-bit mode all jumps must be +// within +/-2GB of RIP. Because of this limitation we may need to use a +// trampoline to jump to the replacement function if it is further than 2GB +// away from the target. The trampoline is 14 bytes. +// +// So 4 bytes + max instruction size (17 bytes) + 5 bytes to jump back to the +// original code + trampoline size. 64 bytes is a nice number :-) +#define MAX_PREAMBLE_STUB_SIZE (64) +#else #define MAX_PREAMBLE_STUB_SIZE (32) +#endif + +// Determines if this is a 64-bit binary. +#ifdef _M_X64 +static const bool kIs64BitBinary = true; +#else +static const bool kIs64BitBinary = false; +#endif namespace sidestep { @@ -68,6 +88,8 @@ enum SideStepError { #define SIDESTEP_TO_HRESULT(error) \ MAKE_HRESULT(SEVERITY_ERROR, FACILITY_NULL, error) +class DeleteUnsignedCharArray; + // Implements a patching mechanism that overwrites the first few bytes of // a function preamble with a jump to our hook function, which is then // able to call the original function via a specially-made preamble-stub @@ -126,7 +148,7 @@ enum SideStepError { // reuse the result of calling the function with a given parameter, which // may mean if you patch the function in between your patch will never get // invoked. See preamble_patcher_test.cc for an example. -class PreamblePatcher { +class PERFTOOLS_DLL_DECL PreamblePatcher { public: // This is a typesafe version of RawPatch(), identical in all other @@ -287,7 +309,55 @@ class PreamblePatcher { return (T)ResolveTargetImpl((unsigned char*)target_function, NULL); } + // Allocates a block of memory of size MAX_PREAMBLE_STUB_SIZE that is as + // close (within 2GB) as possible to target. This is done to ensure that + // we can perform a relative jump from target to a trampoline if the + // replacement function is > +-2GB from target. This means that we only need + // to patch 5 bytes in the target function. + // + // @param target Pointer to target function. + // + // @return Returns a block of memory of size MAX_PREAMBLE_STUB_SIZE that can + // be used to store a function preamble block. + static unsigned char* AllocPreambleBlockNear(void* target); + + // Frees a block allocated by AllocPreambleBlockNear. + // + // @param block Block that was returned by AllocPreambleBlockNear. + static void FreePreambleBlock(unsigned char* block); + private: + friend class DeleteUnsignedCharArray; + + // Used to store data allocated for preamble stubs + struct PreamblePage { + unsigned int magic_; + PreamblePage* next_; + // This member points to a linked list of free blocks within the page + // or NULL if at the end + void* free_; + }; + + // In 64-bit mode, the replacement function must be within 2GB of the original + // target in order to only require 5 bytes for the function patch. To meet + // this requirement we're creating an allocator within this class to + // allocate blocks that are within 2GB of a given target. This member is the + // head of a linked list of pages used to allocate blocks that are within + // 2GB of the target. + static PreamblePage* preamble_pages_; + + // Page granularity + static long granularity_; + + // Page size + static long pagesize_; + + // Determines if the patcher has been initialized. + static bool initialized_; + + // Used to initialize static members. + static void Initialize(); + // Patches a function by overwriting its first few bytes with // a jump to a different function. This is similar to the RawPatch // function except that it uses the stub allocated by the caller @@ -318,7 +388,7 @@ class PreamblePatcher { // @return An error code indicating the result of patching. static SideStepError RawPatchWithStubAndProtections( void* target_function, - void *replacement_function, + void* replacement_function, unsigned char* preamble_stub, unsigned long stub_size, unsigned long* bytes_needed); @@ -348,7 +418,7 @@ class PreamblePatcher { // // @return An error code indicating the result of patching. static SideStepError RawPatchWithStub(void* target_function, - void *replacement_function, + void* replacement_function, unsigned char* preamble_stub, unsigned long stub_size, unsigned long* bytes_needed); @@ -365,12 +435,175 @@ class PreamblePatcher { // target_function, we get to the address stop, we return // immediately, the address that jumps to stop_before. // + // @param stop_before_trampoline When following JMP instructions from + // target_function, stop before a trampoline is detected. See comment in + // PreamblePatcher::RawPatchWithStub for more information. This parameter + // has no effect in 32-bit mode. + // // @return Either target_function (the input parameter), or if // target_function's body consists entirely of a JMP instruction, // the address it JMPs to (or more precisely, the address at the end // of a chain of JMPs). static void* ResolveTargetImpl(unsigned char* target_function, - unsigned char* stop_before); + unsigned char* stop_before, + bool stop_before_trampoline = false); + + // Helper routine that attempts to allocate a page as close (within 2GB) + // as possible to target. + // + // @param target Pointer to target function. + // + // @return Returns an address that is within 2GB of target. + static void* AllocPageNear(void* target); + + // Helper routine that determines if a target instruction is a short + // conditional jump. + // + // @param target Pointer to instruction. + // + // @param instruction_size Size of the instruction in bytes. + // + // @return Returns true if the instruction is a short conditional jump. + static bool IsShortConditionalJump(unsigned char* target, + unsigned int instruction_size); + + // Helper routine that determines if a target instruction is a near + // conditional jump. + // + // @param target Pointer to instruction. + // + // @param instruction_size Size of the instruction in bytes. + // + // @return Returns true if the instruction is a near conditional jump. + static bool IsNearConditionalJump(unsigned char* target, + unsigned int instruction_size); + + // Helper routine that determines if a target instruction is a near + // relative jump. + // + // @param target Pointer to instruction. + // + // @param instruction_size Size of the instruction in bytes. + // + // @return Returns true if the instruction is a near absolute jump. + static bool IsNearRelativeJump(unsigned char* target, + unsigned int instruction_size); + + // Helper routine that determines if a target instruction is a near + // absolute call. + // + // @param target Pointer to instruction. + // + // @param instruction_size Size of the instruction in bytes. + // + // @return Returns true if the instruction is a near absolute call. + static bool IsNearAbsoluteCall(unsigned char* target, + unsigned int instruction_size); + + // Helper routine that determines if a target instruction is a near + // absolute call. + // + // @param target Pointer to instruction. + // + // @param instruction_size Size of the instruction in bytes. + // + // @return Returns true if the instruction is a near absolute call. + static bool IsNearRelativeCall(unsigned char* target, + unsigned int instruction_size); + + // Helper routine that determines if a target instruction is a 64-bit MOV + // that uses a RIP-relative displacement. + // + // @param target Pointer to instruction. + // + // @param instruction_size Size of the instruction in bytes. + // + // @return Returns true if the instruction is a MOV with displacement. + static bool IsMovWithDisplacement(unsigned char* target, + unsigned int instruction_size); + + // Helper routine that converts a short conditional jump instruction + // to a near conditional jump in a target buffer. Note that the target + // buffer must be within 2GB of the source for the near jump to work. + // + // A short conditional jump instruction is in the format: + // 7x xx = Jcc rel8off + // + // @param source Pointer to instruction. + // + // @param instruction_size Size of the instruction. + // + // @param target Target buffer to write the new instruction. + // + // @param target_bytes Pointer to a buffer that contains the size + // of the target instruction, in bytes. + // + // @param target_size Size of the target buffer. + // + // @return Returns SIDESTEP_SUCCESS if successful, otherwise an error. + static SideStepError PatchShortConditionalJump(unsigned char* source, + unsigned int instruction_size, + unsigned char* target, + unsigned int* target_bytes, + unsigned int target_size); + + // Helper routine that converts an instruction that will convert various + // jump-like instructions to corresponding instructions in the target buffer. + // What this routine does is fix up the relative offsets contained in jump + // instructions to point back to the original target routine. Like with + // PatchShortConditionalJump, the target buffer must be within 2GB of the + // source. + // + // We currently handle the following instructions: + // + // E9 xx xx xx xx = JMP rel32off + // 0F 8x xx xx xx xx = Jcc rel32off + // FF /2 xx xx xx xx = CALL reg/mem32/mem64 + // E8 xx xx xx xx = CALL rel32off + // + // It should not be hard to update this function to support other + // instructions that jump to relative targets. + // + // @param source Pointer to instruction. + // + // @param instruction_size Size of the instruction. + // + // @param target Target buffer to write the new instruction. + // + // @param target_bytes Pointer to a buffer that contains the size + // of the target instruction, in bytes. + // + // @param target_size Size of the target buffer. + // + // @return Returns SIDESTEP_SUCCESS if successful, otherwise an error. + static SideStepError PatchNearJumpOrCall(unsigned char* source, + unsigned int instruction_size, + unsigned char* target, + unsigned int* target_bytes, + unsigned int target_size); + + // Helper routine that patches a 64-bit MOV instruction with a RIP-relative + // displacement. The target buffer must be within 2GB of the source. + // + // 48 8B 0D XX XX XX XX = MOV rel32off + // + // @param source Pointer to instruction. + // + // @param instruction_size Size of the instruction. + // + // @param target Target buffer to write the new instruction. + // + // @param target_bytes Pointer to a buffer that contains the size + // of the target instruction, in bytes. + // + // @param target_size Size of the target buffer. + // + // @return Returns SIDESTEP_SUCCESS if successful, otherwise an error. + static SideStepError PatchMovWithDisplacement(unsigned char* source, + unsigned int instruction_size, + unsigned char* target, + unsigned int* target_bytes, + unsigned int target_size); }; }; // namespace sidestep diff --git a/third_party/tcmalloc/chromium/src/windows/preamble_patcher_test.cc b/third_party/tcmalloc/chromium/src/windows/preamble_patcher_test.cc new file mode 100644 index 0000000..41ab551 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/windows/preamble_patcher_test.cc @@ -0,0 +1,367 @@ +/* Copyright (c) 2011, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Joi Sigurdsson + * Author: Scott Francis + * + * Unit tests for PreamblePatcher + */ + +#include "config_for_unittests.h" +#include "preamble_patcher.h" +#include "mini_disassembler.h" +#pragma warning(push) +#pragma warning(disable:4553) +#include "auto_testing_hook.h" +#pragma warning(pop) + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <tchar.h> + +// Turning off all optimizations for this file, since the official build's +// "Whole program optimization" seems to cause the TestPatchUsingDynamicStub +// test to crash with an access violation. We debugged this and found +// that the optimized access a register that is changed by a call to the hook +// function. +#pragma optimize("", off) + +// A convenience macro to avoid a lot of casting in the tests. +// I tried to make this a templated function, but windows complained: +// error C2782: 'sidestep::SideStepError `anonymous-namespace'::Unpatch(T,T,T *)' : template parameter 'T' is ambiguous +// could be 'int (int)' +// or 'int (__cdecl *)(int)' +// My life isn't long enough to try to figure out how to fix this. +#define UNPATCH(target_function, replacement_function, original_function_stub) \ + sidestep::PreamblePatcher::Unpatch((void*)(target_function), \ + (void*)(replacement_function), \ + (void*)(original_function)) + +namespace { + +// Function for testing - this is what we patch +// +// NOTE: Because of the way the compiler optimizes this function in +// release builds, we need to use a different input value every time we +// call it within a function, otherwise the compiler will just reuse the +// last calculated incremented value. +int __declspec(noinline) IncrementNumber(int i) { +#ifdef _M_X64 + __int64 i2 = i + 1; + return (int) i2; +#else + return i + 1; +#endif +} + +extern "C" int TooShortFunction(int); + +extern "C" int JumpShortCondFunction(int); + +extern "C" int JumpNearCondFunction(int); + +extern "C" int JumpAbsoluteFunction(int); + +extern "C" int CallNearRelativeFunction(int); + +typedef int (*IncrementingFunc)(int); +IncrementingFunc original_function = NULL; + +int HookIncrementNumber(int i) { + SIDESTEP_ASSERT(original_function != NULL); + int incremented_once = original_function(i); + return incremented_once + 1; +} + +// For the AutoTestingHook test, we can't use original_function, because +// all that is encapsulated. +// This function "increments" by 10, just to set it apart from the other +// functions. +int __declspec(noinline) AutoHookIncrementNumber(int i) { + return i + 10; +} + +}; // namespace + +namespace sidestep { + +bool TestDisassembler() { + unsigned int instruction_size = 0; + sidestep::MiniDisassembler disassembler; + void * target = reinterpret_cast<unsigned char *>(IncrementNumber); + void * new_target = PreamblePatcher::ResolveTarget(target); + if (target != new_target) + target = new_target; + + while (1) { + sidestep::InstructionType instructionType = disassembler.Disassemble( + reinterpret_cast<unsigned char *>(target) + instruction_size, + instruction_size); + if (sidestep::IT_RETURN == instructionType) { + return true; + } + } +} + +bool TestPatchWithLongJump() { + original_function = NULL; + void *p = ::VirtualAlloc(reinterpret_cast<void *>(0x0000020000000000), 4096, + MEM_RESERVE | MEM_COMMIT, PAGE_EXECUTE_READWRITE); + SIDESTEP_EXPECT_TRUE(p != NULL); + memset(p, 0xcc, 4096); + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + sidestep::PreamblePatcher::Patch(IncrementNumber, + (IncrementingFunc) p, + &original_function)); + SIDESTEP_ASSERT((*original_function)(1) == 2); + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + UNPATCH(IncrementNumber, + (IncrementingFunc)p, + original_function)); + ::VirtualFree(p, 0, MEM_RELEASE); + return true; +} + +bool TestPatchWithPreambleShortCondJump() { + original_function = NULL; + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + sidestep::PreamblePatcher::Patch(JumpShortCondFunction, + HookIncrementNumber, + &original_function)); + (*original_function)(1); + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + UNPATCH(JumpShortCondFunction, + (void*)HookIncrementNumber, + original_function)); + return true; +} + +bool TestPatchWithPreambleNearRelativeCondJump() { + original_function = NULL; + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + sidestep::PreamblePatcher::Patch(JumpNearCondFunction, + HookIncrementNumber, + &original_function)); + (*original_function)(0); + (*original_function)(1); + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + UNPATCH(JumpNearCondFunction, + HookIncrementNumber, + original_function)); + return true; +} + +bool TestPatchWithPreambleAbsoluteJump() { + original_function = NULL; + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + sidestep::PreamblePatcher::Patch(JumpAbsoluteFunction, + HookIncrementNumber, + &original_function)); + (*original_function)(0); + (*original_function)(1); + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + UNPATCH(JumpAbsoluteFunction, + HookIncrementNumber, + original_function)); + return true; +} + +bool TestPatchWithPreambleNearRelativeCall() { + original_function = NULL; + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + sidestep::PreamblePatcher::Patch( + CallNearRelativeFunction, + HookIncrementNumber, + &original_function)); + (*original_function)(0); + (*original_function)(1); + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + UNPATCH(CallNearRelativeFunction, + HookIncrementNumber, + original_function)); + return true; +} + +bool TestPatchUsingDynamicStub() { + original_function = NULL; + SIDESTEP_EXPECT_TRUE(IncrementNumber(1) == 2); + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + sidestep::PreamblePatcher::Patch(IncrementNumber, + HookIncrementNumber, + &original_function)); + SIDESTEP_EXPECT_TRUE(original_function); + SIDESTEP_EXPECT_TRUE(IncrementNumber(2) == 4); + SIDESTEP_EXPECT_TRUE(original_function(3) == 4); + + // Clearbox test to see that the function has been patched. + sidestep::MiniDisassembler disassembler; + unsigned int instruction_size = 0; + SIDESTEP_EXPECT_TRUE(sidestep::IT_JUMP == disassembler.Disassemble( + reinterpret_cast<unsigned char*>(IncrementNumber), + instruction_size)); + + // Since we patched IncrementNumber, its first statement is a + // jmp to the hook function. So verify that we now can not patch + // IncrementNumber because it starts with a jump. +#if 0 + IncrementingFunc dummy = NULL; + // TODO(joi@chromium.org): restore this test once flag is added to + // disable JMP following + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_JUMP_INSTRUCTION == + sidestep::PreamblePatcher::Patch(IncrementNumber, + HookIncrementNumber, + &dummy)); + + // This test disabled because code in preamble_patcher_with_stub.cc + // asserts before returning the error code -- so there is no way + // to get an error code here, in debug build. + dummy = NULL; + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_FUNCTION_TOO_SMALL == + sidestep::PreamblePatcher::Patch(TooShortFunction, + HookIncrementNumber, + &dummy)); +#endif + + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + UNPATCH(IncrementNumber, + HookIncrementNumber, + original_function)); + return true; +} + +bool PatchThenUnpatch() { + original_function = NULL; + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + sidestep::PreamblePatcher::Patch(IncrementNumber, + HookIncrementNumber, + &original_function)); + SIDESTEP_EXPECT_TRUE(original_function); + SIDESTEP_EXPECT_TRUE(IncrementNumber(1) == 3); + SIDESTEP_EXPECT_TRUE(original_function(2) == 3); + + SIDESTEP_EXPECT_TRUE(sidestep::SIDESTEP_SUCCESS == + UNPATCH(IncrementNumber, + HookIncrementNumber, + original_function)); + original_function = NULL; + SIDESTEP_EXPECT_TRUE(IncrementNumber(3) == 4); + + return true; +} + +bool AutoTestingHookTest() { + SIDESTEP_EXPECT_TRUE(IncrementNumber(1) == 2); + + // Inner scope, so we can test what happens when the AutoTestingHook + // goes out of scope + { + AutoTestingHook hook = MakeTestingHook(IncrementNumber, + AutoHookIncrementNumber); + (void) hook; + SIDESTEP_EXPECT_TRUE(IncrementNumber(2) == 12); + } + SIDESTEP_EXPECT_TRUE(IncrementNumber(3) == 4); + + return true; +} + +bool AutoTestingHookInContainerTest() { + SIDESTEP_EXPECT_TRUE(IncrementNumber(1) == 2); + + // Inner scope, so we can test what happens when the AutoTestingHook + // goes out of scope + { + AutoTestingHookHolder hook(MakeTestingHookHolder(IncrementNumber, + AutoHookIncrementNumber)); + (void) hook; + SIDESTEP_EXPECT_TRUE(IncrementNumber(2) == 12); + } + SIDESTEP_EXPECT_TRUE(IncrementNumber(3) == 4); + + return true; +} + +bool TestPreambleAllocation() { + __int64 diff = 0; + void* p1 = reinterpret_cast<void*>(0x110000000); + void* p2 = reinterpret_cast<void*>(0x810000000); + unsigned char* b1 = PreamblePatcher::AllocPreambleBlockNear(p1); + SIDESTEP_EXPECT_TRUE(b1 != NULL); + diff = reinterpret_cast<__int64>(p1) - reinterpret_cast<__int64>(b1); + // Ensure blocks are within 2GB + SIDESTEP_EXPECT_TRUE(diff <= INT_MAX && diff >= INT_MIN); + unsigned char* b2 = PreamblePatcher::AllocPreambleBlockNear(p2); + SIDESTEP_EXPECT_TRUE(b2 != NULL); + diff = reinterpret_cast<__int64>(p2) - reinterpret_cast<__int64>(b2); + SIDESTEP_EXPECT_TRUE(diff <= INT_MAX && diff >= INT_MIN); + + // Ensure we're reusing free blocks + unsigned char* b3 = b1; + unsigned char* b4 = b2; + PreamblePatcher::FreePreambleBlock(b1); + PreamblePatcher::FreePreambleBlock(b2); + b1 = PreamblePatcher::AllocPreambleBlockNear(p1); + SIDESTEP_EXPECT_TRUE(b1 == b3); + b2 = PreamblePatcher::AllocPreambleBlockNear(p2); + SIDESTEP_EXPECT_TRUE(b2 == b4); + PreamblePatcher::FreePreambleBlock(b1); + PreamblePatcher::FreePreambleBlock(b2); + + return true; +} + +bool UnitTests() { + return TestPatchWithPreambleNearRelativeCall() && + TestPatchWithPreambleAbsoluteJump() && + TestPatchWithPreambleNearRelativeCondJump() && + TestPatchWithPreambleShortCondJump() && + TestDisassembler() && TestPatchWithLongJump() && + TestPatchUsingDynamicStub() && PatchThenUnpatch() && + AutoTestingHookTest() && AutoTestingHookInContainerTest() && + TestPreambleAllocation(); +} + +}; // namespace sidestep + +int safe_vsnprintf(char *str, size_t size, const char *format, va_list ap) { + if (size == 0) // not even room for a \0? + return -1; // not what C99 says to do, but what windows does + str[size-1] = '\0'; + return _vsnprintf(str, size-1, format, ap); +} + +int _tmain(int argc, _TCHAR* argv[]) +{ + bool ret = sidestep::UnitTests(); + printf("%s\n", ret ? "PASS" : "FAIL"); + return ret ? 0 : -1; +} + +#pragma optimize("", on) diff --git a/third_party/tcmalloc/chromium/src/windows/preamble_patcher_with_stub.cc b/third_party/tcmalloc/chromium/src/windows/preamble_patcher_with_stub.cc index 4eb391d..b0dc393 100644 --- a/third_party/tcmalloc/chromium/src/windows/preamble_patcher_with_stub.cc +++ b/third_party/tcmalloc/chromium/src/windows/preamble_patcher_with_stub.cc @@ -29,6 +29,7 @@ * * --- * Author: Joi Sigurdsson + * Author: Scott Francis * * Implementation of PreamblePatcher */ @@ -40,12 +41,20 @@ // Definitions of assembly statements we need #define ASM_JMP32REL 0xE9 #define ASM_INT3 0xCC +#define ASM_NOP 0x90 +// X64 opcodes +#define ASM_MOVRAX_IMM 0xB8 +#define ASM_REXW 0x48 +#define ASM_JMP 0xFF +#define ASM_JMP_RAX 0xE0 +#define ASM_PUSH 0x68 +#define ASM_RET 0xC3 namespace sidestep { SideStepError PreamblePatcher::RawPatchWithStub( void* target_function, - void *replacement_function, + void* replacement_function, unsigned char* preamble_stub, unsigned long stub_size, unsigned long* bytes_needed) { @@ -75,23 +84,52 @@ SideStepError PreamblePatcher::RawPatchWithStub( // doing it atomically does not help if one of the other threads happens // to have its eip in the middle of the bytes you change while you change // them. + unsigned char* target = reinterpret_cast<unsigned char*>(target_function); + unsigned int required_trampoline_bytes = 0; + const unsigned int kRequiredStubJumpBytes = 5; + const unsigned int kRequiredTargetPatchBytes = 5; - // First, deal with a special case that we see with functions that - // point into an IAT table (including functions linked statically - // into the application): these function already starts with - // ASM_JMP32REL. For instance, malloc() might be implemented as a - // JMP to __malloc(). In that case, we replace the destination of - // the JMP (__malloc), rather than the JMP itself (malloc). This - // way we get the correct behavior no matter how malloc gets called. - void *new_target = ResolveTarget(target_function); - if (new_target != target_function) { // we're in the IAT case - // I'd like to just say "target = new_target", but I can't, - // because the new target will need to have its protections set. - return RawPatchWithStubAndProtections(new_target, replacement_function, - preamble_stub, stub_size, - bytes_needed); + // Initialize the stub with INT3's just in case. + if (stub_size) { + memset(preamble_stub, 0xcc, stub_size); + } + if (kIs64BitBinary) { + // In 64-bit mode JMP instructions are always relative to RIP. If the + // replacement - target offset is > 2GB, we can't JMP to the replacement + // function. In this case, we're going to use a trampoline - that is, + // we're going to do a relative jump to a small chunk of code in the stub + // that will then do the absolute jump to the replacement function. By + // doing this, we only need to patch 5 bytes in the target function, as + // opposed to patching 12 bytes if we were to do an absolute jump. + // + // Note that the first byte of the trampoline is a NOP instruction. This + // is used as a trampoline signature that will be detected when unpatching + // the function. + // + // jmp <trampoline> + // + // trampoline: + // nop + // mov rax, <replacement_function> + // jmp rax + // + __int64 replacement_target_offset = reinterpret_cast<__int64>( + replacement_function) - reinterpret_cast<__int64>(target) - 5; + if (replacement_target_offset > INT_MAX + || replacement_target_offset < INT_MIN) { + // The stub needs to be within 2GB of the target for the trampoline to + // work! + __int64 trampoline_offset = reinterpret_cast<__int64>(preamble_stub) + - reinterpret_cast<__int64>(target) - 5; + if (trampoline_offset > INT_MAX || trampoline_offset < INT_MIN) { + // We're screwed. + SIDESTEP_ASSERT(false + && "Preamble stub is too far from target to patch."); + return SIDESTEP_UNEXPECTED; + } + required_trampoline_bytes = 13; + } } - unsigned char* target = reinterpret_cast<unsigned char*>(new_target); // Let's disassemble the preamble of the target function to see if we can // patch, and to see how much of the preamble we need to take. We need 5 @@ -99,42 +137,76 @@ SideStepError PreamblePatcher::RawPatchWithStub( // instructions to get 5 bytes. MiniDisassembler disassembler; unsigned int preamble_bytes = 0; - while (preamble_bytes < 5) { + unsigned int stub_bytes = 0; + while (preamble_bytes < kRequiredTargetPatchBytes) { + unsigned int cur_bytes = 0; InstructionType instruction_type = - disassembler.Disassemble(target + preamble_bytes, preamble_bytes); + disassembler.Disassemble(target + preamble_bytes, cur_bytes); if (IT_JUMP == instruction_type) { - SIDESTEP_ASSERT(false && - "Unable to patch because there is a jump instruction " - "in the first 5 bytes."); - return SIDESTEP_JUMP_INSTRUCTION; + unsigned int jump_bytes = 0; + SideStepError jump_ret = SIDESTEP_JUMP_INSTRUCTION; + if (IsShortConditionalJump(target + preamble_bytes, cur_bytes)) { + jump_ret = PatchShortConditionalJump(target + preamble_bytes, cur_bytes, + preamble_stub + stub_bytes, + &jump_bytes, + stub_size - stub_bytes); + } else if (IsNearConditionalJump(target + preamble_bytes, cur_bytes) || + IsNearRelativeJump(target + preamble_bytes, cur_bytes) || + IsNearAbsoluteCall(target + preamble_bytes, cur_bytes) || + IsNearRelativeCall(target + preamble_bytes, cur_bytes)) { + jump_ret = PatchNearJumpOrCall(target + preamble_bytes, cur_bytes, + preamble_stub + stub_bytes, &jump_bytes, + stub_size - stub_bytes); + } + if (jump_ret != SIDESTEP_SUCCESS) { + SIDESTEP_ASSERT(false && + "Unable to patch because there is an unhandled branch " + "instruction in the initial preamble bytes."); + return SIDESTEP_JUMP_INSTRUCTION; + } + stub_bytes += jump_bytes; } else if (IT_RETURN == instruction_type) { SIDESTEP_ASSERT(false && "Unable to patch because function is too short"); return SIDESTEP_FUNCTION_TOO_SMALL; - } else if (IT_GENERIC != instruction_type) { + } else if (IT_GENERIC == instruction_type) { + if (IsMovWithDisplacement(target + preamble_bytes, cur_bytes)) { + unsigned int mov_bytes = 0; + if (PatchMovWithDisplacement(target + preamble_bytes, cur_bytes, + preamble_stub + stub_bytes, &mov_bytes, + stub_size - stub_bytes) + != SIDESTEP_SUCCESS) { + return SIDESTEP_UNSUPPORTED_INSTRUCTION; + } + stub_bytes += mov_bytes; + } else { + memcpy(reinterpret_cast<void*>(preamble_stub + stub_bytes), + reinterpret_cast<void*>(target + preamble_bytes), cur_bytes); + stub_bytes += cur_bytes; + } + } else { SIDESTEP_ASSERT(false && "Disassembler encountered unsupported instruction " "(either unused or unknown"); return SIDESTEP_UNSUPPORTED_INSTRUCTION; } + preamble_bytes += cur_bytes; } if (NULL != bytes_needed) - *bytes_needed = preamble_bytes + 5; + *bytes_needed = stub_bytes + kRequiredStubJumpBytes + + required_trampoline_bytes; // Inv: cbPreamble is the number of bytes (at least 5) that we need to take // from the preamble to have whole instructions that are 5 bytes or more - // in size total. The size of the stub required is cbPreamble + size of - // jmp (5) - if (preamble_bytes + 5 > stub_size) { + // in size total. The size of the stub required is cbPreamble + + // kRequiredStubJumpBytes (5) + required_trampoline_bytes (0 or 13) + if (stub_bytes + kRequiredStubJumpBytes + required_trampoline_bytes + > stub_size) { SIDESTEP_ASSERT(false); return SIDESTEP_INSUFFICIENT_BUFFER; } - // First, copy the preamble that we will overwrite. - memcpy(reinterpret_cast<void*>(preamble_stub), - reinterpret_cast<void*>(target), preamble_bytes); - // Now, make a jmp instruction to the rest of the target function (minus the // preamble bytes we moved into the stub) and copy it into our preamble-stub. // find address to jump to, relative to next address after jmp instruction @@ -144,16 +216,32 @@ SideStepError PreamblePatcher::RawPatchWithStub( #endif int relative_offset_to_target_rest = ((reinterpret_cast<unsigned char*>(target) + preamble_bytes) - - (preamble_stub + preamble_bytes + 5)); + (preamble_stub + stub_bytes + kRequiredStubJumpBytes)); #ifdef _MSC_VER #pragma warning(pop) #endif // jmp (Jump near, relative, displacement relative to next instruction) - preamble_stub[preamble_bytes] = ASM_JMP32REL; + preamble_stub[stub_bytes] = ASM_JMP32REL; // copy the address - memcpy(reinterpret_cast<void*>(preamble_stub + preamble_bytes + 1), + memcpy(reinterpret_cast<void*>(preamble_stub + stub_bytes + 1), reinterpret_cast<void*>(&relative_offset_to_target_rest), 4); + if (kIs64BitBinary && required_trampoline_bytes != 0) { + // Construct the trampoline + unsigned int trampoline_pos = stub_bytes + kRequiredStubJumpBytes; + preamble_stub[trampoline_pos] = ASM_NOP; + preamble_stub[trampoline_pos + 1] = ASM_REXW; + preamble_stub[trampoline_pos + 2] = ASM_MOVRAX_IMM; + memcpy(reinterpret_cast<void*>(preamble_stub + trampoline_pos + 3), + reinterpret_cast<void*>(&replacement_function), + sizeof(void *)); + preamble_stub[trampoline_pos + 11] = ASM_JMP; + preamble_stub[trampoline_pos + 12] = ASM_JMP_RAX; + + // Now update replacement_function to point to the trampoline + replacement_function = preamble_stub + trampoline_pos; + } + // Inv: preamble_stub points to assembly code that will execute the // original function by first executing the first cbPreamble bytes of the // preamble, then jumping to the rest of the function. @@ -177,6 +265,7 @@ SideStepError PreamblePatcher::RawPatchWithStub( // complete the jmp instruction memcpy(reinterpret_cast<void*>(target + 1), reinterpret_cast<void*>(&offset_to_replacement_function), 4); + // Set any remaining bytes that were moved to the preamble-stub to INT3 so // as not to cause confusion (otherwise you might see some strange // instructions if you look at the disassembly, or even invalid @@ -184,8 +273,9 @@ SideStepError PreamblePatcher::RawPatchWithStub( // some code calls into this portion of the code. If this happens, it // means that this function cannot be patched using this patcher without // further thought. - if (preamble_bytes > 5) { - memset(reinterpret_cast<void*>(target + 5), ASM_INT3, preamble_bytes - 5); + if (preamble_bytes > kRequiredTargetPatchBytes) { + memset(reinterpret_cast<void*>(target + kRequiredTargetPatchBytes), + ASM_INT3, preamble_bytes - kRequiredTargetPatchBytes); } // Inv: The memory pointed to by target_function now points to a relative @@ -193,7 +283,13 @@ SideStepError PreamblePatcher::RawPatchWithStub( // stub contains the first stub_size bytes of the original target // function's preamble code, followed by a relative jump back to the next // instruction after the first cbPreamble bytes. - + // + // In 64-bit mode the memory pointed to by target_function *may* point to a + // relative jump instruction that jumps to a trampoline which will then + // perform an absolute jump to the replacement function. The preamble stub + // still contains the original target function's preamble code, followed by a + // jump back to the instructions after the first preamble bytes. + // return SIDESTEP_SUCCESS; } diff --git a/third_party/tcmalloc/chromium/src/windows/shortproc.asm b/third_party/tcmalloc/chromium/src/windows/shortproc.asm new file mode 100644 index 0000000..7e8e3d7 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/windows/shortproc.asm @@ -0,0 +1,169 @@ +; Copyright (c) 2011, Google Inc. +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are +; met: +; +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above +; copyright notice, this list of conditions and the following disclaimer +; in the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Google Inc. nor the names of its +; contributors may be used to endorse or promote products derived from +; this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +; +; --- +; Author: Scott Francis +; +; Unit tests for PreamblePatcher
+
+.MODEL small
+
+.CODE
+
+TooShortFunction PROC
+ ret
+TooShortFunction ENDP
+
+JumpShortCondFunction PROC
+ test cl, 1
+ jnz jumpspot
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+ int 3
+jumpspot:
+ nop
+ nop
+ nop
+ nop
+ mov rax, 1
+ ret
+JumpShortCondFunction ENDP
+
+JumpNearCondFunction PROC
+ test cl, 1
+ jnz jumpspot
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+jumpspot:
+ nop
+ nop
+ mov rax, 1
+ ret
+JumpNearCondFunction ENDP
+
+JumpAbsoluteFunction PROC
+ test cl, 1
+ jmp jumpspot
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+jumpspot:
+ nop
+ nop
+ mov rax, 1
+ ret
+JumpAbsoluteFunction ENDP
+
+CallNearRelativeFunction PROC
+ test cl, 1
+ call TooShortFunction
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ mov rdx, 0ffff1111H
+ nop
+ nop
+ nop
+ ret
+CallNearRelativeFunction ENDP
+
+END
|