diff options
| -rw-r--r-- | third_party/tcmalloc/chromium/src/config.h | 20 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/config_linux.h | 228 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/config_win.h | 270 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/google/heap-checker.h | 8 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/page_heap.cc | 277 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/page_heap.h | 74 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/system-alloc.cc | 11 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/system-alloc.h | 6 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/tcmalloc.cc | 328 | ||||
| -rw-r--r-- | third_party/tcmalloc/chromium/src/windows/port.cc | 87 |
10 files changed, 875 insertions, 434 deletions
diff --git a/third_party/tcmalloc/chromium/src/config.h b/third_party/tcmalloc/chromium/src/config.h new file mode 100644 index 0000000..6155a86 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/config.h @@ -0,0 +1,20 @@ +// Copyright (c) 2009 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONFIG_H_ + +#include "build/build_config.h" + +#define TC_VERSION_MAJOR 1 +#define TC_VERSION_MINOR 4 +#define TC_VERSION_PATCH "" +#define TC_VERSION_STRING "google-perftools 1.4" + +#if defined(OS_WIN) +#include "third_party/tcmalloc/config_win.h" +#elif defined(OS_LINUX) +#include "third_party/tcmalloc/config_linux.h" +#endif + +#endif // CONFIG_H_ diff --git a/third_party/tcmalloc/chromium/src/config_linux.h b/third_party/tcmalloc/chromium/src/config_linux.h new file mode 100644 index 0000000..398f303 --- /dev/null +++ b/third_party/tcmalloc/chromium/src/config_linux.h @@ -0,0 +1,228 @@ +/* src/config.h. Generated from config.h.in by configure. */ +/* src/config.h.in. Generated from configure.ac by autoheader. */ + +/* Define to 1 if compiler supports __builtin_stack_pointer */ +/* #undef HAVE_BUILTIN_STACK_POINTER */ + +/* Define to 1 if you have the <conflict-signal.h> header file. */ +/* #undef HAVE_CONFLICT_SIGNAL_H */ + +/* Define to 1 if you have the declaration of `cfree', and to 0 if you don't. + */ +#define HAVE_DECL_CFREE 1 + +/* Define to 1 if you have the declaration of `memalign', and to 0 if you + don't. */ +#define HAVE_DECL_MEMALIGN 1 + +/* Define to 1 if you have the declaration of `posix_memalign', and to 0 if + you don't. */ +#define HAVE_DECL_POSIX_MEMALIGN 1 + +/* Define to 1 if you have the declaration of `pvalloc', and to 0 if you + don't. */ +#define HAVE_DECL_PVALLOC 1 + +/* Define to 1 if you have the declaration of `uname', and to 0 if you don't. + */ +#define HAVE_DECL_UNAME 1 + +/* Define to 1 if you have the declaration of `valloc', and to 0 if you don't. + */ +#define HAVE_DECL_VALLOC 1 + +/* Define to 1 if you have the <dlfcn.h> header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if the system has the type `Elf32_Versym'. */ +#define HAVE_ELF32_VERSYM 1 + +/* Define to 1 if you have the <execinfo.h> header file. */ +#define HAVE_EXECINFO_H 1 + +/* Define to 1 if you have the <fcntl.h> header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have the <features.h> header file. */ +#define HAVE_FEATURES_H 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getpagesize' function. */ +#define HAVE_GETPAGESIZE 1 + +/* Define to 1 if you have the <glob.h> header file. */ +#define HAVE_GLOB_H 1 + +/* Define to 1 if you have the <grp.h> header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the <inttypes.h> header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the <libunwind.h> header file. */ +/* #undef HAVE_LIBUNWIND_H */ + +/* Define to 1 if you have the <linux/ptrace.h> header file. */ +#define HAVE_LINUX_PTRACE_H 1 + +/* Define to 1 if you have the <malloc.h> header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the <memory.h> header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have a working `mmap' system call. */ +#define HAVE_MMAP 1 + +/* define if the compiler implements namespaces */ +#define HAVE_NAMESPACES 1 + +/* Define to 1 if you have the <poll.h> header file. */ +#define HAVE_POLL_H 1 + +/* define if libc has program_invocation_name */ +#define HAVE_PROGRAM_INVOCATION_NAME 1 + +/* Define if you have POSIX threads libraries and header files. */ +#define HAVE_PTHREAD 1 + +/* Define to 1 if you have the <pwd.h> header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `sbrk' function. */ +#define HAVE_SBRK 1 + +/* Define to 1 if you have the <sched.h> header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the <stdint.h> header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the <stdlib.h> header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the <strings.h> header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if the system has the type `struct mallinfo'. */ +#define HAVE_STRUCT_MALLINFO 1 + +/* Define to 1 if you have the <sys/prctl.h> header file. */ +#define HAVE_SYS_PRCTL_H 1 + +/* Define to 1 if you have the <sys/resource.h> header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the <sys/socket.h> header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the <sys/stat.h> header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the <sys/syscall.h> header file. */ +#define HAVE_SYS_SYSCALL_H 1 + +/* Define to 1 if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the <sys/wait.h> header file. */ +#define HAVE_SYS_WAIT_H 1 + +/* Define to 1 if compiler supports __thread */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the <ucontext.h> header file. */ +#define HAVE_UCONTEXT_H 1 + +/* Define to 1 if you have the <unistd.h> header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the <unwind.h> header file. */ +#define HAVE_UNWIND_H 1 + +/* define if your compiler has __attribute__ */ +#define HAVE___ATTRIBUTE__ 1 + +/* Define to 1 if the system has the type `__int64'. */ +/* #undef HAVE___INT64 */ + +/* prefix where we look for installed files */ +#define INSTALL_PREFIX "/usr/local" + +/* Define to 1 if int32_t is equivalent to intptr_t */ +/* #undef INT32_EQUALS_INTPTR */ + +/* Define to 1 if your C compiler doesn't accept -c and -o together. */ +/* #undef NO_MINUS_C_MINUS_O */ + +/* Name of package */ +#define PACKAGE "google-perftools" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "opensource@google.com" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "google-perftools" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "google-perftools 1.4" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "google-perftools" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "1.4" + +/* How to access the PC from a struct ucontext */ +#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP] + +/* Always the empty-string on non-windows systems. On windows, should be + "__declspec(dllexport)". This way, when we compile the dll, we export our + functions/classes. It's safe to define this here because config.h is only + used internally, to compile the DLL, and every DLL source file #includes + "config.h" before anything else. */ +#define PERFTOOLS_DLL_DECL + +/* printf format code for printing a size_t and ssize_t */ +#define PRIdS "zd" + +/* printf format code for printing a size_t and ssize_t */ +#define PRIuS "zu" + +/* printf format code for printing a size_t and ssize_t */ +#define PRIxS "zx" + +/* Define to necessary symbol if this constant uses a non-standard name on + your system. */ +/* #undef PTHREAD_CREATE_JOINABLE */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* the namespace where STL code like vector<> is defined */ +#define STL_NAMESPACE std + +/* Version number of package */ +#define VERSION "1.4" + +/* C99 says: define this to get the PRI... macros from stdint.h */ +#ifndef __STDC_FORMAT_MACROS +# define __STDC_FORMAT_MACROS 1 +#endif + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +/* #undef inline */ +#endif + + +#ifdef __MINGW32__ +#include "windows/mingw.h" +#endif + diff --git a/third_party/tcmalloc/chromium/src/config_win.h b/third_party/tcmalloc/chromium/src/config_win.h new file mode 100644 index 0000000..30daf4f --- /dev/null +++ b/third_party/tcmalloc/chromium/src/config_win.h @@ -0,0 +1,270 @@ +/* A manual version of config.h fit for windows machines. */ + +/* Sometimes we accidentally #include this config.h instead of the one + in .. -- this is particularly true for msys/mingw, which uses the + unix config.h but also runs code in the windows directory. + */ +#ifdef __MINGW32__ +#include "../config.h" +#define GOOGLE_PERFTOOLS_WINDOWS_CONFIG_H_ +#endif + +#ifndef GOOGLE_PERFTOOLS_WINDOWS_CONFIG_H_ +#define GOOGLE_PERFTOOLS_WINDOWS_CONFIG_H_ + +/* define this if you are linking tcmalloc statically and overriding the + * default allocators. + * For instructions on how to use this mode, see + * http://groups.google.com/group/google-perftools/browse_thread/thread/41cd3710af85e57b + */ +#define WIN32_OVERRIDE_ALLOCATORS + +/* the location of <hash_map> */ +#define HASH_MAP_H <hash_map> + +/* the namespace of hash_map/hash_set */ +#define HASH_NAMESPACE stdext + +/* the location of <hash_set> */ +#define HASH_SET_H <hash_set> + +/* Define to 1 if your libc has a snprintf implementation */ +#define HAVE_SNPRINTF + +/* Define to 1 if compiler supports __builtin_stack_pointer */ +#undef HAVE_BUILTIN_STACK_POINTER + +/* Define to 1 if you have the <conflict-signal.h> header file. */ +#undef HAVE_CONFLICT_SIGNAL_H + +/* Define to 1 if you have the declaration of `cfree', and to 0 if you don't. + */ +#undef HAVE_DECL_CFREE + +/* Define to 1 if you have the declaration of `memalign', and to 0 if you + don't. */ +#undef HAVE_DECL_MEMALIGN + +/* Define to 1 if you have the declaration of `posix_memalign', and to 0 if + you don't. */ +#undef HAVE_DECL_POSIX_MEMALIGN + +/* Define to 1 if you have the declaration of `pvalloc', and to 0 if you + don't. */ +#undef HAVE_DECL_PVALLOC + +/* Define to 1 if you have the declaration of `uname', and to 0 if you don't. + */ +#undef HAVE_DECL_UNAME + +/* Define to 1 if you have the declaration of `valloc', and to 0 if you don't. + */ +#undef HAVE_DECL_VALLOC + +/* Define to 1 if you have the <dlfcn.h> header file. */ +#undef HAVE_DLFCN_H + +/* Define to 1 if the system has the type `Elf32_Versym'. */ +#undef HAVE_ELF32_VERSYM + +/* Define to 1 if you have the <execinfo.h> header file. */ +#undef HAVE_EXECINFO_H + +/* Define to 1 if you have the <fcntl.h> header file. */ +#undef HAVE_FCNTL_H + +/* Define to 1 if you have the `geteuid' function. */ +#undef HAVE_GETEUID + +/* Define to 1 if you have the `getpagesize' function. */ +#define HAVE_GETPAGESIZE 1 /* we define it in windows/port.cc */ + +/* Define to 1 if you have the <glob.h> header file. */ +#undef HAVE_GLOB_H + +/* Define to 1 if you have the <grp.h> header file. */ +#undef HAVE_GRP_H + +/* define if the compiler has hash_map */ +#define HAVE_HASH_MAP 1 + +/* define if the compiler has hash_set */ +#define HAVE_HASH_SET 1 + +/* Define to 1 if you have the <inttypes.h> header file. */ +#undef HAVE_INTTYPES_H + +/* Define to 1 if you have the <libunwind.h> header file. */ +#undef HAVE_LIBUNWIND_H + +/* Define to 1 if you have the <linux/ptrace.h> header file. */ +#undef HAVE_LINUX_PTRACE_H + +/* Define to 1 if you have the <malloc.h> header file. */ +#undef HAVE_MALLOC_H + +/* Define to 1 if you have the <memory.h> header file. */ +#undef HAVE_MEMORY_H + +/* Define to 1 if you have a working `mmap' system call. */ +#undef HAVE_MMAP + +/* define if the compiler implements namespaces */ +#define HAVE_NAMESPACES 1 + +/* define if libc has program_invocation_name */ +#undef HAVE_PROGRAM_INVOCATION_NAME + +/* Define if you have POSIX threads libraries and header files. */ +#undef HAVE_PTHREAD + +/* Define to 1 if you have the <pwd.h> header file. */ +#undef HAVE_PWD_H + +/* Define to 1 if you have the `sbrk' function. */ +#undef HAVE_SBRK + +/* Define to 1 if you have the <stdint.h> header file. */ +#undef HAVE_STDINT_H + +/* Define to 1 if you have the <stdlib.h> header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the <strings.h> header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if the system has the type `struct mallinfo'. */ +#undef HAVE_STRUCT_MALLINFO + +/* Define to 1 if you have the <sys/prctl.h> header file. */ +#undef HAVE_SYS_PRCTL_H + +/* Define to 1 if you have the <sys/resource.h> header file. */ +#undef HAVE_SYS_RESOURCE_H + +/* Define to 1 if you have the <sys/socket.h> header file. */ +#undef HAVE_SYS_SOCKET_H + +/* Define to 1 if you have the <sys/stat.h> header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the <sys/syscall.h> header file. */ +#undef HAVE_SYS_SYSCALL_H + +/* Define to 1 if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the <sys/wait.h> header file. */ +#undef HAVE_SYS_WAIT_H + +/* Define to 1 if compiler supports __thread */ +#undef HAVE_TLS + +/* Define to 1 if you have the <ucontext.h> header file. */ +#undef HAVE_UCONTEXT_H + +/* Define to 1 if you have the <unistd.h> header file. */ +#undef HAVE_UNISTD_H + +/* Define to 1 if you have the <unwind.h> header file. */ +#undef HAVE_UNWIND_H + +/* define if your compiler has __attribute__ */ +#undef HAVE___ATTRIBUTE__ + +/* Define to 1 if the system has the type `__int64'. */ +#define HAVE___INT64 1 + +/* prefix where we look for installed files */ +#undef INSTALL_PREFIX + +/* Define to 1 if int32_t is equivalent to intptr_t */ +#undef INT32_EQUALS_INTPTR + +/* Define to 1 if your C compiler doesn't accept -c and -o together. */ +#undef NO_MINUS_C_MINUS_O + +/* Name of package */ +#undef PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#undef PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME + +/* Define to the version of this package. */ +#undef PACKAGE_VERSION + +/* How to access the PC from a struct ucontext */ +#undef PC_FROM_UCONTEXT + +/* Always the empty-string on non-windows systems. On windows, should be + "__declspec(dllexport)". This way, when we compile the dll, we export our + functions/classes. It's safe to define this here because config.h is only + used internally, to compile the DLL, and every DLL source file #includes + "config.h" before anything else. */ +#ifndef PERFTOOLS_DLL_DECL +# define PERFTOOLS_IS_A_DLL 1 /* not set if you're statically linking */ +# define PERFTOOLS_DLL_DECL __declspec(dllexport) +# define PERFTOOLS_DLL_DECL_FOR_UNITTESTS __declspec(dllimport) +#endif + +/* printf format code for printing a size_t and ssize_t */ +#define PRIdS "Id" + +/* printf format code for printing a size_t and ssize_t */ +#define PRIuS "Iu" + +/* printf format code for printing a size_t and ssize_t */ +#define PRIxS "Ix" + +/* Define to necessary symbol if this constant uses a non-standard name on + your system. */ +#undef PTHREAD_CREATE_JOINABLE + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* the namespace where STL code like vector<> is defined */ +#define STL_NAMESPACE std + +/* Version number of package */ +#undef VERSION + +/* C99 says: define this to get the PRI... macros from stdint.h */ +#ifndef __STDC_FORMAT_MACROS +# define __STDC_FORMAT_MACROS 1 +#endif + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +#undef inline +#endif + +// --------------------------------------------------------------------- +// Extra stuff not found in config.h.in + +// This must be defined before the windows.h is included. It's needed +// for mutex.h, to give access to the TryLock method. +#ifndef _WIN32_WINNT +# define _WIN32_WINNT 0x0400 +#endif + +// We want to make sure not to ever try to #include heap-checker.h +#define NO_HEAP_CHECK 1 + +// TODO(csilvers): include windows/port.h in every relevant source file instead? +#include "windows/port.h" + +#endif /* GOOGLE_PERFTOOLS_WINDOWS_CONFIG_H_ */ diff --git a/third_party/tcmalloc/chromium/src/google/heap-checker.h b/third_party/tcmalloc/chromium/src/google/heap-checker.h index c0ee8a8..751eb9f 100644 --- a/third_party/tcmalloc/chromium/src/google/heap-checker.h +++ b/third_party/tcmalloc/chromium/src/google/heap-checker.h @@ -51,12 +51,10 @@ #ifndef BASE_HEAP_CHECKER_H_ #define BASE_HEAP_CHECKER_H_ +#include "config.h" + #include <sys/types.h> // for size_t -// I can't #include config.h in this public API file, but I should -// really use configure (and make malloc_extension.h a .in file) to -// figure out if the system has stdint.h or not. But I'm lazy, so -// for now I'm assuming it's a problem only with MSVC. -#ifndef _MSC_VER +#ifdef HAVE_STDINT_H #include <stdint.h> // for uintptr_t #endif #include <stdarg.h> // for va_list diff --git a/third_party/tcmalloc/chromium/src/page_heap.cc b/third_party/tcmalloc/chromium/src/page_heap.cc index 1e63cb9..f92cfc4 100644 --- a/third_party/tcmalloc/chromium/src/page_heap.cc +++ b/third_party/tcmalloc/chromium/src/page_heap.cc @@ -49,9 +49,12 @@ namespace tcmalloc { PageHeap::PageHeap() : pagemap_(MetaDataAlloc), pagemap_cache_(0), + free_pages_(0), + system_bytes_(0), + committed_bytes_(0), scavenge_counter_(0), // Start scavenging at kMaxPages list - release_index_(kMaxPages) { + scavenge_index_(kMaxPages-1) { COMPILE_ASSERT(kNumClasses <= (1 << PageMapCache::kValuebits), valuebits); DLL_Init(&large_.normal); DLL_Init(&large_.returned); @@ -148,11 +151,23 @@ Span* PageHeap::Split(Span* span, Length n) { return leftover; } +void PageHeap::CommitSpan(Span* span) { + TCMalloc_SystemCommit(reinterpret_cast<void*>(span->start << kPageShift), + static_cast<size_t>(span->length << kPageShift)); + committed_bytes_ += span->length << kPageShift; +} + +void PageHeap::DecommitSpan(Span* span) { + TCMalloc_SystemRelease(reinterpret_cast<void*>(span->start << kPageShift), + static_cast<size_t>(span->length << kPageShift)); + committed_bytes_ -= span->length << kPageShift; +} + Span* PageHeap::Carve(Span* span, Length n) { ASSERT(n > 0); ASSERT(span->location != Span::IN_USE); const int old_location = span->location; - RemoveFromFreeList(span); + DLL_Remove(span); span->location = Span::IN_USE; Event(span, 'A', n); @@ -163,11 +178,24 @@ Span* PageHeap::Carve(Span* span, Length n) { leftover->location = old_location; Event(leftover, 'S', extra); RecordSpan(leftover); - PrependToFreeList(leftover); // Skip coalescing - no candidates possible + + // Place leftover span on appropriate free list + SpanList* listpair = (extra < kMaxPages) ? &free_[extra] : &large_; + Span* dst = (leftover->location == Span::ON_RETURNED_FREELIST + ? &listpair->returned : &listpair->normal); + DLL_Prepend(dst, leftover); + span->length = n; pagemap_.set(span->start + n - 1, span); } ASSERT(Check()); + free_pages_ -= n; + if (old_location == Span::ON_RETURNED_FREELIST) { + // We need to recommit this address space. + CommitSpan(span); + } + ASSERT(span->location == Span::IN_USE); + ASSERT(span->length == n); return span; } @@ -177,34 +205,46 @@ void PageHeap::Delete(Span* span) { ASSERT(span->length > 0); ASSERT(GetDescriptor(span->start) == span); ASSERT(GetDescriptor(span->start + span->length - 1) == span); - const Length n = span->length; span->sizeclass = 0; span->sample = 0; - span->location = Span::ON_NORMAL_FREELIST; - Event(span, 'D', span->length); - MergeIntoFreeList(span); // Coalesces if possible - IncrementalScavenge(n); - ASSERT(Check()); -} - -void PageHeap::MergeIntoFreeList(Span* span) { - ASSERT(span->location != Span::IN_USE); // Coalesce -- we guarantee that "p" != 0, so no bounds checking // necessary. We do not bother resetting the stale pagemap // entries for the pieces we are merging together because we only // care about the pagemap entries for the boundaries. // - // Note that only similar spans are merged together. For example, - // we do not coalesce "returned" spans with "normal" spans. + // Note that the adjacent spans we merge into "span" may come out of a + // "normal" (committed) list, and cleanly merge with our IN_USE span, which + // is implicitly committed. If the adjacents spans are on the "returned" + // (decommitted) list, then we must get both spans into the same state before + // or after we coalesce them. The current code always decomits. This is + // achieved by blindly decommitting the entire coalesced region, which may + // include any combination of committed and decommitted spans, at the end of + // the method. + + // TODO(jar): "Always decommit" causes some extra calls to commit when we are + // called in GrowHeap() during an allocation :-/. We need to eval the cost of + // that oscillation, and possibly do something to reduce it. + + // TODO(jar): We need a better strategy for deciding to commit, or decommit, + // based on memory usage and free heap sizes. + const PageID p = span->start; const Length n = span->length; Span* prev = GetDescriptor(p-1); - if (prev != NULL && prev->location == span->location) { + if (prev != NULL && prev->location != Span::IN_USE) { // Merge preceding span into this span ASSERT(prev->start + prev->length == p); const Length len = prev->length; - RemoveFromFreeList(prev); + if (prev->location == Span::ON_RETURNED_FREELIST) { + // We're about to put the merge span into the returned freelist and call + // DecommitSpan() on it, which will mark the entire span including this + // one as released and decrease committed_bytes_ by the size of the + // merged span. To make the math work out we temporarily increase the + // committed_bytes_ amount. + committed_bytes_ += prev->length << kPageShift; + } + DLL_Remove(prev); DeleteSpan(prev); span->start -= len; span->length += len; @@ -212,40 +252,33 @@ void PageHeap::MergeIntoFreeList(Span* span) { Event(span, 'L', len); } Span* next = GetDescriptor(p+n); - if (next != NULL && next->location == span->location) { + if (next != NULL && next->location != Span::IN_USE) { // Merge next span into this span ASSERT(next->start == p+n); const Length len = next->length; - RemoveFromFreeList(next); + if (next->location == Span::ON_RETURNED_FREELIST) { + // See the comment below 'if (prev->location ...' for explanation. + committed_bytes_ += next->length << kPageShift; + } + DLL_Remove(next); DeleteSpan(next); span->length += len; pagemap_.set(span->start + span->length - 1, span); Event(span, 'R', len); } - PrependToFreeList(span); -} - -void PageHeap::PrependToFreeList(Span* span) { - ASSERT(span->location != Span::IN_USE); - SpanList* list = (span->length < kMaxPages) ? &free_[span->length] : &large_; - if (span->location == Span::ON_NORMAL_FREELIST) { - stats_.free_bytes += (span->length << kPageShift); - DLL_Prepend(&list->normal, span); + Event(span, 'D', span->length); + span->location = Span::ON_RETURNED_FREELIST; + DecommitSpan(span); + if (span->length < kMaxPages) { + DLL_Prepend(&free_[span->length].returned, span); } else { - stats_.unmapped_bytes += (span->length << kPageShift); - DLL_Prepend(&list->returned, span); + DLL_Prepend(&large_.returned, span); } -} + free_pages_ += n; -void PageHeap::RemoveFromFreeList(Span* span) { - ASSERT(span->location != Span::IN_USE); - if (span->location == Span::ON_NORMAL_FREELIST) { - stats_.free_bytes -= (span->length << kPageShift); - } else { - stats_.unmapped_bytes -= (span->length << kPageShift); - } - DLL_Remove(span); + IncrementalScavenge(n); + ASSERT(Check()); } void PageHeap::IncrementalScavenge(Length n) { @@ -253,6 +286,17 @@ void PageHeap::IncrementalScavenge(Length n) { scavenge_counter_ -= n; if (scavenge_counter_ >= 0) return; // Not yet time to scavenge + // Never delay scavenging for more than the following number of + // deallocated pages. With 4K pages, this comes to 4GB of + // deallocation. + // Chrome: Changed to 64MB + static const int kMaxReleaseDelay = 1 << 14; + + // If there is nothing to release, wait for so many pages before + // scavenging again. With 4K pages, this comes to 1GB of memory. + // Chrome: Changed to 16MB + static const int kDefaultReleaseDelay = 1 << 12; + const double rate = FLAGS_tcmalloc_release_rate; if (rate <= 1e-6) { // Tiny release rate means that releasing is disabled. @@ -260,62 +304,40 @@ void PageHeap::IncrementalScavenge(Length n) { return; } - Length released_pages = ReleaseAtLeastNPages(1); + // Find index of free list to scavenge + int index = scavenge_index_ + 1; + for (int i = 0; i < kMaxPages+1; i++) { + if (index > kMaxPages) index = 0; + SpanList* slist = (index == kMaxPages) ? &large_ : &free_[index]; + if (!DLL_IsEmpty(&slist->normal)) { + // Release the last span on the normal portion of this list + Span* s = slist->normal.prev; + ASSERT(s->location == Span::ON_NORMAL_FREELIST); + DLL_Remove(s); + DecommitSpan(s); + s->location = Span::ON_RETURNED_FREELIST; + DLL_Prepend(&slist->returned, s); + + // Compute how long to wait until we return memory. + // FLAGS_tcmalloc_release_rate==1 means wait for 1000 pages + // after releasing one page. + const double mult = 1000.0 / rate; + double wait = mult * static_cast<double>(s->length); + if (wait > kMaxReleaseDelay) { + // Avoid overflow and bound to reasonable range + wait = kMaxReleaseDelay; + } + scavenge_counter_ = static_cast<int64_t>(wait); - if (released_pages == 0) { - // Nothing to scavenge, delay for a while. - scavenge_counter_ = kDefaultReleaseDelay; - } else { - // Compute how long to wait until we return memory. - // FLAGS_tcmalloc_release_rate==1 means wait for 1000 pages - // after releasing one page. - const double mult = 1000.0 / rate; - double wait = mult * static_cast<double>(released_pages); - if (wait > kMaxReleaseDelay) { - // Avoid overflow and bound to reasonable range. - wait = kMaxReleaseDelay; + scavenge_index_ = index; // Scavenge at index+1 next time + // Note: we stop scavenging after finding one. + return; } - scavenge_counter_ = static_cast<int64_t>(wait); + index++; } -} -Length PageHeap::ReleaseLastNormalSpan(SpanList* slist) { - Span* s = slist->normal.prev; - ASSERT(s->location == Span::ON_NORMAL_FREELIST); - RemoveFromFreeList(s); - const Length n = s->length; - TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift), - static_cast<size_t>(s->length << kPageShift)); - s->location = Span::ON_RETURNED_FREELIST; - MergeIntoFreeList(s); // Coalesces if possible. - return n; -} - -Length PageHeap::ReleaseAtLeastNPages(Length num_pages) { - Length released_pages = 0; - Length prev_released_pages = -1; - - // Round robin through the lists of free spans, releasing the last - // span in each list. Stop after releasing at least num_pages. - while (released_pages < num_pages) { - if (released_pages == prev_released_pages) { - // Last iteration of while loop made no progress. - break; - } - prev_released_pages = released_pages; - - for (int i = 0; i < kMaxPages+1 && released_pages < num_pages; - i++, release_index_++) { - if (release_index_ > kMaxPages) release_index_ = 0; - SpanList* slist = (release_index_ == kMaxPages) ? - &large_ : &free_[release_index_]; - if (!DLL_IsEmpty(&slist->normal)) { - Length released_len = ReleaseLastNormalSpan(slist); - released_pages += released_len; - } - } - } - return released_pages; + // Nothing to scavenge, delay for a while + scavenge_counter_ = kDefaultReleaseDelay; } void PageHeap::RegisterSizeClass(Span* span, size_t sc) { @@ -330,10 +352,6 @@ void PageHeap::RegisterSizeClass(Span* span, size_t sc) { } } -static double MB(uint64_t bytes) { - return bytes / 1048576.0; -} - static double PagesToMB(uint64_t pages) { return (pages << kPageShift) / 1048576.0; } @@ -346,8 +364,8 @@ void PageHeap::Dump(TCMalloc_Printer* out) { } } out->printf("------------------------------------------------\n"); - out->printf("PageHeap: %d sizes; %6.1f MB free; %6.1f MB unmapped\n", - nonempty_sizes, MB(stats_.free_bytes), MB(stats_.unmapped_bytes)); + out->printf("PageHeap: %d sizes; %6.1f MB free\n", + nonempty_sizes, PagesToMB(free_pages_)); out->printf("------------------------------------------------\n"); uint64_t total_normal = 0; uint64_t total_returned = 0; @@ -399,37 +417,6 @@ void PageHeap::Dump(TCMalloc_Printer* out) { PagesToMB(total_returned)); } -bool PageHeap::GetNextRange(PageID start, base::MallocRange* r) { - Span* span = reinterpret_cast<Span*>(pagemap_.Next(start)); - if (span == NULL) { - return false; - } - r->address = span->start << kPageShift; - r->length = span->length << kPageShift; - r->fraction = 0; - switch (span->location) { - case Span::IN_USE: - r->type = base::MallocRange::INUSE; - r->fraction = 1; - if (span->sizeclass > 0) { - // Only some of the objects in this span may be in use. - const size_t osize = Static::sizemap()->class_to_size(span->sizeclass); - r->fraction = (1.0 * osize * span->refcount) / r->length; - } - break; - case Span::ON_NORMAL_FREELIST: - r->type = base::MallocRange::FREE; - break; - case Span::ON_RETURNED_FREELIST: - r->type = base::MallocRange::UNMAPPED; - break; - default: - r->type = base::MallocRange::UNKNOWN; - break; - } - return true; -} - static void RecordGrowth(size_t growth) { StackTrace* t = Static::stacktrace_allocator()->New(); t->depth = GetStackTrace(t->stack, kMaxStackDepth-1, 3); @@ -455,8 +442,9 @@ bool PageHeap::GrowHeap(Length n) { ask = actual_size >> kPageShift; RecordGrowth(ask << kPageShift); - uint64_t old_system_bytes = stats_.system_bytes; - stats_.system_bytes += (ask << kPageShift); + uint64_t old_system_bytes = system_bytes_; + system_bytes_ += (ask << kPageShift); + committed_bytes_ += (ask << kPageShift); const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift; ASSERT(p > 0); @@ -465,7 +453,7 @@ bool PageHeap::GrowHeap(Length n) { // when a program keeps allocating and freeing large blocks. if (old_system_bytes < kPageMapBigAllocationThreshold - && stats_.system_bytes >= kPageMapBigAllocationThreshold) { + && system_bytes_ >= kPageMapBigAllocationThreshold) { pagemap_.PreallocateMoreMemory(); } @@ -473,8 +461,10 @@ bool PageHeap::GrowHeap(Length n) { // Plus ensure one before and one after so coalescing code // does not need bounds-checking. if (pagemap_.Ensure(p-1, ask+2)) { - // Pretend the new area is allocated and then Delete() it to cause - // any necessary coalescing to occur. + // Pretend the new area is allocated and then Delete() it to + // cause any necessary coalescing to occur. + // + // We do not adjust free_pages_ here since Delete() will do it for us. Span* span = NewSpan(p, ask); RecordSpan(span); Delete(span); @@ -516,4 +506,25 @@ bool PageHeap::CheckList(Span* list, Length min_pages, Length max_pages, return true; } +void PageHeap::ReleaseFreeList(Span* list, Span* returned) { + // Walk backwards through list so that when we push these + // spans on the "returned" list, we preserve the order. + while (!DLL_IsEmpty(list)) { + Span* s = list->prev; + DLL_Remove(s); + DLL_Prepend(returned, s); + ASSERT(s->location == Span::ON_NORMAL_FREELIST); + s->location = Span::ON_RETURNED_FREELIST; + DecommitSpan(s); + } +} + +void PageHeap::ReleaseFreePages() { + for (Length s = 0; s < kMaxPages; s++) { + ReleaseFreeList(&free_[s].normal, &free_[s].returned); + } + ReleaseFreeList(&large_.normal, &large_.returned); + ASSERT(Check()); +} + } // namespace tcmalloc diff --git a/third_party/tcmalloc/chromium/src/page_heap.h b/third_party/tcmalloc/chromium/src/page_heap.h index 5ab0d04..100cae4 100644 --- a/third_party/tcmalloc/chromium/src/page_heap.h +++ b/third_party/tcmalloc/chromium/src/page_heap.h @@ -34,7 +34,6 @@ #define TCMALLOC_PAGE_HEAP_H_ #include <config.h> -#include <google/malloc_extension.h> #include "common.h" #include "packed-cache-inl.h" #include "pagemap.h" @@ -111,8 +110,7 @@ class PageHeap { // REQUIRES: span->sizeclass == 0 Span* Split(Span* span, Length n); - // Return the descriptor for the specified page. Returns NULL if - // this PageID was not allocated previously. + // Return the descriptor for the specified page. inline Span* GetDescriptor(PageID p) const { return reinterpret_cast<Span*>(pagemap_.get(p)); } @@ -120,18 +118,15 @@ class PageHeap { // Dump state to stderr void Dump(TCMalloc_Printer* out); - // If this page heap is managing a range with starting page # >= start, - // store info about the range in *r and return true. Else return false. - bool GetNextRange(PageID start, base::MallocRange* r); + // Return number of bytes allocated from system + inline uint64_t SystemBytes() const { return system_bytes_; } - // Page heap statistics - struct Stats { - Stats() : system_bytes(0), free_bytes(0), unmapped_bytes(0) {} - uint64_t system_bytes; // Total bytes allocated from system - uint64_t free_bytes; // Total bytes on normal freelists - uint64_t unmapped_bytes; // Total bytes on returned freelists - }; - inline Stats stats() const { return stats_; } + inline uint64_t CommittedBytes() const { return committed_bytes_; } + + // Return number of free bytes in heap + uint64_t FreeBytes() const { + return (static_cast<uint64_t>(free_pages_) << kPageShift); + } bool Check(); // Like Check() but does some more comprehensive checking. @@ -139,13 +134,8 @@ class PageHeap { bool CheckList(Span* list, Length min_pages, Length max_pages, int freelist); // ON_NORMAL_FREELIST or ON_RETURNED_FREELIST - // Try to release at least num_pages for reuse by the OS. Returns - // the actual number of pages released, which may be less than - // num_pages if there weren't enough pages to release. The result - // may also be larger than num_pages since page_heap might decide to - // release one large range instead of fragmenting it into two - // smaller released and unreleased ranges. - Length ReleaseAtLeastNPages(Length num_pages); + // Release all pages on the free list for reuse by the OS: + void ReleaseFreePages(); // Return 0 if we have no information, or else the correct sizeclass for p. // Reads and writes to pagemap_cache_ do not require locking. @@ -174,15 +164,6 @@ class PageHeap { // REQUIRED: kMaxPages >= kMinSystemAlloc; static const size_t kMaxPages = kMinSystemAlloc; - // Never delay scavenging for more than the following number of - // deallocated pages. With 4K pages, this comes to 4GB of - // deallocation. - static const int kMaxReleaseDelay = 1 << 20; - - // If there is nothing to release, wait for so many pages before - // scavenging again. With 4K pages, this comes to 1GB of memory. - static const int kDefaultReleaseDelay = 1 << 18; - // Pick the appropriate map and cache types based on pointer size typedef MapSelector<8*sizeof(uintptr_t)>::Type PageMap; typedef MapSelector<8*sizeof(uintptr_t)>::CacheType PageMapCache; @@ -203,8 +184,14 @@ class PageHeap { // Array mapping from span length to a doubly linked list of free spans SpanList free_[kMaxPages]; - // Statistics on system, free, and unmapped bytes - Stats stats_; + // Number of pages kept in free lists + uintptr_t free_pages_; + + // Bytes allocated from system + uint64_t system_bytes_; + + // Bytes committed, always <= system_bytes_. + uint64_t committed_bytes_; bool GrowHeap(Length n); @@ -228,30 +215,25 @@ class PageHeap { // span of exactly the specified length. Else, returns NULL. Span* AllocLarge(Length n); - // Coalesce span with neighboring spans if possible, prepend to - // appropriate free list, and adjust stats. - void MergeIntoFreeList(Span* span); + // Commit the span. + void CommitSpan(Span* span); - // Prepends span to appropriate free list, and adjusts stats. - void PrependToFreeList(Span* span); - - // Removes span from its free list, and adjust stats. - void RemoveFromFreeList(Span* span); + // Decommit the span. + void DecommitSpan(Span* span); // Incrementally release some memory to the system. // IncrementalScavenge(n) is called whenever n pages are freed. void IncrementalScavenge(Length n); - // Release the last span on the normal portion of this list. - // Return the length of that span. - Length ReleaseLastNormalSpan(SpanList* slist); - + // Releases all memory held in the given list's 'normal' freelist and adds + // it to the 'released' freelist. + void ReleaseFreeList(Span* list, Span* returned); // Number of pages to deallocate before doing more scavenging int64_t scavenge_counter_; - // Index of last free list where we released memory to the OS. - int release_index_; + // Index of last free list we scavenged + int scavenge_index_; }; } // namespace tcmalloc diff --git a/third_party/tcmalloc/chromium/src/system-alloc.cc b/third_party/tcmalloc/chromium/src/system-alloc.cc index 3341f17..21d9b43 100644 --- a/third_party/tcmalloc/chromium/src/system-alloc.cc +++ b/third_party/tcmalloc/chromium/src/system-alloc.cc @@ -150,10 +150,6 @@ bool RegisterSystemAllocator(SysAllocator *a, int priority) { void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, size_t alignment) { -#ifndef HAVE_SBRK - failed_ = true; - return NULL; -#else // Check if we should use sbrk allocation. // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized // state) and eventually gets initialized to the specified value. Note @@ -220,7 +216,6 @@ void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, ptr += alignment - (ptr & (alignment-1)); } return reinterpret_cast<void*>(ptr); -#endif // HAVE_SBRK } void SbrkSysAllocator::DumpStats(TCMalloc_Printer* printer) { @@ -493,6 +488,12 @@ void TCMalloc_SystemRelease(void* start, size_t length) { #endif } +void TCMalloc_SystemCommit(void* start, size_t length) { + // Nothing to do here. TCMalloc_SystemRelease does not alter pages + // such that they need to be re-committed before they can be used by the + // application. +} + void DumpSystemAllocatorStats(TCMalloc_Printer* printer) { for (int j = 0; j < kMaxAllocators; j++) { SysAllocator *a = allocators[j]; diff --git a/third_party/tcmalloc/chromium/src/system-alloc.h b/third_party/tcmalloc/chromium/src/system-alloc.h index 44b0333..60affed 100644 --- a/third_party/tcmalloc/chromium/src/system-alloc.h +++ b/third_party/tcmalloc/chromium/src/system-alloc.h @@ -65,6 +65,12 @@ extern void* TCMalloc_SystemAlloc(size_t bytes, size_t *actual_bytes, // be released, partial pages will not.) extern void TCMalloc_SystemRelease(void* start, size_t length); +// Called to ressurect memory which has been previously released +// to the system via TCMalloc_SystemRelease. An attempt to +// commit a page that is already committed does not cause this +// function to fail. +extern void TCMalloc_SystemCommit(void* start, size_t length); + // Interface to a pluggable system allocator. class SysAllocator { public: diff --git a/third_party/tcmalloc/chromium/src/tcmalloc.cc b/third_party/tcmalloc/chromium/src/tcmalloc.cc index 450c1ab..66e0ea6 100644 --- a/third_party/tcmalloc/chromium/src/tcmalloc.cc +++ b/third_party/tcmalloc/chromium/src/tcmalloc.cc @@ -136,7 +136,6 @@ # define WIN32_DO_PATCHING 1 #endif -using std::max; using tcmalloc::PageHeap; using tcmalloc::PageHeapAllocator; using tcmalloc::SizeMap; @@ -186,7 +185,6 @@ DEFINE_int64(tcmalloc_large_alloc_report_threshold, // put all callers of MallocHook::Invoke* in this module into // ATTRIBUTE_SECTION(google_malloc) section, so that // MallocHook::GetCallerStackTrace can function accurately. -#ifndef _WIN32 // windows doesn't have attribute_section, so don't bother extern "C" { void* tc_malloc(size_t size) __THROW ATTRIBUTE_SECTION(google_malloc); @@ -231,27 +229,23 @@ extern "C" { ATTRIBUTE_SECTION(google_malloc); void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) __THROW ATTRIBUTE_SECTION(google_malloc); - // Surprisingly, compilers use a nothrow-delete internally. See, eg: - // http://www.dinkumware.com/manuals/?manual=compleat&page=new.html - void tc_delete_nothrow(void* ptr, const std::nothrow_t&) __THROW - ATTRIBUTE_SECTION(google_malloc); - void tc_deletearray_nothrow(void* ptr, const std::nothrow_t&) __THROW - ATTRIBUTE_SECTION(google_malloc); -} // extern "C" -#endif // #ifndef _WIN32 +} // Override the libc functions to prefer our own instead. This comes // first so code in tcmalloc.cc can use the overridden versions. One // exception: in windows, by default, we patch our code into these // functions (via src/windows/patch_function.cc) rather than override // them. In that case, we don't want to do this overriding here. -#if !defined(WIN32_DO_PATCHING) && !defined(TCMALLOC_FOR_DEBUGALLOCATION) +#ifndef WIN32_DO_PATCHING + +// TODO(mbelshe): Turn off TCMalloc's symbols for libc. We do that +// elsewhere. +#if 0 #if defined(__GNUC__) && !defined(__MACH__) // Potentially faster variants that use the gcc alias extension. + // Mach-O (Darwin) does not support weak aliases, hence the __MACH__ check. // FreeBSD does support aliases, but apparently not correctly. :-( - // NOTE: we make many of these symbols weak, but do so in the makefile - // (via objcopy -W) and not here. That ends up being more portable. # define ALIAS(x) __attribute__ ((alias (x))) void* operator new(size_t size) ALIAS("tc_new"); void operator delete(void* p) __THROW ALIAS("tc_delete"); @@ -261,10 +255,6 @@ void* operator new(size_t size, const std::nothrow_t&) __THROW ALIAS("tc_new_nothrow"); void* operator new[](size_t size, const std::nothrow_t&) __THROW ALIAS("tc_newarray_nothrow"); -void operator delete(void* size, const std::nothrow_t&) __THROW - ALIAS("tc_delete_nothrow"); -void operator delete[](void* size, const std::nothrow_t&) __THROW - ALIAS("tc_deletearray_nothrow"); extern "C" { void* malloc(size_t size) __THROW ALIAS("tc_malloc"); void free(void* ptr) __THROW ALIAS("tc_free"); @@ -281,8 +271,26 @@ extern "C" { #ifdef HAVE_STRUCT_MALLINFO struct mallinfo mallinfo(void) __THROW ALIAS("tc_mallinfo"); #endif + // Some library routines on RedHat 9 allocate memory using malloc() + // and free it using __libc_free() (or vice-versa). Since we provide + // our own implementations of malloc/free, we need to make sure that + // the __libc_XXX variants (defined as part of glibc) also point to + // the same implementations. +# if defined(__GLIBC__) + void* __libc_malloc(size_t size) ALIAS("tc_malloc"); + void __libc_free(void* ptr) ALIAS("tc_free"); + void* __libc_realloc(void* ptr, size_t size) ALIAS("tc_realloc"); + void* __libc_calloc(size_t n, size_t size) ALIAS("tc_calloc"); + void __libc_cfree(void* ptr) ALIAS("tc_cfree"); + void* __libc_memalign(size_t align, size_t s) ALIAS("tc_memalign"); + void* __libc_valloc(size_t size) ALIAS("tc_valloc"); + void* __libc_pvalloc(size_t size) ALIAS("tc_pvalloc"); + int __posix_memalign(void** r, size_t a, size_t s) ALIAS("tc_posix_memalign"); +# define HAVE_ALIASED___LIBC 1 +# endif // #if defined(__GLIBC__) } // extern "C" -#else // #if defined(__GNUC__) && !defined(__MACH__) +# undef ALIAS +#else // Portable wrappers void* operator new(size_t size) { return tc_new(size); } void operator delete(void* p) __THROW { tc_delete(p); } @@ -294,12 +302,6 @@ void* operator new(size_t size, const std::nothrow_t& nt) __THROW { void* operator new[](size_t size, const std::nothrow_t& nt) __THROW { return tc_newarray_nothrow(size, nt); } -void operator delete(void* ptr, const std::nothrow_t& nt) __THROW { - return tc_delete_nothrow(ptr, nt); -} -void operator delete[](void* ptr, const std::nothrow_t& nt) __THROW { - return tc_deletearray_nothrow(ptr, nt); -} extern "C" { void* malloc(size_t s) __THROW { return tc_malloc(s); } void free(void* p) __THROW { tc_free(p); } @@ -317,27 +319,11 @@ extern "C" { #ifdef HAVE_STRUCT_MALLINFO struct mallinfo mallinfo(void) __THROW { return tc_mallinfo(); } #endif -} // extern "C" +} // extern C #endif // #if defined(__GNUC__) -// Some library routines on RedHat 9 allocate memory using malloc() -// and free it using __libc_free() (or vice-versa). Since we provide -// our own implementations of malloc/free, we need to make sure that -// the __libc_XXX variants (defined as part of glibc) also point to -// the same implementations. -#ifdef __GLIBC__ // only glibc defines __libc_* +#ifndef HAVE_ALIASED___LIBC extern "C" { -#ifdef ALIAS - void* __libc_malloc(size_t size) ALIAS("tc_malloc"); - void __libc_free(void* ptr) ALIAS("tc_free"); - void* __libc_realloc(void* ptr, size_t size) ALIAS("tc_realloc"); - void* __libc_calloc(size_t n, size_t size) ALIAS("tc_calloc"); - void __libc_cfree(void* ptr) ALIAS("tc_cfree"); - void* __libc_memalign(size_t align, size_t s) ALIAS("tc_memalign"); - void* __libc_valloc(size_t size) ALIAS("tc_valloc"); - void* __libc_pvalloc(size_t size) ALIAS("tc_pvalloc"); - int __posix_memalign(void** r, size_t a, size_t s) ALIAS("tc_posix_memalign"); -#else // #ifdef ALIAS void* __libc_malloc(size_t size) { return malloc(size); } void __libc_free(void* ptr) { free(ptr); } void* __libc_realloc(void* ptr, size_t size) { return realloc(ptr, size); } @@ -349,22 +335,19 @@ extern "C" { int __posix_memalign(void** r, size_t a, size_t s) { return posix_memalign(r, a, s); } -#endif // #ifdef ALIAS } // extern "C" -#endif // ifdef __GLIBC__ +#endif // #ifndef HAVE_ALIASED___LIBC -#undef ALIAS +#endif // #ifdef 0 -#endif // #ifndef(WIN32_DO_PATCHING) && ndef(TCMALLOC_FOR_DEBUGALLOCATION) +#endif // #ifndef WIN32_DO_PATCHING // ----------------------- IMPLEMENTATION ------------------------------- -// Routines such as free() and realloc() catch some erroneous pointers -// passed to them, and invoke the below when they do. (An erroneous pointer -// won't be caught if it's within a valid span or a stale span for which -// the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing -// required) kind of exception handling for these routines. +// These routines are called by free(), realloc(), etc. if the pointer is +// invalid. This is a cheap (source-editing required) kind of exception +// handling for these routines. namespace { void InvalidFree(void* ptr) { CRASH("Attempt to free invalid pointer: %p\n", ptr); @@ -383,11 +366,13 @@ size_t InvalidGetAllocatedSize(void* ptr) { // Extract interesting stats struct TCMallocStats { - uint64_t thread_bytes; // Bytes in thread caches - uint64_t central_bytes; // Bytes in central cache - uint64_t transfer_bytes; // Bytes in central transfer cache - uint64_t metadata_bytes; // Bytes alloced for metadata - PageHeap::Stats pageheap; // Stats from page heap + uint64_t system_bytes; // Bytes alloced from system + uint64_t committed_bytes; // Bytes alloced and committed from system + uint64_t thread_bytes; // Bytes in thread caches + uint64_t central_bytes; // Bytes in central cache + uint64_t transfer_bytes; // Bytes in central transfer cache + uint64_t pageheap_bytes; // Bytes in page heap + uint64_t metadata_bytes; // Bytes alloced for metadata }; // Get stats into "r". Also get per-size-class counts if class_count != NULL @@ -409,8 +394,14 @@ static void ExtractStats(TCMallocStats* r, uint64_t* class_count) { { // scope SpinLockHolder h(Static::pageheap_lock()); ThreadCache::GetThreadStats(&r->thread_bytes, class_count); + } + + { //scope + SpinLockHolder h(Static::pageheap_lock()); + r->system_bytes = Static::pageheap()->SystemBytes(); + r->committed_bytes = Static::pageheap()->CommittedBytes(); r->metadata_bytes = tcmalloc::metadata_system_bytes(); - r->pageheap = Static::pageheap()->stats(); + r->pageheap_bytes = Static::pageheap()->FreeBytes(); } } @@ -422,10 +413,22 @@ static void DumpStats(TCMalloc_Printer* out, int level) { static const double MB = 1048576.0; + const uint64_t bytes_in_use = stats.system_bytes + - stats.pageheap_bytes + - stats.central_bytes + - stats.transfer_bytes + - stats.thread_bytes; + + out->printf("WASTE: %7.1f MB committed but not used\n" + "WASTE: %7.1f MB bytes committed, %7.1f MB bytes in use\n" + "WASTE: committed/used ratio of %f\n", + (stats.committed_bytes - bytes_in_use) / MB, + stats.committed_bytes / MB, + bytes_in_use / MB, + stats.committed_bytes / static_cast<double>(bytes_in_use)); + if (level >= 2) { out->printf("------------------------------------------------\n"); - out->printf("Size class breakdown\n"); - out->printf("------------------------------------------------\n"); uint64_t cumulative = 0; for (int cl = 0; cl < kNumClasses; ++cl) { if (class_count[cl] > 0) { @@ -448,18 +451,11 @@ static void DumpStats(TCMalloc_Printer* out, int level) { DumpSystemAllocatorStats(out); } - const uint64_t bytes_in_use = stats.pageheap.system_bytes - - stats.pageheap.free_bytes - - stats.pageheap.unmapped_bytes - - stats.central_bytes - - stats.transfer_bytes - - stats.thread_bytes; - out->printf("------------------------------------------------\n" "MALLOC: %12" PRIu64 " (%7.1f MB) Heap size\n" + "MALLOC: %12" PRIu64 " (%7.1f MB) Bytes committed\n" "MALLOC: %12" PRIu64 " (%7.1f MB) Bytes in use by application\n" "MALLOC: %12" PRIu64 " (%7.1f MB) Bytes free in page heap\n" - "MALLOC: %12" PRIu64 " (%7.1f MB) Bytes unmapped in page heap\n" "MALLOC: %12" PRIu64 " (%7.1f MB) Bytes free in central cache\n" "MALLOC: %12" PRIu64 " (%7.1f MB) Bytes free in transfer cache\n" "MALLOC: %12" PRIu64 " (%7.1f MB) Bytes free in thread caches\n" @@ -467,10 +463,10 @@ static void DumpStats(TCMalloc_Printer* out, int level) { "MALLOC: %12" PRIu64 " Thread heaps in use\n" "MALLOC: %12" PRIu64 " (%7.1f MB) Metadata allocated\n" "------------------------------------------------\n", - stats.pageheap.system_bytes, stats.pageheap.system_bytes / MB, + stats.system_bytes, stats.system_bytes / MB, + stats.committed_bytes, stats.committed_bytes / MB, bytes_in_use, bytes_in_use / MB, - stats.pageheap.free_bytes, stats.pageheap.free_bytes / MB, - stats.pageheap.unmapped_bytes, stats.pageheap.unmapped_bytes / MB, + stats.pageheap_bytes, stats.pageheap_bytes / MB, stats.central_bytes, stats.central_bytes / MB, stats.transfer_bytes, stats.transfer_bytes / MB, stats.thread_bytes, stats.thread_bytes / MB, @@ -534,50 +530,9 @@ static void** DumpHeapGrowthStackTraces() { return result; } -static void IterateOverRanges(void* arg, MallocExtension::RangeFunction func) { - PageID page = 1; // Some code may assume that page==0 is never used - bool done = false; - while (!done) { - // Accumulate a small number of ranges in a local buffer - static const int kNumRanges = 16; - static base::MallocRange ranges[kNumRanges]; - int n = 0; - { - SpinLockHolder h(Static::pageheap_lock()); - while (n < kNumRanges) { - if (!Static::pageheap()->GetNextRange(page, &ranges[n])) { - done = true; - break; - } else { - uintptr_t limit = ranges[n].address + ranges[n].length; - page = (limit + kPageSize - 1) >> kPageShift; - n++; - } - } - } - - for (int i = 0; i < n; i++) { - (*func)(arg, &ranges[i]); - } - } -} - // TCMalloc's support for extra malloc interfaces class TCMallocImplementation : public MallocExtension { - private: - // ReleaseToSystem() might release more than the requested bytes because - // the page heap releases at the span granularity, and spans are of wildly - // different sizes. This member keeps track of the extra bytes bytes - // released so that the app can periodically call ReleaseToSystem() to - // release memory at a constant rate. - // NOTE: Protected by Static::pageheap_lock(). - size_t extra_bytes_released_; - public: - TCMallocImplementation() - : extra_bytes_released_(0) { - } - virtual void GetStats(char* buffer, int buffer_length) { ASSERT(buffer_length > 0); TCMalloc_Printer printer(buffer, buffer_length); @@ -607,51 +562,39 @@ class TCMallocImplementation : public MallocExtension { return DumpHeapGrowthStackTraces(); } - virtual void Ranges(void* arg, RangeFunction func) { - IterateOverRanges(arg, func); - } - virtual bool GetNumericProperty(const char* name, size_t* value) { ASSERT(name != NULL); if (strcmp(name, "generic.current_allocated_bytes") == 0) { TCMallocStats stats; ExtractStats(&stats, NULL); - *value = stats.pageheap.system_bytes + *value = stats.system_bytes - stats.thread_bytes - stats.central_bytes - stats.transfer_bytes - - stats.pageheap.free_bytes - - stats.pageheap.unmapped_bytes; + - stats.pageheap_bytes; return true; } if (strcmp(name, "generic.heap_size") == 0) { TCMallocStats stats; ExtractStats(&stats, NULL); - *value = stats.pageheap.system_bytes; + *value = stats.system_bytes; return true; } - if (strcmp(name, "tcmalloc.slack_bytes") == 0) { - // We assume that bytes in the page heap are not fragmented too - // badly, and are therefore available for allocation without - // growing the pageheap system byte count. - SpinLockHolder l(Static::pageheap_lock()); - PageHeap::Stats stats = Static::pageheap()->stats(); - *value = stats.free_bytes + stats.unmapped_bytes; - return true; - } - - if (strcmp(name, "tcmalloc.pageheap_free_bytes") == 0) { - SpinLockHolder l(Static::pageheap_lock()); - *value = Static::pageheap()->stats().free_bytes; + if (strcmp(name, "generic.committed_bytes") == 0) { + TCMallocStats stats; + ExtractStats(&stats, NULL); + *value = stats.committed_bytes + stats.metadata_bytes; return true; } - if (strcmp(name, "tcmalloc.pageheap_unmapped_bytes") == 0) { + if (strcmp(name, "tcmalloc.slack_bytes") == 0) { + // We assume that bytes in the page heap are not fragmented too + // badly, and are therefore available for allocation. SpinLockHolder l(Static::pageheap_lock()); - *value = Static::pageheap()->stats().unmapped_bytes; + *value = Static::pageheap()->FreeBytes(); return true; } @@ -687,34 +630,9 @@ class TCMallocImplementation : public MallocExtension { ThreadCache::BecomeIdle(); } - virtual void MarkThreadBusy(); // Implemented below - - virtual void ReleaseToSystem(ssize_t num_bytes) { - if (num_bytes <= 0) { - return; - } + virtual void ReleaseFreeMemory() { SpinLockHolder h(Static::pageheap_lock()); - if (num_bytes <= extra_bytes_released_) { - // We released too much on a prior call, so don't release any - // more this time. - extra_bytes_released_ = extra_bytes_released_ - num_bytes; - return; - } - num_bytes = num_bytes - extra_bytes_released_; - // num_bytes might be less than one page. If we pass zero to - // ReleaseAtLeastNPages, it won't do anything, so we release a whole - // page now and let extra_bytes_released_ smooth it out over time. - Length num_pages = max<Length>(num_bytes >> kPageShift, 1); - size_t bytes_released = Static::pageheap()->ReleaseAtLeastNPages( - num_pages) << kPageShift; - if (bytes_released > num_bytes) { - extra_bytes_released_ = bytes_released - num_bytes; - } else { - // The PageHeap wasn't able to release num_bytes. Don't try to - // compensate with a big release next time. Specifically, - // ReleaseFreeMemory() calls ReleaseToSystem(LONG_MAX). - extra_bytes_released_ = 0; - } + Static::pageheap()->ReleaseFreePages(); } virtual void SetMemoryReleaseRate(double rate) { @@ -1144,18 +1062,16 @@ inline struct mallinfo do_mallinfo() { // Unfortunately, the struct contains "int" field, so some of the // size values will be truncated. - info.arena = static_cast<int>(stats.pageheap.system_bytes); + info.arena = static_cast<int>(stats.system_bytes); info.fsmblks = static_cast<int>(stats.thread_bytes + stats.central_bytes + stats.transfer_bytes); - info.fordblks = static_cast<int>(stats.pageheap.free_bytes + - stats.pageheap.unmapped_bytes); - info.uordblks = static_cast<int>(stats.pageheap.system_bytes + info.fordblks = static_cast<int>(stats.pageheap_bytes); + info.uordblks = static_cast<int>(stats.system_bytes - stats.thread_bytes - stats.central_bytes - stats.transfer_bytes - - stats.pageheap.free_bytes - - stats.pageheap.unmapped_bytes); + - stats.pageheap_bytes); return info; } @@ -1218,55 +1134,39 @@ size_t TCMallocImplementation::GetAllocatedSize(void* ptr) { return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize); } -void TCMallocImplementation::MarkThreadBusy() { - // Allocate to force the creation of a thread cache, but avoid - // invoking any hooks. - do_free(do_malloc(0)); -} - //------------------------------------------------------------------- // Exported routines //------------------------------------------------------------------- -extern "C" PERFTOOLS_DLL_DECL const char* tc_version( - int* major, int* minor, const char** patch) __THROW { - if (major) *major = TC_VERSION_MAJOR; - if (minor) *minor = TC_VERSION_MINOR; - if (patch) *patch = TC_VERSION_PATCH; - return TC_VERSION_STRING; -} - // CAVEAT: The code structure below ensures that MallocHook methods are always // called from the stack frame of the invoked allocation function. // heap-checker.cc depends on this to start a stack trace from // the call to the (de)allocation function. static int tc_new_mode = 0; // See tc_set_new_mode(). -extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW { +extern "C" void* tc_malloc(size_t size) __THROW { void* result = (tc_new_mode ? cpp_alloc(size, false) : do_malloc(size)); MallocHook::InvokeNewHook(result, size); return result; } -extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW { +extern "C" void tc_free(void* ptr) __THROW { MallocHook::InvokeDeleteHook(ptr); do_free(ptr); } -extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t n, - size_t elem_size) __THROW { +extern "C" void* tc_calloc(size_t n, size_t elem_size) __THROW { void* result = do_calloc(n, elem_size); MallocHook::InvokeNewHook(result, n * elem_size); return result; } -extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW { +extern "C" void tc_cfree(void* ptr) __THROW { MallocHook::InvokeDeleteHook(ptr); do_free(ptr); } -extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* old_ptr, - size_t new_size) __THROW { +extern "C" void* tc_realloc(void* old_ptr, size_t new_size) __THROW { if (old_ptr == NULL) { void* result = do_malloc(new_size); MallocHook::InvokeNewHook(result, new_size); @@ -1280,7 +1180,7 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* old_ptr, return do_realloc(old_ptr, new_size); } -extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) { +extern "C" void* tc_new(size_t size) { void* p = cpp_alloc(size, false); // We keep this next instruction out of cpp_alloc for a reason: when // it's in, and new just calls cpp_alloc, the optimizer may fold the @@ -1291,27 +1191,18 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) { return p; } -extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow( - size_t size, const std::nothrow_t&) __THROW { +extern "C" void* tc_new_nothrow(size_t size, const std::nothrow_t&) __THROW { void* p = cpp_alloc(size, true); MallocHook::InvokeNewHook(p, size); return p; } -extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW { +extern "C" void tc_delete(void* p) __THROW { MallocHook::InvokeDeleteHook(p); do_free(p); } -// Compilers define and use this (via ::operator delete(ptr, nothrow)). -// But it's really the same as normal delete, so we just do the same thing. -extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow( - void* p, const std::nothrow_t&) __THROW { - MallocHook::InvokeDeleteHook(p); - do_free(p); -} - -extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) { +extern "C" void* tc_newarray(size_t size) { void* p = cpp_alloc(size, false); // We keep this next instruction out of cpp_alloc for a reason: when // it's in, and new just calls cpp_alloc, the optimizer may fold the @@ -1322,33 +1213,25 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) { return p; } -extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow( - size_t size, const std::nothrow_t&) __THROW { +extern "C" void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) __THROW { void* p = cpp_alloc(size, true); MallocHook::InvokeNewHook(p, size); return p; } -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW { - MallocHook::InvokeDeleteHook(p); - do_free(p); -} - -extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow( - void* p, const std::nothrow_t&) __THROW { +extern "C" void tc_deletearray(void* p) __THROW { MallocHook::InvokeDeleteHook(p); do_free(p); } -extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, - size_t size) __THROW { +extern "C" void* tc_memalign(size_t align, size_t size) __THROW { void* result = do_memalign(align, size); MallocHook::InvokeNewHook(result, size); return result; } -extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign( - void** result_ptr, size_t align, size_t size) __THROW { +extern "C" int tc_posix_memalign(void** result_ptr, size_t align, size_t size) + __THROW { if (((align % sizeof(void*)) != 0) || ((align & (align - 1)) != 0) || (align == 0)) { @@ -1367,7 +1250,7 @@ extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign( static size_t pagesize = 0; -extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) __THROW { +extern "C" void* tc_valloc(size_t size) __THROW { // Allocate page-aligned object of length >= size bytes if (pagesize == 0) pagesize = getpagesize(); void* result = do_memalign(pagesize, size); @@ -1375,28 +1258,25 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) __THROW { return result; } -extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) __THROW { +extern "C" void* tc_pvalloc(size_t size) __THROW { // Round up size to a multiple of pagesize if (pagesize == 0) pagesize = getpagesize(); - if (size == 0) { // pvalloc(0) should allocate one page, according to - size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html - } size = (size + pagesize - 1) & ~(pagesize - 1); void* result = do_memalign(pagesize, size); MallocHook::InvokeNewHook(result, size); return result; } -extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW { +extern "C" void tc_malloc_stats(void) __THROW { do_malloc_stats(); } -extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW { +extern "C" int tc_mallopt(int cmd, int value) __THROW { return do_mallopt(cmd, value); } #ifdef HAVE_STRUCT_MALLINFO -extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { +extern "C" struct mallinfo tc_mallinfo(void) __THROW { return do_mallinfo(); } #endif @@ -1406,7 +1286,7 @@ extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { // If flag is 1, calls to malloc will behave like calls to new, // and the std_new_handler will be invoked on failure. // Returns the previous mode. -extern "C" PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) __THROW { +extern "C" int tc_set_new_mode(int flag) __THROW { int old_mode = tc_new_mode; tc_new_mode = flag; return old_mode; @@ -1420,7 +1300,6 @@ extern "C" PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) __THROW { // This function is an exception to the rule of calling MallocHook method // from the stack frame of the allocation function; // heap-checker handles this special case explicitly. -#ifndef TCMALLOC_FOR_DEBUGALLOCATION static void *MemalignOverride(size_t align, size_t size, const void *caller) __THROW ATTRIBUTE_SECTION(google_malloc); @@ -1431,4 +1310,3 @@ static void *MemalignOverride(size_t align, size_t size, const void *caller) return result; } void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride; -#endif // #ifndef TCMALLOC_FOR_DEBUGALLOCATION diff --git a/third_party/tcmalloc/chromium/src/windows/port.cc b/third_party/tcmalloc/chromium/src/windows/port.cc index 76a9e38..3902ca7 100644 --- a/third_party/tcmalloc/chromium/src/windows/port.cc +++ b/third_party/tcmalloc/chromium/src/windows/port.cc @@ -71,7 +71,8 @@ int getpagesize() { if (pagesize == 0) { SYSTEM_INFO system_info; GetSystemInfo(&system_info); - pagesize = system_info.dwPageSize; + pagesize = std::max(system_info.dwPageSize, + system_info.dwAllocationGranularity); } return pagesize; } @@ -182,44 +183,90 @@ pthread_key_t PthreadKeyCreate(void (*destr_fn)(void*)) { // ----------------------------------------------------------------------- // These functions replace system-alloc.cc +static SpinLock alloc_lock(SpinLock::LINKER_INITIALIZED); + // This is mostly like MmapSysAllocator::Alloc, except it does these weird // munmap's in the middle of the page, which is forbidden in windows. extern void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, size_t alignment) { - // Safest is to make actual_size same as input-size. - if (actual_size) { - *actual_size = size; - } - + SpinLockHolder sh(&alloc_lock); // Align on the pagesize boundary const int pagesize = getpagesize(); if (alignment < pagesize) alignment = pagesize; size = ((size + alignment - 1) / alignment) * alignment; - // Ask for extra memory if alignment > pagesize - size_t extra = 0; - if (alignment > pagesize) { - extra = alignment - pagesize; + // Report the total number of bytes the OS actually delivered. This might be + // greater than |size| because of alignment concerns. The full size is + // necessary so that adjacent spans can be coalesced. + // TODO(antonm): proper processing of alignments + // in actual_size and decommitting. + if (actual_size) { + *actual_size = size; } - void* result = VirtualAlloc(0, size + extra, + // We currently do not support alignments larger than the pagesize or + // alignments that are not multiples of the pagesize after being floored. + // If this ability is needed it can be done by the caller (assuming it knows + // the page size). + assert(alignment <= pagesize); + + void* result = VirtualAlloc(0, size, MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE); if (result == NULL) return NULL; - // Adjust the return memory so it is aligned - uintptr_t ptr = reinterpret_cast<uintptr_t>(result); - size_t adjust = 0; - if ((ptr & (alignment - 1)) != 0) { - adjust = alignment - (ptr & (alignment - 1)); - } + // If the result is not aligned memory fragmentation will result which can + // lead to pathological memory use. + assert((reinterpret_cast<uintptr_t>(result) & (alignment - 1)) == 0); - ptr += adjust; - return reinterpret_cast<void*>(ptr); + return result; } void TCMalloc_SystemRelease(void* start, size_t length) { - // TODO(csilvers): should I be calling VirtualFree here? + if (VirtualFree(start, length, MEM_DECOMMIT)) + return; + + // The decommit may fail if the memory region consists of allocations + // from more than one call to VirtualAlloc. In this case, fall back to + // using VirtualQuery to retrieve the allocation boundaries and decommit + // them each individually. + + char* ptr = static_cast<char*>(start); + char* end = ptr + length; + MEMORY_BASIC_INFORMATION info; + while (ptr < end) { + size_t resultSize = VirtualQuery(ptr, &info, sizeof(info)); + assert(resultSize == sizeof(info)); + size_t decommitSize = std::min<size_t>(info.RegionSize, end - ptr); + BOOL success = VirtualFree(ptr, decommitSize, MEM_DECOMMIT); + assert(success == TRUE); + ptr += decommitSize; + } +} + +void TCMalloc_SystemCommit(void* start, size_t length) +{ + if (VirtualAlloc(start, length, MEM_COMMIT, PAGE_READWRITE) == start) + return; + + // The commit may fail if the memory region consists of allocations + // from more than one call to VirtualAlloc. In this case, fall back to + // using VirtualQuery to retrieve the allocation boundaries and commit them + // each individually. + + char* ptr = static_cast<char*>(start); + char* end = ptr + length; + MEMORY_BASIC_INFORMATION info; + while (ptr < end) { + size_t resultSize = VirtualQuery(ptr, &info, sizeof(info)); + assert(resultSize == sizeof(info)); + + size_t commitSize = std::min<size_t>(info.RegionSize, end - ptr); + void* newAddress = VirtualAlloc(ptr, commitSize, MEM_COMMIT, + PAGE_READWRITE); + assert(newAddress == ptr); + ptr += commitSize; + } } bool RegisterSystemAllocator(SysAllocator *allocator, int priority) { |