diff options
author | glider@chromium.org <glider@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2011-05-31 08:40:56 +0000 |
---|---|---|
committer | glider@chromium.org <glider@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98> | 2011-05-31 08:40:56 +0000 |
commit | 330391569bd285cbdfabc99fb3873ccc06490d2d (patch) | |
tree | 6265244b37e6253f5ab19b831e5bd641b73eb596 /third_party/tcmalloc/vendor/src/base | |
parent | da8a9632f03b1a6c053f0eb51cbe476cc8fcad0c (diff) | |
download | chromium_src-330391569bd285cbdfabc99fb3873ccc06490d2d.zip chromium_src-330391569bd285cbdfabc99fb3873ccc06490d2d.tar.gz chromium_src-330391569bd285cbdfabc99fb3873ccc06490d2d.tar.bz2 |
Update the tcmalloc vendor branch to r109
Review URL: http://codereview.chromium.org/6965012
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@87286 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'third_party/tcmalloc/vendor/src/base')
27 files changed, 1601 insertions, 300 deletions
diff --git a/third_party/tcmalloc/vendor/src/base/arm_instruction_set_select.h b/third_party/tcmalloc/vendor/src/base/arm_instruction_set_select.h new file mode 100644 index 0000000..a47e6bb --- /dev/null +++ b/third_party/tcmalloc/vendor/src/base/arm_instruction_set_select.h @@ -0,0 +1,79 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Alexander Levitskiy +// +// Generalizes the plethora of ARM flavors available to an easier to manage set +// Defs reference is at https://wiki.edubuntu.org/ARM/Thumb2PortingHowto + +#ifndef ARM_INSTRUCTION_SET_SELECT_H_ +#define ARM_INSTRUCTION_SET_SELECT_H_ + +#if defined(__ARM_ARCH_7__) || \ + defined(__ARM_ARCH_7R__) || \ + defined(__ARM_ARCH_7A__) +# define ARMV7 1 +#endif + +#if defined(ARMV7) || \ + defined(__ARM_ARCH_6__) || \ + defined(__ARM_ARCH_6J__) || \ + defined(__ARM_ARCH_6K__) || \ + defined(__ARM_ARCH_6Z__) || \ + defined(__ARM_ARCH_6T2__) || \ + defined(__ARM_ARCH_6ZK__) +# define ARMV6 1 +#endif + +#if defined(ARMV6) || \ + defined(__ARM_ARCH_5T__) || \ + defined(__ARM_ARCH_5E__) || \ + defined(__ARM_ARCH_5TE__) || \ + defined(__ARM_ARCH_5TEJ__) +# define ARMV5 1 +#endif + +#if defined(ARMV5) || \ + defined(__ARM_ARCH_4__) || \ + defined(__ARM_ARCH_4T__) +# define ARMV4 1 +#endif + +#if defined(ARMV4) || \ + defined(__ARM_ARCH_3__) || \ + defined(__ARM_ARCH_3M__) +# define ARMV3 1 +#endif + +#if defined(ARMV3) || \ + defined(__ARM_ARCH_2__) +# define ARMV2 1 +#endif + +#endif // ARM_INSTRUCTION_SET_SELECT_H_ diff --git a/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-gcc.h b/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-gcc.h new file mode 100644 index 0000000..423e993 --- /dev/null +++ b/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-gcc.h @@ -0,0 +1,234 @@ +/* Copyright (c) 2010, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Lei Zhang, Sasha Levitskiy + */ + +// This file is an internal atomic implementation, use base/atomicops.h instead. +// +// LinuxKernelCmpxchg and Barrier_AtomicIncrement are from Google Gears. + +#ifndef BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ +#define BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ + +#include <stdio.h> +#include "base/basictypes.h" // For COMPILE_ASSERT + +typedef int32_t Atomic32; + +namespace base { +namespace subtle { + +typedef int64_t Atomic64; + +// 0xffff0fc0 is the hard coded address of a function provided by +// the kernel which implements an atomic compare-exchange. On older +// ARM architecture revisions (pre-v6) this may be implemented using +// a syscall. This address is stable, and in active use (hard coded) +// by at least glibc-2.7 and the Android C library. +// pLinuxKernelCmpxchg has both acquire and release barrier sematincs. +typedef Atomic32 (*LinuxKernelCmpxchgFunc)(Atomic32 old_value, + Atomic32 new_value, + volatile Atomic32* ptr); +LinuxKernelCmpxchgFunc pLinuxKernelCmpxchg __attribute__((weak)) = + (LinuxKernelCmpxchgFunc) 0xffff0fc0; + +typedef void (*LinuxKernelMemoryBarrierFunc)(void); +LinuxKernelMemoryBarrierFunc pLinuxKernelMemoryBarrier __attribute__((weak)) = + (LinuxKernelMemoryBarrierFunc) 0xffff0fa0; + + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 prev_value = *ptr; + do { + if (!pLinuxKernelCmpxchg(old_value, new_value, + const_cast<Atomic32*>(ptr))) { + return old_value; + } + prev_value = *ptr; + } while (prev_value == old_value); + return prev_value; +} + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + Atomic32 old_value; + do { + old_value = *ptr; + } while (pLinuxKernelCmpxchg(old_value, new_value, + const_cast<Atomic32*>(ptr))); + return old_value; +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + for (;;) { + // Atomic exchange the old value with an incremented one. + Atomic32 old_value = *ptr; + Atomic32 new_value = old_value + increment; + if (pLinuxKernelCmpxchg(old_value, new_value, + const_cast<Atomic32*>(ptr)) == 0) { + // The exchange took place as expected. + return new_value; + } + // Otherwise, *ptr changed mid-loop and we need to retry. + } +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return Barrier_AtomicIncrement(ptr, increment); +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} + +inline void MemoryBarrier() { + pLinuxKernelMemoryBarrier(); +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + MemoryBarrier(); + *ptr = value; +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return *ptr; +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + Atomic32 value = *ptr; + MemoryBarrier(); + return value; +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + MemoryBarrier(); + return *ptr; +} + + +// 64-bit versions are not implemented yet. + +inline void NotImplementedFatalError(const char *function_name) { + fprintf(stderr, "64-bit %s() not implemented on this platform\n", + function_name); + abort(); +} + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + NotImplementedFatalError("NoBarrier_CompareAndSwap"); + return 0; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + NotImplementedFatalError("NoBarrier_AtomicExchange"); + return 0; +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + NotImplementedFatalError("NoBarrier_AtomicIncrement"); + return 0; +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + NotImplementedFatalError("Barrier_AtomicIncrement"); + return 0; +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + NotImplementedFatalError("NoBarrier_Store"); +} + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + NoBarrier_AtomicExchange(ptr, value); + // acts as a barrier in this implementation +} + +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + NotImplementedFatalError("Release_Store"); +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("NoBarrier_Load"); + return 0; +} + +inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { + Atomic64 value = NoBarrier_Load(ptr); + return value; +} + +inline Atomic64 Release_Load(volatile const Atomic64* ptr) { + MemoryBarrier(); + return NoBarrier_Load(ptr); +} + +inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +} // namespace base::subtle +} // namespace base + +#endif // BASE_ATOMICOPS_INTERNALS_ARM_GCC_H_ diff --git a/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-generic.h b/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-generic.h new file mode 100644 index 0000000..7882b0d --- /dev/null +++ b/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-generic.h @@ -0,0 +1,236 @@ +// Copyright (c) 2003, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// --- +// +// Author: Lei Zhang, Sasha Levitskiy +// +// This file is an internal atomic implementation, use base/atomicops.h instead. +// +// LinuxKernelCmpxchg and Barrier_AtomicIncrement are from Google Gears. + +#ifndef BASE_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ +#define BASE_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ + +#include <stdio.h> +#include <stdlib.h> +#include "base/macros.h" // For COMPILE_ASSERT +#include "base/port.h" // ATTRIBUTE_WEAK + +typedef int32_t Atomic32; + +namespace base { +namespace subtle { + +typedef int64_t Atomic64; + +// 0xffff0fc0 is the hard coded address of a function provided by +// the kernel which implements an atomic compare-exchange. On older +// ARM architecture revisions (pre-v6) this may be implemented using +// a syscall. This address is stable, and in active use (hard coded) +// by at least glibc-2.7 and the Android C library. +// pLinuxKernelCmpxchg has both acquire and release barrier sematincs. +typedef Atomic32 (*LinuxKernelCmpxchgFunc)(Atomic32 old_value, + Atomic32 new_value, + volatile Atomic32* ptr); +LinuxKernelCmpxchgFunc pLinuxKernelCmpxchg ATTRIBUTE_WEAK = + (LinuxKernelCmpxchgFunc) 0xffff0fc0; + +typedef void (*LinuxKernelMemoryBarrierFunc)(void); +LinuxKernelMemoryBarrierFunc pLinuxKernelMemoryBarrier ATTRIBUTE_WEAK = + (LinuxKernelMemoryBarrierFunc) 0xffff0fa0; + + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 prev_value = *ptr; + do { + if (!pLinuxKernelCmpxchg(old_value, new_value, + const_cast<Atomic32*>(ptr))) { + return old_value; + } + prev_value = *ptr; + } while (prev_value == old_value); + return prev_value; +} + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + Atomic32 old_value; + do { + old_value = *ptr; + } while (pLinuxKernelCmpxchg(old_value, new_value, + const_cast<Atomic32*>(ptr))); + return old_value; +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + for (;;) { + // Atomic exchange the old value with an incremented one. + Atomic32 old_value = *ptr; + Atomic32 new_value = old_value + increment; + if (pLinuxKernelCmpxchg(old_value, new_value, + const_cast<Atomic32*>(ptr)) == 0) { + // The exchange took place as expected. + return new_value; + } + // Otherwise, *ptr changed mid-loop and we need to retry. + } +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + return Barrier_AtomicIncrement(ptr, increment); +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} + +inline void MemoryBarrier() { + pLinuxKernelMemoryBarrier(); +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + MemoryBarrier(); + *ptr = value; +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return *ptr; +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + Atomic32 value = *ptr; + MemoryBarrier(); + return value; +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + MemoryBarrier(); + return *ptr; +} + + +// 64-bit versions are not implemented yet. + +inline void NotImplementedFatalError(const char *function_name) { + fprintf(stderr, "64-bit %s() not implemented on this platform\n", + function_name); + abort(); +} + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + NotImplementedFatalError("NoBarrier_CompareAndSwap"); + return 0; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + NotImplementedFatalError("NoBarrier_AtomicExchange"); + return 0; +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + NotImplementedFatalError("NoBarrier_AtomicIncrement"); + return 0; +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + NotImplementedFatalError("Barrier_AtomicIncrement"); + return 0; +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + NotImplementedFatalError("NoBarrier_Store"); +} + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + NotImplementedFatalError("Acquire_Store64"); +} + +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + NotImplementedFatalError("Release_Store"); +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("NoBarrier_Load"); + return 0; +} + +inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("Atomic64 Acquire_Load"); + return 0; +} + +inline Atomic64 Release_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("Atomic64 Release_Load"); + return 0; +} + +inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + NotImplementedFatalError("Atomic64 Acquire_CompareAndSwap"); + return 0; +} + +inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + NotImplementedFatalError("Atomic64 Release_CompareAndSwap"); + return 0; +} + +} // namespace base::subtle +} // namespace base + +#endif // BASE_ATOMICOPS_INTERNALS_ARM_GENERIC_H_ diff --git a/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-v6plus.h b/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-v6plus.h new file mode 100644 index 0000000..ee09f32 --- /dev/null +++ b/third_party/tcmalloc/vendor/src/base/atomicops-internals-arm-v6plus.h @@ -0,0 +1,244 @@ +// Copyright (c) 2011, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// --- +// +// Author: Sasha Levitskiy +// based on atomicops-internals by Sanjay Ghemawat +// +// This file is an internal atomic implementation, use base/atomicops.h instead. +// +// This code implements ARM atomics for architectures V6 and newer. + +#ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ +#define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ + +#include <stdio.h> +#include <stdlib.h> +#include "base/basictypes.h" // For COMPILE_ASSERT + +typedef int32_t Atomic32; + +namespace base { +namespace subtle { + +typedef int64_t Atomic64; + +// 32-bit low-level ops + +inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 oldval, res; + do { + __asm__ __volatile__( + "ldrex %1, [%3]\n" + "mov %0, #0\n" + "teq %1, %4\n" + "strexeq %0, %5, [%3]\n" + : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) + : "r" (ptr), "Ir" (old_value), "r" (new_value) + : "cc"); + } while (res); + return oldval; +} + +inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, + Atomic32 new_value) { + Atomic32 tmp, old; + __asm__ __volatile__( + "1:\n" + "ldrex %1, [%2]\n" + "strex %0, %3, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (tmp), "=&r" (old) + : "r" (ptr), "r" (new_value) + : "cc", "memory"); + return old; +} + +inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + Atomic32 tmp, res; + __asm__ __volatile__( + "1:\n" + "ldrex %1, [%2]\n" + "add %1, %1, %3\n" + "strex %0, %1, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (tmp), "=&r"(res) + : "r" (ptr), "r"(increment) + : "cc", "memory"); + return res; +} + +inline void MemoryBarrier() { + __asm__ __volatile__("dmb" : : : "memory"); +} + +inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, + Atomic32 increment) { + Atomic32 tmp, res; + __asm__ __volatile__( + "1:\n" + "ldrex %1, [%2]\n" + "add %1, %1, %3\n" + "dmb\n" + "strex %0, %1, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (tmp), "=&r"(res) + : "r" (ptr), "r"(increment) + : "cc", "memory"); + return res; +} + +inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); + MemoryBarrier(); + return value; +} + +inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, + Atomic32 old_value, + Atomic32 new_value) { + MemoryBarrier(); + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); +} + +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} + +inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; + MemoryBarrier(); +} + +inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { + MemoryBarrier(); + *ptr = value; +} + +inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { + return *ptr; +} + +inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { + Atomic32 value = *ptr; + MemoryBarrier(); + return value; +} + +inline Atomic32 Release_Load(volatile const Atomic32* ptr) { + MemoryBarrier(); + return *ptr; +} + +// 64-bit versions are not implemented yet. + +inline void NotImplementedFatalError(const char *function_name) { + fprintf(stderr, "64-bit %s() not implemented on this platform\n", + function_name); + abort(); +} + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + NotImplementedFatalError("NoBarrier_CompareAndSwap"); + return 0; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + NotImplementedFatalError("NoBarrier_AtomicExchange"); + return 0; +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + NotImplementedFatalError("NoBarrier_AtomicIncrement"); + return 0; +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + NotImplementedFatalError("Barrier_AtomicIncrement"); + return 0; +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + NotImplementedFatalError("NoBarrier_Store"); +} + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + NotImplementedFatalError("Acquire_Store64"); +} + +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + NotImplementedFatalError("Release_Store"); +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("NoBarrier_Load"); + return 0; +} + +inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("Atomic64 Acquire_Load"); + return 0; +} + +inline Atomic64 Release_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("Atomic64 Release_Load"); + return 0; +} + +inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + NotImplementedFatalError("Atomic64 Acquire_CompareAndSwap"); + return 0; +} + +inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + NotImplementedFatalError("Atomic64 Release_CompareAndSwap"); + return 0; +} + +} // namespace subtle ends +} // namespace base ends + +#endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ diff --git a/third_party/tcmalloc/vendor/src/base/atomicops-internals-x86-msvc.h b/third_party/tcmalloc/vendor/src/base/atomicops-internals-windows.h index d50894c..58782a17 100644 --- a/third_party/tcmalloc/vendor/src/base/atomicops-internals-x86-msvc.h +++ b/third_party/tcmalloc/vendor/src/base/atomicops-internals-windows.h @@ -31,12 +31,12 @@ * Author: Sanjay Ghemawat */ -// Implementation of atomic operations for x86. This file should not -// be included directly. Clients should instead include -// "base/atomicops.h". +// Implementation of atomic operations using Windows API +// functions. This file should not be included directly. Clients +// should instead include "base/atomicops.h". -#ifndef BASE_ATOMICOPS_INTERNALS_X86_MSVC_H_ -#define BASE_ATOMICOPS_INTERNALS_X86_MSVC_H_ +#ifndef BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ +#define BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ #include <stdio.h> #include <stdlib.h> @@ -257,8 +257,8 @@ inline Atomic64 Release_Load(volatile const Atomic64* ptr) { // 64-bit low-level operations on 32-bit platform -// TBD(vchen): The GNU assembly below must be converted to MSVC inline -// assembly. +// TODO(vchen): The GNU assembly below must be converted to MSVC inline +// assembly. Then the file should be renamed to ...-x86-mscv.h, probably. inline void NotImplementedFatalError(const char *function_name) { fprintf(stderr, "64-bit %s() not implemented on this platform\n", @@ -411,4 +411,4 @@ inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, } // namespace base::subtle } // namespace base -#endif // BASE_ATOMICOPS_INTERNALS_X86_MSVC_H_ +#endif // BASE_ATOMICOPS_INTERNALS_WINDOWS_H_ diff --git a/third_party/tcmalloc/vendor/src/base/atomicops.h b/third_party/tcmalloc/vendor/src/base/atomicops.h index 0f3d3ef..f36df5f 100644 --- a/third_party/tcmalloc/vendor/src/base/atomicops.h +++ b/third_party/tcmalloc/vendor/src/base/atomicops.h @@ -86,14 +86,18 @@ // TODO(csilvers): figure out ARCH_PIII/ARCH_K8 (perhaps via ./configure?) // ------------------------------------------------------------------------ +#include "base/arm_instruction_set_select.h" + // TODO(csilvers): match piii, not just __i386. Also, match k8 #if defined(__MACH__) && defined(__APPLE__) #include "base/atomicops-internals-macosx.h" -#elif defined(_MSC_VER) && defined(_M_IX86) -#include "base/atomicops-internals-x86-msvc.h" -#elif defined(__MINGW32__) && defined(__i386__) -#include "base/atomicops-internals-x86-msvc.h" -#elif defined(__GNUC__) && (defined(__i386) || defined(ARCH_K8)) +#elif defined(__GNUC__) && defined(ARMV6) +#include "base/atomicops-internals-arm-v6plus.h" +#elif defined(ARMV3) +#include "base/atomicops-internals-arm-generic.h" +#elif defined(_WIN32) +#include "base/atomicops-internals-x86-windows.h" +#elif defined(__GNUC__) && (defined(__i386) || defined(__x86_64__)) #include "base/atomicops-internals-x86.h" #elif defined(__linux__) && defined(__PPC__) #include "base/atomicops-internals-linuxppc.h" diff --git a/third_party/tcmalloc/vendor/src/base/basictypes.h b/third_party/tcmalloc/vendor/src/base/basictypes.h index ab9cdabc..0f21fca 100644 --- a/third_party/tcmalloc/vendor/src/base/basictypes.h +++ b/third_party/tcmalloc/vendor/src/base/basictypes.h @@ -109,7 +109,7 @@ const int64 kint64min = ( ((( int64) kint32min) << 32) | 0 ); // Also allow for printing of a pthread_t. #define GPRIuPTHREAD "lu" #define GPRIxPTHREAD "lx" -#if defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__APPLE__) +#if defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__APPLE__) || defined(__FreeBSD__) #define PRINTABLE_PTHREAD(pthreadt) reinterpret_cast<uintptr_t>(pthreadt) #else #define PRINTABLE_PTHREAD(pthreadt) pthreadt diff --git a/third_party/tcmalloc/vendor/src/base/cycleclock.h b/third_party/tcmalloc/vendor/src/base/cycleclock.h index 8af664ed..11f9c9b 100644 --- a/third_party/tcmalloc/vendor/src/base/cycleclock.h +++ b/third_party/tcmalloc/vendor/src/base/cycleclock.h @@ -46,33 +46,43 @@ #define GOOGLE_BASE_CYCLECLOCK_H_ #include "base/basictypes.h" // make sure we get the def for int64 +#include "base/arm_instruction_set_select.h" #if defined(__MACH__) && defined(__APPLE__) -#include <mach/mach_time.h> +# include <mach/mach_time.h> #endif +// For MSVC, we want the __rdtsc intrinsic, declared in <intrin.h>. +// Unfortunately, in some environments, <windows.h> and <intrin.h> have +// conflicting declarations of some other intrinsics, breaking compilation. +// Therefore, we simply declare __rdtsc ourselves. See also +// http://connect.microsoft.com/VisualStudio/feedback/details/262047 +#if defined(_MSC_VER) +extern "C" uint64 __rdtsc(); +#pragma intrinsic(__rdtsc) +#endif +#include <sys/time.h> // NOTE: only i386 and x86_64 have been well tested. // PPC, sparc, alpha, and ia64 are based on // http://peter.kuscsik.com/wordpress/?p=14 -// with modifications by m3b. cf +// with modifications by m3b. See also // https://setisvn.ssl.berkeley.edu/svn/lib/fftw-3.0.1/kernel/cycle.h struct CycleClock { // This should return the number of cycles since power-on. Thread-safe. static inline int64 Now() { #if defined(__MACH__) && defined(__APPLE__) - // this goes at the top because we need ALL Macs, regardless - // of architecture, to return the number of "mach time units" - // that have passes since startup. See sysinfo.cc where - // InitializeSystemInfo() sets the supposed cpu clock frequency of macs - // to the number of mach time units per second, not actual + // this goes at the top because we need ALL Macs, regardless of + // architecture, to return the number of "mach time units" that + // have passed since startup. See sysinfo.cc where + // InitializeSystemInfo() sets the supposed cpu clock frequency of + // macs to the number of mach time units per second, not actual // CPU clock frequency (which can change in the face of CPU - // frequency scaling). also note that when the Mac sleeps, - // this counter pauses; it does not continue counting, nor resets - // to zero. + // frequency scaling). Also note that when the Mac sleeps, this + // counter pauses; it does not continue counting, nor does it + // reset to zero. return mach_absolute_time(); #elif defined(__i386__) int64 ret; - __asm__ volatile ("rdtsc" - : "=A" (ret) ); + __asm__ volatile ("rdtsc" : "=A" (ret) ); return ret; #elif defined(__x86_64__) || defined(__amd64__) uint64 low, high; @@ -82,7 +92,7 @@ struct CycleClock { // This returns a time-base, which is not always precisely a cycle-count. int64 tbl, tbu0, tbu1; asm("mftbu %0" : "=r" (tbu0)); - asm("mftb %0" : "=r" (tbl )); + asm("mftb %0" : "=r" (tbl)); asm("mftbu %0" : "=r" (tbu1)); tbl &= -static_cast<int64>(tbu0 == tbu1); // high 32 bits in tbu1; low 32 bits in tbl (tbu0 is garbage) @@ -96,11 +106,31 @@ struct CycleClock { int64 itc; asm("mov %0 = ar.itc" : "=r" (itc)); return itc; -#elif defined(_MSC_VER) && defined(_M_IX86) - _asm rdtsc +#elif defined(_MSC_VER) + return __rdtsc(); +#elif defined(ARMV3) +#if defined(ARMV6) // V6 is the earliest arch that has a standard cyclecount + uint32 pmccntr; + uint32 pmuseren; + uint32 pmcntenset; + // Read the user mode perf monitor counter access permissions. + asm("mrc p15, 0, %0, c9, c14, 0" : "=r" (pmuseren)); + if (pmuseren & 1) { // Allows reading perfmon counters for user mode code. + asm("mrc p15, 0, %0, c9, c12, 1" : "=r" (pmcntenset)); + if (pmcntenset & 0x80000000ul) { // Is it counting? + asm("mrc p15, 0, %0, c9, c13, 0" : "=r" (pmccntr)); + // The counter is set up to count every 64th cycle + return static_cast<int64>(pmccntr) * 64; // Should optimize to << 6 + } + } +#endif + struct timeval tv; + gettimeofday(&tv, NULL); + return static_cast<int64>(tv.tv_sec) * 1000000 + tv.tv_usec; #else - // We could define __alpha here as well, but it only has a 32-bit - // timer (good for like 4 seconds), which isn't very useful. +// The soft failover to a generic implementation is automatic only for ARM. +// For other platforms the developer is expected to make an attempt to create +// a fast implementation and use generic version if nothing better is available. #error You need to define CycleTimer for your O/S and CPU #endif } diff --git a/third_party/tcmalloc/vendor/src/base/dynamic_annotations.c b/third_party/tcmalloc/vendor/src/base/dynamic_annotations.c index cdefaa7..1005f90 100644 --- a/third_party/tcmalloc/vendor/src/base/dynamic_annotations.c +++ b/third_party/tcmalloc/vendor/src/base/dynamic_annotations.c @@ -35,6 +35,7 @@ # error "This file should be built as pure C to avoid name mangling" #endif +#include "config.h" #include <stdlib.h> #include <string.h> @@ -127,22 +128,49 @@ static int GetRunningOnValgrind(void) { #ifdef RUNNING_ON_VALGRIND if (RUNNING_ON_VALGRIND) return 1; #endif - // TODO(csilvers): use GetenvBeforeMain() instead? Will need to - // change it to be extern "C". +#ifdef _MSC_VER + /* Visual Studio can complain about getenv, so use a windows equivalent. */ + char value[100] = "1"; /* something that is not "0" */ + int res = GetEnvironmentVariableA("RUNNING_ON_VALGRIND", + value, sizeof(value)); + /* value will remain "1" if the called failed for some reason. */ + return (res > 0 && strcmp(value, "0") != 0); +#else + /* TODO(csilvers): use GetenvBeforeMain() instead? Will need to + * change it to be extern "C". + */ char *running_on_valgrind_str = getenv("RUNNING_ON_VALGRIND"); if (running_on_valgrind_str) { return strcmp(running_on_valgrind_str, "0") != 0; } return 0; +#endif } /* See the comments in dynamic_annotations.h */ int RunningOnValgrind(void) { static volatile int running_on_valgrind = -1; + int local_running_on_valgrind = running_on_valgrind; /* C doesn't have thread-safe initialization of statics, and we don't want to depend on pthread_once here, so hack it. */ - int local_running_on_valgrind = running_on_valgrind; + ANNOTATE_BENIGN_RACE(&running_on_valgrind, "safe hack"); if (local_running_on_valgrind == -1) running_on_valgrind = local_running_on_valgrind = GetRunningOnValgrind(); return local_running_on_valgrind; } + +/* See the comments in dynamic_annotations.h */ +double ValgrindSlowdown(void) { + /* Same initialization hack as in RunningOnValgrind(). */ + static volatile double slowdown = 0.0; + double local_slowdown = slowdown; + ANNOTATE_BENIGN_RACE(&slowdown, "safe hack"); + if (RunningOnValgrind() == 0) { + return 1.0; + } + if (local_slowdown == 0.0) { + char *env = getenv("VALGRIND_SLOWDOWN"); + slowdown = local_slowdown = env ? atof(env) : 50.0; + } + return local_slowdown; +} diff --git a/third_party/tcmalloc/vendor/src/base/dynamic_annotations.h b/third_party/tcmalloc/vendor/src/base/dynamic_annotations.h index dae1a14..811bb5e 100644 --- a/third_party/tcmalloc/vendor/src/base/dynamic_annotations.h +++ b/third_party/tcmalloc/vendor/src/base/dynamic_annotations.h @@ -370,6 +370,41 @@ #endif /* DYNAMIC_ANNOTATIONS_ENABLED */ +/* Macro definitions for GCC attributes that allow static thread safety + analysis to recognize and use some of the dynamic annotations as + escape hatches. + TODO(lcwu): remove the check for __SUPPORT_DYN_ANNOTATION__ once the + default crosstool/GCC supports these GCC attributes. */ + +#define ANNOTALYSIS_STATIC_INLINE +#define ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY ; + +#if defined(__GNUC__) && defined(__SUPPORT_TS_ANNOTATION__) \ + && (!defined(SWIG)) && defined(__SUPPORT_DYN_ANNOTATION__) + +#if DYNAMIC_ANNOTATIONS_ENABLED == 0 +#define ANNOTALYSIS_ONLY 1 +#undef ANNOTALYSIS_STATIC_INLINE +#define ANNOTALYSIS_STATIC_INLINE static inline +#undef ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY +#define ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY { (void)file; (void)line; } +#endif +#define ANNOTALYSIS_IGNORE_READS_BEGIN __attribute__ ((ignore_reads_begin)) +#define ANNOTALYSIS_IGNORE_READS_END __attribute__ ((ignore_reads_end)) +#define ANNOTALYSIS_IGNORE_WRITES_BEGIN __attribute__ ((ignore_writes_begin)) +#define ANNOTALYSIS_IGNORE_WRITES_END __attribute__ ((ignore_writes_end)) +#define ANNOTALYSIS_UNPROTECTED_READ __attribute__ ((unprotected_read)) + +#else + +#define ANNOTALYSIS_IGNORE_READS_BEGIN +#define ANNOTALYSIS_IGNORE_READS_END +#define ANNOTALYSIS_IGNORE_WRITES_BEGIN +#define ANNOTALYSIS_IGNORE_WRITES_END +#define ANNOTALYSIS_UNPROTECTED_READ + +#endif + /* Use the macros above rather than using these functions directly. */ #ifdef __cplusplus extern "C" { @@ -431,10 +466,18 @@ void AnnotateTraceMemory(const char *file, int line, const volatile void *arg); void AnnotateThreadName(const char *file, int line, const char *name); -void AnnotateIgnoreReadsBegin(const char *file, int line); -void AnnotateIgnoreReadsEnd(const char *file, int line); -void AnnotateIgnoreWritesBegin(const char *file, int line); -void AnnotateIgnoreWritesEnd(const char *file, int line); +ANNOTALYSIS_STATIC_INLINE +void AnnotateIgnoreReadsBegin(const char *file, int line) + ANNOTALYSIS_IGNORE_READS_BEGIN ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY +ANNOTALYSIS_STATIC_INLINE +void AnnotateIgnoreReadsEnd(const char *file, int line) + ANNOTALYSIS_IGNORE_READS_END ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY +ANNOTALYSIS_STATIC_INLINE +void AnnotateIgnoreWritesBegin(const char *file, int line) + ANNOTALYSIS_IGNORE_WRITES_BEGIN ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY +ANNOTALYSIS_STATIC_INLINE +void AnnotateIgnoreWritesEnd(const char *file, int line) + ANNOTALYSIS_IGNORE_WRITES_END ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY void AnnotateEnableRaceDetection(const char *file, int line, int enable); void AnnotateNoOp(const char *file, int line, const volatile void *arg); @@ -457,6 +500,19 @@ void AnnotateFlushState(const char *file, int line); */ int RunningOnValgrind(void); +/* ValgrindSlowdown returns: + * 1.0, if (RunningOnValgrind() == 0) + * 50.0, if (RunningOnValgrind() != 0 && getenv("VALGRIND_SLOWDOWN") == NULL) + * atof(getenv("VALGRIND_SLOWDOWN")) otherwise + This function can be used to scale timeout values: + EXAMPLE: + for (;;) { + DoExpensiveBackgroundTask(); + SleepForSeconds(5 * ValgrindSlowdown()); + } + */ +double ValgrindSlowdown(void); + #ifdef __cplusplus } #endif @@ -472,7 +528,8 @@ int RunningOnValgrind(void); one can use ... = ANNOTATE_UNPROTECTED_READ(x); */ template <class T> - inline T ANNOTATE_UNPROTECTED_READ(const volatile T &x) { + inline T ANNOTATE_UNPROTECTED_READ(const volatile T &x) + ANNOTALYSIS_UNPROTECTED_READ { ANNOTATE_IGNORE_READS_BEGIN(); T res = x; ANNOTATE_IGNORE_READS_END(); @@ -498,4 +555,67 @@ int RunningOnValgrind(void); #endif /* DYNAMIC_ANNOTATIONS_ENABLED */ +/* Annotalysis, a GCC based static analyzer, is able to understand and use + some of the dynamic annotations defined in this file. However, dynamic + annotations are usually disabled in the opt mode (to avoid additional + runtime overheads) while Annotalysis only works in the opt mode. + In order for Annotalysis to use these dynamic annotations when they + are disabled, we re-define these annotations here. Note that unlike the + original macro definitions above, these macros are expanded to calls to + static inline functions so that the compiler will be able to remove the + calls after the analysis. */ + +#ifdef ANNOTALYSIS_ONLY + + #undef ANNOTALYSIS_ONLY + + /* Undefine and re-define the macros that the static analyzer understands. */ + #undef ANNOTATE_IGNORE_READS_BEGIN + #define ANNOTATE_IGNORE_READS_BEGIN() \ + AnnotateIgnoreReadsBegin(__FILE__, __LINE__) + + #undef ANNOTATE_IGNORE_READS_END + #define ANNOTATE_IGNORE_READS_END() \ + AnnotateIgnoreReadsEnd(__FILE__, __LINE__) + + #undef ANNOTATE_IGNORE_WRITES_BEGIN + #define ANNOTATE_IGNORE_WRITES_BEGIN() \ + AnnotateIgnoreWritesBegin(__FILE__, __LINE__) + + #undef ANNOTATE_IGNORE_WRITES_END + #define ANNOTATE_IGNORE_WRITES_END() \ + AnnotateIgnoreWritesEnd(__FILE__, __LINE__) + + #undef ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN + #define ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() \ + do { \ + ANNOTATE_IGNORE_READS_BEGIN(); \ + ANNOTATE_IGNORE_WRITES_BEGIN(); \ + }while(0) \ + + #undef ANNOTATE_IGNORE_READS_AND_WRITES_END + #define ANNOTATE_IGNORE_READS_AND_WRITES_END() \ + do { \ + ANNOTATE_IGNORE_WRITES_END(); \ + ANNOTATE_IGNORE_READS_END(); \ + }while(0) \ + + #if defined(__cplusplus) + #undef ANNOTATE_UNPROTECTED_READ + template <class T> + inline T ANNOTATE_UNPROTECTED_READ(const volatile T &x) + __attribute__ ((unprotected_read)) { + ANNOTATE_IGNORE_READS_BEGIN(); + T res = x; + ANNOTATE_IGNORE_READS_END(); + return res; + } + #endif /* __cplusplus */ + +#endif /* ANNOTALYSIS_ONLY */ + +/* Undefine the macros intended only in this file. */ +#undef ANNOTALYSIS_STATIC_INLINE +#undef ANNOTALYSIS_SEMICOLON_OR_EMPTY_BODY + #endif /* BASE_DYNAMIC_ANNOTATIONS_H_ */ diff --git a/third_party/tcmalloc/vendor/src/base/linux_syscall_support.h b/third_party/tcmalloc/vendor/src/base/linux_syscall_support.h index 512805b..79beafa 100644 --- a/third_party/tcmalloc/vendor/src/base/linux_syscall_support.h +++ b/third_party/tcmalloc/vendor/src/base/linux_syscall_support.h @@ -2593,7 +2593,7 @@ struct kernel_statfs { LSS_INLINE _syscall6(void*, mmap2, void*, s, size_t, l, int, p, int, f, int, d, - __off64_t, o) + off_t, o) #endif LSS_INLINE _syscall3(int, _sigaction, int, s, const struct kernel_old_sigaction*, a, diff --git a/third_party/tcmalloc/vendor/src/base/logging.h b/third_party/tcmalloc/vendor/src/base/logging.h index 6aa5c3f..70491ba 100644 --- a/third_party/tcmalloc/vendor/src/base/logging.h +++ b/third_party/tcmalloc/vendor/src/base/logging.h @@ -49,16 +49,26 @@ // On some systems (like freebsd), we can't call write() at all in a // global constructor, perhaps because errno hasn't been set up. +// (In windows, we can't call it because it might call malloc.) // Calling the write syscall is safer (it doesn't set errno), so we // prefer that. Note we don't care about errno for logging: we just // do logging on a best-effort basis. -#ifdef HAVE_SYS_SYSCALL_H +#if defined(_MSC_VER) +#define WRITE_TO_STDERR(buf, len) WriteToStderr(buf, len); // in port.cc +#elif defined(HAVE_SYS_SYSCALL_H) #include <sys/syscall.h> #define WRITE_TO_STDERR(buf, len) syscall(SYS_write, STDERR_FILENO, buf, len) #else #define WRITE_TO_STDERR(buf, len) write(STDERR_FILENO, buf, len) #endif +// MSVC and mingw define their own, safe version of vnsprintf (the +// windows one in broken) in port.cc. Everyone else can use the +// version here. We had to give it a unique name for windows. +#ifndef _WIN32 +# define perftools_vsnprintf vsnprintf +#endif + // We log all messages at this log-level and below. // INFO == -1, WARNING == -2, ERROR == -3, FATAL == -4 @@ -188,7 +198,7 @@ inline void LogPrintf(int severity, const char* pat, va_list ap) { // We write directly to the stderr file descriptor and avoid FILE // buffering because that may invoke malloc() char buf[600]; - vsnprintf(buf, sizeof(buf)-1, pat, ap); + perftools_vsnprintf(buf, sizeof(buf)-1, pat, ap); if (buf[0] != '\0' && buf[strlen(buf)-1] != '\n') { assert(strlen(buf)+1 < sizeof(buf)); strcat(buf, "\n"); @@ -230,6 +240,9 @@ inline void LOG_IF(int lvl, bool cond, const char* pat, ...) { // Like other "raw" routines, these functions are best effort, and // thus don't return error codes (except RawOpenForWriting()). #if defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) +#ifndef NOMINMAX +#define NOMINMAX // @#!$& windows +#endif #include <windows.h> typedef HANDLE RawFD; const RawFD kIllegalRawFD = INVALID_HANDLE_VALUE; diff --git a/third_party/tcmalloc/vendor/src/base/low_level_alloc.cc b/third_party/tcmalloc/vendor/src/base/low_level_alloc.cc index 7ca3953a..532c594 100644 --- a/third_party/tcmalloc/vendor/src/base/low_level_alloc.cc +++ b/third_party/tcmalloc/vendor/src/base/low_level_alloc.cc @@ -59,7 +59,9 @@ // --------------------------------------------------------------------------- static const int kMaxLevel = 30; -namespace { +// We put this class-only struct in a namespace to avoid polluting the +// global namespace with this struct name (thus risking an ODR violation). +namespace low_level_alloc_internal { // This struct describes one allocated block, or one free block. struct AllocList { struct Header { @@ -79,6 +81,8 @@ namespace { // LLA_SkiplistLevels() }; } +using low_level_alloc_internal::AllocList; + // --------------------------------------------------------------------------- // A trivial skiplist implementation. This is used to keep the freelist @@ -208,7 +212,7 @@ static const intptr_t kMagicAllocated = 0x4c833e95; static const intptr_t kMagicUnallocated = ~kMagicAllocated; namespace { - class ArenaLock { + class SCOPED_LOCKABLE ArenaLock { public: explicit ArenaLock(LowLevelAlloc::Arena *arena) EXCLUSIVE_LOCK_FUNCTION(arena->mu) @@ -229,7 +233,7 @@ namespace { this->arena_->mu.Lock(); } ~ArenaLock() { RAW_CHECK(this->left_, "haven't left Arena region"); } - void Leave() UNLOCK_FUNCTION(arena_->mu) { + void Leave() UNLOCK_FUNCTION() { this->arena_->mu.Unlock(); #if 0 if (this->mask_valid_) { diff --git a/third_party/tcmalloc/vendor/src/base/spinlock.cc b/third_party/tcmalloc/vendor/src/base/spinlock.cc index 48cdc89..1413923 100644 --- a/third_party/tcmalloc/vendor/src/base/spinlock.cc +++ b/third_party/tcmalloc/vendor/src/base/spinlock.cc @@ -32,47 +32,28 @@ */ #include <config.h> -#include <time.h> /* For nanosleep() */ -#ifdef HAVE_SCHED_H -#include <sched.h> /* For sched_yield() */ -#endif -#ifdef HAVE_UNISTD_H -#include <unistd.h> /* For read() */ -#endif -#include <fcntl.h> /* for open(), O_RDONLY */ -#include <string.h> /* for strncmp */ -#include <errno.h> #include "base/spinlock.h" +#include "base/synchronization_profiling.h" +#include "base/spinlock_internal.h" #include "base/cycleclock.h" #include "base/sysinfo.h" /* for NumCPUs() */ -// We can do contention-profiling of SpinLocks, but the code is in -// mutex.cc, which is not always linked in with spinlock. Hence we -// provide this weak definition, which is used if mutex.cc isn't linked in. -ATTRIBUTE_WEAK extern void SubmitSpinLockProfileData(const void *, int64); -void SubmitSpinLockProfileData(const void *, int64) {} +// NOTE on the Lock-state values: +// +// kSpinLockFree represents the unlocked state +// kSpinLockHeld represents the locked state with no waiters +// +// Values greater than kSpinLockHeld represent the locked state with waiters, +// where the value is the time the current lock holder had to +// wait before obtaining the lock. The kSpinLockSleeper state is a special +// "locked with waiters" state that indicates that a sleeper needs to +// be woken, but the thread that just released the lock didn't wait. static int adaptive_spin_count = 0; const base::LinkerInitialized SpinLock::LINKER_INITIALIZED = base::LINKER_INITIALIZED; -// The OS-specific header included below must provide two calls: -// Wait until *w becomes zero, atomically set it to 1 and return. -// static void SpinLockWait(volatile Atomic32 *w); -// -// Hint that a thread waiting in SpinLockWait() could now make progress. May -// do nothing. This call may not read or write *w; it must use only the -// address. -// static void SpinLockWake(volatile Atomic32 *w); -#if defined(_WIN32) -#include "base/spinlock_win32-inl.h" -#elif defined(__linux__) -#include "base/spinlock_linux-inl.h" -#else -#include "base/spinlock_posix-inl.h" -#endif - namespace { struct SpinLock_InitHelper { SpinLock_InitHelper() { @@ -91,36 +72,111 @@ static SpinLock_InitHelper init_helper; } // unnamed namespace +// Monitor the lock to see if its value changes within some time period +// (adaptive_spin_count loop iterations). A timestamp indicating +// when the thread initially started waiting for the lock is passed in via +// the initial_wait_timestamp value. The total wait time in cycles for the +// lock is returned in the wait_cycles parameter. The last value read +// from the lock is returned from the method. +Atomic32 SpinLock::SpinLoop(int64 initial_wait_timestamp, + Atomic32* wait_cycles) { + int c = adaptive_spin_count; + while (base::subtle::NoBarrier_Load(&lockword_) != kSpinLockFree && --c > 0) { + } + Atomic32 spin_loop_wait_cycles = CalculateWaitCycles(initial_wait_timestamp); + Atomic32 lock_value = + base::subtle::Acquire_CompareAndSwap(&lockword_, kSpinLockFree, + spin_loop_wait_cycles); + *wait_cycles = spin_loop_wait_cycles; + return lock_value; +} void SpinLock::SlowLock() { - int c = adaptive_spin_count; + // The lock was not obtained initially, so this thread needs to wait for + // it. Record the current timestamp in the local variable wait_start_time + // so the total wait time can be stored in the lockword once this thread + // obtains the lock. + int64 wait_start_time = CycleClock::Now(); + Atomic32 wait_cycles; + Atomic32 lock_value = SpinLoop(wait_start_time, &wait_cycles); - // Spin a few times in the hope that the lock holder releases the lock - while ((c > 0) && (lockword_ != 0)) { - c--; - } + int lock_wait_call_count = 0; + while (lock_value != kSpinLockFree) { + // If the lock is currently held, but not marked as having a sleeper, mark + // it as having a sleeper. + if (lock_value == kSpinLockHeld) { + // Here, just "mark" that the thread is going to sleep. Don't store the + // lock wait time in the lock as that will cause the current lock + // owner to think it experienced contention. + lock_value = base::subtle::Acquire_CompareAndSwap(&lockword_, + kSpinLockHeld, + kSpinLockSleeper); + if (lock_value == kSpinLockHeld) { + // Successfully transitioned to kSpinLockSleeper. Pass + // kSpinLockSleeper to the SpinLockWait routine to properly indicate + // the last lock_value observed. + lock_value = kSpinLockSleeper; + } else if (lock_value == kSpinLockFree) { + // Lock is free again, so try and aquire it before sleeping. The + // new lock state will be the number of cycles this thread waited if + // this thread obtains the lock. + lock_value = base::subtle::Acquire_CompareAndSwap(&lockword_, + kSpinLockFree, + wait_cycles); + continue; // skip the delay at the end of the loop + } + } - if (lockword_ == 1) { - int32 now = (CycleClock::Now() >> PROFILE_TIMESTAMP_SHIFT); - // Don't loose the lock: make absolutely sure "now" is not zero - now |= 1; - // Atomically replace the value of lockword_ with "now" if - // lockword_ is 1, thereby remembering the first timestamp to - // be recorded. - base::subtle::NoBarrier_CompareAndSwap(&lockword_, 1, now); - // base::subtle::NoBarrier_CompareAndSwap() returns: - // 0: the lock is/was available; nothing stored - // 1: our timestamp was stored - // > 1: an older timestamp is already in lockword_; nothing stored + // Wait for an OS specific delay. + base::internal::SpinLockDelay(&lockword_, lock_value, + ++lock_wait_call_count); + // Spin again after returning from the wait routine to give this thread + // some chance of obtaining the lock. + lock_value = SpinLoop(wait_start_time, &wait_cycles); } - - SpinLockWait(&lockword_); // wait until lock acquired; OS specific } -void SpinLock::SlowUnlock(int64 wait_timestamp) { - SpinLockWake(&lockword_); // wake waiter if necessary; OS specific +// The wait time for contentionz lock profiling must fit into 32 bits. +// However, the lower 32-bits of the cycle counter wrap around too quickly +// with high frequency processors, so a right-shift by 7 is performed to +// quickly divide the cycles by 128. Using these 32 bits, reduces the +// granularity of time measurement to 128 cycles, and loses track +// of wait time for waits greater than 109 seconds on a 5 GHz machine +// [(2^32 cycles/5 Ghz)*128 = 109.95 seconds]. Waits this long should be +// very rare and the reduced granularity should not be an issue given +// processors in the Google fleet operate at a minimum of one billion +// cycles/sec. +enum { PROFILE_TIMESTAMP_SHIFT = 7 }; + +void SpinLock::SlowUnlock(uint64 wait_cycles) { + base::internal::SpinLockWake(&lockword_, false); // wake waiter if necessary + + // Collect contentionz profile info, expanding the wait_cycles back out to + // the full value. If wait_cycles is <= kSpinLockSleeper, then no wait + // was actually performed, so don't record the wait time. Note, that the + // CalculateWaitCycles method adds in kSpinLockSleeper cycles + // unconditionally to guarantee the wait time is not kSpinLockFree or + // kSpinLockHeld. The adding in of these small number of cycles may + // overestimate the contention by a slight amount 50% of the time. However, + // if this code tried to correct for that addition by subtracting out the + // kSpinLockSleeper amount that would underestimate the contention slightly + // 50% of the time. Both ways get the wrong answer, so the code + // overestimates to be more conservative. Overestimating also makes the code + // a little simpler. + // + if (wait_cycles > kSpinLockSleeper) { + base::SubmitSpinLockProfileData(this, + wait_cycles << PROFILE_TIMESTAMP_SHIFT); + } +} - // Collect contentionz profile info. Subtract one from wait_timestamp as - // antidote to "now |= 1;" in SlowLock(). - SubmitSpinLockProfileData(this, wait_timestamp - 1); +inline int32 SpinLock::CalculateWaitCycles(int64 wait_start_time) { + int32 wait_cycles = ((CycleClock::Now() - wait_start_time) >> + PROFILE_TIMESTAMP_SHIFT); + // The number of cycles waiting for the lock is used as both the + // wait_cycles and lock value, so it can't be kSpinLockFree or + // kSpinLockHeld. Make sure the value returned is at least + // kSpinLockSleeper. + wait_cycles |= kSpinLockSleeper; + return wait_cycles; } diff --git a/third_party/tcmalloc/vendor/src/base/spinlock.h b/third_party/tcmalloc/vendor/src/base/spinlock.h index 9e633c4..c2be4fd 100644 --- a/third_party/tcmalloc/vendor/src/base/spinlock.h +++ b/third_party/tcmalloc/vendor/src/base/spinlock.h @@ -44,14 +44,14 @@ #define BASE_SPINLOCK_H_ #include <config.h> -#include "base/basictypes.h" #include "base/atomicops.h" +#include "base/basictypes.h" #include "base/dynamic_annotations.h" #include "base/thread_annotations.h" class LOCKABLE SpinLock { public: - SpinLock() : lockword_(0) { } + SpinLock() : lockword_(kSpinLockFree) { } // Special constructor for use with static SpinLock objects. E.g., // @@ -70,18 +70,21 @@ class LOCKABLE SpinLock { // TODO(csilvers): uncomment the annotation when we figure out how to // support this macro with 0 args (see thread_annotations.h) inline void Lock() /*EXCLUSIVE_LOCK_FUNCTION()*/ { - if (Acquire_CompareAndSwap(&lockword_, 0, 1) != 0) { + if (base::subtle::Acquire_CompareAndSwap(&lockword_, kSpinLockFree, + kSpinLockHeld) != kSpinLockFree) { SlowLock(); } ANNOTATE_RWLOCK_ACQUIRED(this, 1); } - // Acquire this SpinLock and return true if the acquisition can be - // done without blocking, else return false. If this SpinLock is - // free at the time of the call, TryLock will return true with high - // probability. + // Try to acquire this SpinLock without blocking and return true if the + // acquisition was successful. If the lock was not acquired, false is + // returned. If this SpinLock is free at the time of the call, TryLock + // will return true with high probability. inline bool TryLock() EXCLUSIVE_TRYLOCK_FUNCTION(true) { - bool res = (Acquire_CompareAndSwap(&lockword_, 0, 1) == 0); + bool res = + (base::subtle::Acquire_CompareAndSwap(&lockword_, kSpinLockFree, + kSpinLockHeld) == kSpinLockFree); if (res) { ANNOTATE_RWLOCK_ACQUIRED(this, 1); } @@ -92,47 +95,37 @@ class LOCKABLE SpinLock { // TODO(csilvers): uncomment the annotation when we figure out how to // support this macro with 0 args (see thread_annotations.h) inline void Unlock() /*UNLOCK_FUNCTION()*/ { - // This is defined in mutex.cc. - extern void SubmitSpinLockProfileData(const void *, int64); - - int64 wait_timestamp = static_cast<uint32>(lockword_); + uint64 wait_cycles = + static_cast<uint64>(base::subtle::NoBarrier_Load(&lockword_)); ANNOTATE_RWLOCK_RELEASED(this, 1); - Release_Store(&lockword_, 0); - if (wait_timestamp != 1) { + base::subtle::Release_Store(&lockword_, kSpinLockFree); + if (wait_cycles != kSpinLockHeld) { // Collect contentionz profile info, and speed the wakeup of any waiter. - // The lockword_ value indicates when the waiter started waiting. - SlowUnlock(wait_timestamp); + // The wait_cycles value indicates how long this thread spent waiting + // for the lock. + SlowUnlock(wait_cycles); } } - // Report if we think the lock can be held by this thread. - // When the lock is truly held by the invoking thread - // we will always return true. - // Indended to be used as CHECK(lock.IsHeld()); + // Determine if the lock is held. When the lock is held by the invoking + // thread, true will always be returned. Intended to be used as + // CHECK(lock.IsHeld()). inline bool IsHeld() const { - return lockword_ != 0; + return base::subtle::NoBarrier_Load(&lockword_) != kSpinLockFree; } - // The timestamp for contention lock profiling must fit into 31 bits. - // as lockword_ is 32 bits and we loose an additional low-order bit due - // to the statement "now |= 1" in SlowLock(). - // To select 31 bits from the 64-bit cycle counter, we shift right by - // PROFILE_TIMESTAMP_SHIFT = 7. - // Using these 31 bits, we reduce granularity of time measurement to - // 256 cycles, and will loose track of wait time for waits greater than - // 109 seconds on a 5 GHz machine, longer for faster clock cycles. - // Waits this long should be very rare. - enum { PROFILE_TIMESTAMP_SHIFT = 7 }; - static const base::LinkerInitialized LINKER_INITIALIZED; // backwards compat private: - // Lock-state: 0 means unlocked; 1 means locked with no waiters; values - // greater than 1 indicate locked with waiters, where the value is the time - // the first waiter started waiting and is used for contention profiling. + enum { kSpinLockFree = 0 }; + enum { kSpinLockHeld = 1 }; + enum { kSpinLockSleeper = 2 }; + volatile Atomic32 lockword_; void SlowLock(); - void SlowUnlock(int64 wait_timestamp); + void SlowUnlock(uint64 wait_cycles); + Atomic32 SpinLoop(int64 initial_wait_timestamp, Atomic32* wait_cycles); + inline int32 CalculateWaitCycles(int64 wait_start_time); DISALLOW_COPY_AND_ASSIGN(SpinLock); }; diff --git a/third_party/tcmalloc/vendor/src/base/spinlock_internal.cc b/third_party/tcmalloc/vendor/src/base/spinlock_internal.cc new file mode 100644 index 0000000..b5b6ca4 --- /dev/null +++ b/third_party/tcmalloc/vendor/src/base/spinlock_internal.cc @@ -0,0 +1,77 @@ +/* Copyright (c) 2010, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// The OS-specific header included below must provide two calls: +// base::internal::SpinLockDelay() and base::internal::SpinLockWake(). +// See spinlock_internal.h for the spec of SpinLockWake(). + +// void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) +// SpinLockDelay() generates an apprproate spin delay on iteration "loop" of a +// spin loop on location *w, whose previously observed value was "value". +// SpinLockDelay() may do nothing, may yield the CPU, may sleep a clock tick, +// or may wait for a delay that can be truncated by a call to SpinlockWake(w). +// In all cases, it must return in bounded time even if SpinlockWake() is not +// called. + +#include "base/spinlock_internal.h" + +#if defined(_WIN32) +#include "base/spinlock_win32-inl.h" +#elif defined(__linux__) +#include "base/spinlock_linux-inl.h" +#else +#include "base/spinlock_posix-inl.h" +#endif + +namespace base { +namespace internal { + +// See spinlock_internal.h for spec. +int32 SpinLockWait(volatile Atomic32 *w, int n, + const SpinLockWaitTransition trans[]) { + int32 v; + bool done = false; + for (int loop = 0; !done; loop++) { + v = base::subtle::Acquire_Load(w); + int i; + for (i = 0; i != n && v != trans[i].from; i++) { + } + if (i == n) { + SpinLockDelay(w, v, loop); // no matching transition + } else if (trans[i].to == v || // null transition + base::subtle::Acquire_CompareAndSwap(w, v, trans[i].to) == v) { + done = trans[i].done; + } + } + return v; +} + +} // namespace internal +} // namespace base diff --git a/third_party/tcmalloc/vendor/src/base/spinlock_internal.h b/third_party/tcmalloc/vendor/src/base/spinlock_internal.h new file mode 100644 index 0000000..4494260 --- /dev/null +++ b/third_party/tcmalloc/vendor/src/base/spinlock_internal.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2010, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * This file is an internal part spinlock.cc and once.cc + * It may not be used directly by code outside of //base. + */ + +#ifndef BASE_SPINLOCK_INTERNAL_H_ +#define BASE_SPINLOCK_INTERNAL_H_ + +#include <config.h> +#include "base/basictypes.h" +#include "base/atomicops.h" + +namespace base { +namespace internal { + +// SpinLockWait() waits until it can perform one of several transitions from +// "from" to "to". It returns when it performs a transition where done==true. +struct SpinLockWaitTransition { + int32 from; + int32 to; + bool done; +}; + +// Wait until *w can transition from trans[i].from to trans[i].to for some i +// satisfying 0<=i<n && trans[i].done, atomically make the transition, +// then return the old value of *w. Make any other atomic tranistions +// where !trans[i].done, but continue waiting. +int32 SpinLockWait(volatile Atomic32 *w, int n, + const SpinLockWaitTransition trans[]); +void SpinLockWake(volatile Atomic32 *w, bool all); +void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop); + +} // namespace internal +} // namespace base +#endif diff --git a/third_party/tcmalloc/vendor/src/base/spinlock_linux-inl.h b/third_party/tcmalloc/vendor/src/base/spinlock_linux-inl.h index 0df09a3..a178dd5 100644 --- a/third_party/tcmalloc/vendor/src/base/spinlock_linux-inl.h +++ b/third_party/tcmalloc/vendor/src/base/spinlock_linux-inl.h @@ -28,11 +28,12 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * --- - * This file is a Linux-specific part of spinlock.cc + * This file is a Linux-specific part of spinlock_internal.cc */ #include <sched.h> #include <time.h> +#include <limits.h> #include "base/linux_syscall_support.h" #define FUTEX_WAIT 0 @@ -48,44 +49,54 @@ static struct InitModule { int x = 0; // futexes are ints, so we can use them only when // that's the same size as the lockword_ in SpinLock. - have_futex = (sizeof (Atomic32) == sizeof (int) && +#ifdef __arm__ + // ARM linux doesn't support sys_futex1(void*, int, int, struct timespec*); + have_futex = 0; +#else + have_futex = (sizeof (Atomic32) == sizeof (int) && sys_futex(&x, FUTEX_WAKE, 1, 0) >= 0); +#endif if (have_futex && sys_futex(&x, FUTEX_WAKE | futex_private_flag, 1, 0) < 0) { futex_private_flag = 0; } } } init_module; + } // anonymous namespace -static void SpinLockWait(volatile Atomic32 *w) { - int save_errno = errno; - struct timespec tm; - tm.tv_sec = 0; - if (have_futex) { - int value; - tm.tv_nsec = 1000000; // 1ms; really we're trying to sleep for one kernel - // clock tick - while ((value = base::subtle::Acquire_CompareAndSwap(w, 0, 1)) != 0) { - sys_futex(reinterpret_cast<int *>(const_cast<Atomic32 *>(w)), - FUTEX_WAIT | futex_private_flag, - value, reinterpret_cast<struct kernel_timespec *>(&tm)); - } - } else { - tm.tv_nsec = 2000001; // above 2ms so linux 2.4 doesn't spin - if (base::subtle::NoBarrier_Load(w) != 0) { - sched_yield(); + +namespace base { +namespace internal { + +void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { + if (loop != 0) { + int save_errno = errno; + struct timespec tm; + tm.tv_sec = 0; + if (have_futex) { + tm.tv_nsec = 1000000; // 1ms; really we're trying to sleep for one + // kernel clock tick + } else { + tm.tv_nsec = 2000001; // above 2ms so linux 2.4 doesn't spin } - while (base::subtle::Acquire_CompareAndSwap(w, 0, 1) != 0) { + if (have_futex) { + sys_futex(reinterpret_cast<int *>(const_cast<Atomic32 *>(w)), + FUTEX_WAIT | futex_private_flag, + value, reinterpret_cast<struct kernel_timespec *>(&tm)); + } else { nanosleep(&tm, NULL); } + errno = save_errno; } - errno = save_errno; } -static void SpinLockWake(volatile Atomic32 *w) { +void SpinLockWake(volatile Atomic32 *w, bool all) { if (have_futex) { sys_futex(reinterpret_cast<int *>(const_cast<Atomic32 *>(w)), - FUTEX_WAKE | futex_private_flag, 1, 0); + FUTEX_WAKE | futex_private_flag, all? INT_MAX : 1, 0); } } + +} // namespace internal +} // namespace base diff --git a/third_party/tcmalloc/vendor/src/base/spinlock_posix-inl.h b/third_party/tcmalloc/vendor/src/base/spinlock_posix-inl.h index 0d933c0..d188ebd 100644 --- a/third_party/tcmalloc/vendor/src/base/spinlock_posix-inl.h +++ b/third_party/tcmalloc/vendor/src/base/spinlock_posix-inl.h @@ -28,25 +28,35 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * --- - * This file is a Posix-specific part of spinlock.cc + * This file is a Posix-specific part of spinlock_internal.cc */ -#include <sched.h> -#include <time.h> +#include <config.h> +#include <errno.h> +#ifdef HAVE_SCHED_H +#include <sched.h> /* For sched_yield() */ +#endif +#include <time.h> /* For nanosleep() */ -static void SpinLockWait(volatile Atomic32 *w) { +namespace base { +namespace internal { + +void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { int save_errno = errno; - struct timespec tm; - tm.tv_sec = 0; - tm.tv_nsec = 1000000; - if (base::subtle::NoBarrier_Load(w) != 0) { + if (loop == 0) { + } else if (loop == 1) { sched_yield(); - } - while (base::subtle::Acquire_CompareAndSwap(w, 0, 1) != 0) { + } else { + struct timespec tm; + tm.tv_sec = 0; + tm.tv_nsec = 1000000; nanosleep(&tm, NULL); } errno = save_errno; } -static void SpinLockWake(volatile Atomic32 *w) { +void SpinLockWake(volatile Atomic32 *w, bool all) { } + +} // namespace internal +} // namespace base diff --git a/third_party/tcmalloc/vendor/src/base/spinlock_win32-inl.h b/third_party/tcmalloc/vendor/src/base/spinlock_win32-inl.h index 9058939..ee23541 100644 --- a/third_party/tcmalloc/vendor/src/base/spinlock_win32-inl.h +++ b/third_party/tcmalloc/vendor/src/base/spinlock_win32-inl.h @@ -28,20 +28,26 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * --- - * This file is a Win32-specific part of spinlock.cc + * This file is a Win32-specific part of spinlock_internal.cc */ #include <windows.h> -static void SpinLockWait(volatile Atomic32 *w) { - if (base::subtle::NoBarrier_Load(w) != 0) { +namespace base { +namespace internal { + +void SpinLockDelay(volatile Atomic32 *w, int32 value, int loop) { + if (loop == 0) { + } else if (loop == 1) { Sleep(0); - } - while (base::subtle::Acquire_CompareAndSwap(w, 0, 1) != 0) { + } else { Sleep(1); } } -static void SpinLockWake(volatile Atomic32 *w) { +void SpinLockWake(volatile Atomic32 *w, bool all) { } + +} // namespace internal +} // namespace base diff --git a/third_party/tcmalloc/vendor/src/base/stl_allocator.h b/third_party/tcmalloc/vendor/src/base/stl_allocator.h index b0ddc68..3152cf9 100644 --- a/third_party/tcmalloc/vendor/src/base/stl_allocator.h +++ b/third_party/tcmalloc/vendor/src/base/stl_allocator.h @@ -37,15 +37,15 @@ #include <config.h> +#include <stddef.h> // for ptrdiff_t #include <limits> -#include "base/basictypes.h" #include "base/logging.h" // Generic allocator class for STL objects // that uses a given type-less allocator Alloc, which must provide: // static void* Alloc::Allocate(size_t size); -// static void Alloc::Free(void* ptr); +// static void Alloc::Free(void* ptr, size_t size); // // STL_Allocator<T, MyAlloc> provides the same thread-safety // guarantees as MyAlloc. @@ -82,7 +82,7 @@ class STL_Allocator { RAW_DCHECK((n * sizeof(T)) / sizeof(T) == n, "n is too big to allocate"); return static_cast<T*>(Alloc::Allocate(n * sizeof(T))); } - void deallocate(pointer p, size_type /*n*/) { Alloc::Free(p); } + void deallocate(pointer p, size_type n) { Alloc::Free(p, n * sizeof(T)); } size_type max_size() const { return size_t(-1) / sizeof(T); } diff --git a/third_party/tcmalloc/vendor/src/base/synchronization_profiling.h b/third_party/tcmalloc/vendor/src/base/synchronization_profiling.h new file mode 100644 index 0000000..cf02c21 --- /dev/null +++ b/third_party/tcmalloc/vendor/src/base/synchronization_profiling.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2010, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Chris Ruemmler + */ + +#ifndef BASE_AUXILIARY_SYNCHRONIZATION_PROFILING_H_ +#define BASE_AUXILIARY_SYNCHRONIZATION_PROFILING_H_ + +#include "base/basictypes.h" + +namespace base { + +// We can do contention-profiling of SpinLocks, but the code is in +// mutex.cc, which is not always linked in with spinlock. Hence we +// provide a weak definition, which are used if mutex.cc isn't linked in. + +// Submit the number of cycles the spinlock spent contending. +ATTRIBUTE_WEAK extern void SubmitSpinLockProfileData(const void *, int64); +extern void SubmitSpinLockProfileData(const void *contendedlock, + int64 wait_cycles) {} +} +#endif // BASE_AUXILIARY_SYNCHRONIZATION_PROFILING_H_ diff --git a/third_party/tcmalloc/vendor/src/base/sysinfo.cc b/third_party/tcmalloc/vendor/src/base/sysinfo.cc index 7af0495..5396743 100644 --- a/third_party/tcmalloc/vendor/src/base/sysinfo.cc +++ b/third_party/tcmalloc/vendor/src/base/sysinfo.cc @@ -56,6 +56,7 @@ #endif #include "base/sysinfo.h" #include "base/commandlineflags.h" +#include "base/dynamic_annotations.h" // for RunningOnValgrind #include "base/logging.h" #include "base/cycleclock.h" @@ -110,20 +111,23 @@ // 8K), so it's not an ideal solution. const char* GetenvBeforeMain(const char* name) { #if defined(HAVE___ENVIRON) // if we have it, it's declared in unistd.h - const int namelen = strlen(name); - for (char** p = __environ; *p; p++) { - if (!memcmp(*p, name, namelen) && (*p)[namelen] == '=') // it's a match - return *p + namelen+1; // point after = + if (__environ) { // can exist but be NULL, if statically linked + const int namelen = strlen(name); + for (char** p = __environ; *p; p++) { + if (!memcmp(*p, name, namelen) && (*p)[namelen] == '=') // it's a match + return *p + namelen+1; // point after = + } + return NULL; } - return NULL; -#elif defined(PLATFORM_WINDOWS) +#endif +#if defined(PLATFORM_WINDOWS) // TODO(mbelshe) - repeated calls to this function will overwrite the // contents of the static buffer. - static char envbuf[1024]; // enough to hold any envvar we care about - if (!GetEnvironmentVariableA(name, envbuf, sizeof(envbuf)-1)) + static char envvar_buf[1024]; // enough to hold any envvar we care about + if (!GetEnvironmentVariableA(name, envvar_buf, sizeof(envvar_buf)-1)) return NULL; - return envbuf; -#else + return envvar_buf; +#endif // static is ok because this function should only be called before // main(), when we're single-threaded. static char envbuf[16<<10]; @@ -151,7 +155,6 @@ const char* GetenvBeforeMain(const char* name) { p = endp + 1; } return NULL; // env var never found -#endif } // This takes as an argument an environment-variable name (like @@ -201,7 +204,7 @@ bool GetUniquePathFromEnv(const char* env_name, char* path) { static double cpuinfo_cycles_per_second = 1.0; // 0.0 might be dangerous static int cpuinfo_num_cpus = 1; // Conservative guess -static void SleepForMilliseconds(int milliseconds) { +void SleepForMilliseconds(int milliseconds) { #ifdef PLATFORM_WINDOWS _sleep(milliseconds); // Windows's _sleep takes milliseconds argument #else @@ -228,6 +231,29 @@ static int64 EstimateCyclesPerSecond(const int estimate_time_ms) { return guess; } +// ReadIntFromFile is only called on linux and cygwin platforms. +#if defined(__linux__) || defined(__CYGWIN__) || defined(__CYGWIN32__) +// Helper function for reading an int from a file. Returns true if successful +// and the memory location pointed to by value is set to the value read. +static bool ReadIntFromFile(const char *file, int *value) { + bool ret = false; + int fd = open(file, O_RDONLY); + if (fd != -1) { + char line[1024]; + char* err; + memset(line, '\0', sizeof(line)); + read(fd, line, sizeof(line) - 1); + const int temp_value = strtol(line, &err, 10); + if (line[0] != '\0' && (*err == '\n' || *err == '\0')) { + *value = temp_value; + ret = true; + } + close(fd); + } + return ret; +} +#endif + // WARNING: logging calls back to InitializeSystemInfo() so it must // not invoke any logging code. Also, InitializeSystemInfo() can be // called before main() -- in fact it *must* be since already_called @@ -240,31 +266,44 @@ static void InitializeSystemInfo() { if (already_called) return; already_called = true; - // I put in a never-called reference to EstimateCyclesPerSecond() here - // to silence the compiler for OS's that don't need it - if (0) EstimateCyclesPerSecond(0); + bool saw_mhz = false; + + if (RunningOnValgrind()) { + // Valgrind may slow the progress of time artificially (--scale-time=N + // option). We thus can't rely on CPU Mhz info stored in /sys or /proc + // files. Thus, actually measure the cps. + cpuinfo_cycles_per_second = EstimateCyclesPerSecond(100); + saw_mhz = true; + } #if defined(__linux__) || defined(__CYGWIN__) || defined(__CYGWIN32__) char line[1024]; char* err; + int freq; + + // If the kernel is exporting the tsc frequency use that. There are issues + // where cpuinfo_max_freq cannot be relied on because the BIOS may be + // exporintg an invalid p-state (on x86) or p-states may be used to put the + // processor in a new mode (turbo mode). Essentially, those frequencies + // cannot always be relied upon. The same reasons apply to /proc/cpuinfo as + // well. + if (!saw_mhz && + ReadIntFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz", &freq)) { + // The value is in kHz (as the file name suggests). For example, on a + // 2GHz warpstation, the file contains the value "2000000". + cpuinfo_cycles_per_second = freq * 1000.0; + saw_mhz = true; + } // If CPU scaling is in effect, we want to use the *maximum* frequency, // not whatever CPU speed some random processor happens to be using now. - bool saw_mhz = false; - const char* pname0 = "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq"; - int fd0 = open(pname0, O_RDONLY); - if (fd0 != -1) { - memset(line, '\0', sizeof(line)); - read(fd0, line, sizeof(line)); - const int max_freq = strtol(line, &err, 10); - if (line[0] != '\0' && (*err == '\n' || *err == '\0')) { - // The value is in kHz. For example, on a 2GHz machine, the file - // contains the value "2000000". Historically this file contained no - // newline, but at some point the kernel started appending a newline. - cpuinfo_cycles_per_second = max_freq * 1000.0; - saw_mhz = true; - } - close(fd0); + if (!saw_mhz && + ReadIntFromFile("/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", + &freq)) { + // The value is in kHz. For example, on a 2GHz machine, the file + // contains the value "2000000". + cpuinfo_cycles_per_second = freq * 1000.0; + saw_mhz = true; } // Read /proc/cpuinfo for other values, and if there is no cpuinfo_max_freq. @@ -272,11 +311,14 @@ static void InitializeSystemInfo() { int fd = open(pname, O_RDONLY); if (fd == -1) { perror(pname); - cpuinfo_cycles_per_second = EstimateCyclesPerSecond(1000); + if (!saw_mhz) { + cpuinfo_cycles_per_second = EstimateCyclesPerSecond(1000); + } return; // TODO: use generic tester instead? } double bogo_clock = 1.0; + bool saw_bogo = false; int num_cpus = 0; line[0] = line[1] = '\0'; int chars_read = 0; @@ -300,29 +342,38 @@ static void InitializeSystemInfo() { if (newline != NULL) *newline = '\0'; - if (!saw_mhz && strncmp(line, "cpu MHz", sizeof("cpu MHz")-1) == 0) { + // When parsing the "cpu MHz" and "bogomips" (fallback) entries, we only + // accept postive values. Some environments (virtual machines) report zero, + // which would cause infinite looping in WallTime_Init. + if (!saw_mhz && strncasecmp(line, "cpu MHz", sizeof("cpu MHz")-1) == 0) { const char* freqstr = strchr(line, ':'); if (freqstr) { cpuinfo_cycles_per_second = strtod(freqstr+1, &err) * 1000000.0; - if (freqstr[1] != '\0' && *err == '\0') + if (freqstr[1] != '\0' && *err == '\0' && cpuinfo_cycles_per_second > 0) saw_mhz = true; } - } else if (strncmp(line, "bogomips", sizeof("bogomips")-1) == 0) { + } else if (strncasecmp(line, "bogomips", sizeof("bogomips")-1) == 0) { const char* freqstr = strchr(line, ':'); - if (freqstr) + if (freqstr) { bogo_clock = strtod(freqstr+1, &err) * 1000000.0; - if (freqstr == NULL || freqstr[1] == '\0' || *err != '\0') - bogo_clock = 1.0; - } else if (strncmp(line, "processor", sizeof("processor")-1) == 0) { + if (freqstr[1] != '\0' && *err == '\0' && bogo_clock > 0) + saw_bogo = true; + } + } else if (strncasecmp(line, "processor", sizeof("processor")-1) == 0) { num_cpus++; // count up every time we see an "processor :" entry } } while (chars_read > 0); close(fd); if (!saw_mhz) { - // If we didn't find anything better, we'll use bogomips, but - // we're not happy about it. - cpuinfo_cycles_per_second = bogo_clock; + if (saw_bogo) { + // If we didn't find anything better, we'll use bogomips, but + // we're not happy about it. + cpuinfo_cycles_per_second = bogo_clock; + } else { + // If we don't even have bogomips, we'll use the slow estimation. + cpuinfo_cycles_per_second = EstimateCyclesPerSecond(1000); + } } if (cpuinfo_cycles_per_second == 0.0) { cpuinfo_cycles_per_second = 1.0; // maybe unnecessary, but safe @@ -453,7 +504,7 @@ static void ConstructFilename(const char* spec, pid_t pid, char* buf, int buf_size) { CHECK_LT(snprintf(buf, buf_size, spec, - pid ? pid : getpid()), buf_size); + static_cast<int>(pid ? pid : getpid())), buf_size); } #endif @@ -766,7 +817,8 @@ bool ProcMapsIterator::NextExt(uint64 *start, uint64 *end, char **flags, Buffer::kBufSize); } else { CHECK_LT(snprintf(object_path.buf_, Buffer::kBufSize, - "/proc/%d/path/%s", pid_, mapinfo->pr_mapname), + "/proc/%d/path/%s", + static_cast<int>(pid_), mapinfo->pr_mapname), Buffer::kBufSize); } ssize_t len = readlink(object_path.buf_, current_filename_, PATH_MAX); @@ -877,9 +929,10 @@ namespace tcmalloc { // Helper to add the list of mapped shared libraries to a profile. // Fill formatted "/proc/self/maps" contents into buffer 'buf' of size 'size' -// and return the actual size occupied in 'buf'. +// and return the actual size occupied in 'buf'. We fill wrote_all to true +// if we successfully wrote all proc lines to buf, false else. // We do not provision for 0-terminating 'buf'. -int FillProcSelfMaps(char buf[], int size) { +int FillProcSelfMaps(char buf[], int size, bool* wrote_all) { ProcMapsIterator::Buffer iterbuf; ProcMapsIterator it(0, &iterbuf); // 0 means "current pid" @@ -887,10 +940,17 @@ int FillProcSelfMaps(char buf[], int size) { int64 inode; char *flags, *filename; int bytes_written = 0; + *wrote_all = true; while (it.Next(&start, &end, &flags, &offset, &inode, &filename)) { - bytes_written += it.FormatLine(buf + bytes_written, size - bytes_written, - start, end, flags, offset, inode, filename, - 0); + const int line_length = it.FormatLine(buf + bytes_written, + size - bytes_written, + start, end, flags, offset, + inode, filename, 0); + if (line_length == 0) + *wrote_all = false; // failed to write this line out + else + bytes_written += line_length; + } return bytes_written; } diff --git a/third_party/tcmalloc/vendor/src/base/sysinfo.h b/third_party/tcmalloc/vendor/src/base/sysinfo.h index 0bcc1f5..7935855 100644 --- a/third_party/tcmalloc/vendor/src/base/sysinfo.h +++ b/third_party/tcmalloc/vendor/src/base/sysinfo.h @@ -71,6 +71,8 @@ extern bool GetUniquePathFromEnv(const char* env_name, char* path); extern int NumCPUs(); +void SleepForMilliseconds(int milliseconds); + // processor cycles per second of each processor. Thread-safe. extern double CyclesPerSecond(void); @@ -226,7 +228,7 @@ class ProcMapsIterator { // Helper routines namespace tcmalloc { -int FillProcSelfMaps(char buf[], int size); +int FillProcSelfMaps(char buf[], int size, bool* wrote_all); void DumpProcSelfMaps(RawFD fd); } diff --git a/third_party/tcmalloc/vendor/src/base/thread_annotations.h b/third_party/tcmalloc/vendor/src/base/thread_annotations.h index ded13d6..f57b299 100644 --- a/third_party/tcmalloc/vendor/src/base/thread_annotations.h +++ b/third_party/tcmalloc/vendor/src/base/thread_annotations.h @@ -45,15 +45,23 @@ #ifndef BASE_THREAD_ANNOTATIONS_H_ #define BASE_THREAD_ANNOTATIONS_H_ -#if defined(__GNUC__) && defined(__SUPPORT_TS_ANNOTATION__) && (!defined(SWIG)) + +#if defined(__GNUC__) \ + && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) \ + && defined(__SUPPORT_TS_ANNOTATION__) && (!defined(SWIG)) +#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#else +#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op +#endif + // Document if a shared variable/field needs to be protected by a lock. // GUARDED_BY allows the user to specify a particular lock that should be // held when accessing the annotated variable, while GUARDED_VAR only // indicates a shared variable should be guarded (by any lock). GUARDED_VAR // is primarily used when the client cannot express the name of the lock. -#define GUARDED_BY(x) __attribute__ ((guarded_by(x))) -#define GUARDED_VAR __attribute__ ((guarded)) +#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) +#define GUARDED_VAR THREAD_ANNOTATION_ATTRIBUTE__(guarded) // Document if the memory location pointed to by a pointer should be guarded // by a lock when dereferencing the pointer. Similar to GUARDED_VAR, @@ -63,90 +71,64 @@ // q, which is guarded by mu1, points to a shared memory location that is // guarded by mu2, q should be annotated as follows: // int *q GUARDED_BY(mu1) PT_GUARDED_BY(mu2); -#define PT_GUARDED_BY(x) __attribute__ ((point_to_guarded_by(x))) -#define PT_GUARDED_VAR __attribute__ ((point_to_guarded)) +#define PT_GUARDED_BY(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(point_to_guarded_by(x)) +#define PT_GUARDED_VAR \ + THREAD_ANNOTATION_ATTRIBUTE__(point_to_guarded) // Document the acquisition order between locks that can be held // simultaneously by a thread. For any two locks that need to be annotated // to establish an acquisition order, only one of them needs the annotation. // (i.e. You don't have to annotate both locks with both ACQUIRED_AFTER // and ACQUIRED_BEFORE.) -#define ACQUIRED_AFTER(...) __attribute__ ((acquired_after(__VA_ARGS__))) -#define ACQUIRED_BEFORE(...) __attribute__ ((acquired_before(__VA_ARGS__))) +#define ACQUIRED_AFTER(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(x)) +#define ACQUIRED_BEFORE(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(x)) // The following three annotations document the lock requirements for // functions/methods. // Document if a function expects certain locks to be held before it is called -#define EXCLUSIVE_LOCKS_REQUIRED(...) \ - __attribute__ ((exclusive_locks_required(__VA_ARGS__))) +#define EXCLUSIVE_LOCKS_REQUIRED(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(x)) -#define SHARED_LOCKS_REQUIRED(...) \ - __attribute__ ((shared_locks_required(__VA_ARGS__))) +#define SHARED_LOCKS_REQUIRED(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(x)) // Document the locks acquired in the body of the function. These locks // cannot be held when calling this function (as google3's Mutex locks are // non-reentrant). -#define LOCKS_EXCLUDED(...) __attribute__ ((locks_excluded(__VA_ARGS__))) +#define LOCKS_EXCLUDED(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(x)) // Document the lock the annotated function returns without acquiring it. -#define LOCK_RETURNED(x) __attribute__ ((lock_returned(x))) +#define LOCK_RETURNED(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) // Document if a class/type is a lockable type (such as the Mutex class). -#define LOCKABLE __attribute__ ((lockable)) +#define LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(lockable) // Document if a class is a scoped lockable type (such as the MutexLock class). -#define SCOPED_LOCKABLE __attribute__ ((scoped_lockable)) +#define SCOPED_LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) // The following annotations specify lock and unlock primitives. -#define EXCLUSIVE_LOCK_FUNCTION(...) \ - __attribute__ ((exclusive_lock(__VA_ARGS__))) +#define EXCLUSIVE_LOCK_FUNCTION(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock(x)) -#define SHARED_LOCK_FUNCTION(...) \ - __attribute__ ((shared_lock(__VA_ARGS__))) +#define SHARED_LOCK_FUNCTION(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_lock(x)) -#define EXCLUSIVE_TRYLOCK_FUNCTION(...) \ - __attribute__ ((exclusive_trylock(__VA_ARGS__))) +#define EXCLUSIVE_TRYLOCK_FUNCTION(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock(x)) -#define SHARED_TRYLOCK_FUNCTION(...) \ - __attribute__ ((shared_trylock(__VA_ARGS__))) +#define SHARED_TRYLOCK_FUNCTION(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock(x)) -#define UNLOCK_FUNCTION(...) __attribute__ ((unlock(__VA_ARGS__))) +#define UNLOCK_FUNCTION(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(unlock(x)) // An escape hatch for thread safety analysis to ignore the annotated function. -#define NO_THREAD_SAFETY_ANALYSIS __attribute__ ((no_thread_safety_analysis)) - - -#else - -// When the compiler is not GCC, these annotations are simply no-ops. - -// NOTE: in theory, the macros that take "arg" below *could* take -// multiple arguments, but in practice so far they only take one. -// Since not all non-gcc compilers support ... -- notably MSVC 7.1 -- -// I just hard-code in a single arg. If this assumption ever breaks, -// we can change it back to "...", or handle it some other way. - -#define GUARDED_BY(x) // no-op -#define GUARDED_VAR // no-op -#define PT_GUARDED_BY(x) // no-op -#define PT_GUARDED_VAR // no-op -#define ACQUIRED_AFTER(arg) // no-op -#define ACQUIRED_BEFORE(arg) // no-op -#define EXCLUSIVE_LOCKS_REQUIRED(arg) // no-op -#define SHARED_LOCKS_REQUIRED(arg) // no-op -#define LOCKS_EXCLUDED(arg) // no-op -#define LOCK_RETURNED(x) // no-op -#define LOCKABLE // no-op -#define SCOPED_LOCKABLE // no-op -#define EXCLUSIVE_LOCK_FUNCTION(arg) // no-op -#define SHARED_LOCK_FUNCTION(arg) // no-op -#define EXCLUSIVE_TRYLOCK_FUNCTION(arg) // no-op -#define SHARED_TRYLOCK_FUNCTION(arg) // no-op -#define UNLOCK_FUNCTION(arg) // no-op -#define NO_THREAD_SAFETY_ANALYSIS // no-op - -#endif // defined(__GNUC__) && defined(__SUPPORT_TS_ANNOTATION__) - // && !defined(SWIG) +#define NO_THREAD_SAFETY_ANALYSIS \ + THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) #endif // BASE_THREAD_ANNOTATIONS_H_ diff --git a/third_party/tcmalloc/vendor/src/base/vdso_support.cc b/third_party/tcmalloc/vendor/src/base/vdso_support.cc index fce7c2c..09288a5 100644 --- a/third_party/tcmalloc/vendor/src/base/vdso_support.cc +++ b/third_party/tcmalloc/vendor/src/base/vdso_support.cc @@ -40,6 +40,7 @@ #ifdef HAVE_VDSO_SUPPORT // defined in vdso_support.h #include <fcntl.h> +#include <stddef.h> // for ptrdiff_t #include "base/atomicops.h" // for MemoryBarrier #include "base/linux_syscall_support.h" @@ -207,6 +208,10 @@ void VDSOSupport::ElfMemImage::Init(const void *base) { if (!base) { return; } + const intptr_t base_as_uintptr_t = reinterpret_cast<uintptr_t>(base); + // Fake VDSO has low bit set. + const bool fake_vdso = ((base_as_uintptr_t & 1) != 0); + base = reinterpret_cast<const void *>(base_as_uintptr_t & ~1); const char *const base_as_char = reinterpret_cast<const char *>(base); if (base_as_char[EI_MAG0] != ELFMAG0 || base_as_char[EI_MAG1] != ELFMAG1 || base_as_char[EI_MAG2] != ELFMAG2 || base_as_char[EI_MAG3] != ELFMAG3) { @@ -266,17 +271,6 @@ void VDSOSupport::ElfMemImage::Init(const void *base) { ElfW(Dyn) *dynamic_entry = reinterpret_cast<ElfW(Dyn) *>(dynamic_program_header->p_vaddr + relocation); - bool fake_vdso = false; // Assume we are dealing with the real VDSO. - for (ElfW(Dyn) *de = dynamic_entry; de->d_tag != DT_NULL; ++de) { - ElfW(Sxword) tag = de->d_tag; - if (tag == DT_PLTGOT || tag == DT_RELA || tag == DT_JMPREL || - tag == DT_NEEDED || tag == DT_RPATH || tag == DT_VERNEED || - tag == DT_INIT || tag == DT_FINI) { - /* Real vdso can not reasonably have any of the above entries. */ - fake_vdso = true; - break; - } - } for (; dynamic_entry->d_tag != DT_NULL; ++dynamic_entry) { ElfW(Xword) value = dynamic_entry->d_un.d_val; if (fake_vdso) { @@ -395,7 +389,7 @@ const void *VDSOSupport::Init() { } // Subtle: this code runs outside of any locks; prevent compiler // from assigning to getcpu_fn_ more than once. - MemoryBarrier(); + base::subtle::MemoryBarrier(); getcpu_fn_ = fn; return vdso_base_; } diff --git a/third_party/tcmalloc/vendor/src/base/vdso_support.h b/third_party/tcmalloc/vendor/src/base/vdso_support.h index c47b3c5..131646a 100644 --- a/third_party/tcmalloc/vendor/src/base/vdso_support.h +++ b/third_party/tcmalloc/vendor/src/base/vdso_support.h @@ -30,6 +30,7 @@ #ifdef HAVE_FEATURES_H #include <features.h> // for __GLIBC__ #endif +#include "base/basictypes.h" // Maybe one day we can rewrite this file not to require the elf // symbol extensions in glibc, but for right now we need them. @@ -39,7 +40,6 @@ #include <stdlib.h> // for NULL #include <link.h> // for ElfW -#include "base/basictypes.h" namespace base { @@ -64,7 +64,7 @@ class VDSOSupport { // Supports iteration over all dynamic symbols. class SymbolIterator { public: - friend struct VDSOSupport; + friend class VDSOSupport; const SymbolInfo *operator->() const; const SymbolInfo &operator*() const; SymbolIterator& operator++(); @@ -147,6 +147,10 @@ class VDSOSupport { // kInvalidBase => value hasn't been determined yet. // 0 => there is no VDSO. // else => vma of VDSO Elf{32,64}_Ehdr. + // + // When testing with mock VDSO, low bit is set. + // The low bit is always available because vdso_base_ is + // page-aligned. static const void *vdso_base_; // NOLINT on 'long' because these routines mimic kernel api. |