diff options
Diffstat (limited to 'third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h')
| -rw-r--r-- | third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h | 156 |
1 files changed, 140 insertions, 16 deletions
diff --git a/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h b/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h index ee09f32..8d5b9b5 100644 --- a/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h +++ b/third_party/tcmalloc/chromium/src/base/atomicops-internals-arm-v6plus.h @@ -42,6 +42,13 @@ #include <stdlib.h> #include "base/basictypes.h" // For COMPILE_ASSERT +// The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, +// only some variants support it. For simplicity, we only use exclusive +// 64-bit load/store in V7 or above. +#if defined(ARMV7) +# define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD +#endif + typedef int32_t Atomic32; namespace base { @@ -60,6 +67,10 @@ inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, "ldrex %1, [%3]\n" "mov %0, #0\n" "teq %1, %4\n" + // The following IT (if-then) instruction is needed for the subsequent + // conditional instruction STREXEQ when compiling in THUMB mode. + // In ARM mode, the compiler/assembler will not generate any code for it. + "it eq\n" "strexeq %0, %5, [%3]\n" : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) : "r" (ptr), "Ir" (old_value), "r" (new_value) @@ -164,7 +175,114 @@ inline Atomic32 Release_Load(volatile const Atomic32* ptr) { return *ptr; } -// 64-bit versions are not implemented yet. +// 64-bit versions are only available if LDREXD and STREXD instructions +// are available. +#ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD + +#define BASE_HAS_ATOMIC64 1 + +inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, + Atomic64 old_value, + Atomic64 new_value) { + Atomic64 oldval, res; + do { + __asm__ __volatile__( + "ldrexd %1, [%3]\n" + "mov %0, #0\n" + "teq %Q1, %Q4\n" + // The following IT (if-then) instructions are needed for the subsequent + // conditional instructions when compiling in THUMB mode. + // In ARM mode, the compiler/assembler will not generate any code for it. + "it eq\n" + "teqeq %R1, %R4\n" + "it eq\n" + "strexdeq %0, %5, [%3]\n" + : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) + : "r" (ptr), "Ir" (old_value), "r" (new_value) + : "cc"); + } while (res); + return oldval; +} + +inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, + Atomic64 new_value) { + int store_failed; + Atomic64 old; + __asm__ __volatile__( + "1:\n" + "ldrexd %1, [%2]\n" + "strexd %0, %3, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (store_failed), "=&r" (old) + : "r" (ptr), "r" (new_value) + : "cc", "memory"); + return old; +} + +inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + int store_failed; + Atomic64 res; + __asm__ __volatile__( + "1:\n" + "ldrexd %1, [%2]\n" + "adds %Q1, %Q1, %Q3\n" + "adc %R1, %R1, %R3\n" + "strexd %0, %1, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (store_failed), "=&r"(res) + : "r" (ptr), "r"(increment) + : "cc", "memory"); + return res; +} + +inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, + Atomic64 increment) { + int store_failed; + Atomic64 res; + __asm__ __volatile__( + "1:\n" + "ldrexd %1, [%2]\n" + "adds %Q1, %Q1, %Q3\n" + "adc %R1, %R1, %R3\n" + "dmb\n" + "strexd %0, %1, [%2]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (store_failed), "=&r"(res) + : "r" (ptr), "r"(increment) + : "cc", "memory"); + return res; +} + +inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { + int store_failed; + Atomic64 dummy; + __asm__ __volatile__( + "1:\n" + // Dummy load to lock cache line. + "ldrexd %1, [%3]\n" + "strexd %0, %2, [%3]\n" + "teq %0, #0\n" + "bne 1b" + : "=&r" (store_failed), "=&r"(dummy) + : "r"(value), "r" (ptr) + : "cc", "memory"); +} + +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + Atomic64 res; + __asm__ __volatile__( + "ldrexd %0, [%1]\n" + "clrex\n" + : "=r" (res) + : "r"(ptr), "Q"(*ptr)); + return res; +} + +#else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD inline void NotImplementedFatalError(const char *function_name) { fprintf(stderr, "64-bit %s() not implemented on this platform\n", @@ -201,41 +319,47 @@ inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { NotImplementedFatalError("NoBarrier_Store"); } -inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Acquire_Store64"); +inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { + NotImplementedFatalError("NoBarrier_Load"); + return 0; } -inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { - NotImplementedFatalError("Release_Store"); +#endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD + +inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { + NoBarrier_Store(ptr, value); + MemoryBarrier(); } -inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("NoBarrier_Load"); - return 0; +inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { + MemoryBarrier(); + NoBarrier_Store(ptr, value); } inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("Atomic64 Acquire_Load"); - return 0; + Atomic64 value = NoBarrier_Load(ptr); + MemoryBarrier(); + return value; } inline Atomic64 Release_Load(volatile const Atomic64* ptr) { - NotImplementedFatalError("Atomic64 Release_Load"); - return 0; + MemoryBarrier(); + return NoBarrier_Load(ptr); } inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, Atomic64 old_value, Atomic64 new_value) { - NotImplementedFatalError("Atomic64 Acquire_CompareAndSwap"); - return 0; + Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); + MemoryBarrier(); + return value; } inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, Atomic64 old_value, Atomic64 new_value) { - NotImplementedFatalError("Atomic64 Release_CompareAndSwap"); - return 0; + MemoryBarrier(); + return NoBarrier_CompareAndSwap(ptr, old_value, new_value); } } // namespace subtle ends |