diff options
| -rw-r--r-- | libc/Android.mk | 5 | ||||
| -rw-r--r-- | libc/arch-arm/bionic/atomics_arm.S | 236 | ||||
| -rw-r--r-- | libc/arch-arm/bionic/atomics_arm.c | 87 | ||||
| -rw-r--r-- | libc/arch-arm/bionic/futex_arm.S | 112 | ||||
| -rw-r--r-- | libc/arch-x86/bionic/futex_x86.S (renamed from libc/arch-x86/bionic/atomics_x86.S) | 0 | ||||
| -rw-r--r-- | libc/arch-x86/include/sys/atomics.h | 65 | ||||
| -rw-r--r-- | libc/include/sys/atomics.h | 46 |
7 files changed, 244 insertions, 307 deletions
diff --git a/libc/Android.mk b/libc/Android.mk index 410f242..207a133 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -346,10 +346,11 @@ libc_common_src_files += \ arch-arm/bionic/__get_sp.S \ arch-arm/bionic/_exit_with_stack_teardown.S \ arch-arm/bionic/_setjmp.S \ - arch-arm/bionic/atomics_arm.S \ + arch-arm/bionic/atomics_arm.c \ arch-arm/bionic/clone.S \ arch-arm/bionic/eabi.c \ arch-arm/bionic/ffs.S \ + arch-arm/bionic/futex_arm.S \ arch-arm/bionic/kill.S \ arch-arm/bionic/libgcc_compat.c \ arch-arm/bionic/tkill.S \ @@ -395,9 +396,9 @@ libc_common_src_files += \ arch-x86/bionic/__get_sp.S \ arch-x86/bionic/__get_tls.c \ arch-x86/bionic/__set_tls.c \ - arch-x86/bionic/atomics_x86.S \ arch-x86/bionic/clone.S \ arch-x86/bionic/_exit_with_stack_teardown.S \ + arch-x86/bionic/futex_x86.S \ arch-x86/bionic/setjmp.S \ arch-x86/bionic/_setjmp.S \ arch-x86/bionic/sigsetjmp.S \ diff --git a/libc/arch-arm/bionic/atomics_arm.S b/libc/arch-arm/bionic/atomics_arm.S deleted file mode 100644 index 4d9cbcf..0000000 --- a/libc/arch-arm/bionic/atomics_arm.S +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#include <sys/linux-syscalls.h> -#include <machine/asm.h> -#include <machine/cpu-features.h> - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - -#if defined(__ARM_HAVE_LDREX_STREX) -/* - * =========================================================================== - * ARMv6+ implementation - * =========================================================================== - */ - -/* r0(addr) -> r0(old) */ -ENTRY(__atomic_dec) - mov r1, r0 @ copy addr so we don't clobber it -1: ldrex r0, [r1] @ load current value into r0 - sub r2, r0, #1 @ generate new value into r2 - strex r3, r2, [r1] @ try to store new value; result in r3 - cmp r3, #0 @ success? - bxeq lr @ yes, return - b 1b @ no, retry -END(__atomic_dec) - -/* r0(addr) -> r0(old) */ -ENTRY(__atomic_inc) - mov r1, r0 -1: ldrex r0, [r1] - add r2, r0, #1 - strex r3, r2, [r1] - cmp r3, #0 - bxeq lr - b 1b -END(__atomic_inc) - -/* r0(old) r1(new) r2(addr) -> r0(zero_if_succeeded) */ -ENTRY(__atomic_cmpxchg) -1: mov ip, #2 @ ip=2 means "new != old" - ldrex r3, [r2] @ load current value into r3 - teq r0, r3 @ new == old? - strexeq ip, r1, [r2] @ yes, try store, set ip to 0 or 1 - teq ip, #1 @ strex failure? - beq 1b @ yes, retry - mov r0, ip @ return 0 on success, 2 on failure - bx lr -END(__atomic_cmpxchg) - -/* r0(new) r1(addr) -> r0(old) */ -ENTRY(__atomic_swap) -1: ldrex r2, [r1] - strex r3, r0, [r1] - teq r3, #0 - bne 1b - mov r0, r2 - bx lr -END(__atomic_swap) - -#else /*not defined __ARM_HAVE_LDREX_STREX*/ -/* - * =========================================================================== - * Pre-ARMv6 implementation - * =========================================================================== - */ - - /* int __kernel_cmpxchg(int oldval, int newval, int* ptr) */ - .equ kernel_cmpxchg, 0xFFFF0FC0 - .equ kernel_atomic_base, 0xFFFF0FFF - -/* r0(addr) -> r0(old) */ -ENTRY(__atomic_dec) - .save {r4, lr} - stmdb sp!, {r4, lr} - mov r2, r0 -1: @ atomic_dec - ldr r0, [r2] - mov r3, #kernel_atomic_base - add lr, pc, #4 - sub r1, r0, #1 - add pc, r3, #(kernel_cmpxchg - kernel_atomic_base) - bcc 1b - add r0, r1, #1 - ldmia sp!, {r4, lr} - bx lr -END(__atomic_dec) - -/* r0(addr) -> r0(old) */ -ENTRY(__atomic_inc) - .save {r4, lr} - stmdb sp!, {r4, lr} - mov r2, r0 -1: @ atomic_inc - ldr r0, [r2] - mov r3, #kernel_atomic_base - add lr, pc, #4 - add r1, r0, #1 - add pc, r3, #(kernel_cmpxchg - kernel_atomic_base) - bcc 1b - sub r0, r1, #1 - ldmia sp!, {r4, lr} - bx lr -END(__atomic_inc) - -/* r0(old) r1(new) r2(addr) -> r0(zero_if_succeeded) */ -ENTRY(__atomic_cmpxchg) - .save {r4, lr} - stmdb sp!, {r4, lr} - mov r4, r0 /* r4 = save oldvalue */ -1: @ atomic_cmpxchg - mov r3, #kernel_atomic_base - add lr, pc, #4 - mov r0, r4 /* r0 = oldvalue */ - add pc, r3, #(kernel_cmpxchg - kernel_atomic_base) - bcs 2f /* swap was made. we're good, return. */ - ldr r3, [r2] /* swap not made, see if it's because *ptr!=oldvalue */ - cmp r3, r4 - beq 1b -2: @ atomic_cmpxchg - ldmia sp!, {r4, lr} - bx lr -END(__atomic_cmpxchg) - -/* r0(new) r1(addr) -> r0(old) */ -ENTRY(__atomic_swap) - swp r0, r0, [r1] - bx lr -END(__atomic_swap) - -#endif /*not defined __ARM_HAVE_LDREX_STREX*/ - - -/* __futex_wait(*ftx, val, *timespec) */ -/* __futex_wake(*ftx, counter) */ -/* __futex_syscall3(*ftx, op, val) */ -/* __futex_syscall4(*ftx, op, val, *timespec) */ - -.global __futex_wait -.type __futex_wait, %function - -.global __futex_wake -.type __futex_wake, %function - -.global __futex_syscall3 -.type __futex_syscall3, %function - -.global __futex_syscall4 -.type __futex_syscall4, %function - -#if __ARM_EABI__ - -ENTRY(__futex_syscall3) - stmdb sp!, {r4, r7} - .save {r4, r7} - ldr r7, =__NR_futex - swi #0 - ldmia sp!, {r4, r7} - bx lr -END(__futex_syscall3) - -ENTRY(__futex_wait) - stmdb sp!, {r4, r7} - .save {r4, r7} - mov r3, r2 - mov r2, r1 - mov r1, #FUTEX_WAIT - ldr r7, =__NR_futex - swi #0 - ldmia sp!, {r4, r7} - bx lr -END(__futex_wait) - -ENTRY(__futex_wake) - .save {r4, r7} - stmdb sp!, {r4, r7} - mov r2, r1 - mov r1, #FUTEX_WAKE - ldr r7, =__NR_futex - swi #0 - ldmia sp!, {r4, r7} - bx lr -END(__futex_wake) - -#else - -ENTRY(__futex_syscall3) - swi #__NR_futex - bx lr -END(__futex_syscall3) - -ENTRY(__futex_wait) - mov r3, r2 - mov r2, r1 - mov r1, #FUTEX_WAIT - swi #__NR_futex - bx lr -END(__futex_wait) - -ENTRY(__futex_wake) - mov r2, r1 - mov r1, #FUTEX_WAKE - swi #__NR_futex - bx lr -END(__futex_wake) - -#endif - -ENTRY(__futex_syscall4) - b __futex_syscall3 -END(__futex_syscall4) diff --git a/libc/arch-arm/bionic/atomics_arm.c b/libc/arch-arm/bionic/atomics_arm.c new file mode 100644 index 0000000..1d7a1e1 --- /dev/null +++ b/libc/arch-arm/bionic/atomics_arm.c @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + +/* The purpose of this file is to export a small set of atomic-related + * functions from the C library, to ensure binary ABI compatibility for + * the NDK. + * + * These functions were initially exposed by the NDK through <sys/atomics.h>, + * which was unfortunate because their implementation didn't provide any + * memory barriers at all. + * + * This wasn't a problem for the platform code that used them, because it + * used explicit barrier instructions around them. On the other hand, it means + * that any NDK-generated machine code that linked against them would not + * perform correctly when running on multi-core devices. + * + * To fix this, the platform code was first modified to not use any of these + * functions (everything is now inlined through assembly statements, see + * libc/private/bionic_arm_inline.h and the headers it includes. + * + * The functions here are thus only for the benefit of NDK applications, + * and now includes full memory barriers to prevent any random memory ordering + * issue from cropping. + * + * Note that we also provide an updated <sys/atomics.h> header that defines + * always_inlined versions of the functions that use the GCC builtin + * intrinsics to perform the same thing. + * + * NOTE: There is no need for a similar file for non-ARM platforms. + */ + +/* DO NOT INCLUDE <sys/atomics.h> HERE ! */ + +int +__android_cmpxchg(int old, int _new, volatile int *ptr) +{ + /* We must return 0 on success */ + return __sync_val_compare_and_swap(ptr, old, _new) != old; +} + +int +__atomic_swap(int _new, volatile int *ptr) +{ + int prev; + do { + prev = *ptr; + } while (__sync_val_compare_and_swap(ptr, prev, _new) != prev); + return prev; +} + +int +__atomic_dec(volatile int *ptr) +{ + return __sync_fetch_and_sub (ptr, 1); +} + +int +__atomic_inc(volatile int *ptr) +{ + return __sync_fetch_and_add (ptr, 1); +} diff --git a/libc/arch-arm/bionic/futex_arm.S b/libc/arch-arm/bionic/futex_arm.S new file mode 100644 index 0000000..7041663 --- /dev/null +++ b/libc/arch-arm/bionic/futex_arm.S @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include <sys/linux-syscalls.h> +#include <machine/asm.h> +#include <machine/cpu-features.h> + +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 + +/* __futex_wait(*ftx, val, *timespec) */ +/* __futex_wake(*ftx, counter) */ +/* __futex_syscall3(*ftx, op, val) */ +/* __futex_syscall4(*ftx, op, val, *timespec) */ + +.global __futex_wait +.type __futex_wait, %function + +.global __futex_wake +.type __futex_wake, %function + +.global __futex_syscall3 +.type __futex_syscall3, %function + +.global __futex_syscall4 +.type __futex_syscall4, %function + +#if __ARM_EABI__ + +ENTRY(__futex_syscall3) + stmdb sp!, {r4, r7} + .save {r4, r7} + ldr r7, =__NR_futex + swi #0 + ldmia sp!, {r4, r7} + bx lr +END(__futex_syscall3) + +ENTRY(__futex_wait) + stmdb sp!, {r4, r7} + .save {r4, r7} + mov r3, r2 + mov r2, r1 + mov r1, #FUTEX_WAIT + ldr r7, =__NR_futex + swi #0 + ldmia sp!, {r4, r7} + bx lr +END(__futex_wait) + +ENTRY(__futex_wake) + .save {r4, r7} + stmdb sp!, {r4, r7} + mov r2, r1 + mov r1, #FUTEX_WAKE + ldr r7, =__NR_futex + swi #0 + ldmia sp!, {r4, r7} + bx lr +END(__futex_wake) + +#else + +ENTRY(__futex_syscall3) + swi #__NR_futex + bx lr +END(__futex_syscall3) + +ENTRY(__futex_wait) + mov r3, r2 + mov r2, r1 + mov r1, #FUTEX_WAIT + swi #__NR_futex + bx lr +END(__futex_wait) + +ENTRY(__futex_wake) + mov r2, r1 + mov r1, #FUTEX_WAKE + swi #__NR_futex + bx lr +END(__futex_wake) + +#endif + +ENTRY(__futex_syscall4) + b __futex_syscall3 +END(__futex_syscall4) diff --git a/libc/arch-x86/bionic/atomics_x86.S b/libc/arch-x86/bionic/futex_x86.S index e98a391..e98a391 100644 --- a/libc/arch-x86/bionic/atomics_x86.S +++ b/libc/arch-x86/bionic/futex_x86.S diff --git a/libc/arch-x86/include/sys/atomics.h b/libc/arch-x86/include/sys/atomics.h deleted file mode 100644 index 7aed3ae..0000000 --- a/libc/arch-x86/include/sys/atomics.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#ifndef _SYS_ATOMICS_H -#define _SYS_ATOMICS_H - -#include <sys/cdefs.h> -#include <sys/time.h> - -__BEGIN_DECLS - -static inline __attribute__((always_inline)) int -__atomic_cmpxchg(int old, int _new, volatile int *ptr) -{ - return !__sync_bool_compare_and_swap (ptr, old, _new); -} - -static inline __attribute__((always_inline)) int -__atomic_swap(int _new, volatile int *ptr) -{ - return __sync_lock_test_and_set(ptr, _new); -} - -static inline __attribute__((always_inline)) int -__atomic_dec(volatile int *ptr) -{ - return __sync_fetch_and_sub (ptr, 1); -} - -static inline __attribute__((always_inline)) int -__atomic_inc(volatile int *ptr) -{ - return __sync_fetch_and_add (ptr, 1); -} - -int __futex_wait(volatile void *ftx, int val, const struct timespec *timeout); -int __futex_wake(volatile void *ftx, int count); - -__END_DECLS - -#endif /* _SYS_ATOMICS_H */ diff --git a/libc/include/sys/atomics.h b/libc/include/sys/atomics.h index d3fa145..3ada8de 100644 --- a/libc/include/sys/atomics.h +++ b/libc/include/sys/atomics.h @@ -33,10 +33,48 @@ __BEGIN_DECLS -extern int __atomic_cmpxchg(int old, int _new, volatile int *ptr); -extern int __atomic_swap(int _new, volatile int *ptr); -extern int __atomic_dec(volatile int *ptr); -extern int __atomic_inc(volatile int *ptr); +/* Note: atomic operations that were exported by the C library didn't + * provide any memory barriers, which created potential issues on + * multi-core devices. We now define them as inlined calls to + * GCC sync builtins, which always provide a full barrier. + * + * NOTE: The C library still exports atomic functions by the same + * name to ensure ABI stability for existing NDK machine code. + * + * If you are an NDK developer, we encourage you to rebuild your + * unmodified sources against this header as soon as possible. + */ +#define __ATOMIC_INLINE__ static __inline__ __attribute__((always_inline)) + +__ATOMIC_INLINE__ int +__atomic_cmpxchg(int old, int _new, volatile int *ptr) +{ + /* We must return 0 on success */ + return __sync_val_compare_and_swap(ptr, old, _new) != old; +} + +__ATOMIC_INLINE__ int +__atomic_swap(int _new, volatile int *ptr) +{ + int prev; + do { + prev = *ptr; + } while (__sync_val_compare_and_swap(ptr, prev, _new) != prev); + return prev; +} + +__ATOMIC_INLINE__ int +__atomic_dec(volatile int *ptr) +{ + return __sync_fetch_and_sub (ptr, 1); +} + +__ATOMIC_INLINE__ int +__atomic_inc(volatile int *ptr) +{ + return __sync_fetch_and_add (ptr, 1); +} + int __futex_wait(volatile void *ftx, int val, const struct timespec *timeout); int __futex_wake(volatile void *ftx, int count); |