diff options
| -rw-r--r-- | libc/arch-arm/bionic/atomics_arm.S | 37 | ||||
| -rw-r--r-- | libc/arch-arm/bionic/memcpy.S | 2 | ||||
| -rw-r--r-- | libc/arch-sh/bionic/atomics_sh.c | 10 | ||||
| -rw-r--r-- | libc/arch-x86/bionic/atomics_x86.S | 32 | ||||
| -rw-r--r-- | libc/arch-x86/bionic/atomics_x86.c | 104 | ||||
| -rw-r--r-- | libc/bionic/pthread.c | 643 | ||||
| -rw-r--r-- | libc/docs/CHANGES.TXT | 13 | ||||
| -rw-r--r-- | libc/include/pthread.h | 5 |
8 files changed, 486 insertions, 360 deletions
diff --git a/libc/arch-arm/bionic/atomics_arm.S b/libc/arch-arm/bionic/atomics_arm.S index 55c642f..047541f 100644 --- a/libc/arch-arm/bionic/atomics_arm.S +++ b/libc/arch-arm/bionic/atomics_arm.S @@ -35,10 +35,6 @@ .type __atomic_dec, %function .global __atomic_inc .type __atomic_inc, %function -.global __futex_wait -.type __futex_wait, %function -.global __futex_wake -.type __futex_wake, %function #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 @@ -159,10 +155,34 @@ __atomic_swap: bx lr /* __futex_wait(*ftx, val, *timespec) */ -/* __futex_syscall(*ftx, op, val, *timespec, *addr2, val3) */ +/* __futex_wake(*ftx, counter) */ +/* __futex_syscall3(*ftx, op, val) */ +/* __futex_syscall4(*ftx, op, val, *timespec) */ + +.global __futex_wait +.type __futex_wait, %function + +.global __futex_wake +.type __futex_wake, %function + +.global __futex_syscall3 +.type __futex_syscall3, %function + +.global __futex_syscall4 +.type __futex_syscall4, %function #if __ARM_EABI__ +__futex_syscall3: + .fnstart + stmdb sp!, {r4, r7} + .save {r4, r7} + ldr r7, =__NR_futex + swi #0 + ldmia sp!, {r4, r7} + bx lr + .fnend + __futex_wait: .fnstart stmdb sp!, {r4, r7} @@ -187,6 +207,10 @@ __futex_wake: #else +__futex_syscall3: + swi #__NR_futex + bx lr + __futex_wait: mov r3, r2 mov r2, r1 @@ -201,3 +225,6 @@ __futex_wake: bx lr #endif + +__futex_syscall4: + b __futex_syscall3 diff --git a/libc/arch-arm/bionic/memcpy.S b/libc/arch-arm/bionic/memcpy.S index 024d885..ba55996 100644 --- a/libc/arch-arm/bionic/memcpy.S +++ b/libc/arch-arm/bionic/memcpy.S @@ -28,7 +28,7 @@ #include <machine/cpu-features.h> -#if __ARM_ARCH__ == 7 || defined(__ARM_NEON__) +#if defined(__ARM_NEON__) .text .fpu neon diff --git a/libc/arch-sh/bionic/atomics_sh.c b/libc/arch-sh/bionic/atomics_sh.c index 16966f7..c7815ff 100644 --- a/libc/arch-sh/bionic/atomics_sh.c +++ b/libc/arch-sh/bionic/atomics_sh.c @@ -98,3 +98,13 @@ int __futex_wake(volatile void *ftx, int count) { return futex(ftx, FUTEX_WAKE, count, NULL, NULL, 0); } + +int __futex_syscall3(volatile void *ftx, int op, int val) +{ + return futex(ftx, op, val, NULL, NULL, 0); +} + +int __futex_syscall4(volative void *ftx, int op, int val, const struct timespec *timeout) +{ + return futex(ftx, op, val, (void *)timeout, NULL, 0); +} diff --git a/libc/arch-x86/bionic/atomics_x86.S b/libc/arch-x86/bionic/atomics_x86.S index 2370f23..666e182 100644 --- a/libc/arch-x86/bionic/atomics_x86.S +++ b/libc/arch-x86/bionic/atomics_x86.S @@ -41,6 +41,38 @@ __futex_wake: popl %ebx ret +/* int __futex_syscall3(volatile void *ftx, int op, int count) */ +.text +.globl __futex_syscall3 +.type __futex_syscall3, @function +.align 4 +__futex_syscall3: + pushl %ebx + movl 8(%esp), %ebx /* ftx */ + movl 12(%esp), %ecx /* op */ + movl 16(%esp), %edx /* value */ + movl $__NR_futex, %eax + int $0x80 + popl %ebx + ret + +/* int __futex_syscall4(volatile void *ftx, int op, int val, const struct timespec *timeout) */ +.text +.globl __futex_syscall4 +.type __futex_syscall4, @function +.align 4 +__futex_syscall4: + pushl %ebx + pushl %esi + movl 12(%esp), %ebx /* ftx */ + movl 16(%esp), %ecx /* op */ + movl 20(%esp), %edx /* val */ + movl 24(%esp), %esi /* timeout */ + movl $__NR_futex, %eax + int $0x80 + popl %esi + popl %ebx + ret /* int __atomic_cmpxchg(int old, int new, volatile int* addr) */ diff --git a/libc/arch-x86/bionic/atomics_x86.c b/libc/arch-x86/bionic/atomics_x86.c deleted file mode 100644 index b7b20e6..0000000 --- a/libc/arch-x86/bionic/atomics_x86.c +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#include <sys/atomics.h> - -#define FUTEX_SYSCALL 240 -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - -int __futex_wait(volatile void *ftx, int val) -{ - int ret; - asm volatile ( - "int $0x80;" - : "=a" (ret) - : "0" (FUTEX_SYSCALL), - "b" (ftx), - "c" (FUTEX_WAIT), - "d" (val), - "S" (0) - ); - return ret; -} - -int __futex_wake(volatile void *ftx, int count) -{ - int ret; - asm volatile ( - "int $0x80;" - : "=a" (ret) - : "0" (FUTEX_SYSCALL), - "b" (ftx), - "c" (FUTEX_WAKE), - "d" (count) - ); - return ret; -} - -int __atomic_cmpxchg(int old, int new, volatile int* addr) { - int xchg; - asm volatile ( - "lock;" - "cmpxchg %%ecx, (%%edx);" - "setne %%al;" - : "=a" (xchg) - : "a" (old), - "c" (new), - "d" (addr) - ); - return xchg; -} - -int __atomic_swap(int new, volatile int* addr) { - int old; - asm volatile ( - "lock;" - "xchg %%ecx, (%%edx);" - : "=c" (old) - : "c" (new), - "d" (addr) - ); - return old; -} - -int __atomic_dec(volatile int* addr) { - int old; - do { - old = *addr; - } while (atomic_cmpxchg(old, old-1, addr)); - return old; -} - -int __atomic_inc(volatile int* addr) { - int old; - do { - old = *addr; - } while (atomic_cmpxchg(old, old+1, addr)); - return old; -} - diff --git a/libc/bionic/pthread.c b/libc/bionic/pthread.c index c1a6a8a..3294cea 100644 --- a/libc/bionic/pthread.c +++ b/libc/bionic/pthread.c @@ -43,12 +43,16 @@ #include <memory.h> #include <assert.h> #include <malloc.h> +#include <linux/futex.h> extern int __pthread_clone(int (*fn)(void*), void *child_stack, int flags, void *arg); extern void _exit_with_stack_teardown(void * stackBase, int stackSize, int retCode); extern void _exit_thread(int retCode); extern int __set_errno(int); +#define __likely(cond) __builtin_expect(!!(cond), 1) +#define __unlikely(cond) __builtin_expect(!!(cond), 0) + void _thread_created_hook(pid_t thread_id) __attribute__((noinline)); #define PTHREAD_ATTR_FLAG_DETACHED 0x00000001 @@ -712,6 +716,21 @@ int pthread_setschedparam(pthread_t thid, int policy, int __futex_wait(volatile void *ftx, int val, const struct timespec *timeout); int __futex_wake(volatile void *ftx, int count); +int __futex_syscall3(volatile void *ftx, int op, int val); +int __futex_syscall4(volatile void *ftx, int op, int val, const struct timespec *timeout); + +#ifndef FUTEX_PRIVATE_FLAG +#define FUTEX_PRIVATE_FLAG 128 +#endif + +#ifndef FUTEX_WAIT_PRIVATE +#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT|FUTEX_PRIVATE_FLAG) +#endif + +#ifndef FUTEX_WAKE_PRIVATE +#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE|FUTEX_PRIVATE_FLAG) +#endif + // mutex lock states // // 0: unlocked @@ -723,7 +742,8 @@ int __futex_wake(volatile void *ftx, int count); * bits: name description * 31-16 tid owner thread's kernel id (recursive and errorcheck only) * 15-14 type mutex type - * 13-2 counter counter of recursive mutexes + * 13 shared process-shared flag + * 12-2 counter counter of recursive mutexes * 1-0 state lock state (0, 1 or 2) */ @@ -737,9 +757,17 @@ int __futex_wake(volatile void *ftx, int count); #define MUTEX_TYPE_ERRORCHECK 0x8000 #define MUTEX_COUNTER_SHIFT 2 -#define MUTEX_COUNTER_MASK 0x3ffc - +#define MUTEX_COUNTER_MASK 0x1ffc +#define MUTEX_SHARED_MASK 0x2000 +/* a mutex attribute holds the following fields + * + * bits: name description + * 0-3 type type of mutex + * 4 shared process-shared flag + */ +#define MUTEXATTR_TYPE_MASK 0x000f +#define MUTEXATTR_SHARED_MASK 0x0010 int pthread_mutexattr_init(pthread_mutexattr_t *attr) @@ -764,10 +792,14 @@ int pthread_mutexattr_destroy(pthread_mutexattr_t *attr) int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type) { - if (attr && *attr >= PTHREAD_MUTEX_NORMAL && - *attr <= PTHREAD_MUTEX_ERRORCHECK ) { - *type = *attr; - return 0; + if (attr) { + int atype = (*attr & MUTEXATTR_TYPE_MASK); + + if (atype >= PTHREAD_MUTEX_NORMAL && + atype <= PTHREAD_MUTEX_ERRORCHECK) { + *type = atype; + return 0; + } } return EINVAL; } @@ -776,7 +808,7 @@ int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) { if (attr && type >= PTHREAD_MUTEX_NORMAL && type <= PTHREAD_MUTEX_ERRORCHECK ) { - *attr = type; + *attr = (*attr & ~MUTEXATTR_TYPE_MASK) | type; return 0; } return EINVAL; @@ -791,54 +823,70 @@ int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) switch (pshared) { case PTHREAD_PROCESS_PRIVATE: + *attr &= ~MUTEXATTR_SHARED_MASK; + return 0; + case PTHREAD_PROCESS_SHARED: /* our current implementation of pthread actually supports shared * mutexes but won't cleanup if a process dies with the mutex held. * Nevertheless, it's better than nothing. Shared mutexes are used * by surfaceflinger and audioflinger. */ + *attr |= MUTEXATTR_SHARED_MASK; return 0; } - - return ENOTSUP; + return EINVAL; } int pthread_mutexattr_getpshared(pthread_mutexattr_t *attr, int *pshared) { - if (!attr) + if (!attr || !pshared) return EINVAL; - *pshared = PTHREAD_PROCESS_PRIVATE; + *pshared = (*attr & MUTEXATTR_SHARED_MASK) ? PTHREAD_PROCESS_SHARED + : PTHREAD_PROCESS_PRIVATE; return 0; } int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr) { - if ( mutex ) { - if (attr == NULL) { - mutex->value = MUTEX_TYPE_NORMAL; - return 0; - } - switch ( *attr ) { - case PTHREAD_MUTEX_NORMAL: - mutex->value = MUTEX_TYPE_NORMAL; - return 0; + int value = 0; - case PTHREAD_MUTEX_RECURSIVE: - mutex->value = MUTEX_TYPE_RECURSIVE; - return 0; + if (mutex == NULL) + return EINVAL; - case PTHREAD_MUTEX_ERRORCHECK: - mutex->value = MUTEX_TYPE_ERRORCHECK; - return 0; - } + if (__likely(attr == NULL)) { + mutex->value = MUTEX_TYPE_NORMAL; + return 0; } - return EINVAL; + + if ((*attr & MUTEXATTR_SHARED_MASK) != 0) + value |= MUTEX_SHARED_MASK; + + switch (*attr & MUTEXATTR_TYPE_MASK) { + case PTHREAD_MUTEX_NORMAL: + value |= MUTEX_TYPE_NORMAL; + break; + case PTHREAD_MUTEX_RECURSIVE: + value |= MUTEX_TYPE_RECURSIVE; + break; + case PTHREAD_MUTEX_ERRORCHECK: + value |= MUTEX_TYPE_ERRORCHECK; + break; + default: + return EINVAL; + } + + mutex->value = value; + return 0; } int pthread_mutex_destroy(pthread_mutex_t *mutex) { + if (__unlikely(mutex == NULL)) + return EINVAL; + mutex->value = 0xdead10cc; return 0; } @@ -859,13 +907,15 @@ int pthread_mutex_destroy(pthread_mutex_t *mutex) static __inline__ void _normal_lock(pthread_mutex_t* mutex) { + /* We need to preserve the shared flag during operations */ + int shared = mutex->value & MUTEX_SHARED_MASK; /* * The common case is an unlocked mutex, so we begin by trying to * change the lock's state from 0 to 1. __atomic_cmpxchg() returns 0 * if it made the swap successfully. If the result is nonzero, this * lock is already held by another thread. */ - if (__atomic_cmpxchg(0, 1, &mutex->value ) != 0) { + if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value ) != 0) { /* * We want to go to sleep until the mutex is available, which * requires promoting it to state 2. We need to swap in the new @@ -882,8 +932,10 @@ _normal_lock(pthread_mutex_t* mutex) * that the mutex is in state 2 when we go to sleep on it, which * guarantees a wake-up call. */ - while (__atomic_swap(2, &mutex->value ) != 0) - __futex_wait(&mutex->value, 2, 0); + int wait_op = shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE; + + while (__atomic_swap(shared|2, &mutex->value ) != (shared|0)) + __futex_syscall4(&mutex->value, wait_op, shared|2, 0); } } @@ -894,12 +946,16 @@ _normal_lock(pthread_mutex_t* mutex) static __inline__ void _normal_unlock(pthread_mutex_t* mutex) { + /* We need to preserve the shared flag during operations */ + int shared = mutex->value & MUTEX_SHARED_MASK; + /* - * The mutex value will be 1 or (rarely) 2. We use an atomic decrement + * The mutex state will be 1 or (rarely) 2. We use an atomic decrement * to release the lock. __atomic_dec() returns the previous value; * if it wasn't 1 we have to do some additional work. */ - if (__atomic_dec(&mutex->value) != 1) { + if (__atomic_dec(&mutex->value) != (shared|1)) { + int wake_op = shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE; /* * Start by releasing the lock. The decrement changed it from * "contended lock" to "uncontended lock", which means we still @@ -914,7 +970,7 @@ _normal_unlock(pthread_mutex_t* mutex) * _normal_lock(), because the __futex_wait() call there will * return immediately if the mutex value isn't 2. */ - mutex->value = 0; + mutex->value = shared; /* * Wake up one waiting thread. We don't know which thread will be @@ -937,7 +993,7 @@ _normal_unlock(pthread_mutex_t* mutex) * Either way we have correct behavior and nobody is orphaned on * the wait queue. */ - __futex_wake(&mutex->value, 1); + __futex_syscall3(&mutex->value, wake_op, 1); } } @@ -946,182 +1002,188 @@ static pthread_mutex_t __recursive_lock = PTHREAD_MUTEX_INITIALIZER; static void _recursive_lock(void) { - _normal_lock( &__recursive_lock); + _normal_lock(&__recursive_lock); } static void _recursive_unlock(void) { - _normal_unlock( &__recursive_lock ); + _normal_unlock(&__recursive_lock ); } -#define __likely(cond) __builtin_expect(!!(cond), 1) -#define __unlikely(cond) __builtin_expect(!!(cond), 0) - int pthread_mutex_lock(pthread_mutex_t *mutex) { - if (__likely(mutex != NULL)) - { - int mtype = (mutex->value & MUTEX_TYPE_MASK); + int mtype, tid, new_lock_type, shared, wait_op; - if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) { - _normal_lock(mutex); - } - else - { - int tid = __get_thread()->kernel_id; + if (__unlikely(mutex == NULL)) + return EINVAL; - if ( tid == MUTEX_OWNER(mutex) ) - { - int oldv, counter; + mtype = (mutex->value & MUTEX_TYPE_MASK); + shared = (mutex->value & MUTEX_SHARED_MASK); - if (mtype == MUTEX_TYPE_ERRORCHECK) { - /* trying to re-lock a mutex we already acquired */ - return EDEADLK; - } - /* - * We own the mutex, but other threads are able to change - * the contents (e.g. promoting it to "contended"), so we - * need to hold the global lock. - */ - _recursive_lock(); - oldv = mutex->value; - counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK; - mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter; - _recursive_unlock(); - } - else - { - /* - * If the new lock is available immediately, we grab it in - * the "uncontended" state. - */ - int new_lock_type = 1; - - for (;;) { - int oldv; - - _recursive_lock(); - oldv = mutex->value; - if (oldv == mtype) { /* uncontended released lock => 1 or 2 */ - mutex->value = ((tid << 16) | mtype | new_lock_type); - } else if ((oldv & 3) == 1) { /* locked state 1 => state 2 */ - oldv ^= 3; - mutex->value = oldv; - } - _recursive_unlock(); - - if (oldv == mtype) - break; - - /* - * The lock was held, possibly contended by others. From - * now on, if we manage to acquire the lock, we have to - * assume that others are still contending for it so that - * we'll wake them when we unlock it. - */ - new_lock_type = 2; - - __futex_wait( &mutex->value, oldv, 0 ); - } - } + /* Handle normal case first */ + if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) { + _normal_lock(mutex); + return 0; + } + + /* Do we already own this recursive or error-check mutex ? */ + tid = __get_thread()->kernel_id; + if ( tid == MUTEX_OWNER(mutex) ) + { + int oldv, counter; + + if (mtype == MUTEX_TYPE_ERRORCHECK) { + /* trying to re-lock a mutex we already acquired */ + return EDEADLK; } + /* + * We own the mutex, but other threads are able to change + * the contents (e.g. promoting it to "contended"), so we + * need to hold the global lock. + */ + _recursive_lock(); + oldv = mutex->value; + counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK; + mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter; + _recursive_unlock(); return 0; } - return EINVAL; + + /* We don't own the mutex, so try to get it. + * + * First, we try to change its state from 0 to 1, if this + * doesn't work, try to change it to state 2. + */ + new_lock_type = 1; + + /* compute futex wait opcode and restore shared flag in mtype */ + wait_op = shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE; + mtype |= shared; + + for (;;) { + int oldv; + + _recursive_lock(); + oldv = mutex->value; + if (oldv == mtype) { /* uncontended released lock => 1 or 2 */ + mutex->value = ((tid << 16) | mtype | new_lock_type); + } else if ((oldv & 3) == 1) { /* locked state 1 => state 2 */ + oldv ^= 3; + mutex->value = oldv; + } + _recursive_unlock(); + + if (oldv == mtype) + break; + + /* + * The lock was held, possibly contended by others. From + * now on, if we manage to acquire the lock, we have to + * assume that others are still contending for it so that + * we'll wake them when we unlock it. + */ + new_lock_type = 2; + + __futex_syscall4(&mutex->value, wait_op, oldv, NULL); + } + return 0; } int pthread_mutex_unlock(pthread_mutex_t *mutex) { - if (__likely(mutex != NULL)) - { - int mtype = (mutex->value & MUTEX_TYPE_MASK); + int mtype, tid, oldv, shared; - if (__likely(mtype == MUTEX_TYPE_NORMAL)) { - _normal_unlock(mutex); - } - else - { - int tid = __get_thread()->kernel_id; + if (__unlikely(mutex == NULL)) + return EINVAL; - if ( tid == MUTEX_OWNER(mutex) ) - { - int oldv; - - _recursive_lock(); - oldv = mutex->value; - if (oldv & MUTEX_COUNTER_MASK) { - mutex->value = oldv - (1 << MUTEX_COUNTER_SHIFT); - oldv = 0; - } else { - mutex->value = mtype; - } - _recursive_unlock(); + mtype = (mutex->value & MUTEX_TYPE_MASK); + shared = (mutex->value & MUTEX_SHARED_MASK); - if ((oldv & 3) == 2) - __futex_wake( &mutex->value, 1 ); - } - else { - /* trying to unlock a lock we do not own */ - return EPERM; - } - } + /* Handle common case first */ + if (__likely(mtype == MUTEX_TYPE_NORMAL)) { + _normal_unlock(mutex); return 0; } - return EINVAL; + + /* Do we already own this recursive or error-check mutex ? */ + tid = __get_thread()->kernel_id; + if ( tid != MUTEX_OWNER(mutex) ) + return EPERM; + + /* We do, decrement counter or release the mutex if it is 0 */ + _recursive_lock(); + oldv = mutex->value; + if (oldv & MUTEX_COUNTER_MASK) { + mutex->value = oldv - (1 << MUTEX_COUNTER_SHIFT); + oldv = 0; + } else { + mutex->value = shared | mtype; + } + _recursive_unlock(); + + /* Wake one waiting thread, if any */ + if ((oldv & 3) == 2) { + int wake_op = shared ? FUTEX_WAIT_PRIVATE : FUTEX_WAIT; + __futex_syscall3(&mutex->value, wake_op, 1); + } + return 0; } int pthread_mutex_trylock(pthread_mutex_t *mutex) { - if (__likely(mutex != NULL)) + int mtype, tid, oldv, shared; + + if (__unlikely(mutex == NULL)) + return EINVAL; + + mtype = (mutex->value & MUTEX_TYPE_MASK); + shared = (mutex->value & MUTEX_SHARED_MASK); + + /* Handle common case first */ + if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) { - int mtype = (mutex->value & MUTEX_TYPE_MASK); + if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value) == 0) + return 0; - if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) - { - if (__atomic_cmpxchg(0, 1, &mutex->value) == 0) - return 0; + return EBUSY; + } - return EBUSY; - } - else - { - int tid = __get_thread()->kernel_id; - int oldv; + /* Do we already own this recursive or error-check mutex ? */ + tid = __get_thread()->kernel_id; + if ( tid == MUTEX_OWNER(mutex) ) + { + int counter; - if ( tid == MUTEX_OWNER(mutex) ) - { - int oldv, counter; + if (mtype == MUTEX_TYPE_ERRORCHECK) { + /* already locked by ourselves */ + return EDEADLK; + } - if (mtype == MUTEX_TYPE_ERRORCHECK) { - /* already locked by ourselves */ - return EDEADLK; - } + _recursive_lock(); + oldv = mutex->value; + counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK; + mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter; + _recursive_unlock(); + return 0; + } - _recursive_lock(); - oldv = mutex->value; - counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK; - mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter; - _recursive_unlock(); - return 0; - } + /* Restore sharing bit in mtype */ + mtype |= shared; - /* try to lock it */ - _recursive_lock(); - oldv = mutex->value; - if (oldv == mtype) /* uncontended released lock => state 1 */ - mutex->value = ((tid << 16) | mtype | 1); - _recursive_unlock(); + /* Try to lock it, just once. */ + _recursive_lock(); + oldv = mutex->value; + if (oldv == mtype) /* uncontended released lock => state 1 */ + mutex->value = ((tid << 16) | mtype | 1); + _recursive_unlock(); - if (oldv != mtype) - return EBUSY; + if (oldv != mtype) + return EBUSY; - return 0; - } - } - return EINVAL; + return 0; } @@ -1164,100 +1226,150 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs) clockid_t clock = CLOCK_MONOTONIC; struct timespec abstime; struct timespec ts; + int mtype, tid, oldv, new_lock_type, shared, wait_op; /* compute absolute expiration time */ __timespec_to_relative_msec(&abstime, msecs, clock); - if (__likely(mutex != NULL)) - { - int mtype = (mutex->value & MUTEX_TYPE_MASK); + if (__unlikely(mutex == NULL)) + return EINVAL; - if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) - { - /* fast path for unconteded lock */ - if (__atomic_cmpxchg(0, 1, &mutex->value) == 0) - return 0; + mtype = (mutex->value & MUTEX_TYPE_MASK); + shared = (mutex->value & MUTEX_SHARED_MASK); - /* loop while needed */ - while (__atomic_swap(2, &mutex->value) != 0) { - if (__timespec_to_absolute(&ts, &abstime, clock) < 0) - return EBUSY; + /* Handle common case first */ + if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) + { + int wait_op = shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE; - __futex_wait(&mutex->value, 2, &ts); - } + /* fast path for unconteded lock */ + if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value) == 0) return 0; + + /* loop while needed */ + while (__atomic_swap(shared|2, &mutex->value) != (shared|0)) { + if (__timespec_to_absolute(&ts, &abstime, clock) < 0) + return EBUSY; + + __futex_syscall4(&mutex->value, wait_op, shared|2, &ts); } - else - { - int tid = __get_thread()->kernel_id; - int oldv; + return 0; + } - if ( tid == MUTEX_OWNER(mutex) ) - { - int oldv, counter; + /* Do we already own this recursive or error-check mutex ? */ + tid = __get_thread()->kernel_id; + if ( tid == MUTEX_OWNER(mutex) ) + { + int oldv, counter; - if (mtype == MUTEX_TYPE_ERRORCHECK) { - /* already locked by ourselves */ - return EDEADLK; - } + if (mtype == MUTEX_TYPE_ERRORCHECK) { + /* already locked by ourselves */ + return EDEADLK; + } - _recursive_lock(); - oldv = mutex->value; - counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK; - mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter; - _recursive_unlock(); - return 0; - } - else - { - /* - * If the new lock is available immediately, we grab it in - * the "uncontended" state. - */ - int new_lock_type = 1; - - for (;;) { - int oldv; - struct timespec ts; - - _recursive_lock(); - oldv = mutex->value; - if (oldv == mtype) { /* uncontended released lock => 1 or 2 */ - mutex->value = ((tid << 16) | mtype | new_lock_type); - } else if ((oldv & 3) == 1) { /* locked state 1 => state 2 */ - oldv ^= 3; - mutex->value = oldv; - } - _recursive_unlock(); - - if (oldv == mtype) - break; - - /* - * The lock was held, possibly contended by others. From - * now on, if we manage to acquire the lock, we have to - * assume that others are still contending for it so that - * we'll wake them when we unlock it. - */ - new_lock_type = 2; - - if (__timespec_to_absolute(&ts, &abstime, clock) < 0) - return EBUSY; - - __futex_wait( &mutex->value, oldv, &ts ); - } - return 0; - } + _recursive_lock(); + oldv = mutex->value; + counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK; + mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter; + _recursive_unlock(); + return 0; + } + + /* We don't own the mutex, so try to get it. + * + * First, we try to change its state from 0 to 1, if this + * doesn't work, try to change it to state 2. + */ + new_lock_type = 1; + + /* Compute wait op and restore sharing bit in mtype */ + wait_op = shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE; + mtype |= shared; + + for (;;) { + int oldv; + struct timespec ts; + + _recursive_lock(); + oldv = mutex->value; + if (oldv == mtype) { /* uncontended released lock => 1 or 2 */ + mutex->value = ((tid << 16) | mtype | new_lock_type); + } else if ((oldv & 3) == 1) { /* locked state 1 => state 2 */ + oldv ^= 3; + mutex->value = oldv; } + _recursive_unlock(); + + if (oldv == mtype) + break; + + /* + * The lock was held, possibly contended by others. From + * now on, if we manage to acquire the lock, we have to + * assume that others are still contending for it so that + * we'll wake them when we unlock it. + */ + new_lock_type = 2; + + if (__timespec_to_absolute(&ts, &abstime, clock) < 0) + return EBUSY; + + __futex_syscall4(&mutex->value, wait_op, oldv, &ts); } - return EINVAL; + return 0; +} + +int pthread_condattr_init(pthread_condattr_t *attr) +{ + if (attr == NULL) + return EINVAL; + + *attr = PTHREAD_PROCESS_PRIVATE; + return 0; +} + +int pthread_condattr_getpshared(pthread_condattr_t *attr, int *pshared) +{ + if (attr == NULL || pshared == NULL) + return EINVAL; + + *pshared = *attr; + return 0; +} + +int pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared) +{ + if (attr == NULL) + return EINVAL; + + if (pshared != PTHREAD_PROCESS_SHARED && + pshared != PTHREAD_PROCESS_PRIVATE) + return EINVAL; + + *attr = pshared; + return 0; +} + +int pthread_condattr_destroy(pthread_condattr_t *attr) +{ + if (attr == NULL) + return EINVAL; + + *attr = 0xdeada11d; + return 0; } +/* We use one bit in condition variable values as the 'shared' flag + * The rest is a counter. + */ +#define COND_SHARING_MASK 0x0001 +#define COND_COUNTER_INCREMENT 0x0002 +#define COND_COUNTER_MASK (~COND_SHARING_MASK) /* XXX *technically* there is a race condition that could allow * XXX a signal to be missed. If thread A is preempted in _wait() * XXX after unlocking the mutex and before waiting, and if other - * XXX threads call signal or broadcast UINT_MAX times (exactly), + * XXX threads call signal or broadcast UINT_MAX/2 times (exactly), * XXX before thread A is scheduled again and calls futex_wait(), * XXX then the signal will be lost. */ @@ -1265,26 +1377,59 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs) int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) { + if (cond == NULL) + return EINVAL; + cond->value = 0; + + if (attr != NULL && *attr == PTHREAD_PROCESS_SHARED) + cond->value |= COND_SHARING_MASK; + return 0; } int pthread_cond_destroy(pthread_cond_t *cond) { + if (cond == NULL) + return EINVAL; + cond->value = 0xdeadc04d; return 0; } +/* This function is used by pthread_cond_broadcast and + * pthread_cond_signal to atomically decrement the counter. + */ +static void +__pthread_cond_pulse(pthread_cond_t *cond) +{ + long flags = (cond->value & ~COND_COUNTER_MASK); + + for (;;) { + long oldval = cond->value; + long newval = ((oldval - COND_COUNTER_INCREMENT) & COND_COUNTER_MASK) + | flags; + if (__atomic_cmpxchg(oldval, newval, &cond->value) == 0) + break; + } +} + int pthread_cond_broadcast(pthread_cond_t *cond) { - __atomic_dec(&cond->value); + if (__unlikely(cond == NULL)) + return EINVAL; + + __pthread_cond_pulse(cond); __futex_wake(&cond->value, INT_MAX); return 0; } int pthread_cond_signal(pthread_cond_t *cond) { - __atomic_dec(&cond->value); + if (__unlikely(cond == NULL)) + return EINVAL; + + __pthread_cond_pulse(cond); __futex_wake(&cond->value, 1); return 0; } diff --git a/libc/docs/CHANGES.TXT b/libc/docs/CHANGES.TXT index 4f87547..08b2009 100644 --- a/libc/docs/CHANGES.TXT +++ b/libc/docs/CHANGES.TXT @@ -78,7 +78,18 @@ Differences between current and Android 2.1: - DNS: get rid of spurious random DNS queries when trying to resolve an unknown domain name. Due to an initialization bug, a random DNS search - list was generated for each thread is net.dns.search is not defined. + list was generated for each thread if net.dns.search is not defined. + +- <pthread.h>: Add pthread_condattr_init/destroy/setpshared/getpshared functions + to enable proper shared conditional variable initialization. + + Modify the pthread_mutex_t implementation to use private futexes for + performance reasons. Mutexes are no longer shareable between processes + by default anymore, unless you use PTHREAD_PROCESS_SHARED with + pthread_mutexattr_setpshared(). + + XXX: The implementation still always uses shared condvars, independent + of the flags being selected. This will change in a later commit. ------------------------------------------------------------------------------- Differences between Android 2.1 and 2.0.1: diff --git a/libc/include/pthread.h b/libc/include/pthread.h index 6603b3f..eb2d169 100644 --- a/libc/include/pthread.h +++ b/libc/include/pthread.h @@ -165,6 +165,11 @@ int pthread_mutex_unlock(pthread_mutex_t *mutex); int pthread_mutex_trylock(pthread_mutex_t *mutex); int pthread_mutex_timedlock(pthread_mutex_t *mutex, struct timespec* ts); +int pthread_condattr_init(pthread_condattr_t *attr); +int pthread_condattr_getpshared(pthread_condattr_t *attr, int *pshared); +int pthread_condattr_setpshared(pthread_condattr_t* attr, int pshared); +int pthread_condattr_destroy(pthread_condattr_t *attr); + int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr); int pthread_cond_destroy(pthread_cond_t *cond); |