Use private futexes for pthread_mutex_t.

This does not change the implementation of conditional variables
since we're waiting for other system components to properly use
pthread_condattr_init/setpshared before that.

Also remove an obsolete x86 source file.

Change-Id: Ia3e3fbac35b87a534fb04d4381c3c66b975bc8f7

6 files changed, 208 insertions, 167 deletions

diff --git a/libc/arch-arm/bionic/atomics_arm.S b/libc/arch-arm/bionic/atomics_arm.S
index 55c642f..047541f 100644
--- a/libc/arch-arm/bionic/atomics_arm.S
+++ b/libc/arch-arm/bionic/atomics_arm.S
@@ -35,10 +35,6 @@
 .type __atomic_dec, %function
 .global __atomic_inc
 .type __atomic_inc, %function
-.global __futex_wait
-.type __futex_wait, %function
-.global __futex_wake
-.type __futex_wake, %function
 
 #define FUTEX_WAIT 0
 #define FUTEX_WAKE 1
@@ -159,10 +155,34 @@ __atomic_swap:
     bx      lr
 
 /* __futex_wait(*ftx, val, *timespec) */
-/* __futex_syscall(*ftx, op, val, *timespec, *addr2, val3) */
+/* __futex_wake(*ftx, counter) */
+/* __futex_syscall3(*ftx, op, val) */
+/* __futex_syscall4(*ftx, op, val, *timespec) */
+
+.global __futex_wait
+.type __futex_wait, %function
+
+.global __futex_wake
+.type __futex_wake, %function
+
+.global __futex_syscall3
+.type __futex_syscall3, %function
+
+.global __futex_syscall4
+.type __futex_syscall4, %function
 
 #if __ARM_EABI__
 
+__futex_syscall3:
+    .fnstart
+    stmdb   sp!, {r4, r7}
+    .save   {r4, r7}
+    ldr     r7, =__NR_futex
+    swi     #0
+    ldmia   sp!, {r4, r7}
+    bx      lr
+    .fnend
+
 __futex_wait:
     .fnstart
     stmdb   sp!, {r4, r7}
@@ -187,6 +207,10 @@ __futex_wake:
 
 #else
 
+__futex_syscall3:
+    swi     #__NR_futex
+    bx      lr
+
 __futex_wait:
     mov     r3, r2
     mov     r2, r1
@@ -201,3 +225,6 @@ __futex_wake:
     bx      lr
 
 #endif
+
+__futex_syscall4:
+    b __futex_syscall3
diff --git a/libc/arch-sh/bionic/atomics_sh.c b/libc/arch-sh/bionic/atomics_sh.c
index 16966f7..c7815ff 100644
--- a/libc/arch-sh/bionic/atomics_sh.c
+++ b/libc/arch-sh/bionic/atomics_sh.c
@@ -98,3 +98,13 @@ int __futex_wake(volatile void *ftx, int count)
 {
     return futex(ftx, FUTEX_WAKE, count, NULL, NULL, 0);
 }
+
+int __futex_syscall3(volatile void *ftx, int op, int val)
+{
+    return futex(ftx, op, val, NULL, NULL, 0);
+}
+
+int __futex_syscall4(volative void *ftx, int op, int val, const struct timespec *timeout)
+{
+    return futex(ftx, op, val, (void *)timeout, NULL, 0);
+}
diff --git a/libc/arch-x86/bionic/atomics_x86.S b/libc/arch-x86/bionic/atomics_x86.S
index 2370f23..666e182 100644
--- a/libc/arch-x86/bionic/atomics_x86.S
+++ b/libc/arch-x86/bionic/atomics_x86.S
@@ -41,6 +41,38 @@ __futex_wake:
     popl    %ebx
     ret
 
+/* int __futex_syscall3(volatile void *ftx, int op, int count) */
+.text
+.globl __futex_syscall3
+.type __futex_syscall3, @function
+.align 4
+__futex_syscall3:
+    pushl   %ebx
+    movl    8(%esp), %ebx      /* ftx */
+    movl    12(%esp), %ecx      /* op */
+    movl    16(%esp), %edx      /* value */
+    movl    $__NR_futex, %eax
+    int     $0x80
+    popl    %ebx
+    ret
+
+/* int __futex_syscall4(volatile void *ftx, int op, int val, const struct timespec *timeout) */
+.text
+.globl __futex_syscall4
+.type __futex_syscall4, @function
+.align 4
+__futex_syscall4:
+    pushl   %ebx
+    pushl   %esi
+    movl    12(%esp), %ebx      /* ftx */
+    movl    16(%esp), %ecx      /* op */
+    movl    20(%esp), %edx      /* val */
+    movl    24(%esp), %esi      /* timeout */
+    movl    $__NR_futex, %eax
+    int     $0x80
+    popl    %esi
+    popl    %ebx
+    ret
 
 /* int __atomic_cmpxchg(int old, int new, volatile int* addr) */
 
diff --git a/libc/arch-x86/bionic/atomics_x86.c b/libc/arch-x86/bionic/atomics_x86.c
deleted file mode 100644
index b7b20e6..0000000
--- a/libc/arch-x86/bionic/atomics_x86.c
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright (C) 2008 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *  * Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-#include <sys/atomics.h>
-
-#define FUTEX_SYSCALL 240
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-
-int __futex_wait(volatile void *ftx, int val)
-{
-    int ret;
-    asm volatile (
-        "int $0x80;"
-        : "=a" (ret)
-        : "0" (FUTEX_SYSCALL),
-          "b" (ftx),
-          "c" (FUTEX_WAIT),
-          "d" (val),
-          "S" (0)
-    );
-    return ret;
-}
-
-int __futex_wake(volatile void *ftx, int count)
-{
-    int ret;
-    asm volatile (
-        "int $0x80;"
-        : "=a" (ret)
-        : "0" (FUTEX_SYSCALL),
-          "b" (ftx),
-          "c" (FUTEX_WAKE),
-          "d" (count)
-    );
-    return ret;
-}
-
-int __atomic_cmpxchg(int old, int new, volatile int* addr) {
-    int xchg;
-    asm volatile (
-        "lock;"
-        "cmpxchg %%ecx, (%%edx);"
-        "setne %%al;"
-        : "=a" (xchg)
-        : "a" (old),
-          "c" (new),
-          "d" (addr)
-    );
-    return xchg;
-}
-
-int __atomic_swap(int new, volatile int* addr) {
-    int old;
-    asm volatile (
-        "lock;"
-        "xchg %%ecx, (%%edx);"
-        : "=c" (old)
-        : "c" (new),
-          "d" (addr)
-    );
-    return old;
-}
-
-int __atomic_dec(volatile int* addr) {
-    int old;
-    do {
-        old = *addr;
-    } while (atomic_cmpxchg(old, old-1, addr));
-    return old;
-}
-
-int __atomic_inc(volatile int* addr) {
-    int old;
-    do {
-        old = *addr;
-    } while (atomic_cmpxchg(old, old+1, addr));
-    return old;
-}
-
diff --git a/libc/bionic/pthread.c b/libc/bionic/pthread.c
index 4ffa9b8..3294cea 100644
--- a/libc/bionic/pthread.c
+++ b/libc/bionic/pthread.c
@@ -43,12 +43,16 @@
 #include <memory.h>
 #include <assert.h>
 #include <malloc.h>
+#include <linux/futex.h>
 
 extern int  __pthread_clone(int (*fn)(void*), void *child_stack, int flags, void *arg);
 extern void _exit_with_stack_teardown(void * stackBase, int stackSize, int retCode);
 extern void _exit_thread(int  retCode);
 extern int  __set_errno(int);
 
+#define  __likely(cond)    __builtin_expect(!!(cond), 1)
+#define  __unlikely(cond)  __builtin_expect(!!(cond), 0)
+
 void _thread_created_hook(pid_t thread_id) __attribute__((noinline));
 
 #define PTHREAD_ATTR_FLAG_DETACHED      0x00000001
@@ -712,6 +716,21 @@ int pthread_setschedparam(pthread_t thid, int policy,
 int __futex_wait(volatile void *ftx, int val, const struct timespec *timeout);
 int __futex_wake(volatile void *ftx, int count);
 
+int __futex_syscall3(volatile void *ftx, int op, int val);
+int __futex_syscall4(volatile void *ftx, int op, int val, const struct timespec *timeout);
+
+#ifndef FUTEX_PRIVATE_FLAG
+#define FUTEX_PRIVATE_FLAG  128
+#endif
+
+#ifndef FUTEX_WAIT_PRIVATE
+#define FUTEX_WAIT_PRIVATE  (FUTEX_WAIT|FUTEX_PRIVATE_FLAG)
+#endif
+
+#ifndef FUTEX_WAKE_PRIVATE
+#define FUTEX_WAKE_PRIVATE  (FUTEX_WAKE|FUTEX_PRIVATE_FLAG)
+#endif
+
 // mutex lock states
 //
 // 0: unlocked
@@ -723,7 +742,8 @@ int __futex_wake(volatile void *ftx, int count);
  * bits:     name     description
  * 31-16     tid      owner thread's kernel id (recursive and errorcheck only)
  * 15-14     type     mutex type
- * 13-2      counter  counter of recursive mutexes
+ * 13        shared   process-shared flag
+ * 12-2      counter  counter of recursive mutexes
  * 1-0       state    lock state (0, 1 or 2)
  */
 
@@ -737,9 +757,17 @@ int __futex_wake(volatile void *ftx, int count);
 #define  MUTEX_TYPE_ERRORCHECK 0x8000
 
 #define  MUTEX_COUNTER_SHIFT  2
-#define  MUTEX_COUNTER_MASK   0x3ffc
-
+#define  MUTEX_COUNTER_MASK   0x1ffc
+#define  MUTEX_SHARED_MASK    0x2000
 
+/* a mutex attribute holds the following fields
+ *
+ * bits:     name       description
+ * 0-3       type       type of mutex
+ * 4         shared     process-shared flag
+ */
+#define  MUTEXATTR_TYPE_MASK   0x000f
+#define  MUTEXATTR_SHARED_MASK 0x0010
 
 
 int pthread_mutexattr_init(pthread_mutexattr_t *attr)
@@ -764,10 +792,14 @@ int pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
 
 int pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type)
 {
-    if (attr && *attr >= PTHREAD_MUTEX_NORMAL &&
-                *attr <= PTHREAD_MUTEX_ERRORCHECK ) {
-        *type = *attr;
-        return 0;
+    if (attr) {
+        int  atype = (*attr & MUTEXATTR_TYPE_MASK);
+
+         if (atype >= PTHREAD_MUTEX_NORMAL &&
+             atype <= PTHREAD_MUTEX_ERRORCHECK) {
+            *type = atype;
+            return 0;
+        }
     }
     return EINVAL;
 }
@@ -776,7 +808,7 @@ int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
 {
     if (attr && type >= PTHREAD_MUTEX_NORMAL &&
                 type <= PTHREAD_MUTEX_ERRORCHECK ) {
-        *attr = type;
+        *attr = (*attr & ~MUTEXATTR_TYPE_MASK) | type;
         return 0;
     }
     return EINVAL;
@@ -791,54 +823,70 @@ int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int  pshared)
 
     switch (pshared) {
     case PTHREAD_PROCESS_PRIVATE:
+        *attr &= ~MUTEXATTR_SHARED_MASK;
+        return 0;
+
     case PTHREAD_PROCESS_SHARED:
         /* our current implementation of pthread actually supports shared
          * mutexes but won't cleanup if a process dies with the mutex held.
          * Nevertheless, it's better than nothing. Shared mutexes are used
          * by surfaceflinger and audioflinger.
          */
+        *attr |= MUTEXATTR_SHARED_MASK;
         return 0;
     }
-
-    return ENOTSUP;
+    return EINVAL;
 }
 
 int pthread_mutexattr_getpshared(pthread_mutexattr_t *attr, int *pshared)
 {
-    if (!attr)
+    if (!attr || !pshared)
         return EINVAL;
 
-    *pshared = PTHREAD_PROCESS_PRIVATE;
+    *pshared = (*attr & MUTEXATTR_SHARED_MASK) ? PTHREAD_PROCESS_SHARED
+                                               : PTHREAD_PROCESS_PRIVATE;
     return 0;
 }
 
 int pthread_mutex_init(pthread_mutex_t *mutex,
                        const pthread_mutexattr_t *attr)
 {
-    if ( mutex ) {
-        if (attr == NULL) {
-            mutex->value = MUTEX_TYPE_NORMAL;
-            return 0;
-        }
-        switch ( *attr ) {
-        case PTHREAD_MUTEX_NORMAL:
-            mutex->value = MUTEX_TYPE_NORMAL;
-            return 0;
+    int value = 0;
 
-        case PTHREAD_MUTEX_RECURSIVE:
-            mutex->value = MUTEX_TYPE_RECURSIVE;
-            return 0;
+    if (mutex == NULL)
+        return EINVAL;
 
-        case PTHREAD_MUTEX_ERRORCHECK:
-            mutex->value = MUTEX_TYPE_ERRORCHECK;
-            return 0;
-        }
+    if (__likely(attr == NULL)) {
+        mutex->value = MUTEX_TYPE_NORMAL;
+        return 0;
     }
-    return EINVAL;
+
+    if ((*attr & MUTEXATTR_SHARED_MASK) != 0)
+        value |= MUTEX_SHARED_MASK;
+
+    switch (*attr & MUTEXATTR_TYPE_MASK) {
+    case PTHREAD_MUTEX_NORMAL:
+        value |= MUTEX_TYPE_NORMAL;
+        break;
+    case PTHREAD_MUTEX_RECURSIVE:
+        value |= MUTEX_TYPE_RECURSIVE;
+        break;
+    case PTHREAD_MUTEX_ERRORCHECK:
+        value |= MUTEX_TYPE_ERRORCHECK;
+        break;
+    default:
+        return EINVAL;
+    }
+
+    mutex->value = value;
+    return 0;
 }
 
 int pthread_mutex_destroy(pthread_mutex_t *mutex)
 {
+    if (__unlikely(mutex == NULL))
+        return EINVAL;
+
     mutex->value = 0xdead10cc;
     return 0;
 }
@@ -859,13 +907,15 @@ int pthread_mutex_destroy(pthread_mutex_t *mutex)
 static __inline__ void
 _normal_lock(pthread_mutex_t*  mutex)
 {
+    /* We need to preserve the shared flag during operations */
+    int  shared = mutex->value & MUTEX_SHARED_MASK;
     /*
      * The common case is an unlocked mutex, so we begin by trying to
      * change the lock's state from 0 to 1.  __atomic_cmpxchg() returns 0
      * if it made the swap successfully.  If the result is nonzero, this
      * lock is already held by another thread.
      */
-    if (__atomic_cmpxchg(0, 1, &mutex->value ) != 0) {
+    if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value ) != 0) {
         /*
          * We want to go to sleep until the mutex is available, which
          * requires promoting it to state 2.  We need to swap in the new
@@ -882,8 +932,10 @@ _normal_lock(pthread_mutex_t*  mutex)
          * that the mutex is in state 2 when we go to sleep on it, which
          * guarantees a wake-up call.
          */
-        while (__atomic_swap(2, &mutex->value ) != 0)
-            __futex_wait(&mutex->value, 2, 0);
+        int  wait_op = shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE;
+
+        while (__atomic_swap(shared|2, &mutex->value ) != (shared|0))
+            __futex_syscall4(&mutex->value, wait_op, shared|2, 0);
     }
 }
 
@@ -894,12 +946,16 @@ _normal_lock(pthread_mutex_t*  mutex)
 static __inline__ void
 _normal_unlock(pthread_mutex_t*  mutex)
 {
+    /* We need to preserve the shared flag during operations */
+    int  shared = mutex->value & MUTEX_SHARED_MASK;
+
     /*
-     * The mutex value will be 1 or (rarely) 2.  We use an atomic decrement
+     * The mutex state will be 1 or (rarely) 2.  We use an atomic decrement
      * to release the lock.  __atomic_dec() returns the previous value;
      * if it wasn't 1 we have to do some additional work.
      */
-    if (__atomic_dec(&mutex->value) != 1) {
+    if (__atomic_dec(&mutex->value) != (shared|1)) {
+        int  wake_op = shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE;
         /*
          * Start by releasing the lock.  The decrement changed it from
          * "contended lock" to "uncontended lock", which means we still
@@ -914,7 +970,7 @@ _normal_unlock(pthread_mutex_t*  mutex)
          * _normal_lock(), because the __futex_wait() call there will
          * return immediately if the mutex value isn't 2.
          */
-        mutex->value = 0;
+        mutex->value = shared;
 
         /*
          * Wake up one waiting thread.  We don't know which thread will be
@@ -937,7 +993,7 @@ _normal_unlock(pthread_mutex_t*  mutex)
          * Either way we have correct behavior and nobody is orphaned on
          * the wait queue.
          */
-        __futex_wake(&mutex->value, 1);
+        __futex_syscall3(&mutex->value, wake_op, 1);
     }
 }
 
@@ -946,26 +1002,24 @@ static pthread_mutex_t  __recursive_lock = PTHREAD_MUTEX_INITIALIZER;
 static void
 _recursive_lock(void)
 {
-    _normal_lock( &__recursive_lock);
+    _normal_lock(&__recursive_lock);
 }
 
 static void
 _recursive_unlock(void)
 {
-    _normal_unlock( &__recursive_lock );
+    _normal_unlock(&__recursive_lock );
 }
 
-#define  __likely(cond)    __builtin_expect(!!(cond), 1)
-#define  __unlikely(cond)  __builtin_expect(!!(cond), 0)
-
 int pthread_mutex_lock(pthread_mutex_t *mutex)
 {
-    int mtype, tid, new_lock_type;
+    int mtype, tid, new_lock_type, shared, wait_op;
 
     if (__unlikely(mutex == NULL))
         return EINVAL;
 
     mtype = (mutex->value & MUTEX_TYPE_MASK);
+    shared = (mutex->value & MUTEX_SHARED_MASK);
 
     /* Handle normal case first */
     if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) {
@@ -1003,6 +1057,10 @@ int pthread_mutex_lock(pthread_mutex_t *mutex)
      */
     new_lock_type = 1;
 
+    /* compute futex wait opcode and restore shared flag in mtype */
+    wait_op = shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE;
+    mtype  |= shared;
+
     for (;;) {
         int  oldv;
 
@@ -1027,7 +1085,7 @@ int pthread_mutex_lock(pthread_mutex_t *mutex)
          */
         new_lock_type = 2;
 
-        __futex_wait( &mutex->value, oldv, 0 );
+        __futex_syscall4(&mutex->value, wait_op, oldv, NULL);
     }
     return 0;
 }
@@ -1035,12 +1093,13 @@ int pthread_mutex_lock(pthread_mutex_t *mutex)
 
 int pthread_mutex_unlock(pthread_mutex_t *mutex)
 {
-    int mtype, tid, oldv;
+    int mtype, tid, oldv, shared;
 
     if (__unlikely(mutex == NULL))
         return EINVAL;
 
-    mtype = (mutex->value & MUTEX_TYPE_MASK);
+    mtype  = (mutex->value & MUTEX_TYPE_MASK);
+    shared = (mutex->value & MUTEX_SHARED_MASK);
 
     /* Handle common case first */
     if (__likely(mtype == MUTEX_TYPE_NORMAL)) {
@@ -1060,31 +1119,33 @@ int pthread_mutex_unlock(pthread_mutex_t *mutex)
         mutex->value = oldv - (1 << MUTEX_COUNTER_SHIFT);
         oldv = 0;
     } else {
-        mutex->value = mtype;
+        mutex->value = shared | mtype;
     }
     _recursive_unlock();
 
     /* Wake one waiting thread, if any */
-    if ((oldv & 3) == 2)
-        __futex_wake( &mutex->value, 1 );
-
+    if ((oldv & 3) == 2) {
+        int wake_op = shared ? FUTEX_WAIT_PRIVATE : FUTEX_WAIT;
+        __futex_syscall3(&mutex->value, wake_op, 1);
+    }
     return 0;
 }
 
 
 int pthread_mutex_trylock(pthread_mutex_t *mutex)
 {
-    int mtype, tid, oldv;
+    int mtype, tid, oldv, shared;
 
     if (__unlikely(mutex == NULL))
         return EINVAL;
 
-    mtype = (mutex->value & MUTEX_TYPE_MASK);
+    mtype  = (mutex->value & MUTEX_TYPE_MASK);
+    shared = (mutex->value & MUTEX_SHARED_MASK);
 
     /* Handle common case first */
     if ( __likely(mtype == MUTEX_TYPE_NORMAL) )
     {
-        if (__atomic_cmpxchg(0, 1, &mutex->value) == 0)
+        if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value) == 0)
             return 0;
 
         return EBUSY;
@@ -1109,6 +1170,9 @@ int pthread_mutex_trylock(pthread_mutex_t *mutex)
         return 0;
     }
 
+    /* Restore sharing bit in mtype */
+    mtype |= shared;
+
     /* Try to lock it, just once. */
     _recursive_lock();
     oldv = mutex->value;
@@ -1162,7 +1226,7 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs)
     clockid_t        clock = CLOCK_MONOTONIC;
     struct timespec  abstime;
     struct timespec  ts;
-    int              mtype, tid, oldv, new_lock_type;
+    int              mtype, tid, oldv, new_lock_type, shared, wait_op;
 
     /* compute absolute expiration time */
     __timespec_to_relative_msec(&abstime, msecs, clock);
@@ -1170,21 +1234,24 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs)
     if (__unlikely(mutex == NULL))
         return EINVAL;
 
-    mtype = (mutex->value & MUTEX_TYPE_MASK);
+    mtype  = (mutex->value & MUTEX_TYPE_MASK);
+    shared = (mutex->value & MUTEX_SHARED_MASK);
 
     /* Handle common case first */
     if ( __likely(mtype == MUTEX_TYPE_NORMAL) )
     {
+        int  wait_op = shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE;
+
         /* fast path for unconteded lock */
-        if (__atomic_cmpxchg(0, 1, &mutex->value) == 0)
+        if (__atomic_cmpxchg(shared|0, shared|1, &mutex->value) == 0)
             return 0;
 
         /* loop while needed */
-        while (__atomic_swap(2, &mutex->value) != 0) {
+        while (__atomic_swap(shared|2, &mutex->value) != (shared|0)) {
             if (__timespec_to_absolute(&ts, &abstime, clock) < 0)
                 return EBUSY;
 
-            __futex_wait(&mutex->value, 2, &ts);
+            __futex_syscall4(&mutex->value, wait_op, shared|2, &ts);
         }
         return 0;
     }
@@ -1215,6 +1282,10 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs)
      */
     new_lock_type = 1;
 
+    /* Compute wait op and restore sharing bit in mtype */
+    wait_op = shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE;
+    mtype  |= shared;
+
     for (;;) {
         int  oldv;
         struct timespec  ts;
@@ -1243,7 +1314,7 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs)
         if (__timespec_to_absolute(&ts, &abstime, clock) < 0)
             return EBUSY;
 
-        __futex_wait( &mutex->value, oldv, &ts );
+        __futex_syscall4(&mutex->value, wait_op, oldv, &ts);
     }
     return 0;
 }
diff --git a/libc/docs/CHANGES.TXT b/libc/docs/CHANGES.TXT
index ec147c2..08b2009 100644
--- a/libc/docs/CHANGES.TXT
+++ b/libc/docs/CHANGES.TXT
@@ -83,6 +83,11 @@ Differences between current and Android 2.1:
 - <pthread.h>: Add pthread_condattr_init/destroy/setpshared/getpshared functions
   to enable proper shared conditional variable initialization.
 
+  Modify the pthread_mutex_t implementation to use private futexes for
+  performance reasons. Mutexes are no longer shareable between processes
+  by default anymore, unless you use PTHREAD_PROCESS_SHARED with
+  pthread_mutexattr_setpshared().
+
   XXX: The implementation still always uses shared condvars, independent
        of the flags being selected. This will change in a later commit.