From 967a0adf8b93a23d2a8fef82e06bd913db94ac19 Mon Sep 17 00:00:00 2001 From: Hiroshi Yamauchi Date: Tue, 10 Sep 2013 16:24:21 -0700 Subject: More allocation code optimizations. - Inline Class::AllocObject() and Array::Alloc(). - Inline some short Mutex functions and add LIKELY/UNLIKELY to some Mutex functions. - This change improves the Ritz MemAllocTest by ~6% on Nexus 4 and ~10% on host. Bug: 9986565 Change-Id: I1606c74ddb21676cbc1de1a40e9b076fc23eaea4 --- runtime/base/mutex-inl.h | 126 +++++++++++++++++++++++++++++++++++++++++ runtime/base/mutex.cc | 142 +++-------------------------------------------- 2 files changed, 134 insertions(+), 134 deletions(-) (limited to 'runtime/base') diff --git a/runtime/base/mutex-inl.h b/runtime/base/mutex-inl.h index 7e8365e..c0cfee2 100644 --- a/runtime/base/mutex-inl.h +++ b/runtime/base/mutex-inl.h @@ -41,6 +41,54 @@ static inline int futex(volatile int *uaddr, int op, int val, const struct times } #endif // ART_USE_FUTEXES +#if defined(__APPLE__) + +// This works on Mac OS 10.6 but hasn't been tested on older releases. +struct __attribute__((__may_alias__)) darwin_pthread_mutex_t { + long padding0; // NOLINT(runtime/int) exact match to darwin type + int padding1; + uint32_t padding2; + int16_t padding3; + int16_t padding4; + uint32_t padding5; + pthread_t darwin_pthread_mutex_owner; + // ...other stuff we don't care about. +}; + +struct __attribute__((__may_alias__)) darwin_pthread_rwlock_t { + long padding0; // NOLINT(runtime/int) exact match to darwin type + pthread_mutex_t padding1; + int padding2; + pthread_cond_t padding3; + pthread_cond_t padding4; + int padding5; + int padding6; + pthread_t darwin_pthread_rwlock_owner; + // ...other stuff we don't care about. +}; + +#endif // __APPLE__ + +#if defined(__GLIBC__) + +struct __attribute__((__may_alias__)) glibc_pthread_mutex_t { + int32_t padding0[2]; + int owner; + // ...other stuff we don't care about. +}; + +struct __attribute__((__may_alias__)) glibc_pthread_rwlock_t { +#ifdef __LP64__ + int32_t padding0[6]; +#else + int32_t padding0[7]; +#endif + int writer; + // ...other stuff we don't care about. +}; + +#endif // __GLIBC__ + class ScopedContentionRecorder { public: ScopedContentionRecorder(BaseMutex* mutex, uint64_t blocked_tid, uint64_t owner_tid) @@ -185,6 +233,84 @@ inline void ReaderWriterMutex::SharedUnlock(Thread* self) { #endif } +inline bool Mutex::IsExclusiveHeld(const Thread* self) const { + DCHECK(self == NULL || self == Thread::Current()); + bool result = (GetExclusiveOwnerTid() == SafeGetTid(self)); + if (kDebugLocking) { + // Sanity debug check that if we think it is locked we have it in our held mutexes. + if (result && self != NULL && level_ != kMonitorLock && !gAborting) { + CHECK_EQ(self->GetHeldMutex(level_), this); + } + } + return result; +} + +inline uint64_t Mutex::GetExclusiveOwnerTid() const { +#if ART_USE_FUTEXES + return exclusive_owner_; +#elif defined(__BIONIC__) + return static_cast((mutex_.value >> 16) & 0xffff); +#elif defined(__GLIBC__) + return reinterpret_cast(&mutex_)->owner; +#elif defined(__APPLE__) + const darwin_pthread_mutex_t* dpmutex = reinterpret_cast(&mutex_); + pthread_t owner = dpmutex->darwin_pthread_mutex_owner; + // 0 for unowned, -1 for PTHREAD_MTX_TID_SWITCHING + // TODO: should we make darwin_pthread_mutex_owner volatile and recheck until not -1? + if ((owner == (pthread_t)0) || (owner == (pthread_t)-1)) { + return 0; + } + uint64_t tid; + CHECK_PTHREAD_CALL(pthread_threadid_np, (owner, &tid), __FUNCTION__); // Requires Mac OS 10.6 + return tid; +#else +#error unsupported C library +#endif +} + +inline bool ReaderWriterMutex::IsExclusiveHeld(const Thread* self) const { + DCHECK(self == NULL || self == Thread::Current()); + bool result = (GetExclusiveOwnerTid() == SafeGetTid(self)); + if (kDebugLocking) { + // Sanity that if the pthread thinks we own the lock the Thread agrees. + if (self != NULL && result) { + CHECK_EQ(self->GetHeldMutex(level_), this); + } + } + return result; +} + +inline uint64_t ReaderWriterMutex::GetExclusiveOwnerTid() const { +#if ART_USE_FUTEXES + int32_t state = state_; + if (state == 0) { + return 0; // No owner. + } else if (state > 0) { + return -1; // Shared. + } else { + return exclusive_owner_; + } +#else +#if defined(__BIONIC__) + return rwlock_.writerThreadId; +#elif defined(__GLIBC__) + return reinterpret_cast(&rwlock_)->writer; +#elif defined(__APPLE__) + const darwin_pthread_rwlock_t* + dprwlock = reinterpret_cast(&rwlock_); + pthread_t owner = dprwlock->darwin_pthread_rwlock_owner; + if (owner == (pthread_t)0) { + return 0; + } + uint64_t tid; + CHECK_PTHREAD_CALL(pthread_threadid_np, (owner, &tid), __FUNCTION__); // Requires Mac OS 10.6 + return tid; +#else +#error unsupported C library +#endif +#endif +} + } // namespace art #endif // ART_RUNTIME_BASE_MUTEX_INL_H_ diff --git a/runtime/base/mutex.cc b/runtime/base/mutex.cc index b99e7c9..b048bbb 100644 --- a/runtime/base/mutex.cc +++ b/runtime/base/mutex.cc @@ -31,54 +31,6 @@ namespace art { -#if defined(__APPLE__) - -// This works on Mac OS 10.6 but hasn't been tested on older releases. -struct __attribute__((__may_alias__)) darwin_pthread_mutex_t { - long padding0; // NOLINT(runtime/int) exact match to darwin type - int padding1; - uint32_t padding2; - int16_t padding3; - int16_t padding4; - uint32_t padding5; - pthread_t darwin_pthread_mutex_owner; - // ...other stuff we don't care about. -}; - -struct __attribute__((__may_alias__)) darwin_pthread_rwlock_t { - long padding0; // NOLINT(runtime/int) exact match to darwin type - pthread_mutex_t padding1; - int padding2; - pthread_cond_t padding3; - pthread_cond_t padding4; - int padding5; - int padding6; - pthread_t darwin_pthread_rwlock_owner; - // ...other stuff we don't care about. -}; - -#endif // __APPLE__ - -#if defined(__GLIBC__) - -struct __attribute__((__may_alias__)) glibc_pthread_mutex_t { - int32_t padding0[2]; - int owner; - // ...other stuff we don't care about. -}; - -struct __attribute__((__may_alias__)) glibc_pthread_rwlock_t { -#ifdef __LP64__ - int32_t padding0[6]; -#else - int32_t padding0[7]; -#endif - int writer; - // ...other stuff we don't care about. -}; - -#endif // __GLIBC__ - #if ART_USE_FUTEXES static bool ComputeRelativeTimeSpec(timespec* result_ts, const timespec& lhs, const timespec& rhs) { const int32_t one_sec = 1000 * 1000 * 1000; // one second in nanoseconds. @@ -346,7 +298,7 @@ void Mutex::ExclusiveLock(Thread* self) { bool done = false; do { int32_t cur_state = state_; - if (cur_state == 0) { + if (LIKELY(cur_state == 0)) { // Change state from 0 to 1. done = android_atomic_acquire_cas(0, 1, &state_) == 0; } else { @@ -432,14 +384,14 @@ void Mutex::ExclusiveUnlock(Thread* self) { bool done = false; do { int32_t cur_state = state_; - if (cur_state == 1) { + if (LIKELY(cur_state == 1)) { // We're no longer the owner. exclusive_owner_ = 0; // Change state to 0. done = android_atomic_release_cas(cur_state, 0, &state_) == 0; - if (done) { // Spurious fail? + if (LIKELY(done)) { // Spurious fail? // Wake a contender - if (num_contenders_ > 0) { + if (UNLIKELY(num_contenders_ > 0)) { futex(&state_, FUTEX_WAKE, 1, NULL, NULL, 0); } } @@ -461,41 +413,6 @@ void Mutex::ExclusiveUnlock(Thread* self) { } } -bool Mutex::IsExclusiveHeld(const Thread* self) const { - DCHECK(self == NULL || self == Thread::Current()); - bool result = (GetExclusiveOwnerTid() == SafeGetTid(self)); - if (kDebugLocking) { - // Sanity debug check that if we think it is locked we have it in our held mutexes. - if (result && self != NULL && level_ != kMonitorLock && !gAborting) { - CHECK_EQ(self->GetHeldMutex(level_), this); - } - } - return result; -} - -uint64_t Mutex::GetExclusiveOwnerTid() const { -#if ART_USE_FUTEXES - return exclusive_owner_; -#elif defined(__BIONIC__) - return static_cast((mutex_.value >> 16) & 0xffff); -#elif defined(__GLIBC__) - return reinterpret_cast(&mutex_)->owner; -#elif defined(__APPLE__) - const darwin_pthread_mutex_t* dpmutex = reinterpret_cast(&mutex_); - pthread_t owner = dpmutex->darwin_pthread_mutex_owner; - // 0 for unowned, -1 for PTHREAD_MTX_TID_SWITCHING - // TODO: should we make darwin_pthread_mutex_owner volatile and recheck until not -1? - if ((owner == (pthread_t)0) || (owner == (pthread_t)-1)) { - return 0; - } - uint64_t tid; - CHECK_PTHREAD_CALL(pthread_threadid_np, (owner, &tid), __FUNCTION__); // Requires Mac OS 10.6 - return tid; -#else -#error unsupported C library -#endif -} - void Mutex::Dump(std::ostream& os) const { os << (recursive_ ? "recursive " : "non-recursive ") << name_ @@ -549,7 +466,7 @@ void ReaderWriterMutex::ExclusiveLock(Thread* self) { bool done = false; do { int32_t cur_state = state_; - if (cur_state == 0) { + if (LIKELY(cur_state == 0)) { // Change state from 0 to -1. done = android_atomic_acquire_cas(0, -1, &state_) == 0; } else { @@ -583,14 +500,14 @@ void ReaderWriterMutex::ExclusiveUnlock(Thread* self) { bool done = false; do { int32_t cur_state = state_; - if (cur_state == -1) { + if (LIKELY(cur_state == -1)) { // We're no longer the owner. exclusive_owner_ = 0; // Change state from -1 to 0. done = android_atomic_release_cas(-1, 0, &state_) == 0; - if (done) { // cmpxchg may fail due to noise? + if (LIKELY(done)) { // cmpxchg may fail due to noise? // Wake any waiters. - if (num_pending_readers_ > 0 || num_pending_writers_ > 0) { + if (UNLIKELY(num_pending_readers_ > 0 || num_pending_writers_ > 0)) { futex(&state_, FUTEX_WAKE, -1, NULL, NULL, 0); } } @@ -687,18 +604,6 @@ bool ReaderWriterMutex::SharedTryLock(Thread* self) { return true; } -bool ReaderWriterMutex::IsExclusiveHeld(const Thread* self) const { - DCHECK(self == NULL || self == Thread::Current()); - bool result = (GetExclusiveOwnerTid() == SafeGetTid(self)); - if (kDebugLocking) { - // Sanity that if the pthread thinks we own the lock the Thread agrees. - if (self != NULL && result) { - CHECK_EQ(self->GetHeldMutex(level_), this); - } - } - return result; -} - bool ReaderWriterMutex::IsSharedHeld(const Thread* self) const { DCHECK(self == NULL || self == Thread::Current()); bool result; @@ -710,37 +615,6 @@ bool ReaderWriterMutex::IsSharedHeld(const Thread* self) const { return result; } -uint64_t ReaderWriterMutex::GetExclusiveOwnerTid() const { -#if ART_USE_FUTEXES - int32_t state = state_; - if (state == 0) { - return 0; // No owner. - } else if (state > 0) { - return -1; // Shared. - } else { - return exclusive_owner_; - } -#else -#if defined(__BIONIC__) - return rwlock_.writerThreadId; -#elif defined(__GLIBC__) - return reinterpret_cast(&rwlock_)->writer; -#elif defined(__APPLE__) - const darwin_pthread_rwlock_t* - dprwlock = reinterpret_cast(&rwlock_); - pthread_t owner = dprwlock->darwin_pthread_rwlock_owner; - if (owner == (pthread_t)0) { - return 0; - } - uint64_t tid; - CHECK_PTHREAD_CALL(pthread_threadid_np, (owner, &tid), __FUNCTION__); // Requires Mac OS 10.6 - return tid; -#else -#error unsupported C library -#endif -#endif -} - void ReaderWriterMutex::Dump(std::ostream& os) const { os << name_ << " level=" << static_cast(level_) -- cgit v1.1