1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
|
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "atomic.h"
#define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__)
#if NEED_SWAP_MUTEXES
#include <vector>
#include "base/mutex.h"
#include "base/stl_util.h"
#include "base/stringprintf.h"
#include "thread.h"
#endif
namespace art {
#if NEED_SWAP_MUTEXES
// We stripe across a bunch of different mutexes to reduce contention.
static const size_t kSwapMutexCount = 32;
static std::vector<Mutex*>* gSwapMutexes;
static Mutex& GetSwapMutex(const volatile int64_t* addr) {
return *(*gSwapMutexes)[(reinterpret_cast<unsigned>(addr) >> 3U) % kSwapMutexCount];
}
#endif
void QuasiAtomic::Startup() {
#if NEED_SWAP_MUTEXES
gSwapMutexes = new std::vector<Mutex*>;
for (size_t i = 0; i < kSwapMutexCount; ++i) {
gSwapMutexes->push_back(new Mutex("QuasiAtomic stripe"));
}
#endif
}
void QuasiAtomic::Shutdown() {
#if NEED_SWAP_MUTEXES
STLDeleteElements(gSwapMutexes);
delete gSwapMutexes;
#endif
}
int64_t QuasiAtomic::Read64(volatile const int64_t* addr) {
int64_t value;
#if NEED_SWAP_MUTEXES
MutexLock mu(Thread::Current(), GetSwapMutex(addr));
value = *addr;
#elif defined(__arm__)
// Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we
// have LPAE (such as Cortex-A15) then ldrd would suffice.
__asm__ __volatile__("@ QuasiAtomic::Read64\n"
"ldrexd %0, %H0, [%1]"
: "=&r" (value)
: "r" (addr));
#elif defined(__i386__)
__asm__ __volatile__(
"movq %1, %0\n"
: "=x" (value)
: "m" (*addr));
#else
#error Unexpected architecture
#endif
return value;
}
void QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) {
#if NEED_SWAP_MUTEXES
MutexLock mu(Thread::Current(), GetSwapMutex(addr));
*addr = value;
#elif defined(__arm__)
// The write is done as a swap so that the cache-line is in the exclusive state for the store. If
// we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will
// suffice.
int64_t prev;
int status;
do {
__asm__ __volatile__("@ QuasiAtomic::Write64\n"
"ldrexd %0, %H0, [%3]\n"
"strexd %1, %4, %H4, [%3]"
: "=&r" (prev), "=&r" (status), "+m"(*addr)
: "r" (addr), "r" (value)
: "cc");
} while (__builtin_expect(status != 0, 0));
#elif defined(__i386__)
__asm__ __volatile__(
"movq %1, %0"
: "=m" (*addr)
: "x" (value));
#else
#error Unexpected architecture
#endif
}
bool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) {
#if NEED_SWAP_MUTEXES
MutexLock mu(Thread::Current(), GetSwapMutex(addr));
if (*addr == old_value) {
*addr = new_value;
return true;
}
return false;
#elif defined(__arm__)
int64_t prev;
int status;
do {
__asm__ __volatile__("@ QuasiAtomic::Cas64\n"
"ldrexd %0, %H0, [%3]\n"
"mov %1, #0\n"
"teq %0, %4\n"
"teqeq %H0, %H4\n"
"strexdeq %1, %5, %H5, [%3]"
: "=&r" (prev), "=&r" (status), "+m"(*addr)
: "r" (addr), "Ir" (old_value), "r" (new_value)
: "cc");
} while (__builtin_expect(status != 0, 0));
return prev == old_value;
#elif defined(__i386__)
// The compiler does the right job and works better than inline assembly, especially with -O0
// compilation.
return __sync_bool_compare_and_swap(addr, old_value, new_value);
#else
#error Unexpected architecture
#endif
}
bool QuasiAtomic::LongAtomicsUseMutexes() {
#if NEED_SWAP_MUTEXES
return true;
#else
return false;
#endif
}
} // namespace art
|
