|
Lines 113-258
struct Atomic {
Source/WTF/wtf/Atomics.h_sec1
|
| 113 |
std::atomic<T> value; |
113 |
std::atomic<T> value; |
| 114 |
}; |
114 |
}; |
| 115 |
|
115 |
|
| 116 |
#if OS(WINDOWS) |
116 |
// This is a weak CAS function that takes a direct pointer and has no portable fencing guarantees. |
| 117 |
inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue) |
117 |
template<typename T> |
| 118 |
{ |
118 |
inline bool weakCompareAndSwap(volatile T* location, T expected, T newValue) |
| 119 |
return InterlockedCompareExchange(reinterpret_cast<LONG volatile*>(location), static_cast<LONG>(newValue), static_cast<LONG>(expected)) == static_cast<LONG>(expected); |
|
|
| 120 |
} |
| 121 |
|
| 122 |
inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue) |
| 123 |
{ |
| 124 |
return InterlockedCompareExchangePointer(location, newValue, expected) == expected; |
| 125 |
} |
| 126 |
#else // OS(WINDOWS) --> not windows |
| 127 |
inline bool weakCompareAndSwap(volatile unsigned* location, unsigned expected, unsigned newValue) |
| 128 |
{ |
| 129 |
#if ENABLE(COMPARE_AND_SWAP) |
| 130 |
#if CPU(X86) || CPU(X86_64) |
| 131 |
unsigned char result; |
| 132 |
asm volatile( |
| 133 |
"lock; cmpxchgl %3, %2\n\t" |
| 134 |
"sete %1" |
| 135 |
: "+a"(expected), "=q"(result), "+m"(*location) |
| 136 |
: "r"(newValue) |
| 137 |
: "memory" |
| 138 |
); |
| 139 |
#elif CPU(ARM_THUMB2) |
| 140 |
unsigned tmp; |
| 141 |
unsigned result; |
| 142 |
asm volatile( |
| 143 |
"movw %1, #1\n\t" |
| 144 |
"ldrex %2, %0\n\t" |
| 145 |
"cmp %3, %2\n\t" |
| 146 |
"bne.n 0f\n\t" |
| 147 |
"strex %1, %4, %0\n\t" |
| 148 |
"0:" |
| 149 |
: "+Q"(*location), "=&r"(result), "=&r"(tmp) |
| 150 |
: "r"(expected), "r"(newValue) |
| 151 |
: "memory"); |
| 152 |
result = !result; |
| 153 |
#elif CPU(ARM64) && COMPILER(GCC_OR_CLANG) |
| 154 |
unsigned tmp; |
| 155 |
unsigned result; |
| 156 |
asm volatile( |
| 157 |
"mov %w1, #1\n\t" |
| 158 |
"ldxr %w2, [%0]\n\t" |
| 159 |
"cmp %w3, %w2\n\t" |
| 160 |
"b.ne 0f\n\t" |
| 161 |
"stxr %w1, %w4, [%0]\n\t" |
| 162 |
"0:" |
| 163 |
: "+r"(location), "=&r"(result), "=&r"(tmp) |
| 164 |
: "r"(expected), "r"(newValue) |
| 165 |
: "memory"); |
| 166 |
result = !result; |
| 167 |
#elif CPU(ARM64) |
| 168 |
unsigned tmp; |
| 169 |
unsigned result; |
| 170 |
asm volatile( |
| 171 |
"mov %w1, #1\n\t" |
| 172 |
"ldxr %w2, %0\n\t" |
| 173 |
"cmp %w3, %w2\n\t" |
| 174 |
"b.ne 0f\n\t" |
| 175 |
"stxr %w1, %w4, %0\n\t" |
| 176 |
"0:" |
| 177 |
: "+m"(*location), "=&r"(result), "=&r"(tmp) |
| 178 |
: "r"(expected), "r"(newValue) |
| 179 |
: "memory"); |
| 180 |
result = !result; |
| 181 |
#else |
| 182 |
#error "Bad architecture for compare and swap." |
| 183 |
#endif |
| 184 |
return result; |
| 185 |
#else |
| 186 |
UNUSED_PARAM(location); |
| 187 |
UNUSED_PARAM(expected); |
| 188 |
UNUSED_PARAM(newValue); |
| 189 |
CRASH(); |
| 190 |
return false; |
| 191 |
#endif |
| 192 |
} |
| 193 |
|
| 194 |
inline bool weakCompareAndSwap(void*volatile* location, void* expected, void* newValue) |
| 195 |
{ |
| 196 |
#if ENABLE(COMPARE_AND_SWAP) |
| 197 |
#if CPU(X86_64) |
| 198 |
bool result; |
| 199 |
asm volatile( |
| 200 |
"lock; cmpxchgq %3, %2\n\t" |
| 201 |
"sete %1" |
| 202 |
: "+a"(expected), "=q"(result), "+m"(*location) |
| 203 |
: "r"(newValue) |
| 204 |
: "memory" |
| 205 |
); |
| 206 |
return result; |
| 207 |
#elif CPU(ARM64) && COMPILER(GCC_OR_CLANG) |
| 208 |
bool result; |
| 209 |
void* tmp; |
| 210 |
asm volatile( |
| 211 |
"mov %w1, #1\n\t" |
| 212 |
"ldxr %x2, [%0]\n\t" |
| 213 |
"cmp %x3, %x2\n\t" |
| 214 |
"b.ne 0f\n\t" |
| 215 |
"stxr %w1, %x4, [%0]\n\t" |
| 216 |
"0:" |
| 217 |
: "+r"(location), "=&r"(result), "=&r"(tmp) |
| 218 |
: "r"(expected), "r"(newValue) |
| 219 |
: "memory"); |
| 220 |
return !result; |
| 221 |
#elif CPU(ARM64) |
| 222 |
bool result; |
| 223 |
void* tmp; |
| 224 |
asm volatile( |
| 225 |
"mov %w1, #1\n\t" |
| 226 |
"ldxr %x2, %0\n\t" |
| 227 |
"cmp %x3, %x2\n\t" |
| 228 |
"b.ne 0f\n\t" |
| 229 |
"stxr %w1, %x4, %0\n\t" |
| 230 |
"0:" |
| 231 |
: "+m"(*location), "=&r"(result), "=&r"(tmp) |
| 232 |
: "r"(expected), "r"(newValue) |
| 233 |
: "memory"); |
| 234 |
return !result; |
| 235 |
#else |
| 236 |
return weakCompareAndSwap(bitwise_cast<unsigned*>(location), bitwise_cast<unsigned>(expected), bitwise_cast<unsigned>(newValue)); |
| 237 |
#endif |
| 238 |
#else // ENABLE(COMPARE_AND_SWAP) |
| 239 |
UNUSED_PARAM(location); |
| 240 |
UNUSED_PARAM(expected); |
| 241 |
UNUSED_PARAM(newValue); |
| 242 |
CRASH(); |
| 243 |
return 0; |
| 244 |
#endif // ENABLE(COMPARE_AND_SWAP) |
| 245 |
} |
| 246 |
#endif // OS(WINDOWS) (end of the not-windows case) |
| 247 |
|
| 248 |
inline bool weakCompareAndSwapUIntPtr(volatile uintptr_t* location, uintptr_t expected, uintptr_t newValue) |
| 249 |
{ |
| 250 |
return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue)); |
| 251 |
} |
| 252 |
|
| 253 |
inline bool weakCompareAndSwapSize(volatile size_t* location, size_t expected, size_t newValue) |
| 254 |
{ |
119 |
{ |
| 255 |
return weakCompareAndSwap(reinterpret_cast<void*volatile*>(location), reinterpret_cast<void*>(expected), reinterpret_cast<void*>(newValue)); |
120 |
return bitwise_cast<Atomic<T>*>(location)->compareExchangeWeak(expected, newValue, std::memory_order_relaxed); |
| 256 |
} |
121 |
} |
| 257 |
|
122 |
|
| 258 |
// Just a compiler fence. Has no effect on the hardware, but tells the compiler |
123 |
// Just a compiler fence. Has no effect on the hardware, but tells the compiler |
|
Lines 322-390
inline void memoryBarrierBeforeUnlock()
Source/WTF/wtf/Atomics.h_sec2
|
| 322 |
|
187 |
|
| 323 |
#endif |
188 |
#endif |
| 324 |
|
189 |
|
| 325 |
inline bool weakCompareAndSwap(uint8_t* location, uint8_t expected, uint8_t newValue) |
|
|
| 326 |
{ |
| 327 |
#if ENABLE(COMPARE_AND_SWAP) |
| 328 |
#if !OS(WINDOWS) && (CPU(X86) || CPU(X86_64)) |
| 329 |
unsigned char result; |
| 330 |
asm volatile( |
| 331 |
"lock; cmpxchgb %3, %2\n\t" |
| 332 |
"sete %1" |
| 333 |
: "+a"(expected), "=q"(result), "+m"(*location) |
| 334 |
: "q"(newValue) |
| 335 |
: "memory" |
| 336 |
); |
| 337 |
return result; |
| 338 |
#elif OS(WINDOWS) && CPU(X86) |
| 339 |
// FIXME: We need a 64-bit ASM implementation, but this cannot be inline due to |
| 340 |
// Microsoft's decision to exclude it from the compiler. |
| 341 |
bool result = false; |
| 342 |
|
| 343 |
__asm { |
| 344 |
mov al, expected |
| 345 |
mov edx, location |
| 346 |
mov cl, newValue |
| 347 |
lock cmpxchg byte ptr[edx], cl |
| 348 |
setz result |
| 349 |
} |
| 350 |
|
| 351 |
return result; |
| 352 |
#else |
| 353 |
uintptr_t locationValue = bitwise_cast<uintptr_t>(location); |
| 354 |
uintptr_t alignedLocationValue = locationValue & ~(sizeof(unsigned) - 1); |
| 355 |
uintptr_t locationOffset = locationValue - alignedLocationValue; |
| 356 |
ASSERT(locationOffset < sizeof(unsigned)); |
| 357 |
unsigned* alignedLocation = bitwise_cast<unsigned*>(alignedLocationValue); |
| 358 |
// Make sure that this load is always issued and never optimized away. |
| 359 |
unsigned oldAlignedValue = *const_cast<volatile unsigned*>(alignedLocation); |
| 360 |
|
| 361 |
struct Splicer { |
| 362 |
static unsigned splice(unsigned value, uint8_t byte, uintptr_t byteIndex) |
| 363 |
{ |
| 364 |
union { |
| 365 |
unsigned word; |
| 366 |
uint8_t bytes[sizeof(unsigned)]; |
| 367 |
} u; |
| 368 |
u.word = value; |
| 369 |
u.bytes[byteIndex] = byte; |
| 370 |
return u.word; |
| 371 |
} |
| 372 |
}; |
| 373 |
|
| 374 |
unsigned expectedAlignedValue = Splicer::splice(oldAlignedValue, expected, locationOffset); |
| 375 |
unsigned newAlignedValue = Splicer::splice(oldAlignedValue, newValue, locationOffset); |
| 376 |
|
| 377 |
return weakCompareAndSwap(alignedLocation, expectedAlignedValue, newAlignedValue); |
| 378 |
#endif |
| 379 |
#else |
| 380 |
UNUSED_PARAM(location); |
| 381 |
UNUSED_PARAM(expected); |
| 382 |
UNUSED_PARAM(newValue); |
| 383 |
CRASH(); |
| 384 |
return false; |
| 385 |
#endif |
| 386 |
} |
| 387 |
|
| 388 |
} // namespace WTF |
190 |
} // namespace WTF |
| 389 |
|
191 |
|
| 390 |
using WTF::Atomic; |
192 |
using WTF::Atomic; |