diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp index 29cfc751ea817..6190e315f72c3 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp @@ -222,78 +222,7 @@ static memory_order to_mo(morder mo) { return memory_order_seq_cst; } -template -static T NoTsanAtomicLoad(const volatile T *a, morder mo) { - return atomic_load(to_atomic(a), to_mo(mo)); -} - -#if __TSAN_HAS_INT128 && !SANITIZER_GO -static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { - SpinMutexLock lock(&mutex128); - return *a; -} -#endif - -template -static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { - DCHECK(IsLoadOrder(mo)); - // This fast-path is critical for performance. - // Assume the access is atomic. - if (!IsAcquireOrder(mo)) { - MemoryAccess(thr, pc, (uptr)a, AccessSize(), - kAccessRead | kAccessAtomic); - return NoTsanAtomicLoad(a, mo); - } - // Don't create sync object if it does not exist yet. For example, an atomic - // pointer is initialized to nullptr and then periodically acquire-loaded. - T v = NoTsanAtomicLoad(a, mo); - SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a); - if (s) { - SlotLocker locker(thr); - ReadLock lock(&s->mtx); - thr->clock.Acquire(s->clock); - // Re-read under sync mutex because we need a consistent snapshot - // of the value and the clock we acquire. - v = NoTsanAtomicLoad(a, mo); - } - MemoryAccess(thr, pc, (uptr)a, AccessSize(), kAccessRead | kAccessAtomic); - return v; -} - -template -static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { - atomic_store(to_atomic(a), v, to_mo(mo)); -} - -#if __TSAN_HAS_INT128 && !SANITIZER_GO -static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { - SpinMutexLock lock(&mutex128); - *a = v; -} -#endif - -template -static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - DCHECK(IsStoreOrder(mo)); - MemoryAccess(thr, pc, (uptr)a, AccessSize(), kAccessWrite | kAccessAtomic); - // This fast-path is critical for performance. - // Assume the access is atomic. - // Strictly saying even relaxed store cuts off release sequence, - // so must reset the clock. - if (!IsReleaseOrder(mo)) { - NoTsanAtomicStore(a, v, mo); - return; - } - SlotLocker locker(thr); - { - auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); - Lock lock(&s->mtx); - thr->clock.ReleaseStore(&s->clock); - NoTsanAtomicStore(a, v, mo); - } - IncrementEpoch(thr); -} +namespace { template static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { @@ -317,164 +246,255 @@ static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { return v; } -template -static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { - return func_xchg(a, v); -} +struct OpLoad { + template + static T NoTsanAtomic(const volatile T *a, morder mo) { + return atomic_load(to_atomic(a), to_mo(mo)); + } -template -static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { - return func_add(a, v); -} +#if __TSAN_HAS_INT128 && !SANITIZER_GO + static a128 NoTsanAtomic(const volatile a128 *a, morder mo) { + SpinMutexLock lock(&mutex128); + return *a; + } +#endif -template -static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { - return func_sub(a, v); -} + template + static T Atomic(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { + DCHECK(IsLoadOrder(mo)); + // This fast-path is critical for performance. + // Assume the access is atomic. + if (!IsAcquireOrder(mo)) { + MemoryAccess(thr, pc, (uptr)a, AccessSize(), + kAccessRead | kAccessAtomic); + return NoTsanAtomic(a, mo); + } + // Don't create sync object if it does not exist yet. For example, an atomic + // pointer is initialized to nullptr and then periodically acquire-loaded. + T v = NoTsanAtomic(a, mo); + SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a); + if (s) { + SlotLocker locker(thr); + ReadLock lock(&s->mtx); + thr->clock.Acquire(s->clock); + // Re-read under sync mutex because we need a consistent snapshot + // of the value and the clock we acquire. + v = NoTsanAtomic(a, mo); + } + MemoryAccess(thr, pc, (uptr)a, AccessSize(), + kAccessRead | kAccessAtomic); + return v; + } +}; -template -static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { - return func_and(a, v); -} +struct OpStore { + template + static void NoTsanAtomic(volatile T *a, T v, morder mo) { + atomic_store(to_atomic(a), v, to_mo(mo)); + } -template -static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { - return func_or(a, v); -} +#if __TSAN_HAS_INT128 && !SANITIZER_GO + static void NoTsanAtomic(volatile a128 *a, a128 v, morder mo) { + SpinMutexLock lock(&mutex128); + *a = v; + } +#endif -template -static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { - return func_xor(a, v); -} + template + static void Atomic(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + DCHECK(IsStoreOrder(mo)); + MemoryAccess(thr, pc, (uptr)a, AccessSize(), + kAccessWrite | kAccessAtomic); + // This fast-path is critical for performance. + // Assume the access is atomic. + // Strictly saying even relaxed store cuts off release sequence, + // so must reset the clock. + if (!IsReleaseOrder(mo)) { + NoTsanAtomic(a, v, mo); + return; + } + SlotLocker locker(thr); + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + Lock lock(&s->mtx); + thr->clock.ReleaseStore(&s->clock); + NoTsanAtomic(a, v, mo); + } + IncrementEpoch(thr); + } +}; -template -static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { - return func_nand(a, v); -} +struct OpExchange { + template + static T NoTsanAtomic(volatile T *a, T v, morder mo) { + return func_xchg(a, v); + } + template + static T Atomic(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + return AtomicRMW(thr, pc, a, v, mo); + } +}; -template -static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW(thr, pc, a, v, mo); -} +struct OpFetchAdd { + template + static T NoTsanAtomic(volatile T *a, T v, morder mo) { + return func_add(a, v); + } -template -static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW(thr, pc, a, v, mo); -} + template + static T Atomic(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + return AtomicRMW(thr, pc, a, v, mo); + } +}; -template -static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW(thr, pc, a, v, mo); -} +struct OpFetchSub { + template + static T NoTsanAtomic(volatile T *a, T v, morder mo) { + return func_sub(a, v); + } -template -static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW(thr, pc, a, v, mo); -} + template + static T Atomic(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + return AtomicRMW(thr, pc, a, v, mo); + } +}; -template -static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW(thr, pc, a, v, mo); -} +struct OpFetchAnd { + template + static T NoTsanAtomic(volatile T *a, T v, morder mo) { + return func_and(a, v); + } -template -static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW(thr, pc, a, v, mo); -} + template + static T Atomic(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + return AtomicRMW(thr, pc, a, v, mo); + } +}; -template -static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, - morder mo) { - return AtomicRMW(thr, pc, a, v, mo); -} +struct OpFetchOr { + template + static T NoTsanAtomic(volatile T *a, T v, morder mo) { + return func_or(a, v); + } -template -static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { - return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); -} + template + static T Atomic(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + return AtomicRMW(thr, pc, a, v, mo); + } +}; -#if __TSAN_HAS_INT128 -static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, morder mo, - morder fmo) { - a128 old = *c; - a128 cur = func_cas(a, old, v); - if (cur == old) - return true; - *c = cur; - return false; -} -#endif +struct OpFetchXor { + template + static T NoTsanAtomic(volatile T *a, T v, morder mo) { + return func_xor(a, v); + } -template -static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { - NoTsanAtomicCAS(a, &c, v, mo, fmo); - return c; -} + template + static T Atomic(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + return AtomicRMW(thr, pc, a, v, mo); + } +}; -template -static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, - morder mo, morder fmo) { - // 31.7.2.18: "The failure argument shall not be memory_order_release - // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic - // (mo_relaxed) when those are used. - DCHECK(IsLoadOrder(fmo)); +struct OpFetchNand { + template + static T NoTsanAtomic(volatile T *a, T v, morder mo) { + return func_nand(a, v); + } - MemoryAccess(thr, pc, (uptr)a, AccessSize(), kAccessWrite | kAccessAtomic); - if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) { - T cc = *c; - T pr = func_cas(a, cc, v); - if (pr == cc) + template + static T Atomic(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { + return AtomicRMW(thr, pc, a, v, mo); + } +}; + +struct OpCAS { + template + static bool NoTsanAtomic(volatile T *a, T *c, T v, morder mo, morder fmo) { + return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); + } + +#if __TSAN_HAS_INT128 + static bool NoTsanAtomic(volatile a128 *a, a128 *c, a128 v, morder mo, + morder fmo) { + a128 old = *c; + a128 cur = func_cas(a, old, v); + if (cur == old) return true; - *c = pr; + *c = cur; return false; } - SlotLocker locker(thr); - bool release = IsReleaseOrder(mo); - bool success; - { - auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); - RWLock lock(&s->mtx, release); - T cc = *c; - T pr = func_cas(a, cc, v); - success = pr == cc; - if (!success) { +#endif + + template + static T NoTsanAtomic(volatile T *a, T c, T v, morder mo, morder fmo) { + NoTsanAtomic(a, &c, v, mo, fmo); + return c; + } + + template + static bool Atomic(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, + morder mo, morder fmo) { + // 31.7.2.18: "The failure argument shall not be memory_order_release + // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic + // (mo_relaxed) when those are used. + DCHECK(IsLoadOrder(fmo)); + + MemoryAccess(thr, pc, (uptr)a, AccessSize(), + kAccessWrite | kAccessAtomic); + if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) { + T cc = *c; + T pr = func_cas(a, cc, v); + if (pr == cc) + return true; *c = pr; - mo = fmo; + return false; } - if (success && IsAcqRelOrder(mo)) - thr->clock.ReleaseAcquire(&s->clock); - else if (success && IsReleaseOrder(mo)) - thr->clock.Release(&s->clock); - else if (IsAcquireOrder(mo)) - thr->clock.Acquire(s->clock); + SlotLocker locker(thr); + bool release = IsReleaseOrder(mo); + bool success; + { + auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false); + RWLock lock(&s->mtx, release); + T cc = *c; + T pr = func_cas(a, cc, v); + success = pr == cc; + if (!success) { + *c = pr; + mo = fmo; + } + if (success && IsAcqRelOrder(mo)) + thr->clock.ReleaseAcquire(&s->clock); + else if (success && IsReleaseOrder(mo)) + thr->clock.Release(&s->clock); + else if (IsAcquireOrder(mo)) + thr->clock.Acquire(s->clock); + } + if (success && release) + IncrementEpoch(thr); + return success; } - if (success && release) - IncrementEpoch(thr); - return success; -} -template -static T AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T c, T v, - morder mo, morder fmo) { - AtomicCAS(thr, pc, a, &c, v, mo, fmo); - return c; -} + template + static T Atomic(ThreadState *thr, uptr pc, volatile T *a, T c, T v, morder mo, + morder fmo) { + Atomic(thr, pc, a, &c, v, mo, fmo); + return c; + } +}; #if !SANITIZER_GO -static void NoTsanAtomicFence(morder mo) { __sync_synchronize(); } +struct OpFence { + static void NoTsanAtomic(morder mo) { __sync_synchronize(); } -static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { - // FIXME(dvyukov): not implemented. - __sync_synchronize(); -} + static void Atomic(ThreadState *thr, uptr pc, morder mo) { + // FIXME(dvyukov): not implemented. + __sync_synchronize(); + } +}; #endif +} // namespace + // Interface functions follow. #if !SANITIZER_GO @@ -501,9 +521,9 @@ static morder convert_morder(morder mo) { ThreadState *const thr = cur_thread(); \ ProcessPendingSignals(thr); \ if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) \ - return NoTsanAtomic##func(__VA_ARGS__); \ + return Op##func::NoTsanAtomic(__VA_ARGS__); \ mo = convert_morder(mo); \ - return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__); + return Op##func::Atomic(thr, GET_CALLER_PC(), __VA_ARGS__); extern "C" { SANITIZER_INTERFACE_ATTRIBUTE @@ -856,22 +876,22 @@ void __tsan_atomic_signal_fence(morder mo) {} // Go -# define ATOMIC(func, ...) \ - if (thr->ignore_sync) { \ - NoTsanAtomic##func(__VA_ARGS__); \ - } else { \ - FuncEntry(thr, cpc); \ - Atomic##func(thr, pc, __VA_ARGS__); \ - FuncExit(thr); \ +# define ATOMIC(func, ...) \ + if (thr->ignore_sync) { \ + Op##func::NoTsanAtomic(__VA_ARGS__); \ + } else { \ + FuncEntry(thr, cpc); \ + Op##func::Atomic(thr, pc, __VA_ARGS__); \ + FuncExit(thr); \ } -# define ATOMIC_RET(func, ret, ...) \ - if (thr->ignore_sync) { \ - (ret) = NoTsanAtomic##func(__VA_ARGS__); \ - } else { \ - FuncEntry(thr, cpc); \ - (ret) = Atomic##func(thr, pc, __VA_ARGS__); \ - FuncExit(thr); \ +# define ATOMIC_RET(func, ret, ...) \ + if (thr->ignore_sync) { \ + (ret) = Op##func::NoTsanAtomic(__VA_ARGS__); \ + } else { \ + FuncEntry(thr, cpc); \ + (ret) = Op##func::Atomic(thr, pc, __VA_ARGS__); \ + FuncExit(thr); \ } extern "C" {