diff --git a/Project.toml b/Project.toml index 4851885..6b01112 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,7 @@ LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" UnsafeAtomicsLLVM = ["LLVM"] [compat] -LLVM = "8.1, 9" +LLVM = "9" julia = "1.10" [extras] diff --git a/ext/UnsafeAtomicsLLVM/atomics.jl b/ext/UnsafeAtomicsLLVM/atomics.jl index 2c5691a..cce568e 100644 --- a/ext/UnsafeAtomicsLLVM/atomics.jl +++ b/ext/UnsafeAtomicsLLVM/atomics.jl @@ -89,68 +89,12 @@ for (julia, llvm) in pairs(_llvm_from_julia_ordering) @eval llvm_from_julia_ordering(::Val{$(QuoteNode(julia))}) = Val{$llvm}() end -""" - @dynamic_order(order) do order - ... use order ... - end -It is expanded to an expression similar to: - if order === :not_atomic - let order = Val(:not_atomic) - ... use order ... - end - elseif order === :unordered - let order = Val(:unordered) - ... use order ... - end - elseif ... - ... - else - throw(ConcurrencyViolationError(...)) - end -This is used for helping the compiler to optimize expressions such as -`atomic_pointerref(ptr, :monotonic)` and also to avoid abstract run-time dispatch. -""" -macro dynamic_order(thunk, order) - @assert Meta.isexpr(thunk, :->, 2) && Meta.isexpr(thunk.args[1], :tuple, 1) - ordervar = esc(thunk.args[1].args[1]) - body = esc(thunk.args[2]) - expr = foldr( - keys(_llvm_from_julia_ordering), - init = :(throw(ConcurrencyViolationError("invalid atomic ordering: ", order))), - ) do key, r - quote - if order === $(QuoteNode(key)) - let $ordervar = Val{$(QuoteNode(key))}() - $body - end - else - $r - end - end - end - quote - order = $(esc(order)) - $expr - end -end - _valueof(::Val{x}) where {x} = x -@inline function atomic_pointerref(pointer, order::Symbol) - @dynamic_order(order) do order - atomic_pointerref(pointer, order) - end -end - -@inline function atomic_pointerset(pointer, x, order::Symbol) - @dynamic_order(order) do order - atomic_pointerset(pointer, x, order) - end -end - -@generated function atomic_pointerref(ptr::LLVMPtr{T,A}, order::AllOrdering) where {T,A} +@generated function atomic_pointerref(ptr::LLVMPtr{T,A}, order::AllOrdering, sync) where {T,A} sizeof(T) == 0 && return T.instance - llvm_order = _valueof(llvm_from_julia_ordering(order())) + llvm_order = _valueof(llvm_from_julia_ordering(order())) + llvm_syncscope = _valueof(sync()) @dispose ctx = Context() begin eltyp = convert(LLVMType, T) @@ -170,6 +114,7 @@ end typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) ld = load!(builder, eltyp, typed_ptr) ordering!(ld, llvm_order) + syncscope!(ld, SyncScope(string(llvm_syncscope))) if A != 0 metadata(ld)[LLVM.MD_tbaa] = tbaa_addrspace(A) @@ -187,6 +132,7 @@ end ptr::LLVMPtr{T,A}, x::T, order::AllOrdering, + sync, ) where {T,A} if sizeof(T) == 0 # Mimicking what `Core.Intrinsics.atomic_pointerset` generates. @@ -197,7 +143,8 @@ end return ptr end end - llvm_order = _valueof(llvm_from_julia_ordering(order())) + llvm_order = _valueof(llvm_from_julia_ordering(order())) + llvm_syncscope = _valueof(sync()) @dispose ctx = Context() begin eltyp = convert(LLVMType, T) T_ptr = convert(LLVMType, ptr) @@ -216,6 +163,7 @@ end val = parameters(llvm_f)[2] st = store!(builder, val, typed_ptr) ordering!(st, llvm_order) + syncscope!(st, SyncScope(string(llvm_syncscope))) if A != 0 metadata(st)[LLVM.MD_tbaa] = tbaa_addrspace(A) @@ -254,14 +202,12 @@ const binoptable = [ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} -# LLVM API accepts string literal as a syncscope argument. -@inline syncscope_to_string(::Type{Val{S}}) where {S} = string(S) - @generated function llvm_atomic_op( binop::AtomicRMWBinOpVal, ptr::LLVMPtr{T,A}, val::T, order::LLVMOrderingVal, + sync, ) where {T,A} @dispose ctx = Context() begin T_val = convert(LLVMType, T) @@ -270,6 +216,7 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} T_typed_ptr = LLVM.PointerType(T_val, A) llvm_f, _ = create_function(T_val, [T_ptr, T_val]) + llvm_syncscope = _valueof(sync()) @dispose builder = IRBuilder() begin entry = BasicBlock(llvm_f, "entry") @@ -277,14 +224,13 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) - single_threaded = false rv = atomic_rmw!( builder, _valueof(binop()), typed_ptr, parameters(llvm_f)[2], _valueof(order()), - single_threaded, + SyncScope(string(llvm_syncscope)) ) ret!(builder, rv) @@ -294,139 +240,40 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} end end -@generated function llvm_atomic_op( - binop::AtomicRMWBinOpVal, - ptr::LLVMPtr{T,A}, - val::T, - order::LLVMOrderingVal, - syncscope::Val{S}, -) where {T,A,S} - @dispose ctx = Context() begin - T_val = convert(LLVMType, T) - T_ptr = convert(LLVMType, ptr) - - T_typed_ptr = LLVM.PointerType(T_val, A) - llvm_f, _ = create_function(T_val, [T_ptr, T_val]) - - @dispose builder = IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) - rv = atomic_rmw!( - builder, - _valueof(binop()), - typed_ptr, - parameters(llvm_f)[2], - _valueof(order()), - syncscope_to_string(syncscope), - ) - - ret!(builder, rv) - end - call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val) - end -end - -@inline function atomic_pointermodify(pointer, op::OP, x, order::Symbol) where {OP} - @dynamic_order(order) do order - atomic_pointermodify(pointer, op, x, order) - end -end - -@inline function atomic_pointermodify( - ptr::LLVMPtr{T}, - op, - x::T, - ::Val{:not_atomic}, -) where {T} - old = atomic_pointerref(ptr, Val(:not_atomic)) - new = op(old, x) - atomic_pointerset(ptr, new, Val(:not_atomic)) - return old => new -end - @inline function atomic_pointermodify( ptr::LLVMPtr{T}, ::typeof(right), x::T, order::AtomicOrdering, -) where {T} + sync::Val{S} +) where {T, S} old = llvm_atomic_op( Val(LLVM.API.LLVMAtomicRMWBinOpXchg), ptr, x, llvm_from_julia_ordering(order), + sync ) return old => x end -const atomictypes = Any[ - Int8, - Int16, - Int32, - Int64, - Int128, - UInt8, - UInt16, - UInt32, - UInt64, - UInt128, - Float16, - Float32, - Float64, -] - -for (opname, op, llvmop) in binoptable - opname === :xchg && continue - types = if opname in (:min, :max) - filter(t -> t <: Signed, atomictypes) - elseif opname in (:umin, :umax) - filter(t -> t <: Unsigned, atomictypes) - elseif opname in (:fadd, :fsub, :fmin, :fmax) - filter(t -> t <: AbstractFloat, atomictypes) - else - filter(t -> t <: Integer, atomictypes) - end - for T in types - @eval @inline function atomic_pointermodify( - ptr::LLVMPtr{$T}, - ::$(typeof(op)), - x::$T, - order::AtomicOrdering, - syncscope::Val{S} = Val{:system}(), - ) where {S} - old = - syncscope isa Val{:system} ? - llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) : - llvm_atomic_op( - $(Val(llvmop)), - ptr, - x, - llvm_from_julia_ordering(order), - syncscope, - ) - return old => $op(old, x) - end - end -end - -@inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) -@inline atomic_pointerswap(pointer, new, order) = - first(atomic_pointermodify(pointer, right, new, order)) +# @inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) +@inline atomic_pointerswap(pointer, new, order, sync) = + first(atomic_pointermodify(pointer, right, new, order, sync)) @inline function atomic_pointermodify( ptr::LLVMPtr{T}, op, x::T, order::AllOrdering, -) where {T} + sync::S, +) where {T, S} # Should `fail_order` be stronger? Ref: https://github.com/JuliaLang/julia/issues/45256 fail_order = Val(:monotonic) - old = atomic_pointerref(ptr, fail_order) + old = atomic_pointerref(ptr, fail_order, sync) while true new = op(old, x) - (old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order) + (old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order, sync) success && return old => new end end @@ -437,9 +284,11 @@ end val::T, success_order::LLVMOrderingVal, fail_order::LLVMOrderingVal, + sync, ) where {T,A} llvm_success = _valueof(success_order()) llvm_fail = _valueof(fail_order()) + llvm_syncscope = _valueof(sync()) @dispose ctx = Context() begin T_val = convert(LLVMType, T) T_pointee = T_val @@ -471,7 +320,6 @@ end val_int = bitcast!(builder, val_int, T_pointee) end - single_threaded = false res = atomic_cmpxchg!( builder, typed_ptr, @@ -479,7 +327,7 @@ end val_int, llvm_success, llvm_fail, - single_threaded, + SyncScope(string(llvm_syncscope)), ) rv = extract_value!(builder, res, 0) @@ -514,42 +362,17 @@ end end end -@inline function atomic_pointerreplace( - pointer, - expected, - desired, - success_order::Symbol, - fail_order::Symbol, -) - # This avoids abstract dispatch at run-time but probably too much codegen? - #= - @dynamic_order(success_order) do success_order - @dynamic_order(fail_order) do fail_order - atomic_pointerreplace(pointer, expected, desired, success_order, fail_order) - end - end - =# - - # This avoids excessive codegen while hopefully imposes no cost when const-prop works: - so = @dynamic_order(success_order) do success_order - success_order - end - fo = @dynamic_order(fail_order) do fail_order - fail_order - end - return atomic_pointerreplace(pointer, expected, desired, so, fo) -end - @inline function atomic_pointerreplace( ptr::LLVMPtr{T}, expected::T, desired::T, ::Val{:not_atomic}, ::Val{:not_atomic}, + sync, ) where {T} - old = atomic_pointerref(ptr, Val(:not_atomic)) + old = atomic_pointerref(ptr, Val(:not_atomic), sync) if old === expected - atomic_pointerset(ptr, desired, Val(:not_atomic)) + atomic_pointerset(ptr, desired, Val(:not_atomic), sync) success = true else success = false @@ -563,10 +386,12 @@ end desired::T, success_order::_julia_ordering(∉((:not_atomic, :unordered))), fail_order::_julia_ordering(∉((:not_atomic, :unordered, :release, :acquire_release))), + sync ) where {T} = llvm_atomic_cas( ptr, expected, desired, llvm_from_julia_ordering(success_order), llvm_from_julia_ordering(fail_order), + sync ) diff --git a/ext/UnsafeAtomicsLLVM/internal.jl b/ext/UnsafeAtomicsLLVM/internal.jl index e4ed556..90fe951 100644 --- a/ext/UnsafeAtomicsLLVM/internal.jl +++ b/ext/UnsafeAtomicsLLVM/internal.jl @@ -4,30 +4,24 @@ julia_ordering_name(::typeof(UnsafeAtomics.acquire_release)) = :acquire_release julia_ordering_name(::typeof(UnsafeAtomics.sequentially_consistent)) = :sequentially_consistent +julia_syncscope_name(::UnsafeAtomics.Internal.LLVMSyncScope{name}) where {name} = name +julia_syncscope_name(::typeof(UnsafeAtomics.none)) = :system + include("atomics.jl") -@inline UnsafeAtomics.load(ptr::LLVMPtr, order::Ordering) = - atomic_pointerref(ptr, Val{julia_ordering_name(order)}()) +@inline UnsafeAtomics.load(ptr::LLVMPtr, order::Ordering, sync::UnsafeAtomics.Internal.SyncScope) = + atomic_pointerref(ptr, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) -@inline function UnsafeAtomics.store!(ptr::LLVMPtr, x, order::Ordering) - atomic_pointerset(ptr, x, Val{julia_ordering_name(order)}()) +@inline function UnsafeAtomics.store!(ptr::LLVMPtr, x, order::Ordering, sync::UnsafeAtomics.Internal.SyncScope) + atomic_pointerset(ptr, x, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) return end mapop(op::OP) where {OP} = op mapop(::typeof(UnsafeAtomics.right)) = right -@inline UnsafeAtomics.modify!(ptr::LLVMPtr, op::OP, x, order::Ordering) where {OP} = - atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}()) - -@inline UnsafeAtomics.modify!( - ptr::LLVMPtr, - op::OP, - x, - order::Ordering, - syncscope::Val{S} = Val(:system), -) where {OP<:Union{typeof(+),typeof(-)},S} = - atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope) +@inline UnsafeAtomics.modify!(ptr::LLVMPtr, op::OP, x, order::Ordering, sync::UnsafeAtomics.Internal.SyncScope) where {OP} = + atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) @inline UnsafeAtomics.cas!( ptr::LLVMPtr, @@ -35,10 +29,12 @@ mapop(::typeof(UnsafeAtomics.right)) = right desired, success_order::Ordering, failure_order::Ordering, + sync::UnsafeAtomics.Internal.SyncScope, ) = atomic_pointerreplace( ptr, expected, desired, Val{julia_ordering_name(success_order)}(), Val{julia_ordering_name(failure_order)}(), + Val{julia_syncscope_name(sync)}(), ) diff --git a/src/UnsafeAtomics.jl b/src/UnsafeAtomics.jl index a83b8d2..988a784 100644 --- a/src/UnsafeAtomics.jl +++ b/src/UnsafeAtomics.jl @@ -1,6 +1,7 @@ baremodule UnsafeAtomics abstract type Ordering end +abstract type SyncScope end function load end function store! end @@ -26,10 +27,11 @@ module Internal using Base.Sys: WORD_SIZE using Base: bitcast, llvmcall -using ..UnsafeAtomics: UnsafeAtomics, Ordering, right +using ..UnsafeAtomics: UnsafeAtomics, Ordering, SyncScope, right include("utils.jl") include("orderings.jl") +include("syncscopes.jl") include("core.jl") end # module Internal @@ -45,4 +47,8 @@ const seq_cst = Internal.seq_cst const acquire_release = acq_rel const sequentially_consistent = seq_cst +# SyncScope +const none = Internal.none +const singlethread = Internal.singlethread + end # baremodule UnsafeAtomics diff --git a/src/core.jl b/src/core.jl index aff57f8..733d19e 100644 --- a/src/core.jl +++ b/src/core.jl @@ -4,6 +4,13 @@ @inline UnsafeAtomics.modify!(ptr, op, x) = UnsafeAtomics.modify!(ptr, op, x, seq_cst) @inline UnsafeAtomics.fence() = UnsafeAtomics.fence(seq_cst) +@inline UnsafeAtomics.load(x, ord) = UnsafeAtomics.load(x, ord, none) +@inline UnsafeAtomics.store!(x, v, ord) = UnsafeAtomics.store!(x, v, ord, none) +@inline UnsafeAtomics.cas!(x, cmp, new, ord) = UnsafeAtomics.cas!(x, cmp, new, ord, ord, none) +@inline UnsafeAtomics.cas!(x, cmp, new, success_ord, failure_order) = UnsafeAtomics.cas!(x, cmp, new, success_ord, failure_order, none) +@inline UnsafeAtomics.modify!(ptr, op, x, ord) = UnsafeAtomics.modify!(ptr, op, x, ord, none) +@inline UnsafeAtomics.fence(ord) = UnsafeAtomics.fence(ord, none) + #! format: off # https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl#L23-L30 if Sys.ARCH == :i686 || startswith(string(Sys.ARCH), "arm") || @@ -45,8 +52,9 @@ const OP_RMW_TABLE = [ for (op, rmwop) in OP_RMW_TABLE fn = Symbol(rmwop, "!") @eval @inline UnsafeAtomics.$fn(x, v) = UnsafeAtomics.$fn(x, v, seq_cst) - @eval @inline UnsafeAtomics.$fn(ptr, x, ord) = - first(UnsafeAtomics.modify!(ptr, $op, x, ord)) + @eval @inline UnsafeAtomics.$fn(x, v, ord) = UnsafeAtomics.$fn(x, v, ord, none) + @eval @inline UnsafeAtomics.$fn(ptr, x, ord, scope) = + first(UnsafeAtomics.modify!(ptr, $op, x, ord, scope)) end const ATOMIC_INTRINSICS = isdefined(Core.Intrinsics, :atomic_pointerref) @@ -67,47 +75,51 @@ for typ in (inttypes..., floattypes...) for ord in orderings ord in (release, acq_rel) && continue - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) - return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord)) - end - else - @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) - return llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rv = load atomic $rt %ptr $ord, align $(sizeof(typ)) - ret $lt %rv - """), - $typ, - Tuple{Ptr{$typ}}, - x, - ) + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync))) + return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord)) + end + else + @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = load atomic $rt %ptr $ord, align $(sizeof(typ)) + ret $lt %rv + """), + $typ, + Tuple{Ptr{$typ}}, + x, + ) + end end end end for ord in orderings ord in (acquire, acq_rel) && continue - - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) - Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord)) - return nothing - end - else - @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) - return llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ)) - ret void - """), - Cvoid, - Tuple{Ptr{$typ},$typ}, - x, - v, - ) + + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync))) + Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord)) + return nothing + end + else + @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ)) + ret void + """), + Cvoid, + Tuple{Ptr{$typ},$typ}, + x, + v, + ) + end end end end @@ -117,54 +129,58 @@ for typ in (inttypes..., floattypes...) typ <: AbstractFloat && break - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.cas!( - x::Ptr{$typ}, - cmp::$typ, - new::$typ, - ::$(typeof(success_ordering)), - ::$(typeof(failure_ordering)), - ) - return Core.Intrinsics.atomic_pointerreplace( - x, - cmp, - new, - base_ordering($success_ordering), - base_ordering($failure_ordering) + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.cas!( + x::Ptr{$typ}, + cmp::$typ, + new::$typ, + ::$(typeof(success_ordering)), + ::$(typeof(failure_ordering)), + ::$(typeof(sync)), ) - end - else - @eval function UnsafeAtomics.cas!( - x::Ptr{$typ}, - cmp::$typ, - new::$typ, - ::$(typeof(success_ordering)), - ::$(typeof(failure_ordering)), - ) - success = Ref{Int8}() - GC.@preserve success begin - old = llvmcall( - $( - """ - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering - %rv = extractvalue { $lt, i1 } %rs, 0 - %s1 = extractvalue { $lt, i1 } %rs, 1 - %s8 = zext i1 %s1 to i8 - %sptr = inttoptr i$WORD_SIZE %3 to i8* - store i8 %s8, i8* %sptr - ret $lt %rv - """ - ), - $typ, - Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}}, + return Core.Intrinsics.atomic_pointerreplace( x, cmp, new, - Ptr{Int8}(pointer_from_objref(success)), + base_ordering($success_ordering), + base_ordering($failure_ordering) ) end - return (old = old, success = !iszero(success[])) + else + @eval function UnsafeAtomics.cas!( + x::Ptr{$typ}, + cmp::$typ, + new::$typ, + ::$(typeof(success_ordering)), + ::$(typeof(failure_ordering)), + ::$(typeof(sync)), + ) + success = Ref{Int8}() + GC.@preserve success begin + old = llvmcall( + $( + """ + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering + %rv = extractvalue { $lt, i1 } %rs, 0 + %s1 = extractvalue { $lt, i1 } %rs, 1 + %s8 = zext i1 %s1 to i8 + %sptr = inttoptr i$WORD_SIZE %3 to i8* + store i8 %s8, i8* %sptr + ret $lt %rv + """ + ), + $typ, + Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}}, + x, + cmp, + new, + Ptr{Int8}(pointer_from_objref(success)), + ) + end + return (old = old, success = !iszero(success[])) + end end end end @@ -186,60 +202,81 @@ for typ in (inttypes..., floattypes...) end end for ord in orderings - # Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged - if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.modify!( + for sync in syncscopes + # Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged + if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.modify!( + x::Ptr{$typ}, + op::typeof($op), + v::$typ, + ::$(typeof(ord)), + ::$(typeof(sync)), + ) + return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord)) + end + else + @eval function UnsafeAtomics.modify!( x::Ptr{$typ}, - op::typeof($op), + ::typeof($op), v::$typ, ::$(typeof(ord)), + ::$(typeof(sync)), ) - return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord)) - end - else - @eval function UnsafeAtomics.modify!( - x::Ptr{$typ}, - ::typeof($op), - v::$typ, - ::$(typeof(ord)), - ) - old = llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord - ret $lt %rv - """), - $typ, - Tuple{Ptr{$typ},$typ}, - x, - v, - ) - return old => $op(old, v) + old = llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord + ret $lt %rv + """), + $typ, + Tuple{Ptr{$typ},$typ}, + x, + v, + ) + return old => $op(old, v) + end end end end end end -# Core.Intrinsics.atomic_fence was introduced in 1.10 -function UnsafeAtomics.fence(ord::Ordering) - Core.Intrinsics.atomic_fence(base_ordering(ord)) - return nothing -end -if Sys.ARCH == :x86_64 - # FIXME: Disable this once on LLVM 19 - # This is unfortunatly required for good-performance on AMD - # https://github.com/llvm/llvm-project/pull/106555 - function UnsafeAtomics.fence(::typeof(seq_cst)) - Base.llvmcall( - (raw""" - define void @fence() #0 { - entry: - tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory} - ret void - } - attributes #0 = { alwaysinline } - """, "fence"), Nothing, Tuple{}) +for sync in syncscopes + if sync == none + # Core.Intrinsics.atomic_fence was introduced in 1.10 + @eval function UnsafeAtomics.fence(ord::Ordering, ::$(typeof(sync))) + Core.Intrinsics.atomic_fence(base_ordering(ord)) + return nothing + end + if Sys.ARCH == :x86_64 + # FIXME: Disable this once on LLVM 19 + # This is unfortunatly required for good-performance on AMD + # https://github.com/llvm/llvm-project/pull/106555 + @eval function UnsafeAtomics.fence(::typeof(seq_cst), ::$(typeof(sync))) + Base.llvmcall( + (raw""" + define void @fence() #0 { + entry: + tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory} + ret void + } + attributes #0 = { alwaysinline } + """, "fence"), Nothing, Tuple{}) + end + end + else + for ord in orderings + @eval function UnsafeAtomics.fence(::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + fence $sync $ord + ret void + """), + Cvoid, + Tuple{}, + ) + end + end end end @@ -258,20 +295,20 @@ as_native_uint(::Type{T}) where {T} = error(LazyString("unsupported size: ", sizeof(T))) end -function UnsafeAtomics.load(x::Ptr{T}, ordering) where {T} +function UnsafeAtomics.load(x::Ptr{T}, ordering, syncscope) where {T} UI = as_native_uint(T) - v = UnsafeAtomics.load(Ptr{UI}(x), ordering) + v = UnsafeAtomics.load(Ptr{UI}(x), ordering, syncscope) return bitcast(T, v) end -function UnsafeAtomics.store!(x::Ptr{T}, v::T, ordering) where {T} +function UnsafeAtomics.store!(x::Ptr{T}, v::T, ordering, syncscope) where {T} UI = as_native_uint(T) - UnsafeAtomics.store!(Ptr{UI}(x), bitcast(UI, v), ordering)::Nothing + UnsafeAtomics.store!(Ptr{UI}(x), bitcast(UI, v), ordering, syncscope)::Nothing end -function UnsafeAtomics.modify!(x::Ptr{T}, ::typeof(right), v::T, ordering) where {T} +function UnsafeAtomics.modify!(x::Ptr{T}, ::typeof(right), v::T, ordering, syncscope) where {T} UI = as_native_uint(T) - old, _ = UnsafeAtomics.modify!(Ptr{UI}(x), right, bitcast(UI, v), ordering) + old, _ = UnsafeAtomics.modify!(Ptr{UI}(x), right, bitcast(UI, v), ordering, syncscope) return bitcast(T, old) => v end @@ -281,6 +318,7 @@ function UnsafeAtomics.cas!( new::T, success_ordering, failure_ordering, + syncscope, ) where {T} UI = as_native_uint(T) (old, success) = UnsafeAtomics.cas!( @@ -289,6 +327,7 @@ function UnsafeAtomics.cas!( bitcast(UI, new), success_ordering, failure_ordering, + syncscope ) return (old = bitcast(T, old), success = success) end diff --git a/src/syncscopes.jl b/src/syncscopes.jl new file mode 100644 index 0000000..3c4413e --- /dev/null +++ b/src/syncscopes.jl @@ -0,0 +1,16 @@ +struct LLVMSyncScope{name} <: SyncScope end + +const none = LLVMSyncScope{:none}() +const singlethread = LLVMSyncScope{:singlethread}() + +const syncscopes = (none, singlethread) +const ConcreteSyncScopes = Union{map(typeof, syncscopes)...} + +llvm_syncscope(::LLVMSyncScope{name}) where {name} = name + +Base.string(s::LLVMSyncScope) = string("syncscope(\"", llvm_syncscope(s), "\")") +Base.string(s::typeof(none)) = "" + +Base.print(io::IO, s::LLVMSyncScope) = print(io, string(s)) + +Base.show(io::IO, o::ConcreteSyncScopes) = print(io, UnsafeAtomics, '.', llvm_syncscope(o)) \ No newline at end of file diff --git a/test/UnsafeAtomicsLLVM.jl b/test/UnsafeAtomicsLLVM.jl index 81c0fa5..5fa02fa 100644 --- a/test/UnsafeAtomicsLLVM.jl +++ b/test/UnsafeAtomicsLLVM.jl @@ -71,12 +71,12 @@ function test_explicit_ordering(xs::AbstractArray{T}, x1::T, x2::T) where T # Test syncscopes. if (op == +) || (op == -) xs[1] = x1 - @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:system)) === + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, UnsafeAtomics.none) === (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) xs[1] = x1 - @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:singlethread)) === + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, UnsafeAtomics.singlethread) === (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) end diff --git a/test/UnsafeAtomicsTests/src/UnsafeAtomicsTests.jl b/test/UnsafeAtomicsTests/src/UnsafeAtomicsTests.jl index 89dd87d..e417af1 100644 --- a/test/UnsafeAtomicsTests/src/UnsafeAtomicsTests.jl +++ b/test/UnsafeAtomicsTests/src/UnsafeAtomicsTests.jl @@ -2,6 +2,7 @@ module UnsafeAtomicsTests include("bits.jl") include("test_orderings.jl") +include("test_syncscopes.jl") include("test_core.jl") end # module UnsafeAtomicsTests diff --git a/test/UnsafeAtomicsTests/src/test_core.jl b/test/UnsafeAtomicsTests/src/test_core.jl index 4d4b1cf..ffeede0 100644 --- a/test/UnsafeAtomicsTests/src/test_core.jl +++ b/test/UnsafeAtomicsTests/src/test_core.jl @@ -1,6 +1,7 @@ module TestCore using UnsafeAtomics: UnsafeAtomics, monotonic, acquire, release, acq_rel, seq_cst, right +using UnsafeAtomics: none, singlethread using UnsafeAtomics.Internal: OP_RMW_TABLE, inttypes, floattypes using Test @@ -94,4 +95,44 @@ function test_explicit_ordering(T::Type) end end +function test_explicit_syncscope(T::Type) + xs = T[rand(T), rand(T)] + x1 = rand(T) + x2 = rand(T) + @debug "xs=$(repr(xs)) x1=$(repr(x1)) x2=$(repr(x2))" + + ptr = pointer(xs, 1) + GC.@preserve xs begin + + @test UnsafeAtomics.load(ptr, acquire, none) === xs[1] + @test UnsafeAtomics.load(ptr, acquire, singlethread) === xs[1] + UnsafeAtomics.store!(ptr, x1, release, singlethread) + @test xs[1] === x1 + desired = (old = x1, success = true) + @test UnsafeAtomics.cas!(ptr, x1, x2, acq_rel, acquire, singlethread) === desired + @test xs[1] === x2 + @testset for (op, name) in rmw_table_for(T) + xs[1] = x1 + @test UnsafeAtomics.modify!(ptr, op, x2, acq_rel, singlethread) === (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + + rmw = getfield(UnsafeAtomics, Symbol(name, :!)) + xs[1] = x1 + @test rmw(ptr, x2, acquire, singlethread) === x1 + @test xs[1] === op(x1, x2) + end + end +end + +function test_explicit_syncscope() + @testset for T in [UInt, Float64] + test_explicit_syncscope(T) + end + UnsafeAtomics.fence(monotonic, none) + UnsafeAtomics.fence(acquire, singlethread) + UnsafeAtomics.fence(release, singlethread) + UnsafeAtomics.fence(acq_rel, none) + UnsafeAtomics.fence(seq_cst, none) +end + end # module diff --git a/test/UnsafeAtomicsTests/src/test_syncscopes.jl b/test/UnsafeAtomicsTests/src/test_syncscopes.jl new file mode 100644 index 0000000..55279d3 --- /dev/null +++ b/test/UnsafeAtomicsTests/src/test_syncscopes.jl @@ -0,0 +1,21 @@ +module TestSyncScopes + +using UnsafeAtomics: UnsafeAtomics, SyncScope +using Test + +const SYNCSCOPES = [:none, :singlethread] + +function check_syncscope(name::Symbol) + sync = getfield(UnsafeAtomics, name) + @test sync isa SyncScope + @test string(sync) == sprint(print, sync) == string(sync) + @test repr(sync) == "UnsafeAtomics.$name" +end + +function test_syncscope() + @testset for name in SYNCSCOPES + check_syncscope(name) + end +end + +end # module \ No newline at end of file