From 828833a684e441daec97d6e5d548d2051721c222 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Mon, 2 Dec 2024 14:36:24 +0100 Subject: [PATCH 1/7] implement syncscopes everywhere --- src/UnsafeAtomics.jl | 8 +- src/core.jl | 282 ++++++++++++++++++++++++------------------- src/syncscopes.jl | 16 +++ 3 files changed, 182 insertions(+), 124 deletions(-) create mode 100644 src/syncscopes.jl diff --git a/src/UnsafeAtomics.jl b/src/UnsafeAtomics.jl index a83b8d2..988a784 100644 --- a/src/UnsafeAtomics.jl +++ b/src/UnsafeAtomics.jl @@ -1,6 +1,7 @@ baremodule UnsafeAtomics abstract type Ordering end +abstract type SyncScope end function load end function store! end @@ -26,10 +27,11 @@ module Internal using Base.Sys: WORD_SIZE using Base: bitcast, llvmcall -using ..UnsafeAtomics: UnsafeAtomics, Ordering, right +using ..UnsafeAtomics: UnsafeAtomics, Ordering, SyncScope, right include("utils.jl") include("orderings.jl") +include("syncscopes.jl") include("core.jl") end # module Internal @@ -45,4 +47,8 @@ const seq_cst = Internal.seq_cst const acquire_release = acq_rel const sequentially_consistent = seq_cst +# SyncScope +const none = Internal.none +const singlethread = Internal.singlethread + end # baremodule UnsafeAtomics diff --git a/src/core.jl b/src/core.jl index aff57f8..3210d8d 100644 --- a/src/core.jl +++ b/src/core.jl @@ -4,6 +4,12 @@ @inline UnsafeAtomics.modify!(ptr, op, x) = UnsafeAtomics.modify!(ptr, op, x, seq_cst) @inline UnsafeAtomics.fence() = UnsafeAtomics.fence(seq_cst) +@inline UnsafeAtomics.load(x, ord) = UnsafeAtomics.load(x, ord, none) +@inline UnsafeAtomics.store!(x, v, ord) = UnsafeAtomics.store!(x, v, ord, none) +@inline UnsafeAtomics.cas!(x, cmp, new, ord) = UnsafeAtomics.cas!(x, cmp, new, ord, ord, none) +@inline UnsafeAtomics.modify!(ptr, op, x, ord) = UnsafeAtomics.modify!(ptr, op, x, ord, none) +@inline UnsafeAtomics.fence(ord) = UnsafeAtomics.fence(ord, none) + #! format: off # https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl#L23-L30 if Sys.ARCH == :i686 || startswith(string(Sys.ARCH), "arm") || @@ -45,8 +51,9 @@ const OP_RMW_TABLE = [ for (op, rmwop) in OP_RMW_TABLE fn = Symbol(rmwop, "!") @eval @inline UnsafeAtomics.$fn(x, v) = UnsafeAtomics.$fn(x, v, seq_cst) - @eval @inline UnsafeAtomics.$fn(ptr, x, ord) = - first(UnsafeAtomics.modify!(ptr, $op, x, ord)) + @eval @inline UnsafeAtomics.$fn(x, v, ord) = UnsafeAtomics.$fn(x, v, ord, none) + @eval @inline UnsafeAtomics.$fn(ptr, x, ord, scope) = + first(UnsafeAtomics.modify!(ptr, $op, x, ord, scope)) end const ATOMIC_INTRINSICS = isdefined(Core.Intrinsics, :atomic_pointerref) @@ -67,47 +74,51 @@ for typ in (inttypes..., floattypes...) for ord in orderings ord in (release, acq_rel) && continue - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) - return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord)) - end - else - @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) - return llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rv = load atomic $rt %ptr $ord, align $(sizeof(typ)) - ret $lt %rv - """), - $typ, - Tuple{Ptr{$typ}}, - x, - ) + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync))) + return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord)) + end + else + @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = load atomic $rt %ptr $ord, align $(sizeof(typ)) + ret $lt %rv + """), + $typ, + Tuple{Ptr{$typ}}, + x, + ) + end end end end for ord in orderings ord in (acquire, acq_rel) && continue - - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) - Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord)) - return nothing - end - else - @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) - return llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ)) - ret void - """), - Cvoid, - Tuple{Ptr{$typ},$typ}, - x, - v, - ) + + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync))) + Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord)) + return nothing + end + else + @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ)) + ret void + """), + Cvoid, + Tuple{Ptr{$typ},$typ}, + x, + v, + ) + end end end end @@ -117,54 +128,58 @@ for typ in (inttypes..., floattypes...) typ <: AbstractFloat && break - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.cas!( - x::Ptr{$typ}, - cmp::$typ, - new::$typ, - ::$(typeof(success_ordering)), - ::$(typeof(failure_ordering)), - ) - return Core.Intrinsics.atomic_pointerreplace( - x, - cmp, - new, - base_ordering($success_ordering), - base_ordering($failure_ordering) + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.cas!( + x::Ptr{$typ}, + cmp::$typ, + new::$typ, + ::$(typeof(success_ordering)), + ::$(typeof(failure_ordering)), + ::$(typeof(sync)), ) - end - else - @eval function UnsafeAtomics.cas!( - x::Ptr{$typ}, - cmp::$typ, - new::$typ, - ::$(typeof(success_ordering)), - ::$(typeof(failure_ordering)), - ) - success = Ref{Int8}() - GC.@preserve success begin - old = llvmcall( - $( - """ - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering - %rv = extractvalue { $lt, i1 } %rs, 0 - %s1 = extractvalue { $lt, i1 } %rs, 1 - %s8 = zext i1 %s1 to i8 - %sptr = inttoptr i$WORD_SIZE %3 to i8* - store i8 %s8, i8* %sptr - ret $lt %rv - """ - ), - $typ, - Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}}, + return Core.Intrinsics.atomic_pointerreplace( x, cmp, new, - Ptr{Int8}(pointer_from_objref(success)), + base_ordering($success_ordering), + base_ordering($failure_ordering) ) end - return (old = old, success = !iszero(success[])) + else + @eval function UnsafeAtomics.cas!( + x::Ptr{$typ}, + cmp::$typ, + new::$typ, + ::$(typeof(success_ordering)), + ::$(typeof(failure_ordering)), + ::$(typeof(sync)), + ) + success = Ref{Int8}() + GC.@preserve success begin + old = llvmcall( + $( + """ + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering + %rv = extractvalue { $lt, i1 } %rs, 0 + %s1 = extractvalue { $lt, i1 } %rs, 1 + %s8 = zext i1 %s1 to i8 + %sptr = inttoptr i$WORD_SIZE %3 to i8* + store i8 %s8, i8* %sptr + ret $lt %rv + """ + ), + $typ, + Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}}, + x, + cmp, + new, + Ptr{Int8}(pointer_from_objref(success)), + ) + end + return (old = old, success = !iszero(success[])) + end end end end @@ -186,60 +201,81 @@ for typ in (inttypes..., floattypes...) end end for ord in orderings - # Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged - if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.modify!( + for sync in syncscopes + # Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged + if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.modify!( + x::Ptr{$typ}, + op::typeof($op), + v::$typ, + ::$(typeof(ord)), + ::$(typeof(sync)), + ) + return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord)) + end + else + @eval function UnsafeAtomics.modify!( x::Ptr{$typ}, - op::typeof($op), + ::typeof($op), v::$typ, ::$(typeof(ord)), + ::$(typeof(sync)), ) - return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord)) - end - else - @eval function UnsafeAtomics.modify!( - x::Ptr{$typ}, - ::typeof($op), - v::$typ, - ::$(typeof(ord)), - ) - old = llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord - ret $lt %rv - """), - $typ, - Tuple{Ptr{$typ},$typ}, - x, - v, - ) - return old => $op(old, v) + old = llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord + ret $lt %rv + """), + $typ, + Tuple{Ptr{$typ},$typ}, + x, + v, + ) + return old => $op(old, v) + end end end end end end -# Core.Intrinsics.atomic_fence was introduced in 1.10 -function UnsafeAtomics.fence(ord::Ordering) - Core.Intrinsics.atomic_fence(base_ordering(ord)) - return nothing -end -if Sys.ARCH == :x86_64 - # FIXME: Disable this once on LLVM 19 - # This is unfortunatly required for good-performance on AMD - # https://github.com/llvm/llvm-project/pull/106555 - function UnsafeAtomics.fence(::typeof(seq_cst)) - Base.llvmcall( - (raw""" - define void @fence() #0 { - entry: - tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory} - ret void - } - attributes #0 = { alwaysinline } - """, "fence"), Nothing, Tuple{}) +for sync in syncscopes + if sync == none + # Core.Intrinsics.atomic_fence was introduced in 1.10 + @eval function UnsafeAtomics.fence(ord::Ordering, ::$(typeof(sync))) + Core.Intrinsics.atomic_fence(base_ordering(ord)) + return nothing + end + if Sys.ARCH == :x86_64 + # FIXME: Disable this once on LLVM 19 + # This is unfortunatly required for good-performance on AMD + # https://github.com/llvm/llvm-project/pull/106555 + @eval function UnsafeAtomics.fence(::typeof(seq_cst), ::$(typeof(sync))) + Base.llvmcall( + (raw""" + define void @fence() #0 { + entry: + tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory} + ret void + } + attributes #0 = { alwaysinline } + """, "fence"), Nothing, Tuple{}) + end + end + else + for ord in orderings + @eval function UnsafeAtomics.fence(::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + fence $sync $ord + ret void + """), + Cvoid, + Tuple{}, + ) + end + end end end diff --git a/src/syncscopes.jl b/src/syncscopes.jl new file mode 100644 index 0000000..6a201c6 --- /dev/null +++ b/src/syncscopes.jl @@ -0,0 +1,16 @@ +struct LLVMSyncScope{name} <: SyncScope end + +const none = LLVMSyncScope{:none}() +const singlethread = LLVMSyncScope{:singlethread}() + +const syncscopes = (none, singlethread) +const ConcreteSyncScopes = Union{map(typeof, syncscopes)...} + +llvm_syncscope(::LLVMSyncScope{name}) where {name} = name + +Base.string(s::LLVMSyncScope) = string("syncscope(\"", llvm_syncscope(s), "\")") +Base.string(s::typeof(none)) = "" + +Base.print(io::IO, s::LLVMSyncScope) = print(io, string(s)) + +Base.show(io::IO, o::ConcreteSyncScopes) = print(io, UnsafeAtomics, '.', llvm_ordering(o)) \ No newline at end of file From 84ec64e9e499cfbfaad1788506f4c5b81fcb7814 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 5 Dec 2024 12:37:55 +0100 Subject: [PATCH 2/7] Support syncscope in UnsafeAtomicsLLVM --- Project.toml | 2 +- ext/UnsafeAtomicsLLVM/atomics.jl | 249 +++--------------------------- ext/UnsafeAtomicsLLVM/internal.jl | 21 ++- 3 files changed, 33 insertions(+), 239 deletions(-) diff --git a/Project.toml b/Project.toml index 4851885..6b01112 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,7 @@ LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" UnsafeAtomicsLLVM = ["LLVM"] [compat] -LLVM = "8.1, 9" +LLVM = "9" julia = "1.10" [extras] diff --git a/ext/UnsafeAtomicsLLVM/atomics.jl b/ext/UnsafeAtomicsLLVM/atomics.jl index 2c5691a..78ac3f9 100644 --- a/ext/UnsafeAtomicsLLVM/atomics.jl +++ b/ext/UnsafeAtomicsLLVM/atomics.jl @@ -89,68 +89,12 @@ for (julia, llvm) in pairs(_llvm_from_julia_ordering) @eval llvm_from_julia_ordering(::Val{$(QuoteNode(julia))}) = Val{$llvm}() end -""" - @dynamic_order(order) do order - ... use order ... - end -It is expanded to an expression similar to: - if order === :not_atomic - let order = Val(:not_atomic) - ... use order ... - end - elseif order === :unordered - let order = Val(:unordered) - ... use order ... - end - elseif ... - ... - else - throw(ConcurrencyViolationError(...)) - end -This is used for helping the compiler to optimize expressions such as -`atomic_pointerref(ptr, :monotonic)` and also to avoid abstract run-time dispatch. -""" -macro dynamic_order(thunk, order) - @assert Meta.isexpr(thunk, :->, 2) && Meta.isexpr(thunk.args[1], :tuple, 1) - ordervar = esc(thunk.args[1].args[1]) - body = esc(thunk.args[2]) - expr = foldr( - keys(_llvm_from_julia_ordering), - init = :(throw(ConcurrencyViolationError("invalid atomic ordering: ", order))), - ) do key, r - quote - if order === $(QuoteNode(key)) - let $ordervar = Val{$(QuoteNode(key))}() - $body - end - else - $r - end - end - end - quote - order = $(esc(order)) - $expr - end -end - _valueof(::Val{x}) where {x} = x -@inline function atomic_pointerref(pointer, order::Symbol) - @dynamic_order(order) do order - atomic_pointerref(pointer, order) - end -end - -@inline function atomic_pointerset(pointer, x, order::Symbol) - @dynamic_order(order) do order - atomic_pointerset(pointer, x, order) - end -end - -@generated function atomic_pointerref(ptr::LLVMPtr{T,A}, order::AllOrdering) where {T,A} +@generated function atomic_pointerref(ptr::LLVMPtr{T,A}, order::AllOrdering, sync) where {T,A} sizeof(T) == 0 && return T.instance - llvm_order = _valueof(llvm_from_julia_ordering(order())) + llvm_order = _valueof(llvm_from_julia_ordering(order())) + llvm_syncscope = _valueof(sync()) @dispose ctx = Context() begin eltyp = convert(LLVMType, T) @@ -170,6 +114,7 @@ end typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) ld = load!(builder, eltyp, typed_ptr) ordering!(ld, llvm_order) + syncscope!(ld, SyncScope(string(llvm_syncscope))) if A != 0 metadata(ld)[LLVM.MD_tbaa] = tbaa_addrspace(A) @@ -187,6 +132,7 @@ end ptr::LLVMPtr{T,A}, x::T, order::AllOrdering, + sync, ) where {T,A} if sizeof(T) == 0 # Mimicking what `Core.Intrinsics.atomic_pointerset` generates. @@ -197,7 +143,8 @@ end return ptr end end - llvm_order = _valueof(llvm_from_julia_ordering(order())) + llvm_order = _valueof(llvm_from_julia_ordering(order())) + llvm_syncscope = _valueof(sync()) @dispose ctx = Context() begin eltyp = convert(LLVMType, T) T_ptr = convert(LLVMType, ptr) @@ -216,6 +163,7 @@ end val = parameters(llvm_f)[2] st = store!(builder, val, typed_ptr) ordering!(st, llvm_order) + syncscope!(st, SyncScope(string(llvm_syncscope))) if A != 0 metadata(st)[LLVM.MD_tbaa] = tbaa_addrspace(A) @@ -254,14 +202,12 @@ const binoptable = [ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} -# LLVM API accepts string literal as a syncscope argument. -@inline syncscope_to_string(::Type{Val{S}}) where {S} = string(S) - @generated function llvm_atomic_op( binop::AtomicRMWBinOpVal, ptr::LLVMPtr{T,A}, val::T, order::LLVMOrderingVal, + sync, ) where {T,A} @dispose ctx = Context() begin T_val = convert(LLVMType, T) @@ -270,6 +216,7 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} T_typed_ptr = LLVM.PointerType(T_val, A) llvm_f, _ = create_function(T_val, [T_ptr, T_val]) + llvm_syncscope = _valueof(sync()) @dispose builder = IRBuilder() begin entry = BasicBlock(llvm_f, "entry") @@ -277,14 +224,13 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) - single_threaded = false rv = atomic_rmw!( builder, _valueof(binop()), typed_ptr, parameters(llvm_f)[2], _valueof(order()), - single_threaded, + SyncScope(string(llvm_syncscope)) ) ret!(builder, rv) @@ -299,8 +245,8 @@ end ptr::LLVMPtr{T,A}, val::T, order::LLVMOrderingVal, - syncscope::Val{S}, -) where {T,A,S} + sync, +) where {T,A} @dispose ctx = Context() begin T_val = convert(LLVMType, T) T_ptr = convert(LLVMType, ptr) @@ -308,6 +254,8 @@ end T_typed_ptr = LLVM.PointerType(T_val, A) llvm_f, _ = create_function(T_val, [T_ptr, T_val]) + llvm_syncscope = _valueof(sync()) + @dispose builder = IRBuilder() begin entry = BasicBlock(llvm_f, "entry") position!(builder, entry) @@ -319,7 +267,7 @@ end typed_ptr, parameters(llvm_f)[2], _valueof(order()), - syncscope_to_string(syncscope), + SyncScope(string(llvm_syncscope)), ) ret!(builder, rv) @@ -328,118 +276,17 @@ end end end -@inline function atomic_pointermodify(pointer, op::OP, x, order::Symbol) where {OP} - @dynamic_order(order) do order - atomic_pointermodify(pointer, op, x, order) - end -end - -@inline function atomic_pointermodify( - ptr::LLVMPtr{T}, - op, - x::T, - ::Val{:not_atomic}, -) where {T} - old = atomic_pointerref(ptr, Val(:not_atomic)) - new = op(old, x) - atomic_pointerset(ptr, new, Val(:not_atomic)) - return old => new -end - -@inline function atomic_pointermodify( - ptr::LLVMPtr{T}, - ::typeof(right), - x::T, - order::AtomicOrdering, -) where {T} - old = llvm_atomic_op( - Val(LLVM.API.LLVMAtomicRMWBinOpXchg), - ptr, - x, - llvm_from_julia_ordering(order), - ) - return old => x -end - -const atomictypes = Any[ - Int8, - Int16, - Int32, - Int64, - Int128, - UInt8, - UInt16, - UInt32, - UInt64, - UInt128, - Float16, - Float32, - Float64, -] - -for (opname, op, llvmop) in binoptable - opname === :xchg && continue - types = if opname in (:min, :max) - filter(t -> t <: Signed, atomictypes) - elseif opname in (:umin, :umax) - filter(t -> t <: Unsigned, atomictypes) - elseif opname in (:fadd, :fsub, :fmin, :fmax) - filter(t -> t <: AbstractFloat, atomictypes) - else - filter(t -> t <: Integer, atomictypes) - end - for T in types - @eval @inline function atomic_pointermodify( - ptr::LLVMPtr{$T}, - ::$(typeof(op)), - x::$T, - order::AtomicOrdering, - syncscope::Val{S} = Val{:system}(), - ) where {S} - old = - syncscope isa Val{:system} ? - llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) : - llvm_atomic_op( - $(Val(llvmop)), - ptr, - x, - llvm_from_julia_ordering(order), - syncscope, - ) - return old => $op(old, x) - end - end -end - -@inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) -@inline atomic_pointerswap(pointer, new, order) = - first(atomic_pointermodify(pointer, right, new, order)) - -@inline function atomic_pointermodify( - ptr::LLVMPtr{T}, - op, - x::T, - order::AllOrdering, -) where {T} - # Should `fail_order` be stronger? Ref: https://github.com/JuliaLang/julia/issues/45256 - fail_order = Val(:monotonic) - old = atomic_pointerref(ptr, fail_order) - while true - new = op(old, x) - (old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order) - success && return old => new - end -end - @generated function llvm_atomic_cas( ptr::LLVMPtr{T,A}, cmp::T, val::T, success_order::LLVMOrderingVal, fail_order::LLVMOrderingVal, + sync, ) where {T,A} llvm_success = _valueof(success_order()) llvm_fail = _valueof(fail_order()) + llvm_syncscope = _valueof(sync()) @dispose ctx = Context() begin T_val = convert(LLVMType, T) T_pointee = T_val @@ -471,7 +318,6 @@ end val_int = bitcast!(builder, val_int, T_pointee) end - single_threaded = false res = atomic_cmpxchg!( builder, typed_ptr, @@ -479,7 +325,7 @@ end val_int, llvm_success, llvm_fail, - single_threaded, + SyncScope(string(llvm_syncscope)), ) rv = extract_value!(builder, res, 0) @@ -513,60 +359,3 @@ end end end end - -@inline function atomic_pointerreplace( - pointer, - expected, - desired, - success_order::Symbol, - fail_order::Symbol, -) - # This avoids abstract dispatch at run-time but probably too much codegen? - #= - @dynamic_order(success_order) do success_order - @dynamic_order(fail_order) do fail_order - atomic_pointerreplace(pointer, expected, desired, success_order, fail_order) - end - end - =# - - # This avoids excessive codegen while hopefully imposes no cost when const-prop works: - so = @dynamic_order(success_order) do success_order - success_order - end - fo = @dynamic_order(fail_order) do fail_order - fail_order - end - return atomic_pointerreplace(pointer, expected, desired, so, fo) -end - -@inline function atomic_pointerreplace( - ptr::LLVMPtr{T}, - expected::T, - desired::T, - ::Val{:not_atomic}, - ::Val{:not_atomic}, -) where {T} - old = atomic_pointerref(ptr, Val(:not_atomic)) - if old === expected - atomic_pointerset(ptr, desired, Val(:not_atomic)) - success = true - else - success = false - end - return (; old, success) -end - -@inline atomic_pointerreplace( - ptr::LLVMPtr{T}, - expected::T, - desired::T, - success_order::_julia_ordering(∉((:not_atomic, :unordered))), - fail_order::_julia_ordering(∉((:not_atomic, :unordered, :release, :acquire_release))), -) where {T} = llvm_atomic_cas( - ptr, - expected, - desired, - llvm_from_julia_ordering(success_order), - llvm_from_julia_ordering(fail_order), -) diff --git a/ext/UnsafeAtomicsLLVM/internal.jl b/ext/UnsafeAtomicsLLVM/internal.jl index e4ed556..1419e7d 100644 --- a/ext/UnsafeAtomicsLLVM/internal.jl +++ b/ext/UnsafeAtomicsLLVM/internal.jl @@ -4,30 +4,33 @@ julia_ordering_name(::typeof(UnsafeAtomics.acquire_release)) = :acquire_release julia_ordering_name(::typeof(UnsafeAtomics.sequentially_consistent)) = :sequentially_consistent +julia_syncscope_name(::UnsafeAtomics.Internal.LLVMSyncScope{name}) where {name} = name +julia_syncscope_name(::typeof(UnsafeAtomics.none)) = :system + include("atomics.jl") -@inline UnsafeAtomics.load(ptr::LLVMPtr, order::Ordering) = - atomic_pointerref(ptr, Val{julia_ordering_name(order)}()) +@inline UnsafeAtomics.load(ptr::LLVMPtr, order::Ordering, sync::SyncScope) = + atomic_pointerref(ptr, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) -@inline function UnsafeAtomics.store!(ptr::LLVMPtr, x, order::Ordering) - atomic_pointerset(ptr, x, Val{julia_ordering_name(order)}()) +@inline function UnsafeAtomics.store!(ptr::LLVMPtr, x, order::Ordering, sync::SyncScope) + atomic_pointerset(ptr, x, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) return end mapop(op::OP) where {OP} = op mapop(::typeof(UnsafeAtomics.right)) = right -@inline UnsafeAtomics.modify!(ptr::LLVMPtr, op::OP, x, order::Ordering) where {OP} = - atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}()) +@inline UnsafeAtomics.modify!(ptr::LLVMPtr, op::OP, x, order::Ordering, sync::SyncScope) where {OP} = + atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) @inline UnsafeAtomics.modify!( ptr::LLVMPtr, op::OP, x, order::Ordering, - syncscope::Val{S} = Val(:system), + sync::SyncScope, ) where {OP<:Union{typeof(+),typeof(-)},S} = - atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope) + atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) @inline UnsafeAtomics.cas!( ptr::LLVMPtr, @@ -35,10 +38,12 @@ mapop(::typeof(UnsafeAtomics.right)) = right desired, success_order::Ordering, failure_order::Ordering, + sync::SyncScope, ) = atomic_pointerreplace( ptr, expected, desired, Val{julia_ordering_name(success_order)}(), Val{julia_ordering_name(failure_order)}(), + Val{julia_syncscope_name(sync)}(), ) From 43d9bd4d20bbce4a074adba46af60fc02eed312e Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 5 Dec 2024 13:23:22 +0100 Subject: [PATCH 3/7] fixup! Support syncscope in UnsafeAtomicsLLVM --- ext/UnsafeAtomicsLLVM/atomics.jl | 36 ------------------------------- ext/UnsafeAtomicsLLVM/internal.jl | 17 ++++----------- src/core.jl | 15 +++++++------ test/UnsafeAtomicsLLVM.jl | 4 ++-- 4 files changed, 15 insertions(+), 57 deletions(-) diff --git a/ext/UnsafeAtomicsLLVM/atomics.jl b/ext/UnsafeAtomicsLLVM/atomics.jl index 78ac3f9..a924ed2 100644 --- a/ext/UnsafeAtomicsLLVM/atomics.jl +++ b/ext/UnsafeAtomicsLLVM/atomics.jl @@ -240,42 +240,6 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} end end -@generated function llvm_atomic_op( - binop::AtomicRMWBinOpVal, - ptr::LLVMPtr{T,A}, - val::T, - order::LLVMOrderingVal, - sync, -) where {T,A} - @dispose ctx = Context() begin - T_val = convert(LLVMType, T) - T_ptr = convert(LLVMType, ptr) - - T_typed_ptr = LLVM.PointerType(T_val, A) - llvm_f, _ = create_function(T_val, [T_ptr, T_val]) - - llvm_syncscope = _valueof(sync()) - - @dispose builder = IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) - rv = atomic_rmw!( - builder, - _valueof(binop()), - typed_ptr, - parameters(llvm_f)[2], - _valueof(order()), - SyncScope(string(llvm_syncscope)), - ) - - ret!(builder, rv) - end - call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val) - end -end - @generated function llvm_atomic_cas( ptr::LLVMPtr{T,A}, cmp::T, diff --git a/ext/UnsafeAtomicsLLVM/internal.jl b/ext/UnsafeAtomicsLLVM/internal.jl index 1419e7d..90fe951 100644 --- a/ext/UnsafeAtomicsLLVM/internal.jl +++ b/ext/UnsafeAtomicsLLVM/internal.jl @@ -9,10 +9,10 @@ julia_syncscope_name(::typeof(UnsafeAtomics.none)) = :system include("atomics.jl") -@inline UnsafeAtomics.load(ptr::LLVMPtr, order::Ordering, sync::SyncScope) = +@inline UnsafeAtomics.load(ptr::LLVMPtr, order::Ordering, sync::UnsafeAtomics.Internal.SyncScope) = atomic_pointerref(ptr, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) -@inline function UnsafeAtomics.store!(ptr::LLVMPtr, x, order::Ordering, sync::SyncScope) +@inline function UnsafeAtomics.store!(ptr::LLVMPtr, x, order::Ordering, sync::UnsafeAtomics.Internal.SyncScope) atomic_pointerset(ptr, x, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) return end @@ -20,16 +20,7 @@ end mapop(op::OP) where {OP} = op mapop(::typeof(UnsafeAtomics.right)) = right -@inline UnsafeAtomics.modify!(ptr::LLVMPtr, op::OP, x, order::Ordering, sync::SyncScope) where {OP} = - atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) - -@inline UnsafeAtomics.modify!( - ptr::LLVMPtr, - op::OP, - x, - order::Ordering, - sync::SyncScope, -) where {OP<:Union{typeof(+),typeof(-)},S} = +@inline UnsafeAtomics.modify!(ptr::LLVMPtr, op::OP, x, order::Ordering, sync::UnsafeAtomics.Internal.SyncScope) where {OP} = atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), Val{julia_syncscope_name(sync)}()) @inline UnsafeAtomics.cas!( @@ -38,7 +29,7 @@ mapop(::typeof(UnsafeAtomics.right)) = right desired, success_order::Ordering, failure_order::Ordering, - sync::SyncScope, + sync::UnsafeAtomics.Internal.SyncScope, ) = atomic_pointerreplace( ptr, expected, diff --git a/src/core.jl b/src/core.jl index 3210d8d..733d19e 100644 --- a/src/core.jl +++ b/src/core.jl @@ -7,6 +7,7 @@ @inline UnsafeAtomics.load(x, ord) = UnsafeAtomics.load(x, ord, none) @inline UnsafeAtomics.store!(x, v, ord) = UnsafeAtomics.store!(x, v, ord, none) @inline UnsafeAtomics.cas!(x, cmp, new, ord) = UnsafeAtomics.cas!(x, cmp, new, ord, ord, none) +@inline UnsafeAtomics.cas!(x, cmp, new, success_ord, failure_order) = UnsafeAtomics.cas!(x, cmp, new, success_ord, failure_order, none) @inline UnsafeAtomics.modify!(ptr, op, x, ord) = UnsafeAtomics.modify!(ptr, op, x, ord, none) @inline UnsafeAtomics.fence(ord) = UnsafeAtomics.fence(ord, none) @@ -294,20 +295,20 @@ as_native_uint(::Type{T}) where {T} = error(LazyString("unsupported size: ", sizeof(T))) end -function UnsafeAtomics.load(x::Ptr{T}, ordering) where {T} +function UnsafeAtomics.load(x::Ptr{T}, ordering, syncscope) where {T} UI = as_native_uint(T) - v = UnsafeAtomics.load(Ptr{UI}(x), ordering) + v = UnsafeAtomics.load(Ptr{UI}(x), ordering, syncscope) return bitcast(T, v) end -function UnsafeAtomics.store!(x::Ptr{T}, v::T, ordering) where {T} +function UnsafeAtomics.store!(x::Ptr{T}, v::T, ordering, syncscope) where {T} UI = as_native_uint(T) - UnsafeAtomics.store!(Ptr{UI}(x), bitcast(UI, v), ordering)::Nothing + UnsafeAtomics.store!(Ptr{UI}(x), bitcast(UI, v), ordering, syncscope)::Nothing end -function UnsafeAtomics.modify!(x::Ptr{T}, ::typeof(right), v::T, ordering) where {T} +function UnsafeAtomics.modify!(x::Ptr{T}, ::typeof(right), v::T, ordering, syncscope) where {T} UI = as_native_uint(T) - old, _ = UnsafeAtomics.modify!(Ptr{UI}(x), right, bitcast(UI, v), ordering) + old, _ = UnsafeAtomics.modify!(Ptr{UI}(x), right, bitcast(UI, v), ordering, syncscope) return bitcast(T, old) => v end @@ -317,6 +318,7 @@ function UnsafeAtomics.cas!( new::T, success_ordering, failure_ordering, + syncscope, ) where {T} UI = as_native_uint(T) (old, success) = UnsafeAtomics.cas!( @@ -325,6 +327,7 @@ function UnsafeAtomics.cas!( bitcast(UI, new), success_ordering, failure_ordering, + syncscope ) return (old = bitcast(T, old), success = success) end diff --git a/test/UnsafeAtomicsLLVM.jl b/test/UnsafeAtomicsLLVM.jl index 81c0fa5..5fa02fa 100644 --- a/test/UnsafeAtomicsLLVM.jl +++ b/test/UnsafeAtomicsLLVM.jl @@ -71,12 +71,12 @@ function test_explicit_ordering(xs::AbstractArray{T}, x1::T, x2::T) where T # Test syncscopes. if (op == +) || (op == -) xs[1] = x1 - @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:system)) === + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, UnsafeAtomics.none) === (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) xs[1] = x1 - @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:singlethread)) === + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, UnsafeAtomics.singlethread) === (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) end From 3ff51d271f76662895aa167846dc7ff6ab68fbc7 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 5 Dec 2024 13:27:35 +0100 Subject: [PATCH 4/7] add back old code --- ext/UnsafeAtomicsLLVM/atomics.jl | 194 +++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) diff --git a/ext/UnsafeAtomicsLLVM/atomics.jl b/ext/UnsafeAtomicsLLVM/atomics.jl index a924ed2..1f5be12 100644 --- a/ext/UnsafeAtomicsLLVM/atomics.jl +++ b/ext/UnsafeAtomicsLLVM/atomics.jl @@ -240,6 +240,143 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} end end +@generated function llvm_atomic_op( + binop::AtomicRMWBinOpVal, + ptr::LLVMPtr{T,A}, + val::T, + order::LLVMOrderingVal, + syncscope::Val{S}, +) where {T,A,S} + @dispose ctx = Context() begin + T_val = convert(LLVMType, T) + T_ptr = convert(LLVMType, ptr) + + T_typed_ptr = LLVM.PointerType(T_val, A) + llvm_f, _ = create_function(T_val, [T_ptr, T_val]) + + @dispose builder = IRBuilder() begin + entry = BasicBlock(llvm_f, "entry") + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + rv = atomic_rmw!( + builder, + _valueof(binop()), + typed_ptr, + parameters(llvm_f)[2], + _valueof(order()), + syncscope_to_string(syncscope), + ) + + ret!(builder, rv) + end + call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val) + end +end + +@inline function atomic_pointermodify(pointer, op::OP, x, order::Symbol) where {OP} + @dynamic_order(order) do order + atomic_pointermodify(pointer, op, x, order) + end +end + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + op, + x::T, + ::Val{:not_atomic}, +) where {T} + old = atomic_pointerref(ptr, Val(:not_atomic)) + new = op(old, x) + atomic_pointerset(ptr, new, Val(:not_atomic)) + return old => new +end + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + ::typeof(right), + x::T, + order::AtomicOrdering, +) where {T} + old = llvm_atomic_op( + Val(LLVM.API.LLVMAtomicRMWBinOpXchg), + ptr, + x, + llvm_from_julia_ordering(order), + ) + return old => x +end + +const atomictypes = Any[ + Int8, + Int16, + Int32, + Int64, + Int128, + UInt8, + UInt16, + UInt32, + UInt64, + UInt128, + Float16, + Float32, + Float64, +] + +for (opname, op, llvmop) in binoptable + opname === :xchg && continue + types = if opname in (:min, :max) + filter(t -> t <: Signed, atomictypes) + elseif opname in (:umin, :umax) + filter(t -> t <: Unsigned, atomictypes) + elseif opname in (:fadd, :fsub, :fmin, :fmax) + filter(t -> t <: AbstractFloat, atomictypes) + else + filter(t -> t <: Integer, atomictypes) + end + for T in types + @eval @inline function atomic_pointermodify( + ptr::LLVMPtr{$T}, + ::$(typeof(op)), + x::$T, + order::AtomicOrdering, + syncscope::Val{S} = Val{:system}(), + ) where {S} + old = + syncscope isa Val{:system} ? + llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) : + llvm_atomic_op( + $(Val(llvmop)), + ptr, + x, + llvm_from_julia_ordering(order), + syncscope, + ) + return old => $op(old, x) + end + end +end + +@inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) +@inline atomic_pointerswap(pointer, new, order) = + first(atomic_pointermodify(pointer, right, new, order)) + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + op, + x::T, + order::AllOrdering, +) where {T} + # Should `fail_order` be stronger? Ref: https://github.com/JuliaLang/julia/issues/45256 + fail_order = Val(:monotonic) + old = atomic_pointerref(ptr, fail_order) + while true + new = op(old, x) + (old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order) + success && return old => new + end +end + @generated function llvm_atomic_cas( ptr::LLVMPtr{T,A}, cmp::T, @@ -323,3 +460,60 @@ end end end end + +@inline function atomic_pointerreplace( + pointer, + expected, + desired, + success_order::Symbol, + fail_order::Symbol, +) + # This avoids abstract dispatch at run-time but probably too much codegen? + #= + @dynamic_order(success_order) do success_order + @dynamic_order(fail_order) do fail_order + atomic_pointerreplace(pointer, expected, desired, success_order, fail_order) + end + end + =# + + # This avoids excessive codegen while hopefully imposes no cost when const-prop works: + so = @dynamic_order(success_order) do success_order + success_order + end + fo = @dynamic_order(fail_order) do fail_order + fail_order + end + return atomic_pointerreplace(pointer, expected, desired, so, fo) +end + +@inline function atomic_pointerreplace( + ptr::LLVMPtr{T}, + expected::T, + desired::T, + ::Val{:not_atomic}, + ::Val{:not_atomic}, +) where {T} + old = atomic_pointerref(ptr, Val(:not_atomic)) + if old === expected + atomic_pointerset(ptr, desired, Val(:not_atomic)) + success = true + else + success = false + end + return (; old, success) +end + +@inline atomic_pointerreplace( + ptr::LLVMPtr{T}, + expected::T, + desired::T, + success_order::_julia_ordering(∉((:not_atomic, :unordered))), + fail_order::_julia_ordering(∉((:not_atomic, :unordered, :release, :acquire_release))), +) where {T} = llvm_atomic_cas( + ptr, + expected, + desired, + llvm_from_julia_ordering(success_order), + llvm_from_julia_ordering(fail_order), +) From f66264e504afef4e47e74a1446d613ac73bf6556 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 5 Dec 2024 13:28:55 +0100 Subject: [PATCH 5/7] fixup! add back old code --- ext/UnsafeAtomicsLLVM/atomics.jl | 44 -------------------------------- 1 file changed, 44 deletions(-) diff --git a/ext/UnsafeAtomicsLLVM/atomics.jl b/ext/UnsafeAtomicsLLVM/atomics.jl index 1f5be12..cc5ef64 100644 --- a/ext/UnsafeAtomicsLLVM/atomics.jl +++ b/ext/UnsafeAtomicsLLVM/atomics.jl @@ -274,24 +274,6 @@ end end end -@inline function atomic_pointermodify(pointer, op::OP, x, order::Symbol) where {OP} - @dynamic_order(order) do order - atomic_pointermodify(pointer, op, x, order) - end -end - -@inline function atomic_pointermodify( - ptr::LLVMPtr{T}, - op, - x::T, - ::Val{:not_atomic}, -) where {T} - old = atomic_pointerref(ptr, Val(:not_atomic)) - new = op(old, x) - atomic_pointerset(ptr, new, Val(:not_atomic)) - return old => new -end - @inline function atomic_pointermodify( ptr::LLVMPtr{T}, ::typeof(right), @@ -461,32 +443,6 @@ end end end -@inline function atomic_pointerreplace( - pointer, - expected, - desired, - success_order::Symbol, - fail_order::Symbol, -) - # This avoids abstract dispatch at run-time but probably too much codegen? - #= - @dynamic_order(success_order) do success_order - @dynamic_order(fail_order) do fail_order - atomic_pointerreplace(pointer, expected, desired, success_order, fail_order) - end - end - =# - - # This avoids excessive codegen while hopefully imposes no cost when const-prop works: - so = @dynamic_order(success_order) do success_order - success_order - end - fo = @dynamic_order(fail_order) do fail_order - fail_order - end - return atomic_pointerreplace(pointer, expected, desired, so, fo) -end - @inline function atomic_pointerreplace( ptr::LLVMPtr{T}, expected::T, From 06675ab04a81d312c5a891be0c666f0282ff9fe4 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 5 Dec 2024 13:32:27 +0100 Subject: [PATCH 6/7] fixup! fixup! add back old code --- ext/UnsafeAtomicsLLVM/atomics.jl | 88 ++------------------------------ 1 file changed, 3 insertions(+), 85 deletions(-) diff --git a/ext/UnsafeAtomicsLLVM/atomics.jl b/ext/UnsafeAtomicsLLVM/atomics.jl index cc5ef64..452b8ae 100644 --- a/ext/UnsafeAtomicsLLVM/atomics.jl +++ b/ext/UnsafeAtomicsLLVM/atomics.jl @@ -240,105 +240,23 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} end end -@generated function llvm_atomic_op( - binop::AtomicRMWBinOpVal, - ptr::LLVMPtr{T,A}, - val::T, - order::LLVMOrderingVal, - syncscope::Val{S}, -) where {T,A,S} - @dispose ctx = Context() begin - T_val = convert(LLVMType, T) - T_ptr = convert(LLVMType, ptr) - - T_typed_ptr = LLVM.PointerType(T_val, A) - llvm_f, _ = create_function(T_val, [T_ptr, T_val]) - - @dispose builder = IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) - rv = atomic_rmw!( - builder, - _valueof(binop()), - typed_ptr, - parameters(llvm_f)[2], - _valueof(order()), - syncscope_to_string(syncscope), - ) - - ret!(builder, rv) - end - call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val) - end -end - @inline function atomic_pointermodify( ptr::LLVMPtr{T}, ::typeof(right), x::T, order::AtomicOrdering, -) where {T} + sync::Val{S} +) where {T, S} old = llvm_atomic_op( Val(LLVM.API.LLVMAtomicRMWBinOpXchg), ptr, x, llvm_from_julia_ordering(order), + sync ) return old => x end -const atomictypes = Any[ - Int8, - Int16, - Int32, - Int64, - Int128, - UInt8, - UInt16, - UInt32, - UInt64, - UInt128, - Float16, - Float32, - Float64, -] - -for (opname, op, llvmop) in binoptable - opname === :xchg && continue - types = if opname in (:min, :max) - filter(t -> t <: Signed, atomictypes) - elseif opname in (:umin, :umax) - filter(t -> t <: Unsigned, atomictypes) - elseif opname in (:fadd, :fsub, :fmin, :fmax) - filter(t -> t <: AbstractFloat, atomictypes) - else - filter(t -> t <: Integer, atomictypes) - end - for T in types - @eval @inline function atomic_pointermodify( - ptr::LLVMPtr{$T}, - ::$(typeof(op)), - x::$T, - order::AtomicOrdering, - syncscope::Val{S} = Val{:system}(), - ) where {S} - old = - syncscope isa Val{:system} ? - llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) : - llvm_atomic_op( - $(Val(llvmop)), - ptr, - x, - llvm_from_julia_ordering(order), - syncscope, - ) - return old => $op(old, x) - end - end -end - @inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) @inline atomic_pointerswap(pointer, new, order) = first(atomic_pointermodify(pointer, right, new, order)) From 95ff90377557c55b8e961a2194a9ea7236a40e41 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 6 Dec 2024 15:20:24 +0100 Subject: [PATCH 7/7] add some tests --- ext/UnsafeAtomicsLLVM/atomics.jl | 20 +++++---- src/syncscopes.jl | 2 +- .../src/UnsafeAtomicsTests.jl | 1 + test/UnsafeAtomicsTests/src/test_core.jl | 41 +++++++++++++++++++ .../UnsafeAtomicsTests/src/test_syncscopes.jl | 21 ++++++++++ 5 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 test/UnsafeAtomicsTests/src/test_syncscopes.jl diff --git a/ext/UnsafeAtomicsLLVM/atomics.jl b/ext/UnsafeAtomicsLLVM/atomics.jl index 452b8ae..cce568e 100644 --- a/ext/UnsafeAtomicsLLVM/atomics.jl +++ b/ext/UnsafeAtomicsLLVM/atomics.jl @@ -257,22 +257,23 @@ end return old => x end -@inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) -@inline atomic_pointerswap(pointer, new, order) = - first(atomic_pointermodify(pointer, right, new, order)) +# @inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) +@inline atomic_pointerswap(pointer, new, order, sync) = + first(atomic_pointermodify(pointer, right, new, order, sync)) @inline function atomic_pointermodify( ptr::LLVMPtr{T}, op, x::T, order::AllOrdering, -) where {T} + sync::S, +) where {T, S} # Should `fail_order` be stronger? Ref: https://github.com/JuliaLang/julia/issues/45256 fail_order = Val(:monotonic) - old = atomic_pointerref(ptr, fail_order) + old = atomic_pointerref(ptr, fail_order, sync) while true new = op(old, x) - (old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order) + (old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order, sync) success && return old => new end end @@ -367,10 +368,11 @@ end desired::T, ::Val{:not_atomic}, ::Val{:not_atomic}, + sync, ) where {T} - old = atomic_pointerref(ptr, Val(:not_atomic)) + old = atomic_pointerref(ptr, Val(:not_atomic), sync) if old === expected - atomic_pointerset(ptr, desired, Val(:not_atomic)) + atomic_pointerset(ptr, desired, Val(:not_atomic), sync) success = true else success = false @@ -384,10 +386,12 @@ end desired::T, success_order::_julia_ordering(∉((:not_atomic, :unordered))), fail_order::_julia_ordering(∉((:not_atomic, :unordered, :release, :acquire_release))), + sync ) where {T} = llvm_atomic_cas( ptr, expected, desired, llvm_from_julia_ordering(success_order), llvm_from_julia_ordering(fail_order), + sync ) diff --git a/src/syncscopes.jl b/src/syncscopes.jl index 6a201c6..3c4413e 100644 --- a/src/syncscopes.jl +++ b/src/syncscopes.jl @@ -13,4 +13,4 @@ Base.string(s::typeof(none)) = "" Base.print(io::IO, s::LLVMSyncScope) = print(io, string(s)) -Base.show(io::IO, o::ConcreteSyncScopes) = print(io, UnsafeAtomics, '.', llvm_ordering(o)) \ No newline at end of file +Base.show(io::IO, o::ConcreteSyncScopes) = print(io, UnsafeAtomics, '.', llvm_syncscope(o)) \ No newline at end of file diff --git a/test/UnsafeAtomicsTests/src/UnsafeAtomicsTests.jl b/test/UnsafeAtomicsTests/src/UnsafeAtomicsTests.jl index 89dd87d..e417af1 100644 --- a/test/UnsafeAtomicsTests/src/UnsafeAtomicsTests.jl +++ b/test/UnsafeAtomicsTests/src/UnsafeAtomicsTests.jl @@ -2,6 +2,7 @@ module UnsafeAtomicsTests include("bits.jl") include("test_orderings.jl") +include("test_syncscopes.jl") include("test_core.jl") end # module UnsafeAtomicsTests diff --git a/test/UnsafeAtomicsTests/src/test_core.jl b/test/UnsafeAtomicsTests/src/test_core.jl index 4d4b1cf..ffeede0 100644 --- a/test/UnsafeAtomicsTests/src/test_core.jl +++ b/test/UnsafeAtomicsTests/src/test_core.jl @@ -1,6 +1,7 @@ module TestCore using UnsafeAtomics: UnsafeAtomics, monotonic, acquire, release, acq_rel, seq_cst, right +using UnsafeAtomics: none, singlethread using UnsafeAtomics.Internal: OP_RMW_TABLE, inttypes, floattypes using Test @@ -94,4 +95,44 @@ function test_explicit_ordering(T::Type) end end +function test_explicit_syncscope(T::Type) + xs = T[rand(T), rand(T)] + x1 = rand(T) + x2 = rand(T) + @debug "xs=$(repr(xs)) x1=$(repr(x1)) x2=$(repr(x2))" + + ptr = pointer(xs, 1) + GC.@preserve xs begin + + @test UnsafeAtomics.load(ptr, acquire, none) === xs[1] + @test UnsafeAtomics.load(ptr, acquire, singlethread) === xs[1] + UnsafeAtomics.store!(ptr, x1, release, singlethread) + @test xs[1] === x1 + desired = (old = x1, success = true) + @test UnsafeAtomics.cas!(ptr, x1, x2, acq_rel, acquire, singlethread) === desired + @test xs[1] === x2 + @testset for (op, name) in rmw_table_for(T) + xs[1] = x1 + @test UnsafeAtomics.modify!(ptr, op, x2, acq_rel, singlethread) === (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + + rmw = getfield(UnsafeAtomics, Symbol(name, :!)) + xs[1] = x1 + @test rmw(ptr, x2, acquire, singlethread) === x1 + @test xs[1] === op(x1, x2) + end + end +end + +function test_explicit_syncscope() + @testset for T in [UInt, Float64] + test_explicit_syncscope(T) + end + UnsafeAtomics.fence(monotonic, none) + UnsafeAtomics.fence(acquire, singlethread) + UnsafeAtomics.fence(release, singlethread) + UnsafeAtomics.fence(acq_rel, none) + UnsafeAtomics.fence(seq_cst, none) +end + end # module diff --git a/test/UnsafeAtomicsTests/src/test_syncscopes.jl b/test/UnsafeAtomicsTests/src/test_syncscopes.jl new file mode 100644 index 0000000..55279d3 --- /dev/null +++ b/test/UnsafeAtomicsTests/src/test_syncscopes.jl @@ -0,0 +1,21 @@ +module TestSyncScopes + +using UnsafeAtomics: UnsafeAtomics, SyncScope +using Test + +const SYNCSCOPES = [:none, :singlethread] + +function check_syncscope(name::Symbol) + sync = getfield(UnsafeAtomics, name) + @test sync isa SyncScope + @test string(sync) == sprint(print, sync) == string(sync) + @test repr(sync) == "UnsafeAtomics.$name" +end + +function test_syncscope() + @testset for name in SYNCSCOPES + check_syncscope(name) + end +end + +end # module \ No newline at end of file