Skip to content

Commit 828833a

Browse files
vchuravyvchuravy
committed
implement syncscopes everywhere
1 parent b099010 commit 828833a

File tree

3 files changed

+182
-124
lines changed

3 files changed

+182
-124
lines changed

‎src/UnsafeAtomics.jl

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
baremodule UnsafeAtomics
22

33
abstract type Ordering end
4+
abstract type SyncScope end
45

56
function load end
67
function store! end
@@ -26,10 +27,11 @@ module Internal
2627
using Base.Sys: WORD_SIZE
2728
using Base: bitcast, llvmcall
2829

29-
using ..UnsafeAtomics: UnsafeAtomics, Ordering, right
30+
using ..UnsafeAtomics: UnsafeAtomics, Ordering, SyncScope, right
3031

3132
include("utils.jl")
3233
include("orderings.jl")
34+
include("syncscopes.jl")
3335
include("core.jl")
3436

3537
end # module Internal
@@ -45,4 +47,8 @@ const seq_cst = Internal.seq_cst
4547
const acquire_release = acq_rel
4648
const sequentially_consistent = seq_cst
4749

50+
# SyncScope
51+
const none = Internal.none
52+
const singlethread = Internal.singlethread
53+
4854
end # baremodule UnsafeAtomics

‎src/core.jl

Lines changed: 159 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@
44
@inline UnsafeAtomics.modify!(ptr, op, x) = UnsafeAtomics.modify!(ptr, op, x, seq_cst)
55
@inline UnsafeAtomics.fence() = UnsafeAtomics.fence(seq_cst)
66

7+
@inline UnsafeAtomics.load(x, ord) = UnsafeAtomics.load(x, ord, none)
8+
@inline UnsafeAtomics.store!(x, v, ord) = UnsafeAtomics.store!(x, v, ord, none)
9+
@inline UnsafeAtomics.cas!(x, cmp, new, ord) = UnsafeAtomics.cas!(x, cmp, new, ord, ord, none)
10+
@inline UnsafeAtomics.modify!(ptr, op, x, ord) = UnsafeAtomics.modify!(ptr, op, x, ord, none)
11+
@inline UnsafeAtomics.fence(ord) = UnsafeAtomics.fence(ord, none)
12+
713
#! format: off
814
# https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl#L23-L30
915
if Sys.ARCH == :i686 || startswith(string(Sys.ARCH), "arm") ||
@@ -45,8 +51,9 @@ const OP_RMW_TABLE = [
4551
for (op, rmwop) in OP_RMW_TABLE
4652
fn = Symbol(rmwop, "!")
4753
@eval @inline UnsafeAtomics.$fn(x, v) = UnsafeAtomics.$fn(x, v, seq_cst)
48-
@eval @inline UnsafeAtomics.$fn(ptr, x, ord) =
49-
first(UnsafeAtomics.modify!(ptr, $op, x, ord))
54+
@eval @inline UnsafeAtomics.$fn(x, v, ord) = UnsafeAtomics.$fn(x, v, ord, none)
55+
@eval @inline UnsafeAtomics.$fn(ptr, x, ord, scope) =
56+
first(UnsafeAtomics.modify!(ptr, $op, x, ord, scope))
5057
end
5158

5259
const ATOMIC_INTRINSICS = isdefined(Core.Intrinsics, :atomic_pointerref)
@@ -67,47 +74,51 @@ for typ in (inttypes..., floattypes...)
6774
for ord in orderings
6875
ord in (release, acq_rel) && continue
6976

70-
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
71-
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
72-
return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord))
73-
end
74-
else
75-
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
76-
return llvmcall(
77-
$("""
78-
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
79-
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
80-
ret $lt %rv
81-
"""),
82-
$typ,
83-
Tuple{Ptr{$typ}},
84-
x,
85-
)
77+
for sync in syncscopes
78+
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
79+
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync)))
80+
return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord))
81+
end
82+
else
83+
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync)))
84+
return llvmcall(
85+
$("""
86+
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
87+
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
88+
ret $lt %rv
89+
"""),
90+
$typ,
91+
Tuple{Ptr{$typ}},
92+
x,
93+
)
94+
end
8695
end
8796
end
8897
end
8998

9099
for ord in orderings
91100
ord in (acquire, acq_rel) && continue
92-
93-
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
94-
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
95-
Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord))
96-
return nothing
97-
end
98-
else
99-
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
100-
return llvmcall(
101-
$("""
102-
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
103-
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
104-
ret void
105-
"""),
106-
Cvoid,
107-
Tuple{Ptr{$typ},$typ},
108-
x,
109-
v,
110-
)
101+
102+
for sync in syncscopes
103+
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
104+
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync)))
105+
Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord))
106+
return nothing
107+
end
108+
else
109+
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync)))
110+
return llvmcall(
111+
$("""
112+
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
113+
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
114+
ret void
115+
"""),
116+
Cvoid,
117+
Tuple{Ptr{$typ},$typ},
118+
x,
119+
v,
120+
)
121+
end
111122
end
112123
end
113124
end
@@ -117,54 +128,58 @@ for typ in (inttypes..., floattypes...)
117128

118129
typ <: AbstractFloat && break
119130

120-
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
121-
@eval function UnsafeAtomics.cas!(
122-
x::Ptr{$typ},
123-
cmp::$typ,
124-
new::$typ,
125-
::$(typeof(success_ordering)),
126-
::$(typeof(failure_ordering)),
127-
)
128-
return Core.Intrinsics.atomic_pointerreplace(
129-
x,
130-
cmp,
131-
new,
132-
base_ordering($success_ordering),
133-
base_ordering($failure_ordering)
131+
for sync in syncscopes
132+
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
133+
@eval function UnsafeAtomics.cas!(
134+
x::Ptr{$typ},
135+
cmp::$typ,
136+
new::$typ,
137+
::$(typeof(success_ordering)),
138+
::$(typeof(failure_ordering)),
139+
::$(typeof(sync)),
134140
)
135-
end
136-
else
137-
@eval function UnsafeAtomics.cas!(
138-
x::Ptr{$typ},
139-
cmp::$typ,
140-
new::$typ,
141-
::$(typeof(success_ordering)),
142-
::$(typeof(failure_ordering)),
143-
)
144-
success = Ref{Int8}()
145-
GC.@preserve success begin
146-
old = llvmcall(
147-
$(
148-
"""
149-
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
150-
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
151-
%rv = extractvalue { $lt, i1 } %rs, 0
152-
%s1 = extractvalue { $lt, i1 } %rs, 1
153-
%s8 = zext i1 %s1 to i8
154-
%sptr = inttoptr i$WORD_SIZE %3 to i8*
155-
store i8 %s8, i8* %sptr
156-
ret $lt %rv
157-
"""
158-
),
159-
$typ,
160-
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
141+
return Core.Intrinsics.atomic_pointerreplace(
161142
x,
162143
cmp,
163144
new,
164-
Ptr{Int8}(pointer_from_objref(success)),
145+
base_ordering($success_ordering),
146+
base_ordering($failure_ordering)
165147
)
166148
end
167-
return (old = old, success = !iszero(success[]))
149+
else
150+
@eval function UnsafeAtomics.cas!(
151+
x::Ptr{$typ},
152+
cmp::$typ,
153+
new::$typ,
154+
::$(typeof(success_ordering)),
155+
::$(typeof(failure_ordering)),
156+
::$(typeof(sync)),
157+
)
158+
success = Ref{Int8}()
159+
GC.@preserve success begin
160+
old = llvmcall(
161+
$(
162+
"""
163+
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
164+
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
165+
%rv = extractvalue { $lt, i1 } %rs, 0
166+
%s1 = extractvalue { $lt, i1 } %rs, 1
167+
%s8 = zext i1 %s1 to i8
168+
%sptr = inttoptr i$WORD_SIZE %3 to i8*
169+
store i8 %s8, i8* %sptr
170+
ret $lt %rv
171+
"""
172+
),
173+
$typ,
174+
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
175+
x,
176+
cmp,
177+
new,
178+
Ptr{Int8}(pointer_from_objref(success)),
179+
)
180+
end
181+
return (old = old, success = !iszero(success[]))
182+
end
168183
end
169184
end
170185
end
@@ -186,60 +201,81 @@ for typ in (inttypes..., floattypes...)
186201
end
187202
end
188203
for ord in orderings
189-
# Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged
190-
if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
191-
@eval function UnsafeAtomics.modify!(
204+
for sync in syncscopes
205+
# Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged
206+
if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
207+
@eval function UnsafeAtomics.modify!(
208+
x::Ptr{$typ},
209+
op::typeof($op),
210+
v::$typ,
211+
::$(typeof(ord)),
212+
::$(typeof(sync)),
213+
)
214+
return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord))
215+
end
216+
else
217+
@eval function UnsafeAtomics.modify!(
192218
x::Ptr{$typ},
193-
op::typeof($op),
219+
::typeof($op),
194220
v::$typ,
195221
::$(typeof(ord)),
222+
::$(typeof(sync)),
196223
)
197-
return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord))
198-
end
199-
else
200-
@eval function UnsafeAtomics.modify!(
201-
x::Ptr{$typ},
202-
::typeof($op),
203-
v::$typ,
204-
::$(typeof(ord)),
205-
)
206-
old = llvmcall(
207-
$("""
208-
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
209-
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
210-
ret $lt %rv
211-
"""),
212-
$typ,
213-
Tuple{Ptr{$typ},$typ},
214-
x,
215-
v,
216-
)
217-
return old => $op(old, v)
224+
old = llvmcall(
225+
$("""
226+
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
227+
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
228+
ret $lt %rv
229+
"""),
230+
$typ,
231+
Tuple{Ptr{$typ},$typ},
232+
x,
233+
v,
234+
)
235+
return old => $op(old, v)
236+
end
218237
end
219238
end
220239
end
221240
end
222241
end
223242

224-
# Core.Intrinsics.atomic_fence was introduced in 1.10
225-
function UnsafeAtomics.fence(ord::Ordering)
226-
Core.Intrinsics.atomic_fence(base_ordering(ord))
227-
return nothing
228-
end
229-
if Sys.ARCH == :x86_64
230-
# FIXME: Disable this once on LLVM 19
231-
# This is unfortunatly required for good-performance on AMD
232-
# https://github.com/llvm/llvm-project/pull/106555
233-
function UnsafeAtomics.fence(::typeof(seq_cst))
234-
Base.llvmcall(
235-
(raw"""
236-
define void @fence() #0 {
237-
entry:
238-
tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory}
239-
ret void
240-
}
241-
attributes #0 = { alwaysinline }
242-
""", "fence"), Nothing, Tuple{})
243+
for sync in syncscopes
244+
if sync == none
245+
# Core.Intrinsics.atomic_fence was introduced in 1.10
246+
@eval function UnsafeAtomics.fence(ord::Ordering, ::$(typeof(sync)))
247+
Core.Intrinsics.atomic_fence(base_ordering(ord))
248+
return nothing
249+
end
250+
if Sys.ARCH == :x86_64
251+
# FIXME: Disable this once on LLVM 19
252+
# This is unfortunatly required for good-performance on AMD
253+
# https://github.com/llvm/llvm-project/pull/106555
254+
@eval function UnsafeAtomics.fence(::typeof(seq_cst), ::$(typeof(sync)))
255+
Base.llvmcall(
256+
(raw"""
257+
define void @fence() #0 {
258+
entry:
259+
tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory}
260+
ret void
261+
}
262+
attributes #0 = { alwaysinline }
263+
""", "fence"), Nothing, Tuple{})
264+
end
265+
end
266+
else
267+
for ord in orderings
268+
@eval function UnsafeAtomics.fence(::$(typeof(ord)), ::$(typeof(sync)))
269+
return llvmcall(
270+
$("""
271+
fence $sync $ord
272+
ret void
273+
"""),
274+
Cvoid,
275+
Tuple{},
276+
)
277+
end
278+
end
243279
end
244280
end
245281

‎src/syncscopes.jl

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
struct LLVMSyncScope{name} <: SyncScope end
2+
3+
const none = LLVMSyncScope{:none}()
4+
const singlethread = LLVMSyncScope{:singlethread}()
5+
6+
const syncscopes = (none, singlethread)
7+
const ConcreteSyncScopes = Union{map(typeof, syncscopes)...}
8+
9+
llvm_syncscope(::LLVMSyncScope{name}) where {name} = name
10+
11+
Base.string(s::LLVMSyncScope) = string("syncscope(\"", llvm_syncscope(s), "\")")
12+
Base.string(s::typeof(none)) = ""
13+
14+
Base.print(io::IO, s::LLVMSyncScope) = print(io, string(s))
15+
16+
Base.show(io::IO, o::ConcreteSyncScopes) = print(io, UnsafeAtomics, '.', llvm_ordering(o))

0 commit comments

Comments
 (0)