Skip to content

Commit ce7a883

Browse files
jpsamaroovchuravy
jpsamaroo
and
vchuravy
authored
Use atomic_pointer* methods when available (#14)
Co-authored-by: Valentin Churavy <[email protected]>
1 parent 2eaec48 commit ce7a883

File tree

2 files changed

+124
-66
lines changed

2 files changed

+124
-66
lines changed

‎src/core.jl

Lines changed: 119 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,16 @@ for (op, rmwop) in OP_RMW_TABLE
4848
first(UnsafeAtomics.modify!(ptr, $op, x, ord))
4949
end
5050

51+
const ATOMIC_INTRINSICS = isdefined(Core.Intrinsics, :atomic_pointerref)
52+
53+
if VERSION >= v"1.12.0-DEV.161" && Int == Int64
54+
const MAX_ATOMIC_SIZE = 16
55+
const MAX_POINTERATOMIC_SIZE = 16
56+
else
57+
const MAX_ATOMIC_SIZE = 8
58+
const MAX_POINTERATOMIC_SIZE = 8
59+
end
60+
5161
# Based on: https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl
5262
for typ in (inttypes..., floattypes...)
5363
lt = llvmtypes[typ]
@@ -56,35 +66,48 @@ for typ in (inttypes..., floattypes...)
5666
for ord in orderings
5767
ord in (release, acq_rel) && continue
5868

59-
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
60-
return llvmcall(
61-
$("""
62-
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
63-
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
64-
ret $lt %rv
65-
"""),
66-
$typ,
67-
Tuple{Ptr{$typ}},
68-
x,
69-
)
69+
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
70+
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
71+
return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord))
72+
end
73+
else
74+
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
75+
return llvmcall(
76+
$("""
77+
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
78+
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
79+
ret $lt %rv
80+
"""),
81+
$typ,
82+
Tuple{Ptr{$typ}},
83+
x,
84+
)
85+
end
7086
end
7187
end
7288

7389
for ord in orderings
7490
ord in (acquire, acq_rel) && continue
7591

76-
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
77-
return llvmcall(
78-
$("""
79-
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
80-
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
81-
ret void
82-
"""),
83-
Cvoid,
84-
Tuple{Ptr{$typ},$typ},
85-
x,
86-
v,
87-
)
92+
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
93+
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
94+
Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord))
95+
return nothing
96+
end
97+
else
98+
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
99+
return llvmcall(
100+
$("""
101+
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
102+
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
103+
ret void
104+
"""),
105+
Cvoid,
106+
Tuple{Ptr{$typ},$typ},
107+
x,
108+
v,
109+
)
110+
end
88111
end
89112
end
90113

@@ -93,37 +116,55 @@ for typ in (inttypes..., floattypes...)
93116

94117
typ <: AbstractFloat && break
95118

96-
@eval function UnsafeAtomics.cas!(
97-
x::Ptr{$typ},
98-
cmp::$typ,
99-
new::$typ,
100-
::$(typeof(success_ordering)),
101-
::$(typeof(failure_ordering)),
102-
)
103-
success = Ref{Int8}()
104-
GC.@preserve success begin
105-
old = llvmcall(
106-
$(
107-
"""
108-
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
109-
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
110-
%rv = extractvalue { $lt, i1 } %rs, 0
111-
%s1 = extractvalue { $lt, i1 } %rs, 1
112-
%s8 = zext i1 %s1 to i8
113-
%sptr = inttoptr i$WORD_SIZE %3 to i8*
114-
store i8 %s8, i8* %sptr
115-
ret $lt %rv
116-
"""
117-
),
118-
$typ,
119-
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
119+
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
120+
@eval function UnsafeAtomics.cas!(
121+
x::Ptr{$typ},
122+
cmp::$typ,
123+
new::$typ,
124+
::$(typeof(success_ordering)),
125+
::$(typeof(failure_ordering)),
126+
)
127+
return Core.Intrinsics.atomic_pointerreplace(
120128
x,
121129
cmp,
122130
new,
123-
Ptr{Int8}(pointer_from_objref(success)),
131+
base_ordering($success_ordering),
132+
base_ordering($failure_ordering)
124133
)
125134
end
126-
return (old = old, success = !iszero(success[]))
135+
else
136+
@eval function UnsafeAtomics.cas!(
137+
x::Ptr{$typ},
138+
cmp::$typ,
139+
new::$typ,
140+
::$(typeof(success_ordering)),
141+
::$(typeof(failure_ordering)),
142+
)
143+
success = Ref{Int8}()
144+
GC.@preserve success begin
145+
old = llvmcall(
146+
$(
147+
"""
148+
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
149+
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
150+
%rv = extractvalue { $lt, i1 } %rs, 0
151+
%s1 = extractvalue { $lt, i1 } %rs, 1
152+
%s8 = zext i1 %s1 to i8
153+
%sptr = inttoptr i$WORD_SIZE %3 to i8*
154+
store i8 %s8, i8* %sptr
155+
ret $lt %rv
156+
"""
157+
),
158+
$typ,
159+
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
160+
x,
161+
cmp,
162+
new,
163+
Ptr{Int8}(pointer_from_objref(success)),
164+
)
165+
end
166+
return (old = old, success = !iszero(success[]))
167+
end
127168
end
128169
end
129170

@@ -144,24 +185,36 @@ for typ in (inttypes..., floattypes...)
144185
end
145186
end
146187
for ord in orderings
147-
@eval function UnsafeAtomics.modify!(
148-
x::Ptr{$typ},
149-
::typeof($op),
150-
v::$typ,
151-
::$(typeof(ord)),
152-
)
153-
old = llvmcall(
154-
$("""
155-
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
156-
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
157-
ret $lt %rv
158-
"""),
159-
$typ,
160-
Tuple{Ptr{$typ},$typ},
161-
x,
162-
v,
188+
# Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged
189+
if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
190+
@eval function UnsafeAtomics.modify!(
191+
x::Ptr{$typ},
192+
op::typeof($op),
193+
v::$typ,
194+
::$(typeof(ord)),
195+
)
196+
return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord))
197+
end
198+
else
199+
@eval function UnsafeAtomics.modify!(
200+
x::Ptr{$typ},
201+
::typeof($op),
202+
v::$typ,
203+
::$(typeof(ord)),
163204
)
164-
return old => $op(old, v)
205+
old = llvmcall(
206+
$("""
207+
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
208+
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
209+
ret $lt %rv
210+
"""),
211+
$typ,
212+
Tuple{Ptr{$typ},$typ},
213+
x,
214+
v,
215+
)
216+
return old => $op(old, v)
217+
end
165218
end
166219
end
167220
end

‎src/orderings.jl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@ const orderings = (unordered, monotonic, acquire, release, acq_rel, seq_cst)
1212
const ConcreteOrdering = Union{map(typeof, orderings)...}
1313

1414
llvm_ordering(::LLVMOrdering{name}) where {name} = name
15+
1516
Base.string(o::LLVMOrdering) = String(llvm_ordering(o))
1617
Base.print(io::IO, o::LLVMOrdering) = print(io, string(o))
1718

1819
Base.show(io::IO, o::ConcreteOrdering) = print(io, UnsafeAtomics, '.', llvm_ordering(o))
20+
21+
base_ordering(::LLVMOrdering{name}) where {name} = name
22+
base_ordering(::LLVMOrdering{:seq_cst}) = :sequentially_consistent
23+
base_ordering(::LLVMOrdering{:acq_rel}) = :acquire_release

0 commit comments

Comments
 (0)