Skip to content

Commit ac74d6a

Browse files
Renaud-KRenaud-K
committed
Adding support for more atomic calls
1 parent 754b946 commit ac74d6a

File tree

4 files changed

+248
-4
lines changed

4 files changed

+248
-4
lines changed

‎flang/include/flang/Optimizer/Builder/IntrinsicCall.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,13 @@ struct IntrinsicLibrary {
186186
fir::ExtendedValue genAny(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
187187
mlir::Value genAtanpi(mlir::Type, llvm::ArrayRef<mlir::Value>);
188188
mlir::Value genAtomicAdd(mlir::Type, llvm::ArrayRef<mlir::Value>);
189+
mlir::Value genAtomicAnd(mlir::Type, llvm::ArrayRef<mlir::Value>);
190+
mlir::Value genAtomicOr(mlir::Type, llvm::ArrayRef<mlir::Value>);
191+
mlir::Value genAtomicDec(mlir::Type, llvm::ArrayRef<mlir::Value>);
192+
mlir::Value genAtomicInc(mlir::Type, llvm::ArrayRef<mlir::Value>);
193+
mlir::Value genAtomicMax(mlir::Type, llvm::ArrayRef<mlir::Value>);
194+
mlir::Value genAtomicMin(mlir::Type, llvm::ArrayRef<mlir::Value>);
195+
mlir::Value genAtomicSub(mlir::Type, llvm::ArrayRef<mlir::Value>);
189196
fir::ExtendedValue
190197
genCommandArgumentCount(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
191198
mlir::Value genAsind(mlir::Type, llvm::ArrayRef<mlir::Value>);

‎flang/lib/Optimizer/Builder/IntrinsicCall.cpp‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,15 @@
4646
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
4747
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
4848
#include "mlir/Dialect/Math/IR/Math.h"
49+
#include "mlir/Dialect/MemRef/IR/MemRef.h"
4950
#include "mlir/Dialect/Vector/IR/VectorOps.h"
5051
#include "llvm/Support/CommandLine.h"
5152
#include "llvm/Support/Debug.h"
5253
#include "llvm/Support/MathExtras.h"
5354
#include "llvm/Support/raw_ostream.h"
5455
#include <cfenv> // temporary -- only used in genIeeeGetOrSetModesOrStatus
56+
#include <mlir/IR/BuiltinTypes.h>
57+
#include <mlir/IR/Types.h>
5558
#include <optional>
5659

5760
#define DEBUG_TYPE "flang-lower-intrinsic"
@@ -151,6 +154,22 @@ static constexpr IntrinsicHandler handlers[]{
151154
{"atomicaddf", &I::genAtomicAdd, {{{"a", asAddr}, {"v", asValue}}}, false},
152155
{"atomicaddi", &I::genAtomicAdd, {{{"a", asAddr}, {"v", asValue}}}, false},
153156
{"atomicaddl", &I::genAtomicAdd, {{{"a", asAddr}, {"v", asValue}}}, false},
157+
{"atomicandi", &I::genAtomicAnd, {{{"a", asAddr}, {"v", asValue}}}, false},
158+
{"atomicdeci", &I::genAtomicDec, {{{"a", asAddr}, {"v", asValue}}}, false},
159+
{"atomicinci", &I::genAtomicInc, {{{"a", asAddr}, {"v", asValue}}}, false},
160+
{"atomicmaxd", &I::genAtomicMax, {{{"a", asAddr}, {"v", asValue}}}, false},
161+
{"atomicmaxf", &I::genAtomicMax, {{{"a", asAddr}, {"v", asValue}}}, false},
162+
{"atomicmaxi", &I::genAtomicMax, {{{"a", asAddr}, {"v", asValue}}}, false},
163+
{"atomicmaxl", &I::genAtomicMax, {{{"a", asAddr}, {"v", asValue}}}, false},
164+
{"atomicmind", &I::genAtomicMin, {{{"a", asAddr}, {"v", asValue}}}, false},
165+
{"atomicminf", &I::genAtomicMin, {{{"a", asAddr}, {"v", asValue}}}, false},
166+
{"atomicmini", &I::genAtomicMin, {{{"a", asAddr}, {"v", asValue}}}, false},
167+
{"atomicminl", &I::genAtomicMin, {{{"a", asAddr}, {"v", asValue}}}, false},
168+
{"atomicori", &I::genAtomicOr, {{{"a", asAddr}, {"v", asValue}}}, false},
169+
{"atomicsubd", &I::genAtomicSub, {{{"a", asAddr}, {"v", asValue}}}, false},
170+
{"atomicsubf", &I::genAtomicSub, {{{"a", asAddr}, {"v", asValue}}}, false},
171+
{"atomicsubi", &I::genAtomicSub, {{{"a", asAddr}, {"v", asValue}}}, false},
172+
{"atomicsubl", &I::genAtomicSub, {{{"a", asAddr}, {"v", asValue}}}, false},
154173
{"bessel_jn",
155174
&I::genBesselJn,
156175
{{{"n1", asValue}, {"n2", asValue}, {"x", asValue}}},
@@ -2600,6 +2619,75 @@ mlir::Value IntrinsicLibrary::genAtomicAdd(mlir::Type resultType,
26002619
return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
26012620
}
26022621

2622+
mlir::Value IntrinsicLibrary::genAtomicSub(mlir::Type resultType,
2623+
llvm::ArrayRef<mlir::Value> args) {
2624+
assert(args.size() == 2);
2625+
2626+
mlir::LLVM::AtomicBinOp binOp =
2627+
mlir::isa<mlir::IntegerType>(args[1].getType())
2628+
? mlir::LLVM::AtomicBinOp::sub
2629+
: mlir::LLVM::AtomicBinOp::fsub;
2630+
return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
2631+
}
2632+
2633+
mlir::Value IntrinsicLibrary::genAtomicAnd(mlir::Type resultType,
2634+
llvm::ArrayRef<mlir::Value> args) {
2635+
assert(args.size() == 2);
2636+
assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
2637+
2638+
mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::_and;
2639+
return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
2640+
}
2641+
2642+
mlir::Value IntrinsicLibrary::genAtomicOr(mlir::Type resultType,
2643+
llvm::ArrayRef<mlir::Value> args) {
2644+
assert(args.size() == 2);
2645+
assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
2646+
2647+
mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::_or;
2648+
return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
2649+
}
2650+
2651+
mlir::Value IntrinsicLibrary::genAtomicDec(mlir::Type resultType,
2652+
llvm::ArrayRef<mlir::Value> args) {
2653+
assert(args.size() == 2);
2654+
assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
2655+
2656+
mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::udec_wrap;
2657+
return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
2658+
}
2659+
2660+
mlir::Value IntrinsicLibrary::genAtomicInc(mlir::Type resultType,
2661+
llvm::ArrayRef<mlir::Value> args) {
2662+
assert(args.size() == 2);
2663+
assert(mlir::isa<mlir::IntegerType>(args[1].getType()));
2664+
2665+
mlir::LLVM::AtomicBinOp binOp = mlir::LLVM::AtomicBinOp::uinc_wrap;
2666+
return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
2667+
}
2668+
2669+
mlir::Value IntrinsicLibrary::genAtomicMax(mlir::Type resultType,
2670+
llvm::ArrayRef<mlir::Value> args) {
2671+
assert(args.size() == 2);
2672+
2673+
mlir::LLVM::AtomicBinOp binOp =
2674+
mlir::isa<mlir::IntegerType>(args[1].getType())
2675+
? mlir::LLVM::AtomicBinOp::max
2676+
: mlir::LLVM::AtomicBinOp::fmax;
2677+
return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
2678+
}
2679+
2680+
mlir::Value IntrinsicLibrary::genAtomicMin(mlir::Type resultType,
2681+
llvm::ArrayRef<mlir::Value> args) {
2682+
assert(args.size() == 2);
2683+
2684+
mlir::LLVM::AtomicBinOp binOp =
2685+
mlir::isa<mlir::IntegerType>(args[1].getType())
2686+
? mlir::LLVM::AtomicBinOp::min
2687+
: mlir::LLVM::AtomicBinOp::fmin;
2688+
return genAtomBinOp(builder, loc, binOp, args[0], args[1]);
2689+
}
2690+
26032691
// ASSOCIATED
26042692
fir::ExtendedValue
26052693
IntrinsicLibrary::genAssociated(mlir::Type resultType,

‎flang/module/cudadevice.f90‎

Lines changed: 113 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,17 +106,126 @@ attributes(device) pure real function atomicaddf(address, val)
106106
real, intent(inout) :: address
107107
real, value :: val
108108
end function
109-
attributes(device) pure real*8 function atomicaddd(address, val)
109+
attributes(device) pure real(8) function atomicaddd(address, val)
110110
!dir$ ignore_tkr (d) address, (d) val
111-
real*8, intent(inout) :: address
112-
real*8, value :: val
111+
real(8), intent(inout) :: address
112+
real(8), value :: val
113113
end function
114114
attributes(device) pure integer(8) function atomicaddl(address, val)
115115
!dir$ ignore_tkr (d) address, (d) val
116116
integer(8), intent(inout) :: address
117117
integer(8), value :: val
118118
end function
119119
end interface
120-
public :: atomicadd
120+
public :: atomicadd
121+
122+
interface atomicsub
123+
attributes(device) pure integer function atomicsubi(address, val)
124+
!dir$ ignore_tkr (d) address, (d) val
125+
integer, intent(inout) :: address
126+
integer, value :: val
127+
end function
128+
attributes(device) pure real function atomicsubf(address, val)
129+
!dir$ ignore_tkr (d) address, (d) val
130+
real, intent(inout) :: address
131+
real, value :: val
132+
end function
133+
attributes(device) pure real(8) function atomicsubd(address, val)
134+
!dir$ ignore_tkr (d) address, (d) val
135+
real(8), intent(inout) :: address
136+
real(8), value :: val
137+
end function
138+
attributes(device) pure integer(8) function atomicsubl(address, val)
139+
!dir$ ignore_tkr (d) address, (dk) val
140+
integer(8), intent(inout) :: address
141+
integer(8), value :: val
142+
end function
143+
end interface
144+
public :: atomicsub
145+
146+
interface atomicmax
147+
attributes(device) pure integer function atomicmaxi(address, val)
148+
!dir$ ignore_tkr (d) address, (d) val
149+
integer, intent(inout) :: address
150+
integer, value :: val
151+
end function
152+
attributes(device) pure real function atomicmaxf(address, val)
153+
!dir$ ignore_tkr (d) address, (d) val
154+
real, intent(inout) :: address
155+
real, value :: val
156+
end function
157+
attributes(device) pure real(8) function atomicmaxd(address, val)
158+
!dir$ ignore_tkr (d) address, (d) val
159+
real(8), intent(inout) :: address
160+
real(8), value :: val
161+
end function
162+
attributes(device) pure integer(8) function atomicmaxl(address, val)
163+
!dir$ ignore_tkr (d) address, (dk) val
164+
integer(8), intent(inout) :: address
165+
integer(8), value :: val
166+
end function
167+
end interface
168+
public :: atomicmax
169+
170+
interface atomicmin
171+
attributes(device) pure integer function atomicmini(address, val)
172+
!dir$ ignore_tkr (d) address, (d) val
173+
integer, intent(inout) :: address
174+
integer, value :: val
175+
end function
176+
attributes(device) pure real function atomicminf(address, val)
177+
!dir$ ignore_tkr (d) address, (d) val
178+
real, intent(inout) :: address
179+
real, value :: val
180+
end function
181+
attributes(device) pure real(8) function atomicmind(address, val)
182+
!dir$ ignore_tkr (d) address, (d) val
183+
real(8), intent(inout) :: address
184+
real(8), value :: val
185+
end function
186+
attributes(device) pure integer(8) function atomicminl(address, val)
187+
!dir$ ignore_tkr (d) address, (dk) val
188+
integer(8), intent(inout) :: address
189+
integer(8), value :: val
190+
end function
191+
end interface
192+
public :: atomicmin
193+
194+
interface atomicand
195+
attributes(device) pure integer function atomicandi(address, val)
196+
!dir$ ignore_tkr (d) address, (d) val
197+
integer, intent(inout) :: address
198+
integer, value :: val
199+
end function
200+
end interface
201+
public :: atomicand
202+
203+
interface atomicor
204+
attributes(device) pure integer function atomicori(address, val)
205+
!dir$ ignore_tkr (d) address, (d) val
206+
integer, intent(inout) :: address
207+
integer, value :: val
208+
end function
209+
end interface
210+
public :: atomicor
211+
212+
interface atomicinc
213+
attributes(device) pure integer function atomicinci(address, val)
214+
!dir$ ignore_tkr (d) address, (d) val
215+
integer, intent(inout) :: address
216+
integer, value :: val
217+
end function
218+
end interface
219+
public :: atomicinc
220+
221+
interface atomicdec
222+
attributes(device) pure integer function atomicdeci(address, val)
223+
!dir$ ignore_tkr (d) address, (d) val
224+
integer, intent(inout) :: address
225+
integer, value :: val
226+
end function
227+
end interface
228+
public :: atomicdec
229+
121230

122231
end module

‎flang/test/Lower/CUDA/cuda-device-proc.cuf‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,26 @@ attributes(global) subroutine devsub()
2323
al = atomicadd(al, 1_8)
2424
af = atomicadd(af, 1.0_4)
2525
ad = atomicadd(ad, 1.0_8)
26+
27+
ai = atomicsub(ai, 1_4)
28+
al = atomicsub(al, 1_8)
29+
af = atomicsub(af, 1.0_4)
30+
ad = atomicsub(ad, 1.0_8)
31+
32+
ai = atomicmax(ai, 1_4)
33+
al = atomicmax(al, 1_8)
34+
af = atomicmax(af, 1.0_4)
35+
ad = atomicmax(ad, 1.0_8)
36+
37+
ai = atomicmin(ai, 1_4)
38+
al = atomicmin(al, 1_8)
39+
af = atomicmin(af, 1.0_4)
40+
ad = atomicmin(ad, 1.0_8)
41+
42+
ai = atomicand(ai, 1_4)
43+
ai = atomicor(ai, 1_4)
44+
ai = atomicinc(ai, 1_4)
45+
ai = atomicdec(ai, 1_4)
2646
end
2747

2848
! CHECK-LABEL: func.func @_QPdevsub() attributes {cuf.proc_attr = #cuf.cuda_proc<global>}
@@ -39,6 +59,26 @@ end
3959
! CHECK: %{{.*}} = llvm.atomicrmw fadd %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
4060
! CHECK: %{{.*}} = llvm.atomicrmw fadd %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f64
4161

62+
! CHECK: %{{.*}} = llvm.atomicrmw sub %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
63+
! CHECK: %{{.*}} = llvm.atomicrmw sub %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i64
64+
! CHECK: %{{.*}} = llvm.atomicrmw fsub %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
65+
! CHECK: %{{.*}} = llvm.atomicrmw fsub %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f64
66+
67+
! CHECK: %{{.*}} = llvm.atomicrmw max %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
68+
! CHECK: %{{.*}} = llvm.atomicrmw max %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i64
69+
! CHECK: %{{.*}} = llvm.atomicrmw fmax %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
70+
! CHECK: %{{.*}} = llvm.atomicrmw fmax %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f64
71+
72+
! CHECK: %{{.*}} = llvm.atomicrmw min %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
73+
! CHECK: %{{.*}} = llvm.atomicrmw min %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i64
74+
! CHECK: %{{.*}} = llvm.atomicrmw fmin %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f32
75+
! CHECK: %{{.*}} = llvm.atomicrmw fmin %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, f64
76+
77+
! CHECK: %{{.*}} = llvm.atomicrmw _and %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
78+
! CHECK: %{{.*}} = llvm.atomicrmw _or %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
79+
! CHECK: %{{.*}} = llvm.atomicrmw uinc_wrap %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
80+
! CHECK: %{{.*}} = llvm.atomicrmw udec_wrap %{{.*}}, %{{.*}} seq_cst : !llvm.ptr, i32
81+
4282
! CHECK: func.func private @llvm.nvvm.barrier0()
4383
! CHECK: func.func private @__syncwarp(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncwarp", fir.proc_attrs = #fir.proc_attrs<bind_c>}
4484
! CHECK: func.func private @llvm.nvvm.membar.gl()

0 commit comments

Comments
 (0)