Skip to content

Commit 0452807

Browse files
arsenmarsenm
committed
AMDGPU: Remove flat/global atomic fadd v2bf16 intrinsics
These are now fully covered by atomicrmw.
1 parent 7e0bed6 commit 0452807

File tree

11 files changed

+33
-538
lines changed

11 files changed

+33
-538
lines changed

‎llvm/include/llvm/IR/IntrinsicsAMDGPU.td‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2990,10 +2990,6 @@ multiclass AMDGPUMFp8SmfmacIntrinsic<LLVMType DestTy> {
29902990
def NAME#"_"#kind : AMDGPUMFp8SmfmacIntrinsic<DestTy>;
29912991
}
29922992

2993-
// bf16 atomics use v2i16 argument since there is no bf16 data type in the llvm.
2994-
def int_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUAtomicRtn<llvm_v2i16_ty>;
2995-
def int_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUAtomicRtn<llvm_v2i16_ty>;
2996-
29972993
defset list<Intrinsic> AMDGPUMFMAIntrinsics940 = {
29982994
def int_amdgcn_mfma_i32_16x16x32_i8 : AMDGPUMfmaIntrinsic<llvm_v4i32_ty, llvm_i64_ty>;
29992995
def int_amdgcn_mfma_i32_32x32x16_i8 : AMDGPUMfmaIntrinsic<llvm_v16i32_ty, llvm_i64_ty>;

‎llvm/lib/IR/AutoUpgrade.cpp‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1034,7 +1034,9 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
10341034
}
10351035

10361036
if (Name.starts_with("ds.fadd") || Name.starts_with("ds.fmin") ||
1037-
Name.starts_with("ds.fmax")) {
1037+
Name.starts_with("ds.fmax") ||
1038+
Name.starts_with("global.atomic.fadd.v2bf16") ||
1039+
Name.starts_with("flat.atomic.fadd.v2bf16")) {
10381040
// Replaced with atomicrmw fadd/fmin/fmax, so there's no new
10391041
// declaration.
10401042
NewFn = nullptr;
@@ -4042,7 +4044,9 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
40424044
.StartsWith("ds.fmin", AtomicRMWInst::FMin)
40434045
.StartsWith("ds.fmax", AtomicRMWInst::FMax)
40444046
.StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
4045-
.StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap);
4047+
.StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
4048+
.StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
4049+
.StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd);
40464050

40474051
unsigned NumOperands = CI->getNumOperands();
40484052
if (NumOperands < 3) // Malformed bitcode.
@@ -4097,8 +4101,10 @@ static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
40974101
Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
40984102

40994103
if (PtrTy->getAddressSpace() != 3) {
4100-
RMW->setMetadata("amdgpu.no.fine.grained.memory",
4101-
MDNode::get(F->getContext(), {}));
4104+
MDNode *EmptyMD = MDNode::get(F->getContext(), {});
4105+
RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
4106+
if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
4107+
RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
41024108
}
41034109

41044110
if (IsVolatile)

‎llvm/lib/Target/AMDGPU/AMDGPUInstructions.td‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -620,12 +620,10 @@ multiclass local_addr_space_atomic_op {
620620

621621
defm int_amdgcn_flat_atomic_fadd : noret_op;
622622
defm int_amdgcn_flat_atomic_fadd : flat_addr_space_atomic_op;
623-
defm int_amdgcn_flat_atomic_fadd_v2bf16 : noret_op;
624623
defm int_amdgcn_flat_atomic_fmin : noret_op;
625624
defm int_amdgcn_flat_atomic_fmax : noret_op;
626625
defm int_amdgcn_global_atomic_fadd : global_addr_space_atomic_op;
627626
defm int_amdgcn_flat_atomic_fadd : global_addr_space_atomic_op;
628-
defm int_amdgcn_global_atomic_fadd_v2bf16 : noret_op;
629627
defm int_amdgcn_global_atomic_fmin : noret_op;
630628
defm int_amdgcn_global_atomic_fmax : noret_op;
631629
defm int_amdgcn_global_atomic_csub : noret_op;

‎llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4907,8 +4907,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
49074907
case Intrinsic::amdgcn_flat_atomic_fmax:
49084908
case Intrinsic::amdgcn_flat_atomic_fmin_num:
49094909
case Intrinsic::amdgcn_flat_atomic_fmax_num:
4910-
case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
4911-
case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16:
49124910
case Intrinsic::amdgcn_atomic_cond_sub_u32:
49134911
case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
49144912
case Intrinsic::amdgcn_global_load_tr_b64:

‎llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,6 @@ def : SourceOfDivergence<int_amdgcn_flat_atomic_fmin>;
250250
def : SourceOfDivergence<int_amdgcn_flat_atomic_fmax>;
251251
def : SourceOfDivergence<int_amdgcn_flat_atomic_fmin_num>;
252252
def : SourceOfDivergence<int_amdgcn_flat_atomic_fmax_num>;
253-
def : SourceOfDivergence<int_amdgcn_global_atomic_fadd_v2bf16>;
254-
def : SourceOfDivergence<int_amdgcn_flat_atomic_fadd_v2bf16>;
255253
def : SourceOfDivergence<int_amdgcn_raw_buffer_atomic_swap>;
256254
def : SourceOfDivergence<int_amdgcn_raw_buffer_atomic_add>;
257255
def : SourceOfDivergence<int_amdgcn_raw_buffer_atomic_sub>;

‎llvm/lib/Target/AMDGPU/FLATInstructions.td‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1674,13 +1674,11 @@ defm : FlatAtomicIntrPat <"FLAT_ATOMIC_ADD_F32", "int_amdgcn_flat_atomic_fadd",
16741674

16751675
let OtherPredicates = [HasAtomicFlatPkAdd16Insts] in {
16761676
defm : FlatAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_F16", "int_amdgcn_flat_atomic_fadd", v2f16>;
1677-
defm : FlatAtomicIntrPat <"FLAT_ATOMIC_PK_ADD_BF16", "int_amdgcn_flat_atomic_fadd_v2bf16", v2i16>;
16781677
defm : FlatAtomicPat <"FLAT_ATOMIC_PK_ADD_F16", "atomic_load_fadd_flat", v2f16>;
16791678
defm : FlatAtomicPat <"FLAT_ATOMIC_PK_ADD_BF16", "atomic_load_fadd_flat", v2bf16>;
16801679
}
16811680

16821681
let OtherPredicates = [HasAtomicGlobalPkAddBF16Inst] in
1683-
defm : GlobalFLATAtomicIntrPats <"GLOBAL_ATOMIC_PK_ADD_BF16", "int_amdgcn_global_atomic_fadd_v2bf16", v2i16>;
16841682
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_PK_ADD_BF16", "atomic_load_fadd_global", v2bf16>;
16851683
} // End OtherPredicates = [HasFlatGlobalInsts], AddedComplexity = 10
16861684

‎llvm/lib/Target/AMDGPU/SIISelLowering.cpp‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1362,9 +1362,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
13621362
case Intrinsic::amdgcn_flat_atomic_fmax:
13631363
case Intrinsic::amdgcn_flat_atomic_fmin_num:
13641364
case Intrinsic::amdgcn_flat_atomic_fmax_num:
1365-
case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
1366-
case Intrinsic::amdgcn_atomic_cond_sub_u32:
1367-
case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16: {
1365+
case Intrinsic::amdgcn_atomic_cond_sub_u32: {
13681366
Info.opc = ISD::INTRINSIC_W_CHAIN;
13691367
Info.memVT = MVT::getVT(CI.getType());
13701368
Info.ptrVal = CI.getOperand(0);
@@ -1467,14 +1465,12 @@ bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
14671465
case Intrinsic::amdgcn_ds_ordered_add:
14681466
case Intrinsic::amdgcn_ds_ordered_swap:
14691467
case Intrinsic::amdgcn_flat_atomic_fadd:
1470-
case Intrinsic::amdgcn_flat_atomic_fadd_v2bf16:
14711468
case Intrinsic::amdgcn_flat_atomic_fmax:
14721469
case Intrinsic::amdgcn_flat_atomic_fmax_num:
14731470
case Intrinsic::amdgcn_flat_atomic_fmin:
14741471
case Intrinsic::amdgcn_flat_atomic_fmin_num:
14751472
case Intrinsic::amdgcn_global_atomic_csub:
14761473
case Intrinsic::amdgcn_global_atomic_fadd:
1477-
case Intrinsic::amdgcn_global_atomic_fadd_v2bf16:
14781474
case Intrinsic::amdgcn_global_atomic_fmax:
14791475
case Intrinsic::amdgcn_global_atomic_fmax_num:
14801476
case Intrinsic::amdgcn_global_atomic_fmin:

‎llvm/test/Bitcode/amdgcn-atomic.ll‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,4 +300,26 @@ define float @upgrade_amdgcn_ds_fmax_f32_no_suffix(ptr addrspace(3) %ptr, float
300300
ret float %result0
301301
}
302302

303+
declare <2 x i16> @llvm.amdgcn.flat.atomic.fadd.v2bf16.p0(ptr, <2 x i16>)
304+
305+
define <2 x i16> @upgrade_amdgcn_flat_atomic_fadd_v2bf16_p0(ptr %ptr, <2 x i16> %data) {
306+
; CHECK: [[BC0:%.+]] = bitcast <2 x i16> %data to <2 x bfloat>
307+
; CHECK-NEXT: [[ATOMIC:%.+]] = atomicrmw fadd ptr %ptr, <2 x bfloat> [[BC0]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}
308+
; CHECK-NEXT: [[BC1:%.+]] = bitcast <2 x bfloat> [[ATOMIC]] to <2 x i16>
309+
; CHECK-NEXT: ret <2 x i16> [[BC1]]
310+
%result = call <2 x i16> @llvm.amdgcn.flat.atomic.fadd.v2bf16.p0(ptr %ptr, <2 x i16> %data)
311+
ret <2 x i16> %result
312+
}
313+
314+
declare <2 x i16> @llvm.amdgcn.global.atomic.fadd.v2bf16.p1(ptr addrspace(1), <2 x i16>)
315+
316+
define <2 x i16> @upgrade_amdgcn_global_atomic_fadd_v2bf16_p1(ptr addrspace(1) %ptr, <2 x i16> %data) {
317+
; CHECK: [[BC0:%.+]] = bitcast <2 x i16> %data to <2 x bfloat>
318+
; CHECK-NEXT: [[ATOMIC:%.+]] = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> [[BC0]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}
319+
; CHECK-NEXT: [[BC1:%.+]] = bitcast <2 x bfloat> [[ATOMIC]] to <2 x i16>
320+
; CHECK-NEXT: ret <2 x i16> [[BC1]]
321+
%result = call <2 x i16> @llvm.amdgcn.global.atomic.fadd.v2bf16.p1(ptr addrspace(1) %ptr, <2 x i16> %data)
322+
ret <2 x i16> %result
323+
}
324+
303325
attributes #0 = { argmemonly nounwind willreturn }

‎llvm/test/CodeGen/AMDGPU/GlobalISel/fp-atomics-gfx940.ll‎

Lines changed: 0 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,6 @@
44
declare float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr %ptr, float %data)
55
declare <2 x half> @llvm.amdgcn.flat.atomic.fadd.v2f16.p0.v2f16(ptr %ptr, <2 x half> %data)
66

7-
; bf16 atomics use v2i16 argument since there is no bf16 data type in the llvm.
8-
declare <2 x i16> @llvm.amdgcn.flat.atomic.fadd.v2bf16.p0(ptr %ptr, <2 x i16> %data)
9-
declare <2 x i16> @llvm.amdgcn.global.atomic.fadd.v2bf16.p1(ptr addrspace(1) %ptr, <2 x i16> %data)
10-
declare <2 x half> @llvm.amdgcn.ds.fadd.v2f16(ptr addrspace(3) %ptr, <2 x half> %data, i32, i32, i1)
11-
declare <2 x i16> @llvm.amdgcn.ds.fadd.v2bf16(ptr addrspace(3) %ptr, <2 x i16> %data)
12-
137
define amdgpu_kernel void @flat_atomic_fadd_f32_noret(ptr %ptr, float %data) {
148
; GFX940-LABEL: flat_atomic_fadd_f32_noret:
159
; GFX940: ; %bb.0:
@@ -106,106 +100,6 @@ define <2 x half> @flat_atomic_fadd_v2f16_rtn(ptr %ptr, <2 x half> %data) {
106100
ret <2 x half> %ret
107101
}
108102

109-
define amdgpu_kernel void @flat_atomic_fadd_v2bf16_noret(ptr %ptr, <2 x i16> %data) {
110-
; GFX940-LABEL: flat_atomic_fadd_v2bf16_noret:
111-
; GFX940: ; %bb.0:
112-
; GFX940-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24
113-
; GFX940-NEXT: s_load_dword s4, s[2:3], 0x2c
114-
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
115-
; GFX940-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
116-
; GFX940-NEXT: v_mov_b32_e32 v2, s4
117-
; GFX940-NEXT: flat_atomic_pk_add_bf16 v[0:1], v2
118-
; GFX940-NEXT: s_endpgm
119-
%ret = call <2 x i16> @llvm.amdgcn.flat.atomic.fadd.v2bf16.p0(ptr %ptr, <2 x i16> %data)
120-
ret void
121-
}
122-
123-
define <2 x i16> @flat_atomic_fadd_v2bf16_rtn(ptr %ptr, <2 x i16> %data) {
124-
; GFX940-LABEL: flat_atomic_fadd_v2bf16_rtn:
125-
; GFX940: ; %bb.0:
126-
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
127-
; GFX940-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 sc0
128-
; GFX940-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
129-
; GFX940-NEXT: s_setpc_b64 s[30:31]
130-
%ret = call <2 x i16> @llvm.amdgcn.flat.atomic.fadd.v2bf16.p0(ptr %ptr, <2 x i16> %data)
131-
ret <2 x i16> %ret
132-
}
133-
134-
define amdgpu_kernel void @global_atomic_fadd_v2bf16_noret(ptr addrspace(1) %ptr, <2 x i16> %data) {
135-
; GFX940-LABEL: global_atomic_fadd_v2bf16_noret:
136-
; GFX940: ; %bb.0:
137-
; GFX940-NEXT: s_load_dword s4, s[2:3], 0x2c
138-
; GFX940-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24
139-
; GFX940-NEXT: v_mov_b32_e32 v1, 0
140-
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
141-
; GFX940-NEXT: v_mov_b32_e32 v0, s4
142-
; GFX940-NEXT: global_atomic_pk_add_bf16 v1, v0, s[0:1]
143-
; GFX940-NEXT: s_endpgm
144-
%ret = call <2 x i16> @llvm.amdgcn.global.atomic.fadd.v2bf16.p1(ptr addrspace(1) %ptr, <2 x i16> %data)
145-
ret void
146-
}
147-
148-
define <2 x i16> @global_atomic_fadd_v2bf16_rtn(ptr addrspace(1) %ptr, <2 x i16> %data) {
149-
; GFX940-LABEL: global_atomic_fadd_v2bf16_rtn:
150-
; GFX940: ; %bb.0:
151-
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
152-
; GFX940-NEXT: global_atomic_pk_add_bf16 v0, v[0:1], v2, off sc0
153-
; GFX940-NEXT: s_waitcnt vmcnt(0)
154-
; GFX940-NEXT: s_setpc_b64 s[30:31]
155-
%ret = call <2 x i16> @llvm.amdgcn.global.atomic.fadd.v2bf16.p1(ptr addrspace(1) %ptr, <2 x i16> %data)
156-
ret <2 x i16> %ret
157-
}
158-
159-
define amdgpu_kernel void @local_atomic_fadd_v2f16_noret(ptr addrspace(3) %ptr, <2 x half> %data) {
160-
; GFX940-LABEL: local_atomic_fadd_v2f16_noret:
161-
; GFX940: ; %bb.0:
162-
; GFX940-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24
163-
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
164-
; GFX940-NEXT: v_mov_b32_e32 v0, s0
165-
; GFX940-NEXT: v_mov_b32_e32 v1, s1
166-
; GFX940-NEXT: ds_pk_add_f16 v0, v1
167-
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
168-
; GFX940-NEXT: s_endpgm
169-
%ret = call <2 x half> @llvm.amdgcn.ds.fadd.v2f16(ptr addrspace(3) %ptr, <2 x half> %data, i32 0, i32 0, i1 0)
170-
ret void
171-
}
172-
173-
define <2 x half> @local_atomic_fadd_v2f16_rtn(ptr addrspace(3) %ptr, <2 x half> %data) {
174-
; GFX940-LABEL: local_atomic_fadd_v2f16_rtn:
175-
; GFX940: ; %bb.0:
176-
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
177-
; GFX940-NEXT: ds_pk_add_rtn_f16 v0, v0, v1
178-
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
179-
; GFX940-NEXT: s_setpc_b64 s[30:31]
180-
%ret = call <2 x half> @llvm.amdgcn.ds.fadd.v2f16(ptr addrspace(3) %ptr, <2 x half> %data, i32 0, i32 0, i1 0)
181-
ret <2 x half> %ret
182-
}
183-
184-
define amdgpu_kernel void @local_atomic_fadd_v2bf16_noret(ptr addrspace(3) %ptr, <2 x i16> %data) {
185-
; GFX940-LABEL: local_atomic_fadd_v2bf16_noret:
186-
; GFX940: ; %bb.0:
187-
; GFX940-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x24
188-
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
189-
; GFX940-NEXT: v_mov_b32_e32 v0, s0
190-
; GFX940-NEXT: v_mov_b32_e32 v1, s1
191-
; GFX940-NEXT: ds_pk_add_f16 v0, v1
192-
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
193-
; GFX940-NEXT: s_endpgm
194-
%ret = call <2 x i16> @llvm.amdgcn.ds.fadd.v2bf16(ptr addrspace(3) %ptr, <2 x i16> %data)
195-
ret void
196-
}
197-
198-
define <2 x i16> @local_atomic_fadd_v2bf16_rtn(ptr addrspace(3) %ptr, <2 x i16> %data) {
199-
; GFX940-LABEL: local_atomic_fadd_v2bf16_rtn:
200-
; GFX940: ; %bb.0:
201-
; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
202-
; GFX940-NEXT: ds_pk_add_rtn_f16 v0, v0, v1
203-
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
204-
; GFX940-NEXT: s_setpc_b64 s[30:31]
205-
%ret = call <2 x i16> @llvm.amdgcn.ds.fadd.v2bf16(ptr addrspace(3) %ptr, <2 x i16> %data)
206-
ret <2 x i16> %ret
207-
}
208-
209103
define <2 x half> @local_atomic_fadd_ret_v2f16_offset(ptr addrspace(3) %ptr, <2 x half> %val) {
210104
; GFX940-LABEL: local_atomic_fadd_ret_v2f16_offset:
211105
; GFX940: ; %bb.0:

0 commit comments

Comments
 (0)