Skip to content

Commit 3a30594

Browse files
Ana MihajlovicAna Mihajlovic
Ana Mihajlovic
authored and
Ana Mihajlovic
committed
[AMDGPU] Fix bad removal of s_delay_alu
1 parent a19ddff commit 3a30594

File tree

55 files changed

+1078
-446
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+1078
-446
lines changed

‎llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,12 @@ class AMDGPUInsertDelayAlu {
5050
static bool instructionWaitsForSGPRWrites(const MachineInstr &MI) {
5151
// These instruction types wait for VA_SDST==0 before issuing.
5252
const uint64_t VA_SDST_0 = SIInstrFlags::SALU | SIInstrFlags::SMRD;
53-
54-
return MI.getDesc().TSFlags & VA_SDST_0;
53+
if (MI.getDesc().TSFlags & VA_SDST_0) {
54+
for (auto &Op : MI.uses())
55+
if (Op.isReg())
56+
return true;
57+
}
58+
return false;
5559
}
5660

5761
// Types of delay that can be encoded in an s_delay_alu instruction.

‎llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3070,6 +3070,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
30703070
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
30713071
; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
30723072
; GFX12-NEXT: s_wait_alu 0xfffd
3073+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
30733074
; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
30743075
; GFX12-NEXT: flat_store_b32 v[0:1], v3
30753076
; GFX12-NEXT: s_endpgm
@@ -4161,6 +4162,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
41614162
; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
41624163
; GFX12-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
41634164
; GFX12-NEXT: s_wait_alu 0xfffd
4165+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
41644166
; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
41654167
; GFX12-NEXT: flat_store_b64 v[2:3], v[0:1]
41664168
; GFX12-NEXT: s_endpgm

‎llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,11 +1025,12 @@ define i128 @v_mul_i128(i128 %num, i128 %den) {
10251025
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10261026
; GFX12-NEXT: v_mov_b32_e32 v2, v11
10271027
; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], vcc_lo, v8, v5, v[1:2]
1028-
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
1028+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
10291029
; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], s0, v9, v4, v[1:2]
10301030
; GFX12-NEXT: s_wait_alu 0xf1ff
10311031
; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, v12, v7, s0
10321032
; GFX12-NEXT: s_wait_alu 0xfffd
1033+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
10331034
; GFX12-NEXT: v_add_co_ci_u32_e64 v6, null, v7, v6, vcc_lo
10341035
; GFX12-NEXT: v_mad_co_u64_u32 v[5:6], null, v10, v5, v[6:7]
10351036
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2387,33 +2388,39 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
23872388
; GFX12-NEXT: v_mad_co_u64_u32 v[18:19], null, v16, v12, 0
23882389
; GFX12-NEXT: v_mul_lo_u32 v30, v17, v14
23892390
; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v17, v13, v[0:1]
2390-
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
2391+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
23912392
; GFX12-NEXT: v_mad_co_u64_u32 v[18:19], s0, v17, v11, v[18:19]
23922393
; GFX12-NEXT: s_wait_alu 0xf1ff
23932394
; GFX12-NEXT: v_cndmask_b32_e64 v20, 0, 1, s0
23942395
; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, v12, v[0:1]
2396+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
23952397
; GFX12-NEXT: v_mad_co_u64_u32 v[18:19], vcc_lo, v2, v10, v[18:19]
23962398
; GFX12-NEXT: s_wait_alu 0xfffd
23972399
; GFX12-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v20, vcc_lo
23982400
; GFX12-NEXT: v_mad_co_u64_u32 v[20:21], null, v16, v10, 0
2401+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
23992402
; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v3, v11, v[0:1]
24002403
; GFX12-NEXT: v_mad_co_u64_u32 v[18:19], vcc_lo, v3, v9, v[18:19]
24012404
; GFX12-NEXT: s_wait_alu 0xfffd
24022405
; GFX12-NEXT: v_add_co_ci_u32_e64 v24, null, 0, v22, vcc_lo
2406+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
24032407
; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v4, v10, v[0:1]
24042408
; GFX12-NEXT: v_mad_co_u64_u32 v[18:19], vcc_lo, v4, v8, v[18:19]
24052409
; GFX12-NEXT: s_wait_alu 0xfffd
2410+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
24062411
; GFX12-NEXT: v_add_co_ci_u32_e64 v26, null, 0, v24, vcc_lo
24072412
; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], null, v5, v9, v[0:1]
2408-
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
2413+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
24092414
; GFX12-NEXT: v_mad_co_u64_u32 v[22:23], null, v6, v8, v[0:1]
24102415
; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], s0, v17, v9, v[20:21]
24112416
; GFX12-NEXT: s_wait_alu 0xf1ff
24122417
; GFX12-NEXT: v_cndmask_b32_e64 v25, 0, 1, s0
24132418
; GFX12-NEXT: v_mov_b32_e32 v20, v22
2419+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
24142420
; GFX12-NEXT: v_mad_co_u64_u32 v[21:22], vcc_lo, v2, v8, v[0:1]
24152421
; GFX12-NEXT: s_wait_alu 0xfffd
24162422
; GFX12-NEXT: v_add_co_ci_u32_e64 v29, null, 0, v25, vcc_lo
2423+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
24172424
; GFX12-NEXT: v_mad_co_u64_u32 v[0:1], s0, v16, v13, v[19:20]
24182425
; GFX12-NEXT: v_mov_b32_e32 v19, v22
24192426
; GFX12-NEXT: v_mul_lo_u32 v22, v16, v15
@@ -2434,6 +2441,7 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
24342441
; GFX12-NEXT: s_wait_alu 0xf1ff
24352442
; GFX12-NEXT: v_add_co_ci_u32_e64 v6, null, 0, v6, s2
24362443
; GFX12-NEXT: v_dual_mov_b32 v13, v1 :: v_dual_mov_b32 v14, v21
2444+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
24372445
; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], s2, v2, v9, v[11:12]
24382446
; GFX12-NEXT: s_wait_alu 0xf1ff
24392447
; GFX12-NEXT: v_add_co_ci_u32_e64 v6, null, 0, v6, s2
@@ -2447,6 +2455,7 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
24472455
; GFX12-NEXT: v_mad_co_u64_u32 v[5:6], s4, v5, v8, v[10:11]
24482456
; GFX12-NEXT: v_mad_co_u64_u32 v[1:2], s5, v17, v8, v[12:13]
24492457
; GFX12-NEXT: s_wait_alu 0xf1ff
2458+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
24502459
; GFX12-NEXT: v_add_co_ci_u32_e64 v3, s5, v9, v3, s5
24512460
; GFX12-NEXT: s_wait_alu 0xf1ff
24522461
; GFX12-NEXT: v_add_co_ci_u32_e64 v4, s5, v29, v4, s5
@@ -2463,9 +2472,10 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
24632472
; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, v9, v25, s3
24642473
; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, v9, v20, s1
24652474
; GFX12-NEXT: s_wait_alu 0xfffd
2466-
; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, v9, v28, vcc_lo
24672475
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2476+
; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, v9, v28, vcc_lo
24682477
; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, v9, v27, s0
2478+
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
24692479
; GFX12-NEXT: v_mad_co_u64_u32 v[7:8], null, v7, v8, v[9:10]
24702480
; GFX12-NEXT: s_setpc_b64 s[30:31]
24712481
%result = mul i256 %num, %den

‎llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
240240
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
241241
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
242242
; GFX12W64-NEXT: s_wait_alu 0xf1ff
243+
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
243244
; GFX12W64-NEXT: v_mad_u32_u24 v0, v0, 5, s2
244245
; GFX12W64-NEXT: s_wait_kmcnt 0x0
245246
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -453,6 +454,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
453454
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
454455
; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
455456
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
457+
; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
456458
; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
457459
; GFX11W64-NEXT: v_mov_b32_e32 v0, 0
458460
; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -482,6 +484,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
482484
; GFX11W32-NEXT: s_waitcnt vmcnt(0)
483485
; GFX11W32-NEXT: v_readfirstlane_b32 s4, v1
484486
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
487+
; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
485488
; GFX11W32-NEXT: v_mad_u64_u32 v[1:2], null, s0, v0, s[4:5]
486489
; GFX11W32-NEXT: v_mov_b32_e32 v0, 0
487490
; GFX11W32-NEXT: global_store_b32 v0, v1, s[2:3]
@@ -514,6 +517,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
514517
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
515518
; GFX12W64-NEXT: s_wait_kmcnt 0x0
516519
; GFX12W64-NEXT: s_wait_alu 0xf1ff
520+
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
517521
; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
518522
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
519523
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -544,6 +548,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
544548
; GFX12W32-NEXT: s_wait_loadcnt 0x0
545549
; GFX12W32-NEXT: v_readfirstlane_b32 s4, v1
546550
; GFX12W32-NEXT: s_wait_kmcnt 0x0
551+
; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
547552
; GFX12W32-NEXT: v_mad_co_u64_u32 v[0:1], null, s0, v0, s[4:5]
548553
; GFX12W32-NEXT: v_mov_b32_e32 v1, 0
549554
; GFX12W32-NEXT: global_store_b32 v1, v0, s[2:3]
@@ -882,6 +887,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
882887
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
883888
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
884889
; GFX12W64-NEXT: s_wait_alu 0xf1ff
890+
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
885891
; GFX12W64-NEXT: v_add_nc_u32_e32 v0, s2, v0
886892
; GFX12W64-NEXT: s_wait_kmcnt 0x0
887893
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -926,6 +932,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
926932
; GFX12W32-NEXT: s_wait_loadcnt 0x0
927933
; GFX12W32-NEXT: v_readfirstlane_b32 s2, v1
928934
; GFX12W32-NEXT: s_wait_alu 0xf1ff
935+
; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
929936
; GFX12W32-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
930937
; GFX12W32-NEXT: s_wait_kmcnt 0x0
931938
; GFX12W32-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -1285,6 +1292,7 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
12851292
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
12861293
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
12871294
; GFX12W64-NEXT: s_wait_alu 0xf1ff
1295+
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
12881296
; GFX12W64-NEXT: v_add_nc_u32_e32 v0, s2, v0
12891297
; GFX12W64-NEXT: s_wait_kmcnt 0x0
12901298
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -1331,6 +1339,7 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
13311339
; GFX12W32-NEXT: s_wait_loadcnt 0x0
13321340
; GFX12W32-NEXT: v_readfirstlane_b32 s2, v1
13331341
; GFX12W32-NEXT: s_wait_alu 0xf1ff
1342+
; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
13341343
; GFX12W32-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
13351344
; GFX12W32-NEXT: s_wait_kmcnt 0x0
13361345
; GFX12W32-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -1968,6 +1977,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
19681977
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
19691978
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
19701979
; GFX12W64-NEXT: s_wait_alu 0xf1ff
1980+
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
19711981
; GFX12W64-NEXT: v_sub_nc_u32_e32 v0, s2, v0
19721982
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
19731983
; GFX12W64-NEXT: s_endpgm
@@ -2000,6 +2010,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
20002010
; GFX12W32-NEXT: v_readfirstlane_b32 s0, v1
20012011
; GFX12W32-NEXT: v_mov_b32_e32 v1, 0
20022012
; GFX12W32-NEXT: s_wait_alu 0xf1ff
2013+
; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
20032014
; GFX12W32-NEXT: v_sub_nc_u32_e32 v0, s0, v0
20042015
; GFX12W32-NEXT: global_store_b32 v1, v0, s[2:3]
20052016
; GFX12W32-NEXT: s_endpgm
@@ -2338,6 +2349,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
23382349
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
23392350
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
23402351
; GFX12W64-NEXT: s_wait_alu 0xf1ff
2352+
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
23412353
; GFX12W64-NEXT: v_sub_nc_u32_e32 v0, s2, v0
23422354
; GFX12W64-NEXT: s_wait_kmcnt 0x0
23432355
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -2383,6 +2395,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
23832395
; GFX12W32-NEXT: v_readfirstlane_b32 s2, v1
23842396
; GFX12W32-NEXT: v_mov_b32_e32 v1, 0
23852397
; GFX12W32-NEXT: s_wait_alu 0xf1ff
2398+
; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
23862399
; GFX12W32-NEXT: v_sub_nc_u32_e32 v0, s2, v0
23872400
; GFX12W32-NEXT: s_wait_kmcnt 0x0
23882401
; GFX12W32-NEXT: global_store_b32 v1, v0, s[0:1]

0 commit comments

Comments
 (0)