Skip to content

Commit 3395676

Browse files
rampitecjayfoad
rampitec
and
jayfoad
authored
[AMDGPU] Fold copies of constant physical registers into their uses (#154183)
With current codegen this only affects src_flat_scratch_base_lo/hi. Co-authored-by: Jay Foad <[email protected]> Co-authored-by: Jay Foad <[email protected]>
1 parent c328c5d commit 3395676

File tree

6 files changed

+597
-787
lines changed

6 files changed

+597
-787
lines changed

‎llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -709,7 +709,10 @@ bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
709709
// 16-bit SGPRs instead of 32-bit ones.
710710
if (Old.getSubReg() == AMDGPU::lo16 && TRI->isSGPRReg(*MRI, New->getReg()))
711711
Old.setSubReg(AMDGPU::NoSubRegister);
712-
Old.substVirtReg(New->getReg(), New->getSubReg(), *TRI);
712+
if (New->getReg().isPhysical())
713+
Old.substPhysReg(New->getReg(), *TRI);
714+
else
715+
Old.substVirtReg(New->getReg(), New->getSubReg(), *TRI);
713716
Old.setIsUndef(New->isUndef());
714717
return true;
715718
}
@@ -1986,7 +1989,9 @@ bool SIFoldOperandsImpl::tryFoldFoldableCopy(
19861989
if (!FoldingImm && !OpToFold.isReg())
19871990
return false;
19881991

1989-
if (OpToFold.isReg() && !OpToFold.getReg().isVirtual())
1992+
// Fold virtual registers and constant physical registers.
1993+
if (OpToFold.isReg() && OpToFold.getReg().isPhysical() &&
1994+
!TRI->isConstantPhysReg(OpToFold.getReg()))
19901995
return false;
19911996

19921997
// Prevent folding operands backwards in the function. For example,

‎llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll

Lines changed: 22 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,14 @@ target triple = "amdgcn-amd-amdhsa"
99
define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
1010
; GFX1250-SDAG-LABEL: use_private_to_flat_addrspacecast:
1111
; GFX1250-SDAG: ; %bb.0:
12-
; GFX1250-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x24
12+
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x24
1313
; GFX1250-SDAG-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
14-
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_flat_scratch_base_lo
1514
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
1615
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
17-
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_lshlrev_b32 v1, 20, v0
18-
; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, -1
16+
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_lshlrev_b32 v1, 20, v0
17+
; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s0, -1
1918
; GFX1250-SDAG-NEXT: s_cselect_b32 vcc_lo, -1, 0
20-
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
19+
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], src_flat_scratch_base_lo, v[0:1]
2120
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
2221
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v1, 0, v1
2322
; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
@@ -27,20 +26,20 @@ define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %p
2726
;
2827
; GFX1250-GISEL-LABEL: use_private_to_flat_addrspacecast:
2928
; GFX1250-GISEL: ; %bb.0:
30-
; GFX1250-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x24
31-
; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_flat_scratch_base_lo
29+
; GFX1250-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x24
30+
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], src_flat_scratch_base_lo
3231
; GFX1250-GISEL-NEXT: v_mbcnt_lo_u32_b32 v2, -1, 0
33-
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3432
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
35-
; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s2, -1
36-
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
37-
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
33+
; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s0, -1
34+
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
35+
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
3836
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v2, 20, v2
39-
; GFX1250-GISEL-NEXT: s_cselect_b32 s0, 1, 0
40-
; GFX1250-GISEL-NEXT: s_and_b32 s0, 1, s0
41-
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
37+
; GFX1250-GISEL-NEXT: s_cselect_b32 s1, 1, 0
38+
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
39+
; GFX1250-GISEL-NEXT: s_and_b32 s1, 1, s1
4240
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v1, vcc_lo
43-
; GFX1250-GISEL-NEXT: v_cmp_ne_u32_e64 vcc_lo, 0, s0
41+
; GFX1250-GISEL-NEXT: v_cmp_ne_u32_e64 vcc_lo, 0, s1
42+
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
4443
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v1, 0, v1
4544
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
4645
; GFX1250-GISEL-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
@@ -56,27 +55,24 @@ define amdgpu_kernel void @use_private_to_flat_addrspacecast_nonnull(ptr addrspa
5655
; GFX1250-SDAG: ; %bb.0:
5756
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x24
5857
; GFX1250-SDAG-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
59-
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
58+
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
6059
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v1, 20, v0
6160
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
6261
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, s0
63-
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_flat_scratch_base_lo
64-
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
65-
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
62+
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], src_flat_scratch_base_lo, v[0:1]
6663
; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
6764
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
6865
; GFX1250-SDAG-NEXT: s_endpgm
6966
;
7067
; GFX1250-GISEL-LABEL: use_private_to_flat_addrspacecast_nonnull:
7168
; GFX1250-GISEL: ; %bb.0:
72-
; GFX1250-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x24
73-
; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_flat_scratch_base_lo
69+
; GFX1250-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x24
7470
; GFX1250-GISEL-NEXT: v_mbcnt_lo_u32_b32 v2, -1, 0
75-
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
71+
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], src_flat_scratch_base_lo
7672
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
7773
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v2, 20, v2
7874
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
79-
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
75+
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0
8076
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
8177
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v1, vcc_lo
8278
; GFX1250-GISEL-NEXT: flat_store_b32 v[0:1], v3 scope:SCOPE_SYS
@@ -91,10 +87,9 @@ define amdgpu_kernel void @use_flat_to_private_addrspacecast(ptr %ptr) {
9187
; GFX1250-LABEL: use_flat_to_private_addrspacecast:
9288
; GFX1250: ; %bb.0:
9389
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
94-
; GFX1250-NEXT: s_mov_b32 s2, src_flat_scratch_base_lo
9590
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
9691
; GFX1250-NEXT: s_wait_kmcnt 0x0
97-
; GFX1250-NEXT: s_sub_co_i32 s2, s0, s2
92+
; GFX1250-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo
9893
; GFX1250-NEXT: s_cmp_lg_u64 s[0:1], 0
9994
; GFX1250-NEXT: s_cselect_b32 s0, s2, -1
10095
; GFX1250-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
@@ -110,9 +105,8 @@ define amdgpu_kernel void @use_flat_to_private_addrspacecast_nonnull(ptr %ptr) {
110105
; GFX1250-SDAG: ; %bb.0:
111106
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x24
112107
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
113-
; GFX1250-SDAG-NEXT: s_mov_b32 s1, src_flat_scratch_base_lo
114108
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
115-
; GFX1250-SDAG-NEXT: s_sub_co_i32 s0, s0, s1
109+
; GFX1250-SDAG-NEXT: s_sub_co_i32 s0, s0, src_flat_scratch_base_lo
116110
; GFX1250-SDAG-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
117111
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
118112
; GFX1250-SDAG-NEXT: s_endpgm
@@ -122,9 +116,7 @@ define amdgpu_kernel void @use_flat_to_private_addrspacecast_nonnull(ptr %ptr) {
122116
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
123117
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0
124118
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
125-
; GFX1250-GISEL-NEXT: s_mov_b32 s1, src_flat_scratch_base_lo
126-
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
127-
; GFX1250-GISEL-NEXT: s_sub_co_i32 s0, s0, s1
119+
; GFX1250-GISEL-NEXT: s_sub_co_i32 s0, s0, src_flat_scratch_base_lo
128120
; GFX1250-GISEL-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
129121
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
130122
; GFX1250-GISEL-NEXT: s_endpgm

0 commit comments

Comments
 (0)