[AMDGPU] Ensure divergence for v_alignbit #129159

jrbyrnes · 2025-02-28T00:10:26Z

Selecting vgpr for the uniform version of this pattern may lead to unnecessary vgpr and waterfall loops.

llvmbot · 2025-02-28T00:11:02Z

@llvm/pr-subscribers-backend-amdgpu

Author: Jeffrey Byrnes (jrbyrnes)

Changes

Selecting vgpr for the uniform version of this pattern may lead to unnecessary vgpr and waterfall loops.

Patch is 273.24 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/129159.diff

20 Files Affected:

(modified) llvm/lib/Target/AMDGPU/SIInstructions.td (+2-2)
(modified) llvm/test/CodeGen/AMDGPU/alignbit-pat.ll (+4-2)
(modified) llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll (+369-381)
(modified) llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll (+434-412)
(modified) llvm/test/CodeGen/AMDGPU/call-argument-types.ll (+4-4)
(modified) llvm/test/CodeGen/AMDGPU/cgp-bitfield-extract.ll (+2-3)
(modified) llvm/test/CodeGen/AMDGPU/fmed3.ll (+4-7)
(modified) llvm/test/CodeGen/AMDGPU/function-returns.ll (+5-5)
(modified) llvm/test/CodeGen/AMDGPU/insert-delay-alu-bug.ll (+5-7)
(modified) llvm/test/CodeGen/AMDGPU/minimummaximum.ll (+23-11)
(modified) llvm/test/CodeGen/AMDGPU/minmax.ll (+21-10)
(modified) llvm/test/CodeGen/AMDGPU/mul_int24.ll (+9-8)
(modified) llvm/test/CodeGen/AMDGPU/sdiv64.ll (+34-36)
(modified) llvm/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll (+2-2)
(modified) llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll (+6-11)
(modified) llvm/test/CodeGen/AMDGPU/srem64.ll (+40-42)
(modified) llvm/test/CodeGen/AMDGPU/udiv64.ll (+14-18)
(added) llvm/test/CodeGen/AMDGPU/uniform-alignbit.ll (+38)
(modified) llvm/test/CodeGen/AMDGPU/urem64.ll (+12-12)
(modified) llvm/test/CodeGen/AMDGPU/v_pack.ll (+12-24)

diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 6f80dbcfe5e71..e618cbebadb7c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2500,11 +2500,11 @@ def : AMDGPUPat <
 
 def : ROTRPattern <V_ALIGNBIT_B32_e64>;
 
-def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
+def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (and i32:$src1, (i32 31))))),
           (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),
                           (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>;
 
-def : GCNPat<(i32 (trunc (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),
+def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (i32 ShiftAmt32Imm:$src1)))),
           (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),
                           (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>;
 
diff --git a/llvm/test/CodeGen/AMDGPU/alignbit-pat.ll b/llvm/test/CodeGen/AMDGPU/alignbit-pat.ll
index fc1326268fd1e..6451677db313c 100644
--- a/llvm/test/CodeGen/AMDGPU/alignbit-pat.ll
+++ b/llvm/test/CodeGen/AMDGPU/alignbit-pat.ll
@@ -3,7 +3,8 @@
 ; GCN-LABEL: {{^}}alignbit_shr_pat:
 ; GCN-DAG: s_load_dword s[[SHR:[0-9]+]]
 ; GCN-DAG: load_dwordx2 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]]
-; GCN: v_alignbit_b32 v{{[0-9]+}}, v[[HI]], v[[LO]], s[[SHR]]
+; GCN-NEXT: s_mov_b32
+; GCN-NEXT: s_mov_b32
 
 define amdgpu_kernel void @alignbit_shr_pat(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1, i32 %arg2) {
 bb:
@@ -70,7 +71,8 @@ bb:
 
 ; GCN-LABEL: {{^}}alignbit_shr_pat_const30:
 ; GCN: load_dwordx2 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]]
-; GCN: v_alignbit_b32 v{{[0-9]+}}, v[[HI]], v[[LO]], 30
+; GCN-NEXT: s_mov_b32
+; GCN-NEXT: s_mov_b32
 
 define amdgpu_kernel void @alignbit_shr_pat_const30(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) nocapture %arg1) {
 bb:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 8e16889c72e65..8e8aa409eafec 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -4610,50 +4610,48 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[10:11], 30
+; GFX6-NEXT:    s_mov_b32 s0, s8
+; GFX6-NEXT:    s_and_b32 s8, s6, 0x7fff
+; GFX6-NEXT:    s_and_b32 s6, s4, 0x7fff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s6
 ; GFX6-NEXT:    s_and_b32 s6, s10, 0x7fff
-; GFX6-NEXT:    s_and_b32 s7, s4, 0x7fff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s7
-; GFX6-NEXT:    v_mov_b32_e32 v2, s4
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s6
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[4:5], 30
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
 ; GFX6-NEXT:    s_bfe_u32 s4, s4, 0xf000f
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s6
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v1
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT:    s_bfe_u32 s7, s10, 0xf000f
-; GFX6-NEXT:    v_alignbit_b32 v2, s5, v2, 30
-; GFX6-NEXT:    v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v6, s7
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v7, v5
-; GFX6-NEXT:    v_and_b32_e32 v2, 0x7fff, v2
-; GFX6-NEXT:    v_trunc_f32_e32 v4, v4
-; GFX6-NEXT:    v_mad_f32 v3, -v4, v1, v3
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, v2
-; GFX6-NEXT:    v_mov_b32_e32 v0, s10
-; GFX6-NEXT:    v_alignbit_b32 v0, s11, v0, 30
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v1
-; GFX6-NEXT:    v_mul_f32_e32 v1, v6, v7
-; GFX6-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_mad_f32 v4, -v1, v5, v6
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, v0
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v2
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, v5
-; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX6-NEXT:    v_mul_f32_e32 v1, v0, v6
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v5, v1
-; GFX6-NEXT:    v_mad_f32 v0, -v1, v2, v0
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, v2
-; GFX6-NEXT:    v_and_b32_e32 v2, 0x7fff, v3
-; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v5, vcc
-; GFX6-NEXT:    v_and_b32_e32 v3, 0x7fff, v4
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT:    s_mov_b32 s1, s9
+; GFX6-NEXT:    s_bfe_u32 s9, s10, 0xf000f
+; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, s9
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v5, v3
+; GFX6-NEXT:    s_and_b32 s5, s6, 0x7fff
+; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s5
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT:    v_mul_f32_e32 v0, v4, v5
+; GFX6-NEXT:    v_trunc_f32_e32 v0, v0
+; GFX6-NEXT:    v_mad_f32 v4, -v0, v3, v4
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s8
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v1
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, v3
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v0, vcc
+; GFX6-NEXT:    v_mul_f32_e32 v0, v5, v6
+; GFX6-NEXT:    v_trunc_f32_e32 v0, v0
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v4, v0
+; GFX6-NEXT:    v_mad_f32 v0, -v0, v1, v5
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, v1
+; GFX6-NEXT:    v_and_b32_e32 v3, 0x7fff, v3
+; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v4, vcc
 ; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
+; GFX6-NEXT:    v_and_b32_e32 v2, 0x7fff, v2
 ; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
 ; GFX6-NEXT:    v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT:    s_mov_b32 s0, s8
-; GFX6-NEXT:    s_mov_b32 s1, s9
 ; GFX6-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -4667,46 +4665,44 @@ define amdgpu_kernel void @udiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
 ; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    s_and_b32 s5, s6, 0x7fff
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[2:3], 30
+; GFX9-NEXT:    s_and_b32 s3, s6, 0x7fff
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX9-NEXT:    s_and_b32 s5, s2, 0x7fff
+; GFX9-NEXT:    s_bfe_u32 s8, s2, 0xf000f
+; GFX9-NEXT:    s_lshr_b64 s[2:3], s[6:7], 30
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s5
-; GFX9-NEXT:    s_and_b32 s4, s2, 0x7fff
-; GFX9-NEXT:    v_alignbit_b32 v0, s3, v0, 30
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v0
 ; GFX9-NEXT:    s_bfe_u32 s3, s6, 0xf000f
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s4
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v1
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v6, s3
-; GFX9-NEXT:    s_bfe_u32 s2, s2, 0xf000f
-; GFX9-NEXT:    v_mov_b32_e32 v3, s6
-; GFX9-NEXT:    v_alignbit_b32 v3, s7, v3, 30
-; GFX9-NEXT:    v_mul_f32_e32 v5, v4, v5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, s2
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v6
-; GFX9-NEXT:    v_and_b32_e32 v3, 0x7fff, v3
-; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
-; GFX9-NEXT:    v_mad_f32 v4, -v5, v1, v4
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v3
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, v1
-; GFX9-NEXT:    v_mul_f32_e32 v1, v7, v8
-; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_mad_f32 v5, -v1, v6, v7
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, v0
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v7, v3
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v1, v0, v7
-; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v6, v1
-; GFX9-NEXT:    v_mad_f32 v0, -v1, v3, v0
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, v3
-; GFX9-NEXT:    v_and_b32_e32 v3, 0x7fff, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v6, vcc
-; GFX9-NEXT:    v_and_b32_e32 v4, 0x7fff, v5
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s8
+; GFX9-NEXT:    v_mul_f32_e32 v3, v1, v3
+; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v6, v4
+; GFX9-NEXT:    s_and_b32 s2, s2, 0x7fff
+; GFX9-NEXT:    v_mad_f32 v1, -v3, v0, v1
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    v_mul_f32_e32 v0, v5, v6
+; GFX9-NEXT:    s_and_b32 s4, s4, 0x7fff
+; GFX9-NEXT:    v_trunc_f32_e32 v0, v0
+; GFX9-NEXT:    v_mad_f32 v5, -v0, v4, v5
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v6, s4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v7, v1
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v0, vcc
+; GFX9-NEXT:    v_mul_f32_e32 v0, v6, v7
+; GFX9-NEXT:    v_trunc_f32_e32 v0, v0
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v0
+; GFX9-NEXT:    v_mad_f32 v0, -v0, v1, v6
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, v1
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x7fff, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v5, vcc
 ; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 30, v[0:1]
+; GFX9-NEXT:    v_and_b32_e32 v3, 0x7fff, v3
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 15, v4
 ; GFX9-NEXT:    v_or_b32_e32 v3, v3, v4
 ; GFX9-NEXT:    v_or_b32_e32 v0, v3, v0
@@ -4797,58 +4793,56 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[10:11], 30
 ; GFX6-NEXT:    s_mov_b32 s0, s8
-; GFX6-NEXT:    s_and_b32 s8, s4, 0x7fff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s8
-; GFX6-NEXT:    s_and_b32 s7, s10, 0x7fff
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s7
-; GFX6-NEXT:    v_mov_b32_e32 v2, s4
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v1
-; GFX6-NEXT:    v_alignbit_b32 v2, s5, v2, 30
-; GFX6-NEXT:    s_bfe_u32 s5, s4, 0xf000f
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s5
-; GFX6-NEXT:    v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT:    v_trunc_f32_e32 v4, v4
-; GFX6-NEXT:    v_mad_f32 v3, -v4, v1, v3
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v1
-; GFX6-NEXT:    s_bfe_u32 s8, s10, 0xf000f
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s8
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v4, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s4
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v5
-; GFX6-NEXT:    v_and_b32_e32 v2, 0x7fff, v2
-; GFX6-NEXT:    v_mov_b32_e32 v0, s10
-; GFX6-NEXT:    v_sub_i32_e32 v6, vcc, s10, v1
-; GFX6-NEXT:    v_mul_f32_e32 v1, v3, v4
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v4, v2
-; GFX6-NEXT:    v_alignbit_b32 v0, s11, v0, 30
-; GFX6-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX6-NEXT:    v_cvt_f32_u32_e32 v7, v0
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v8, v4
-; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
-; GFX6-NEXT:    v_mad_f32 v3, -v1, v5, v3
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX6-NEXT:    v_mul_f32_e32 v3, v7, v8
-; GFX6-NEXT:    v_trunc_f32_e32 v3, v3
-; GFX6-NEXT:    v_cvt_u32_f32_e32 v5, v3
-; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT:    v_mad_f32 v3, -v3, v4, v7
+; GFX6-NEXT:    s_and_b32 s8, s6, 0x7fff
+; GFX6-NEXT:    s_and_b32 s6, s4, 0x7fff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX6-NEXT:    s_and_b32 s6, s10, 0x7fff
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s6
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[4:5], 30
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GFX6-NEXT:    s_and_b32 s5, s6, 0x7fff
+; GFX6-NEXT:    s_bfe_u32 s6, s4, 0xf000f
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v3, s6
+; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
+; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX6-NEXT:    s_bfe_u32 s11, s10, 0xf000f
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v1, s11
+; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s4
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v3
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v5, s8
 ; GFX6-NEXT:    s_lshr_b32 s4, s4, 15
-; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s4
-; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
-; GFX6-NEXT:    v_mul_lo_u32 v2, v3, v2
-; GFX6-NEXT:    s_lshr_b32 s6, s10, 15
-; GFX6-NEXT:    v_sub_i32_e32 v3, vcc, s6, v1
-; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT:    v_and_b32_e32 v3, 0x7fff, v3
-; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
-; GFX6-NEXT:    v_and_b32_e32 v2, 0x7fff, v6
-; GFX6-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
-; GFX6-NEXT:    v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT:    v_sub_i32_e32 v4, vcc, s10, v0
+; GFX6-NEXT:    v_mul_f32_e32 v0, v1, v2
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v2, s5
+; GFX6-NEXT:    v_trunc_f32_e32 v0, v0
+; GFX6-NEXT:    v_mad_f32 v1, -v0, v3, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v6, v2
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v3
 ; GFX6-NEXT:    s_mov_b32 s1, s9
+; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v0, vcc
+; GFX6-NEXT:    v_mul_f32_e32 v1, v5, v6
+; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v3, v1
+; GFX6-NEXT:    v_mad_f32 v1, -v1, v2, v5
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v2
+; GFX6-NEXT:    v_mul_lo_u32 v0, v0, s4
+; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v1, v1, s5
+; GFX6-NEXT:    s_lshr_b32 s9, s10, 15
+; GFX6-NEXT:    v_sub_i32_e32 v2, vcc, s9, v0
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, s8, v1
+; GFX6-NEXT:    v_and_b32_e32 v2, 0x7fff, v2
+; GFX6-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
+; GFX6-NEXT:    v_and_b32_e32 v3, 0x7fff, v4
+; GFX6-NEXT:    v_lshlrev_b32_e32 v2, 15, v2
+; GFX6-NEXT:    v_or_b32_e32 v2, v2, v3
 ; GFX6-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX6-NEXT:    s_waitcnt expcnt(0)
@@ -4862,54 +4856,52 @@ define amdgpu_kernel void @urem_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
 ; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_alignbit_b32 v0, s3, v0, 30
-; GFX9-NEXT:    s_and_b32 s3, s6, 0x7fff
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[2:3], 30
+; GFX9-NEXT:    s_and_b32 s5, s6, 0x7fff
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT:    s_and_b32 s3, s2, 0x7fff
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, s3
-; GFX9-NEXT:    s_and_b32 s4, s2, 0x7fff
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s4
-; GFX9-NEXT:    s_bfe_u32 s4, s6, 0xf000f
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v5, v1
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v6, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s6
-; GFX9-NEXT:    v_alignbit_b32 v3, s7, v3, 30
-; GFX9-NEXT:    v_mul_f32_e32 v5, v4, v5
-; GFX9-NEXT:    v_trunc_f32_e32 v5, v5
-; GFX9-NEXT:    v_mad_f32 v4, -v5, v1, v4
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0xf000f
-; GFX9-NEXT:    v_and_b32_e32 v3, 0x7fff, v3
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, v1
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v7, s5
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v8, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v5, vcc
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, v3
-; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
-; GFX9-NEXT:    v_mul_f32_e32 v4, v7, v8
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v8, v0
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v9, v5
+; GFX9-NEXT:    s_and_b32 s8, s4, 0x7fff
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v3, v0
+; GFX9-NEXT:    s_lshr_b64 s[4:5], s[6:7], 30
+; GFX9-NEXT:    s_bfe_u32 s5, s6, 0xf000f
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s5
+; GFX9-NEXT:    v_mul_f32_e32 v3, v1, v3
+; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX9-NEXT:    v_mad_f32 v1, -v3, v0, v1
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GFX9-NEXT:    s_bfe_u32 s9, s2, 0xf000f
+; GFX9-NEXT:    s_and_b32 s3, s4, 0x7fff
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v5, s9
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, s3
+; GFX9-NEXT:    v_mul_f32_e32 v1, v5, v6
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v6, s8
+; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v7, v3
+; GFX9-NEXT:    v_mad_f32 v5, -v1, v4, v5
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, v4
+; GFX9-NEXT:    v_mul_f32_e32 v4, v6, v7
 ; GFX9-NEXT:    v_trunc_f32_e32 v4, v4
-; GFX9-NEXT:    v_mad_f32 v7, -v4, v6, v7
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v7|, v6
-; GFX9-NEXT:    v_mul_f32_e32 v6, v8, v9
-; GFX9-NEXT:    v_trunc_f32_e32 v6, v6
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v7, v6
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
-; GFX9-NEXT:    v_mad_f32 v6, -v6, v5, v8
-; GFX9-NEXT:    s_lshr_b32 s3, s6, 15
-; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v6|, v5
-; GFX9-NEXT:    v_mul_lo_u32 v4, v4, s3
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s6
-; GFX9-NEXT:    v_mul_lo_u32 v3, v5, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v5, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mad_f32 v4, -v4, v3, v6
+; GFX9-NEXT:    s_lshr_b32 s4, s6, 15
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, v3
+; GFX9-NEXT:    v_mul_lo_u32 v0, v0, s6
+; GFX9-NEXT:    v_mul_lo_u32 v1, v1, s4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, s3
 ; GFX9-NEXT:    s_lshr_b32 s3, s2, 15
-; GFX9-NEXT:    v_sub_u32_e32 v4, s3, v4
-; GFX9-NEXT:    v_sub_u32_e32 v5, s2, v1
-; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT:    v_and_b32_e32 v4, 0x7fff, v4
+; GFX9-NEXT:    v_sub_u32_e32 v4, s2, v0
+; GFX9-NEXT:    v_sub_u32_e32 v5, s3, v1
+; GFX9-NEXT:    v_sub_u32_e32 v0, s8, v3
+; GFX9-NEXT:    v_and_b32_e32 v3, 0x7fff, v4
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x7fff, v5
 ; GFX9-NEXT:    v_lshlrev_b64 v[0:1], 30, v[0:1]
-; GFX9-NEXT:    v_and_b32_e32 v3, 0x7fff, v5
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 15, v4
 ; GFX9-NEXT:    v_or_b32_e32 v3, v3, v4
 ; GFX9-NEXT:    v_or_b32_e32 v0, v3, v0
@@ -5006,64 +4998,63 @@ define amdgpu_kernel void @sdiv_v3i15(ptr addrspace(1) %out, <3 x i15> %x, <3 x
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NEXT:    v_mov_b32_e32 v0, s10
-; GFX6-NEXT:    s_bfe_i32 s6, s4, 0xf0000
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s6
-; GFX6-NEXT:    v_mov_b32_e32 v1, s4
-; GFX6-NEXT:    v_alignbit_b32 v1, s5, v1, 30
+; GFX6-NEXT:    s_lshr_b64 s[6:7], s[10:11], 30
+; GFX6-NEXT:    s_bfe_i32 s7, s4, 0xf0000
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v0, s7
+; GFX6-NEXT:    s_mov_b32 s0, s8
+; GFX6-NEXT:    s_mov_b32 s1, s9
+; GFX6-NEXT:    s_lshr_b64 s[8:9], s[4:5], 30
 ; GFX6-NEXT:    s_bfe_i32 s5, s10, 0xf0000
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v3, s5
-; GFX6-NEXT:    v_rcp_iflag_f32_e32 v4, v2
-; GFX6-NEXT:    s_xor_b32 s5, s5, s6
+; GFX6-NEXT:    v_cvt_f32_i32_e32 v1, s5
+; GFX6-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GFX6-NEXT:    s_xor_b32 s5, s5, s7
 ; GFX6-NEXT:    s_ashr_i32 s5, s5, 30
 ; GFX6-NEXT:    s_or_b32 s5, s5, 1
-; GFX6-NEXT:    v_mul_f32_e32 v4, v3, v4
-; GFX6-NEXT:    v_trunc_f32_e32 v4, v4
-; GFX6-NEXT:    v_mad_f32 v3, -v4, v2, v3
-; GFX6-NEXT:    v_cmp_ge_f32_e64 s[6:7], |v3|, |v2|
-; GFX6-NEXT:    s_and_b64 s[6:7], s[6:7], exec
-; GFX6-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GFX6-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT:    v_trunc_f32_e32 v2, v2
+; GFX6-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT:    v_cmp_ge_f32_e64 s[12:13], |v1|, |v0|
+; GFX6-NEXT:    s_and_b64 s[12:13], s[12:13], exec
+; GFX6-NEXT:    v_cvt_i32_f32_e32 v2, v2
 ; GFX6-NEXT:    s_cselect_b32 s5, s5, 0
 ; GFX6-NEXT:    s_bfe_i32 s4, s4, 0xf000f
-; GFX6-NEXT:    v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: ...
[truncated]

arsenm · 2025-02-28T04:44:11Z

llvm/test/CodeGen/AMDGPU/call-argument-types.ll

+; CI-NEXT:    v_lshr_b64 v[1:2], v[3:4], 16
+; CI-NEXT:    v_mov_b32_e32 v0, v3
+; CI-NEXT:    v_mov_b32_e32 v2, v4


Regression here. The uniformity is a flawed proxy for is SGPR

arsenm · 2025-02-28T04:44:56Z

llvm/lib/Target/AMDGPU/SIInstructions.td


-def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
+def : GCNPat<(i32 (DivergentUnaryFrag<trunc> (srl i64:$src0, (and i32:$src1, (i32 31))))),
          (V_ALIGNBIT_B32_e64 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),


Why do we handle these as patterns? Can we do this as a combine to produce fshr?

jrbyrnes · 2025-08-15T17:11:40Z

ping due to increased importance.

The uniformity is a flawed proxy for is SGPR

Should this really block using it in the tablegen pattern (i.e. the path to solving the regressions is to improve uniformity analysis)? We use it everywhere else basically.

llvm/lib/Target/AMDGPU/SIInstructions.td

jayfoad

Not disputing the other reviewers' comments, but I think this patch is a step in the right direction.

Change-Id: I72dcb47e39e5d78a0d770ddcd0fa1023b30a0eab

Change-Id: Ie919caa7e7707ab3102f2e352eae83edbb27dbf5

Selecting vgpr for the uniform version of this pattern may lead to unnecessary vgpr and waterfall loops.

jrbyrnes requested a review from arsenm February 28, 2025 00:10

llvmbot added the backend:AMDGPU label Feb 28, 2025

arsenm reviewed Feb 28, 2025

View reviewed changes

Sisyph mentioned this pull request Mar 4, 2025

[AMDGPU][True16][CodeGen] fshr true16 pattern #129085

Merged

arsenm mentioned this pull request Apr 5, 2025

[AMDGPU] illegal VGPR to SGPR copy #134463

Open

broxigarchen reviewed Aug 15, 2025

View reviewed changes

llvm/lib/Target/AMDGPU/SIInstructions.td Show resolved Hide resolved

jayfoad approved these changes Aug 17, 2025

View reviewed changes

jayfoad mentioned this pull request Sep 24, 2025

[AMDGPU] Optimize rotate instruction selection patterns #143551

Open

jrbyrnes force-pushed the UnifAB branch from 7767fc6 to e4526ad Compare September 25, 2025 22:34

jrbyrnes added 2 commits September 26, 2025 09:32

Precommit autogen checks

47c7afd

Change-Id: I72dcb47e39e5d78a0d770ddcd0fa1023b30a0eab

[AMDGPU] Ensure divergence for trunc -> v_alignbit pattern

cbab59c

Change-Id: Ie919caa7e7707ab3102f2e352eae83edbb27dbf5

jrbyrnes force-pushed the UnifAB branch from e4526ad to cbab59c Compare September 26, 2025 16:56

Merge branch 'main' into UnifAB

3adcdd0

jrbyrnes merged commit b427182 into llvm:main Sep 26, 2025
9 checks passed

mahesh-attarde pushed a commit to mahesh-attarde/llvm-project that referenced this pull request Oct 3, 2025

[AMDGPU] Ensure divergence for v_alignbit (llvm#129159)

affeeab

Selecting vgpr for the uniform version of this pattern may lead to unnecessary vgpr and waterfall loops.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[AMDGPU] Ensure divergence for v_alignbit #129159

[AMDGPU] Ensure divergence for v_alignbit #129159

Uh oh!

jrbyrnes commented Feb 28, 2025

Uh oh!

llvmbot commented Feb 28, 2025

Uh oh!

arsenm Feb 28, 2025

Uh oh!

arsenm Feb 28, 2025

Uh oh!

jrbyrnes commented Aug 15, 2025 •

edited

Loading

Uh oh!

Uh oh!

jayfoad left a comment

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

5 participants

[AMDGPU] Ensure divergence for v_alignbit #129159

[AMDGPU] Ensure divergence for v_alignbit #129159

Uh oh!

Conversation

jrbyrnes commented Feb 28, 2025

Uh oh!

llvmbot commented Feb 28, 2025

Uh oh!

arsenm Feb 28, 2025

Choose a reason for hiding this comment

Uh oh!

arsenm Feb 28, 2025

Choose a reason for hiding this comment

Uh oh!

jrbyrnes commented Aug 15, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

Uh oh!

jayfoad left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

5 participants

jrbyrnes commented Aug 15, 2025 •

edited

Loading