Skip to content

[AMDGPU][MISched] Allow memory ops of different base pointers to be clustered #140674

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

[AMDGPU][MISched] Allow memory ops of different base pointers to be clustered #140674

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
584a643
[AMDGPU][MISched] Allow memory ops of different base pointers to be c…
choikwa May 12, 2025
b453d6c
Fix lit test failures, remove no-cluster test since we now expect to …
choikwa May 20, 2025
608b002
NFC formatting
choikwa May 20, 2025
8dba56c
Addressed feedback: Change to Enable, Reject scalar and vector cluste…
choikwa May 21, 2025
952d5f7
Add flag test .ll file. Couldn't get a contrived .mir testcase to not…
choikwa May 22, 2025
a05a360
NFC formatting
choikwa May 22, 2025
0f00dab
use isVMEM & isSMRD
choikwa May 22, 2025
031db99
use != and isVMEM only
choikwa May 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ namespace llvm::AMDGPU {
#include "AMDGPUGenSearchableTables.inc"
} // namespace llvm::AMDGPU

static cl::opt<bool> EnableDiffBasePtrMemClustering(
"amdgpu-enable-diff-baseptr-mem-clustering",
cl::desc("Enable clustering memory ops with different base pointers"),
cl::init(true), cl::Hidden);

// Must be at least 4 to be able to branch over minimum unconditional branch
// code. This is only for making it possible to write reasonably small tests for
// long branches.
Expand Down Expand Up @@ -522,6 +527,22 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
return false;
}

static bool memOpsHaveSameAddrspace(const MachineInstr &MI1,
ArrayRef<const MachineOperand *> BaseOps1,
const MachineInstr &MI2,
ArrayRef<const MachineOperand *> BaseOps2) {
// If base is identical, assume identical addrspace
if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
return true;

if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
return false;

auto *MO1 = *MI1.memoperands_begin();
auto *MO2 = *MI2.memoperands_begin();
return MO1->getAddrSpace() == MO2->getAddrSpace();
}

static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
ArrayRef<const MachineOperand *> BaseOps1,
const MachineInstr &MI2,
Expand Down Expand Up @@ -559,14 +580,33 @@ bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
int64_t Offset2, bool OffsetIsScalable2,
unsigned ClusterSize,
unsigned NumBytes) const {
// If the mem ops (to be clustered) do not have the same base ptr, then they
// should not be clustered
unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
if (!BaseOps1.empty() && !BaseOps2.empty()) {
const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
return false;

if (EnableDiffBasePtrMemClustering) {
// Only consider memory ops from same addrspace for clustering
if (!memOpsHaveSameAddrspace(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
return false;

// Don't cluster scalar and vecter memory ops
const MachineFunction &MF = *FirstLdSt.getParent()->getParent();
const MachineRegisterInfo &MRI = MF.getRegInfo();
if (FirstLdSt.getOperand(0).isReg() && SecondLdSt.getOperand(0).isReg()) {
bool isFirstVecReg =
RI.isVectorRegister(MRI, FirstLdSt.getOperand(0).getReg());
bool isSecondVecReg =
RI.isVectorRegister(MRI, SecondLdSt.getOperand(0).getReg());
if (isFirstVecReg ^ isSecondVecReg)
return false;
}
} else {
// If the mem ops (to be clustered) do not have the same base ptr, then
// they should not be clustered
if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
return false;
}

const SIMachineFunctionInfo *MFI =
FirstLdSt.getMF()->getInfo<SIMachineFunctionInfo>();
Expand Down
228 changes: 114 additions & 114 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1030-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX1030-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1030-NEXT: s_clause 0x1
; GFX1030-NEXT: flat_load_dword v0, v[0:1]
; GFX1030-NEXT: flat_load_dword v1, v[2:3]
; GFX1030-NEXT: v_mov_b32_e32 v2, 0
Expand Down Expand Up @@ -674,6 +675,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
; GFX1013-NEXT: v_add_co_u32 v2, vcc_lo, v2, v6
; GFX1013-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX1013-NEXT: v_mov_b32_e32 v6, 4.0
; GFX1013-NEXT: s_clause 0x1
; GFX1013-NEXT: flat_load_dword v0, v[4:5]
; GFX1013-NEXT: flat_load_dword v1, v[2:3]
; GFX1013-NEXT: v_mov_b32_e32 v2, 0
Expand Down Expand Up @@ -711,6 +713,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: flat_load_b32 v9, v[0:1]
; GFX11-NEXT: flat_load_b32 v10, v[2:3]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s8
Expand Down Expand Up @@ -757,6 +760,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1030-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX1030-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1030-NEXT: s_clause 0x1
; GFX1030-NEXT: flat_load_dword v0, v[0:1]
; GFX1030-NEXT: flat_load_dword v1, v[2:3]
; GFX1030-NEXT: v_mov_b32_e32 v2, 0
Expand All @@ -783,6 +787,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1013-NEXT: v_add_co_u32 v2, vcc_lo, v2, v6
; GFX1013-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
; GFX1013-NEXT: v_mov_b32_e32 v6, 0x46004500
; GFX1013-NEXT: s_clause 0x1
; GFX1013-NEXT: flat_load_dword v0, v[4:5]
; GFX1013-NEXT: flat_load_dword v1, v[2:3]
; GFX1013-NEXT: v_mov_b32_e32 v2, 0
Expand Down Expand Up @@ -816,6 +821,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX11-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: flat_load_b32 v6, v[0:1]
; GFX11-NEXT: flat_load_b32 v7, v[2:3]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s8
Expand Down
12 changes: 4 additions & 8 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_32.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,9 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_untied(<16 x half> %A.0, <16 x
; W32: ; %bb.0: ; %bb
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[44:51], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
; W32-NEXT: s_clause 0x3
; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: s_endpgm
Expand All @@ -102,10 +101,9 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_tied(<16 x half> %A.0, <16 x h
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
; W32-NEXT: v_wmma_f16_16x16x16_f16 v[44:51], v[0:7], v[8:15], v[44:51]
; W32-NEXT: s_clause 0x1
; W32-NEXT: s_clause 0x3
; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: s_endpgm
Expand Down Expand Up @@ -152,10 +150,9 @@ define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_untied(<16 x i16> %A.0, <16
; W32: ; %bb.0: ; %bb
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[44:51], v[0:7], v[8:15], v[32:39]
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_clause 0x1
; W32-NEXT: s_clause 0x3
; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: s_endpgm
Expand All @@ -177,10 +174,9 @@ define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_tied(<16 x i16> %A.0, <16 x
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:39], v[16:23], v[24:31], v[32:39]
; W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
; W32-NEXT: v_wmma_bf16_16x16x16_bf16 v[44:51], v[0:7], v[8:15], v[44:51]
; W32-NEXT: s_clause 0x1
; W32-NEXT: s_clause 0x3
; W32-NEXT: global_store_b128 v[40:41], v[44:47], off
; W32-NEXT: global_store_b128 v[40:41], v[48:51], off offset:16
; W32-NEXT: s_clause 0x1
; W32-NEXT: global_store_b128 v[42:43], v[32:35], off
; W32-NEXT: global_store_b128 v[42:43], v[36:39], off offset:16
; W32-NEXT: s_endpgm
Expand Down
4 changes: 4 additions & 0 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wmma_64.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_untied(<16 x half> %A.0, <16 x
; W64: ; %bb.0: ; %bb
; W64-NEXT: v_wmma_f16_16x16x16_f16 v[40:43], v[0:7], v[8:15], v[32:35]
; W64-NEXT: v_wmma_f16_16x16x16_f16 v[32:35], v[16:23], v[24:31], v[32:35]
; W64-NEXT: s_clause 0x1
; W64-NEXT: global_store_b128 v[36:37], v[40:43], off
; W64-NEXT: global_store_b128 v[38:39], v[32:35], off
; W64-NEXT: s_endpgm
Expand All @@ -90,6 +91,7 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_tied(<16 x half> %A.0, <16 x h
; W64-NEXT: v_wmma_f16_16x16x16_f16 v[32:35], v[16:23], v[24:31], v[32:35]
; W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; W64-NEXT: v_wmma_f16_16x16x16_f16 v[40:43], v[0:7], v[8:15], v[40:43]
; W64-NEXT: s_clause 0x1
; W64-NEXT: global_store_b128 v[36:37], v[40:43], off
; W64-NEXT: global_store_b128 v[38:39], v[32:35], off
; W64-NEXT: s_endpgm
Expand Down Expand Up @@ -132,6 +134,7 @@ define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_untied(<16 x i16> %A.0, <16
; W64: ; %bb.0: ; %bb
; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[40:43], v[0:7], v[8:15], v[32:35]
; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:35], v[16:23], v[24:31], v[32:35]
; W64-NEXT: s_clause 0x1
; W64-NEXT: global_store_b128 v[36:37], v[40:43], off
; W64-NEXT: global_store_b128 v[38:39], v[32:35], off
; W64-NEXT: s_endpgm
Expand All @@ -153,6 +156,7 @@ define amdgpu_ps void @test_wmma_bf16_16x16x16_bf16_tied(<16 x i16> %A.0, <16 x
; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[32:35], v[16:23], v[24:31], v[32:35]
; W64-NEXT: s_delay_alu instid0(VALU_DEP_2)
; W64-NEXT: v_wmma_bf16_16x16x16_bf16 v[40:43], v[0:7], v[8:15], v[40:43]
; W64-NEXT: s_clause 0x1
; W64-NEXT: global_store_b128 v[36:37], v[40:43], off
; W64-NEXT: global_store_b128 v[38:39], v[32:35], off
; W64-NEXT: s_endpgm
Expand Down
8 changes: 4 additions & 4 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/localizer.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,17 +126,17 @@ define amdgpu_kernel void @localize_globals(i1 %cond) {
; GFX9-NEXT: s_getpc_b64 s[0:1]
; GFX9-NEXT: s_add_u32 s0, s0, gv0@gotpcrel32@lo+4
; GFX9-NEXT: s_addc_u32 s1, s1, gv0@gotpcrel32@hi+12
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX9-NEXT: s_getpc_b64 s[2:3]
; GFX9-NEXT: s_add_u32 s2, s2, gv1@gotpcrel32@lo+4
; GFX9-NEXT: s_addc_u32 s3, s3, gv1@gotpcrel32@hi+12
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v0, v0, s[0:1]
; GFX9-NEXT: global_store_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
; GFX9-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: .LBB1_4: ; %bb2
; GFX9-NEXT: s_endpgm
Expand Down
4 changes: 4 additions & 0 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/mul-known-bits.i64.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ define amdgpu_kernel void @v_mul_i64_zext_src1(ptr addrspace(1) %out, ptr addrsp
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
; GFX10-NEXT: global_load_dword v4, v3, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
Expand All @@ -82,6 +83,7 @@ define amdgpu_kernel void @v_mul_i64_zext_src1(ptr addrspace(1) %out, ptr addrsp
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_b64 v[0:1], v1, s[2:3]
; GFX11-NEXT: global_load_b32 v5, v2, s[4:5]
; GFX11-NEXT: s_waitcnt vmcnt(0)
Expand Down Expand Up @@ -113,6 +115,7 @@ define amdgpu_kernel void @v_mul_i64_zext_src0(ptr addrspace(1) %out, ptr addrsp
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: global_load_dword v4, v2, s[2:3]
; GFX10-NEXT: global_load_dwordx2 v[0:1], v3, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
Expand All @@ -134,6 +137,7 @@ define amdgpu_kernel void @v_mul_i64_zext_src0(ptr addrspace(1) %out, ptr addrsp
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_b32 v5, v1, s[2:3]
; GFX11-NEXT: global_load_b64 v[0:1], v0, s[4:5]
; GFX11-NEXT: s_waitcnt vmcnt(0)
Expand Down
26 changes: 10 additions & 16 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/wmma-gfx12-w32-swmmac-index_key.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,9 @@ define amdgpu_ps void @test_swmmac_f32_16x16x32_f16_index_key(<8 x half> %A, <16
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[26:33], v[0:3], v[4:11], v20
; GFX12-NEXT: v_swmmac_f32_16x16x32_f16 v[12:19], v[0:3], v[4:11], v20 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[22:23], v[26:29], off
; GFX12-NEXT: global_store_b128 v[22:23], v[30:33], off offset:16
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[24:25], v[12:15], off
; GFX12-NEXT: global_store_b128 v[24:25], v[16:19], off offset:16
; GFX12-NEXT: s_endpgm
Expand All @@ -43,10 +42,9 @@ define amdgpu_ps void @test_swmmac_f32_16x16x32_bf16_index_key(<8 x i16> %A, <16
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_f32_16x16x32_bf16 v[26:33], v[0:3], v[4:11], v20
; GFX12-NEXT: v_swmmac_f32_16x16x32_bf16 v[12:19], v[0:3], v[4:11], v20 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[22:23], v[26:29], off
; GFX12-NEXT: global_store_b128 v[22:23], v[30:33], off offset:16
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[24:25], v[12:15], off
; GFX12-NEXT: global_store_b128 v[24:25], v[16:19], off offset:16
; GFX12-NEXT: s_endpgm
Expand All @@ -71,6 +69,7 @@ define amdgpu_ps void @test_swmmac_f16_16x16x32_f16_index_key(<8 x half> %A, <16
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[22:25], v[0:3], v[4:11], v16
; GFX12-NEXT: v_swmmac_f16_16x16x32_f16 v[12:15], v[0:3], v[4:11], v16 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[18:19], v[22:25], off
; GFX12-NEXT: global_store_b128 v[20:21], v[12:15], off
; GFX12-NEXT: s_endpgm
Expand All @@ -95,6 +94,7 @@ define amdgpu_ps void @test_swmmac_bf16_16x16x32_bf16_index_key(<8 x i16> %A, <1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_bf16_16x16x32_bf16 v[22:25], v[0:3], v[4:11], v16
; GFX12-NEXT: v_swmmac_bf16_16x16x32_bf16 v[12:15], v[0:3], v[4:11], v16 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[18:19], v[22:25], off
; GFX12-NEXT: global_store_b128 v[20:21], v[12:15], off
; GFX12-NEXT: s_endpgm
Expand All @@ -121,10 +121,9 @@ define amdgpu_ps void @test_swmmac_i32_16x16x32_iu8_index_key(<2 x i32> %A, <4 x
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_i32_16x16x32_iu8 v[20:27], v[0:1], v[2:5], v14
; GFX12-NEXT: v_swmmac_i32_16x16x32_iu8 v[6:13], v[0:1], v[2:5], v14 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[16:17], v[20:23], off
; GFX12-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[18:19], v[6:9], off
; GFX12-NEXT: global_store_b128 v[18:19], v[10:13], off offset:16
; GFX12-NEXT: s_endpgm
Expand All @@ -151,10 +150,9 @@ define amdgpu_ps void @test_swmmac_i32_16x16x32_iu4_index_key(i32 %A, <2 x i32>
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_i32_16x16x32_iu4 v[17:24], v0, v[1:2], v11
; GFX12-NEXT: v_swmmac_i32_16x16x32_iu4 v[3:10], v0, v[1:2], v11 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[13:14], v[17:20], off
; GFX12-NEXT: global_store_b128 v[13:14], v[21:24], off offset:16
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[15:16], v[3:6], off
; GFX12-NEXT: global_store_b128 v[15:16], v[7:10], off offset:16
; GFX12-NEXT: s_endpgm
Expand All @@ -181,10 +179,9 @@ define amdgpu_ps void @test_swmmac_f32_16x16x32_fp8_fp8_index_key(<2 x i32> %A,
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_fp8 v[20:27], v[0:1], v[2:5], v14
; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_fp8 v[6:13], v[0:1], v[2:5], v14 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[16:17], v[20:23], off
; GFX12-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[18:19], v[6:9], off
; GFX12-NEXT: global_store_b128 v[18:19], v[10:13], off offset:16
; GFX12-NEXT: s_endpgm
Expand All @@ -211,10 +208,9 @@ define amdgpu_ps void @test_swmmac_f32_16x16x32_fp8_bf8_index_key(<2 x i32> %A,
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_bf8 v[20:27], v[0:1], v[2:5], v14
; GFX12-NEXT: v_swmmac_f32_16x16x32_fp8_bf8 v[6:13], v[0:1], v[2:5], v14 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[16:17], v[20:23], off
; GFX12-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[18:19], v[6:9], off
; GFX12-NEXT: global_store_b128 v[18:19], v[10:13], off offset:16
; GFX12-NEXT: s_endpgm
Expand All @@ -241,10 +237,9 @@ define amdgpu_ps void @test_swmmac_f32_16x16x32_bf8_fp8_index_key(<2 x i32> %A,
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_fp8 v[20:27], v[0:1], v[2:5], v14
; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_fp8 v[6:13], v[0:1], v[2:5], v14 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[16:17], v[20:23], off
; GFX12-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[18:19], v[6:9], off
; GFX12-NEXT: global_store_b128 v[18:19], v[10:13], off offset:16
; GFX12-NEXT: s_endpgm
Expand All @@ -271,10 +266,9 @@ define amdgpu_ps void @test_swmmac_f32_16x16x32_bf8_bf8_index_key(<2 x i32> %A,
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_bf8 v[20:27], v[0:1], v[2:5], v14
; GFX12-NEXT: v_swmmac_f32_16x16x32_bf8_bf8 v[6:13], v[0:1], v[2:5], v14 index_key:1
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[16:17], v[20:23], off
; GFX12-NEXT: global_store_b128 v[16:17], v[24:27], off offset:16
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v[18:19], v[6:9], off
; GFX12-NEXT: global_store_b128 v[18:19], v[10:13], off offset:16
; GFX12-NEXT: s_endpgm
Expand Down
Loading
Loading