Skip to content

AMDGPU: Add agpr versions of global return atomics #156890

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 4, 2025
Merged

AMDGPU: Add agpr versions of global return atomics #156890

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
522784f
AMDGPU: Fix mis-selecting saddr flat atomics on gfx9
arsenm Sep 4, 2025
0b03d7f
AMDGPU: Add more tests for flat/global atomicrmw with agprs
arsenm Sep 4, 2025
c0b9ff9
AMDGPU: Add agpr versions of global return atomics
arsenm Sep 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions llvm/lib/Target/AMDGPU/AMDGPU.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,15 @@ def FeatureFlatInstOffsets : SubtargetFeature<"flat-inst-offsets",
def FeatureFlatGlobalInsts : SubtargetFeature<"flat-global-insts",
"FlatGlobalInsts",
"true",
"Have global_* flat memory instructions"
"Have global_* flat memory instructions",
[FeatureFlatAddressSpace]
>;

def FeatureFlatScratchInsts : SubtargetFeature<"flat-scratch-insts",
"FlatScratchInsts",
"true",
"Have scratch_* flat memory instructions"
"Have scratch_* flat memory instructions",
[FeatureFlatAddressSpace]
>;

def FeatureScalarFlatScratchInsts : SubtargetFeature<"scalar-flat-scratch-insts",
Expand All @@ -92,7 +94,8 @@ def FeatureEnableFlatScratch : SubtargetFeature<"enable-flat-scratch",
def FeatureFlatGVSMode : SubtargetFeature<"flat-gvs-mode",
"FlatGVSMode",
"true",
"Have GVS addressing mode with flat_* instructions"
"Have GVS addressing mode with flat_* instructions",
[FeatureFlatAddressSpace]
>;

def FeatureAddNoCarryInsts : SubtargetFeature<"add-no-carry-insts",
Expand Down Expand Up @@ -934,13 +937,15 @@ def FeatureAtomicFMinFMaxF64GlobalInsts : SubtargetFeature<"atomic-fmin-fmax-glo
def FeatureAtomicFMinFMaxF32FlatInsts : SubtargetFeature<"atomic-fmin-fmax-flat-f32",
"HasAtomicFMinFMaxF32FlatInsts",
"true",
"Has flat memory instructions for atomicrmw fmin/fmax for float"
"Has flat memory instructions for atomicrmw fmin/fmax for float",
[FeatureFlatAddressSpace]
>;

def FeatureAtomicFMinFMaxF64FlatInsts : SubtargetFeature<"atomic-fmin-fmax-flat-f64",
"HasAtomicFMinFMaxF64FlatInsts",
"true",
"Has flat memory instructions for atomicrmw fmin/fmax for double"
"Has flat memory instructions for atomicrmw fmin/fmax for double",
[FeatureFlatAddressSpace]
>;

def FeatureAtomicFaddNoRtnInsts : SubtargetFeature<"atomic-fadd-no-rtn-insts",
Expand Down Expand Up @@ -992,7 +997,8 @@ def FeatureFlatAtomicFaddF32Inst
: SubtargetFeature<"flat-atomic-fadd-f32-inst",
"HasFlatAtomicFaddF32Inst",
"true",
"Has flat_atomic_add_f32 instruction"
"Has flat_atomic_add_f32 instruction",
[FeatureFlatAddressSpace]
>;

def FeatureFlatBufferGlobalAtomicFaddF64Inst
Expand Down
24 changes: 4 additions & 20 deletions llvm/lib/Target/AMDGPU/DSInstructions.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,22 +51,6 @@ class DS_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> patt
let Uses = !if(has_m0_read, [M0, EXEC], [EXEC]);
}

class DstOperandIsAV<dag OperandList> {
bit ret = OperandIsAV<!getdagarg<DAGOperand>(OperandList, "vdst")>.ret;
}

class DstOperandIsAGPR<dag OperandList> {
bit ret = OperandIsAGPR<!getdagarg<DAGOperand>(OperandList, "vdst")>.ret;
}

class DataOperandIsAV<dag OperandList> {
bit ret = OperandIsAV<!getdagarg<DAGOperand>(OperandList, "data0")>.ret;
}

class DataOperandIsAGPR<dag OperandList> {
bit ret = OperandIsAGPR<!getdagarg<DAGOperand>(OperandList, "data0")>.ret;
}

class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> :
InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands>,
Enc64 {
Expand Down Expand Up @@ -115,13 +99,13 @@ class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> :
// register fields are only 8-bit, so data operands must all be AGPR
// or VGPR.
defvar DstOpIsAV = !if(ps.has_vdst,
DstOperandIsAV<ps.OutOperandList>.ret, 0);
VDstOperandIsAV<ps.OutOperandList>.ret, 0);
defvar DstOpIsAGPR = !if(ps.has_vdst,
DstOperandIsAGPR<ps.OutOperandList>.ret, 0);
VDstOperandIsAGPR<ps.OutOperandList>.ret, 0);
defvar DataOpIsAV = !if(!or(ps.has_data0, ps.has_gws_data0),
DataOperandIsAV<ps.InOperandList>.ret, 0);
Data0OperandIsAV<ps.InOperandList>.ret, 0);
defvar DataOpIsAGPR = !if(!or(ps.has_data0, ps.has_gws_data0),
DataOperandIsAGPR<ps.InOperandList>.ret, 0);
Data0OperandIsAGPR<ps.InOperandList>.ret, 0);

bits<1> acc = !if(ps.has_vdst,
!if(DstOpIsAV, vdst{9}, DstOpIsAGPR),
Expand Down
Loading