Skip to content

Commit e55172f

Browse files
ro-iro-i
authored
[AMDGPU] Classify FLAT instructions as VMEM (#137148)
Also adapt hazard and wait handling.
1 parent 2b05c7c commit e55172f

File tree

8 files changed

+45
-51
lines changed

8 files changed

+45
-51
lines changed

‎llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2409,17 +2409,15 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const {
24092409
Result = true;
24102410

24112411
else if (((SGMask & SchedGroupMask::VMEM) != SchedGroupMask::NONE) &&
2412-
(TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2412+
TII->isVMEM(MI))
24132413
Result = true;
24142414

24152415
else if (((SGMask & SchedGroupMask::VMEM_READ) != SchedGroupMask::NONE) &&
2416-
MI.mayLoad() &&
2417-
(TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2416+
MI.mayLoad() && TII->isVMEM(MI))
24182417
Result = true;
24192418

24202419
else if (((SGMask & SchedGroupMask::VMEM_WRITE) != SchedGroupMask::NONE) &&
2421-
MI.mayStore() &&
2422-
(TII->isVMEM(MI) || (TII->isFLAT(MI) && !TII->isDS(MI))))
2420+
MI.mayStore() && TII->isVMEM(MI))
24232421
Result = true;
24242422

24252423
else if (((SGMask & SchedGroupMask::DS) != SchedGroupMask::NONE) &&

‎llvm/lib/Target/AMDGPU/AMDGPUWaitSGPRHazards.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,9 @@ class AMDGPUWaitSGPRHazards {
232232
State.ActiveFlat = true;
233233

234234
// SMEM or VMEM clears hazards
235-
if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isSMRD(*MI)) {
235+
// FIXME: adapt to add FLAT without VALU (so !isLDSDMA())?
236+
if ((SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI)) ||
237+
SIInstrInfo::isSMRD(*MI)) {
236238
State.VCCHazard = HazardState::None;
237239
State.SALUHazards.reset();
238240
State.VALUHazards.reset();

‎llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp‎

Lines changed: 24 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -183,10 +183,7 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
183183
if (ST.hasNoDataDepHazard())
184184
return NoHazard;
185185

186-
// FIXME: Should flat be considered vmem?
187-
if ((SIInstrInfo::isVMEM(*MI) ||
188-
SIInstrInfo::isFLAT(*MI))
189-
&& checkVMEMHazards(MI) > 0)
186+
if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0)
190187
return HazardType;
191188

192189
if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
@@ -202,8 +199,8 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
202199
return HazardType;
203200

204201
if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
205-
SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
206-
SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
202+
SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) &&
203+
checkMAIVALUHazards(MI) > 0)
207204
return HazardType;
208205

209206
if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
@@ -229,9 +226,8 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
229226
if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
230227
return HazardType;
231228

232-
if ((SIInstrInfo::isVMEM(*MI) ||
233-
SIInstrInfo::isFLAT(*MI) ||
234-
SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
229+
if ((SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI)) &&
230+
checkMAILdStHazards(MI) > 0)
235231
return HazardType;
236232

237233
if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
@@ -324,7 +320,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
324320
if (ST.hasNoDataDepHazard())
325321
return WaitStates;
326322

327-
if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isFLAT(*MI))
323+
if (SIInstrInfo::isVMEM(*MI))
328324
WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
329325

330326
if (SIInstrInfo::isVALU(*MI))
@@ -340,8 +336,8 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
340336
WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
341337

342338
if ((SIInstrInfo::isVALU(*MI) || SIInstrInfo::isVMEM(*MI) ||
343-
SIInstrInfo::isFLAT(*MI) || SIInstrInfo::isDS(*MI) ||
344-
SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
339+
SIInstrInfo::isDS(*MI) || SIInstrInfo::isEXP(*MI)) &&
340+
checkMAIVALUHazards(MI) > 0)
345341
WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
346342

347343
if (MI->isInlineAsm())
@@ -369,9 +365,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
369365
if (SIInstrInfo::isMAI(*MI))
370366
return std::max(WaitStates, checkMAIHazards(MI));
371367

372-
if (SIInstrInfo::isVMEM(*MI) ||
373-
SIInstrInfo::isFLAT(*MI) ||
374-
SIInstrInfo::isDS(*MI))
368+
if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI))
375369
return std::max(WaitStates, checkMAILdStHazards(MI));
376370

377371
if (ST.hasGFX950Insts() && isPermlane(*MI))
@@ -598,7 +592,7 @@ static bool breaksSMEMSoftClause(MachineInstr *MI) {
598592
}
599593

600594
static bool breaksVMEMSoftClause(MachineInstr *MI) {
601-
return !SIInstrInfo::isVMEM(*MI) && !SIInstrInfo::isFLAT(*MI);
595+
return !SIInstrInfo::isVMEM(*MI);
602596
}
603597

604598
int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
@@ -1250,8 +1244,7 @@ bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
12501244
const SIRegisterInfo *TRI = ST.getRegisterInfo();
12511245

12521246
auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
1253-
if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I) &&
1254-
!SIInstrInfo::isFLAT(I))
1247+
if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I))
12551248
return false;
12561249

12571250
for (const MachineOperand &Def : MI->defs()) {
@@ -1425,8 +1418,8 @@ static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
14251418
for (auto &MBB : MF) {
14261419
for (auto &MI : MBB) {
14271420
HasLds |= SIInstrInfo::isDS(MI);
1428-
HasVmem |=
1429-
SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI);
1421+
HasVmem |= (SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) ||
1422+
SIInstrInfo::isSegmentSpecificFLAT(MI);
14301423
if (HasLds && HasVmem)
14311424
return true;
14321425
}
@@ -1450,7 +1443,8 @@ bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
14501443
auto IsHazardInst = [](const MachineInstr &MI) {
14511444
if (SIInstrInfo::isDS(MI))
14521445
return 1;
1453-
if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI))
1446+
if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) ||
1447+
SIInstrInfo::isSegmentSpecificFLAT(MI))
14541448
return 2;
14551449
return 0;
14561450
};
@@ -1517,8 +1511,8 @@ bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
15171511
if (WaitStates >= NoHazardWaitStates)
15181512
return true;
15191513
// Instructions which cause va_vdst==0 expire hazard
1520-
return SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1521-
SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I);
1514+
return SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) ||
1515+
SIInstrInfo::isEXP(I);
15221516
};
15231517
auto GetWaitStatesFn = [](const MachineInstr &MI) {
15241518
return SIInstrInfo::isVALU(MI) ? 1 : 0;
@@ -1549,8 +1543,7 @@ bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
15491543
const Register VDSTReg = VDST->getReg();
15501544

15511545
auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
1552-
if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I) &&
1553-
!SIInstrInfo::isDS(I))
1546+
if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I))
15541547
return false;
15551548
return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
15561549
};
@@ -1635,8 +1628,8 @@ bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
16351628
return HazardExpired;
16361629

16371630
// Instructions which cause va_vdst==0 expire hazard
1638-
if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1639-
SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) ||
1631+
if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) ||
1632+
SIInstrInfo::isEXP(I) ||
16401633
(I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
16411634
AMDGPU::DepCtr::decodeFieldVaVdst(I.getOperand(0).getImm()) == 0))
16421635
return HazardExpired;
@@ -1772,8 +1765,8 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
17721765
return HazardExpired;
17731766

17741767
// Instructions which cause va_vdst==0 expire hazard
1775-
if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isFLAT(I) ||
1776-
SIInstrInfo::isDS(I) || SIInstrInfo::isEXP(I) ||
1768+
if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) ||
1769+
SIInstrInfo::isEXP(I) ||
17771770
(I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
17781771
I.getOperand(0).getImm() == 0x0fff))
17791772
return HazardExpired;
@@ -2003,7 +1996,7 @@ int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
20031996
return 0;
20041997

20051998
auto IsHazardFn = [](const MachineInstr &I) {
2006-
if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isFLAT(I))
1999+
if (!SIInstrInfo::isVMEM(I))
20072000
return false;
20082001
return SIInstrInfo::isFPAtomic(I);
20092002
};
@@ -2625,9 +2618,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
26252618

26262619
int WaitStatesNeeded = 0;
26272620

2628-
bool IsMem = SIInstrInfo::isVMEM(*MI) ||
2629-
SIInstrInfo::isFLAT(*MI) ||
2630-
SIInstrInfo::isDS(*MI);
2621+
bool IsMem = SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI);
26312622
bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI);
26322623
bool IsVALU = SIInstrInfo::isVALU(*MI);
26332624

‎llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ void AMDGPUCustomBehaviour::generateWaitCntInfo() {
303303
bool AMDGPUCustomBehaviour::isVMEM(const MCInstrDesc &MCID) {
304304
return MCID.TSFlags & SIInstrFlags::MUBUF ||
305305
MCID.TSFlags & SIInstrFlags::MTBUF ||
306-
MCID.TSFlags & SIInstrFlags::MIMG;
306+
MCID.TSFlags & SIInstrFlags::MIMG || MCID.TSFlags & SIInstrFlags::FLAT;
307307
}
308308

309309
// taken from SIInstrInfo::hasModifiersSet()

‎llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ FunctionPass *llvm::createSIFormMemoryClausesLegacyPass() {
100100
}
101101

102102
static bool isVMEMClauseInst(const MachineInstr &MI) {
103-
return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI);
103+
return SIInstrInfo::isVMEM(MI);
104104
}
105105

106106
static bool isSMEMClauseInst(const MachineInstr &MI) {

‎llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ class SIInsertHardClauses {
9797
HardClauseType getHardClauseType(const MachineInstr &MI) {
9898
if (MI.mayLoad() || (MI.mayStore() && ST->shouldClusterStores())) {
9999
if (ST->getGeneration() == AMDGPUSubtarget::GFX10) {
100-
if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) {
100+
if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) ||
101+
SIInstrInfo::isSegmentSpecificFLAT(MI)) {
101102
if (ST->hasNSAClauseBug()) {
102103
const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode());
103104
if (Info && Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA)
@@ -121,7 +122,8 @@ class SIInsertHardClauses {
121122
: HARDCLAUSE_MIMG_LOAD
122123
: HARDCLAUSE_MIMG_STORE;
123124
}
124-
if (SIInstrInfo::isVMEM(MI) || SIInstrInfo::isSegmentSpecificFLAT(MI)) {
125+
if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) ||
126+
SIInstrInfo::isSegmentSpecificFLAT(MI)) {
125127
return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_VMEM_ATOMIC
126128
: HARDCLAUSE_VMEM_LOAD
127129
: HARDCLAUSE_VMEM_STORE;

‎llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,8 @@ static const unsigned instrsForExtendedCounterTypes[NUM_EXTENDED_INST_CNTS] = {
168168
AMDGPU::S_WAIT_KMCNT};
169169

170170
static bool updateVMCntOnly(const MachineInstr &Inst) {
171-
return SIInstrInfo::isVMEM(Inst) || SIInstrInfo::isFLATGlobal(Inst) ||
172-
SIInstrInfo::isFLATScratch(Inst);
171+
return (SIInstrInfo::isVMEM(Inst) && !SIInstrInfo::isFLAT(Inst)) ||
172+
SIInstrInfo::isFLATGlobal(Inst) || SIInstrInfo::isFLATScratch(Inst);
173173
}
174174

175175
#ifndef NDEBUG
@@ -695,8 +695,8 @@ class SIInsertWaitcnts {
695695
#endif // NDEBUG
696696
}
697697

698-
// Return the appropriate VMEM_*_ACCESS type for Inst, which must be a VMEM or
699-
// FLAT instruction.
698+
// Return the appropriate VMEM_*_ACCESS type for Inst, which must be a VMEM
699+
// instruction.
700700
WaitEventType getVmemWaitEventType(const MachineInstr &Inst) const {
701701
switch (Inst.getOpcode()) {
702702
case AMDGPU::GLOBAL_INV:
@@ -712,7 +712,7 @@ class SIInsertWaitcnts {
712712
static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
713713
VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
714714

715-
assert(SIInstrInfo::isVMEM(Inst) || SIInstrInfo::isFLAT(Inst));
715+
assert(SIInstrInfo::isVMEM(Inst));
716716
// LDS DMA loads are also stores, but on the LDS side. On the VMEM side
717717
// these should use VM_CNT.
718718
if (!ST->hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
@@ -2466,8 +2466,9 @@ bool SIInsertWaitcnts::isPreheaderToFlush(
24662466
}
24672467

24682468
bool SIInsertWaitcnts::isVMEMOrFlatVMEM(const MachineInstr &MI) const {
2469-
return SIInstrInfo::isVMEM(MI) ||
2470-
(SIInstrInfo::isFLAT(MI) && mayAccessVMEMThroughFlat(MI));
2469+
if (SIInstrInfo::isFLAT(MI))
2470+
return mayAccessVMEMThroughFlat(MI);
2471+
return SIInstrInfo::isVMEM(MI);
24712472
}
24722473

24732474
// Return true if it is better to flush the vmcnt counter in the preheader of

‎llvm/lib/Target/AMDGPU/SIInstrInfo.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,7 +449,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
449449
}
450450

451451
static bool isVMEM(const MachineInstr &MI) {
452-
return isMUBUF(MI) || isMTBUF(MI) || isImage(MI);
452+
return isMUBUF(MI) || isMTBUF(MI) || isImage(MI) || isFLAT(MI);
453453
}
454454

455455
bool isVMEM(uint16_t Opcode) const {

0 commit comments

Comments
 (0)