Skip to content

Commit 531c485

Browse files
sstipanosstipano
authored
[AMDGPU][NFC] Move isXDL and isDGEMM to SIInstrInfo. (#129103)
1 parent 28d7671 commit 531c485

File tree

3 files changed

+39
-38
lines changed

3 files changed

+39
-38
lines changed

‎llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp‎

Lines changed: 21 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -111,25 +111,6 @@ static bool isSMovRel(unsigned Opcode) {
111111
}
112112
}
113113

114-
static bool isDGEMM(unsigned Opcode) {
115-
return AMDGPU::getMAIIsDGEMM(Opcode);
116-
}
117-
118-
static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
119-
unsigned Opcode = MI.getOpcode();
120-
121-
if (!SIInstrInfo::isMAI(MI) ||
122-
isDGEMM(Opcode) ||
123-
Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
124-
Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
125-
return false;
126-
127-
if (!ST.hasGFX940Insts())
128-
return true;
129-
130-
return AMDGPU::getMAIIsGFX940XDL(Opcode);
131-
}
132-
133114
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
134115
const MachineInstr &MI) {
135116
if (TII.isAlwaysGDS(MI.getOpcode()))
@@ -2375,7 +2356,8 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
23752356
unsigned Opc1 = MI1->getOpcode();
23762357
int NeedWaitStates = 0;
23772358
if (OpNo == SrcCIdx) {
2378-
if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) {
2359+
if (!SIInstrInfo::isDGEMM(Opc) &&
2360+
(!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) {
23792361
NeedWaitStates = 0;
23802362
} else if (FullReg) {
23812363
if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
@@ -2392,26 +2374,26 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
23922374
case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
23932375
case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
23942376
case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
2395-
if (!isXDL(ST, *MI))
2377+
if (!TII.isXDL(*MI))
23962378
NeedWaitStates =
23972379
ST.hasGFX950Insts()
23982380
? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
23992381
: DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
24002382
break;
24012383
case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
24022384
case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
2403-
if (!isXDL(ST, *MI))
2385+
if (!TII.isXDL(*MI))
24042386
NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
24052387
break;
24062388
default:
24072389
int NumPasses = TSchedModel.computeInstrLatency(MI1);
24082390
if (ST.hasGFX940Insts()) {
2409-
if (isXDL(ST, *MI) && !isXDL(ST, *MI1))
2391+
if (TII.isXDL(*MI) && !TII.isXDL(*MI1))
24102392
break;
24112393

24122394
NeedWaitStates =
2413-
isXDL(ST, *MI1)
2414-
? (isXDL(ST, *MI)
2395+
TII.isXDL(*MI1)
2396+
? (TII.isXDL(*MI)
24152397
? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
24162398
NumPasses, ST.hasGFX950Insts())
24172399
: GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
@@ -2424,18 +2406,19 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
24242406
switch (NumPasses) {
24252407
case 2:
24262408
NeedWaitStates =
2427-
isDGEMM(Opc) ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
2428-
: SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
2409+
SIInstrInfo::isDGEMM(Opc)
2410+
? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
2411+
: SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
24292412
break;
24302413
case 8:
24312414
NeedWaitStates =
2432-
isDGEMM(Opc)
2415+
SIInstrInfo::isDGEMM(Opc)
24332416
? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
24342417
: SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
24352418
break;
24362419
case 16:
24372420
NeedWaitStates =
2438-
isDGEMM(Opc)
2421+
SIInstrInfo::isDGEMM(Opc)
24392422
? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
24402423
: SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
24412424
break;
@@ -2464,7 +2447,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
24642447

24652448
if (ST.hasGFX940Insts()) {
24662449
NeedWaitStates =
2467-
isXDL(ST, *MI1)
2450+
TII.isXDL(*MI1)
24682451
? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
24692452
NumPasses, ST.hasGFX950Insts())
24702453
: GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
@@ -2631,7 +2614,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
26312614
return 0;
26322615

26332616
auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
2634-
return isDGEMM(MI.getOpcode());
2617+
return SIInstrInfo::isDGEMM(MI.getOpcode());
26352618
};
26362619

26372620
// This is checked in checkMAIHazards90A()
@@ -2670,7 +2653,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
26702653
bool DGEMMAfterVALUWrite = false;
26712654
auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
26722655
// Found DGEMM on reverse traversal to def.
2673-
if (isDGEMM(MI.getOpcode()))
2656+
if (SIInstrInfo::isDGEMM(MI.getOpcode()))
26742657
DGEMMAfterVALUWrite = true;
26752658

26762659
// Only hazard if register is defined by a VALU and a DGEMM is found after
@@ -2745,7 +2728,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
27452728
int NumPasses = HazardDefLatency;
27462729
int NeedWaitStates = MaxWaitStates;
27472730

2748-
if (isDGEMM(MFMA->getOpcode())) {
2731+
if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
27492732
switch (HazardDefLatency) {
27502733
case 4:
27512734
NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
@@ -2765,7 +2748,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
27652748
}
27662749
} else if (ST.hasGFX940Insts()) {
27672750
NeedWaitStates =
2768-
isXDL(ST, *MFMA)
2751+
TII.isXDL(*MFMA)
27692752
? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
27702753
NumPasses, ST.hasGFX950Insts())
27712754
: GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
@@ -2838,7 +2821,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
28382821
int NeedWaitStates = MaxWaitStates;
28392822
int NumPasses = TSchedModel.computeInstrLatency(MFMA);
28402823

2841-
if (isDGEMM(MFMA->getOpcode())) {
2824+
if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
28422825
switch (NumPasses) {
28432826
case 4:
28442827
NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
@@ -2852,7 +2835,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
28522835
}
28532836
} else if (ST.hasGFX940Insts()) {
28542837
NeedWaitStates =
2855-
isXDL(ST, *MFMA)
2838+
TII.isXDL(*MFMA)
28562839
? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
28572840
NumPasses, ST.hasGFX950Insts())
28582841
: GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
@@ -2880,11 +2863,11 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
28802863
}
28812864

28822865
auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
2883-
if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) ||
2866+
if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) ||
28842867
!MI.readsRegister(Reg, &TRI))
28852868
return false;
28862869

2887-
if (ST.hasGFX940Insts() && !isXDL(ST, MI))
2870+
if (ST.hasGFX940Insts() && !TII.isXDL(MI))
28882871
return false;
28892872

28902873
const MachineOperand *SrcC =

‎llvm/lib/Target/AMDGPU/SIInstrInfo.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10251,3 +10251,17 @@ bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
1025110251

1025210252
return TargetInstrInfo::isGlobalMemoryObject(MI);
1025310253
}
10254+
10255+
bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
10256+
unsigned Opcode = MI.getOpcode();
10257+
10258+
if (!SIInstrInfo::isMAI(MI) || isDGEMM(Opcode) ||
10259+
Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
10260+
Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
10261+
return false;
10262+
10263+
if (!ST.hasGFX940Insts())
10264+
return true;
10265+
10266+
return AMDGPU::getMAIIsGFX940XDL(Opcode);
10267+
}

‎llvm/lib/Target/AMDGPU/SIInstrInfo.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
847847
return get(Opcode).TSFlags & SIInstrFlags::IsDOT;
848848
}
849849

850+
bool isXDL(const MachineInstr &MI) const;
851+
852+
static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opcode); }
853+
850854
static bool isLDSDIR(const MachineInstr &MI) {
851855
return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR;
852856
}

0 commit comments

Comments
 (0)