From 04733e9e9a3b7af47a45950a4d459281d6e9ee42 Mon Sep 17 00:00:00 2001 From: Stefan Stipanovic Date: Thu, 27 Feb 2025 20:22:23 +0100 Subject: [PATCH] [AMDGPU][NFC] Move isXDL and isDGEMM to SIInstrInfo --- .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 59 +++++++------------ llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 14 +++++ llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 ++ 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index 1ff75095b220a..582da42a0dc4e 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -111,25 +111,6 @@ static bool isSMovRel(unsigned Opcode) { } } -static bool isDGEMM(unsigned Opcode) { - return AMDGPU::getMAIIsDGEMM(Opcode); -} - -static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { - unsigned Opcode = MI.getOpcode(); - - if (!SIInstrInfo::isMAI(MI) || - isDGEMM(Opcode) || - Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || - Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) - return false; - - if (!ST.hasGFX940Insts()) - return true; - - return AMDGPU::getMAIIsGFX940XDL(Opcode); -} - static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI) { if (TII.isAlwaysGDS(MI.getOpcode())) @@ -2375,7 +2356,8 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { unsigned Opc1 = MI1->getOpcode(); int NeedWaitStates = 0; if (OpNo == SrcCIdx) { - if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) { + if (!SIInstrInfo::isDGEMM(Opc) && + (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) { NeedWaitStates = 0; } else if (FullReg) { if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 || @@ -2392,7 +2374,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64: case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: - if (!isXDL(ST, *MI)) + if (!TII.isXDL(*MI)) NeedWaitStates = ST.hasGFX950Insts() ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates @@ -2400,18 +2382,18 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { break; case AMDGPU::V_MFMA_F64_4X4X4F64_e64: case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64: - if (!isXDL(ST, *MI)) + if (!TII.isXDL(*MI)) NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates; break; default: int NumPasses = TSchedModel.computeInstrLatency(MI1); if (ST.hasGFX940Insts()) { - if (isXDL(ST, *MI) && !isXDL(ST, *MI1)) + if (TII.isXDL(*MI) && !TII.isXDL(*MI1)) break; NeedWaitStates = - isXDL(ST, *MI1) - ? (isXDL(ST, *MI) + TII.isXDL(*MI1) + ? (TII.isXDL(*MI) ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates( @@ -2424,18 +2406,19 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { switch (NumPasses) { case 2: NeedWaitStates = - isDGEMM(Opc) ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates - : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; + SIInstrInfo::isDGEMM(Opc) + ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates + : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates; break; case 8: NeedWaitStates = - isDGEMM(Opc) + SIInstrInfo::isDGEMM(Opc) ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates; break; case 16: NeedWaitStates = - isDGEMM(Opc) + SIInstrInfo::isDGEMM(Opc) ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates; break; @@ -2464,7 +2447,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { if (ST.hasGFX940Insts()) { NeedWaitStates = - isXDL(ST, *MI1) + TII.isXDL(*MI1) ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates( @@ -2631,7 +2614,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { return 0; auto IsDGEMMFn = [](const MachineInstr &MI) -> bool { - return isDGEMM(MI.getOpcode()); + return SIInstrInfo::isDGEMM(MI.getOpcode()); }; // This is checked in checkMAIHazards90A() @@ -2670,7 +2653,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { bool DGEMMAfterVALUWrite = false; auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) { // Found DGEMM on reverse traversal to def. - if (isDGEMM(MI.getOpcode())) + if (SIInstrInfo::isDGEMM(MI.getOpcode())) DGEMMAfterVALUWrite = true; // Only hazard if register is defined by a VALU and a DGEMM is found after @@ -2745,7 +2728,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { int NumPasses = HazardDefLatency; int NeedWaitStates = MaxWaitStates; - if (isDGEMM(MFMA->getOpcode())) { + if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { switch (HazardDefLatency) { case 4: NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates @@ -2765,7 +2748,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { } } else if (ST.hasGFX940Insts()) { NeedWaitStates = - isXDL(ST, *MFMA) + TII.isXDL(*MFMA) ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates( @@ -2838,7 +2821,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { int NeedWaitStates = MaxWaitStates; int NumPasses = TSchedModel.computeInstrLatency(MFMA); - if (isDGEMM(MFMA->getOpcode())) { + if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) { switch (NumPasses) { case 4: NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates; @@ -2852,7 +2835,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { } } else if (ST.hasGFX940Insts()) { NeedWaitStates = - isXDL(ST, *MFMA) + TII.isXDL(*MFMA) ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates( NumPasses, ST.hasGFX950Insts()) : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses); @@ -2880,11 +2863,11 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { } auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) { - if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) || + if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) || !MI.readsRegister(Reg, &TRI)) return false; - if (ST.hasGFX940Insts() && !isXDL(ST, MI)) + if (ST.hasGFX940Insts() && !TII.isXDL(MI)) return false; const MachineOperand *SrcC = diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 9aec2bef0c18a..0face8108d249 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -10251,3 +10251,17 @@ bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const { return TargetInstrInfo::isGlobalMemoryObject(MI); } + +bool SIInstrInfo::isXDL(const MachineInstr &MI) const { + unsigned Opcode = MI.getOpcode(); + + if (!SIInstrInfo::isMAI(MI) || isDGEMM(Opcode) || + Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || + Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64) + return false; + + if (!ST.hasGFX940Insts()) + return true; + + return AMDGPU::getMAIIsGFX940XDL(Opcode); +} diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index 428322a5a2f04..88b1e477f13e4 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -847,6 +847,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { return get(Opcode).TSFlags & SIInstrFlags::IsDOT; } + bool isXDL(const MachineInstr &MI) const; + + static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opcode); } + static bool isLDSDIR(const MachineInstr &MI) { return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR; }