diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index c5e8f95748cf1..887092182f7d1 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4572,19 +4572,24 @@ static bool compareMachineOp(const MachineOperand &Op0, } } -bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, - const MachineOperand &MO) const { - const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; - - assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); - +bool SIInstrInfo::isLiteralOperandLegal(const MCInstrDesc &InstDesc, + const MCOperandInfo &OpInfo) const { if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE) return true; - if (OpInfo.RegClass < 0) + if (!RI.opCanUseLiteralConstant(OpInfo.OperandType)) return false; - if (MO.isImm() && isInlineConstant(MO, OpInfo)) { + if (!isVOP3(InstDesc) || !AMDGPU::isSISrcOperand(OpInfo)) + return true; + + return ST.hasVOP3Literal(); +} + +bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, + int64_t ImmVal) const { + const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; + if (isInlineConstant(ImmVal, OpInfo.OperandType)) { if (isMAI(InstDesc) && ST.hasMFMAInlineLiteralBug() && OpNo == (unsigned)AMDGPU::getNamedOperandIdx(InstDesc.getOpcode(), AMDGPU::OpName::src2)) @@ -4592,13 +4597,18 @@ bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, return RI.opCanUseInlineConstant(OpInfo.OperandType); } - if (!RI.opCanUseLiteralConstant(OpInfo.OperandType)) - return false; + return isLiteralOperandLegal(InstDesc, OpInfo); +} - if (!isVOP3(InstDesc) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) - return true; +bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, + const MachineOperand &MO) const { + if (MO.isImm()) + return isImmOperandLegal(InstDesc, OpNo, MO.getImm()); - return ST.hasVOP3Literal(); + assert((MO.isTargetIndex() || MO.isFI() || MO.isGlobal()) && + "unexpected imm-like operand kind"); + const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; + return isLiteralOperandLegal(InstDesc, OpInfo); } bool SIInstrInfo::isLegalAV64PseudoImm(uint64_t Imm) const { @@ -4758,6 +4768,31 @@ MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI, return Inst32; } +bool SIInstrInfo::physRegUsesConstantBus(const MachineOperand &RegOp) const { + // Null is free + Register Reg = RegOp.getReg(); + if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64) + return false; + + // SGPRs use the constant bus + + // FIXME: implicit registers that are not part of the MCInstrDesc's implicit + // physical register operands should also count, except for exec. + if (RegOp.isImplicit()) + return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0; + + // SGPRs use the constant bus + return AMDGPU::SReg_32RegClass.contains(Reg) || + AMDGPU::SReg_64RegClass.contains(Reg); +} + +bool SIInstrInfo::regUsesConstantBus(const MachineOperand &RegOp, + const MachineRegisterInfo &MRI) const { + Register Reg = RegOp.getReg(); + return Reg.isVirtual() ? RI.isSGPRClass(MRI.getRegClass(Reg)) + : physRegUsesConstantBus(RegOp); +} + bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, const MachineOperand &MO, const MCOperandInfo &OpInfo) const { @@ -4765,23 +4800,9 @@ bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, if (!MO.isReg()) return !isInlineConstant(MO, OpInfo); - if (!MO.isUse()) - return false; - - if (MO.getReg().isVirtual()) - return RI.isSGPRClass(MRI.getRegClass(MO.getReg())); - - // Null is free - if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64) - return false; - - // SGPRs use the constant bus - if (MO.isImplicit()) { - return MO.getReg() == AMDGPU::M0 || MO.getReg() == AMDGPU::VCC || - MO.getReg() == AMDGPU::VCC_LO; - } - return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || - AMDGPU::SReg_64RegClass.contains(MO.getReg()); + Register Reg = MO.getReg(); + return Reg.isVirtual() ? RI.isSGPRClass(MRI.getRegClass(Reg)) + : physRegUsesConstantBus(MO); } static Register findImplicitSGPRRead(const MachineInstr &MI) { @@ -6250,15 +6271,14 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx, continue; const MachineOperand &Op = MI.getOperand(i); if (Op.isReg()) { - RegSubRegPair SGPR(Op.getReg(), Op.getSubReg()); - if (!SGPRsUsed.count(SGPR) && - // FIXME: This can access off the end of the operands() array. - usesConstantBus(MRI, Op, InstDesc.operands().begin()[i])) { - if (--ConstantBusLimit <= 0) - return false; - SGPRsUsed.insert(SGPR); + if (Op.isUse()) { + RegSubRegPair SGPR(Op.getReg(), Op.getSubReg()); + if (regUsesConstantBus(Op, MRI) && SGPRsUsed.insert(SGPR).second) { + if (--ConstantBusLimit <= 0) + return false; + } } - } else if (AMDGPU::isSISrcOperand(InstDesc, i) && + } else if (AMDGPU::isSISrcOperand(InstDesc.operands()[i]) && !isInlineConstant(Op, InstDesc.operands()[i])) { // The same literal may be used multiple times. if (!UsedLiteral) diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index 958af0ff1147f..1070d4824aa14 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -1183,6 +1183,12 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, const MachineOperand &MO) const; + bool isLiteralOperandLegal(const MCInstrDesc &InstDesc, + const MCOperandInfo &OpInfo) const; + + bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, + int64_t ImmVal) const; + bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, const MachineOperand &MO) const { return isImmOperandLegal(MI.getDesc(), OpNo, MO); @@ -1195,6 +1201,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { /// This function will return false if you pass it a 32-bit instruction. bool hasVALU32BitEncoding(unsigned Opcode) const; + bool physRegUsesConstantBus(const MachineOperand &Reg) const; + bool regUsesConstantBus(const MachineOperand &Reg, + const MachineRegisterInfo &MRI) const; + /// Returns true if this operand uses the constant bus. bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineOperand &MO, diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 18ee9c16b3ff9..da19a6faa9e0f 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -2720,13 +2720,6 @@ bool isInlineValue(unsigned Reg) { #undef CASE_GFXPRE11_GFX11PLUS_TO #undef MAP_REG2REG -bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { - assert(OpNo < Desc.NumOperands); - unsigned OpType = Desc.operands()[OpNo].OperandType; - return OpType >= AMDGPU::OPERAND_SRC_FIRST && - OpType <= AMDGPU::OPERAND_SRC_LAST; -} - bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) { assert(OpNo < Desc.NumOperands); unsigned OpType = Desc.operands()[OpNo].OperandType; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 70dfb63cbe040..7c5c1e85b2014 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -1590,7 +1590,14 @@ bool isInlineValue(unsigned Reg); /// Is this an AMDGPU specific source operand? These include registers, /// inline constants, literals and mandatory literals (KImm). -bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo); +constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo) { + return OpInfo.OperandType >= AMDGPU::OPERAND_SRC_FIRST && + OpInfo.OperandType <= AMDGPU::OPERAND_SRC_LAST; +} + +constexpr bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { + return isSISrcOperand(Desc.operands()[OpNo]); +} /// Is this a KImm operand? bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo);