diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index 3e037affe1cfd..785570da45387 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -283,6 +283,8 @@ class RegisterClass regTypes, int alignment, // value means copying is extremely expensive or impossible. int CopyCost = 1; + RegisterClass CrossCopyRegClass = ?; + // MemberList - Specify which registers are in this class. If the // allocation_order_* method are not specified, this also defines the order of // allocation used by the register allocator. diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td index f3a962eea7539..f81ae2c475517 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td @@ -362,13 +362,6 @@ foreach Index = 0...255 in { // Groupings using register classes and tuples //===----------------------------------------------------------------------===// -def SCC_CLASS : SIRegisterClass<"AMDGPU", [i1], 1, (add SCC)> { - let CopyCost = -1; - let isAllocatable = 0; - let HasSGPR = 1; - let BaseClassOrder = 10000; -} - def M0_CLASS : SIRegisterClass<"AMDGPU", [i32], 32, (add M0)> { let CopyCost = 1; let isAllocatable = 0; @@ -801,6 +794,14 @@ def SReg_32_XM0 : SIRegisterClass<"AMDGPU", [i32, f32, i16, f16, bf16, v2i16, v2 } // End GeneratePressureSet = 0 +def SCC_CLASS : SIRegisterClass<"AMDGPU", [i1], 1, (add SCC)> { + let CopyCost = -1; + let isAllocatable = 0; + let CrossCopyRegClass = SReg_32_XM0_XEXEC; + let HasSGPR = 1; + let BaseClassOrder = 10000; +} + // Register class for all scalar registers (SGPRs + Special Registers) def SReg_32 : SIRegisterClass<"AMDGPU", [i32, f32, i16, f16, bf16, v2i16, v2f16, v2bf16, i1], 32, (add SReg_32_XM0, M0_CLASS)> { diff --git a/llvm/lib/Target/X86/X86InstrArithmetic.td b/llvm/lib/Target/X86/X86InstrArithmetic.td index 16ca2882a84da..63c20af1dac1d 100644 --- a/llvm/lib/Target/X86/X86InstrArithmetic.td +++ b/llvm/lib/Target/X86/X86InstrArithmetic.td @@ -64,16 +64,13 @@ class MulDivOpM o, Format f, string m, X86TypeInfo t, sched.ReadAfterFold, sched.ReadAfterFold]; } -multiclass Mul o, string m, Format RegMRM, Format MemMRM, SDPatternOperator node> { +multiclass Mul o, string m, Format RegMRM, Format MemMRM> { // AL is really implied by AX, but the registers in Defs must match the // SDNode results (i8, i32). // // FIXME: Used for 8-bit mul, ignore result upper 8 bits. - // This probably ought to be moved to a def : Pat<> if the - // syntax can be accepted. let Defs = [AL, EFLAGS, AX], Uses = [AL] in - def 8r : MulDivOpR; + def 8r : MulDivOpR; let Defs = [AX, DX, EFLAGS], Uses = [AX] in def 16r : MulDivOpR, OpSize16; let Defs = [EAX, EDX, EFLAGS], Uses = [EAX] in @@ -81,8 +78,7 @@ multiclass Mul o, string m, Format RegMRM, Format MemMRM, SDPatternOpera let Defs = [RAX, RDX, EFLAGS], Uses = [RAX] in def 64r : MulDivOpR; let Defs = [AL, EFLAGS, AX], Uses = [AL] in - def 8m : MulDivOpM; + def 8m : MulDivOpM; let Defs = [AX, DX, EFLAGS], Uses = [AX] in def 16m : MulDivOpM, OpSize16; let Defs = [EAX, EDX, EFLAGS], Uses = [EAX] in @@ -127,8 +123,14 @@ multiclass Mul o, string m, Format RegMRM, Format MemMRM, SDPatternOpera } } -defm MUL : Mul<0xF7, "mul", MRM4r, MRM4m, mul>; -defm IMUL : Mul<0xF7, "imul", MRM5r, MRM5m, null_frag>; +defm MUL : Mul<0xF7, "mul", MRM4r, MRM4m>; +defm IMUL : Mul<0xF7, "imul", MRM5r, MRM5m>; + +// These nodes are selected by custom C++ code. +let GISelShouldIgnore = true in { + def : Pat<(mul AL, i8:$src1), (MUL8r $src1)>; + def : Pat<(mul AL, (loadi8 addr:$src1)), (MUL8m addr:$src1)>; +} multiclass Div o, string m, Format RegMRM, Format MemMRM> { defvar sched8 = !if(!eq(m, "div"), WriteDiv8, WriteIDiv8); diff --git a/llvm/test/TableGen/Common/GlobalISelEmitterCommon.td b/llvm/test/TableGen/Common/GlobalISelEmitterCommon.td index 8f11fee375184..8f625811eb9b2 100644 --- a/llvm/test/TableGen/Common/GlobalISelEmitterCommon.td +++ b/llvm/test/TableGen/Common/GlobalISelEmitterCommon.td @@ -7,7 +7,8 @@ class MyTargetGenericInstruction : GenericInstruction { } def R0 : Register<"r0"> { let Namespace = "MyTarget"; } -def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0)>; +def R1 : Register<"r1"> { let Namespace = "MyTarget"; } +def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0, R1)>; def GPR32Op : RegisterOperand; def F0 : Register<"f0"> { let Namespace = "MyTarget"; } def FPR32 : RegisterClass<"MyTarget", [f32], 32, (add F0)>; diff --git a/llvm/test/TableGen/GlobalISelEmitter/implicit-defs.td b/llvm/test/TableGen/GlobalISelEmitter/implicit-defs.td index 06e5e39a68175..64f2330411ed8 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/implicit-defs.td +++ b/llvm/test/TableGen/GlobalISelEmitter/implicit-defs.td @@ -1,12 +1,75 @@ -// RUN: llvm-tblgen -gen-global-isel -warn-on-skipped-patterns -I %p/../../../include -I %p/../Common %s -o /dev/null 2>&1 | FileCheck %s --implicit-check-not="Skipped pattern" +// RUN: llvm-tblgen -gen-global-isel -optimize-match-table=false \ +// RUN: -I %p/../../../include -I %p/../Common %s include "llvm/Target/Target.td" include "GlobalISelEmitterCommon.td" -// CHECK: Skipped pattern: Pattern defines a physical register -let Uses = [B0], Defs = [B0] in -def tst1 : I<(outs), (ins), [(set B0, (add B0, 1))]>; +let Defs = [R0, B0] in +def tst1 : I<(outs), (ins), [(set R0, (i32 42))]>; -// CHECK: Skipped pattern: Src pattern result has 1 def(s) without the HasNoUse predicate set to true but Dst MI has no def -let Uses = [B0] in -def tst2 : I<(outs), (ins), [(set B0, (add B0, 1))]>; +let Defs = [R0, R1] in +def tst2 : I<(outs GPR32:$rd), (ins GPR32:$rs1, GPR32:$rs2), + [(set GPR32:$rd, R1, (sdivrem i32:$rs1, i32:$rs2))]>; + +let Defs = [R0, R1] in +def tst3 : I<(outs), (ins GPR32:$rs1, GPR32:$rs2), + [(set R1, R0, (udivrem i32:$rs1, i32:$rs2))]>; + +let Defs = [R0] in +def tst4 : I<(outs GPR32:$rd), (ins GPR32:$rs), []>; + +def : Pat<(sdivrem i32:$rs, 42), (tst4 (tst4 $rs))>; + +// CHECK-LABEL: // (sdivrem:{ *:[i32] }:{ *:[i32] } i32:{ *:[i32] }:$rs, 42:{ *:[i32] }) => (tst4:{ *:[i32] }:{ *:[i32] } (tst4:{ *:[i32] }:{ *:[i32] } ?:{ *:[i32] }:$rs)) +// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32, +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(MyTarget::tst4), +// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/2, /*OldInsnID*/0, /*OpIdx*/2, // rs +// CHECK-NEXT: GIR_SetImplicitDefDead, /*InsnID*/2, /*OpIdx for MyTarget::R0*/0, +// CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/2, +// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::tst4), +// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[rd] +// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // DstI[R0] +// CHECK-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(0), +// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, +// CHECK-NEXT: // GIR_Coverage, 3, +// CHECK-NEXT: GIR_EraseRootFromParent_Done, + +// CHECK-LABEL: // 42:{ *:[i32] } => (tst1:{ *:[i32] }) +// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::tst1), +// CHECK-NEXT: GIR_SetImplicitDefDead, /*InsnID*/0, /*OpIdx for MyTarget::B0*/1, +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // DstI[R0] +// CHECK-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(0), +// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, +// CHECK-NEXT: // GIR_Coverage, 0, +// CHECK-NEXT: GIR_EraseRootFromParent_Done, + +// CHECK-LABEL: // (sdivrem:{ *:[i32] }:{ *:[i32] } i32:{ *:[i32] }:$rs1, i32:{ *:[i32] }:$rs2) => (tst2:{ *:[i32] }:{ *:[i32] } i32:{ *:[i32] }:$rs1, i32:{ *:[i32] }:$rs2) +// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::tst2), +// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[rd] +// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/2, // rs1 +// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/3, // rs2 +// CHECK-NEXT: GIR_SetImplicitDefDead, /*InsnID*/0, /*OpIdx for MyTarget::R1*/1, +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // DstI[R0] +// CHECK-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(0), +// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, +// CHECK-NEXT: // GIR_Coverage, 1, +// CHECK-NEXT: GIR_EraseRootFromParent_Done, + +// CHECK-LABEL: // (udivrem:{ *:[i32] }:{ *:[i32] } i32:{ *:[i32] }:$rs1, i32:{ *:[i32] }:$rs2) => (tst3:{ *:[i32] } i32:{ *:[i32] }:$rs1, i32:{ *:[i32] }:$rs2) +// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::tst3), +// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/2, // rs1 +// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/3, // rs2 +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // DstI[R0] +// CHECK-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(0), +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/2, /*OldInsnID*/0, /*OpIdx*/1, // DstI[R1] +// CHECK-NEXT: GIR_AddRegister, /*InsnID*/2, GIMT_Encode2(MyTarget::R1), /*AddRegisterRegFlags*/GIMT_Encode2(0), +// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, +// CHECK-NEXT: // GIR_Coverage, 2, +// CHECK-NEXT: GIR_EraseRootFromParent_Done, diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp index 2dbee94d7e540..89836055ff396 100644 --- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp +++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp @@ -1306,6 +1306,18 @@ CodeGenRegBank::CodeGenRegBank(const RecordKeeper &Records, addToMaps(&RC); } + // Resolve cross references. + for (const Record *RCRec : RCs) { + if (const Record *CrossCopyRCRec = + RCRec->getValueAsOptionalDef("CrossCopyRegClass")) { + const CodeGenRegisterClass *CrossCopyRC = getRegClass(CrossCopyRCRec); + if (!CrossCopyRC->Allocatable) + PrintFatalError(RCRec->getFieldLoc("CrossCopyRegClass"), + "cross-copy register class must be allocatable"); + getRegClass(RCRec)->CrossCopyRC = CrossCopyRC; + } + } + // Infer missing classes to create a full algebra. computeInferredRegisterClasses(); diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.h b/llvm/utils/TableGen/Common/CodeGenRegisters.h index 2fa6cab2afb89..2517d399f1554 100644 --- a/llvm/utils/TableGen/Common/CodeGenRegisters.h +++ b/llvm/utils/TableGen/Common/CodeGenRegisters.h @@ -351,6 +351,7 @@ class CodeGenRegisterClass { RegSizeInfoByHwMode RSI; int CopyCost; bool Allocatable; + const CodeGenRegisterClass *CrossCopyRC = nullptr; StringRef AltOrderSelect; uint8_t AllocationPriority; bool GlobalPriority; diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 3b334ea4ce152..5bfc70b5060a3 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -404,10 +404,11 @@ class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter { createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst) const; - Expected - importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M, - BuildMIAction &DstMIBuilder, - const TreePatternNode &Dst, bool IsRoot) const; + Expected importDefRenderers(action_iterator InsertPt, + RuleMatcher &M, + BuildMIAction &DstMIBuilder, + const TreePatternNode &Dst, + bool IsRoot) const; Expected importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M, @@ -432,9 +433,6 @@ class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter { const TreePatternNode &N, action_iterator &InsertPt) const; - Error importImplicitDefRenderers(BuildMIAction &DstMIBuilder, - ArrayRef ImplicitDefs) const; - /// Analyze pattern \p P, returning a matcher for it if possible. /// Otherwise, return an Error explaining why we don't support it. Expected runOnPattern(const PatternToMatch &P); @@ -1440,9 +1438,9 @@ Expected GlobalISelEmitter::createAndImportInstructionRenderer( CopyToPhysRegMIBuilder.addRenderer(PhysOp.first); } - if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst, - /*IsRoot=*/true) - .takeError()) + if (auto Error = + importDefRenderers(InsertPt, M, DstMIBuilder, Dst, /*IsRoot=*/true) + .takeError()) return std::move(Error); if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst) @@ -1470,8 +1468,8 @@ GlobalISelEmitter::createAndImportSubInstructionRenderer( DstMIBuilder.addRenderer(TempRegID, true); // Handle additional (ignored) results. - InsertPtOrError = importExplicitDefRenderers( - std::prev(*InsertPtOrError), M, DstMIBuilder, Dst, /*IsRoot=*/false); + InsertPtOrError = importDefRenderers(std::prev(*InsertPtOrError), M, + DstMIBuilder, Dst, /*IsRoot=*/false); if (auto Error = InsertPtOrError.takeError()) return std::move(Error); @@ -1510,7 +1508,7 @@ GlobalISelEmitter::createInstructionRenderer(action_iterator InsertPt, DstI); } -Expected GlobalISelEmitter::importExplicitDefRenderers( +Expected GlobalISelEmitter::importDefRenderers( action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, const TreePatternNode &Dst, bool IsRoot) const { const CodeGenInstruction *DstI = DstMIBuilder.getCGI(); @@ -1568,11 +1566,32 @@ Expected GlobalISelEmitter::importExplicitDefRenderers( TempRegID, /*IsDef=*/true, /*SubReg=*/nullptr, /*IsDead=*/true); } - // Implicit defs are not currently supported, mark all of them as dead. + // Process implicit defs. for (const Record *Reg : DstI->ImplicitDefs) { std::string OpName = getMangledRootDefName(Reg->getName()); - assert(!M.hasOperand(OpName) && "The pattern should've been rejected"); - DstMIBuilder.setDeadImplicitDef(Reg); + + if (!IsRoot || !M.hasOperand(OpName)) { + DstMIBuilder.setDeadImplicitDef(Reg); + continue; + } + + BuildMIAction &CopyBuilder = M.addAction( + M.allocateOutputInsnID(), &Target.getInstruction(RK.getDef("COPY"))); + + StringRef PermanentRef = M.getOperandMatcher(OpName).getSymbolicName(); + CopyBuilder.addRenderer(PermanentRef); + CopyBuilder.addRenderer(Target, Reg); + + const CodeGenRegisterClass *RC = CGRegs.getRegClassForRegister(Reg); + if (RC && !RC->Allocatable) + RC = RC->CrossCopyRC; + + if (!RC) + return failedImport("could not infer register class for " + + Reg->getName()); + + M.addAction(CopyBuilder.getInsnID(), + /*OpIdx=*/0, *RC); } return InsertPt; @@ -1751,13 +1770,6 @@ Expected GlobalISelEmitter::importExplicitUseRenderers( return InsertPt; } -Error GlobalISelEmitter::importImplicitDefRenderers( - BuildMIAction &DstMIBuilder, ArrayRef ImplicitDefs) const { - if (!ImplicitDefs.empty()) - return failedImport("Pattern defines a physical register"); - return Error::success(); -} - Error GlobalISelEmitter::constrainOperands(action_iterator InsertPt, RuleMatcher &M, unsigned InsnID, const TreePatternNode &Dst) const { @@ -2134,13 +2146,14 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { unsigned DstExpDefs = DstI.Operands.NumDefs, DstNumDefs = DstI.ImplicitDefs.size() + DstExpDefs, SrcNumDefs = Src.getExtTypes().size(); + + bool FoundNoUsePred = false; if (DstNumDefs < SrcNumDefs) { if (DstNumDefs != 0) return failedImport("Src pattern result has more defs than dst MI (" + to_string(SrcNumDefs) + " def(s) vs " + to_string(DstNumDefs) + " def(s))"); - bool FoundNoUsePred = false; for (const auto &Pred : InsnMatcher.predicates()) { if ((FoundNoUsePred = isa(Pred.get()))) break; @@ -2153,15 +2166,24 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { // The root of the match also has constraints on the register bank so that it // matches the result instruction. - unsigned N = std::min(DstExpDefs, SrcNumDefs); - for (unsigned I = 0; I < N; ++I) { - const auto &DstIOperand = DstI.Operands[I]; + for (unsigned I = 0; I < SrcNumDefs; ++I) { + if (FoundNoUsePred) + continue; OperandMatcher &OM = InsnMatcher.getOperand(I); + + if (I >= DstExpDefs) { + const Record *Reg = DstI.ImplicitDefs[I - DstExpDefs]; + OM.setSymbolicName(getMangledRootDefName(Reg->getName())); + M.defineOperand(OM.getSymbolicName(), OM); + continue; + } + // The operand names declared in the DstI instruction are unrelated to // those used in pattern's source and destination DAGs, so mangle the // former to prevent implicitly adding unexpected // GIM_CheckIsSameOperand predicates by the defineOperand method. + const CGIOperandList::OperandInfo &DstIOperand = DstI.Operands[I]; OM.setSymbolicName(getMangledRootDefName(DstIOperand.Name)); M.defineOperand(OM.getSymbolicName(), OM); @@ -2179,11 +2201,6 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { return std::move(Error); BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get(); - // Render the implicit defs. - // These are only added to the root of the result. - if (auto Error = importImplicitDefRenderers(DstMIBuilder, P.getDstRegs())) - return std::move(Error); - DstMIBuilder.chooseInsnToMutate(M); // Constrain the registers to classes. This is normally derived from the