[TableGen][GISel] Learn to import patterns with physreg defs

llvm/include/llvm/Target/Target.td

@@ -283,6 +283,8 @@ class RegisterClass<string namespace, list<ValueType> regTypes, int alignment,
   // value means copying is extremely expensive or impossible.
   int CopyCost = 1;
 
+  RegisterClass CrossCopyRegClass = ?;
+
   // MemberList - Specify which registers are in this class.  If the
   // allocation_order_* method are not specified, this also defines the order of
   // allocation used by the register allocator.

llvm/lib/Target/AMDGPU/SIRegisterInfo.td

@@ -362,13 +362,6 @@ foreach Index = 0...255 in {
 //  Groupings using register classes and tuples
 //===----------------------------------------------------------------------===//
 
-def SCC_CLASS : SIRegisterClass<"AMDGPU", [i1], 1, (add SCC)> {
-  let CopyCost = -1;
-  let isAllocatable = 0;
-  let HasSGPR = 1;
-  let BaseClassOrder = 10000;
-}
-
 def M0_CLASS : SIRegisterClass<"AMDGPU", [i32], 32, (add M0)> {
   let CopyCost = 1;
   let isAllocatable = 0;
@@ -801,6 +794,14 @@ def SReg_32_XM0 : SIRegisterClass<"AMDGPU", [i32, f32, i16, f16, bf16, v2i16, v2
 
 } // End GeneratePressureSet = 0
 
+def SCC_CLASS : SIRegisterClass<"AMDGPU", [i1], 1, (add SCC)> {
+  let CopyCost = -1;
+  let isAllocatable = 0;
+  let CrossCopyRegClass = SReg_32_XM0_XEXEC;
+  let HasSGPR = 1;
+  let BaseClassOrder = 10000;
+}
+
 // Register class for all scalar registers (SGPRs + Special Registers)
 def SReg_32 : SIRegisterClass<"AMDGPU", [i32, f32, i16, f16, bf16, v2i16, v2f16, v2bf16, i1], 32,
   (add SReg_32_XM0, M0_CLASS)> {

llvm/lib/Target/X86/X86InstrArithmetic.td

@@ -64,25 +64,21 @@ class MulDivOpM<bits<8> o, Format f, string m, X86TypeInfo t,
      sched.ReadAfterFold, sched.ReadAfterFold];
 }
 
-multiclass Mul<bits<8> o, string m, Format RegMRM, Format MemMRM, SDPatternOperator node> {
+multiclass Mul<bits<8> o, string m, Format RegMRM, Format MemMRM> {
   // AL is really implied by AX, but the registers in Defs must match the
   // SDNode results (i8, i32).
   //
   // FIXME: Used for 8-bit mul, ignore result upper 8 bits.
-  // This probably ought to be moved to a def : Pat<> if the
-  // syntax can be accepted.
   let Defs = [AL, EFLAGS, AX], Uses = [AL] in
-    def 8r : MulDivOpR<o, RegMRM, m, Xi8, WriteIMul8,
-                       [(set AL, EFLAGS, (node AL, GR8:$src1))]>;
+    def 8r : MulDivOpR<o, RegMRM, m, Xi8, WriteIMul8, []>;
   let Defs = [AX, DX, EFLAGS], Uses = [AX] in
     def 16r : MulDivOpR<o, RegMRM, m, Xi16, WriteIMul16, []>, OpSize16;
   let Defs = [EAX, EDX, EFLAGS], Uses = [EAX] in
     def 32r : MulDivOpR<o, RegMRM, m, Xi32, WriteIMul32, []>, OpSize32;
   let Defs = [RAX, RDX, EFLAGS], Uses = [RAX] in
     def 64r : MulDivOpR<o, RegMRM, m, Xi64, WriteIMul64, []>;
   let Defs = [AL, EFLAGS, AX], Uses = [AL] in
-    def 8m : MulDivOpM<o, MemMRM, m, Xi8, WriteIMul8,
-                       [(set AL, EFLAGS, (node AL, (loadi8 addr:$src1)))]>;
+    def 8m : MulDivOpM<o, MemMRM, m, Xi8, WriteIMul8, []>;
   let Defs = [AX, DX, EFLAGS], Uses = [AX] in
     def 16m : MulDivOpM<o, MemMRM, m, Xi16, WriteIMul16, []>, OpSize16;
   let Defs = [EAX, EDX, EFLAGS], Uses = [EAX] in
@@ -127,8 +123,14 @@ multiclass Mul<bits<8> o, string m, Format RegMRM, Format MemMRM, SDPatternOpera
   }
 }
 
-defm MUL : Mul<0xF7, "mul", MRM4r, MRM4m, mul>;
-defm IMUL : Mul<0xF7, "imul", MRM5r, MRM5m, null_frag>;
+defm MUL : Mul<0xF7, "mul", MRM4r, MRM4m>;
+defm IMUL : Mul<0xF7, "imul", MRM5r, MRM5m>;
+
+// These nodes are selected by custom C++ code.
+let GISelShouldIgnore = true in {
+  def : Pat<(mul AL, i8:$src1), (MUL8r $src1)>;
+  def : Pat<(mul AL, (loadi8 addr:$src1)), (MUL8m addr:$src1)>;
+}
 
 multiclass Div<bits<8> o, string m, Format RegMRM, Format MemMRM> {
   defvar sched8 = !if(!eq(m, "div"), WriteDiv8, WriteIDiv8);

llvm/test/TableGen/Common/GlobalISelEmitterCommon.td

@@ -7,7 +7,8 @@ class MyTargetGenericInstruction : GenericInstruction {
 }
 
 def R0 : Register<"r0"> { let Namespace = "MyTarget"; }
-def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0)>;
+def R1 : Register<"r1"> { let Namespace = "MyTarget"; }
+def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0, R1)>;
 def GPR32Op : RegisterOperand<GPR32>;
 def F0 : Register<"f0"> { let Namespace = "MyTarget"; }
 def FPR32 : RegisterClass<"MyTarget", [f32], 32, (add F0)>;

llvm/test/TableGen/GlobalISelEmitter/implicit-defs.td

@@ -1,12 +1,75 @@
-// RUN: llvm-tblgen -gen-global-isel -warn-on-skipped-patterns -I %p/../../../include -I %p/../Common %s -o /dev/null 2>&1 | FileCheck %s --implicit-check-not="Skipped pattern"
+// RUN: llvm-tblgen -gen-global-isel -optimize-match-table=false \
+// RUN:   -I %p/../../../include -I %p/../Common %s
 
 include "llvm/Target/Target.td"
 include "GlobalISelEmitterCommon.td"
 
-// CHECK: Skipped pattern: Pattern defines a physical register
-let Uses = [B0], Defs = [B0] in
-def tst1 : I<(outs), (ins), [(set B0, (add B0, 1))]>;
+let Defs = [R0, B0] in
+def tst1 : I<(outs), (ins), [(set R0, (i32 42))]>;
 
-// CHECK: Skipped pattern: Src pattern result has 1 def(s) without the HasNoUse predicate set to true but Dst MI has no def
-let Uses = [B0] in
-def tst2 : I<(outs), (ins), [(set B0, (add B0, 1))]>;
+let Defs = [R0, R1] in
+def tst2 : I<(outs GPR32:$rd), (ins GPR32:$rs1, GPR32:$rs2),
+             [(set GPR32:$rd, R1, (sdivrem i32:$rs1, i32:$rs2))]>;
+
+let Defs = [R0, R1] in
+def tst3 : I<(outs), (ins GPR32:$rs1, GPR32:$rs2),
+             [(set R1, R0, (udivrem i32:$rs1, i32:$rs2))]>;
+
+let Defs = [R0] in
+def tst4 : I<(outs GPR32:$rd), (ins GPR32:$rs), []>;
+
+def : Pat<(sdivrem i32:$rs, 42), (tst4 (tst4 $rs))>;
+
+// CHECK-LABEL: // (sdivrem:{ *:[i32] }:{ *:[i32] } i32:{ *:[i32] }:$rs, 42:{ *:[i32] })  =>  (tst4:{ *:[i32] }:{ *:[i32] } (tst4:{ *:[i32] }:{ *:[i32] } ?:{ *:[i32] }:$rs))
+// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(MyTarget::tst4),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/2, /*OldInsnID*/0, /*OpIdx*/2, // rs
+// CHECK-NEXT: GIR_SetImplicitDefDead, /*InsnID*/2, /*OpIdx for MyTarget::R0*/0,
+// CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/2,
+// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::tst4),
+// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[rd]
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // DstI[R0]
+// CHECK-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(0),
+// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands,
+// CHECK-NEXT: // GIR_Coverage, 3,
+// CHECK-NEXT: GIR_EraseRootFromParent_Done,
+
+// CHECK-LABEL: // 42:{ *:[i32] }  =>  (tst1:{ *:[i32] })
+// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::tst1),
+// CHECK-NEXT: GIR_SetImplicitDefDead, /*InsnID*/0, /*OpIdx for MyTarget::B0*/1,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // DstI[R0]
+// CHECK-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(0),
+// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands,
+// CHECK-NEXT: // GIR_Coverage, 0,
+// CHECK-NEXT: GIR_EraseRootFromParent_Done,
+
+// CHECK-LABEL: // (sdivrem:{ *:[i32] }:{ *:[i32] } i32:{ *:[i32] }:$rs1, i32:{ *:[i32] }:$rs2)  =>  (tst2:{ *:[i32] }:{ *:[i32] } i32:{ *:[i32] }:$rs1, i32:{ *:[i32] }:$rs2)
+// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::tst2),
+// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[rd]
+// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/2, // rs1
+// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/3, // rs2
+// CHECK-NEXT: GIR_SetImplicitDefDead, /*InsnID*/0, /*OpIdx for MyTarget::R1*/1,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // DstI[R0]
+// CHECK-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(0),
+// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands,
+// CHECK-NEXT: // GIR_Coverage, 1,
+// CHECK-NEXT: GIR_EraseRootFromParent_Done,
+
+// CHECK-LABEL: // (udivrem:{ *:[i32] }:{ *:[i32] } i32:{ *:[i32] }:$rs1, i32:{ *:[i32] }:$rs2)  =>  (tst3:{ *:[i32] } i32:{ *:[i32] }:$rs1, i32:{ *:[i32] }:$rs2)
+// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::tst3),
+// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/2, // rs1
+// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/3, // rs2
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // DstI[R0]
+// CHECK-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(0),
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_Copy, /*NewInsnID*/2, /*OldInsnID*/0, /*OpIdx*/1, // DstI[R1]
+// CHECK-NEXT: GIR_AddRegister, /*InsnID*/2, GIMT_Encode2(MyTarget::R1), /*AddRegisterRegFlags*/GIMT_Encode2(0),
+// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands,
+// CHECK-NEXT: // GIR_Coverage, 2,
+// CHECK-NEXT: GIR_EraseRootFromParent_Done,

llvm/utils/TableGen/Common/CodeGenRegisters.cpp

@@ -1306,6 +1306,18 @@ CodeGenRegBank::CodeGenRegBank(const RecordKeeper &Records,
       addToMaps(&RC);
   }
 
+  // Resolve cross references.
+  for (const Record *RCRec : RCs) {
+    if (const Record *CrossCopyRCRec =
+            RCRec->getValueAsOptionalDef("CrossCopyRegClass")) {
+      const CodeGenRegisterClass *CrossCopyRC = getRegClass(CrossCopyRCRec);
+      if (!CrossCopyRC->Allocatable)
+        PrintFatalError(RCRec->getFieldLoc("CrossCopyRegClass"),
+                        "cross-copy register class must be allocatable");
+      getRegClass(RCRec)->CrossCopyRC = CrossCopyRC;
+    }
+  }
+
   // Infer missing classes to create a full algebra.
   computeInferredRegisterClasses();
 

llvm/utils/TableGen/Common/CodeGenRegisters.h

@@ -351,6 +351,7 @@ class CodeGenRegisterClass {
   RegSizeInfoByHwMode RSI;
   int CopyCost;
   bool Allocatable;
+  const CodeGenRegisterClass *CrossCopyRC = nullptr;
   StringRef AltOrderSelect;
   uint8_t AllocationPriority;
   bool GlobalPriority;

llvm/utils/TableGen/GlobalISelEmitter.cpp

@@ -404,10 +404,11 @@ class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
   createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
                             const TreePatternNode &Dst) const;
 
-  Expected<action_iterator>
-  importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
-                             BuildMIAction &DstMIBuilder,
-                             const TreePatternNode &Dst, bool IsRoot) const;
+  Expected<action_iterator> importDefRenderers(action_iterator InsertPt,
+                                               RuleMatcher &M,
+                                               BuildMIAction &DstMIBuilder,
+                                               const TreePatternNode &Dst,
+                                               bool IsRoot) const;
 
   Expected<action_iterator>
   importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M,
@@ -432,9 +433,6 @@ class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
                            const TreePatternNode &N,
                            action_iterator &InsertPt) const;
 
-  Error importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
-                                   ArrayRef<const Record *> ImplicitDefs) const;
-
   /// Analyze pattern \p P, returning a matcher for it if possible.
   /// Otherwise, return an Error explaining why we don't support it.
   Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
@@ -1440,9 +1438,9 @@ Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
     CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(PhysOp.first);
   }
 
-  if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst,
-                                              /*IsRoot=*/true)
-                       .takeError())
+  if (auto Error =
+          importDefRenderers(InsertPt, M, DstMIBuilder, Dst, /*IsRoot=*/true)
+              .takeError())
     return std::move(Error);
 
   if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst)
@@ -1470,8 +1468,8 @@ GlobalISelEmitter::createAndImportSubInstructionRenderer(
   DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true);
 
   // Handle additional (ignored) results.
-  InsertPtOrError = importExplicitDefRenderers(
-      std::prev(*InsertPtOrError), M, DstMIBuilder, Dst, /*IsRoot=*/false);
+  InsertPtOrError = importDefRenderers(std::prev(*InsertPtOrError), M,
+                                       DstMIBuilder, Dst, /*IsRoot=*/false);
   if (auto Error = InsertPtOrError.takeError())
     return std::move(Error);
 
@@ -1510,7 +1508,7 @@ GlobalISelEmitter::createInstructionRenderer(action_iterator InsertPt,
                                        DstI);
 }
 
-Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
+Expected<action_iterator> GlobalISelEmitter::importDefRenderers(
     action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
     const TreePatternNode &Dst, bool IsRoot) const {
   const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
@@ -1568,11 +1566,32 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
         TempRegID, /*IsDef=*/true, /*SubReg=*/nullptr, /*IsDead=*/true);
   }
 
-  // Implicit defs are not currently supported, mark all of them as dead.
+  // Process implicit defs.
   for (const Record *Reg : DstI->ImplicitDefs) {
     std::string OpName = getMangledRootDefName(Reg->getName());
-    assert(!M.hasOperand(OpName) && "The pattern should've been rejected");
-    DstMIBuilder.setDeadImplicitDef(Reg);
+
+    if (!IsRoot || !M.hasOperand(OpName)) {
+      DstMIBuilder.setDeadImplicitDef(Reg);
+      continue;
+    }
+
+    BuildMIAction &CopyBuilder = M.addAction<BuildMIAction>(
+        M.allocateOutputInsnID(), &Target.getInstruction(RK.getDef("COPY")));
+
+    StringRef PermanentRef = M.getOperandMatcher(OpName).getSymbolicName();
+    CopyBuilder.addRenderer<CopyRenderer>(PermanentRef);
+    CopyBuilder.addRenderer<AddRegisterRenderer>(Target, Reg);
+
+    const CodeGenRegisterClass *RC = CGRegs.getRegClassForRegister(Reg);
+    if (RC && !RC->Allocatable)
+      RC = RC->CrossCopyRC;
+
+    if (!RC)
+      return failedImport("could not infer register class for " +
+                          Reg->getName());
+
+    M.addAction<ConstrainOperandToRegClassAction>(CopyBuilder.getInsnID(),
+                                                  /*OpIdx=*/0, *RC);
   }
 
   return InsertPt;
@@ -1751,13 +1770,6 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
   return InsertPt;
 }
 
-Error GlobalISelEmitter::importImplicitDefRenderers(
-    BuildMIAction &DstMIBuilder, ArrayRef<const Record *> ImplicitDefs) const {
-  if (!ImplicitDefs.empty())
-    return failedImport("Pattern defines a physical register");
-  return Error::success();
-}
-
 Error GlobalISelEmitter::constrainOperands(action_iterator InsertPt,
                                            RuleMatcher &M, unsigned InsnID,
                                            const TreePatternNode &Dst) const {
@@ -2134,13 +2146,14 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
   unsigned DstExpDefs = DstI.Operands.NumDefs,
            DstNumDefs = DstI.ImplicitDefs.size() + DstExpDefs,
            SrcNumDefs = Src.getExtTypes().size();
+
+  bool FoundNoUsePred = false;
   if (DstNumDefs < SrcNumDefs) {
     if (DstNumDefs != 0)
       return failedImport("Src pattern result has more defs than dst MI (" +
                           to_string(SrcNumDefs) + " def(s) vs " +
                           to_string(DstNumDefs) + " def(s))");
 
-    bool FoundNoUsePred = false;
     for (const auto &Pred : InsnMatcher.predicates()) {
       if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Pred.get())))
         break;
@@ -2153,15 +2166,24 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
 
   // The root of the match also has constraints on the register bank so that it
   // matches the result instruction.
-  unsigned N = std::min(DstExpDefs, SrcNumDefs);
-  for (unsigned I = 0; I < N; ++I) {
-    const auto &DstIOperand = DstI.Operands[I];
+  for (unsigned I = 0; I < SrcNumDefs; ++I) {
+    if (FoundNoUsePred)
+      continue;
 
     OperandMatcher &OM = InsnMatcher.getOperand(I);
+
+    if (I >= DstExpDefs) {
+      const Record *Reg = DstI.ImplicitDefs[I - DstExpDefs];
+      OM.setSymbolicName(getMangledRootDefName(Reg->getName()));
+      M.defineOperand(OM.getSymbolicName(), OM);
+      continue;
+    }
+
     // The operand names declared in the DstI instruction are unrelated to
     // those used in pattern's source and destination DAGs, so mangle the
     // former to prevent implicitly adding unexpected
     // GIM_CheckIsSameOperand predicates by the defineOperand method.
+    const CGIOperandList::OperandInfo &DstIOperand = DstI.Operands[I];
     OM.setSymbolicName(getMangledRootDefName(DstIOperand.Name));
     M.defineOperand(OM.getSymbolicName(), OM);
 
@@ -2179,11 +2201,6 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
     return std::move(Error);
   BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get();
 
-  // Render the implicit defs.
-  // These are only added to the root of the result.
-  if (auto Error = importImplicitDefRenderers(DstMIBuilder, P.getDstRegs()))
-    return std::move(Error);
-
   DstMIBuilder.chooseInsnToMutate(M);
 
   // Constrain the registers to classes. This is normally derived from the