From 53f67158fcb689bf1f8f14e5647675c5b4c70ae5 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Fri, 27 Dec 2024 14:46:08 -0800 Subject: [PATCH 1/5] [GlobalISel] Support physical register inputs in nested patterns When importing nested patterns, we create InsnMatcher for each pattern and miss them if consider only the top level InsnMatcher. Iterate all InsnMatchers of PhysRegOperands when generating COPYs for physical registers. --- .../GlobalISelEmitter/gisel-physreg-input.td | 43 ++++++++++++++++++- .../Common/GlobalISel/GlobalISelMatchTable.h | 8 +++- llvm/utils/TableGen/GlobalISelEmitter.cpp | 21 +++++---- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/llvm/test/TableGen/GlobalISelEmitter/gisel-physreg-input.td b/llvm/test/TableGen/GlobalISelEmitter/gisel-physreg-input.td index a05f364eb3f05..dbee154e31c9a 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/gisel-physreg-input.td +++ b/llvm/test/TableGen/GlobalISelEmitter/gisel-physreg-input.td @@ -22,6 +22,45 @@ class I Pat> let Pattern = Pat; } +// Try a nested physical register + +// GISEL: GIM_Try, +// GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, +// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), +// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// GISEL-NEXT: // MIs[0] src0 +// GISEL-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, +// GISEL-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// GISEL-NEXT: // MIs[0] Operand 1 +// GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32, +// GISEL-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// GISEL-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3, +// GISEL-NEXT: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_MUL), +// GISEL-NEXT: // MIs[1] Operand 0 +// GISEL-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, +// GISEL-NEXT: // MIs[1] src1 +// GISEL-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, +// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// GISEL-NEXT: // MIs[1] Operand 2 +// GISEL-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, +// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::Special32RegClassID), +// GISEL-NEXT: GIM_CheckIsSafeToFold, /*NumInsns*/1, +// GISEL-NEXT: // (st GPR32:{ *:[i32] }:$src0, (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src1, SPECIAL:{ *:[i32] })) => (MULM_PHYS GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) +// GISEL-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), +// GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::SPECIAL), /*AddRegisterRegFlags*/GIMT_Encode2(RegState::Define), +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/1, /*OpIdx*/2, // SPECIAL +// GISEL-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::MULM_PHYS), +// GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // src0 +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 +// GISEL-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1, +// GISEL-NEXT: GIR_RootConstrainSelectedInstOperands, +// GISEL-NEXT: // GIR_Coverage, 0, +// GISEL-NEXT: GIR_EraseRootFromParent_Done, +def MULM_PHYS : I<(outs), (ins GPR32:$src0, GPR32:$src1), + [(st GPR32:$src0, (mul GPR32:$src1, SPECIAL))]> { + let Uses = [SPECIAL]; +} + // Try a normal physical register use. // GISEL: GIM_Try, @@ -44,7 +83,7 @@ class I Pat> // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // src0 // GISEL-NEXT: GIR_RootConstrainSelectedInstOperands, -// GISEL-NEXT: // GIR_Coverage, 0, +// GISEL-NEXT: // GIR_Coverage, 1, // GISEL-NEXT: GIR_EraseRootFromParent_Done, def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0), [(set GPR32:$dst, (add GPR32:$src0, SPECIAL))]> { @@ -73,7 +112,7 @@ def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0), // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // SPECIAL // GISEL-NEXT: GIR_RootConstrainSelectedInstOperands, -// GISEL-NEXT: // GIR_Coverage, 1, +// GISEL-NEXT: // GIR_Coverage, 2, // GISEL-NEXT: GIR_EraseRootFromParent_Done, def MUL_PHYS : I<(outs GPR32:$dst), (ins GPR32:$SPECIAL), [(set GPR32:$dst, (mul GPR32:$SPECIAL, SPECIAL))]> { diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index 48ce71be677c0..abf145a2507c0 100644 --- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -492,9 +492,11 @@ class RuleMatcher : public Matcher { /// the renderers. StringMap DefinedOperands; + using PhysRegOperandsTy = DenseMap; + /// A map of anonymous physical register operands defined by the matchers that /// may be referenced by the renderers. - DenseMap PhysRegOperands; + PhysRegOperandsTy PhysRegOperands; /// ID for the next instruction variable defined with /// implicitlyDefineInsnVar() @@ -695,6 +697,10 @@ class RuleMatcher : public Matcher { unsigned allocateOutputInsnID() { return NextOutputInsnID++; } unsigned allocateTempRegID() { return NextTempRegID++; } + iterator_range physoperands() const { + return make_range(PhysRegOperands.begin(), PhysRegOperands.end()); + } + iterator_range insnmatchers() { return make_range(Matchers.begin(), Matchers.end()); } diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index f0fb11625883e..62ee6e735be74 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -1412,15 +1412,18 @@ Expected GlobalISelEmitter::createAndImportInstructionRenderer( action_iterator InsertPt = InsertPtOrError.get(); BuildMIAction &DstMIBuilder = *static_cast(InsertPt->get()); - for (auto PhysInput : InsnMatcher.getPhysRegInputs()) { - InsertPt = M.insertAction( - InsertPt, M.allocateOutputInsnID(), - &Target.getInstruction(RK.getDef("COPY"))); - BuildMIAction &CopyToPhysRegMIBuilder = - *static_cast(InsertPt->get()); - CopyToPhysRegMIBuilder.addRenderer( - Target, PhysInput.first, true); - CopyToPhysRegMIBuilder.addRenderer(PhysInput.first); + for (auto PhysOp : M.physoperands()) { + auto &OpInsnMatcher = PhysOp.second->getInstructionMatcher(); + for (auto PhysInput : OpInsnMatcher.getPhysRegInputs()) { + InsertPt = M.insertAction( + InsertPt, M.allocateOutputInsnID(), + &Target.getInstruction(RK.getDef("COPY"))); + BuildMIAction &CopyToPhysRegMIBuilder = + *static_cast(InsertPt->get()); + CopyToPhysRegMIBuilder.addRenderer( + Target, PhysInput.first, true); + CopyToPhysRegMIBuilder.addRenderer(PhysInput.first); + } } if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst, From 05972736e0a5ec0a0c96947703256a00f2c336eb Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Sat, 28 Dec 2024 08:44:04 -0800 Subject: [PATCH 2/5] Eliminate copies --- .../GlobalISelEmitter/gisel-physreg-input.td | 45 ++++++++++++++++++- .../Common/GlobalISel/GlobalISelMatchTable.h | 3 +- llvm/utils/TableGen/GlobalISelEmitter.cpp | 19 ++++---- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/llvm/test/TableGen/GlobalISelEmitter/gisel-physreg-input.td b/llvm/test/TableGen/GlobalISelEmitter/gisel-physreg-input.td index dbee154e31c9a..1f1b557ace608 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/gisel-physreg-input.td +++ b/llvm/test/TableGen/GlobalISelEmitter/gisel-physreg-input.td @@ -61,6 +61,47 @@ def MULM_PHYS : I<(outs), (ins GPR32:$src0, GPR32:$src1), let Uses = [SPECIAL]; } +// Try nested physical registers and check on duplicated copies + +// GISEL: GIM_Try, +// GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, +// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), +// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// GISEL-NEXT: // MIs[0] src0 +// GISEL-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, +// GISEL-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// GISEL-NEXT: // MIs[0] Operand 1 +// GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32, +// GISEL-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// GISEL-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3, +// GISEL-NEXT: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_MUL), +// GISEL-NEXT: // MIs[1] Operand 0 +// GISEL-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, +// GISEL-NEXT: // MIs[1] Operand 1 +// GISEL-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, +// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// GISEL-NEXT: // MIs[1] Operand 2 +// GISEL-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, +// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::Special32RegClassID), +// GISEL-NEXT: GIM_CheckIsSafeToFold, /*NumInsns*/1, +// GISEL-NEXT: // (st GPR32:{ *:[i32] }:$src0, (mul:{ *:[i32] } R0:{ *:[i32] }, SPECIAL:{ *:[i32] })) => (MULMR0_PHYS GPR32:{ *:[i32] }:$src0) +// GISEL-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), +// GISEL-NEXT: GIR_AddRegister, /*InsnID*/2, GIMT_Encode2(MyTarget::SPECIAL), /*AddRegisterRegFlags*/GIMT_Encode2(RegState::Define), +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/2, /*OldInsnID*/1, /*OpIdx*/2, // SPECIAL +// GISEL-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY), +// GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::R0), /*AddRegisterRegFlags*/GIMT_Encode2(RegState::Define), +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/1, /*OpIdx*/1, // R0 +// GISEL-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::MULMR0_PHYS), +// GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // src0 +// GISEL-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1, +// GISEL-NEXT: GIR_RootConstrainSelectedInstOperands, +// GISEL-NEXT: // GIR_Coverage, 1, +// GISEL-NEXT: GIR_EraseRootFromParent_Done, +def MULMR0_PHYS : I<(outs), (ins GPR32:$src0), + [(st GPR32:$src0, (mul R0, SPECIAL))]> { + let Uses = [R0, SPECIAL]; +} + // Try a normal physical register use. // GISEL: GIM_Try, @@ -83,7 +124,7 @@ def MULM_PHYS : I<(outs), (ins GPR32:$src0, GPR32:$src1), // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // src0 // GISEL-NEXT: GIR_RootConstrainSelectedInstOperands, -// GISEL-NEXT: // GIR_Coverage, 1, +// GISEL-NEXT: // GIR_Coverage, 2, // GISEL-NEXT: GIR_EraseRootFromParent_Done, def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0), [(set GPR32:$dst, (add GPR32:$src0, SPECIAL))]> { @@ -112,7 +153,7 @@ def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0), // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // SPECIAL // GISEL-NEXT: GIR_RootConstrainSelectedInstOperands, -// GISEL-NEXT: // GIR_Coverage, 2, +// GISEL-NEXT: // GIR_Coverage, 3, // GISEL-NEXT: GIR_EraseRootFromParent_Done, def MUL_PHYS : I<(outs GPR32:$dst), (ins GPR32:$SPECIAL), [(set GPR32:$dst, (mul GPR32:$SPECIAL, SPECIAL))]> { diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index abf145a2507c0..1617c9a3b9b98 100644 --- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -19,6 +19,7 @@ #include "Common/CodeGenDAGPatterns.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" @@ -492,7 +493,7 @@ class RuleMatcher : public Matcher { /// the renderers. StringMap DefinedOperands; - using PhysRegOperandsTy = DenseMap; + using PhysRegOperandsTy = MapVector; /// A map of anonymous physical register operands defined by the matchers that /// may be referenced by the renderers. diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 62ee6e735be74..15610319bc7a5 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -1413,17 +1413,14 @@ Expected GlobalISelEmitter::createAndImportInstructionRenderer( BuildMIAction &DstMIBuilder = *static_cast(InsertPt->get()); for (auto PhysOp : M.physoperands()) { - auto &OpInsnMatcher = PhysOp.second->getInstructionMatcher(); - for (auto PhysInput : OpInsnMatcher.getPhysRegInputs()) { - InsertPt = M.insertAction( - InsertPt, M.allocateOutputInsnID(), - &Target.getInstruction(RK.getDef("COPY"))); - BuildMIAction &CopyToPhysRegMIBuilder = - *static_cast(InsertPt->get()); - CopyToPhysRegMIBuilder.addRenderer( - Target, PhysInput.first, true); - CopyToPhysRegMIBuilder.addRenderer(PhysInput.first); - } + InsertPt = M.insertAction( + InsertPt, M.allocateOutputInsnID(), + &Target.getInstruction(RK.getDef("COPY"))); + BuildMIAction &CopyToPhysRegMIBuilder = + *static_cast(InsertPt->get()); + CopyToPhysRegMIBuilder.addRenderer(Target, + PhysOp.first, true); + CopyToPhysRegMIBuilder.addRenderer(PhysOp.first); } if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst, From 0452bd0bdc11c504a159ff1b94262c115c15ee14 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Wed, 1 Jan 2025 16:17:04 -0800 Subject: [PATCH 3/5] From MapVector to SmallMapVector --- llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index 1617c9a3b9b98..9fb49f0e30296 100644 --- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -493,7 +493,7 @@ class RuleMatcher : public Matcher { /// the renderers. StringMap DefinedOperands; - using PhysRegOperandsTy = MapVector; + using PhysRegOperandsTy = SmallMapVector; /// A map of anonymous physical register operands defined by the matchers that /// may be referenced by the renderers. From 381d594ee9e44bc34ace2bd5f7620d5c3243eaf5 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Wed, 1 Jan 2025 16:18:32 -0800 Subject: [PATCH 4/5] Remove getPhysRegInputs as no uses left --- llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index 9fb49f0e30296..283c72bf38071 100644 --- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -1809,10 +1809,6 @@ class InstructionMatcher final : public PredicateListMatcher { OperandMatcher &addPhysRegInput(const Record *Reg, unsigned OpIdx, unsigned TempOpIdx); - ArrayRef> getPhysRegInputs() const { - return PhysRegInputs; - } - StringRef getSymbolicName() const { return SymbolicName; } unsigned getNumOperandMatchers() const { return Operands.size(); } From 0d3084ea36a440c0c4fe2a4c40563709647dce19 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Thu, 2 Jan 2025 08:28:11 -0800 Subject: [PATCH 5/5] Drop PhysRegInputs from InstructionMatcher as no users anymore --- .../TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp | 1 - llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h | 5 ----- 2 files changed, 6 deletions(-) diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp index 619e7a4790c88..a81f2b53f2846 100644 --- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp @@ -1723,7 +1723,6 @@ OperandMatcher &InstructionMatcher::addPhysRegInput(const Record *Reg, OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx); Operands.emplace_back(OM); Rule.definePhysRegOperand(Reg, *OM); - PhysRegInputs.emplace_back(Reg, OpIdx); return *OM; } diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h index 283c72bf38071..8e6de80d6083c 100644 --- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.h @@ -1763,11 +1763,6 @@ class InstructionMatcher final : public PredicateListMatcher { unsigned InsnVarID; bool AllowNumOpsCheck; - /// PhysRegInputs - List list has an entry for each explicitly specified - /// physreg input to the pattern. The first elt is the Register node, the - /// second is the recorded slot number the input pattern match saved it in. - SmallVector, 2> PhysRegInputs; - bool canAddNumOperandsCheck() const { // Add if it's allowed, and: // - We don't have a variadic operand