Skip to content

Commit 53a65ba

Browse files
committed
[VPlan] Don't look up recipe for IV step via RecipeBuilder. (NFC)
Directly update induction increments with step value created for wide inductions in createWidenInductionRecipes, which does not require looking up via RecipeBuilder.
1 parent 6636659 commit 53a65ba

File tree

2 files changed

+14
-14
lines changed

2 files changed

+14
-14
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7633,6 +7633,14 @@ createWidenInductionRecipes(VPInstruction *PhiR,
76337633

76347634
VPValue *Step =
76357635
vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
7636+
7637+
// Update wide induction increments to use the same step as the corresponding
7638+
// wide induction. This enables detecting induction increments directly in
7639+
// VPlan and removes redundant splats.
7640+
using namespace llvm::VPlanPatternMatch;
7641+
if (match(PhiR->getOperand(1), m_Add(m_Specific(PhiR), m_VPValue())))
7642+
PhiR->getOperand(1)->getDefiningRecipe()->setOperand(1, Step);
7643+
76367644
PHINode *Phi = cast<PHINode>(PhiR->getUnderlyingInstr());
76377645
return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
76387646
IndDesc, PhiR->getDebugLoc());
@@ -8473,20 +8481,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
84738481
"entry block must be set to a VPRegionBlock having a non-empty entry "
84748482
"VPBasicBlock");
84758483

8476-
// Update wide induction increments to use the same step as the corresponding
8477-
// wide induction. This enables detecting induction increments directly in
8478-
// VPlan and removes redundant splats.
8479-
for (const auto &[Phi, ID] : Legal->getInductionVars()) {
8480-
auto *IVInc = cast<Instruction>(
8481-
Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
8482-
if (IVInc->getOperand(0) != Phi || IVInc->getOpcode() != Instruction::Add)
8483-
continue;
8484-
VPWidenInductionRecipe *WideIV =
8485-
cast<VPWidenInductionRecipe>(RecipeBuilder.getRecipe(Phi));
8486-
VPRecipeBase *R = RecipeBuilder.getRecipe(IVInc);
8487-
R->setOperand(1, WideIV->getStepValue());
8488-
}
8489-
84908484
// TODO: We can't call runPass on these transforms yet, due to verifier
84918485
// failures.
84928486
VPlanTransforms::addExitUsersForFirstOrderRecurrences(*Plan, Range);

llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,12 @@ m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
496496
return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
497497
}
498498

499+
template <typename Op0_t, typename Op1_t>
500+
inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0,
501+
const Op1_t &Op1) {
502+
return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
503+
}
504+
499505
template <typename Op0_t, typename Op1_t>
500506
inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>
501507
m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {

0 commit comments

Comments
 (0)