Skip to content

Commit 2764023

Browse files
fhahnfhahn
committed
Step
1 parent 0f66ee1 commit 2764023

File tree

4 files changed

+87
-17
lines changed

4 files changed

+87
-17
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 56 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8953,14 +8953,65 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan) {
89538953
}
89548954
}
89558955

8956+
static bool isIVUse(VPValue *Incoming) {
8957+
VPRecipeBase *IncomingDef = Incoming->getDefiningRecipe();
8958+
if (!IncomingDef)
8959+
return false;
8960+
auto *WideIV = dyn_cast<VPWidenInductionRecipe>(IncomingDef);
8961+
if (WideIV) {
8962+
return isa<VPWidenPointerInductionRecipe>(WideIV) || !cast<VPWidenIntOrFpInductionRecipe>(WideIV)->getTruncInst();
8963+
}
8964+
8965+
if (IncomingDef->getNumOperands() != 2)
8966+
return false;
8967+
WideIV = dyn_cast<VPWidenInductionRecipe>(IncomingDef->getOperand(0));
8968+
if (!WideIV)
8969+
WideIV = dyn_cast<VPWidenInductionRecipe>(IncomingDef->getOperand(1));
8970+
if (!WideIV)
8971+
return false;
8972+
8973+
using namespace VPlanPatternMatch;
8974+
auto &ID = WideIV->getInductionDescriptor();
8975+
switch (ID.getInductionOpcode()) {
8976+
case Instruction::Add:
8977+
return match(Incoming,
8978+
m_c_Binary<Instruction::Add>(
8979+
m_VPValue(), m_Specific(WideIV->getStepValue())));
8980+
case Instruction::FAdd:
8981+
return match(Incoming,
8982+
m_c_Binary<Instruction::FAdd>(
8983+
m_VPValue(), m_Specific(WideIV->getStepValue())));
8984+
case Instruction::FSub:
8985+
return match(Incoming,
8986+
m_Binary<Instruction::FSub>(
8987+
m_VPValue(), m_Specific(WideIV->getStepValue())));
8988+
case Instruction::Sub: {
8989+
VPValue *Step;
8990+
return match(Incoming,
8991+
m_Binary<Instruction::Sub>(m_VPValue(), m_VPValue(Step))) &&
8992+
Step->isLiveIn() && WideIV->getStepValue()->isLiveIn() &&
8993+
(cast<ConstantInt>(Step->getLiveInIRValue())->getValue() +
8994+
cast<ConstantInt>(WideIV->getStepValue()->getLiveInIRValue())
8995+
->getValue())
8996+
.isZero();
8997+
}
8998+
default:
8999+
return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
9000+
match(
9001+
Incoming,
9002+
m_GetElementPtr(m_VPValue(), m_Specific(WideIV->getStepValue())));
9003+
}
9004+
llvm_unreachable("should have been covered by switch above");
9005+
}
9006+
89569007
// Collect VPIRInstructions for phis in the exit blocks that are modeled
89579008
// in VPlan and add the exiting VPValue as operand. Some exiting values are not
89589009
// modeled explicitly yet and won't be included. Those are un-truncated
89599010
// VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe and induction
89609011
// increments.
89619012
static SetVector<VPIRInstruction *> collectUsersInExitBlocks(
8962-
Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan,
8963-
const MapVector<PHINode *, InductionDescriptor> &Inductions) {
9013+
Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan
9014+
) {
89649015
auto *MiddleVPBB = Plan.getMiddleBlock();
89659016
SetVector<VPIRInstruction *> ExitUsersToFix;
89669017
for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
@@ -8985,18 +9036,8 @@ static SetVector<VPIRInstruction *> collectUsersInExitBlocks(
89859036
// Exit values for inductions are computed and updated outside of VPlan
89869037
// and independent of induction recipes.
89879038
// TODO: Compute induction exit values in VPlan.
8988-
if ((isa<VPWidenIntOrFpInductionRecipe>(V) &&
8989-
!cast<VPWidenIntOrFpInductionRecipe>(V)->getTruncInst()) ||
8990-
isa<VPWidenPointerInductionRecipe>(V) ||
8991-
(isa<Instruction>(IncomingValue) &&
8992-
OrigLoop->contains(cast<Instruction>(IncomingValue)) &&
8993-
any_of(IncomingValue->users(), [&Inductions](User *U) {
8994-
auto *P = dyn_cast<PHINode>(U);
8995-
return P && Inductions.contains(P);
8996-
}))) {
8997-
if (ExitVPBB->getSinglePredecessor() == MiddleVPBB)
8998-
continue;
8999-
}
9039+
if (isIVUse(V) && ExitVPBB->getSinglePredecessor() == MiddleVPBB)
9040+
continue;
90009041
ExitUsersToFix.insert(ExitIRI);
90019042
ExitIRI->addOperand(V);
90029043
}
@@ -9332,7 +9373,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
93329373
}
93339374
addScalarResumePhis(RecipeBuilder, *Plan);
93349375
SetVector<VPIRInstruction *> ExitUsersToFix = collectUsersInExitBlocks(
9335-
OrigLoop, RecipeBuilder, *Plan, Legal->getInductionVars());
9376+
OrigLoop, RecipeBuilder, *Plan);
93369377
addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix);
93379378
if (!addUsersInExitBlocks(*Plan, ExitUsersToFix)) {
93389379
reportVectorizationFailure(

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2095,6 +2095,15 @@ class VPWidenInductionRecipe : public VPHeaderPHIRecipe {
20952095
R->getVPDefID() == VPDef::VPWidenPointerInductionSC;
20962096
}
20972097

2098+
static inline bool classof(const VPValue *V) {
2099+
auto *R = V->getDefiningRecipe();
2100+
return R && classof(R);
2101+
}
2102+
2103+
static inline bool classof(const VPHeaderPHIRecipe *R) {
2104+
return classof(static_cast<const VPRecipeBase *>(R));
2105+
}
2106+
20982107
virtual void execute(VPTransformState &State) override = 0;
20992108

21002109
/// Returns the step value of the induction.

‎llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,8 @@ struct MatchRecipeAndOpcode<Opcode, RecipeTy> {
139139
if constexpr (std::is_same<RecipeTy, VPScalarIVStepsRecipe>::value ||
140140
std::is_same<RecipeTy, VPCanonicalIVPHIRecipe>::value ||
141141
std::is_same<RecipeTy, VPWidenSelectRecipe>::value ||
142-
std::is_same<RecipeTy, VPDerivedIVRecipe>::value)
142+
std::is_same<RecipeTy, VPDerivedIVRecipe>::value ||
143+
std::is_same<RecipeTy, VPWidenGEPRecipe>::value)
143144
return DefR;
144145
else
145146
return DefR && DefR->getOpcode() == Opcode;
@@ -309,6 +310,12 @@ m_Binary(const Op0_t &Op0, const Op1_t &Op1) {
309310
return AllBinaryRecipe_match<Op0_t, Op1_t, Opcode, Commutative>(Op0, Op1);
310311
}
311312

313+
template <unsigned Opcode, typename Op0_t, typename Op1_t>
314+
inline AllBinaryRecipe_match<Op0_t, Op1_t, Opcode, true>
315+
m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
316+
return AllBinaryRecipe_match<Op0_t, Op1_t, Opcode, true>(Op0, Op1);
317+
}
318+
312319
template <typename Op0_t, typename Op1_t>
313320
inline AllBinaryRecipe_match<Op0_t, Op1_t, Instruction::Mul>
314321
m_Mul(const Op0_t &Op0, const Op1_t &Op1) {
@@ -339,6 +346,18 @@ m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
339346
return m_BinaryOr<Op0_t, Op1_t, /*Commutative*/ true>(Op0, Op1);
340347
}
341348

349+
template <typename Op0_t, typename Op1_t>
350+
using GEPLikeRecipe_match =
351+
BinaryRecipe_match<Op0_t, Op1_t, Instruction::GetElementPtr, false,
352+
VPWidenRecipe, VPReplicateRecipe, VPWidenGEPRecipe,
353+
VPInstruction>;
354+
355+
template <typename Op0_t, typename Op1_t>
356+
inline GEPLikeRecipe_match<Op0_t, Op1_t> m_GetElementPtr(const Op0_t &Op0,
357+
const Op1_t &Op1) {
358+
return GEPLikeRecipe_match<Op0_t, Op1_t>(Op0, Op1);
359+
}
360+
342361
template <typename Op0_t, typename Op1_t, typename Op2_t, unsigned Opcode>
343362
using AllTernaryRecipe_match =
344363
Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, Opcode, false,

‎llvm/test/Transforms/LoopVectorize/X86/induction-step.ll‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ define i16 @wide_sub_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
115115
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
116116
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
117117
; CHECK: middle.block:
118+
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i16> [[TMP10]], i32 3
118119
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
119120
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
120121
; CHECK: scalar.ph:
@@ -131,7 +132,7 @@ define i16 @wide_sub_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
131132
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
132133
; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
133134
; CHECK: exit:
134-
; CHECK-NEXT: [[SUB_LCSSA:%.*]] = phi i16 [ [[SUB]], [[LOOP]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ]
135+
; CHECK-NEXT: [[SUB_LCSSA:%.*]] = phi i16 [ [[SUB]], [[LOOP]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
135136
; CHECK-NEXT: ret i16 [[SUB_LCSSA]]
136137
;
137138
entry:

0 commit comments

Comments
 (0)