diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 5df1061691a67..02dc407e6d56b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7564,14 +7564,10 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(), CM, CM.CostKind); precomputeCosts(BestPlan, BestFactor.Width, CostCtx); - // Set PlanForEarlyExitLoop to true if the BestPlan has been built from a - // loop with an uncountable early exit. The legacy cost model doesn't - // properly model costs for such loops. - bool PlanForEarlyExitLoop = - BestPlan.getVectorLoopRegion() && - BestPlan.getVectorLoopRegion()->getSingleSuccessor() != - BestPlan.getMiddleBlock(); - assert((BestFactor.Width == LegacyVF.Width || PlanForEarlyExitLoop || + // Verify that the VPlan-based and legacy cost models agree, except for VPlans + // with early exits and plans with additional VPlan simplifications. The + // legacy cost model doesn't properly model costs for such loops. + assert((BestFactor.Width == LegacyVF.Width || BestPlan.hasEarlyExit() || planContainsAdditionalSimplifications(getPlanFor(BestFactor.Width), CostCtx, OrigLoop) || planContainsAdditionalSimplifications(getPlanFor(LegacyVF.Width), @@ -7782,7 +7778,7 @@ DenseMap LoopVectorizationPlanner::executePlan( // 2.5 When vectorizing the epilogue, fix reduction resume values from the // additional bypass block. if (VectorizingEpilogue) { - assert(!ILV.Legal->hasUncountableEarlyExit() && + assert(!BestVPlan.hasEarlyExit() && "Epilogue vectorisation not yet supported with early exits"); BasicBlock *PH = OrigLoop->getLoopPreheader(); BasicBlock *BypassBlock = ILV.getAdditionalBypassBlock(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index a98d0ecb9a33b..da7aef73f9df3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3768,6 +3768,14 @@ class VPlan { /// successors of the block in VPlan. The returned block is owned by the VPlan /// and deleted once the VPlan is destroyed. VPIRBasicBlock *createVPIRBasicBlock(BasicBlock *IRBB); + + /// Returns true if the VPlan is based on a loop with an early exit. That is + /// the case if the VPlan has either more than one exit block or a single exit + /// block with multiple predecessors (one for the exit via the latch and one + /// via the other early exit). + bool hasEarlyExit() const { + return ExitBlocks.size() > 1 || ExitBlocks[0]->getNumPredecessors() > 1; + } }; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)