@@ -2447,12 +2447,26 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
24472447 };
24482448
24492449 TailFoldingStyle Style = Cost->getTailFoldingStyle ();
2450- if (Style == TailFoldingStyle::None)
2451- CheckMinIters =
2452- Builder.CreateICmp (P, Count, CreateStep (), " min.iters.check" );
2453- else if (VF.isScalable () &&
2454- !isIndvarOverflowCheckKnownFalse (Cost, VF, UF) &&
2455- Style != TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck) {
2450+ if (Style == TailFoldingStyle::None) {
2451+ Value *Step = CreateStep ();
2452+ ScalarEvolution &SE = *PSE.getSE ();
2453+ // TODO: Emit unconditional branch to vector preheader instead of
2454+ // conditional branch with known condition.
2455+ const SCEV *TripCountSCEV = SE.applyLoopGuards (SE.getSCEV (Count), OrigLoop);
2456+ // Check if the trip count is < the step.
2457+ if (SE.isKnownPredicate (P, TripCountSCEV, SE.getSCEV (Step))) {
2458+ // TODO: Ensure step is at most the trip count when determining max VF and
2459+ // UF, w/o tail folding.
2460+ CheckMinIters = Builder.getTrue ();
2461+ } else if (!SE.isKnownPredicate (CmpInst::getInversePredicate (P),
2462+ TripCountSCEV, SE.getSCEV (Step))) {
2463+ // Generate the minimum iteration check only if we cannot prove the
2464+ // check is known to be true, or known to be false.
2465+ CheckMinIters = Builder.CreateICmp (P, Count, Step, " min.iters.check" );
2466+ } // else step known to be < trip count, use CheckMinIters preset to false.
2467+ } else if (VF.isScalable () &&
2468+ !isIndvarOverflowCheckKnownFalse (Cost, VF, UF) &&
2469+ Style != TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck) {
24562470 // vscale is not necessarily a power-of-2, which means we cannot guarantee
24572471 // an overflow to zero when updating induction variables and so an
24582472 // additional overflow check is required before entering the vector loop.
@@ -2462,8 +2476,18 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
24622476 ConstantInt::get (CountTy, cast<IntegerType>(CountTy)->getMask ());
24632477 Value *LHS = Builder.CreateSub (MaxUIntTripCount, Count);
24642478
2479+ Value *Step = CreateStep ();
2480+ #ifndef NDEBUG
2481+ ScalarEvolution &SE = *PSE.getSE ();
2482+ const SCEV *TC2OverflowSCEV = SE.applyLoopGuards (SE.getSCEV (LHS), OrigLoop);
2483+ assert (
2484+ !isIndvarOverflowCheckKnownFalse (Cost, VF * UF) &&
2485+ !SE.isKnownPredicate (CmpInst::getInversePredicate (ICmpInst::ICMP_ULT),
2486+ TC2OverflowSCEV, SE.getSCEV (Step)) &&
2487+ " unexpectedly proved overflow check to be known" );
2488+ #endif
24652489 // Don't execute the vector loop if (UMax - n) < (VF * UF).
2466- CheckMinIters = Builder.CreateICmp (ICmpInst::ICMP_ULT, LHS, CreateStep () );
2490+ CheckMinIters = Builder.CreateICmp (ICmpInst::ICMP_ULT, LHS, Step );
24672491 }
24682492
24692493 // Create new preheader for vector loop.
0 commit comments