Skip to content

Commit 8633fcb

Browse files
committed
[LV]: Improve accuract of calculating remaining iterations out of MainLoop.
Account for vscale for vscale-based TC when calculating remaining iterations.
1 parent 8e4bda1 commit 8633fcb

File tree

2 files changed

+276
-327
lines changed

2 files changed

+276
-327
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4411,8 +4411,21 @@ VectorizationFactor LoopVectorizationPlanner::selectEpilogueVectorizationFactor(
44114411
const SCEV *TC =
44124412
vputils::getSCEVExprForVPValue(getPlanFor(MainLoopVF).getTripCount(), SE);
44134413
assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
4414+
4415+
// TODO: Maybe this could be removed when SCEV can evaluate expressions with
4416+
// 'vscale'.
4417+
// If TC is multiple of vscale, try to get estimated value:
4418+
if (match(TC, m_scev_Mul(m_SCEV(), m_SCEVVScale()))) {
4419+
std::optional<ElementCount> BestKnownTC =
4420+
getSmallBestKnownTC(PSE, OrigLoop);
4421+
if (BestKnownTC) {
4422+
unsigned EstimatedRuntimeTC =
4423+
estimateElementCount(*BestKnownTC, CM.getVScaleForTuning());
4424+
TC = SE.getConstant(TCType, EstimatedRuntimeTC);
4425+
}
4426+
}
44144427
RemainingIterations =
4415-
SE.getURemExpr(TC, SE.getElementCount(TCType, MainLoopVF * IC));
4428+
SE.getURemExpr(TC, SE.getElementCount(TCType, EstimatedRuntimeVF * IC));
44164429

44174430
// No iterations left to process in the epilogue.
44184431
if (RemainingIterations->isZero())

0 commit comments

Comments
 (0)