Skip to content

Commit 0aac227

Browse files
committed
[LV] Correctly cost chains of replicating calls in legacy CM.
Check for scalarized calls in needsExtract to fix a divergence between legacy and VPlan-based cost model. The legacy cost model was missing a check for scalarized calls in needsExtract, which meant if incorrectly assumed the result of a scalarized call needs extracting. Exposed by #154617. Fixes #156091.
1 parent 86c4ef5 commit 0aac227

File tree

3 files changed

+606
-418
lines changed

3 files changed

+606
-418
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1134,7 +1134,10 @@ class LoopVectorizationCostModel {
11341134
CallWideningDecision getCallWideningDecision(CallInst *CI,
11351135
ElementCount VF) const {
11361136
assert(!VF.isScalar() && "Expected vector VF");
1137-
return CallWideningDecisions.at({CI, VF});
1137+
auto I = CallWideningDecisions.find({CI, VF});
1138+
if (I == CallWideningDecisions.end())
1139+
return {CM_Unknown, nullptr, Intrinsic::not_intrinsic, std::nullopt, 0};
1140+
return I->second;
11381141
}
11391142

11401143
/// Return True if instruction \p I is an optimizable truncate whose operand
@@ -1657,7 +1660,9 @@ class LoopVectorizationCostModel {
16571660
Instruction *I = dyn_cast<Instruction>(V);
16581661
if (VF.isScalar() || !I || !TheLoop->contains(I) ||
16591662
TheLoop->isLoopInvariant(I) ||
1660-
getWideningDecision(I, VF) == CM_Scalarize)
1663+
getWideningDecision(I, VF) == CM_Scalarize ||
1664+
(isa<CallInst>(I) &&
1665+
getCallWideningDecision(cast<CallInst>(I), VF).Kind == CM_Scalarize))
16611666
return false;
16621667

16631668
// Assume we can vectorize V (and hence we need extraction) if the

0 commit comments

Comments
 (0)