Skip to content

Commit 054cc9f

Browse files
committed
[LV] Correctly cost chains of replicating calls in legacy CM.
Check for scalarized calls in needsExtract to fix a divergence between legacy and VPlan-based cost model. The legacy cost model was missing a check for scalarized calls in needsExtract, which meant if incorrectly assumed the result of a scalarized call needs extracting. Exposed by llvm#154617. Fixes llvm#156091. (cherry picked from commit 0aac227)
1 parent b367457 commit 054cc9f

File tree

3 files changed

+610
-418
lines changed

3 files changed

+610
-418
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,7 +1152,10 @@ class LoopVectorizationCostModel {
11521152
CallWideningDecision getCallWideningDecision(CallInst *CI,
11531153
ElementCount VF) const {
11541154
assert(!VF.isScalar() && "Expected vector VF");
1155-
return CallWideningDecisions.at({CI, VF});
1155+
auto I = CallWideningDecisions.find({CI, VF});
1156+
if (I == CallWideningDecisions.end())
1157+
return {CM_Unknown, nullptr, Intrinsic::not_intrinsic, std::nullopt, 0};
1158+
return I->second;
11561159
}
11571160

11581161
/// Return True if instruction \p I is an optimizable truncate whose operand
@@ -1665,7 +1668,9 @@ class LoopVectorizationCostModel {
16651668
Instruction *I = dyn_cast<Instruction>(V);
16661669
if (VF.isScalar() || !I || !TheLoop->contains(I) ||
16671670
TheLoop->isLoopInvariant(I) ||
1668-
getWideningDecision(I, VF) == CM_Scalarize)
1671+
getWideningDecision(I, VF) == CM_Scalarize ||
1672+
(isa<CallInst>(I) &&
1673+
getCallWideningDecision(cast<CallInst>(I), VF).Kind == CM_Scalarize))
16691674
return false;
16701675

16711676
// Assume we can vectorize V (and hence we need extraction) if the

0 commit comments

Comments
 (0)