Skip to content

Commit e89fd9a

Browse files
committed
[LV] Set correct costs for interleave group members.
This ensures each scalarized member has an accurate cost, matching the cost it would have if it would not have been considered for an interleave group. (cherry picked from commit 49605a4)
1 parent 302beb5 commit e89fd9a

File tree

3 files changed

+73
-299
lines changed

3 files changed

+73
-299
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5695,9 +5695,18 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
56955695
// If the instructions belongs to an interleave group, the whole group
56965696
// receives the same decision. The whole group receives the cost, but
56975697
// the cost will actually be assigned to one instruction.
5698-
if (const auto *Group = getInterleavedAccessGroup(&I))
5699-
setWideningDecision(Group, VF, Decision, Cost);
5700-
else
5698+
if (const auto *Group = getInterleavedAccessGroup(&I)) {
5699+
if (Decision == CM_Scalarize) {
5700+
for (unsigned Idx = 0; Idx < Group->getFactor(); ++Idx) {
5701+
if (auto *I = Group->getMember(Idx)) {
5702+
setWideningDecision(I, VF, Decision,
5703+
getMemInstScalarizationCost(I, VF));
5704+
}
5705+
}
5706+
} else {
5707+
setWideningDecision(Group, VF, Decision, Cost);
5708+
}
5709+
} else
57015710
setWideningDecision(&I, VF, Decision, Cost);
57025711
}
57035712
}

0 commit comments

Comments
 (0)