Skip to content

Commit c2e9fc6

Browse files
fhahnaokblast
authored andcommitted
[LV] Only skip scalarization overhead for members used as address.
Refine logic to scalarize interleave group member: only skip scalarization overhead for member being used as addresses. For others, use the regular scalar memory op cost. This currently doesn't trigger in practice as far as I could find, but fixes a potential divergence between VPlan- and legacy cost models. It fixes a concrete divergence with a follow-up patch, llvm#161276.
1 parent 65a34f6 commit c2e9fc6

File tree

1 file changed

+12
-7
lines changed

1 file changed

+12
-7
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5750,13 +5750,18 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
57505750
getMemoryInstructionCost(I, ElementCount::getFixed(1))));
57515751
UpdateMemOpUserCost(cast<LoadInst>(I));
57525752
} else if (const auto *Group = getInterleavedAccessGroup(I)) {
5753-
// Scalarize an interleave group of address loads.
5754-
for (unsigned I = 0; I < Group->getFactor(); ++I) {
5755-
if (Instruction *Member = Group->getMember(I)) {
5756-
setWideningDecision(
5757-
Member, VF, CM_Scalarize,
5758-
(VF.getKnownMinValue() *
5759-
getMemoryInstructionCost(Member, ElementCount::getFixed(1))));
5753+
// Scalarize all members of this interleaved group when any member
5754+
// is used as an address. The address-used load skips scalarization
5755+
// overhead, other members include it.
5756+
for (unsigned Idx = 0; Idx < Group->getFactor(); ++Idx) {
5757+
if (Instruction *Member = Group->getMember(Idx)) {
5758+
InstructionCost Cost =
5759+
AddrDefs.contains(Member)
5760+
? (VF.getKnownMinValue() *
5761+
getMemoryInstructionCost(Member,
5762+
ElementCount::getFixed(1)))
5763+
: getMemInstScalarizationCost(Member, VF);
5764+
setWideningDecision(Member, VF, CM_Scalarize, Cost);
57605765
UpdateMemOpUserCost(cast<LoadInst>(Member));
57615766
}
57625767
}

0 commit comments

Comments
 (0)