Skip to content

Commit 88aab08

Browse files
committed
[LV] Check for hoisted safe-div selects in planContainsAdditionalSimp.
In some cases, safe-divisor selects can be hoisted out of the vector loop. Catching all cases in the legacy cost model isn't possible, in particular checking if all conditions guarding a division are loop invariant. Instead, check in planContainsAdditionalSimplifications if there are any hoisted safe-divisor selects. If so, don't compare to the more inaccurate legacy cost model. Fixes #160354. Fixes #160356.
1 parent fc3a27f commit 88aab08

File tree

2 files changed

+286
-193
lines changed

2 files changed

+286
-193
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2907,15 +2907,12 @@ LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
29072907

29082908
InstructionCost SafeDivisorCost = 0;
29092909
auto *VecTy = toVectorTy(I->getType(), VF);
2910-
auto *DivisorI = dyn_cast<Instruction>(I->getOperand(1));
2911-
if (DivisorI && !Legal->isInvariant(DivisorI)) {
2912-
// The cost of the select guard to ensure all lanes are well defined
2913-
// after we speculate above any internal control flow.
2914-
SafeDivisorCost +=
2915-
TTI.getCmpSelInstrCost(Instruction::Select, VecTy,
2916-
toVectorTy(Type::getInt1Ty(I->getContext()), VF),
2917-
CmpInst::BAD_ICMP_PREDICATE, CostKind);
2918-
}
2910+
// The cost of the select guard to ensure all lanes are well defined
2911+
// after we speculate above any internal control flow.
2912+
SafeDivisorCost +=
2913+
TTI.getCmpSelInstrCost(Instruction::Select, VecTy,
2914+
toVectorTy(Type::getInt1Ty(I->getContext()), VF),
2915+
CmpInst::BAD_ICMP_PREDICATE, CostKind);
29192916

29202917
SmallVector<const Value *, 4> Operands(I->operand_values());
29212918
SafeDivisorCost += TTI.getArithmeticInstrCost(
@@ -6908,6 +6905,28 @@ static bool planContainsAdditionalSimplifications(VPlan &Plan,
69086905
return nullptr;
69096906
};
69106907

6908+
// Check if a select for a safe divisor was hoisted to the pre-header. If so,
6909+
// the select doesn't need to be considered for the vector loop cost; go with
6910+
// the more accurate VPlan-based cost model.
6911+
for (VPRecipeBase &R : *Plan.getVectorPreheader()) {
6912+
auto *VPI = dyn_cast<VPInstruction>(&R);
6913+
if (!VPI || VPI->getOpcode() != Instruction::Select ||
6914+
VPI->getNumUsers() != 1)
6915+
continue;
6916+
6917+
if (auto *WR = dyn_cast<VPWidenRecipe>(*VPI->user_begin())) {
6918+
switch (WR->getOpcode()) {
6919+
case Instruction::UDiv:
6920+
case Instruction::SDiv:
6921+
case Instruction::URem:
6922+
case Instruction::SRem:
6923+
return true;
6924+
default:
6925+
break;
6926+
}
6927+
}
6928+
}
6929+
69116930
DenseSet<Instruction *> SeenInstrs;
69126931
auto Iter = vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry());
69136932
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {

0 commit comments

Comments
 (0)