Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 2 additions & 22 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7278,34 +7278,14 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,

// The legacy cost model has special logic to compute the cost of in-loop
// reductions, which may be smaller than the sum of all instructions involved
// in the reduction. For AnyOf reductions, VPlan codegen may remove the select
// which the legacy cost model uses to assign cost. Pre-compute their costs
// for now.
// in the reduction.
// TODO: Switch to costing based on VPlan once the logic has been ported.
for (const auto &[RedPhi, RdxDesc] : Legal->getReductionVars()) {
if (ForceTargetInstructionCost.getNumOccurrences())
continue;

if (!CM.isInLoopReduction(RedPhi) &&
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
RdxDesc.getRecurrenceKind()))
continue;

// AnyOf reduction codegen may remove the select. To match the legacy cost
// model, pre-compute the cost for AnyOf reductions here.
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
RdxDesc.getRecurrenceKind())) {
auto *Select = cast<SelectInst>(*find_if(
RedPhi->users(), [](User *U) { return isa<SelectInst>(U); }));
assert(!CostCtx.SkipCostComputation.contains(Select) &&
"reduction op visited multiple times");
CostCtx.SkipCostComputation.insert(Select);
auto ReductionCost = CostCtx.getLegacyCost(Select, VF);
LLVM_DEBUG(dbgs() << "Cost of " << ReductionCost << " for VF " << VF
<< ":\n any-of reduction " << *Select << "\n");
Cost += ReductionCost;
if (!CM.isInLoopReduction(RedPhi))
continue;
}

const auto &ChainOps = RdxDesc.getReductionOpChain(RedPhi, OrigLoop);
SetVector<Instruction *> ChainOpsAndOperands(ChainOps.begin(),
Expand Down
Loading