Skip to content

[LV] Use VPReductionRecipe for partial reductions #147513

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: users/SamTebbs33/bundle-constant-mul
Choose a base branch
from
Open

[LV] Use VPReductionRecipe for partial reductions #147513

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9130747
[LV] Use VPReductionRecipe for partial reductions
SamTebbs33 Jun 18, 2025
901297f
Remove outer-loop comment and re-design getVFScaleFactor interface
SamTebbs33 Oct 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 20 additions & 13 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6937,7 +6937,8 @@ static bool planContainsAdditionalSimplifications(VPlan &Plan,
}
// The VPlan-based cost model is more accurate for partial reduction and
// comparing against the legacy cost isn't desirable.
if (isa<VPPartialReductionRecipe>(&R))
if (auto *VPR = dyn_cast<VPReductionRecipe>(&R);
VPR && VPR->isPartialReduction())
return true;

// The VPlan-based cost model can analyze if recipes are scalar
Expand Down Expand Up @@ -8056,11 +8057,15 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));

// If the PHI is used by a partial reduction, set the scale factor.
bool UseInLoopReduction = CM.isInLoopReduction(Phi);
bool UseOrderedReductions = CM.useOrderedReductions(RdxDesc);
unsigned ScaleFactor =
getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
PhiRecipe = new VPReductionPHIRecipe(
Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi),
CM.useOrderedReductions(RdxDesc), ScaleFactor);
(UseOrderedReductions || UseInLoopReduction)
? 0
: getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
PhiRecipe =
new VPReductionPHIRecipe(Phi, RdxDesc.getRecurrenceKind(), *StartV,
UseOrderedReductions, ScaleFactor);
} else {
// TODO: Currently fixed-order recurrences are modeled as chains of
// first-order recurrences. If there are no users of the intermediate
Expand Down Expand Up @@ -8128,7 +8133,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
VPValue *Accumulator = Operands[1];
VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
isa<VPPartialReductionRecipe>(BinOpRecipe))
(isa<VPReductionRecipe>(BinOpRecipe) &&
cast<VPReductionRecipe>(BinOpRecipe)->isPartialReduction()))
std::swap(BinOp, Accumulator);

if (ScaleFactor !=
Expand All @@ -8153,12 +8159,10 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
"Expected an ADD or SUB operation for predicated partial "
"reductions (because the neutral element in the mask is zero)!");
Cond = getBlockInMask(Builder.getInsertBlock());
VPValue *Zero =
Plan.getOrAddLiveIn(ConstantInt::get(Reduction->getType(), 0));
BinOp = Builder.createSelect(Cond, BinOp, Zero, Reduction->getDebugLoc());
}
return new VPPartialReductionRecipe(ReductionOpcode, Accumulator, BinOp, Cond,
ScaleFactor, Reduction);

return new VPReductionRecipe(RecurKind::Add, FastMathFlags(), Reduction,
Accumulator, BinOp, Cond, false, ScaleFactor);
}

void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
Expand Down Expand Up @@ -8669,9 +8673,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
? RdxDesc.getFastMathFlags()
: FastMathFlags();
bool UseOrderedReductions = PhiR->isOrdered();
unsigned VFScaleFactor = !UseOrderedReductions;
auto *RedRecipe = new VPReductionRecipe(
Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
PhiR->isOrdered(), CurrentLinkI->getDebugLoc());
UseOrderedReductions, VFScaleFactor, CurrentLinkI->getDebugLoc());
// Append the recipe to the end of the VPBasicBlock because we need to
// ensure that it comes after all of it's inputs, including CondOp.
// Delete CurrentLink as it will be invalid if its operand is replaced
Expand Down Expand Up @@ -8706,8 +8712,9 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
// Don't output selects for partial reductions because they have an output
// with fewer lanes than the VF. So the operands of the select would have
// different numbers of lanes. Partial reductions mask the input instead.
auto *RR = dyn_cast<VPReductionRecipe>(OrigExitingVPV->getDefiningRecipe());
if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
!isa<VPPartialReductionRecipe>(OrigExitingVPV->getDefiningRecipe())) {
(!RR || !RR->isPartialReduction())) {
VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
std::optional<FastMathFlags> FMFs =
PhiTy->isFloatingPointTy()
Expand Down
135 changes: 46 additions & 89 deletions llvm/lib/Transforms/Vectorize/VPlan.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -558,7 +558,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPWidenIntOrFpInductionSC:
case VPRecipeBase::VPWidenPointerInductionSC:
case VPRecipeBase::VPReductionPHISC:
case VPRecipeBase::VPPartialReductionSC:
return true;
case VPRecipeBase::VPBranchOnMaskSC:
case VPRecipeBase::VPInterleaveEVLSC:
Expand Down Expand Up @@ -2333,32 +2332,35 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
/// The recurrence kind of the reduction.
const RecurKind Kind;

/// The phi is part of an in-loop reduction.
bool IsInLoop;

/// The phi is part of an ordered reduction. Requires IsInLoop to be true.
bool IsOrdered;

/// When expanding the reduction PHI, the plan's VF element count is divided
/// by this factor to form the reduction phi's VF.
unsigned VFScaleFactor = 1;
/// The scaling factor, relative to the VF, that this recipe's output is
/// divided by.
/// For outer-loop reductions this is equal to 1.
/// For in-loop reductions this is equal to 0, to specify that this is equal
/// to the VF (which may not be known yet). For partial-reductions this is
/// equal to another scalar value.
unsigned VFScaleFactor;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer not to overload the meaning of this variable, and keep IsInLoop semantically separate from the scale factor for partial reductions.

I wonder if a variant with empty types would help here, something like:

struct RdxOrderedInLoop {};
struct RdxInLoop {};
struct RdxNormal {};
struct RdxPartial {
  unsigned VFScaleFactor;
};
std::variant<RdxOrderedInLoop, RdxInLoop, RdxNormal, RdxPartial> RdxStyle;

That way we can remove the separate boolean flags and asserts that they are consistent, and we don't need to overload the meaning of the scale factor.

(I checked that this has been used elsewhere in the codebase; there's a couple places so far, but LockFileManager.h has the best example).

@fhahn any objections?


public:
/// Create a new VPReductionPHIRecipe for the reduction \p Phi.
VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
bool IsInLoop = false, bool IsOrdered = false,
unsigned VFScaleFactor = 1)
bool IsOrdered = false, unsigned VFScaleFactor = 1)
: VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind),
IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
assert((!IsOrdered || isInLoop()) &&
"IsOrdered requires the reduction to be in-loop");
assert(((!isInLoop() && !IsOrdered) || isInLoop()) &&
"Invalid VFScaleFactor");
}

~VPReductionPHIRecipe() override = default;

VPReductionPHIRecipe *clone() override {
auto *R = new VPReductionPHIRecipe(
dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
*getOperand(0), IsInLoop, IsOrdered, VFScaleFactor);
*getOperand(0), IsOrdered, VFScaleFactor);
R->addOperand(getBackedgeValue());
return R;
}
Expand Down Expand Up @@ -2388,8 +2390,11 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
/// Returns true, if the phi is part of an ordered reduction.
bool isOrdered() const { return IsOrdered; }

/// Returns true, if the phi is part of an in-loop reduction.
bool isInLoop() const { return IsInLoop; }
/// Returns true if the phi is part of an in-loop reduction.
bool isInLoop() const { return VFScaleFactor == 0; }

/// Returns true if the reduction outputs a vector with a scaled down VF.
bool isPartialReduction() const { return VFScaleFactor > 1; }

/// Returns true if the recipe only uses the first lane of operand \p Op.
bool onlyFirstLaneUsed(const VPValue *Op) const override {
Expand Down Expand Up @@ -2661,23 +2666,32 @@ class LLVM_ABI_FOR_TEST VPInterleaveEVLRecipe final : public VPInterleaveBase {
}
};

/// A recipe to represent inloop reduction operations, performing a reduction on
/// a vector operand into a scalar value, and adding the result to a chain.
/// The Operands are {ChainOp, VecOp, [Condition]}.
/// A recipe to represent inloop, ordered or partial reduction operations. It
/// performs a reduction on a vector operand into a scalar (vector in the case
/// of a partial reduction) value, and adds the result to a chain. The Operands
/// are {ChainOp, VecOp, [Condition]}.
class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
/// The recurrence kind for the reduction in question.
RecurKind RdxKind;
bool IsOrdered;
/// Whether the reduction is conditional.
bool IsConditional = false;
/// The scaling factor, relative to the VF, that this recipe's output is
/// divided by.
/// For outer-loop reductions this is equal to 1.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I'm getting a bit confused. In which case do we get a VPReductionRecipe for "outer-loop" reductions? I thought it was only used when the reduction result was computed inside the loop (sometimes partially). For "outer-loop" reductions, don't we "widen everything" and compute the reduction result in the middle block?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you're right actually! I've removed that part of the comment.

/// For in-loop reductions this is equal to 0, to specify that this is equal
/// to the VF (which may not be known yet).
/// For partial-reductions this is equal to another scalar value.
unsigned VFScaleFactor;

protected:
VPReductionRecipe(const unsigned char SC, RecurKind RdxKind,
FastMathFlags FMFs, Instruction *I,
ArrayRef<VPValue *> Operands, VPValue *CondOp,
bool IsOrdered, DebugLoc DL)
bool IsOrdered, unsigned VFScaleFactor, DebugLoc DL)
: VPRecipeWithIRFlags(SC, Operands, FMFs, DL), RdxKind(RdxKind),
IsOrdered(IsOrdered) {
IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
assert((!IsOrdered || VFScaleFactor == 0) && "Invalid scale factor");
if (CondOp) {
IsConditional = true;
addOperand(CondOp);
Expand All @@ -2688,30 +2702,31 @@ class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
public:
VPReductionRecipe(RecurKind RdxKind, FastMathFlags FMFs, Instruction *I,
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
bool IsOrdered, DebugLoc DL = DebugLoc::getUnknown())
bool IsOrdered, unsigned VFScaleFactor,
DebugLoc DL = DebugLoc::getUnknown())
: VPReductionRecipe(VPDef::VPReductionSC, RdxKind, FMFs, I,
ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
IsOrdered, DL) {}
IsOrdered, VFScaleFactor, DL) {}

VPReductionRecipe(const RecurKind RdxKind, FastMathFlags FMFs,
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
bool IsOrdered, DebugLoc DL = DebugLoc::getUnknown())
bool IsOrdered, unsigned VFScaleFactor,
DebugLoc DL = DebugLoc::getUnknown())
: VPReductionRecipe(VPDef::VPReductionSC, RdxKind, FMFs, nullptr,
ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
IsOrdered, DL) {}
IsOrdered, VFScaleFactor, DL) {}

~VPReductionRecipe() override = default;

VPReductionRecipe *clone() override {
return new VPReductionRecipe(RdxKind, getFastMathFlags(),
getUnderlyingInstr(), getChainOp(), getVecOp(),
getCondOp(), IsOrdered, getDebugLoc());
return new VPReductionRecipe(
RdxKind, getFastMathFlags(), getUnderlyingInstr(), getChainOp(),
getVecOp(), getCondOp(), IsOrdered, VFScaleFactor, getDebugLoc());
}

static inline bool classof(const VPRecipeBase *R) {
return R->getVPDefID() == VPRecipeBase::VPReductionSC ||
R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
R->getVPDefID() == VPRecipeBase::VPPartialReductionSC;
R->getVPDefID() == VPRecipeBase::VPReductionEVLSC;
}

static inline bool classof(const VPUser *U) {
Expand All @@ -2738,6 +2753,8 @@ class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
bool isOrdered() const { return IsOrdered; };
/// Return true if the in-loop reduction is conditional.
bool isConditional() const { return IsConditional; };
/// Returns true if the reduction outputs a vector with a scaled down VF.
bool isPartialReduction() const { return VFScaleFactor > 1; }
/// The VPValue of the scalar Chain being accumulated.
VPValue *getChainOp() const { return getOperand(0); }
/// The VPValue of the vector value to be reduced.
Expand All @@ -2746,68 +2763,8 @@ class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
VPValue *getCondOp() const {
return isConditional() ? getOperand(getNumOperands() - 1) : nullptr;
}
};

/// A recipe for forming partial reductions. In the loop, an accumulator and
/// vector operand are added together and passed to the next iteration as the
/// next accumulator. After the loop body, the accumulator is reduced to a
/// scalar value.
class VPPartialReductionRecipe : public VPReductionRecipe {
unsigned Opcode;

/// The divisor by which the VF of this recipe's output should be divided
/// during execution.
unsigned VFScaleFactor;

public:
VPPartialReductionRecipe(Instruction *ReductionInst, VPValue *Op0,
VPValue *Op1, VPValue *Cond, unsigned VFScaleFactor)
: VPPartialReductionRecipe(ReductionInst->getOpcode(), Op0, Op1, Cond,
VFScaleFactor, ReductionInst) {}
VPPartialReductionRecipe(unsigned Opcode, VPValue *Op0, VPValue *Op1,
VPValue *Cond, unsigned ScaleFactor,
Instruction *ReductionInst = nullptr)
: VPReductionRecipe(VPDef::VPPartialReductionSC, RecurKind::Add,
FastMathFlags(), ReductionInst,
ArrayRef<VPValue *>({Op0, Op1}), Cond, false, {}),
Opcode(Opcode), VFScaleFactor(ScaleFactor) {
[[maybe_unused]] auto *AccumulatorRecipe =
getChainOp()->getDefiningRecipe();
// When cloning as part of a VPExpressionRecipe the chain op could have
// replaced by a temporary VPValue, so it doesn't have a defining recipe.
assert((!AccumulatorRecipe ||
isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
isa<VPPartialReductionRecipe>(AccumulatorRecipe)) &&
"Unexpected operand order for partial reduction recipe");
}
~VPPartialReductionRecipe() override = default;

VPPartialReductionRecipe *clone() override {
return new VPPartialReductionRecipe(Opcode, getOperand(0), getOperand(1),
getCondOp(), VFScaleFactor,
getUnderlyingInstr());
}

VP_CLASSOF_IMPL(VPDef::VPPartialReductionSC)

/// Generate the reduction in the loop.
void execute(VPTransformState &State) override;

/// Return the cost of this VPPartialReductionRecipe.
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;

/// Get the binary op's opcode.
unsigned getOpcode() const { return Opcode; }

/// Get the factor that the VF of this recipe's output should be scaled by.
unsigned getVFScaleFactor() const { return VFScaleFactor; }
Copy link
Contributor

@gbossu gbossu Oct 20, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this interface is now a bit misleading. We might return 0, which actually means the reduction is "scaled" by whatever the loop VF is, not 0.

Maybe something like below would be clearer:

/// Get the factor that the VF of this recipe's output should be scaled by,
/// if it represents a partial reduction.
std::optional<unsigned> getPartialVFScaleFactor() const {
  return isPartial() ? VFScaleFactor : std::nullopt;
}

This way, it forces users of getPartialVFScaleFactor() to question what the result (or lack thereof) means. What do you think?

Edit: getPartialReductionScaleFactor() might be a clearer name.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like that, thanks. I'd prefer to keep the current name though.


#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
#endif
};

/// A recipe to represent inloop reduction operations with vector-predication
Expand All @@ -2823,7 +2780,7 @@ class LLVM_ABI_FOR_TEST VPReductionEVLRecipe : public VPReductionRecipe {
R.getFastMathFlags(),
cast_or_null<Instruction>(R.getUnderlyingValue()),
ArrayRef<VPValue *>({R.getChainOp(), R.getVecOp(), &EVL}), CondOp,
R.isOrdered(), DL) {}
R.isOrdered(), 0, DL) {}

~VPReductionEVLRecipe() override = default;

Expand Down
19 changes: 10 additions & 9 deletions llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,10 +286,10 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
[](const auto *R) { return R->getScalarType(); })
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
VPScalarIVStepsRecipe, VPWidenGEPRecipe, VPVectorPointerRecipe,
VPVectorEndPointerRecipe, VPWidenCanonicalIVRecipe,
VPPartialReductionRecipe>([this](const VPRecipeBase *R) {
return inferScalarType(R->getOperand(0));
})
VPVectorEndPointerRecipe, VPWidenCanonicalIVRecipe>(
[this](const VPRecipeBase *R) {
return inferScalarType(R->getOperand(0));
})
// VPInstructionWithType must be handled before VPInstruction.
.Case<VPInstructionWithType, VPWidenIntrinsicRecipe,
VPWidenCastRecipe>(
Expand Down Expand Up @@ -559,11 +559,12 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
// fewer lanes than the VF.
unsigned ScaleFactor =
vputils::getVFScaleFactor(VPV->getDefiningRecipe());
ElementCount VF = VFs[J].divideCoefficientBy(ScaleFactor);
LLVM_DEBUG(if (VF != VFs[J]) {
dbgs() << "LV(REG): Scaled down VF from " << VFs[J] << " to " << VF
<< " for " << *R << "\n";
});
ElementCount VF = VFs[J];
if (ScaleFactor > 1) {
VF = VFs[J].divideCoefficientBy(ScaleFactor);
LLVM_DEBUG(dbgs() << "LV(REG): Scaled down VF from " << VFs[J]
<< " to " << VF << " for " << *R << "\n";);
}

Type *ScalarTy = TypeInfo.inferScalarType(VPV);
unsigned ClassID = TTI.getRegisterClassForType(true, ScalarTy);
Expand Down
Loading
Loading