Skip to content

[SLP][NFC] Avoid the getRdxOperand hack #148672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[SLP][NFC] Avoid the getRdxOperand hack #148672

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8c3aacd
[SLP][NFC] Avoid trivial calls to getRdxOperand
pfusik Jul 14, 2025
d7589ce
[SLP][NFC] Avoid the getRdxOperand hack
pfusik Jul 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 32 additions & 47 deletions llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23469,16 +23469,6 @@ class HorizontalReduction {
return I->isAssociative();
}

static Value *getRdxOperand(Instruction *I, unsigned Index) {
// Poison-safe 'or' takes the form: select X, true, Y
// To make that work with the normal operand processing, we skip the
// true value operand.
// TODO: Change the code and data structures to handle this without a hack.
if (getRdxKind(I) == RecurKind::Or && isa<SelectInst>(I) && Index == 1)
return I->getOperand(2);
return I->getOperand(Index);
}

/// Creates reduction operation with the current opcode.
static Value *createOp(IRBuilderBase &Builder, RecurKind Kind, Value *LHS,
Value *RHS, const Twine &Name, bool UseSelect) {
Expand Down Expand Up @@ -23670,11 +23660,6 @@ class HorizontalReduction {
}

private:
/// Total number of operands in the reduction operation.
static unsigned getNumberOfOperands(Instruction *I) {
return isCmpSelMinMax(I) ? 3 : 2;
}

/// Checks if the instruction is in basic block \p BB.
/// For a cmp+sel min/max reduction check that both ops are in \p BB.
static bool hasSameParent(Instruction *I, BasicBlock *BB) {
Expand Down Expand Up @@ -23780,31 +23765,27 @@ class HorizontalReduction {
// Checks if the operands of the \p TreeN instruction are also reduction
// operations or should be treated as reduced values or an extra argument,
// which is not part of the reduction.
auto CheckOperands = [&](Instruction *TreeN,
SmallVectorImpl<Value *> &PossibleReducedVals,
SmallVectorImpl<Instruction *> &ReductionOps,
unsigned Level) {
for (int I : reverse(seq<int>(getFirstOperandIndex(TreeN),
getNumberOfOperands(TreeN)))) {
Value *EdgeVal = getRdxOperand(TreeN, I);
ReducedValsToOps[EdgeVal].push_back(TreeN);
auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
// If the edge is not an instruction, or it is different from the main
// reduction opcode or has too many uses - possible reduced value.
// Also, do not try to reduce const values, if the operation is not
// foldable.
if (!EdgeInst || Level > RecursionMaxDepth ||
getRdxKind(EdgeInst) != RdxKind ||
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst) ||
!hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) ||
!isVectorizable(RdxKind, EdgeInst) ||
(R.isAnalyzedReductionRoot(EdgeInst) &&
all_of(EdgeInst->operands(), IsaPred<Constant>))) {
PossibleReducedVals.push_back(EdgeVal);
continue;
}
auto CheckOperand = [&](Instruction *TreeN, int OperandIndex,
SmallVectorImpl<Value *> &PossibleReducedVals,
SmallVectorImpl<Instruction *> &ReductionOps,
unsigned Level) {
Value *EdgeVal = TreeN->getOperand(OperandIndex);
ReducedValsToOps[EdgeVal].push_back(TreeN);
auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
// If the edge is not an instruction, or it is different from the main
// reduction opcode or has too many uses - possible reduced value.
// Also, do not try to reduce const values, if the operation is not
// foldable.
if (!EdgeInst || Level > RecursionMaxDepth ||
getRdxKind(EdgeInst) != RdxKind ||
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst) ||
!hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) ||
!isVectorizable(RdxKind, EdgeInst) ||
(R.isAnalyzedReductionRoot(EdgeInst) &&
all_of(EdgeInst->operands(), IsaPred<Constant>)))
PossibleReducedVals.push_back(EdgeVal);
else
ReductionOps.push_back(EdgeInst);
}
};
// Try to regroup reduced values so that it gets more profitable to try to
// reduce them. Values are grouped by their value ids, instructions - by
Expand Down Expand Up @@ -23855,7 +23836,12 @@ class HorizontalReduction {
auto [TreeN, Level] = Worklist.pop_back_val();
SmallVector<Value *> PossibleRedVals;
SmallVector<Instruction *> PossibleReductionOps;
CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps, Level);
int I1 = getFirstOperandIndex(TreeN);
int I2 = isa<SelectInst>(TreeN) && match(TreeN, m_LogicalOr())
? 2 // Skip "true" in "select X, true, Y"
: I1 + 1;
CheckOperand(TreeN, I2, PossibleRedVals, PossibleReductionOps, Level);
CheckOperand(TreeN, I1, PossibleRedVals, PossibleReductionOps, Level);
addReductionOps(TreeN);
// Add reduction values. The values are sorted for better vectorization
// results.
Expand Down Expand Up @@ -23970,15 +23956,14 @@ class HorizontalReduction {
isGuaranteedNotToBePoison(VectorizedTree, AC) ||
(It != ReducedValsToOps.end() &&
any_of(It->getSecond(), [&](Instruction *I) {
return isBoolLogicOp(I) &&
getRdxOperand(I, 0) == VectorizedTree;
return isBoolLogicOp(I) && I->getOperand(0) == VectorizedTree;
}))) {
;
} else if (isGuaranteedNotToBePoison(Res, AC) ||
(It1 != ReducedValsToOps.end() &&
any_of(It1->getSecond(), [&](Instruction *I) {
return isBoolLogicOp(I) && getRdxOperand(I, 0) == Res;
}))) {
any_of(It1->getSecond(), [&](Instruction *I) {
return isBoolLogicOp(I) && I->getOperand(0) == Res;
}))) {
std::swap(VectorizedTree, Res);
} else {
VectorizedTree = Builder.CreateFreeze(VectorizedTree);
Expand Down Expand Up @@ -24467,11 +24452,11 @@ class HorizontalReduction {
if (!AnyBoolLogicOp)
return;
if (isBoolLogicOp(RedOp1) && ((!InitStep && LHS == VectorizedTree) ||
getRdxOperand(RedOp1, 0) == LHS ||
RedOp1->getOperand(0) == LHS ||
isGuaranteedNotToBePoison(LHS, AC)))
return;
if (isBoolLogicOp(RedOp2) && ((!InitStep && RHS == VectorizedTree) ||
getRdxOperand(RedOp2, 0) == RHS ||
RedOp2->getOperand(0) == RHS ||
isGuaranteedNotToBePoison(RHS, AC))) {
std::swap(LHS, RHS);
return;
Expand Down