Skip to content

Commit 040d9c9

Browse files
committed
[VPlan] Collect FMFs for in-loop reduction chain in VPlan. (NFC)
Replace retrieving FMFs for in-loop reduction via underlying instruction + legal by collecting the flags during reduction chain traversal in VPlan.
1 parent 5c43385 commit 040d9c9

File tree

1 file changed

+14
-7
lines changed

1 file changed

+14
-7
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8665,6 +8665,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
86658665
!RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
86668666
"AnyOf and FindIV reductions are not allowed for in-loop reductions");
86678667

8668+
bool IsFPRecurrence =
8669+
RecurrenceDescriptor::isFloatingPointRecurrenceKind(Kind);
8670+
FastMathFlags FMFs =
8671+
IsFPRecurrence ? FastMathFlags::getFast() : FastMathFlags();
8672+
86688673
// Collect the chain of "link" recipes for the reduction starting at PhiR.
86698674
SetVector<VPSingleDefRecipe *> Worklist;
86708675
Worklist.insert(PhiR);
@@ -8705,6 +8710,15 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
87058710
continue;
87068711
}
87078712

8713+
if (IsFPRecurrence) {
8714+
FastMathFlags CurFMF =
8715+
cast<VPRecipeWithIRFlags>(CurrentLink)->getFastMathFlags();
8716+
if (match(CurrentLink, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
8717+
CurFMF |= cast<VPRecipeWithIRFlags>(CurrentLink->getOperand(0))
8718+
->getFastMathFlags();
8719+
FMFs &= CurFMF;
8720+
}
8721+
87088722
Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
87098723

87108724
// Index of the first operand which holds a non-mask vector operand.
@@ -8772,13 +8786,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
87728786
if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
87738787
CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
87748788

8775-
// TODO: Retrieve FMFs from recipes directly.
8776-
RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor(
8777-
cast<PHINode>(PhiR->getUnderlyingInstr()));
8778-
// Non-FP RdxDescs will have all fast math flags set, so clear them.
8779-
FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
8780-
? RdxDesc.getFastMathFlags()
8781-
: FastMathFlags();
87828789
auto *RedRecipe = new VPReductionRecipe(
87838790
Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
87848791
PhiR->isOrdered(), CurrentLinkI->getDebugLoc());

0 commit comments

Comments
 (0)