@@ -8665,6 +8665,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
86658665 !RecurrenceDescriptor::isFindIVRecurrenceKind (Kind) &&
86668666 " AnyOf and FindIV reductions are not allowed for in-loop reductions" );
86678667
8668+ bool IsFPRecurrence =
8669+ RecurrenceDescriptor::isFloatingPointRecurrenceKind (Kind);
8670+ FastMathFlags FMFs =
8671+ IsFPRecurrence ? FastMathFlags::getFast () : FastMathFlags ();
8672+
86688673 // Collect the chain of "link" recipes for the reduction starting at PhiR.
86698674 SetVector<VPSingleDefRecipe *> Worklist;
86708675 Worklist.insert (PhiR);
@@ -8705,6 +8710,15 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
87058710 continue ;
87068711 }
87078712
8713+ if (IsFPRecurrence) {
8714+ FastMathFlags CurFMF =
8715+ cast<VPRecipeWithIRFlags>(CurrentLink)->getFastMathFlags ();
8716+ if (match (CurrentLink, m_Select (m_VPValue (), m_VPValue (), m_VPValue ())))
8717+ CurFMF |= cast<VPRecipeWithIRFlags>(CurrentLink->getOperand (0 ))
8718+ ->getFastMathFlags ();
8719+ FMFs &= CurFMF;
8720+ }
8721+
87088722 Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr ();
87098723
87108724 // Index of the first operand which holds a non-mask vector operand.
@@ -8772,13 +8786,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
87728786 if (CM.blockNeedsPredicationForAnyReason (CurrentLinkI->getParent ()))
87738787 CondOp = RecipeBuilder.getBlockInMask (CurrentLink->getParent ());
87748788
8775- // TODO: Retrieve FMFs from recipes directly.
8776- RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor (
8777- cast<PHINode>(PhiR->getUnderlyingInstr ()));
8778- // Non-FP RdxDescs will have all fast math flags set, so clear them.
8779- FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
8780- ? RdxDesc.getFastMathFlags ()
8781- : FastMathFlags ();
87828789 auto *RedRecipe = new VPReductionRecipe (
87838790 Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
87848791 PhiR->isOrdered (), CurrentLinkI->getDebugLoc ());
0 commit comments