@@ -9141,7 +9141,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
91419141 continue ;
91429142
91439143 const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor ();
9144- auto *Result = PhiR->getBackedgeValue ()-> getDefiningRecipe ();
9144+ auto *NewExitingVPV = PhiR->getBackedgeValue ();
91459145 // If tail is folded by masking, introduce selects between the phi
91469146 // and the live-out instruction of each reduction, at the beginning of the
91479147 // dedicated latch block.
@@ -9151,21 +9151,20 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
91519151 VPValue *Red = PhiR->getBackedgeValue ();
91529152 assert (Red->getDefiningRecipe ()->getParent () != LatchVPBB &&
91539153 " reduction recipe must be defined before latch" );
9154- FastMathFlags FMFs = RdxDesc.getFastMathFlags ();
91559154 Type *PhiTy = PhiR->getOperand (0 )->getLiveInIRValue ()->getType ();
9156- Result =
9155+ std::optional<FastMathFlags> FMFs =
91579156 PhiTy->isFloatingPointTy ()
9158- ? new VPInstruction (Instruction::Select, {Cond, Red, PhiR}, FMFs )
9159- : new VPInstruction (Instruction::Select, {Cond, Red, PhiR}) ;
9160- Result-> insertBefore (&* Builder.getInsertPoint () );
9161- Red->replaceUsesWithIf (
9162- Result-> getVPSingleValue (),
9163- [](VPUser &U, unsigned ) { return isa<VPLiveOut>(&U); });
9157+ ? std::make_optional (RdxDesc. getFastMathFlags () )
9158+ : std:: nullopt ;
9159+ NewExitingVPV = Builder.createSelect (Cond, Red, PhiR, {}, " " , FMFs );
9160+ Red->replaceUsesWithIf (NewExitingVPV, [](VPUser &U, unsigned ) {
9161+ return isa<VPLiveOut>(&U);
9162+ });
91649163 if (PreferPredicatedReductionSelect ||
91659164 TTI.preferPredicatedReductionSelect (
91669165 PhiR->getRecurrenceDescriptor ().getOpcode (), PhiTy,
91679166 TargetTransformInfo::ReductionFlags ()))
9168- PhiR->setOperand (1 , Result-> getVPSingleValue () );
9167+ PhiR->setOperand (1 , NewExitingVPV );
91699168 }
91709169 // If the vector reduction can be performed in a smaller type, we truncate
91719170 // then extend the loop exit value to enable InstCombine to evaluate the
@@ -9174,17 +9173,17 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
91749173 if (MinVF.isVector () && PhiTy != RdxDesc.getRecurrenceType ()) {
91759174 assert (!PhiR->isInLoop () && " Unexpected truncated inloop reduction!" );
91769175 Type *RdxTy = RdxDesc.getRecurrenceType ();
9177- auto *Trunc = new VPWidenCastRecipe (Instruction::Trunc,
9178- Result-> getVPSingleValue () , RdxTy);
9176+ auto *Trunc =
9177+ new VPWidenCastRecipe (Instruction::Trunc, NewExitingVPV , RdxTy);
91799178 auto *Extnd =
91809179 RdxDesc.isSigned ()
91819180 ? new VPWidenCastRecipe (Instruction::SExt, Trunc, PhiTy)
91829181 : new VPWidenCastRecipe (Instruction::ZExt, Trunc, PhiTy);
91839182
9184- Trunc->insertAfter (Result );
9183+ Trunc->insertAfter (NewExitingVPV-> getDefiningRecipe () );
91859184 Extnd->insertAfter (Trunc);
9186- Result-> getVPSingleValue () ->replaceAllUsesWith (Extnd);
9187- Trunc->setOperand (0 , Result-> getVPSingleValue () );
9185+ NewExitingVPV ->replaceAllUsesWith (Extnd);
9186+ Trunc->setOperand (0 , NewExitingVPV );
91889187 }
91899188 }
91909189
0 commit comments