@@ -874,34 +874,31 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
874874
875875 VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock ();
876876 VPBuilder LatchBuilder (LatchVPBB->getTerminator ());
877- VPValue *IsNaNLane = nullptr ;
877+ VPValue *AllNaNLanes = nullptr ;
878878 SmallPtrSet<VPValue *, 2 > RdxResults;
879- for (const auto &[RedPhiR, MinMaxOp] : MinMaxNumReductionsToHandle) {
880- assert (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind (
881- RedPhiR->getRecurrenceKind ()) &&
882- " unsupported reduction" );
883-
884- VPValue *IsNaN =
879+ for (const auto &[_, MinMaxOp] : MinMaxNumReductionsToHandle) {
880+ VPValue *RedNaNLanes =
885881 LatchBuilder.createFCmp (CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
886- IsNaNLane = IsNaNLane ? LatchBuilder.createOr (IsNaNLane, IsNaN) : IsNaN;
882+ AllNaNLanes = AllNaNLanes ? LatchBuilder.createOr (AllNaNLanes, RedNaNLanes)
883+ : RedNaNLanes;
887884 }
888885
889886 VPValue *AnyNaNLane =
890- LatchBuilder.createNaryOp (VPInstruction::AnyOf, {IsNaNLane });
887+ LatchBuilder.createNaryOp (VPInstruction::AnyOf, {AllNaNLanes });
891888 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock ();
892889 VPBuilder MiddleBuilder (MiddleVPBB, MiddleVPBB->begin ());
893- for (const auto &[RedPhiR, MinMaxOp] : MinMaxNumReductionsToHandle) {
894- // If we exit early due to NaNs, compute the final reduction result based
895- // on the reduction phi at the beginning of the last vector iteration.
890+ for (const auto &[RedPhiR, _] : MinMaxNumReductionsToHandle) {
891+ assert (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind (
892+ RedPhiR->getRecurrenceKind ()) &&
893+ " unsupported reduction" );
894+
895+ // If we exit early due to NaNs, compute the final reduction result based on
896+ // the reduction phi at the beginning of the last vector iteration.
896897 auto *RdxResult = find_singleton<VPSingleDefRecipe>(
897- RedPhiR->getBackedgeValue ()->users (),
898- [RedPhiR = RedPhiR](VPUser *U, bool ) -> VPSingleDefRecipe * {
898+ RedPhiR->users (), [](VPUser *U, bool ) -> VPSingleDefRecipe * {
899899 auto *VPI = dyn_cast<VPInstruction>(U);
900900 if (VPI && VPI->getOpcode () == VPInstruction::ComputeReductionResult)
901901 return VPI;
902- assert (U == RedPhiR &&
903- " Backedge value must only be used by "
904- " ComputeReductionResult and the reduction phi" );
905902 return nullptr ;
906903 });
907904
0 commit comments