@@ -2391,61 +2391,57 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
23912391}
23922392
23932393void VPlanTransforms::simplifyEVLIVs (VPlan &Plan) {
2394- auto ConvertEVLPhi = [](VPlan &Plan, VPBasicBlock *Entry,
2395- VPEVLBasedIVPHIRecipe *EVLPhi) {
2396- using namespace llvm ::VPlanPatternMatch;
2397- VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2398-
2399- // Convert EVLPhi to concrete recipe.
2400- auto *ScalarR = VPBuilder (EVLPhi).createScalarPhi (
2401- {EVLPhi->getStartValue (), EVLIncrement}, EVLPhi->getDebugLoc (),
2402- " evl.based.iv" );
2403- EVLPhi->replaceAllUsesWith (ScalarR);
2404- EVLPhi->eraseFromParent ();
2405-
2406- // Find the latch-exiting block and convert to variable-length stepping.
2407- // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2408- // After: (branch-on-count EVLIVInc, TripCount)
2409- auto Range =
2410- VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2411- auto It = find_if (Range, [&](VPBasicBlock *VPBB) {
2412- return any_of (VPBB->successors (),
2413- [&](VPBlockBase *Succ) { return Succ == Entry; });
2414- });
2415- assert ((It != Range.end ()) && " LatchExiting is not found" );
2416- VPBasicBlock *LatchExiting = *It;
2417- auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2418- VPValue *ScalarIVInc;
2419- assert (LatchExitingBr &&
2420- match (LatchExitingBr,
2421- m_BranchOnCount (m_VPValue (ScalarIVInc),
2422- m_Specific (&Plan.getVectorTripCount ()))) &&
2423- " Unexpected terminator in EVL loop" );
2424- LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2425- ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2426- VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2427- VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2428- IVIncR->eraseFromParent ();
2429- ScalarIV->eraseFromParent ();
2430- };
2431-
2394+ using namespace llvm ::VPlanPatternMatch;
24322395 // Find EVL loop entries by locating VPEVLBasedIVPHIRecipe
24332396 // There should be only one EVL PHI in the entire plan
24342397 VPEVLBasedIVPHIRecipe *EVLPhi = nullptr ;
2435- VPBasicBlock *EVLPhiBlock = nullptr ;
24362398
24372399 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
24382400 vp_depth_first_shallow (Plan.getEntry ())))
24392401 for (VPRecipeBase &R : VPBB->phis ())
24402402 if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
2441- assert (!EVLPhi && " Found multiple EVL PHIs - only one expected" );
2403+ assert (!EVLPhi && " Found multiple EVL PHIs. Only one expected" );
24422404 EVLPhi = PhiR;
2443- EVLPhiBlock = VPBB;
24442405 }
24452406
2446- // Process the single EVL PHI if found
2447- if (EVLPhi)
2448- ConvertEVLPhi (Plan, EVLPhiBlock, EVLPhi);
2407+ // Early return if no EVL PHI is found
2408+ if (!EVLPhi)
2409+ return ;
2410+
2411+ VPBasicBlock *Entry = EVLPhi->getParent ();
2412+ VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2413+
2414+ // Convert EVLPhi to concrete recipe.
2415+ auto *ScalarR =
2416+ VPBuilder (EVLPhi).createScalarPhi ({EVLPhi->getStartValue (), EVLIncrement},
2417+ EVLPhi->getDebugLoc (), " evl.based.iv" );
2418+ EVLPhi->replaceAllUsesWith (ScalarR);
2419+ EVLPhi->eraseFromParent ();
2420+
2421+ // Find the latch-exiting block and convert to variable-length stepping.
2422+ // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2423+ // After: (branch-on-count EVLIVInc, TripCount)
2424+ auto Range =
2425+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2426+ auto It = find_if (Range, [&Entry](VPBasicBlock *VPBB) {
2427+ return any_of (VPBB->successors (),
2428+ [&Entry](VPBlockBase *Succ) { return Succ == Entry; });
2429+ });
2430+ assert ((It != Range.end ()) && " LatchExiting is not found" );
2431+ VPBasicBlock *LatchExiting = *It;
2432+ auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2433+ VPValue *ScalarIVInc;
2434+ assert (LatchExitingBr &&
2435+ match (LatchExitingBr,
2436+ m_BranchOnCount (m_VPValue (ScalarIVInc),
2437+ m_Specific (&Plan.getVectorTripCount ()))) &&
2438+ " Unexpected terminator in EVL loop" );
2439+ LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2440+ ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2441+ VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2442+ VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2443+ IVIncR->eraseFromParent ();
2444+ ScalarIV->eraseFromParent ();
24492445}
24502446
24512447void VPlanTransforms::dropPoisonGeneratingRecipes (
0 commit comments