@@ -2391,61 +2391,57 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
2391
2391
}
2392
2392
2393
2393
void VPlanTransforms::simplifyEVLIVs (VPlan &Plan) {
2394
- auto ConvertEVLPhi = [](VPlan &Plan, VPBasicBlock *Entry,
2395
- VPEVLBasedIVPHIRecipe *EVLPhi) {
2396
- using namespace llvm ::VPlanPatternMatch;
2397
- VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2398
-
2399
- // Convert EVLPhi to concrete recipe.
2400
- auto *ScalarR = VPBuilder (EVLPhi).createScalarPhi (
2401
- {EVLPhi->getStartValue (), EVLIncrement}, EVLPhi->getDebugLoc (),
2402
- " evl.based.iv" );
2403
- EVLPhi->replaceAllUsesWith (ScalarR);
2404
- EVLPhi->eraseFromParent ();
2405
-
2406
- // Find the latch-exiting block and convert to variable-length stepping.
2407
- // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2408
- // After: (branch-on-count EVLIVInc, TripCount)
2409
- auto Range =
2410
- VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2411
- auto It = find_if (Range, [&](VPBasicBlock *VPBB) {
2412
- return any_of (VPBB->successors (),
2413
- [&](VPBlockBase *Succ) { return Succ == Entry; });
2414
- });
2415
- assert ((It != Range.end ()) && " LatchExiting is not found" );
2416
- VPBasicBlock *LatchExiting = *It;
2417
- auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2418
- VPValue *ScalarIVInc;
2419
- assert (LatchExitingBr &&
2420
- match (LatchExitingBr,
2421
- m_BranchOnCount (m_VPValue (ScalarIVInc),
2422
- m_Specific (&Plan.getVectorTripCount ()))) &&
2423
- " Unexpected terminator in EVL loop" );
2424
- LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2425
- ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2426
- VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2427
- VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2428
- IVIncR->eraseFromParent ();
2429
- ScalarIV->eraseFromParent ();
2430
- };
2431
-
2394
+ using namespace llvm ::VPlanPatternMatch;
2432
2395
// Find EVL loop entries by locating VPEVLBasedIVPHIRecipe
2433
2396
// There should be only one EVL PHI in the entire plan
2434
2397
VPEVLBasedIVPHIRecipe *EVLPhi = nullptr ;
2435
- VPBasicBlock *EVLPhiBlock = nullptr ;
2436
2398
2437
2399
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2438
2400
vp_depth_first_shallow (Plan.getEntry ())))
2439
2401
for (VPRecipeBase &R : VPBB->phis ())
2440
2402
if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
2441
- assert (!EVLPhi && " Found multiple EVL PHIs - only one expected" );
2403
+ assert (!EVLPhi && " Found multiple EVL PHIs. Only one expected" );
2442
2404
EVLPhi = PhiR;
2443
- EVLPhiBlock = VPBB;
2444
2405
}
2445
2406
2446
- // Process the single EVL PHI if found
2447
- if (EVLPhi)
2448
- ConvertEVLPhi (Plan, EVLPhiBlock, EVLPhi);
2407
+ // Early return if no EVL PHI is found
2408
+ if (!EVLPhi)
2409
+ return ;
2410
+
2411
+ VPBasicBlock *Entry = EVLPhi->getParent ();
2412
+ VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2413
+
2414
+ // Convert EVLPhi to concrete recipe.
2415
+ auto *ScalarR =
2416
+ VPBuilder (EVLPhi).createScalarPhi ({EVLPhi->getStartValue (), EVLIncrement},
2417
+ EVLPhi->getDebugLoc (), " evl.based.iv" );
2418
+ EVLPhi->replaceAllUsesWith (ScalarR);
2419
+ EVLPhi->eraseFromParent ();
2420
+
2421
+ // Find the latch-exiting block and convert to variable-length stepping.
2422
+ // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2423
+ // After: (branch-on-count EVLIVInc, TripCount)
2424
+ auto Range =
2425
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2426
+ auto It = find_if (Range, [&Entry](VPBasicBlock *VPBB) {
2427
+ return any_of (VPBB->successors (),
2428
+ [&Entry](VPBlockBase *Succ) { return Succ == Entry; });
2429
+ });
2430
+ assert ((It != Range.end ()) && " LatchExiting is not found" );
2431
+ VPBasicBlock *LatchExiting = *It;
2432
+ auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2433
+ VPValue *ScalarIVInc;
2434
+ assert (LatchExitingBr &&
2435
+ match (LatchExitingBr,
2436
+ m_BranchOnCount (m_VPValue (ScalarIVInc),
2437
+ m_Specific (&Plan.getVectorTripCount ()))) &&
2438
+ " Unexpected terminator in EVL loop" );
2439
+ LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2440
+ ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2441
+ VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2442
+ VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2443
+ IVIncR->eraseFromParent ();
2444
+ ScalarIV->eraseFromParent ();
2449
2445
}
2450
2446
2451
2447
void VPlanTransforms::dropPoisonGeneratingRecipes (
0 commit comments