@@ -2366,61 +2366,57 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
2366
2366
}
2367
2367
2368
2368
void VPlanTransforms::simplifyEVLIVs (VPlan &Plan) {
2369
- auto ConvertEVLPhi = [](VPlan &Plan, VPBasicBlock *Entry,
2370
- VPEVLBasedIVPHIRecipe *EVLPhi) {
2371
- using namespace llvm ::VPlanPatternMatch;
2372
- VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2373
-
2374
- // Convert EVLPhi to concrete recipe.
2375
- auto *ScalarR = VPBuilder (EVLPhi).createScalarPhi (
2376
- {EVLPhi->getStartValue (), EVLIncrement}, EVLPhi->getDebugLoc (),
2377
- " evl.based.iv" );
2378
- EVLPhi->replaceAllUsesWith (ScalarR);
2379
- EVLPhi->eraseFromParent ();
2380
-
2381
- // Find the latch-exiting block and convert to variable-length stepping.
2382
- // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2383
- // After: (branch-on-count EVLIVInc, TripCount)
2384
- auto Range =
2385
- VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2386
- auto It = find_if (Range, [&](VPBasicBlock *VPBB) {
2387
- return any_of (VPBB->successors (),
2388
- [&](VPBlockBase *Succ) { return Succ == Entry; });
2389
- });
2390
- assert ((It != Range.end ()) && " LatchExiting is not found" );
2391
- VPBasicBlock *LatchExiting = *It;
2392
- auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2393
- VPValue *ScalarIVInc;
2394
- assert (LatchExitingBr &&
2395
- match (LatchExitingBr,
2396
- m_BranchOnCount (m_VPValue (ScalarIVInc),
2397
- m_Specific (&Plan.getVectorTripCount ()))) &&
2398
- " Unexpected terminator in EVL loop" );
2399
- LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2400
- ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2401
- VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2402
- VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2403
- IVIncR->eraseFromParent ();
2404
- ScalarIV->eraseFromParent ();
2405
- };
2406
-
2369
+ using namespace llvm ::VPlanPatternMatch;
2407
2370
// Find EVL loop entries by locating VPEVLBasedIVPHIRecipe
2408
2371
// There should be only one EVL PHI in the entire plan
2409
2372
VPEVLBasedIVPHIRecipe *EVLPhi = nullptr ;
2410
- VPBasicBlock *EVLPhiBlock = nullptr ;
2411
2373
2412
2374
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2413
2375
vp_depth_first_shallow (Plan.getEntry ())))
2414
2376
for (VPRecipeBase &R : VPBB->phis ())
2415
2377
if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
2416
- assert (!EVLPhi && " Found multiple EVL PHIs - only one expected" );
2378
+ assert (!EVLPhi && " Found multiple EVL PHIs. Only one expected" );
2417
2379
EVLPhi = PhiR;
2418
- EVLPhiBlock = VPBB;
2419
2380
}
2420
2381
2421
- // Process the single EVL PHI if found
2422
- if (EVLPhi)
2423
- ConvertEVLPhi (Plan, EVLPhiBlock, EVLPhi);
2382
+ // Early return if no EVL PHI is found
2383
+ if (!EVLPhi)
2384
+ return ;
2385
+
2386
+ VPBasicBlock *Entry = EVLPhi->getParent ();
2387
+ VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2388
+
2389
+ // Convert EVLPhi to concrete recipe.
2390
+ auto *ScalarR =
2391
+ VPBuilder (EVLPhi).createScalarPhi ({EVLPhi->getStartValue (), EVLIncrement},
2392
+ EVLPhi->getDebugLoc (), " evl.based.iv" );
2393
+ EVLPhi->replaceAllUsesWith (ScalarR);
2394
+ EVLPhi->eraseFromParent ();
2395
+
2396
+ // Find the latch-exiting block and convert to variable-length stepping.
2397
+ // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2398
+ // After: (branch-on-count EVLIVInc, TripCount)
2399
+ auto Range =
2400
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2401
+ auto It = find_if (Range, [&Entry](VPBasicBlock *VPBB) {
2402
+ return any_of (VPBB->successors (),
2403
+ [&Entry](VPBlockBase *Succ) { return Succ == Entry; });
2404
+ });
2405
+ assert ((It != Range.end ()) && " LatchExiting is not found" );
2406
+ VPBasicBlock *LatchExiting = *It;
2407
+ auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2408
+ VPValue *ScalarIVInc;
2409
+ assert (LatchExitingBr &&
2410
+ match (LatchExitingBr,
2411
+ m_BranchOnCount (m_VPValue (ScalarIVInc),
2412
+ m_Specific (&Plan.getVectorTripCount ()))) &&
2413
+ " Unexpected terminator in EVL loop" );
2414
+ LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2415
+ ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2416
+ VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2417
+ VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2418
+ IVIncR->eraseFromParent ();
2419
+ ScalarIV->eraseFromParent ();
2424
2420
}
2425
2421
2426
2422
void VPlanTransforms::dropPoisonGeneratingRecipes (
0 commit comments