@@ -2370,61 +2370,57 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
2370
2370
}
2371
2371
2372
2372
void VPlanTransforms::simplifyEVLIVs (VPlan &Plan) {
2373
- auto ConvertEVLPhi = [](VPlan &Plan, VPBasicBlock *Entry,
2374
- VPEVLBasedIVPHIRecipe *EVLPhi) {
2375
- using namespace llvm ::VPlanPatternMatch;
2376
- VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2377
-
2378
- // Convert EVLPhi to concrete recipe.
2379
- auto *ScalarR = VPBuilder (EVLPhi).createScalarPhi (
2380
- {EVLPhi->getStartValue (), EVLIncrement}, EVLPhi->getDebugLoc (),
2381
- " evl.based.iv" );
2382
- EVLPhi->replaceAllUsesWith (ScalarR);
2383
- EVLPhi->eraseFromParent ();
2384
-
2385
- // Find the latch-exiting block and convert to variable-length stepping.
2386
- // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2387
- // After: (branch-on-count EVLIVInc, TripCount)
2388
- auto Range =
2389
- VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2390
- auto It = find_if (Range, [&](VPBasicBlock *VPBB) {
2391
- return any_of (VPBB->successors (),
2392
- [&](VPBlockBase *Succ) { return Succ == Entry; });
2393
- });
2394
- assert ((It != Range.end ()) && " LatchExiting is not found" );
2395
- VPBasicBlock *LatchExiting = *It;
2396
- auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2397
- VPValue *ScalarIVInc;
2398
- assert (LatchExitingBr &&
2399
- match (LatchExitingBr,
2400
- m_BranchOnCount (m_VPValue (ScalarIVInc),
2401
- m_Specific (&Plan.getVectorTripCount ()))) &&
2402
- " Unexpected terminator in EVL loop" );
2403
- LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2404
- ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2405
- VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2406
- VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2407
- IVIncR->eraseFromParent ();
2408
- ScalarIV->eraseFromParent ();
2409
- };
2410
-
2373
+ using namespace llvm ::VPlanPatternMatch;
2411
2374
// Find EVL loop entries by locating VPEVLBasedIVPHIRecipe
2412
2375
// There should be only one EVL PHI in the entire plan
2413
2376
VPEVLBasedIVPHIRecipe *EVLPhi = nullptr ;
2414
- VPBasicBlock *EVLPhiBlock = nullptr ;
2415
2377
2416
2378
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2417
2379
vp_depth_first_shallow (Plan.getEntry ())))
2418
2380
for (VPRecipeBase &R : VPBB->phis ())
2419
2381
if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
2420
- assert (!EVLPhi && " Found multiple EVL PHIs - only one expected" );
2382
+ assert (!EVLPhi && " Found multiple EVL PHIs. Only one expected" );
2421
2383
EVLPhi = PhiR;
2422
- EVLPhiBlock = VPBB;
2423
2384
}
2424
2385
2425
- // Process the single EVL PHI if found
2426
- if (EVLPhi)
2427
- ConvertEVLPhi (Plan, EVLPhiBlock, EVLPhi);
2386
+ // Early return if no EVL PHI is found
2387
+ if (!EVLPhi)
2388
+ return ;
2389
+
2390
+ VPBasicBlock *Entry = EVLPhi->getParent ();
2391
+ VPValue *EVLIncrement = EVLPhi->getBackedgeValue ();
2392
+
2393
+ // Convert EVLPhi to concrete recipe.
2394
+ auto *ScalarR =
2395
+ VPBuilder (EVLPhi).createScalarPhi ({EVLPhi->getStartValue (), EVLIncrement},
2396
+ EVLPhi->getDebugLoc (), " evl.based.iv" );
2397
+ EVLPhi->replaceAllUsesWith (ScalarR);
2398
+ EVLPhi->eraseFromParent ();
2399
+
2400
+ // Find the latch-exiting block and convert to variable-length stepping.
2401
+ // Before: (branch-on-count CanonicalIVInc, VectorTripCount)
2402
+ // After: (branch-on-count EVLIVInc, TripCount)
2403
+ auto Range =
2404
+ VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow (Entry));
2405
+ auto It = find_if (Range, [&Entry](VPBasicBlock *VPBB) {
2406
+ return any_of (VPBB->successors (),
2407
+ [&Entry](VPBlockBase *Succ) { return Succ == Entry; });
2408
+ });
2409
+ assert ((It != Range.end ()) && " LatchExiting is not found" );
2410
+ VPBasicBlock *LatchExiting = *It;
2411
+ auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator ());
2412
+ VPValue *ScalarIVInc;
2413
+ assert (LatchExitingBr &&
2414
+ match (LatchExitingBr,
2415
+ m_BranchOnCount (m_VPValue (ScalarIVInc),
2416
+ m_Specific (&Plan.getVectorTripCount ()))) &&
2417
+ " Unexpected terminator in EVL loop" );
2418
+ LatchExitingBr->setOperand (1 , Plan.getTripCount ());
2419
+ ScalarIVInc->replaceAllUsesWith (EVLIncrement);
2420
+ VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe ();
2421
+ VPRecipeBase *ScalarIV = IVIncR->getOperand (0 )->getDefiningRecipe ();
2422
+ IVIncR->eraseFromParent ();
2423
+ ScalarIV->eraseFromParent ();
2428
2424
}
2429
2425
2430
2426
void VPlanTransforms::dropPoisonGeneratingRecipes (
0 commit comments