@@ -627,14 +627,12 @@ static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
627627static void legalizeAndOptimizeInductions (VPlan &Plan) {
628628 using namespace llvm ::VPlanPatternMatch;
629629 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion ()->getEntryBasicBlock ();
630- SmallVector<VPWidenInductionRecipe *, 4 > InductionPhis;
631- for (VPRecipeBase &R : HeaderVPBB->phis ())
632- if (auto *IV = dyn_cast<VPWidenInductionRecipe>(&R))
633- InductionPhis.push_back (IV);
634-
635630 bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly ();
636- VPBuilder Builder;
637- for (VPWidenInductionRecipe *PhiR : reverse (InductionPhis)) {
631+ VPBuilder Builder (HeaderVPBB, HeaderVPBB->getFirstNonPhi ());
632+ for (VPRecipeBase &Phi : HeaderVPBB->phis ()) {
633+ auto *PhiR = dyn_cast<VPWidenInductionRecipe>(&Phi);
634+ if (!PhiR)
635+ continue ;
638636 // Try to narrow wide and replicating recipes to uniform recipes, based on
639637 // VPlan analysis.
640638 // TODO: Apply to all recipes in the future, to replace legacy uniformity
@@ -644,8 +642,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
644642 auto *Def = dyn_cast<VPSingleDefRecipe>(U);
645643 auto *RepR = dyn_cast<VPReplicateRecipe>(U);
646644 // Skip recipes that shouldn't be narrowed.
647- if (!Def ||
648- !isa<VPReplicateRecipe, VPWidenRecipe, VPWidenGEPRecipe>(Def) ||
645+ if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
649646 Def->getNumUsers () == 0 || !Def->getUnderlyingValue () ||
650647 (RepR && (RepR->isSingleScalar () || RepR->isPredicated ())))
651648 continue ;
@@ -658,13 +655,11 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
658655 Def->operands (), /* IsUniform*/ true );
659656 Clone->insertAfter (Def);
660657 Def->replaceAllUsesWith (Clone);
661- Def->eraseFromParent ();
662658 }
663659
664- Builder.setInsertPoint (HeaderVPBB, HeaderVPBB->getFirstNonPhi ());
665660 // Replace wide pointer inductions which have only their scalars used by
666661 // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
667- if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(PhiR )) {
662+ if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(&Phi )) {
668663 if (!PtrIV->onlyScalarsGenerated (Plan.hasScalableVF ()))
669664 continue ;
670665
@@ -685,7 +680,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
685680
686681 // Replace widened induction with scalar steps for users that only use
687682 // scalars.
688- auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(PhiR );
683+ auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(&Phi );
689684 if (HasOnlyVectorVFs && none_of (WideIV->users (), [WideIV](VPUser *U) {
690685 return U->usesScalars (WideIV);
691686 }))
0 commit comments