[VPlan] Convert EVL loops to variable-length stepping after dissolution

Loop regions require fixed-length steps and rounded-up trip counts, but
after dissolution creates explicit control flow, EVL loops can leverage
variable-length stepping with original trip counts.

This patch adds a post-dissolution transform pass to convert EVL loops
from fixed-length to variable-length stepping .

Author: arcbbb

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -7315,6 +7315,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   // Regions are dissolved after optimizing for VF and UF, which completely
   // removes unneeded loop regions first.
   VPlanTransforms::dissolveLoopRegions(BestVPlan);
+  // Enable variable-length stepping for EVL loops after regions are dissolved
+  VPlanTransforms::simplifyEVLIVs(BestVPlan);
   // Perform the actual loop transformation.
   VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
                          OrigLoop->getParentLoop(),

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -2390,6 +2390,58 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
   return true;
 }
 
+void VPlanTransforms::simplifyEVLIVs(VPlan &Plan) {
+  auto ConvertEVLPhi = [](VPlan &Plan, VPBasicBlock *Entry,
+                          VPEVLBasedIVPHIRecipe *EVLPhi) {
+    using namespace llvm::VPlanPatternMatch;
+    VPValue *EVLIncrement = EVLPhi->getBackedgeValue();
+
+    // Convert EVLPhi to concrete recipe.
+    auto *ScalarR = VPBuilder(EVLPhi).createScalarPhi(
+        {EVLPhi->getStartValue(), EVLIncrement}, EVLPhi->getDebugLoc(),
+        "evl.based.iv");
+    EVLPhi->replaceAllUsesWith(ScalarR);
+    EVLPhi->eraseFromParent();
+
+    // Find the latch-exiting block and convert to variable-length stepping.
+    // Before: (branch-on-cond CanonicalIVInc, VectorTripCount)
+    // After: (branch-on-cond EVLIVInc, TripCount)
+    auto FindLatchExiting = [](VPBasicBlock *Entry) {
+      auto Range =
+          VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(Entry));
+      auto It = find_if(Range, [&](VPBasicBlock *VPBB) {
+        return any_of(VPBB->successors(),
+                      [&](VPBlockBase *Succ) { return Succ == Entry; });
+      });
+      return It != Range.end() ? *It : nullptr;
+    };
+    VPBasicBlock *LatchExiting = FindLatchExiting(Entry);
+    assert(LatchExiting && "LatchExiting is not found");
+    auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator());
+    VPValue *ScalarIVInc;
+    if (!LatchExitingBr ||
+        !match(LatchExitingBr,
+               m_BranchOnCount(m_VPValue(ScalarIVInc),
+                               m_Specific(&Plan.getVectorTripCount()))))
+      return;
+    LatchExitingBr->setOperand(1, Plan.getTripCount());
+    ScalarIVInc->replaceAllUsesWith(EVLIncrement);
+    VPRecipeBase *IVIncR = ScalarIVInc->getDefiningRecipe();
+    VPRecipeBase *ScalarIV = IVIncR->getOperand(0)->getDefiningRecipe();
+    IVIncR->eraseFromParent();
+    ScalarIV->eraseFromParent();
+  };
+
+  // Find EVL loop entries by locating VPEVLBasedIVPHIRecipe
+  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+           vp_depth_first_shallow(Plan.getEntry())))
+    for (VPRecipeBase &R : VPBB->phis())
+      if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
+        ConvertEVLPhi(Plan, VPBB, PhiR);
+        break;
+      }
+}
+
 void VPlanTransforms::dropPoisonGeneratingRecipes(
     VPlan &Plan,
     const std::function<bool(BasicBlock *)> &BlockNeedsPredication) {
@@ -2721,15 +2773,6 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getEntry()))) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
-      if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
-        auto *ScalarR = VPBuilder(PhiR).createScalarPhi(
-            {PhiR->getStartValue(), PhiR->getBackedgeValue()},
-            PhiR->getDebugLoc(), "evl.based.iv");
-        PhiR->replaceAllUsesWith(ScalarR);
-        ToRemove.push_back(PhiR);
-        continue;
-      }
-
       if (auto *WidenIVR = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
         expandVPWidenIntOrFpInduction(WidenIVR, TypeInfo);
         ToRemove.push_back(WidenIVR);

llvm/lib/Transforms/Vectorize/VPlanTransforms.h

@@ -209,6 +209,17 @@ struct VPlanTransforms {
   /// Replace loop regions with explicit CFG.
   static void dissolveLoopRegions(VPlan &Plan);
 
+  /// Transform EVL loops to use variable-length stepping after region
+  /// dissolution.
+  ///
+  /// Once loop regions are replaced with explicit CFG, EVL loops can step with
+  /// variable vector lengths instead of fixed lengths. This transformation:
+  ///  * EVL-Phi concretization (makes them concrete)
+  ///  * Replaces fixed-length stepping (branch-on-cond CanonicalIVInc,
+  ///    VectorTripCount) with variable-length stepping (branch-on-cond
+  ///    EVLIVInc, TripCount).
+  static void simplifyEVLIVs(VPlan &Plan);
+
   /// Lower abstract recipes to concrete ones, that can be codegen'd. Use \p
   /// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis.
   static void convertToConcreteRecipes(VPlan &Plan, Type &CanonicalIVTy);

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

@@ -193,7 +193,9 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
             errs() << "EVL used by unexpected VPInstruction\n";
             return false;
           }
-          if (I->getNumUsers() != 1) {
+          // EVLIVIncrement is only used by EVLIV & BranchOnCount.
+          // More than two is unexpected.
+          if (I->getNumUsers() > 2) {
             errs() << "EVL is used in VPInstruction with multiple users\n";
             return false;
           }