Address comments

Author: arcbbb

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -7290,8 +7290,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   // Regions are dissolved after optimizing for VF and UF, which completely
   // removes unneeded loop regions first.
   VPlanTransforms::dissolveLoopRegions(BestVPlan);
-  // Enable variable-length stepping for EVL loops after regions are dissolved
-  VPlanTransforms::simplifyEVLIVs(BestVPlan);
+  // Canonicalize EVL loops after regions are dissolved.
+  VPlanTransforms::canonicalizeEVLLoops(BestVPlan);
   // Perform the actual loop transformation.
   VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
                          OrigLoop->getParentLoop(),

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -2369,10 +2369,10 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
   return true;
 }
 
-void VPlanTransforms::simplifyEVLIVs(VPlan &Plan) {
+void VPlanTransforms::canonicalizeEVLLoops(VPlan &Plan) {
   using namespace llvm::VPlanPatternMatch;
-  // Find EVL loop entries by locating VPEVLBasedIVPHIRecipe
-  // There should be only one EVL PHI in the entire plan
+  // Find EVL loop entries by locating VPEVLBasedIVPHIRecipe.
+  // There should be only one EVL PHI in the entire plan.
   VPEVLBasedIVPHIRecipe *EVLPhi = nullptr;
 
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
@@ -2383,11 +2383,11 @@ void VPlanTransforms::simplifyEVLIVs(VPlan &Plan) {
         EVLPhi = PhiR;
       }
 
-  // Early return if no EVL PHI is found
+  // Early return if no EVL PHI is found.
   if (!EVLPhi)
     return;
 
-  VPBasicBlock *Entry = EVLPhi->getParent();
+  VPBasicBlock *HeaderVPBB = EVLPhi->getParent();
   VPValue *EVLIncrement = EVLPhi->getBackedgeValue();
 
   // Convert EVLPhi to concrete recipe.
@@ -2397,32 +2397,29 @@ void VPlanTransforms::simplifyEVLIVs(VPlan &Plan) {
   EVLPhi->replaceAllUsesWith(ScalarR);
   EVLPhi->eraseFromParent();
 
-  // Replace CanonicalIVInc with EVL-PHI increment
-  VPRecipeBase *CanonicalIV = &*Entry->begin();
-  assert(dyn_cast<VPPhi>(CanonicalIV) && "Unexpected canoincal iv");
+  // Replace CanonicalIVInc with EVL-PHI increment.
+  VPRecipeBase *CanonicalIV = &*HeaderVPBB->begin();
+  assert(
+      isa<VPPhi>(CanonicalIV) &&
+      match(CanonicalIV->getOperand(1),
+            m_c_Binary<Instruction::Add>(m_Specific(cast<VPPhi>(CanonicalIV)),
+                                         m_Specific(&Plan.getVFxUF()))) &&
+      "Unexpected canoincal iv");
   VPValue *Backedge = CanonicalIV->getOperand(1);
   Backedge->replaceAllUsesWith(EVLIncrement);
 
-  // Remove unused phi
+  // Remove unused phi and increment.
   VPRecipeBase *CanonicalIVIncrement = Backedge->getDefiningRecipe();
   CanonicalIVIncrement->eraseFromParent();
   CanonicalIV->eraseFromParent();
 
-  // Find the latch-exiting block and replace use of VectorTripCount
+  // Replace the use of VectorTripCount in the latch-exiting block.
   // Before: (branch-on-count EVLIVInc, VectorTripCount)
   // After: (branch-on-count EVLIVInc, TripCount)
-  auto Range =
-      VPBlockUtils::blocksOnly<VPBasicBlock>(vp_depth_first_shallow(Entry));
-  auto It = find_if(Range, [&Entry](VPBasicBlock *VPBB) {
-    return any_of(VPBB->successors(),
-                  [&Entry](VPBlockBase *Succ) { return Succ == Entry; });
-  });
-  assert((It != Range.end()) && "LatchExiting is not found");
-
-  VPBasicBlock *LatchExiting = *It;
 
+  VPBasicBlock *LatchExiting =
+      HeaderVPBB->getPredecessors()[1]->getEntryBasicBlock();
   auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator());
-
   // Skip single-iteration loop region
   if (match(LatchExitingBr, m_BranchOnCond(m_True())))
     return;

llvm/lib/Transforms/Vectorize/VPlanTransforms.h

@@ -214,11 +214,12 @@ struct VPlanTransforms {
   ///
   /// Once loop regions are replaced with explicit CFG, EVL loops can step with
   /// variable vector lengths instead of fixed lengths. This transformation:
-  ///  * EVL-Phi concretization (makes them concrete)
+  ///  * Make EVL-Phi concrete.
+  //   * Remove CanonicalIV and increment.
   ///  * Replaces fixed-length stepping (branch-on-cond CanonicalIVInc,
   ///    VectorTripCount) with variable-length stepping (branch-on-cond
   ///    EVLIVInc, TripCount).
-  static void simplifyEVLIVs(VPlan &Plan);
+  static void canonicalizeEVLLoops(VPlan &Plan);
 
   /// Lower abstract recipes to concrete ones, that can be codegen'd. Use \p
   /// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis.

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

@@ -17,6 +17,7 @@
 #include "VPlanCFG.h"
 #include "VPlanDominatorTree.h"
 #include "VPlanHelpers.h"
+#include "VPlanPatternMatch.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 
@@ -193,8 +194,12 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
             return false;
           }
           // EVLIVIncrement is only used by EVLIV & BranchOnCount.
-          // More than two is unexpected.
-          if (I->getNumUsers() > 2) {
+          // Having more than two users is unexpected.
+          if ((I->getNumUsers() != 1) &&
+              (I->getNumUsers() != 2 || !any_of(I->users(), [&I](VPUser *U) {
+                 using namespace llvm::VPlanPatternMatch;
+                 return match(U, m_BranchOnCount(m_Specific(I), m_VPValue()));
+               }))) {
             errs() << "EVL is used in VPInstruction with multiple users\n";
             return false;
           }