[VPlan] Refactor HCFG builder to preserve original vector latch (NFC).

Update HCFG builder to preserve the original latch block of the initial
VPlan, ensuring there is always a latch.

It also skips creating the BranchOnCond for the latch of the top-level
loop, instead of removing it later. Exiting via the latch is controlled
by later recipes.

This further unifies HCFG construction and prepares for use to also
build an initial VPlan (VPlan0) for inner loops.

Author: fhahn

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -9509,12 +9509,6 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
       [this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); },
       *PSE.getSE(), *TLI);
 
-  // Remove the existing terminator of the exiting block of the top-most region.
-  // A BranchOnCount will be added instead when adding the canonical IV recipes.
-  auto *Term =
-      Plan->getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
-  Term->eraseFromParent();
-
   // Tail folding is not supported for outer loops, so the induction increment
   // is guaranteed to not wrap.
   bool HasNUW = true;

llvm/lib/Transforms/Vectorize/VPlan.cpp

@@ -1628,6 +1628,9 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
     VFRange SubRange = {VF, MaxVFTimes2};
     auto Plan = buildVPlan(SubRange);
     VPlanTransforms::optimize(*Plan);
+    // Update the name of the latch of the top-level vector loop region region
+    // after optimizations which includes block folding.
+    Plan->getVectorLoopRegion()->getExiting()->setName("vector.latch");
     VPlans.push_back(std::move(Plan));
     VF = SubRange.End;
   }

llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp

@@ -292,6 +292,11 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
            "Instruction shouldn't have been visited.");
 
     if (auto *Br = dyn_cast<BranchInst>(Inst)) {
+      if (TheLoop->getLoopLatch() == BB ||
+          any_of(successors(BB),
+                 [this](BasicBlock *Succ) { return !TheLoop->contains(Succ); }))
+        continue;
+
       // Conditional branch instruction are represented using BranchOnCond
       // recipes.
       if (Br->isConditional()) {
@@ -356,11 +361,6 @@ void PlainCFGBuilder::buildPlainCFG() {
   VPBasicBlock *VectorLatchVPBB = TheRegion->getExitingBasicBlock();
   BB2VPBB[TheLoop->getHeader()] = VectorHeaderVPBB;
   VectorHeaderVPBB->clearSuccessors();
-  VectorLatchVPBB->clearPredecessors();
-  if (TheLoop->getHeader() != TheLoop->getLoopLatch())
-    BB2VPBB[TheLoop->getLoopLatch()] = VectorLatchVPBB;
-  else
-    TheRegion->setExiting(VectorHeaderVPBB);
 
   // 1. Scan the body of the loop in a topological order to visit each basic
   // block after having visited its predecessor basic blocks. Create a VPBB for
@@ -398,6 +398,13 @@ void PlainCFGBuilder::buildPlainCFG() {
         setRegionPredsFromBB(Region, BB);
     }
 
+    if (TheLoop->getLoopLatch() == BB) {
+      VPBB->setOneSuccessor(VectorLatchVPBB);
+      VectorLatchVPBB->clearPredecessors();
+      VectorLatchVPBB->setPredecessors({VPBB});
+      continue;
+    }
+
     // Set VPBB successors. We create empty VPBBs for successors if they don't
     // exist already. Recipes will be created when the successor is visited
     // during the RPO traversal.

llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll

@@ -35,12 +35,14 @@ define void @foo(i64 %n) {
 ; CHECK-NEXT:       EMIT branch-on-cond ir<%inner.ec>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  Successor(s): vector.latch
+; CHECK-NEXT:  Successor(s): outer.latch
 ; CHECK-EMPTY:
-; CHECK-NEXT:   vector.latch:
+; CHECK-NEXT:  outer.latch:
 ; CHECK-NEXT:     EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
 ; CHECK-NEXT:     EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
-; CHECK-NEXT:     EMIT branch-on-cond ir<%outer.ec>
+; CHECK-NEXT:  Successor(s): vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT:   vector.latch:
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT:  }
 ; CHECK-NEXT: Successor(s): middle.block

llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp

@@ -48,16 +48,19 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
   EXPECT_EQ(0u, Entry->getNumPredecessors());
   EXPECT_EQ(1u, Entry->getNumSuccessors());
 
-  // Check that the region following the preheader is a single basic-block
-  // region (loop).
+  // Check that the region following the preheader consists of a block for the
+  // original header and a separate latch.
   VPBasicBlock *VecBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
-  EXPECT_EQ(8u, VecBB->size());
+  EXPECT_EQ(7u, VecBB->size());
   EXPECT_EQ(0u, VecBB->getNumPredecessors());
-  EXPECT_EQ(0u, VecBB->getNumSuccessors());
+  EXPECT_EQ(1u, VecBB->getNumSuccessors());
   EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
-  EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
   EXPECT_EQ(&*Plan, VecBB->getPlan());
 
+  VPBlockBase *VecLatch = VecBB->getSingleSuccessor();
+  EXPECT_EQ(VecLatch->getParent()->getExitingBasicBlock(), VecLatch);
+  EXPECT_EQ(0u, VecLatch->getNumSuccessors());
+
   auto Iter = VecBB->begin();
   VPWidenPHIRecipe *Phi = dyn_cast<VPWidenPHIRecipe>(&*Iter++);
   EXPECT_NE(nullptr, Phi);
@@ -127,29 +130,33 @@ compound=true
       "  EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
       "  EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\>, ir\<1\>\l" +
       "  EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\>, ir\<%N\>\l" +
-      "  EMIT branch-on-cond ir\<%exitcond\>\l" +
+      "Successor(s): vector.latch\l"
+    ]
+    N2 -> N4 [ label=""]
+    N4 [label =
+      "vector.latch:\l" +
       "No successors\l"
     ]
   }
-  N2 -> N4 [ label="" ltail=cluster_N3]
-  N4 [label =
+  N4 -> N5 [ label="" ltail=cluster_N3]
+  N5 [label =
     "middle.block:\l" +
     "  EMIT vp\<%cmp.n\> = icmp eq ir\<%N\>, vp\<%0\>\l" +
     "  EMIT branch-on-cond vp\<%cmp.n\>\l" +
     "Successor(s): ir-bb\<for.end\>, scalar.ph\l"
   ]
-  N4 -> N5 [ label="T"]
-  N4 -> N6 [ label="F"]
-  N5 [label =
+  N5 -> N6 [ label="T"]
+  N5 -> N7 [ label="F"]
+  N6 [label =
     "ir-bb\<for.end\>:\l" +
     "No successors\l"
   ]
-  N6 [label =
+  N7 [label =
     "scalar.ph:\l" +
     "Successor(s): ir-bb\<for.body\>\l"
   ]
-  N6 -> N7 [ label=""]
-  N7 [label =
+  N7 -> N8 [ label=""]
+  N8 [label =
     "ir-bb\<for.body\>:\l" +
     "  IR   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\l" +
     "  IR   %arr.idx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv\l" +
@@ -204,14 +211,17 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
   EXPECT_EQ(0u, Entry->getNumPredecessors());
   EXPECT_EQ(1u, Entry->getNumSuccessors());
 
-  // Check that the region following the preheader is a single basic-block
-  // region (loop).
+  // Check that the region following the preheader consists of a block for the
+  // original header and a separate latch.
   VPBasicBlock *VecBB = Plan->getVectorLoopRegion()->getEntryBasicBlock();
-  EXPECT_EQ(8u, VecBB->size());
+  EXPECT_EQ(7u, VecBB->size());
   EXPECT_EQ(0u, VecBB->getNumPredecessors());
-  EXPECT_EQ(0u, VecBB->getNumSuccessors());
+  EXPECT_EQ(1u, VecBB->getNumSuccessors());
   EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
-  EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
+
+  VPBlockBase *VecLatch = VecBB->getSingleSuccessor();
+  EXPECT_EQ(VecLatch->getParent()->getExitingBasicBlock(), VecLatch);
+  EXPECT_EQ(0u, VecLatch->getNumSuccessors());
 
   auto Iter = VecBB->begin();
   EXPECT_NE(nullptr, dyn_cast<VPWidenPHIRecipe>(&*Iter++));
@@ -221,7 +231,6 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
   EXPECT_NE(nullptr, dyn_cast<VPWidenMemoryRecipe>(&*Iter++));
   EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
   EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
-  EXPECT_NE(nullptr, dyn_cast<VPInstruction>(&*Iter++));
   EXPECT_EQ(VecBB->end(), Iter);
 }