!fixup address latest comments, thanks

Author: fhahn

llvm/lib/Transforms/Vectorize/VPlan.cpp

@@ -450,10 +450,12 @@ void VPBasicBlock::connectToPredecessors(VPTransformState &State) {
   if (ParentLoop && !State.LI->getLoopFor(NewBB))
     ParentLoop->addBasicBlockToLoop(NewBB, *State.LI);
 
-  auto Preds = to_vector(getHierarchicalPredecessors());
+  SmallVector<VPBlockBase *> Preds;
   if (VPBlockUtils::isHeader(this, State.VPDT)) {
     // There's no block for the latch yet, connect to the preheader only.
-    Preds = {Preds[0]};
+    Preds = {getPredecessors()[0]};
+  } else {
+    Preds = to_vector(getPredecessors());
   }
 
   // Hook up the new basic block to its predecessors.
@@ -879,11 +881,10 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
-void VPRegionBlock::removeRegion() {
+void VPRegionBlock::dissolveToCFGLoop() {
   auto *Header = cast<VPBasicBlock>(getEntry());
   VPBlockBase *Preheader = getSinglePredecessor();
-  auto *Exiting = cast<VPBasicBlock>(getExiting());
-
+  auto *ExitingLatch = cast<VPBasicBlock>(getExiting());
   VPBlockBase *Middle = getSingleSuccessor();
   VPBlockUtils::disconnectBlocks(Preheader, this);
   VPBlockUtils::disconnectBlocks(this, Middle);
@@ -892,8 +893,8 @@ void VPRegionBlock::removeRegion() {
     VPB->setParent(getParent());
 
   VPBlockUtils::connectBlocks(Preheader, Header);
-  VPBlockUtils::connectBlocks(Exiting, Middle);
-  VPBlockUtils::connectBlocks(Exiting, Header);
+  VPBlockUtils::connectBlocks(ExitingLatch, Middle);
+  VPBlockUtils::connectBlocks(ExitingLatch, Header);
 }
 
 VPlan::VPlan(Loop *L) {
@@ -1005,6 +1006,8 @@ void VPlan::execute(VPTransformState *State) {
   for (VPBlockBase *Block : RPOT)
     Block->execute(State);
 
+  State->CFG.DTU.flush();
+
   VPBasicBlock *Header = vputils::getFirstLoopHeader(*this, State->VPDT);
   if (!Header)
     return;
@@ -1427,7 +1430,7 @@ static bool isDefinedInsideLoopRegions(const VPValue *VPV) {
                   DefR->getParent()->getEnclosingLoopRegion());
 }
 
-bool VPValue::isDefinedOutsideLoop() const {
+bool VPValue::isDefinedOutsideLoopRegions() const {
   return !isDefinedInsideLoopRegions(this);
 }
 void VPValue::replaceAllUsesWith(VPValue *New) {

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -1631,7 +1631,9 @@ struct VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
     return getOperand(0);
   }
 
-  bool isInvariantCond() const { return getCond()->isDefinedOutsideLoop(); }
+  bool isInvariantCond() const {
+    return getCond()->isDefinedOutsideLoopRegions();
+  }
 
   /// Returns true if the recipe only uses the first lane of operand \p Op.
   bool onlyFirstLaneUsed(const VPValue *Op) const override {
@@ -1644,16 +1646,17 @@ struct VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
 /// A recipe for handling GEP instructions.
 class VPWidenGEPRecipe : public VPRecipeWithIRFlags {
   bool isPointerLoopInvariant() const {
-    return getOperand(0)->isDefinedOutsideLoop();
+    return getOperand(0)->isDefinedOutsideLoopRegions();
   }
 
   bool isIndexLoopInvariant(unsigned I) const {
-    return getOperand(I + 1)->isDefinedOutsideLoop();
+    return getOperand(I + 1)->isDefinedOutsideLoopRegions();
   }
 
   bool areAllOperandsInvariant() const {
-    return all_of(operands(),
-                  [](VPValue *Op) { return Op->isDefinedOutsideLoop(); });
+    return all_of(operands(), [](VPValue *Op) {
+      return Op->isDefinedOutsideLoopRegions();
+    });
   }
 
 public:
@@ -3868,8 +3871,8 @@ class VPRegionBlock : public VPBlockBase {
   VPRegionBlock *clone() override;
 
   /// Remove the current region from its VPlan, connecting its predecessor to
-  /// its entry and exiting block to its successor.
-  void removeRegion();
+  /// its entry, and its exiting block to its successor.
+  void dissolveToCFGLoop();
 };
 
 /// VPlan models a candidate for vectorization, encoding various decisions take

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -465,7 +465,7 @@ Value *VPInstruction::generatePerLane(VPTransformState &State,
 /// Create a conditional branch using \p Cond branching to the successors of \p
 /// VPBB. Note that the first successor is always forward (i.e. not created yet)
 /// while the second successor may already have been created (if it is a header
-/// block and VPBB is a header).
+/// block and VPBB is a latch).
 static BranchInst *createCondBranch(Value *Cond, VPBasicBlock *VPBB,
                                     VPTransformState &State) {
   // Replace the temporary unreachable terminator with a new conditional
@@ -1557,7 +1557,7 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
 InstructionCost VPWidenSelectRecipe::computeCost(ElementCount VF,
                                                  VPCostContext &Ctx) const {
   SelectInst *SI = cast<SelectInst>(getUnderlyingValue());
-  bool ScalarCond = getOperand(0)->isDefinedOutsideLoop();
+  bool ScalarCond = getOperand(0)->isDefinedOutsideLoopRegions();
   Type *ScalarTy = Ctx.Types.inferScalarType(this);
   Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);
 
@@ -1778,7 +1778,7 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
     TargetTransformInfo::OperandValueInfo RHSInfo = Ctx.getOperandInfo(RHS);
 
     if (RHSInfo.Kind == TargetTransformInfo::OK_AnyValue &&
-        getOperand(1)->isDefinedOutsideLoop())
+        getOperand(1)->isDefinedOutsideLoopRegions())
       RHSInfo.Kind = TargetTransformInfo::OK_UniformValue;
     Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);
     Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
@@ -2677,12 +2677,13 @@ static void scalarizeInstruction(const Instruction *Instr,
   if (auto *II = dyn_cast<AssumeInst>(Cloned))
     State.AC->registerAssumption(II);
 
-  assert((RepRecipe->getParent()->getParent() ||
-          !RepRecipe->getParent()->getPlan()->getVectorLoopRegion() ||
-          all_of(RepRecipe->operands(),
-                 [](VPValue *Op) { return Op->isDefinedOutsideLoop(); })) &&
-         "Expected a recipe is either within a region or all of its operands "
-         "are defined outside the vectorized region.");
+  assert(
+      (RepRecipe->getParent()->getParent() ||
+       !RepRecipe->getParent()->getPlan()->getVectorLoopRegion() ||
+       all_of(RepRecipe->operands(),
+              [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); })) &&
+      "Expected a recipe is either within a region or all of its operands "
+      "are defined outside the vectorized region.");
 }
 
 void VPReplicateRecipe::execute(VPTransformState &State) {

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -1736,8 +1736,9 @@ static void licm(VPlan &Plan) {
       // TODO: Relax checks in the future, e.g. we could also hoist reads, if
       // their memory location is not modified in the vector loop.
       if (R.mayHaveSideEffects() || R.mayReadFromMemory() || R.isPhi() ||
-          any_of(R.operands(),
-                 [](VPValue *Op) { return !Op->isDefinedOutsideLoop(); }))
+          any_of(R.operands(), [](VPValue *Op) {
+            return !Op->isDefinedOutsideLoopRegions();
+          }))
         continue;
       R.moveBefore(*Preheader, Preheader->end());
     }
@@ -2514,7 +2515,7 @@ void VPlanTransforms::dissolveLoopRegions(VPlan &Plan) {
       LoopRegions.push_back(R);
   }
   for (VPRegionBlock *R : LoopRegions)
-    R->removeRegion();
+    R->dissolveToCFGLoop();
 }
 
 // Expand VPExtendedReductionRecipe to VPWidenCastRecipe + VPReductionRecipe.

llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp

@@ -246,7 +246,7 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
   }
   if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
     if (isa<StoreInst>(RepR->getUnderlyingValue()) &&
-        RepR->getOperand(1)->isDefinedOutsideLoop()) {
+        RepR->getOperand(1)->isDefinedOutsideLoopRegions()) {
       // Stores to an invariant address only need to store the last part.
       remapOperands(&R, UF - 1);
       return;

llvm/lib/Transforms/Vectorize/VPlanUtils.cpp

@@ -90,7 +90,7 @@ bool vputils::isUniformAcrossVFsAndUFs(VPValue *V) {
     return true;
 
   VPRecipeBase *R = V->getDefiningRecipe();
-  if (R && V->isDefinedOutsideLoop()) {
+  if (R && V->isDefinedOutsideLoopRegions()) {
     if (match(V->getDefiningRecipe(),
               m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>(
                   m_VPValue())))

llvm/lib/Transforms/Vectorize/VPlanValue.h

@@ -182,7 +182,7 @@ class VPValue {
   }
 
   /// Returns true if the VPValue is defined outside any loop.
-  bool isDefinedOutsideLoop() const;
+  bool isDefinedOutsideLoopRegions() const;
 
   // Set \p Val as the underlying Value of this VPValue.
   void setUnderlyingValue(Value *Val) {