[VPlan] Add VPSymbolicValueSC for typed VPValues w/o underlying IR value

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -4478,7 +4478,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
 static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
                                 const TargetTransformInfo &TTI) {
   assert(VF.isVector() && "Checking a scalar VF?");
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
   DenseSet<VPRecipeBase *> EphemeralRecipes;
   collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
   // Set of already visited types.
@@ -9083,7 +9083,7 @@ static VPInstruction *addResumePhiRecipeForInduction(
 /// \p IVEndValues.
 static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
                                 DenseMap<VPValue *, VPValue *> &IVEndValues) {
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
   auto *ScalarPH = Plan.getScalarPreheader();
   auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getSinglePredecessor());
   VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
@@ -9326,7 +9326,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
             return !CM.requiresScalarEpilogue(VF.isVector());
           },
           Range);
-  auto Plan = std::make_unique<VPlan>(OrigLoop);
+  auto Plan =
+      std::make_unique<VPlan>(OrigLoop, Legal->getWidestInductionType());
   // Build hierarchical CFG.
   // Convert to VPlan-transform and consoliate all transforms for VPlan
   // creation.
@@ -9632,7 +9633,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
   assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
 
   // Create new empty VPlan
-  auto Plan = std::make_unique<VPlan>(OrigLoop);
+  auto Plan =
+      std::make_unique<VPlan>(OrigLoop, Legal->getWidestInductionType());
   // Build hierarchical CFG
   VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
   HCFGBuilder.buildHierarchicalCFG();

llvm/lib/Transforms/Vectorize/VPlan.cpp

@@ -222,7 +222,7 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
                                    Loop *CurrentParentLoop, Type *CanonicalIVTy)
     : TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
       CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
-      TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
+      TypeAnalysis(CanonicalIVTy->getContext()), VPDT(*Plan) {}
 
 Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
   if (Def->isLiveIn())
@@ -850,7 +850,8 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
-VPlan::VPlan(Loop *L) {
+VPlan::VPlan(Loop *L, Type *InductionTy)
+    : VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) {
   setEntry(createVPIRBasicBlock(L->getLoopPreheader()));
   ScalarHeader = createVPIRBasicBlock(L->getHeader());
 
@@ -861,7 +862,7 @@ VPlan::VPlan(Loop *L) {
 }
 
 VPlan::~VPlan() {
-  VPValue DummyValue;
+  VPValue DummyValue((Type *)nullptr);
 
   for (auto *VPB : CreatedBlocks) {
     if (auto *VPBB = dyn_cast<VPBasicBlock>(VPB)) {
@@ -891,10 +892,10 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
     IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
     auto *TCMO = Builder.CreateSub(TripCountV, ConstantInt::get(TCTy, 1),
                                    "trip.count.minus.1");
-    BackedgeTakenCount->setUnderlyingValue(TCMO);
+    BackedgeTakenCount->replaceAllUsesWith(getOrAddLiveIn(TCMO));
   }
 
-  VectorTripCount.setUnderlyingValue(VectorTripCountV);
+  VectorTripCount.replaceAllUsesWith(getOrAddLiveIn(VectorTripCountV));
 
   IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
   // FIXME: Model VF * UF computation completely in VPlan.
@@ -903,12 +904,13 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
   unsigned UF = getUF();
   if (VF.getNumUsers()) {
     Value *RuntimeVF = getRuntimeVF(Builder, TCTy, State.VF);
-    VF.setUnderlyingValue(RuntimeVF);
-    VFxUF.setUnderlyingValue(
+    VF.replaceAllUsesWith(getOrAddLiveIn(RuntimeVF));
+    VFxUF.replaceAllUsesWith(getOrAddLiveIn(
         UF > 1 ? Builder.CreateMul(RuntimeVF, ConstantInt::get(TCTy, UF))
-               : RuntimeVF);
+               : RuntimeVF));
   } else {
-    VFxUF.setUnderlyingValue(createStepForVF(Builder, TCTy, State.VF, UF));
+    VFxUF.replaceAllUsesWith(
+        getOrAddLiveIn(createStepForVF(Builder, TCTy, State.VF, UF)));
   }
 }
 
@@ -1175,7 +1177,8 @@ VPlan *VPlan::duplicate() {
         return VPIRBB && VPIRBB->getIRBasicBlock() == ScalarHeaderIRBB;
       }));
   // Create VPlan, clone live-ins and remap operands in the cloned blocks.
-  auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader);
+  auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader,
+                            getCanonicalIV()->getScalarType());
   DenseMap<VPValue *, VPValue *> Old2NewVPValues;
   for (VPValue *OldLiveIn : getLiveIns()) {
     Old2NewVPValues[OldLiveIn] =
@@ -1185,7 +1188,7 @@ VPlan *VPlan::duplicate() {
   Old2NewVPValues[&VF] = &NewPlan->VF;
   Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF;
   if (BackedgeTakenCount) {
-    NewPlan->BackedgeTakenCount = new VPValue();
+    NewPlan->BackedgeTakenCount = new VPValue((Type *)nullptr);
     Old2NewVPValues[BackedgeTakenCount] = NewPlan->BackedgeTakenCount;
   }
   assert(TripCount && "trip count must be set");
@@ -1371,6 +1374,11 @@ static bool isDefinedInsideLoopRegions(const VPValue *VPV) {
                   DefR->getParent()->getEnclosingLoopRegion());
 }
 
+Type *VPValue::getType() const {
+  assert(isLiveIn() && "can only return the type for a live-in");
+  return SubclassID == VPSymbolicValueSC ? Ty : getUnderlyingValue()->getType();
+}
+
 bool VPValue::isDefinedOutsideLoopRegions() const {
   return !isDefinedInsideLoopRegions(this);
 }

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -3439,8 +3439,9 @@ class VPlan {
 
   /// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
   /// wrapping the original header of the scalar loop.
-  VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
-      : Entry(Entry), ScalarHeader(ScalarHeader) {
+  VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader, Type *InductionTy)
+      : Entry(Entry), ScalarHeader(ScalarHeader), VectorTripCount(InductionTy),
+        VF(InductionTy), VFxUF(InductionTy) {
     Entry->setPlan(this);
     assert(ScalarHeader->getNumSuccessors() == 0 &&
            "scalar header must be a leaf node");
@@ -3450,11 +3451,13 @@ class VPlan {
   /// Construct a VPlan for \p L. This will create VPIRBasicBlocks wrapping the
   /// original preheader and scalar header of \p L, to be used as entry and
   /// scalar header blocks of the new VPlan.
-  VPlan(Loop *L);
+  VPlan(Loop *L, Type *InductionTy);
 
   /// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
-  /// wrapping \p ScalarHeaderBB and a trip count of \p TC.
-  VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC) {
+  /// wrapping \p ScalarHeaderBB and a trip count of \p TC. Also creates
+  /// symbolic VectorTripCount, VF and VFxUF VPValues using \p InductionTy.
+  VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC, Type *InductionTy)
+      : VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) {
     setEntry(createVPBasicBlock("preheader"));
     ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB);
     TripCount = TC;
@@ -3546,7 +3549,7 @@ class VPlan {
   /// The backedge taken count of the original loop.
   VPValue *getOrCreateBackedgeTakenCount() {
     if (!BackedgeTakenCount)
-      BackedgeTakenCount = new VPValue();
+      BackedgeTakenCount = new VPValue(getCanonicalIV()->getScalarType());
     return BackedgeTakenCount;
   }
 

llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

@@ -231,13 +231,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
   if (Type *CachedTy = CachedTypes.lookup(V))
     return CachedTy;
 
-  if (V->isLiveIn()) {
-    if (auto *IRValue = V->getLiveInIRValue())
-      return IRValue->getType();
-    // All VPValues without any underlying IR value (like the vector trip count
-    // or the backedge-taken count) have the same type as the canonical IV.
-    return CanonicalIVTy;
-  }
+  if (V->isLiveIn())
+    return V->getType();
 
   Type *ResultTy =
       TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())

llvm/lib/Transforms/Vectorize/VPlanAnalysis.h

@@ -39,10 +39,6 @@ class Type;
 /// of the previously inferred types.
 class VPTypeAnalysis {
   DenseMap<const VPValue *, Type *> CachedTypes;
-  /// Type of the canonical induction variable. Used for all VPValues without
-  /// any underlying IR value (like the vector trip count or the backedge-taken
-  /// count).
-  Type *CanonicalIVTy;
   LLVMContext &Ctx;
 
   Type *inferScalarTypeForRecipe(const VPBlendRecipe *R);
@@ -55,8 +51,7 @@ class VPTypeAnalysis {
   Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
 
 public:
-  VPTypeAnalysis(Type *CanonicalIVTy)
-      : CanonicalIVTy(CanonicalIVTy), Ctx(CanonicalIVTy->getContext()) {}
+  VPTypeAnalysis(LLVMContext &Ctx) : Ctx(Ctx) {}
 
   /// Infer the type of \p V. Returns the scalar type of \p V.
   Type *inferScalarType(const VPValue *V);

llvm/lib/Transforms/Vectorize/VPlanHelpers.h

@@ -379,8 +379,8 @@ struct VPCostContext {
   VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
                 Type *CanIVTy, LoopVectorizationCostModel &CM,
                 TargetTransformInfo::TargetCostKind CostKind)
-      : TTI(TTI), TLI(TLI), Types(CanIVTy), LLVMCtx(CanIVTy->getContext()),
-        CM(CM), CostKind(CostKind) {}
+      : TTI(TTI), TLI(TLI), Types(CanIVTy->getContext()),
+        LLVMCtx(CanIVTy->getContext()), CM(CM), CostKind(CostKind) {}
 
   /// Return the cost for \p UI with \p VF using the legacy cost model as
   /// fallback until computing the cost of all recipes migrates to VPlan.

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -549,7 +549,7 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
 
   // Truncate base induction if needed.
   Type *CanonicalIVType = CanonicalIV->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
+  VPTypeAnalysis TypeInfo(CanonicalIVType->getContext());
   Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
   if (TruncI) {
     Type *TruncTy = TruncI->getType();
@@ -795,13 +795,14 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
 void VPlanTransforms::optimizeInductionExitUsers(
     VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
   VPBlockBase *MiddleVPBB = Plan.getMiddleBlock();
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
   for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
     for (VPRecipeBase &R : *ExitVPBB) {
       auto *ExitIRI = cast<VPIRInstruction>(&R);
       if (!isa<PHINode>(ExitIRI->getInstruction()))
         break;
 
+
       for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
         if (PredVPBB == MiddleVPBB)
           if (VPValue *Escape = optimizeLatchExitInductionUser(
@@ -957,8 +958,11 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
 #ifndef NDEBUG
     // Verify that the cached type info is for both A and its users is still
     // accurate by comparing it to freshly computed types.
-    VPTypeAnalysis TypeInfo2(
-        R.getParent()->getPlan()->getCanonicalIV()->getScalarType());
+    VPTypeAnalysis TypeInfo2(R.getParent()
+                                 ->getPlan()
+                                 ->getCanonicalIV()
+                                 ->getScalarType()
+                                 ->getContext());
     assert(TypeInfo.inferScalarType(A) == TypeInfo2.inferScalarType(A));
     for (VPUser *U : A->users()) {
       auto *R = cast<VPRecipeBase>(U);
@@ -1001,7 +1005,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
 void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
       Plan.getEntry());
-  VPTypeAnalysis TypeInfo(&CanonicalIVTy);
+  VPTypeAnalysis TypeInfo(CanonicalIVTy.getContext());
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
       simplifyRecipe(R, TypeInfo);
@@ -1351,7 +1355,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
   // typed.
   DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
   Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
+  VPTypeAnalysis TypeInfo(CanonicalIVType->getContext());
   VPBasicBlock *PH = Plan.getVectorPreheader();
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
@@ -1743,8 +1747,8 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
 /// Replace recipes with their EVL variants.
 static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
   Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
   LLVMContext &Ctx = CanonicalIVType->getContext();
+  VPTypeAnalysis TypeInfo(Ctx);
   VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();

llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp

@@ -73,7 +73,7 @@ class UnrollState {
 
 public:
   UnrollState(VPlan &Plan, unsigned UF, LLVMContext &Ctx)
-      : Plan(Plan), UF(UF), TypeInfo(Plan.getCanonicalIV()->getScalarType()) {}
+      : Plan(Plan), UF(UF), TypeInfo(Ctx) {}
 
   void unrollBlock(VPBlockBase *VPB);
 

llvm/lib/Transforms/Vectorize/VPlanValue.h

@@ -61,17 +61,22 @@ class VPValue {
   SmallVector<VPUser *, 1> Users;
 
 protected:
-  // Hold the underlying Value, if any, attached to this VPValue.
-  Value *UnderlyingVal;
+  union {
+    // Hold the underlying Value, if any, attached to this non-symbolic VPValue.
+    Value *UnderlyingVal;
+    // Hold the type of this VPValue, if it is symbolic.
+    Type *Ty;
+  };
 
   /// Pointer to the VPDef that defines this VPValue. If it is nullptr, the
   /// VPValue is not defined by any recipe modeled in VPlan.
   VPDef *Def;
 
   VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr);
 
-  /// Create a live-in VPValue.
-  VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {}
+  /// Create a live-in IR VPValue.
+  VPValue(Value *UV) : VPValue(VPValueSC, UV, nullptr) {}
+  VPValue(Type *Ty) : SubclassID(VPSymbolicValueSC), Ty(Ty), Def(nullptr) {}
   /// Create a VPValue for a \p Def which is a subclass of VPValue.
   VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {}
   /// Create a VPValue for a \p Def which defines multiple values.
@@ -86,14 +91,18 @@ class VPValue {
 
 public:
   /// Return the underlying Value attached to this VPValue.
-  Value *getUnderlyingValue() const { return UnderlyingVal; }
+  Value *getUnderlyingValue() const {
+    return isSymbolic() ? nullptr : UnderlyingVal;
+  }
 
   /// An enumeration for keeping track of the concrete subclass of VPValue that
   /// are actually instantiated.
   enum {
-    VPValueSC, /// A generic VPValue, like live-in values or defined by a recipe
-               /// that defines multiple values.
-    VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase.
+    VPValueSC, /// A generic non-symbolic VPValue, like live-in IR values or
+               /// defined by a recipe that defines multiple values.
+    VPSymbolicValueSC, /// A generic VPValue, like live-in values or defined by
+                       /// a recipe that defines multiple values.
+    VPVRecipeSC        /// A VPValue sub-class that is a VPRecipeBase.
   };
 
   VPValue(const VPValue &) = delete;
@@ -172,6 +181,13 @@ class VPValue {
   /// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
   bool isLiveIn() const { return !hasDefiningRecipe(); }
 
+  /// Returns true if the VPValue is symbolic, that is a live-in without
+  /// underlying value.
+  bool isSymbolic() const { return SubclassID == VPSymbolicValueSC; }
+
+  /// If the VPValue is a live-in, return its scalar type.
+  Type *getScalarType() const;
+
   /// Returns the underlying IR value, if this VPValue is defined outside the
   /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef
   /// inside a VPlan.

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

@@ -420,8 +420,10 @@ bool VPlanVerifier::verify(const VPlan &Plan) {
 bool llvm::verifyVPlanIsValid(const VPlan &Plan) {
   VPDominatorTree VPDT;
   VPDT.recalculate(const_cast<VPlan &>(Plan));
-  VPTypeAnalysis TypeInfo(
-      const_cast<VPlan &>(Plan).getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(const_cast<VPlan &>(Plan)
+                              .getCanonicalIV()
+                              ->getScalarType()
+                              ->getContext());
   VPlanVerifier Verifier(VPDT, TypeInfo);
   return Verifier.verify(Plan);
 }

llvm/unittests/Transforms/Vectorize/VPlanTestBase.h

@@ -71,7 +71,7 @@ class VPlanTestIRBase : public testing::Test {
 
     Loop *L = LI->getLoopFor(LoopHeader);
     PredicatedScalarEvolution PSE(*SE, *L);
-    auto Plan = std::make_unique<VPlan>(L);
+    auto Plan = std::make_unique<VPlan>(L, IntegerType::get(*Ctx, 64));
     VPlanHCFGBuilder HCFGBuilder(L, LI.get(), *Plan);
     HCFGBuilder.buildHierarchicalCFG();
     VPlanTransforms::introduceTopLevelVectorLoopRegion(
@@ -91,7 +91,8 @@ class VPlanTestBase : public testing::Test {
   }
 
   VPlan &getPlan(VPValue *TC = nullptr) {
-    Plans.push_back(std::make_unique<VPlan>(&*ScalarHeader, TC));
+    Plans.push_back(
+        std::make_unique<VPlan>(&*ScalarHeader, TC, IntegerType::get(C, 64)));
     return *Plans.back();
   }
 };