[VPlan] Add VPSymbolicValueSC for typed VPValues w/o underlying IR value

This introduces a new VPSymbolicValueSC to be used for typed live-in VPValues
without underlying IR. VPValue is updated to store either the type or
an underlying value in an union. This allows keeping the size of VPValue
unchanged.

The main motivation for adding the type is to not require passing the
canonical IV type to VPTypeAnalysis.

While with this patch, we still need to pass the LLVMContext, this can
also be removed in a future patch (see
9493c38)

Author: fhahn

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -4478,7 +4478,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
 static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
                                 const TargetTransformInfo &TTI) {
   assert(VF.isVector() && "Checking a scalar VF?");
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
   DenseSet<VPRecipeBase *> EphemeralRecipes;
   collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
   // Set of already visited types.
@@ -9083,7 +9083,7 @@ static VPInstruction *addResumePhiRecipeForInduction(
 /// \p IVEndValues.
 static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
                                 DenseMap<VPValue *, VPValue *> &IVEndValues) {
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
   auto *ScalarPH = Plan.getScalarPreheader();
   auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getSinglePredecessor());
   VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
@@ -9326,7 +9326,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
             return !CM.requiresScalarEpilogue(VF.isVector());
           },
           Range);
-  auto Plan = std::make_unique<VPlan>(OrigLoop);
+  auto Plan =
+      std::make_unique<VPlan>(OrigLoop, Legal->getWidestInductionType());
   // Build hierarchical CFG.
   // Convert to VPlan-transform and consoliate all transforms for VPlan
   // creation.
@@ -9632,7 +9633,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
   assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
 
   // Create new empty VPlan
-  auto Plan = std::make_unique<VPlan>(OrigLoop);
+  auto Plan =
+      std::make_unique<VPlan>(OrigLoop, Legal->getWidestInductionType());
   // Build hierarchical CFG
   VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
   HCFGBuilder.buildHierarchicalCFG();

llvm/lib/Transforms/Vectorize/VPlan.cpp

@@ -222,7 +222,7 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
                                    Loop *CurrentParentLoop, Type *CanonicalIVTy)
     : TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
       CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
-      TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
+      TypeAnalysis(CanonicalIVTy->getContext()), VPDT(*Plan) {}
 
 Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
   if (Def->isLiveIn())
@@ -850,7 +850,8 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
-VPlan::VPlan(Loop *L) {
+VPlan::VPlan(Loop *L, Type *InductionTy)
+    : VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) {
   setEntry(createVPIRBasicBlock(L->getLoopPreheader()));
   ScalarHeader = createVPIRBasicBlock(L->getHeader());
 
@@ -861,7 +862,7 @@ VPlan::VPlan(Loop *L) {
 }
 
 VPlan::~VPlan() {
-  VPValue DummyValue;
+  VPValue DummyValue((Type *)nullptr);
 
   for (auto *VPB : CreatedBlocks) {
     if (auto *VPBB = dyn_cast<VPBasicBlock>(VPB)) {
@@ -891,10 +892,10 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
     IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
     auto *TCMO = Builder.CreateSub(TripCountV, ConstantInt::get(TCTy, 1),
                                    "trip.count.minus.1");
-    BackedgeTakenCount->setUnderlyingValue(TCMO);
+    BackedgeTakenCount->replaceAllUsesWith(getOrAddLiveIn(TCMO));
   }
 
-  VectorTripCount.setUnderlyingValue(VectorTripCountV);
+  VectorTripCount.replaceAllUsesWith(getOrAddLiveIn(VectorTripCountV));
 
   IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
   // FIXME: Model VF * UF computation completely in VPlan.
@@ -903,12 +904,13 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
   unsigned UF = getUF();
   if (VF.getNumUsers()) {
     Value *RuntimeVF = getRuntimeVF(Builder, TCTy, State.VF);
-    VF.setUnderlyingValue(RuntimeVF);
-    VFxUF.setUnderlyingValue(
+    VF.replaceAllUsesWith(getOrAddLiveIn(RuntimeVF));
+    VFxUF.replaceAllUsesWith(getOrAddLiveIn(
         UF > 1 ? Builder.CreateMul(RuntimeVF, ConstantInt::get(TCTy, UF))
-               : RuntimeVF);
+               : RuntimeVF));
   } else {
-    VFxUF.setUnderlyingValue(createStepForVF(Builder, TCTy, State.VF, UF));
+    VFxUF.replaceAllUsesWith(
+        getOrAddLiveIn(createStepForVF(Builder, TCTy, State.VF, UF)));
   }
 }
 
@@ -1175,7 +1177,8 @@ VPlan *VPlan::duplicate() {
         return VPIRBB && VPIRBB->getIRBasicBlock() == ScalarHeaderIRBB;
       }));
   // Create VPlan, clone live-ins and remap operands in the cloned blocks.
-  auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader);
+  auto *NewPlan = new VPlan(cast<VPBasicBlock>(NewEntry), NewScalarHeader,
+                            getCanonicalIV()->getScalarType());
   DenseMap<VPValue *, VPValue *> Old2NewVPValues;
   for (VPValue *OldLiveIn : getLiveIns()) {
     Old2NewVPValues[OldLiveIn] =
@@ -1185,7 +1188,7 @@ VPlan *VPlan::duplicate() {
   Old2NewVPValues[&VF] = &NewPlan->VF;
   Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF;
   if (BackedgeTakenCount) {
-    NewPlan->BackedgeTakenCount = new VPValue();
+    NewPlan->BackedgeTakenCount = new VPValue((Type *)nullptr);
     Old2NewVPValues[BackedgeTakenCount] = NewPlan->BackedgeTakenCount;
   }
   assert(TripCount && "trip count must be set");
@@ -1371,6 +1374,11 @@ static bool isDefinedInsideLoopRegions(const VPValue *VPV) {
                   DefR->getParent()->getEnclosingLoopRegion());
 }
 
+Type *VPValue::getType() const {
+  assert(isLiveIn());
+  return SubclassID == VPSymbolicValueSC ? Ty : getUnderlyingValue()->getType();
+}
+
 bool VPValue::isDefinedOutsideLoopRegions() const {
   return !isDefinedInsideLoopRegions(this);
 }

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -3439,8 +3439,9 @@ class VPlan {
 
   /// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
   /// wrapping the original header of the scalar loop.
-  VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
-      : Entry(Entry), ScalarHeader(ScalarHeader) {
+  VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader, Type *InductionTy)
+      : Entry(Entry), ScalarHeader(ScalarHeader), VectorTripCount(InductionTy),
+        VF(InductionTy), VFxUF(InductionTy) {
     Entry->setPlan(this);
     assert(ScalarHeader->getNumSuccessors() == 0 &&
            "scalar header must be a leaf node");
@@ -3450,11 +3451,12 @@ class VPlan {
   /// Construct a VPlan for \p L. This will create VPIRBasicBlocks wrapping the
   /// original preheader and scalar header of \p L, to be used as entry and
   /// scalar header blocks of the new VPlan.
-  VPlan(Loop *L);
+  VPlan(Loop *L, Type *InductionTy);
 
   /// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
   /// wrapping \p ScalarHeaderBB and a trip count of \p TC.
-  VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC) {
+  VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC, Type *InductionTy)
+      : VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) {
     setEntry(createVPBasicBlock("preheader"));
     ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB);
     TripCount = TC;
@@ -3546,7 +3548,7 @@ class VPlan {
   /// The backedge taken count of the original loop.
   VPValue *getOrCreateBackedgeTakenCount() {
     if (!BackedgeTakenCount)
-      BackedgeTakenCount = new VPValue();
+      BackedgeTakenCount = new VPValue(getCanonicalIV()->getScalarType());
     return BackedgeTakenCount;
   }
 

llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

@@ -231,13 +231,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
   if (Type *CachedTy = CachedTypes.lookup(V))
     return CachedTy;
 
-  if (V->isLiveIn()) {
-    if (auto *IRValue = V->getLiveInIRValue())
-      return IRValue->getType();
-    // All VPValues without any underlying IR value (like the vector trip count
-    // or the backedge-taken count) have the same type as the canonical IV.
-    return CanonicalIVTy;
-  }
+  if (V->isLiveIn())
+    return V->getType();
 
   Type *ResultTy =
       TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())

llvm/lib/Transforms/Vectorize/VPlanAnalysis.h

@@ -39,10 +39,6 @@ class Type;
 /// of the previously inferred types.
 class VPTypeAnalysis {
   DenseMap<const VPValue *, Type *> CachedTypes;
-  /// Type of the canonical induction variable. Used for all VPValues without
-  /// any underlying IR value (like the vector trip count or the backedge-taken
-  /// count).
-  Type *CanonicalIVTy;
   LLVMContext &Ctx;
 
   Type *inferScalarTypeForRecipe(const VPBlendRecipe *R);
@@ -55,8 +51,7 @@ class VPTypeAnalysis {
   Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
 
 public:
-  VPTypeAnalysis(Type *CanonicalIVTy)
-      : CanonicalIVTy(CanonicalIVTy), Ctx(CanonicalIVTy->getContext()) {}
+  VPTypeAnalysis(LLVMContext &Ctx) : Ctx(Ctx) {}
 
   /// Infer the type of \p V. Returns the scalar type of \p V.
   Type *inferScalarType(const VPValue *V);

llvm/lib/Transforms/Vectorize/VPlanHelpers.h

@@ -379,8 +379,8 @@ struct VPCostContext {
   VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
                 Type *CanIVTy, LoopVectorizationCostModel &CM,
                 TargetTransformInfo::TargetCostKind CostKind)
-      : TTI(TTI), TLI(TLI), Types(CanIVTy), LLVMCtx(CanIVTy->getContext()),
-        CM(CM), CostKind(CostKind) {}
+      : TTI(TTI), TLI(TLI), Types(CanIVTy->getContext()),
+        LLVMCtx(CanIVTy->getContext()), CM(CM), CostKind(CostKind) {}
 
   /// Return the cost for \p UI with \p VF using the legacy cost model as
   /// fallback until computing the cost of all recipes migrates to VPlan.

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -549,7 +549,7 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
 
   // Truncate base induction if needed.
   Type *CanonicalIVType = CanonicalIV->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
+  VPTypeAnalysis TypeInfo(CanonicalIVType->getContext());
   Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
   if (TruncI) {
     Type *TruncTy = TruncI->getType();
@@ -795,13 +795,14 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
 void VPlanTransforms::optimizeInductionExitUsers(
     VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
   VPBlockBase *MiddleVPBB = Plan.getMiddleBlock();
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext());
   for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
     for (VPRecipeBase &R : *ExitVPBB) {
       auto *ExitIRI = cast<VPIRInstruction>(&R);
       if (!isa<PHINode>(ExitIRI->getInstruction()))
         break;
 
+
       for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
         if (PredVPBB == MiddleVPBB)
           if (VPValue *Escape = optimizeLatchExitInductionUser(
@@ -957,8 +958,11 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
 #ifndef NDEBUG
     // Verify that the cached type info is for both A and its users is still
     // accurate by comparing it to freshly computed types.
-    VPTypeAnalysis TypeInfo2(
-        R.getParent()->getPlan()->getCanonicalIV()->getScalarType());
+    VPTypeAnalysis TypeInfo2(R.getParent()
+                                 ->getPlan()
+                                 ->getCanonicalIV()
+                                 ->getScalarType()
+                                 ->getContext());
     assert(TypeInfo.inferScalarType(A) == TypeInfo2.inferScalarType(A));
     for (VPUser *U : A->users()) {
       auto *R = cast<VPRecipeBase>(U);
@@ -1001,7 +1005,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
 void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
       Plan.getEntry());
-  VPTypeAnalysis TypeInfo(&CanonicalIVTy);
+  VPTypeAnalysis TypeInfo(CanonicalIVTy.getContext());
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
       simplifyRecipe(R, TypeInfo);
@@ -1351,7 +1355,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
   // typed.
   DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
   Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
+  VPTypeAnalysis TypeInfo(CanonicalIVType->getContext());
   VPBasicBlock *PH = Plan.getVectorPreheader();
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
@@ -1743,8 +1747,8 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
 /// Replace recipes with their EVL variants.
 static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
   Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
   LLVMContext &Ctx = CanonicalIVType->getContext();
+  VPTypeAnalysis TypeInfo(Ctx);
   VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx));
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();

llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp

@@ -73,7 +73,7 @@ class UnrollState {
 
 public:
   UnrollState(VPlan &Plan, unsigned UF, LLVMContext &Ctx)
-      : Plan(Plan), UF(UF), TypeInfo(Plan.getCanonicalIV()->getScalarType()) {}
+      : Plan(Plan), UF(UF), TypeInfo(Ctx) {}
 
   void unrollBlock(VPBlockBase *VPB);
 

llvm/lib/Transforms/Vectorize/VPlanValue.h

@@ -61,17 +61,22 @@ class VPValue {
   SmallVector<VPUser *, 1> Users;
 
 protected:
-  // Hold the underlying Value, if any, attached to this VPValue.
-  Value *UnderlyingVal;
+  union {
+    // Hold the underlying Value, if any, attached to this non-symbolic VPValue.
+    Value *UnderlyingVal;
+    // Hold the type of this VPValue, if it is symbolic.
+    Type *Ty;
+  };
 
   /// Pointer to the VPDef that defines this VPValue. If it is nullptr, the
   /// VPValue is not defined by any recipe modeled in VPlan.
   VPDef *Def;
 
   VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr);
 
-  /// Create a live-in VPValue.
-  VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {}
+  /// Create a live-in IR VPValue.
+  VPValue(Value *UV) : VPValue(VPValueSC, UV, nullptr) {}
+  VPValue(Type *Ty) : SubclassID(VPSymbolicValueSC), Ty(Ty), Def(nullptr) {}
   /// Create a VPValue for a \p Def which is a subclass of VPValue.
   VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {}
   /// Create a VPValue for a \p Def which defines multiple values.
@@ -86,14 +91,18 @@ class VPValue {
 
 public:
   /// Return the underlying Value attached to this VPValue.
-  Value *getUnderlyingValue() const { return UnderlyingVal; }
+  Value *getUnderlyingValue() const {
+    return SubclassID == VPSymbolicValueSC ? nullptr : UnderlyingVal;
+  }
 
   /// An enumeration for keeping track of the concrete subclass of VPValue that
   /// are actually instantiated.
   enum {
-    VPValueSC, /// A generic VPValue, like live-in values or defined by a recipe
-               /// that defines multiple values.
-    VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase.
+    VPValueSC, /// A generic non-symbolic VPValue, like live-in IR values or
+               /// defined by a recipe that defines multiple values.
+    VPSymbolicValueSC, /// A generic VPValue, like live-in values or defined by
+                       /// a recipe that defines multiple values.
+    VPVRecipeSC        /// A VPValue sub-class that is a VPRecipeBase.
   };
 
   VPValue(const VPValue &) = delete;
@@ -172,6 +181,10 @@ class VPValue {
   /// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
   bool isLiveIn() const { return !hasDefiningRecipe(); }
 
+  bool isSymbolic() const { return SubclassID == VPSymbolicValueSC; }
+
+  Type *getType() const;
+
   /// Returns the underlying IR value, if this VPValue is defined outside the
   /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef
   /// inside a VPlan.

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

@@ -420,8 +420,10 @@ bool VPlanVerifier::verify(const VPlan &Plan) {
 bool llvm::verifyVPlanIsValid(const VPlan &Plan) {
   VPDominatorTree VPDT;
   VPDT.recalculate(const_cast<VPlan &>(Plan));
-  VPTypeAnalysis TypeInfo(
-      const_cast<VPlan &>(Plan).getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(const_cast<VPlan &>(Plan)
+                              .getCanonicalIV()
+                              ->getScalarType()
+                              ->getContext());
   VPlanVerifier Verifier(VPDT, TypeInfo);
   return Verifier.verify(Plan);
 }