diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 584da9b7baef5..17fe21e481360 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4478,7 +4478,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks( static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI) { assert(VF.isVector() && "Checking a scalar VF?"); - VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()); + VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext()); DenseSet EphemeralRecipes; collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes); // Set of already visited types. @@ -9083,7 +9083,7 @@ static VPInstruction *addResumePhiRecipeForInduction( /// \p IVEndValues. static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, DenseMap &IVEndValues) { - VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()); + VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext()); auto *ScalarPH = Plan.getScalarPreheader(); auto *MiddleVPBB = cast(ScalarPH->getSinglePredecessor()); VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion(); @@ -9326,7 +9326,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { return !CM.requiresScalarEpilogue(VF.isVector()); }, Range); - auto Plan = std::make_unique(OrigLoop); + auto Plan = + std::make_unique(OrigLoop, Legal->getWidestInductionType()); // Build hierarchical CFG. // Convert to VPlan-transform and consoliate all transforms for VPlan // creation. @@ -9632,7 +9633,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) { assert(EnableVPlanNativePath && "VPlan-native path is not enabled."); // Create new empty VPlan - auto Plan = std::make_unique(OrigLoop); + auto Plan = + std::make_unique(OrigLoop, Legal->getWidestInductionType()); // Build hierarchical CFG VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan); HCFGBuilder.buildHierarchicalCFG(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index ac8884cfc17ae..7ad1340cdea60 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -222,7 +222,7 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI, Loop *CurrentParentLoop, Type *CanonicalIVTy) : TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan), CurrentParentLoop(CurrentParentLoop), LVer(nullptr), - TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {} + TypeAnalysis(CanonicalIVTy->getContext()), VPDT(*Plan) {} Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) { if (Def->isLiveIn()) @@ -850,7 +850,8 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent, } #endif -VPlan::VPlan(Loop *L) { +VPlan::VPlan(Loop *L, Type *InductionTy) + : VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) { setEntry(createVPIRBasicBlock(L->getLoopPreheader())); ScalarHeader = createVPIRBasicBlock(L->getHeader()); @@ -861,7 +862,7 @@ VPlan::VPlan(Loop *L) { } VPlan::~VPlan() { - VPValue DummyValue; + VPValue DummyValue((Type *)nullptr); for (auto *VPB : CreatedBlocks) { if (auto *VPBB = dyn_cast(VPB)) { @@ -891,10 +892,10 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV, IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); auto *TCMO = Builder.CreateSub(TripCountV, ConstantInt::get(TCTy, 1), "trip.count.minus.1"); - BackedgeTakenCount->setUnderlyingValue(TCMO); + BackedgeTakenCount->replaceAllUsesWith(getOrAddLiveIn(TCMO)); } - VectorTripCount.setUnderlyingValue(VectorTripCountV); + VectorTripCount.replaceAllUsesWith(getOrAddLiveIn(VectorTripCountV)); IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); // FIXME: Model VF * UF computation completely in VPlan. @@ -903,12 +904,13 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV, unsigned UF = getUF(); if (VF.getNumUsers()) { Value *RuntimeVF = getRuntimeVF(Builder, TCTy, State.VF); - VF.setUnderlyingValue(RuntimeVF); - VFxUF.setUnderlyingValue( + VF.replaceAllUsesWith(getOrAddLiveIn(RuntimeVF)); + VFxUF.replaceAllUsesWith(getOrAddLiveIn( UF > 1 ? Builder.CreateMul(RuntimeVF, ConstantInt::get(TCTy, UF)) - : RuntimeVF); + : RuntimeVF)); } else { - VFxUF.setUnderlyingValue(createStepForVF(Builder, TCTy, State.VF, UF)); + VFxUF.replaceAllUsesWith( + getOrAddLiveIn(createStepForVF(Builder, TCTy, State.VF, UF))); } } @@ -1175,7 +1177,8 @@ VPlan *VPlan::duplicate() { return VPIRBB && VPIRBB->getIRBasicBlock() == ScalarHeaderIRBB; })); // Create VPlan, clone live-ins and remap operands in the cloned blocks. - auto *NewPlan = new VPlan(cast(NewEntry), NewScalarHeader); + auto *NewPlan = new VPlan(cast(NewEntry), NewScalarHeader, + getCanonicalIV()->getScalarType()); DenseMap Old2NewVPValues; for (VPValue *OldLiveIn : getLiveIns()) { Old2NewVPValues[OldLiveIn] = @@ -1185,7 +1188,7 @@ VPlan *VPlan::duplicate() { Old2NewVPValues[&VF] = &NewPlan->VF; Old2NewVPValues[&VFxUF] = &NewPlan->VFxUF; if (BackedgeTakenCount) { - NewPlan->BackedgeTakenCount = new VPValue(); + NewPlan->BackedgeTakenCount = new VPValue((Type *)nullptr); Old2NewVPValues[BackedgeTakenCount] = NewPlan->BackedgeTakenCount; } assert(TripCount && "trip count must be set"); @@ -1371,6 +1374,11 @@ static bool isDefinedInsideLoopRegions(const VPValue *VPV) { DefR->getParent()->getEnclosingLoopRegion()); } +Type *VPValue::getType() const { + assert(isLiveIn() && "can only return the type for a live-in"); + return SubclassID == VPSymbolicValueSC ? Ty : getUnderlyingValue()->getType(); +} + bool VPValue::isDefinedOutsideLoopRegions() const { return !isDefinedInsideLoopRegions(this); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 3059b87ae63c8..d81afa1bf406e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3439,8 +3439,9 @@ class VPlan { /// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader /// wrapping the original header of the scalar loop. - VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader) - : Entry(Entry), ScalarHeader(ScalarHeader) { + VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader, Type *InductionTy) + : Entry(Entry), ScalarHeader(ScalarHeader), VectorTripCount(InductionTy), + VF(InductionTy), VFxUF(InductionTy) { Entry->setPlan(this); assert(ScalarHeader->getNumSuccessors() == 0 && "scalar header must be a leaf node"); @@ -3450,11 +3451,13 @@ class VPlan { /// Construct a VPlan for \p L. This will create VPIRBasicBlocks wrapping the /// original preheader and scalar header of \p L, to be used as entry and /// scalar header blocks of the new VPlan. - VPlan(Loop *L); + VPlan(Loop *L, Type *InductionTy); /// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock - /// wrapping \p ScalarHeaderBB and a trip count of \p TC. - VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC) { + /// wrapping \p ScalarHeaderBB and a trip count of \p TC. Also creates + /// symbolic VectorTripCount, VF and VFxUF VPValues using \p InductionTy. + VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC, Type *InductionTy) + : VectorTripCount(InductionTy), VF(InductionTy), VFxUF(InductionTy) { setEntry(createVPBasicBlock("preheader")); ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB); TripCount = TC; @@ -3546,7 +3549,7 @@ class VPlan { /// The backedge taken count of the original loop. VPValue *getOrCreateBackedgeTakenCount() { if (!BackedgeTakenCount) - BackedgeTakenCount = new VPValue(); + BackedgeTakenCount = new VPValue(getCanonicalIV()->getScalarType()); return BackedgeTakenCount; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index 38bec733dbf73..5f05e776dfa22 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -231,13 +231,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) { if (Type *CachedTy = CachedTypes.lookup(V)) return CachedTy; - if (V->isLiveIn()) { - if (auto *IRValue = V->getLiveInIRValue()) - return IRValue->getType(); - // All VPValues without any underlying IR value (like the vector trip count - // or the backedge-taken count) have the same type as the canonical IV. - return CanonicalIVTy; - } + if (V->isLiveIn()) + return V->getType(); Type *ResultTy = TypeSwitch(V->getDefiningRecipe()) diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h index cc21870bee2e3..7d4771d178924 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h @@ -39,10 +39,6 @@ class Type; /// of the previously inferred types. class VPTypeAnalysis { DenseMap CachedTypes; - /// Type of the canonical induction variable. Used for all VPValues without - /// any underlying IR value (like the vector trip count or the backedge-taken - /// count). - Type *CanonicalIVTy; LLVMContext &Ctx; Type *inferScalarTypeForRecipe(const VPBlendRecipe *R); @@ -55,8 +51,7 @@ class VPTypeAnalysis { Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R); public: - VPTypeAnalysis(Type *CanonicalIVTy) - : CanonicalIVTy(CanonicalIVTy), Ctx(CanonicalIVTy->getContext()) {} + VPTypeAnalysis(LLVMContext &Ctx) : Ctx(Ctx) {} /// Infer the type of \p V. Returns the scalar type of \p V. Type *inferScalarType(const VPValue *V); diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h index d53b72bb2258a..1ba55d6ac15ee 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h +++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h @@ -379,8 +379,8 @@ struct VPCostContext { VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, Type *CanIVTy, LoopVectorizationCostModel &CM, TargetTransformInfo::TargetCostKind CostKind) - : TTI(TTI), TLI(TLI), Types(CanIVTy), LLVMCtx(CanIVTy->getContext()), - CM(CM), CostKind(CostKind) {} + : TTI(TTI), TLI(TLI), Types(CanIVTy->getContext()), + LLVMCtx(CanIVTy->getContext()), CM(CM), CostKind(CostKind) {} /// Return the cost for \p UI with \p VF using the legacy cost model as /// fallback until computing the cost of all recipes migrates to VPlan. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 6c8680ac96610..2de6f3b83c1d2 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -549,7 +549,7 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind, // Truncate base induction if needed. Type *CanonicalIVType = CanonicalIV->getScalarType(); - VPTypeAnalysis TypeInfo(CanonicalIVType); + VPTypeAnalysis TypeInfo(CanonicalIVType->getContext()); Type *ResultTy = TypeInfo.inferScalarType(BaseIV); if (TruncI) { Type *TruncTy = TruncI->getType(); @@ -795,13 +795,14 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo, void VPlanTransforms::optimizeInductionExitUsers( VPlan &Plan, DenseMap &EndValues) { VPBlockBase *MiddleVPBB = Plan.getMiddleBlock(); - VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()); + VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()->getContext()); for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) { for (VPRecipeBase &R : *ExitVPBB) { auto *ExitIRI = cast(&R); if (!isa(ExitIRI->getInstruction())) break; + for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) { if (PredVPBB == MiddleVPBB) if (VPValue *Escape = optimizeLatchExitInductionUser( @@ -957,8 +958,11 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { #ifndef NDEBUG // Verify that the cached type info is for both A and its users is still // accurate by comparing it to freshly computed types. - VPTypeAnalysis TypeInfo2( - R.getParent()->getPlan()->getCanonicalIV()->getScalarType()); + VPTypeAnalysis TypeInfo2(R.getParent() + ->getPlan() + ->getCanonicalIV() + ->getScalarType() + ->getContext()); assert(TypeInfo.inferScalarType(A) == TypeInfo2.inferScalarType(A)); for (VPUser *U : A->users()) { auto *R = cast(U); @@ -1001,7 +1005,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { ReversePostOrderTraversal> RPOT( Plan.getEntry()); - VPTypeAnalysis TypeInfo(&CanonicalIVTy); + VPTypeAnalysis TypeInfo(CanonicalIVTy.getContext()); for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) { for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { simplifyRecipe(R, TypeInfo); @@ -1351,7 +1355,7 @@ void VPlanTransforms::truncateToMinimalBitwidths( // typed. DenseMap ProcessedTruncs; Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType(); - VPTypeAnalysis TypeInfo(CanonicalIVType); + VPTypeAnalysis TypeInfo(CanonicalIVType->getContext()); VPBasicBlock *PH = Plan.getVectorPreheader(); for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly( vp_depth_first_deep(Plan.getVectorLoopRegion()))) { @@ -1743,8 +1747,8 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask, /// Replace recipes with their EVL variants. static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType(); - VPTypeAnalysis TypeInfo(CanonicalIVType); LLVMContext &Ctx = CanonicalIVType->getContext(); + VPTypeAnalysis TypeInfo(Ctx); VPValue *AllOneMask = Plan.getOrAddLiveIn(ConstantInt::getTrue(Ctx)); VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); VPBasicBlock *Header = LoopRegion->getEntryBasicBlock(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index a36c2aeb3da5c..0e33bd77bc30d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -73,7 +73,7 @@ class UnrollState { public: UnrollState(VPlan &Plan, unsigned UF, LLVMContext &Ctx) - : Plan(Plan), UF(UF), TypeInfo(Plan.getCanonicalIV()->getScalarType()) {} + : Plan(Plan), UF(UF), TypeInfo(Ctx) {} void unrollBlock(VPBlockBase *VPB); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index d0fa62978c1b4..c1f95aa24d5e5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -61,8 +61,12 @@ class VPValue { SmallVector Users; protected: - // Hold the underlying Value, if any, attached to this VPValue. - Value *UnderlyingVal; + union { + // Hold the underlying Value, if any, attached to this non-symbolic VPValue. + Value *UnderlyingVal; + // Hold the type of this VPValue, if it is symbolic. + Type *Ty; + }; /// Pointer to the VPDef that defines this VPValue. If it is nullptr, the /// VPValue is not defined by any recipe modeled in VPlan. @@ -70,8 +74,9 @@ class VPValue { VPValue(const unsigned char SC, Value *UV = nullptr, VPDef *Def = nullptr); - /// Create a live-in VPValue. - VPValue(Value *UV = nullptr) : VPValue(VPValueSC, UV, nullptr) {} + /// Create a live-in IR VPValue. + VPValue(Value *UV) : VPValue(VPValueSC, UV, nullptr) {} + VPValue(Type *Ty) : SubclassID(VPSymbolicValueSC), Ty(Ty), Def(nullptr) {} /// Create a VPValue for a \p Def which is a subclass of VPValue. VPValue(VPDef *Def, Value *UV = nullptr) : VPValue(VPVRecipeSC, UV, Def) {} /// Create a VPValue for a \p Def which defines multiple values. @@ -86,14 +91,18 @@ class VPValue { public: /// Return the underlying Value attached to this VPValue. - Value *getUnderlyingValue() const { return UnderlyingVal; } + Value *getUnderlyingValue() const { + return isSymbolic() ? nullptr : UnderlyingVal; + } /// An enumeration for keeping track of the concrete subclass of VPValue that /// are actually instantiated. enum { - VPValueSC, /// A generic VPValue, like live-in values or defined by a recipe - /// that defines multiple values. - VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase. + VPValueSC, /// A generic non-symbolic VPValue, like live-in IR values or + /// defined by a recipe that defines multiple values. + VPSymbolicValueSC, /// A generic VPValue, like live-in values or defined by + /// a recipe that defines multiple values. + VPVRecipeSC /// A VPValue sub-class that is a VPRecipeBase. }; VPValue(const VPValue &) = delete; @@ -172,6 +181,13 @@ class VPValue { /// Returns true if this VPValue is a live-in, i.e. defined outside the VPlan. bool isLiveIn() const { return !hasDefiningRecipe(); } + /// Returns true if the VPValue is symbolic, that is a live-in without + /// underlying value. + bool isSymbolic() const { return SubclassID == VPSymbolicValueSC; } + + /// If the VPValue is a live-in, return its scalar type. + Type *getScalarType() const; + /// Returns the underlying IR value, if this VPValue is defined outside the /// scope of VPlan. Returns nullptr if the VPValue is defined by a VPDef /// inside a VPlan. diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 6fe131879b1a2..16f1e70b4e753 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -420,8 +420,10 @@ bool VPlanVerifier::verify(const VPlan &Plan) { bool llvm::verifyVPlanIsValid(const VPlan &Plan) { VPDominatorTree VPDT; VPDT.recalculate(const_cast(Plan)); - VPTypeAnalysis TypeInfo( - const_cast(Plan).getCanonicalIV()->getScalarType()); + VPTypeAnalysis TypeInfo(const_cast(Plan) + .getCanonicalIV() + ->getScalarType() + ->getContext()); VPlanVerifier Verifier(VPDT, TypeInfo); return Verifier.verify(Plan); } diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h index caf5d2357411d..85ac0c3ac8a75 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h +++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h @@ -71,7 +71,7 @@ class VPlanTestIRBase : public testing::Test { Loop *L = LI->getLoopFor(LoopHeader); PredicatedScalarEvolution PSE(*SE, *L); - auto Plan = std::make_unique(L); + auto Plan = std::make_unique(L, IntegerType::get(*Ctx, 64)); VPlanHCFGBuilder HCFGBuilder(L, LI.get(), *Plan); HCFGBuilder.buildHierarchicalCFG(); VPlanTransforms::introduceTopLevelVectorLoopRegion( @@ -91,7 +91,8 @@ class VPlanTestBase : public testing::Test { } VPlan &getPlan(VPValue *TC = nullptr) { - Plans.push_back(std::make_unique(&*ScalarHeader, TC)); + Plans.push_back( + std::make_unique(&*ScalarHeader, TC, IntegerType::get(C, 64))); return *Plans.back(); } };