diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 275b3d5678560..56a2b09539fe0 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -527,6 +527,8 @@ class InnerLoopVectorizer { return AdditionalBypassBlock; } + IRBuilderBase &getIRBuilder() { return Builder; } + protected: friend class LoopVectorizationPlanner; @@ -2321,7 +2323,8 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) { return VectorTripCount; Value *TC = getTripCount(); - IRBuilder<> Builder(InsertBlock->getTerminator()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(InsertBlock->getTerminator()); Type *Ty = TC->getType(); // This is where we can make the step a runtime constant. @@ -2395,7 +2398,8 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { // Reuse existing vector loop preheader for TC checks. // Note that new preheader block is generated for vector loop. BasicBlock *const TCCheckBlock = LoopVectorPreHeader; - IRBuilder<> Builder(TCCheckBlock->getTerminator()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(TCCheckBlock->getTerminator()); // Generate code to check if the loop's trip count is less than VF * UF, or // equal to it in case a scalar epilogue is required; this implies that the @@ -7840,7 +7844,8 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, // Reuse existing vector loop preheader for TC checks. // Note that new preheader block is generated for vector loop. BasicBlock *const TCCheckBlock = LoopVectorPreHeader; - IRBuilder<> Builder(TCCheckBlock->getTerminator()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(TCCheckBlock->getTerminator()); // Generate code to check if the loop's trip count is less than VF * UF of the // main vector loop. @@ -7965,7 +7970,8 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck( assert(EPI.TripCount && "Expected trip count to have been saved in the first pass."); Value *TC = EPI.TripCount; - IRBuilder<> Builder(Insert->getTerminator()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(Insert->getTerminator()); Value *Count = Builder.CreateSub(TC, EPI.VectorTripCount, "n.vec.remaining"); // Generate code to check if the loop's trip count is less than VF * UF of the @@ -9608,7 +9614,7 @@ void VPDerivedIVRecipe::execute(VPTransformState &State) { assert(!State.Lane && "VPDerivedIVRecipe being replicated."); // Fast-math-flags propagate from the original induction instruction. - IRBuilder<>::FastMathFlagGuard FMFG(State.Builder); + IRBuilderBase::FastMathFlagGuard FMFG(State.Builder); if (FPBinOp) State.Builder.setFastMathFlags(FPBinOp->getFastMathFlags()); @@ -10011,10 +10017,9 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) { /// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded /// SCEVs from \p ExpandedSCEVs and set resume values for header recipes. -static void -preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, - const SCEV2ValueTy &ExpandedSCEVs, - const EpilogueLoopVectorizationInfo &EPI) { +static void preparePlanForEpilogueVectorLoop( + VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, + const EpilogueLoopVectorizationInfo &EPI, IRBuilderBase &Builder) { VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion(); VPBasicBlock *Header = VectorLoop->getEntryBasicBlock(); Header->setName("vec.epilog.vector.body"); @@ -10081,7 +10086,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, // start value; compare the final value from the main vector loop // to the start value. BasicBlock *PBB = cast(ResumeV)->getParent(); - IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(PBB, PBB->getFirstNonPHIIt()); ResumeV = Builder.CreateICmpNE(ResumeV, RdxDesc.getRecurrenceStartValue()); } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) { @@ -10096,7 +10102,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, // less than the minimum value of a monotonically increasing induction // variable. BasicBlock *ResumeBB = cast(ResumeV)->getParent(); - IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(ResumeBB, ResumeBB->getFirstNonPHIIt()); Value *Cmp = Builder.CreateICmpEQ( ResumeV, ToFrozen[RdxDesc.getRecurrenceStartValue()]); ResumeV = @@ -10150,9 +10157,9 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, // resume value for the induction variable comes from the trip count of the // main vector loop, passed as the second argument. static Value *createInductionAdditionalBypassValues( - PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder, - const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount, - Instruction *OldInduction) { + PHINode *OrigPhi, const InductionDescriptor &II, + IRBuilderBase &BypassBuilder, const SCEV2ValueTy &ExpandedSCEVs, + Value *MainVectorTripCount, Instruction *OldInduction) { Value *Step = getExpandedStep(II, ExpandedSCEVs); // For the primary induction the additional bypass end value is known. // Otherwise it is computed. @@ -10541,15 +10548,18 @@ bool LoopVectorizePass::processLoop(Loop *L) { ORE, EPI, &CM, BFI, PSI, Checks, BestEpiPlan); EpilogILV.setTripCount(MainILV.getTripCount()); - preparePlanForEpilogueVectorLoop(BestEpiPlan, L, ExpandedSCEVs, EPI); + preparePlanForEpilogueVectorLoop(BestEpiPlan, L, ExpandedSCEVs, EPI, + EpilogILV.getIRBuilder()); LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT, true); // Fix induction resume values from the additional bypass block. BasicBlock *BypassBlock = EpilogILV.getAdditionalBypassBlock(); - IRBuilder<> BypassBuilder(BypassBlock, - BypassBlock->getFirstInsertionPt()); + IRBuilderBase &BypassBuilder = EpilogILV.getIRBuilder(); + IRBuilderBase::InsertPointGuard Guard(BypassBuilder); + BypassBuilder.SetInsertPoint(BypassBlock, + BypassBlock->getFirstInsertionPt()); BasicBlock *PH = L->getLoopPreheader(); for (const auto &[IVPhi, II] : LVL.getInductionVars()) { auto *Inc = cast(IVPhi->getIncomingValueForBlock(PH)); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 06b738afbd221..16f84a36e85e4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -272,7 +272,7 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) { if (VF.isScalar()) return V; // Place the code for broadcasting invariant variables in the new preheader. - IRBuilder<>::InsertPointGuard Guard(Builder); + IRBuilderBase::InsertPointGuard Guard(Builder); if (SafeToHoist) { BasicBlock *LoopVectorPreHeader = CFG.VPBB2IRBB[Plan->getVectorPreheader()]; @@ -887,9 +887,11 @@ VPlan::~VPlan() { void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV, VPTransformState &State) { Type *TCTy = TripCountV->getType(); + IRBuilderBase &Builder = State.Builder; // Check if the backedge taken count is needed, and if so build it. if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) { - IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(State.CFG.PrevBB->getTerminator()); auto *TCMO = Builder.CreateSub(TripCountV, ConstantInt::get(TCTy, 1), "trip.count.minus.1"); BackedgeTakenCount->setUnderlyingValue(TCMO); @@ -897,7 +899,8 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV, VectorTripCount.setUnderlyingValue(VectorTripCountV); - IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(State.CFG.PrevBB->getTerminator()); // FIXME: Model VF * UF computation completely in VPlan. unsigned UF = getUF(); if (VF.getNumUsers()) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 14ed40f16683a..5ed4e0c040cf9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1986,7 +1986,7 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { Instruction *EntryVal = Trunc ? cast(Trunc) : getPHINode(); // Fast-math-flags propagate from the original induction instruction. - IRBuilder<>::FastMathFlagGuard FMFG(Builder); + IRBuilderBase::FastMathFlagGuard FMFG(Builder); if (ID.getInductionBinOp() && isa(ID.getInductionBinOp())) Builder.setFastMathFlags(ID.getInductionBinOp()->getFastMathFlags()); @@ -2110,7 +2110,7 @@ void VPDerivedIVRecipe::print(raw_ostream &O, const Twine &Indent, void VPScalarIVStepsRecipe::execute(VPTransformState &State) { // Fast-math-flags propagate from the original induction instruction. - IRBuilder<>::FastMathFlagGuard FMFG(State.Builder); + IRBuilderBase::FastMathFlagGuard FMFG(State.Builder); if (hasFastMathFlags()) State.Builder.setFastMathFlags(getFastMathFlags()); @@ -3629,7 +3629,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0)); PHINode *NewPointerPhi = nullptr; if (CurrentPart == 0) { - IRBuilder<>::InsertPointGuard Guard(State.Builder); + IRBuilderBase::InsertPointGuard Guard(State.Builder); if (State.Builder.GetInsertPoint() != State.Builder.GetInsertBlock()->getFirstNonPHIIt()) State.Builder.SetInsertPoint( @@ -3733,7 +3733,9 @@ void VPExpandSCEVRecipe::print(raw_ostream &O, const Twine &Indent, void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) { Value *CanonicalIV = State.get(getOperand(0), /*IsScalar*/ true); Type *STy = CanonicalIV->getType(); - IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); + IRBuilderBase &Builder = State.Builder; + IRBuilderBase::InsertPointGuard Guard(Builder); + Builder.SetInsertPoint(State.CFG.PrevBB->getTerminator()); ElementCount VF = State.VF; Value *VStart = VF.isScalar() ? CanonicalIV @@ -3772,7 +3774,7 @@ void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) { if (State.VF.isVector()) { auto *IdxTy = Builder.getInt32Ty(); auto *One = ConstantInt::get(IdxTy, 1); - IRBuilder<>::InsertPointGuard Guard(Builder); + IRBuilderBase::InsertPointGuard Guard(Builder); Builder.SetInsertPoint(VectorPH->getTerminator()); auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF); auto *LastIdx = Builder.CreateSub(RuntimeVF, One);