From bbdc0bbd017a9a75f9b155df8950665d2e9cff95 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 15 Dec 2024 14:45:48 +0000 Subject: [PATCH 1/3] [VPlan] Get types and step from VPWidenPointerInductionRecipe (NFC). --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +++-- llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 022c0f5a173a6..1a891e5b79feb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3194,7 +3194,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { // A pointer induction, performed by using a gep BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint(); Value *ScalarStepValue = State.get(getOperand(1), VPLane(0)); - Type *PhiType = IndDesc.getStep()->getType(); + Type *PhiType = State.TypeAnalysis.inferScalarType(getOperand(1)); Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF); // Add induction update using an incorrect block temporarily. The phi node // will be fixed after VPlan execution. Note that at this point the latch @@ -3246,7 +3246,8 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent, printAsOperand(O, SlotTracker); O << " = WIDEN-POINTER-INDUCTION "; getStartValue()->printAsOperand(O, SlotTracker); - O << ", " << *IndDesc.getStep(); + O << ", "; + getOperand(1)->printAsOperand(O, SlotTracker); if (getNumOperands() == 4) { O << ", "; getOperand(2)->printAsOperand(O, SlotTracker); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll index 1ee6083eb59a5..de1500421a915 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -22,7 +22,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 +; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, ir<1> ; CHECK-NEXT: vp<[[PTR_IDX:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<8> ; CHECK-NEXT: vp<[[PTR_IDX_STEPS:%.+]]> = SCALAR-STEPS vp<[[PTR_IDX]]>, ir<8> ; CHECK-NEXT: EMIT vp<[[PTR_IV_1:%.+]]> = ptradd ir<%start.1>, vp<[[PTR_IDX_STEPS]]> From 89a1b3a915d8752c51be122e1ee1feedf5168722 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 15 Dec 2024 15:03:44 +0000 Subject: [PATCH 2/3] [VPlan] Add VPWidenInduction recipe as common base class (NFC). This helps to simplify some existing code and new code (https://github.com/llvm/llvm-project/pull/112145) --- .../Transforms/Vectorize/LoopVectorize.cpp | 12 +- llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlan.h | 110 ++++++++++-------- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 23 ++-- .../LoopVectorize/vplan-printing.ll | 4 +- 5 files changed, 78 insertions(+), 73 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index d37e826412004..e8b1201eb2742 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7941,10 +7941,8 @@ BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton( SmallPtrSet WideIVs; for (VPRecipeBase &H : EPI.EpiloguePlan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) { - if (auto *WideIV = dyn_cast(&H)) + if (auto *WideIV = dyn_cast(&H)) WideIVs.insert(WideIV->getPHINode()); - else if (auto *PtrIV = dyn_cast(&H)) - WideIVs.insert(cast(PtrIV->getUnderlyingValue())); } createInductionResumeVPValues(ExpandedSCEVs, nullptr, &WideIVs); @@ -10131,13 +10129,7 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, } else { // Retrieve the induction resume values for wide inductions from // their original phi nodes in the scalar loop. - PHINode *IndPhi = nullptr; - if (auto *Ind = dyn_cast(&R)) { - IndPhi = cast(Ind->getUnderlyingValue()); - } else { - auto *WidenInd = cast(&R); - IndPhi = WidenInd->getPHINode(); - } + PHINode *IndPhi = dyn_cast(&R)->getPHINode(); // Hook up to the PHINode generated by a ResumePhi recipe of main // loop VPlan, which feeds the scalar loop. ResumeV = IndPhi->getIncomingValueForBlock(L->getLoopPreheader()); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index d3476399dabf1..432030a7b1adf 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1047,7 +1047,7 @@ void VPlan::execute(VPTransformState *State) { if (isa(&R)) continue; - if (isa(&R)) { + if (isa(&R)) { PHINode *Phi = nullptr; if (isa(&R)) { Phi = cast(State->get(R.getVPSingleValue())); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index e8902009455ef..12208a7968338 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2099,38 +2099,81 @@ class VPHeaderPHIRecipe : public VPSingleDefRecipe { } }; +/// Base class for widened induction (VPWidenIntOrFpInductionRecipe and +/// VPWidenPointerInductionRecipe), providing shared functionality, including +/// retrieving the step value, induction descriptor and original phi node. +class VPWidenInductionRecipe : public VPHeaderPHIRecipe { + const InductionDescriptor &IndDesc; + +public: + VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start, + VPValue *Step, const InductionDescriptor &IndDesc, + DebugLoc DL) + : VPHeaderPHIRecipe(Kind, IV, Start, DL), IndDesc(IndDesc) { + addOperand(Step); + } + + static inline bool classof(const VPRecipeBase *R) { + return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC || + R->getVPDefID() == VPDef::VPWidenPointerInductionSC; + } + + virtual void execute(VPTransformState &State) override = 0; + + /// Returns the step value of the induction. + VPValue *getStepValue() { return getOperand(1); } + const VPValue *getStepValue() const { return getOperand(1); } + + PHINode *getPHINode() const { return cast(getUnderlyingValue()); } + + /// Returns the induction descriptor for the recipe. + const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } + + VPValue *getBackedgeValue() override { + // TODO: All operands of base recipe must exist and be at same index in + // derived recipe. + llvm_unreachable( + "VPWidenIntOrFpInductionRecipe generates its own backedge value"); + } + + VPRecipeBase &getBackedgeRecipe() override { + // TODO: All operands of base recipe must exist and be at same index in + // derived recipe. + llvm_unreachable( + "VPWidenIntOrFpInductionRecipe generates its own backedge value"); + } +}; + /// A recipe for handling phi nodes of integer and floating-point inductions, /// producing their vector values. -class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe { - PHINode *IV; +class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe { TruncInst *Trunc; - const InductionDescriptor &IndDesc; public: VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, VPValue *VF, const InductionDescriptor &IndDesc, DebugLoc DL) - : VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start, DL), - IV(IV), Trunc(nullptr), IndDesc(IndDesc) { - addOperand(Step); + : VPWidenInductionRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start, + Step, IndDesc, DL), + Trunc(nullptr) { addOperand(VF); } VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, VPValue *VF, const InductionDescriptor &IndDesc, TruncInst *Trunc, DebugLoc DL) - : VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionSC, Trunc, Start, DL), - IV(IV), Trunc(Trunc), IndDesc(IndDesc) { - addOperand(Step); + : VPWidenInductionRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start, + Step, IndDesc, DL), + Trunc(Trunc) { addOperand(VF); } ~VPWidenIntOrFpInductionRecipe() override = default; VPWidenIntOrFpInductionRecipe *clone() override { - return new VPWidenIntOrFpInductionRecipe(IV, getStartValue(), - getStepValue(), getVFValue(), - IndDesc, Trunc, getDebugLoc()); + return new VPWidenIntOrFpInductionRecipe( + getPHINode(), getStartValue(), getStepValue(), getVFValue(), + getInductionDescriptor(), Trunc, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC) @@ -2145,24 +2188,6 @@ class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe { VPSlotTracker &SlotTracker) const override; #endif - VPValue *getBackedgeValue() override { - // TODO: All operands of base recipe must exist and be at same index in - // derived recipe. - llvm_unreachable( - "VPWidenIntOrFpInductionRecipe generates its own backedge value"); - } - - VPRecipeBase &getBackedgeRecipe() override { - // TODO: All operands of base recipe must exist and be at same index in - // derived recipe. - llvm_unreachable( - "VPWidenIntOrFpInductionRecipe generates its own backedge value"); - } - - /// Returns the step value of the induction. - VPValue *getStepValue() { return getOperand(1); } - const VPValue *getStepValue() const { return getOperand(1); } - VPValue *getVFValue() { return getOperand(2); } const VPValue *getVFValue() const { return getOperand(2); } @@ -2177,11 +2202,6 @@ class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe { TruncInst *getTruncInst() { return Trunc; } const TruncInst *getTruncInst() const { return Trunc; } - PHINode *getPHINode() { return IV; } - - /// Returns the induction descriptor for the recipe. - const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } - /// Returns true if the induction is canonical, i.e. starting at 0 and /// incremented by UF * VF (= the original IV is incremented by 1) and has the /// same type as the canonical induction. @@ -2189,7 +2209,7 @@ class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe { /// Returns the scalar type of the induction. Type *getScalarType() const { - return Trunc ? Trunc->getType() : IV->getType(); + return Trunc ? Trunc->getType() : getPHINode()->getType(); } /// Returns the VPValue representing the value of this induction at @@ -2200,10 +2220,8 @@ class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe { } }; -class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe, +class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe, public VPUnrollPartAccessor<3> { - const InductionDescriptor &IndDesc; - bool IsScalarAfterVectorization; public: @@ -2212,19 +2230,16 @@ class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe, VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization, DebugLoc DL) - : VPHeaderPHIRecipe(VPDef::VPWidenPointerInductionSC, Phi, nullptr, DL), - IndDesc(IndDesc), - IsScalarAfterVectorization(IsScalarAfterVectorization) { - addOperand(Start); - addOperand(Step); - } + : VPWidenInductionRecipe(VPDef::VPWidenPointerInductionSC, Phi, Start, + Step, IndDesc, DL), + IsScalarAfterVectorization(IsScalarAfterVectorization) {} ~VPWidenPointerInductionRecipe() override = default; VPWidenPointerInductionRecipe *clone() override { return new VPWidenPointerInductionRecipe( cast(getUnderlyingInstr()), getOperand(0), getOperand(1), - IndDesc, IsScalarAfterVectorization, getDebugLoc()); + getInductionDescriptor(), IsScalarAfterVectorization, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC) @@ -2235,9 +2250,6 @@ class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe, /// Returns true if only scalar values will be generated. bool onlyScalarsGenerated(bool IsScalable); - /// Returns the induction descriptor for the recipe. - const InductionDescriptor &getInductionDescriptor() const { return IndDesc; } - /// Returns the VPValue representing the value of this induction at /// the first unrolled part, if it exists. Returns itself if unrolling did not /// take place. diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1a891e5b79feb..b830e104539fd 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1660,12 +1660,13 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { const InductionDescriptor &ID = getInductionDescriptor(); TruncInst *Trunc = getTruncInst(); IRBuilderBase &Builder = State.Builder; - assert(IV->getType() == ID.getStartValue()->getType() && "Types must match"); + assert(getPHINode()->getType() == ID.getStartValue()->getType() && + "Types must match"); assert(State.VF.isVector() && "must have vector VF"); // The value from the original loop to which we are mapping the new induction // variable. - Instruction *EntryVal = Trunc ? cast(Trunc) : IV; + Instruction *EntryVal = Trunc ? cast(Trunc) : getPHINode(); // Fast-math-flags propagate from the original induction instruction. IRBuilder<>::FastMathFlagGuard FMFG(Builder); @@ -1755,13 +1756,12 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { O << Indent << "WIDEN-INDUCTION"; - if (getTruncInst()) { + if (auto *TI = getTruncInst()) { O << "\\l\""; - O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\""; - O << " +\n" << Indent << "\" "; - getVPValue(0)->printAsOperand(O, SlotTracker); + O << " +\n" << Indent << "\" " << VPlanIngredient(getPHINode()) << "\\l\""; + O << " +\n" << Indent << "\" " << VPlanIngredient(TI); } else - O << " " << VPlanIngredient(IV); + O << " " << VPlanIngredient(getPHINode()); O << ", "; getStepValue()->printAsOperand(O, SlotTracker); @@ -3157,7 +3157,8 @@ bool VPWidenPointerInductionRecipe::onlyScalarsGenerated(bool IsScalable) { } void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { - assert(IndDesc.getKind() == InductionDescriptor::IK_PtrInduction && + assert(getInductionDescriptor().getKind() == + InductionDescriptor::IK_PtrInduction && "Not a pointer induction according to InductionDescriptor!"); assert(cast(getUnderlyingInstr())->getType()->isPointerTy() && "Unexpected type."); @@ -3193,8 +3194,8 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { // A pointer induction, performed by using a gep BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint(); - Value *ScalarStepValue = State.get(getOperand(1), VPLane(0)); - Type *PhiType = State.TypeAnalysis.inferScalarType(getOperand(1)); + Value *ScalarStepValue = State.get(getStepValue(), VPLane(0)); + Type *PhiType = State.TypeAnalysis.inferScalarType(getStepValue()); Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF); // Add induction update using an incorrect block temporarily. The phi node // will be fixed after VPlan execution. Note that at this point the latch @@ -3247,7 +3248,7 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent, O << " = WIDEN-POINTER-INDUCTION "; getStartValue()->printAsOperand(O, SlotTracker); O << ", "; - getOperand(1)->printAsOperand(O, SlotTracker); + getStepValue()->printAsOperand(O, SlotTracker); if (getNumOperands() == 4) { O << ", "; getOperand(2)->printAsOperand(O, SlotTracker); diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index c526c53dbea06..dccbed63a161b 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -655,10 +655,10 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION\l" + ; CHECK-NEXT: " %iv = phi %iv.next, 0\l" + -; CHECK-NEXT: " ir<%v2>, vp<[[EXP_SCEV]]>, vp<[[VF]]> +; CHECK-NEXT: " %v2 = trunc %iv, vp<[[EXP_SCEV]]>, vp<[[VF]]> ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * vp<[[EXP_SCEV]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, vp<[[EXP_SCEV]]> -; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%v2>, ir<1> +; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%iv>, ir<1> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> From 438067dc41b8eff3a42d33844dd3ee36c1bed842 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 15 Dec 2024 20:29:34 +0000 Subject: [PATCH 3/3] !fixup address comments, thanks --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index e8b1201eb2742..61f7bd8490281 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -10129,7 +10129,7 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, } else { // Retrieve the induction resume values for wide inductions from // their original phi nodes in the scalar loop. - PHINode *IndPhi = dyn_cast(&R)->getPHINode(); + PHINode *IndPhi = cast(&R)->getPHINode(); // Hook up to the PHINode generated by a ResumePhi recipe of main // loop VPlan, which feeds the scalar loop. ResumeV = IndPhi->getIncomingValueForBlock(L->getLoopPreheader());