From 324659a08ea1695b095bb780c8cd360d7fce0edb Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Wed, 4 Sep 2024 20:52:14 -0700 Subject: [PATCH 1/7] [VPlan] Implment VPReductionRecipe::computeCost(). NFC Implementation of `computeCost()` function for `VPReductionRecipe`. --- llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++++ .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 68a62638b9d58..6a61ef63c2a05 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2476,6 +2476,10 @@ class VPReductionRecipe : public VPSingleDefRecipe { /// Generate the reduction in the loop void execute(VPTransformState &State) override; + /// Return the cost of VPReductionRecipe. + InstructionCost computeCost(ElementCount VF, + VPCostContext &Ctx) const override; + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 2948ecc580edc..ea28a089812cb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2071,6 +2071,30 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) { State.set(this, NewRed, /*IsScalar*/ true); } +InstructionCost VPReductionRecipe::computeCost(ElementCount VF, + VPCostContext &Ctx) const { + RecurKind RdxKind = RdxDesc.getRecurrenceKind(); + Type *ElementTy = RdxDesc.getRecurrenceType(); + auto *VectorTy = dyn_cast(ToVectorTy(ElementTy, VF)); + TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; + unsigned Opcode = RdxDesc.getOpcode(); + + if (VectorTy == nullptr) + return InstructionCost::getInvalid(); + + // Cost = Reduction cost + BinOp cost + InstructionCost Cost = + Ctx.TTI.getArithmeticInstrCost(Opcode, ElementTy, CostKind); + if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind)) { + Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind); + return Cost + Ctx.TTI.getMinMaxReductionCost( + Id, VectorTy, RdxDesc.getFastMathFlags(), CostKind); + } + + return Cost + Ctx.TTI.getArithmeticReductionCost( + Opcode, VectorTy, RdxDesc.getFastMathFlags(), CostKind); +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPReductionRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { From 91fcf39b061742a04e18c5c948cad4bca51ca608 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Tue, 24 Sep 2024 23:31:24 -0700 Subject: [PATCH 2/7] Address comments. --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index ea28a089812cb..4bf5394b31773 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2075,13 +2075,10 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF, VPCostContext &Ctx) const { RecurKind RdxKind = RdxDesc.getRecurrenceKind(); Type *ElementTy = RdxDesc.getRecurrenceType(); - auto *VectorTy = dyn_cast(ToVectorTy(ElementTy, VF)); + auto *VectorTy = cast(ToVectorTy(ElementTy, VF)); TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; unsigned Opcode = RdxDesc.getOpcode(); - if (VectorTy == nullptr) - return InstructionCost::getInvalid(); - // Cost = Reduction cost + BinOp cost InstructionCost Cost = Ctx.TTI.getArithmeticInstrCost(Opcode, ElementTy, CostKind); From 4f2cc46e8d0af29c7db640619abe413f3103f64a Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Fri, 4 Oct 2024 03:19:36 -0700 Subject: [PATCH 3/7] Address comments. --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 4bf5394b31773..b5bc489891dca 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2074,7 +2074,14 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) { InstructionCost VPReductionRecipe::computeCost(ElementCount VF, VPCostContext &Ctx) const { RecurKind RdxKind = RdxDesc.getRecurrenceKind(); - Type *ElementTy = RdxDesc.getRecurrenceType(); + // TODO: Support any-of reduction and in-loop reduction + assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) && + "Not support any-of reduction in VPlan-based cost model currently."); + + Type *ElementTy = Ctx.Types.inferScalarType(this->getVPSingleValue()); + assert(ElementTy->getTypeID() == RdxDesc.getRecurrenceType()->getTypeID() && + "Infered type and recurrence type mismatch."); + auto *VectorTy = cast(ToVectorTy(ElementTy, VF)); TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; unsigned Opcode = RdxDesc.getOpcode(); From 28b82ca60b243b78117c1e798424bf77891c1d32 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Tue, 8 Oct 2024 20:03:47 -0700 Subject: [PATCH 4/7] Address comments and add `isInLoopReduction()`. --- .../Transforms/Vectorize/LoopVectorize.cpp | 8 +++++++ llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++++ .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 22 ++++++++++++------- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 027ee21527d22..acc3c4290e8c4 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7137,6 +7137,14 @@ bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const { SkipCostComputation.contains(UI); } +bool VPCostContext::isInLoopReduction(const Instruction *UI, ElementCount VF, + Type *VectorTy) const { + return CM + .getReductionPatternCost(const_cast(UI), VF, VectorTy, + TTI::TCK_RecipThroughput) + .has_value(); +} + InstructionCost LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF, VPCostContext &CostCtx) const { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 6a61ef63c2a05..dc3beafd8dabe 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -718,6 +718,10 @@ struct VPCostContext { /// Return true if the cost for \p UI shouldn't be computed, e.g. because it /// has already been pre-computed. bool skipCostComputation(Instruction *UI, bool IsVector) const; + + /// Return true if the \p UI is part of the in-loop reduction. + bool isInLoopReduction(const Instruction *UI, ElementCount VF, + Type *VectorTy) const; }; /// VPRecipeBase is a base class modeling a sequence of one or more output IR diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index b5bc489891dca..b8612927e6ff4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2074,18 +2074,24 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) { InstructionCost VPReductionRecipe::computeCost(ElementCount VF, VPCostContext &Ctx) const { RecurKind RdxKind = RdxDesc.getRecurrenceKind(); - // TODO: Support any-of reduction and in-loop reduction - assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) && - "Not support any-of reduction in VPlan-based cost model currently."); - - Type *ElementTy = Ctx.Types.inferScalarType(this->getVPSingleValue()); - assert(ElementTy->getTypeID() == RdxDesc.getRecurrenceType()->getTypeID() && - "Infered type and recurrence type mismatch."); - + Type *ElementTy = Ctx.Types.inferScalarType(this); auto *VectorTy = cast(ToVectorTy(ElementTy, VF)); TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; unsigned Opcode = RdxDesc.getOpcode(); + // TODO: Support any-of reduction and in-loop reductions. + assert( + (!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) || + ForceTargetInstructionCost.getNumOccurrences() > 0) && + "Any-of reduction not implemented in VPlan-based cost model currently."); + assert( + (!Ctx.isInLoopReduction(getUnderlyingInstr(), VF, VectorTy) || + ForceTargetInstructionCost.getNumOccurrences() > 0) && + "In-loop reduction not implemented in VPlan-based cost model currently."); + + assert(ElementTy->getTypeID() == RdxDesc.getRecurrenceType()->getTypeID() && + "Infered type and recurrence type mismatch."); + // Cost = Reduction cost + BinOp cost InstructionCost Cost = Ctx.TTI.getArithmeticInstrCost(Opcode, ElementTy, CostKind); From 041f41b153327090c31ca97ba263c3c3100475e6 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Wed, 9 Oct 2024 15:33:08 -0700 Subject: [PATCH 5/7] Fixup! Remove legacy model query --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 -------- llvm/lib/Transforms/Vectorize/VPlan.h | 4 ---- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +++-- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index acc3c4290e8c4..027ee21527d22 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7137,14 +7137,6 @@ bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const { SkipCostComputation.contains(UI); } -bool VPCostContext::isInLoopReduction(const Instruction *UI, ElementCount VF, - Type *VectorTy) const { - return CM - .getReductionPatternCost(const_cast(UI), VF, VectorTy, - TTI::TCK_RecipThroughput) - .has_value(); -} - InstructionCost LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF, VPCostContext &CostCtx) const { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index dc3beafd8dabe..6a61ef63c2a05 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -718,10 +718,6 @@ struct VPCostContext { /// Return true if the cost for \p UI shouldn't be computed, e.g. because it /// has already been pre-computed. bool skipCostComputation(Instruction *UI, bool IsVector) const; - - /// Return true if the \p UI is part of the in-loop reduction. - bool isInLoopReduction(const Instruction *UI, ElementCount VF, - Type *VectorTy) const; }; /// VPRecipeBase is a base class modeling a sequence of one or more output IR diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index b8612927e6ff4..6ade6a74811a3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2079,13 +2079,14 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; unsigned Opcode = RdxDesc.getOpcode(); - // TODO: Support any-of reduction and in-loop reductions. + // TODO: Support any-of and in-loop reductions. assert( (!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) || ForceTargetInstructionCost.getNumOccurrences() > 0) && "Any-of reduction not implemented in VPlan-based cost model currently."); + auto *ReductionPHIRecipe = dyn_cast(getOperand(0)); assert( - (!Ctx.isInLoopReduction(getUnderlyingInstr(), VF, VectorTy) || + (ReductionPHIRecipe && !ReductionPHIRecipe->isInLoop() || ForceTargetInstructionCost.getNumOccurrences() > 0) && "In-loop reduction not implemented in VPlan-based cost model currently."); From 7acd294a3c2d4ca69698d8ff78d810fe6da0fdac Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Sun, 13 Oct 2024 19:26:00 -0700 Subject: [PATCH 6/7] Fixup! Using cast to avoid warning when compiling without assertion. --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 6ade6a74811a3..966dae5bf45ea 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2084,9 +2084,8 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF, (!RecurrenceDescriptor::isAnyOfRecurrenceKind(RdxKind) || ForceTargetInstructionCost.getNumOccurrences() > 0) && "Any-of reduction not implemented in VPlan-based cost model currently."); - auto *ReductionPHIRecipe = dyn_cast(getOperand(0)); assert( - (ReductionPHIRecipe && !ReductionPHIRecipe->isInLoop() || + (!cast(getOperand(0))->isInLoop() || ForceTargetInstructionCost.getNumOccurrences() > 0) && "In-loop reduction not implemented in VPlan-based cost model currently."); From 80faae5f2ef98756e0f5e2b13acfdee2201d3a87 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Mon, 14 Oct 2024 18:14:18 -0700 Subject: [PATCH 7/7] Fixup! typo --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 966dae5bf45ea..368d6e58a5578 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2090,7 +2090,7 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF, "In-loop reduction not implemented in VPlan-based cost model currently."); assert(ElementTy->getTypeID() == RdxDesc.getRecurrenceType()->getTypeID() && - "Infered type and recurrence type mismatch."); + "Inferred type and recurrence type mismatch."); // Cost = Reduction cost + BinOp cost InstructionCost Cost =