@@ -2056,55 +2056,6 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
20562056 }
20572057};
20582058
2059- // / A recipe for forming partial reductions. In the loop, an accumulator and
2060- // / vector operand are added together and passed to the next iteration as the
2061- // / next accumulator. After the loop body, the accumulator is reduced to a
2062- // / scalar value.
2063- class VPPartialReductionRecipe : public VPSingleDefRecipe {
2064- unsigned Opcode;
2065-
2066- public:
2067- VPPartialReductionRecipe (Instruction *ReductionInst, VPValue *Op0,
2068- VPValue *Op1)
2069- : VPPartialReductionRecipe(ReductionInst->getOpcode (), Op0, Op1,
2070- ReductionInst) {}
2071- VPPartialReductionRecipe (unsigned Opcode, VPValue *Op0, VPValue *Op1,
2072- Instruction *ReductionInst = nullptr )
2073- : VPSingleDefRecipe(VPDef::VPPartialReductionSC,
2074- ArrayRef<VPValue *>({Op0, Op1}), ReductionInst),
2075- Opcode(Opcode) {
2076- [[maybe_unused]] auto *AccumulatorRecipe =
2077- getOperand (1 )->getDefiningRecipe ();
2078- assert ((isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
2079- isa<VPPartialReductionRecipe>(AccumulatorRecipe)) &&
2080- " Unexpected operand order for partial reduction recipe" );
2081- }
2082- ~VPPartialReductionRecipe () override = default ;
2083-
2084- VPPartialReductionRecipe *clone () override {
2085- return new VPPartialReductionRecipe (Opcode, getOperand (0 ), getOperand (1 ),
2086- getUnderlyingInstr ());
2087- }
2088-
2089- VP_CLASSOF_IMPL (VPDef::VPPartialReductionSC)
2090-
2091- // / Generate the reduction in the loop.
2092- void execute(VPTransformState &State) override ;
2093-
2094- // / Return the cost of this VPPartialReductionRecipe.
2095- InstructionCost computeCost (ElementCount VF,
2096- VPCostContext &Ctx) const override ;
2097-
2098- // / Get the binary op's opcode.
2099- unsigned getOpcode () const { return Opcode; }
2100-
2101- #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2102- // / Print the recipe.
2103- void print (raw_ostream &O, const Twine &Indent,
2104- VPSlotTracker &SlotTracker) const override ;
2105- #endif
2106- };
2107-
21082059// / A recipe for vectorizing a phi-node as a sequence of mask-based select
21092060// / instructions.
21102061class VPBlendRecipe : public VPSingleDefRecipe {
@@ -2376,6 +2327,56 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
23762327 }
23772328};
23782329
2330+ // / A recipe for forming partial reductions. In the loop, an accumulator and
2331+ // / vector operand are added together and passed to the next iteration as the
2332+ // / next accumulator. After the loop body, the accumulator is reduced to a
2333+ // / scalar value.
2334+ class VPPartialReductionRecipe : public VPSingleDefRecipe {
2335+ unsigned Opcode;
2336+
2337+ public:
2338+ VPPartialReductionRecipe (Instruction *ReductionInst, VPValue *Op0,
2339+ VPValue *Op1)
2340+ : VPPartialReductionRecipe(ReductionInst->getOpcode (), Op0, Op1,
2341+ ReductionInst) {}
2342+ VPPartialReductionRecipe (unsigned Opcode, VPValue *Op0, VPValue *Op1,
2343+ Instruction *ReductionInst = nullptr )
2344+ : VPSingleDefRecipe(VPDef::VPPartialReductionSC,
2345+ ArrayRef<VPValue *>({Op0, Op1}), ReductionInst),
2346+ Opcode(Opcode) {
2347+ [[maybe_unused]] auto *AccumulatorRecipe =
2348+ getOperand (1 )->getDefiningRecipe ();
2349+ assert ((isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
2350+ isa<VPPartialReductionRecipe>(AccumulatorRecipe)) &&
2351+ " Unexpected operand order for partial reduction recipe" );
2352+ }
2353+ ~VPPartialReductionRecipe () override = default ;
2354+
2355+ VPPartialReductionRecipe *clone () override {
2356+ return new VPPartialReductionRecipe (Opcode, getOperand (0 ), getOperand (1 ),
2357+ getUnderlyingInstr ());
2358+ }
2359+
2360+ VP_CLASSOF_IMPL (VPDef::VPPartialReductionSC)
2361+
2362+ // / Generate the reduction in the loop.
2363+ void execute(VPTransformState &State) override ;
2364+
2365+ // / Return the cost of this VPPartialReductionRecipe.
2366+ InstructionCost computeCost (ElementCount VF,
2367+ VPCostContext &Ctx) const override ;
2368+
2369+ // / Get the binary op's opcode.
2370+ unsigned getOpcode () const { return Opcode; }
2371+
2372+ #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2373+ // / Print the recipe.
2374+ void print (raw_ostream &O, const Twine &Indent,
2375+ VPSlotTracker &SlotTracker) const override ;
2376+ #endif
2377+ };
2378+
2379+
23792380// / A recipe to represent inloop reduction operations with vector-predication
23802381// / intrinsics, performing a reduction on a vector operand with the explicit
23812382// / vector length (EVL) into a scalar value, and adding the result to a chain.
0 commit comments