Skip to content

Commit 7c436bf

Browse files
SamTebbs33SamTebbs33
committed
Make VFScaleFactor an unsigned instead of ElementCount
1 parent d5caa62 commit 7c436bf

File tree

6 files changed

+31
-40
lines changed

6 files changed

+31
-40
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8281,10 +8281,10 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
82818281
// If the PHI is used by a partial reduction, set the scale factor.
82828282
bool UseInLoopReduction = CM.isInLoopReduction(Phi);
82838283
bool UseOrderedReductions = CM.useOrderedReductions(RdxDesc);
8284-
auto ScaleFactor = ElementCount::getFixed(
8284+
auto ScaleFactor =
82858285
(UseOrderedReductions || UseInLoopReduction)
82868286
? 0
8287-
: getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1));
8287+
: getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
82888288
PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV,
82898289
CM.isInLoopReduction(Phi),
82908290
UseOrderedReductions, ScaleFactor);
@@ -8320,8 +8320,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
83208320
return tryToWidenMemory(Instr, Operands, Range);
83218321

83228322
if (std::optional<unsigned> ScaleFactor = getScalingForReduction(Instr))
8323-
return tryToCreatePartialReduction(
8324-
Instr, Operands, ElementCount::getFixed(ScaleFactor.value()));
8323+
return tryToCreatePartialReduction(Instr, Operands, ScaleFactor.value());
83258324

83268325
if (!shouldWiden(Instr, Range))
83278326
return nullptr;
@@ -8344,7 +8343,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
83448343
VPRecipeBase *
83458344
VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
83468345
ArrayRef<VPValue *> Operands,
8347-
ElementCount ScaleFactor) {
8346+
unsigned ScaleFactor) {
83488347
assert(Operands.size() == 2 &&
83498348
"Unexpected number of operands for partial reduction");
83508349

@@ -9145,8 +9144,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
91459144
? RdxDesc.getFastMathFlags()
91469145
: FastMathFlags();
91479146
bool UseOrderedReductions = CM.useOrderedReductions(RdxDesc);
9148-
ElementCount VFScaleFactor =
9149-
ElementCount::getFixed(!UseOrderedReductions);
9147+
unsigned VFScaleFactor = !UseOrderedReductions;
91509148
auto *RedRecipe = new VPReductionRecipe(
91519149
Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
91529150
UseOrderedReductions, VFScaleFactor, CurrentLinkI->getDebugLoc());

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ class VPRecipeBuilder {
172172
/// along with binary operation and reduction phi operands.
173173
VPRecipeBase *tryToCreatePartialReduction(Instruction *Reduction,
174174
ArrayRef<VPValue *> Operands,
175-
ElementCount ScaleFactor);
175+
unsigned ScaleFactor);
176176

177177
/// Set the recipe created for given ingredient.
178178
void setRecipe(Instruction *I, VPRecipeBase *R) {

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2188,20 +2188,19 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
21882188

21892189
/// When expanding the reduction PHI, the plan's VF element count is divided
21902190
/// by this factor to form the reduction phi's VF.
2191-
ElementCount VFScaleFactor;
2191+
unsigned VFScaleFactor;
21922192

21932193
public:
21942194
/// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
21952195
/// RdxDesc.
21962196
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc,
21972197
VPValue &Start, bool IsInLoop = false,
2198-
bool IsOrdered = false,
2199-
ElementCount VFScaleFactor = ElementCount::getFixed(1))
2198+
bool IsOrdered = false, unsigned VFScaleFactor = 1)
22002199
: VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
22012200
RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered),
22022201
VFScaleFactor(VFScaleFactor) {
22032202
assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
2204-
assert(((!IsInLoop && !IsOrdered) || VFScaleFactor.isZero()) &&
2203+
assert(((!IsInLoop && !IsOrdered) || VFScaleFactor == 0) &&
22052204
"Invalid VFScaleFactor");
22062205
}
22072206

@@ -2221,7 +2220,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
22212220
void execute(VPTransformState &State) override;
22222221

22232222
/// Get the factor that the VF of this recipe's output should be scaled by.
2224-
ElementCount getVFScaleFactor() const { return VFScaleFactor; }
2223+
unsigned getVFScaleFactor() const { return VFScaleFactor; }
22252224

22262225
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
22272226
/// Print the recipe.
@@ -2239,9 +2238,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
22392238
/// Returns true, if the phi is part of an in-loop reduction.
22402239
bool isInLoop() const { return IsInLoop; }
22412240

2242-
bool isPartialReduction() const {
2243-
return ElementCount::isKnownGT(VFScaleFactor, ElementCount::getFixed(1));
2244-
}
2241+
bool isPartialReduction() const { return VFScaleFactor > 1; }
22452242

22462243
/// Returns true if the recipe only uses the first lane of operand \p Op.
22472244
bool onlyFirstLaneUsed(const VPValue *Op) const override {
@@ -2430,16 +2427,16 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
24302427
/// For in-loop reductions this is equal to 0, to specify that this is equal
24312428
/// to the VF (which may not be known yet). For partial-reductions this is
24322429
/// equal to another scalar value.
2433-
ElementCount VFScaleFactor;
2430+
unsigned VFScaleFactor;
24342431

24352432
protected:
24362433
VPReductionRecipe(const unsigned char SC, RecurKind RdxKind,
24372434
FastMathFlags FMFs, Instruction *I,
24382435
ArrayRef<VPValue *> Operands, VPValue *CondOp,
2439-
bool IsOrdered, ElementCount VFScaleFactor, DebugLoc DL)
2436+
bool IsOrdered, unsigned VFScaleFactor, DebugLoc DL)
24402437
: VPRecipeWithIRFlags(SC, Operands, FMFs, DL), RdxKind(RdxKind),
24412438
IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
2442-
assert((!IsOrdered || VFScaleFactor.isZero()) && "Invalid scale factor");
2439+
assert((!IsOrdered || VFScaleFactor == 0) && "Invalid scale factor");
24432440
if (CondOp) {
24442441
IsConditional = true;
24452442
addOperand(CondOp);
@@ -2451,11 +2448,11 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
24512448
/// Note that the debug location is from the extend.
24522449
VPReductionRecipe(const unsigned char SC, const RecurKind RdxKind,
24532450
ArrayRef<VPValue *> Operands, VPValue *CondOp,
2454-
bool IsOrdered, ElementCount VFScaleFactor, DebugLoc DL)
2451+
bool IsOrdered, unsigned VFScaleFactor, DebugLoc DL)
24552452
: VPRecipeWithIRFlags(SC, Operands, DL), RdxKind(RdxKind),
24562453
IsOrdered(IsOrdered), IsConditional(CondOp),
24572454
VFScaleFactor(VFScaleFactor) {
2458-
assert((!IsOrdered || VFScaleFactor.isZero()) && "Invalid scale factor");
2455+
assert((!IsOrdered || VFScaleFactor == 0) && "Invalid scale factor");
24592456
if (CondOp)
24602457
addOperand(CondOp);
24612458
}
@@ -2464,29 +2461,27 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
24642461
/// Note that the NUW/NSW flags and the debug location are from the Mul.
24652462
VPReductionRecipe(const unsigned char SC, const RecurKind RdxKind,
24662463
ArrayRef<VPValue *> Operands, VPValue *CondOp,
2467-
bool IsOrdered, ElementCount VFScaleFactor,
2464+
bool IsOrdered, unsigned VFScaleFactor,
24682465
WrapFlagsTy WrapFlags, DebugLoc DL)
24692466
: VPRecipeWithIRFlags(SC, Operands, WrapFlags, DL), RdxKind(RdxKind),
24702467
IsOrdered(IsOrdered), IsConditional(CondOp),
24712468
VFScaleFactor(VFScaleFactor) {
2472-
assert((!IsOrdered || VFScaleFactor.isZero()) && "Invalid scale factor");
2469+
assert((!IsOrdered || VFScaleFactor == 0) && "Invalid scale factor");
24732470
if (CondOp)
24742471
addOperand(CondOp);
24752472
}
24762473

24772474
public:
24782475
VPReductionRecipe(RecurKind RdxKind, FastMathFlags FMFs, Instruction *I,
24792476
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
2480-
bool IsOrdered, ElementCount VFScaleFactor,
2481-
DebugLoc DL = {})
2477+
bool IsOrdered, unsigned VFScaleFactor, DebugLoc DL = {})
24822478
: VPReductionRecipe(VPDef::VPReductionSC, RdxKind, FMFs, I,
24832479
ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
24842480
IsOrdered, VFScaleFactor, DL) {}
24852481

24862482
VPReductionRecipe(const RecurKind RdxKind, FastMathFlags FMFs,
24872483
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
2488-
bool IsOrdered, ElementCount VFScaleFactor,
2489-
DebugLoc DL = {})
2484+
bool IsOrdered, unsigned VFScaleFactor, DebugLoc DL = {})
24902485
: VPReductionRecipe(VPDef::VPReductionSC, RdxKind, FMFs, nullptr,
24912486
ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
24922487
IsOrdered, VFScaleFactor, DL) {}
@@ -2531,9 +2526,7 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
25312526
/// Return true if the in-loop reduction is conditional.
25322527
bool isConditional() const { return IsConditional; };
25332528
/// Return true if the reduction is a partial reduction.
2534-
bool isPartialReduction() const {
2535-
return ElementCount::isKnownGT(VFScaleFactor, ElementCount::getFixed(1));
2536-
}
2529+
bool isPartialReduction() const { return VFScaleFactor > 1; }
25372530
/// The VPValue of the scalar Chain being accumulated.
25382531
VPValue *getChainOp() const { return getOperand(0); }
25392532
/// The VPValue of the vector value to be reduced.
@@ -2543,7 +2536,7 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
25432536
return isConditional() ? getOperand(getNumOperands() - 1) : nullptr;
25442537
}
25452538
/// Get the factor that the VF of this recipe's output should be scaled by.
2546-
ElementCount getVFScaleFactor() const { return VFScaleFactor; }
2539+
unsigned getVFScaleFactor() const { return VFScaleFactor; }
25472540
};
25482541

25492542
/// A recipe to represent inloop reduction operations with vector-predication
@@ -2559,7 +2552,7 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
25592552
R.getFastMathFlags(),
25602553
cast_or_null<Instruction>(R.getUnderlyingValue()),
25612554
ArrayRef<VPValue *>({R.getChainOp(), R.getVecOp(), &EVL}), CondOp,
2562-
R.isOrdered(), ElementCount::getFixed(0), DL) {}
2555+
R.isOrdered(), 0, DL) {}
25632556

25642557
~VPReductionEVLRecipe() override = default;
25652558

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -393,9 +393,9 @@ bool VPDominatorTree::properlyDominates(const VPRecipeBase *A,
393393
/// one.
394394
static unsigned getVFScaleFactor(VPRecipeBase *R) {
395395
if (auto *RR = dyn_cast<VPReductionPHIRecipe>(R))
396-
return RR->getVFScaleFactor().getFixedValue();
396+
return RR->getVFScaleFactor();
397397
if (auto *RR = dyn_cast<VPReductionRecipe>(R))
398-
return RR->getVFScaleFactor().getFixedValue();
398+
return RR->getVFScaleFactor();
399399
assert(
400400
(!isa<VPInstruction>(R) || cast<VPInstruction>(R)->getOpcode() !=
401401
VPInstruction::ReductionStartVector) &&

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3836,8 +3836,8 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
38363836
printAsOperand(O, SlotTracker);
38373837
O << " = phi ";
38383838
printOperands(O, SlotTracker);
3839-
if (VFScaleFactor != ElementCount::getFixed(1))
3840-
O << " (VF scaled by 1/" << VFScaleFactor.getFixedValue() << ")";
3839+
if (VFScaleFactor != 1)
3840+
O << " (VF scaled by 1/" << VFScaleFactor << ")";
38413841
}
38423842
#endif
38433843

‎llvm/unittests/Transforms/Vectorize/VPlanTest.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1214,7 +1214,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
12141214
VPValue *VecOp = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
12151215
VPValue *CondOp = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 3));
12161216
VPReductionRecipe Recipe(RecurKind::Add, FastMathFlags(), Add, ChainOp,
1217-
CondOp, VecOp, false, ElementCount::getFixed(1));
1217+
CondOp, VecOp, false, 1);
12181218
EXPECT_FALSE(Recipe.mayHaveSideEffects());
12191219
EXPECT_FALSE(Recipe.mayReadFromMemory());
12201220
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1229,7 +1229,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
12291229
VPValue *VecOp = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
12301230
VPValue *CondOp = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 3));
12311231
VPReductionRecipe Recipe(RecurKind::Add, FastMathFlags(), Add, ChainOp,
1232-
CondOp, VecOp, false, ElementCount::getFixed(1));
1232+
CondOp, VecOp, false, 1);
12331233
VPValue *EVL = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 4));
12341234
VPReductionEVLRecipe EVLRecipe(Recipe, *EVL, CondOp);
12351235
EXPECT_FALSE(EVLRecipe.mayHaveSideEffects());
@@ -1586,7 +1586,7 @@ TEST_F(VPRecipeTest, CastVPReductionRecipeToVPUser) {
15861586
VPValue *VecOp = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 2));
15871587
VPValue *CondOp = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 3));
15881588
VPReductionRecipe Recipe(RecurKind::Add, FastMathFlags(), Add, ChainOp,
1589-
CondOp, VecOp, false, ElementCount::getFixed(1));
1589+
CondOp, VecOp, false, 1);
15901590
EXPECT_TRUE(isa<VPUser>(&Recipe));
15911591
VPRecipeBase *BaseR = &Recipe;
15921592
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -1601,7 +1601,7 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
16011601
VPValue *VecOp = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 2));
16021602
VPValue *CondOp = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 3));
16031603
VPReductionRecipe Recipe(RecurKind::Add, FastMathFlags(), Add, ChainOp,
1604-
CondOp, VecOp, false, ElementCount::getFixed(1));
1604+
CondOp, VecOp, false, 1);
16051605
VPValue *EVL = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 0));
16061606
VPReductionEVLRecipe EVLRecipe(Recipe, *EVL, CondOp);
16071607
EXPECT_TRUE(isa<VPUser>(&EVLRecipe));

0 commit comments

Comments
 (0)