Skip to content

Commit b05ac80

Browse files
fhahnfhahn
committed
[VPlan] Use VPInstruction for VPScalarPHIRecipe. (NFCI)
Now that all phi nodes manage their incoming blocks through the VPlan-predecessors, there should be no need for having a dedicate recipe, it should be sufficient to allow PHI opcodes in VPInstruction. Follow-ups will also migrate VPWidenPHIRecipe and possibly others, building on top of #129388.
1 parent f9a6ea4 commit b05ac80

File tree

10 files changed

+63
-88
lines changed

10 files changed

+63
-88
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1074,9 +1074,15 @@ void VPlan::execute(VPTransformState *State) {
10741074
Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
10751075
continue;
10761076
}
1077+
if (auto *VPI = dyn_cast<VPInstruction>(&R)) {
1078+
Value *Phi = State->get(VPI, true);
1079+
Value *Val = State->get(VPI->getOperand(1), true);
1080+
cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
1081+
continue;
1082+
}
10771083

10781084
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
1079-
bool NeedsScalar = isa<VPScalarPHIRecipe>(PhiR) ||
1085+
bool NeedsScalar =
10801086
(isa<VPReductionPHIRecipe>(PhiR) &&
10811087
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
10821088
Value *Phi = State->get(PhiR, NeedsScalar);

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 8 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -441,9 +441,7 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
441441
bool mayHaveSideEffects() const;
442442

443443
/// Returns true for PHI-like recipes.
444-
bool isPhi() const {
445-
return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
446-
}
444+
bool isPhi() const;
447445

448446
/// Returns true if the recipe may read from memory.
449447
bool mayReadFromMemory() const;
@@ -1887,45 +1885,6 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe,
18871885
#endif
18881886
};
18891887

1890-
/// Recipe to generate a scalar PHI. Used to generate code for recipes that
1891-
/// produce scalar header phis, including VPCanonicalIVPHIRecipe and
1892-
/// VPEVLBasedIVPHIRecipe.
1893-
class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
1894-
std::string Name;
1895-
1896-
public:
1897-
VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
1898-
StringRef Name)
1899-
: VPHeaderPHIRecipe(VPDef::VPScalarPHISC, nullptr, Start, DL),
1900-
Name(Name.str()) {
1901-
addOperand(BackedgeValue);
1902-
}
1903-
1904-
~VPScalarPHIRecipe() override = default;
1905-
1906-
VPScalarPHIRecipe *clone() override {
1907-
llvm_unreachable("cloning not implemented yet");
1908-
}
1909-
1910-
VP_CLASSOF_IMPL(VPDef::VPScalarPHISC)
1911-
1912-
/// Generate the phi/select nodes.
1913-
void execute(VPTransformState &State) override;
1914-
1915-
/// Returns true if the recipe only uses the first lane of operand \p Op.
1916-
bool onlyFirstLaneUsed(const VPValue *Op) const override {
1917-
assert(is_contained(operands(), Op) &&
1918-
"Op must be an operand of the recipe");
1919-
return true;
1920-
}
1921-
1922-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1923-
/// Print the recipe.
1924-
void print(raw_ostream &O, const Twine &Indent,
1925-
VPSlotTracker &SlotTracker) const override;
1926-
#endif
1927-
};
1928-
19291888
/// A recipe for widened phis. Incoming values are operands of the recipe and
19301889
/// their operand index corresponds to the incoming predecessor block. If the
19311890
/// recipe is placed in an entry block to a (non-replicate) region, it must have
@@ -3004,6 +2963,13 @@ class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
30042963
return 0;
30052964
}
30062965

2966+
/// Returns true if the recipe only uses the first lane of operand \p Op.
2967+
bool onlyFirstLaneUsed(const VPValue *Op) const override {
2968+
assert(is_contained(operands(), Op) &&
2969+
"Op must be an operand of the recipe");
2970+
return true;
2971+
}
2972+
30072973
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
30082974
/// Print the recipe.
30092975
void print(raw_ostream &O, const Twine &Indent,

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
7171
}
7272
case VPInstruction::ExplicitVectorLength:
7373
return Type::getIntNTy(Ctx, 32);
74+
case Instruction::PHI:
75+
// Avoid inferring the type for other operands, as this may lead to infinite
76+
// recursions for cycles.
77+
return inferScalarType(R->getOperand(0));
7478
case VPInstruction::FirstOrderRecurrenceSplice:
7579
case VPInstruction::Not:
7680
case VPInstruction::ResumePhi:
@@ -236,14 +240,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
236240
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
237241
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
238242
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
239-
VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe,
240-
VPScalarPHIRecipe>([this](const auto *R) {
241-
// Handle header phi recipes, except VPWidenIntOrFpInduction
242-
// which needs special handling due it being possibly truncated.
243-
// TODO: consider inferring/caching type of siblings, e.g.,
244-
// backedge value, here and in cases below.
245-
return inferScalarType(R->getStartValue());
246-
})
243+
VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
244+
[this](const auto *R) {
245+
// Handle header phi recipes, except VPWidenIntOrFpInduction
246+
// which needs special handling due it being possibly truncated.
247+
// TODO: consider inferring/caching type of siblings, e.g.,
248+
// backedge value, here and in cases below.
249+
return inferScalarType(R->getStartValue());
250+
})
247251
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
248252
[](const auto *R) { return R->getScalarType(); })
249253
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 21 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,11 @@ InstructionCost VPRecipeBase::computeCost(ElementCount VF,
277277
VPCostContext &Ctx) const {
278278
llvm_unreachable("subclasses should implement computeCost");
279279
}
280+
bool VPRecipeBase::isPhi() const {
281+
return (getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC) ||
282+
(isa<VPInstruction>(this) &&
283+
cast<VPInstruction>(this)->getOpcode() == Instruction::PHI);
284+
}
280285

281286
InstructionCost
282287
VPPartialReductionRecipe::computeCost(ElementCount VF,
@@ -418,6 +423,7 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
418423
if (isSingleScalar() || isVectorToScalar())
419424
return true;
420425
switch (Opcode) {
426+
case Instruction::PHI:
421427
case Instruction::ICmp:
422428
case Instruction::Select:
423429
case VPInstruction::BranchOnCond:
@@ -458,6 +464,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
458464
}
459465

460466
switch (getOpcode()) {
467+
case Instruction::PHI: {
468+
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
469+
Value *Start = State.get(getOperand(0), VPLane(0));
470+
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
471+
Phi->addIncoming(Start, VectorPH);
472+
return Phi;
473+
}
461474
case VPInstruction::Not: {
462475
Value *A = State.get(getOperand(0));
463476
return Builder.CreateNot(A, Name);
@@ -838,6 +851,8 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
838851
switch (getOpcode()) {
839852
default:
840853
return false;
854+
case Instruction::PHI:
855+
return true;
841856
case Instruction::ICmp:
842857
case Instruction::Select:
843858
case Instruction::Or:
@@ -3283,11 +3298,12 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
32833298
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
32843299
PHINode *NewPointerPhi = nullptr;
32853300
if (CurrentPart == 0) {
3286-
auto *IVR = cast<VPHeaderPHIRecipe>(&getParent()
3287-
->getPlan()
3288-
->getVectorLoopRegion()
3289-
->getEntryBasicBlock()
3290-
->front());
3301+
auto *IVR = getParent()
3302+
->getPlan()
3303+
->getVectorLoopRegion()
3304+
->getEntryBasicBlock()
3305+
->front()
3306+
.getVPSingleValue();
32913307
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
32923308
NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",
32933309
CanonicalIV->getIterator());
@@ -3666,22 +3682,3 @@ void VPEVLBasedIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
36663682
printOperands(O, SlotTracker);
36673683
}
36683684
#endif
3669-
3670-
void VPScalarPHIRecipe::execute(VPTransformState &State) {
3671-
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
3672-
Value *Start = State.get(getStartValue(), VPLane(0));
3673-
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
3674-
Phi->addIncoming(Start, VectorPH);
3675-
Phi->setDebugLoc(getDebugLoc());
3676-
State.set(this, Phi, /*IsScalar=*/true);
3677-
}
3678-
3679-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3680-
void VPScalarPHIRecipe::print(raw_ostream &O, const Twine &Indent,
3681-
VPSlotTracker &SlotTracker) const {
3682-
O << Indent << "SCALAR-PHI ";
3683-
printAsOperand(O, SlotTracker);
3684-
O << " = phi ";
3685-
printOperands(O, SlotTracker);
3686-
}
3687-
#endif

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1738,7 +1738,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
17381738

17391739
// Create a scalar phi to track the previous EVL if fixed-order recurrence is
17401740
// contained.
1741-
VPScalarPHIRecipe *PrevEVL = nullptr;
1741+
VPInstruction *PrevEVL = nullptr;
17421742
bool ContainsFORs =
17431743
any_of(Header->phis(), IsaPred<VPFirstOrderRecurrencePHIRecipe>);
17441744
if (ContainsFORs) {
@@ -1753,7 +1753,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
17531753
VFSize > 32 ? Instruction::Trunc : Instruction::ZExt, MaxEVL,
17541754
Type::getInt32Ty(Ctx), DebugLoc());
17551755
}
1756-
PrevEVL = new VPScalarPHIRecipe(MaxEVL, &EVL, DebugLoc(), "prev.evl");
1756+
PrevEVL = new VPInstruction(Instruction::PHI, {MaxEVL, &EVL}, DebugLoc(),
1757+
"prev.evl");
17571758
PrevEVL->insertBefore(*Header, Header->getFirstNonPhi());
17581759
}
17591760

@@ -2080,9 +2081,9 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
20802081
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
20812082
StringRef Name =
20822083
isa<VPCanonicalIVPHIRecipe>(PhiR) ? "index" : "evl.based.iv";
2083-
auto *ScalarR =
2084-
new VPScalarPHIRecipe(PhiR->getStartValue(), PhiR->getBackedgeValue(),
2085-
PhiR->getDebugLoc(), Name);
2084+
auto *ScalarR = new VPInstruction(
2085+
Instruction::PHI, {PhiR->getStartValue(), PhiR->getBackedgeValue()},
2086+
PhiR->getDebugLoc(), Name);
20862087
ScalarR->insertBefore(PhiR);
20872088
PhiR->replaceAllUsesWith(ScalarR);
20882089
PhiR->eraseFromParent();

‎llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,12 +143,13 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
143143
})
144144
.Case<VPWidenStoreEVLRecipe, VPReductionEVLRecipe>(
145145
[&](const VPRecipeBase *S) { return VerifyEVLUse(*S, 2); })
146-
.Case<VPWidenLoadEVLRecipe, VPReverseVectorPointerRecipe,
147-
VPScalarPHIRecipe>(
146+
.Case<VPWidenLoadEVLRecipe, VPReverseVectorPointerRecipe>(
148147
[&](const VPRecipeBase *R) { return VerifyEVLUse(*R, 1); })
149148
.Case<VPScalarCastRecipe>(
150149
[&](const VPScalarCastRecipe *S) { return VerifyEVLUse(*S, 0); })
151150
.Case<VPInstruction>([&](const VPInstruction *I) {
151+
if (I->getOpcode() == Instruction::PHI)
152+
return VerifyEVLUse(*I, I->getNumOperands() - 1);
152153
if (I->getOpcode() != Instruction::Add) {
153154
errs() << "EVL is used as an operand in non-VPInstruction::Add\n";
154155
return false;

‎llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
8686
; CHECK-EMPTY:
8787
; CHECK-NEXT: <x1> vector loop: {
8888
; CHECK-NEXT: vector.body:
89-
; CHECK-NEXT: SCALAR-PHI vp<[[EP_IV:%.+]]> = phi ir<0>, vp<%index.next>
89+
; CHECK-NEXT: EMIT vp<[[EP_IV:%.+]]> = phi ir<0>, vp<%index.next>
9090
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi ir<0>, ir<%add> (VF scaled by 1/4)
9191
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[EP_IV]]>, ir<1>
9292
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>

‎llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
193193
; CHECK-EMPTY:
194194
; CHECK-NEXT: <x1> vector loop: {
195195
; CHECK-NEXT: vector.body:
196-
; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
196+
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
197197
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
198198
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
199199
; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
@@ -442,7 +442,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
442442
; CHECK-EMPTY:
443443
; CHECK-NEXT: <x1> vector loop: {
444444
; CHECK-NEXT: vector.body:
445-
; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
445+
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
446446
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
447447
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
448448
; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>

‎llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@
3232

3333
; IF-EVL: <x1> vector loop: {
3434
; IF-EVL-NEXT: vector.body:
35-
; IF-EVL-NEXT: SCALAR-PHI vp<[[IV:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT_EXIT:%.+]]>
36-
; IF-EVL-NEXT: SCALAR-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEX:%.+]]>
35+
; IF-EVL-NEXT: EMIT vp<[[IV:%.+]]> = phi ir<0>, vp<[[IV_NEXT_EXIT:%.+]]>
36+
; IF-EVL-NEXT: EMIT vp<[[EVL_PHI:%.+]]> = phi ir<0>, vp<[[IV_NEX:%.+]]>
3737
; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
3838
; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
3939
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>

‎llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
1919
; CHECK-EMPTY:
2020
; CHECK-NEXT: <x1> vector loop: {
2121
; CHECK-NEXT: vector.body:
22-
; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
22+
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
2323
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
2424
; CHECK-NEXT: EMIT vp<[[PTR:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]>
2525
; CHECK-NEXT: vp<[[WIDE_PTR:%.+]]> = vector-pointer vp<[[PTR]]>

0 commit comments

Comments
 (0)