Skip to content

Commit df21288

Browse files
fhahnfhahn
authored
[VPlan] Replace ExtractFromEnd with Extract(Last|Penultimate)Element (NFC). (#137030)
ExtractFromEnd only has 2 uses, extracting the last and penultimate elements. Replace it with 2 separate opcodes, removing the need to materialize and handle a constant argument. PR: llvm/llvm-project#137030
1 parent 4ea2278 commit df21288

13 files changed

+60
-57
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9237,8 +9237,6 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
92379237
cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
92389238
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
92399239
VPBuilder ScalarPHBuilder(ScalarPH);
9240-
VPValue *OneVPV = Plan.getOrAddLiveIn(
9241-
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
92429240
for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) {
92439241
auto *ScalarPhiIRI = cast<VPIRPhi>(&ScalarPhiR);
92449242

@@ -9273,7 +9271,7 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
92739271
"Cannot handle loops with uncountable early exits");
92749272
if (IsFOR)
92759273
ResumeFromVectorLoop = MiddleBuilder.createNaryOp(
9276-
VPInstruction::ExtractFromEnd, {ResumeFromVectorLoop, OneVPV}, {},
9274+
VPInstruction::ExtractLastElement, {ResumeFromVectorLoop}, {},
92779275
"vector.recur.extract");
92789276
StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx";
92799277
auto *ResumePhiR = ScalarPHBuilder.createNaryOp(
@@ -9341,8 +9339,6 @@ static void addExitUsersForFirstOrderRecurrences(
93419339
auto *MiddleVPBB = Plan.getMiddleBlock();
93429340
VPBuilder ScalarPHBuilder(ScalarPHVPBB);
93439341
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
9344-
VPValue *TwoVPV = Plan.getOrAddLiveIn(
9345-
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 2));
93469342

93479343
for (auto &HeaderPhi : VectorRegion->getEntryBasicBlock()->phis()) {
93489344
auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&HeaderPhi);
@@ -9426,8 +9422,8 @@ static void addExitUsersForFirstOrderRecurrences(
94269422
if (ExitIRI->getOperand(0) != FOR)
94279423
continue;
94289424
VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
9429-
VPInstruction::ExtractFromEnd, {FOR->getBackedgeValue(), TwoVPV}, {},
9430-
"vector.recur.extract.for.phi");
9425+
VPInstruction::ExtractPenultimateElement, {FOR->getBackedgeValue()},
9426+
{}, "vector.recur.extract.for.phi");
94319427
ExitIRI->setOperand(0, PenultimateElement);
94329428
ExitUsersToFix.remove(ExitIRI);
94339429
}

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -878,10 +878,14 @@ class VPInstruction : public VPRecipeWithIRFlags,
878878
Broadcast,
879879
ComputeFindLastIVResult,
880880
ComputeReductionResult,
881-
// Takes the VPValue to extract from as first operand and the lane or part
882-
// to extract as second operand, counting from the end starting with 1 for
883-
// last. The second operand must be a positive constant and <= VF.
884-
ExtractFromEnd,
881+
// Extracts the last lane from its operand if it is a vector, or the last
882+
// part if scalar. In the latter case, the recipe will be removed during
883+
// unrolling.
884+
ExtractLastElement,
885+
// Extracts the second-to-last lane from its operand or the second-to-last
886+
// part if it is scalar. In the latter case, the recipe will be removed
887+
// during unrolling.
888+
ExtractPenultimateElement,
885889
LogicalAnd, // Non-poison propagating logical And.
886890
// Add an offset in bytes (second operand) to a base pointer (first
887891
// operand). Only generates scalar values (either for the first lane only or

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
8888
return SetResultTyFromOp();
8989
case VPInstruction::FirstActiveLane:
9090
return Type::getIntNTy(Ctx, 64);
91-
case VPInstruction::ExtractFromEnd: {
91+
case VPInstruction::ExtractLastElement:
92+
case VPInstruction::ExtractPenultimateElement: {
9293
Type *BaseTy = inferScalarType(R->getOperand(0));
9394
if (auto *VecTy = dyn_cast<VectorType>(BaseTy))
9495
return VecTy->getElementType();

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -725,10 +725,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
725725

726726
return ReducedPartRdx;
727727
}
728-
case VPInstruction::ExtractFromEnd: {
729-
auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
730-
unsigned Offset = CI->getZExtValue();
731-
assert(Offset > 0 && "Offset from end must be positive");
728+
case VPInstruction::ExtractLastElement:
729+
case VPInstruction::ExtractPenultimateElement: {
730+
unsigned Offset = getOpcode() == VPInstruction::ExtractLastElement ? 1 : 2;
732731
Value *Res;
733732
if (State.VF.isVector()) {
734733
assert(Offset <= State.VF.getKnownMinValue() &&
@@ -854,7 +853,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
854853
}
855854

856855
bool VPInstruction::isVectorToScalar() const {
857-
return getOpcode() == VPInstruction::ExtractFromEnd ||
856+
return getOpcode() == VPInstruction::ExtractLastElement ||
857+
getOpcode() == VPInstruction::ExtractPenultimateElement ||
858858
getOpcode() == Instruction::ExtractElement ||
859859
getOpcode() == VPInstruction::FirstActiveLane ||
860860
getOpcode() == VPInstruction::ComputeFindLastIVResult ||
@@ -924,7 +924,8 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
924924
case VPInstruction::AnyOf:
925925
case VPInstruction::CalculateTripCountMinusVF:
926926
case VPInstruction::CanonicalIVIncrementForPart:
927-
case VPInstruction::ExtractFromEnd:
927+
case VPInstruction::ExtractLastElement:
928+
case VPInstruction::ExtractPenultimateElement:
928929
case VPInstruction::FirstActiveLane:
929930
case VPInstruction::FirstOrderRecurrenceSplice:
930931
case VPInstruction::LogicalAnd:
@@ -1042,8 +1043,11 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
10421043
case VPInstruction::Broadcast:
10431044
O << "broadcast";
10441045
break;
1045-
case VPInstruction::ExtractFromEnd:
1046-
O << "extract-from-end";
1046+
case VPInstruction::ExtractLastElement:
1047+
O << "extract-last-element";
1048+
break;
1049+
case VPInstruction::ExtractPenultimateElement:
1050+
O << "extract-penultimate-element";
10471051
break;
10481052
case VPInstruction::ComputeFindLastIVResult:
10491053
O << "compute-find-last-iv-result";
@@ -1145,11 +1149,7 @@ void VPIRInstruction::extractLastLaneOfFirstOperand(VPBuilder &Builder) {
11451149
if (Exiting->isLiveIn())
11461150
return;
11471151

1148-
LLVMContext &Ctx = getInstruction().getContext();
1149-
auto &Plan = *getParent()->getPlan();
1150-
Exiting = Builder.createNaryOp(VPInstruction::ExtractFromEnd,
1151-
{Exiting, Plan.getOrAddLiveIn(ConstantInt::get(
1152-
IntegerType::get(Ctx, 32), 1))});
1152+
Exiting = Builder.createNaryOp(VPInstruction::ExtractLastElement, {Exiting});
11531153
setOperand(0, Exiting);
11541154
}
11551155

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -827,8 +827,8 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
827827
using namespace VPlanPatternMatch;
828828

829829
VPValue *Incoming;
830-
if (!match(Op, m_VPInstruction<VPInstruction::ExtractFromEnd>(
831-
m_VPValue(Incoming), m_SpecificInt(1))))
830+
if (!match(Op, m_VPInstruction<VPInstruction::ExtractLastElement>(
831+
m_VPValue(Incoming))))
832832
return nullptr;
833833

834834
auto *WideIV = getOptimizableIVOf(Incoming);

‎llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -337,16 +337,18 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
337337
continue;
338338
}
339339
VPValue *Op0;
340-
if (match(&R, m_VPInstruction<VPInstruction::ExtractFromEnd>(
341-
m_VPValue(Op0), m_VPValue(Op1)))) {
340+
if (match(&R, m_VPInstruction<VPInstruction::ExtractLastElement>(
341+
m_VPValue(Op0))) ||
342+
match(&R, m_VPInstruction<VPInstruction::ExtractPenultimateElement>(
343+
m_VPValue(Op0)))) {
342344
addUniformForAllParts(cast<VPSingleDefRecipe>(&R));
343345
if (Plan.hasScalarVFOnly()) {
344-
// Extracting from end with VF = 1 implies retrieving the scalar part UF
345-
// - Op1.
346+
auto *I = cast<VPInstruction>(&R);
347+
// Extracting from end with VF = 1 implies retrieving the last or
348+
// penultimate scalar part (UF-1 or UF-2).
346349
unsigned Offset =
347-
cast<ConstantInt>(Op1->getLiveInIRValue())->getZExtValue();
348-
R.getVPSingleValue()->replaceAllUsesWith(
349-
getValueForPart(Op0, UF - Offset));
350+
I->getOpcode() == VPInstruction::ExtractLastElement ? 1 : 2;
351+
I->replaceAllUsesWith(getValueForPart(Op0, UF - Offset));
350352
R.eraseFromParent();
351353
} else {
352354
// Otherwise we extract from the last part.

‎llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
4242
; CHECK-EMPTY:
4343
; CHECK-NEXT: middle.block:
4444
; CHECK-NEXT: EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<[[ACC]]>, ir<[[REDUCE]]>
45-
; CHECK-NEXT: EMIT vp<[[EXTRACT:%.+]]> = extract-from-end vp<[[RED_RESULT]]>, ir<1>
45+
; CHECK-NEXT: EMIT vp<[[EXTRACT:%.+]]> = extract-last-element vp<[[RED_RESULT]]>
4646
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VEC_TC]]>
4747
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
4848
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -107,7 +107,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
107107
; CHECK-EMPTY:
108108
; CHECK-NEXT: middle.block:
109109
; CHECK-NEXT: EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<%accum>, ir<%add>
110-
; CHECK-NEXT: EMIT vp<[[EXTRACT:%.+]]> = extract-from-end vp<[[RED_RESULT]]>, ir<1>
110+
; CHECK-NEXT: EMIT vp<[[EXTRACT:%.+]]> = extract-last-element vp<[[RED_RESULT]]>
111111
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, ir<1024>
112112
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
113113
; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>

‎llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
6262
; IF-EVL-OUTLOOP-EMPTY:
6363
; IF-EVL-OUTLOOP-NEXT: middle.block:
6464
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, vp<[[RDX_SELECT]]>
65-
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-from-end vp<[[RDX]]>, ir<1>
65+
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
6666
; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>
6767
; IF-EVL-OUTLOOP-EMPTY:
6868
; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
@@ -102,7 +102,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
102102
; IF-EVL-INLOOP-EMPTY:
103103
; IF-EVL-INLOOP-NEXT: middle.block:
104104
; IF-EVL-INLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
105-
; IF-EVL-INLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-from-end vp<[[RDX]]>, ir<1>
105+
; IF-EVL-INLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
106106
; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.end>
107107
; IF-EVL-INLOOP-EMPTY:
108108
; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
@@ -137,7 +137,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
137137
; NO-VP-OUTLOOP-EMPTY:
138138
; NO-VP-OUTLOOP-NEXT: middle.block:
139139
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
140-
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-from-end vp<[[RDX]]>, ir<1>
140+
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
141141
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[BOC:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
142142
; NO-VP-OUTLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>
143143
; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
@@ -185,7 +185,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
185185
; NO-VP-INLOOP-EMPTY:
186186
; NO-VP-INLOOP-NEXT: middle.block:
187187
; NO-VP-INLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
188-
; NO-VP-INLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-from-end vp<[[RDX]]>, ir<1>
188+
; NO-VP-INLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
189189
; NO-VP-INLOOP-NEXT: EMIT vp<[[BOC:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
190190
; NO-VP-INLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>
191191
; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph

‎llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
3737
; CHECK-NEXT: Successor(s): middle.block
3838
; CHECK-EMPTY:
3939
; CHECK-NEXT: middle.block:
40-
; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%for.1.next>, ir<1>
41-
; CHECK-NEXT: EMIT vp<[[RESUME_2:%.+]]>.1 = extract-from-end vp<[[FOR1_SPLICE]]>, ir<1>
40+
; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-last-element ir<%for.1.next>
41+
; CHECK-NEXT: EMIT vp<[[RESUME_2:%.+]]>.1 = extract-last-element vp<[[FOR1_SPLICE]]>
4242
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1000>, vp<[[VTC]]>
4343
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
4444
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -117,9 +117,9 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
117117
; CHECK-NEXT: Successor(s): middle.block
118118
; CHECK-EMPTY:
119119
; CHECK-NEXT: middle.block:
120-
; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-from-end ir<%for.1.next>, ir<1>
121-
; CHECK-NEXT: EMIT vp<[[RESUME_2:%.+]]>.1 = extract-from-end vp<[[FOR1_SPLICE]]>, ir<1>
122-
; CHECK-NEXT: EMIT vp<[[RESUME_3:%.+]]>.2 = extract-from-end vp<[[FOR2_SPLICE]]>, ir<1>
120+
; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-last-element ir<%for.1.next>
121+
; CHECK-NEXT: EMIT vp<[[RESUME_2:%.+]]>.1 = extract-last-element vp<[[FOR1_SPLICE]]>
122+
; CHECK-NEXT: EMIT vp<[[RESUME_3:%.+]]>.2 = extract-last-element vp<[[FOR2_SPLICE]]>
123123
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1000>, vp<[[VTC]]>
124124
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
125125
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -203,8 +203,8 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) {
203203
; CHECK-NEXT: Successor(s): middle.block
204204
; CHECK-EMPTY:
205205
; CHECK-NEXT: middle.block:
206-
; CHECK-NEXT: EMIT vp<[[EXT_X:%.+]]> = extract-from-end ir<%for.x.next>, ir<1>
207-
; CHECK-NEXT: EMIT vp<[[EXT_Y:%.+]]>.1 = extract-from-end ir<%for.x.prev>, ir<1>
206+
; CHECK-NEXT: EMIT vp<[[EXT_X:%.+]]> = extract-last-element ir<%for.x.next>
207+
; CHECK-NEXT: EMIT vp<[[EXT_Y:%.+]]>.1 = extract-last-element ir<%for.x.prev>
208208
; CHECK-NEXT: EMIT vp<[[MIDDLE_C:%.+]]> = icmp eq ir<4098>, vp<[[VTC]]>
209209
; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_C]]>
210210
; CHECK-NEXT: Successor(s): ir-bb<ret>, scalar.ph
@@ -282,8 +282,8 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) {
282282
; CHECK-NEXT: Successor(s): middle.block
283283
; CHECK-EMPTY:
284284
; CHECK-NEXT: middle.block:
285-
; CHECK-NEXT: EMIT vp<[[EXT_X:%.+]]> = extract-from-end ir<%for.x.next>, ir<1>
286-
; CHECK-NEXT: EMIT vp<[[EXT_Y:%.+]]>.1 = extract-from-end ir<%for.x.prev>, ir<1>
285+
; CHECK-NEXT: EMIT vp<[[EXT_X:%.+]]> = extract-last-element ir<%for.x.next>
286+
; CHECK-NEXT: EMIT vp<[[EXT_Y:%.+]]>.1 = extract-last-element ir<%for.x.prev>
287287
; CHECK-NEXT: EMIT vp<[[MIDDLE_C:%.+]]> = icmp eq ir<4098>, vp<[[VTC]]>
288288
; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_C]]>
289289
; CHECK-NEXT: Successor(s): ir-bb<ret>, scalar.ph

‎llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
214214
; CHECK-EMPTY:
215215
; CHECK-NEXT: middle.block:
216216
; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]>
217-
; CHECK-NEXT: EMIT vp<[[RED_EX:%.+]]> = extract-from-end vp<[[RED_RES]]>, ir<1>
217+
; CHECK-NEXT: EMIT vp<[[RED_EX:%.+]]> = extract-last-element vp<[[RED_RES]]>
218218
; CHECK-NEXT: Successor(s): ir-bb<exit>
219219
; CHECK-EMPTY:
220220
; CHECK-NEXT: ir-bb<exit>

0 commit comments

Comments
 (0)