Skip to content

Commit cca24cc

Browse files
fhahnrlavaee
fhahn
authored and
rlavaee
committed
[LV] Replace redundant ExtractLastElement of reduction result (NFC).
Replace redundant ExtractLastElement VPInstructions early. This is NFC, as the VPInstruction computing the final result is vector-to-scalar, producing a single scalar already. This enables follow-up changes to model more aspects of reductions directly in VPlan.
1 parent 903c9f9 commit cca24cc

File tree

5 files changed

+20
-24
lines changed

5 files changed

+20
-24
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9271,12 +9271,17 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92719271
Builder.createNaryOp(VPInstruction::ComputeReductionResult,
92729272
{PhiR, NewExitingVPV}, Flags, ExitDL);
92739273
}
9274-
// Update all users outside the vector region.
9275-
OrigExitingVPV->replaceUsesWithIf(
9276-
FinalReductionResult, [FinalReductionResult](VPUser &User, unsigned) {
9277-
auto *Parent = cast<VPRecipeBase>(&User)->getParent();
9278-
return FinalReductionResult != &User && !Parent->getParent();
9279-
});
9274+
// Update all users outside the vector region. Also replace redundant
9275+
// ExtractLastElement.
9276+
for (auto *U : to_vector(OrigExitingVPV->users())) {
9277+
auto *Parent = cast<VPRecipeBase>(U)->getParent();
9278+
if (FinalReductionResult == U || Parent->getParent())
9279+
continue;
9280+
U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
9281+
if (match(U, m_VPInstruction<VPInstruction::ExtractLastElement>(
9282+
m_VPValue())))
9283+
cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult);
9284+
}
92809285

92819286
// Adjust AnyOf reductions; replace the reduction phi for the selected value
92829287
// with a boolean reduction phi node to check if the condition is true in

‎llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,12 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
4343
; CHECK-EMPTY:
4444
; CHECK-NEXT: middle.block:
4545
; CHECK-NEXT: EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<[[ACC]]>, ir<[[REDUCE]]>
46-
; CHECK-NEXT: EMIT vp<[[EXTRACT:%.+]]> = extract-last-element vp<[[RED_RESULT]]>
4746
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VEC_TC]]>
4847
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
4948
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
5049
; CHECK-EMPTY:
5150
; CHECK-NEXT: ir-bb<exit>:
52-
; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[EXTRACT]]> from middle.block)
51+
; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RED_RESULT]]> from middle.block)
5352
; CHECK-NEXT: No successors
5453
; CHECK-EMPTY:
5554
; CHECK-NEXT: scalar.ph:

‎llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
6363
; IF-EVL-OUTLOOP-EMPTY:
6464
; IF-EVL-OUTLOOP-NEXT: middle.block:
6565
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, vp<[[RDX_SELECT]]>
66-
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
6766
; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>
6867
; IF-EVL-OUTLOOP-EMPTY:
6968
; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
70-
; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
69+
; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)
7170
; IF-EVL-OUTLOOP-NEXT: No successors
7271
;
7372

@@ -103,11 +102,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
103102
; IF-EVL-INLOOP-EMPTY:
104103
; IF-EVL-INLOOP-NEXT: middle.block:
105104
; IF-EVL-INLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
106-
; IF-EVL-INLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
107105
; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.end>
108106
; IF-EVL-INLOOP-EMPTY:
109107
; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
110-
; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
108+
; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)
111109
; IF-EVL-INLOOP-NEXT: No successors
112110
;
113111

@@ -138,13 +136,12 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
138136
; NO-VP-OUTLOOP-EMPTY:
139137
; NO-VP-OUTLOOP-NEXT: middle.block:
140138
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
141-
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
142139
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[BOC:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
143140
; NO-VP-OUTLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>
144141
; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
145142
; NO-VP-OUTLOOP-EMPTY:
146143
; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:
147-
; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
144+
; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)
148145
; NO-VP-OUTLOOP-NEXT: No successors
149146
; NO-VP-OUTLOOP-EMPTY:
150147
; NO-VP-OUTLOOP-NEXT: scalar.ph:
@@ -187,13 +184,12 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
187184
; NO-VP-INLOOP-EMPTY:
188185
; NO-VP-INLOOP-NEXT: middle.block:
189186
; NO-VP-INLOOP-NEXT: EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
190-
; NO-VP-INLOOP-NEXT: EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
191187
; NO-VP-INLOOP-NEXT: EMIT vp<[[BOC:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
192188
; NO-VP-INLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]>
193189
; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
194190
; NO-VP-INLOOP-EMPTY:
195191
; NO-VP-INLOOP-NEXT: ir-bb<for.end>:
196-
; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
192+
; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)
197193
; NO-VP-INLOOP-NEXT: No successors
198194
; NO-VP-INLOOP-EMPTY:
199195
; NO-VP-INLOOP-NEXT: scalar.ph:

‎llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -213,11 +213,10 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
213213
; CHECK-EMPTY:
214214
; CHECK-NEXT: middle.block:
215215
; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]>
216-
; CHECK-NEXT: EMIT vp<[[RED_EX:%.+]]> = extract-last-element vp<[[RED_RES]]>
217216
; CHECK-NEXT: Successor(s): ir-bb<exit>
218217
; CHECK-EMPTY:
219218
; CHECK-NEXT: ir-bb<exit>
220-
; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block)
219+
; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_RES]]> from middle.block)
221220
; CHECK-NEXT: No successors
222221
;
223222
entry:

‎llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,12 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
3636
; CHECK-EMPTY:
3737
; CHECK-NEXT: middle.block:
3838
; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result fast ir<%red>, ir<%red.next>
39-
; CHECK-NEXT: EMIT vp<[[RED_EX:%.+]]> = extract-last-element vp<[[RED_RES]]>
4039
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
4140
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
4241
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
4342
; CHECK-EMPTY:
4443
; CHECK-NEXT: ir-bb<exit>
45-
; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block)
44+
; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %loop ] (extra operand: vp<[[RED_RES]]> from middle.block)
4645
; CHECK-NEXT: No successors
4746
; CHECK-EMPTY:
4847
; CHECK-NEXT: scalar.ph
@@ -179,13 +178,12 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
179178
; CHECK-EMPTY:
180179
; CHECK-NEXT: middle.block:
181180
; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result nnan ninf nsz ir<%sum.07>, ir<[[MULADD]]>
182-
; CHECK-NEXT: EMIT vp<[[RED_EX:%.+]]> = extract-last-element vp<[[RED_RES]]>
183181
; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
184182
; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]>
185183
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
186184
; CHECK-EMPTY:
187185
; CHECK-NEXT: ir-bb<exit>
188-
; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block)
186+
; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %loop ] (extra operand: vp<[[RED_RES]]> from middle.block)
189187
; CHECK-NEXT: No successors
190188
; CHECK-EMPTY:
191189
; CHECK-NEXT: scalar.ph
@@ -241,13 +239,12 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
241239
; CHECK-EMPTY:
242240
; CHECK-NEXT: middle.block:
243241
; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
244-
; CHECK-NEXT: EMIT vp<[[EXT:%.+]]> = extract-last-element vp<[[RDX_RES]]>
245242
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}>
246243
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
247244
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
248245
; CHECK-EMPTY:
249246
; CHECK: ir-bb<exit>:
250-
; CHECK-NEXT: IR %cond.lcssa = phi i64 [ %cond, %loop ] (extra operand: vp<[[EXT]]> from middle.block)
247+
; CHECK-NEXT: IR %cond.lcssa = phi i64 [ %cond, %loop ] (extra operand: vp<[[RDX_RES]]> from middle.block)
251248
; CHECK-NEXT: No successors
252249
; CHECK-EMPTY:
253250
; CHECK-NEXT: scalar.ph:

0 commit comments

Comments
 (0)