Skip to content

Commit 553bad4

Browse files
david-armdavid-arm
committed
Address review comments + regen CHECK lines
1 parent cff198a commit 553bad4

File tree

6 files changed

+299
-302
lines changed

6 files changed

+299
-302
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -700,8 +700,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
700700
case VPInstruction::ExtractFirstActive: {
701701
Value *Vec = State.get(getOperand(0));
702702
Value *Mask = State.get(getOperand(1));
703-
Value *Ctz =
704-
Builder.CreateCountTrailingZeroElems(Builder.getInt64Ty(), Mask);
703+
Value *Ctz = Builder.CreateCountTrailingZeroElems(
704+
Builder.getInt64Ty(), Mask, true, "first.active.lane");
705705
return Builder.CreateExtractElement(Vec, Ctz);
706706
}
707707
default:

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2128,7 +2128,6 @@ void VPlanTransforms::handleUncountableEarlyExit(
21282128
ExitIRI->addOperand(IncomingFromLatch);
21292129
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
21302130
}
2131-
21322131
// Add the incoming value from the early exit.
21332132
if (!IncomingFromEarlyExit->isLiveIn())
21342133
IncomingFromEarlyExit =

‎llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,9 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
209209
auto *UI = cast<VPRecipeBase>(U);
210210
// TODO: check dominance of incoming values for phis properly.
211211
if (!UI ||
212-
isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI))
212+
isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI) ||
213+
(isa<VPIRInstruction>(UI) &&
214+
isa<PHINode>(cast<VPIRInstruction>(UI)->getInstruction())))
213215
continue;
214216

215217
// If the user is in the same block, check it comes after R in the
@@ -222,11 +224,7 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
222224
continue;
223225
}
224226

225-
// Now that we support vectorising loops with uncountable early exits
226-
// we can end up in situations where VPBB does not dominate the exit
227-
// block. Only do the check if the user is not in a VPIRBasicBlock.
228-
if (!isa<VPIRBasicBlock>(UI->getParent()) &&
229-
!VPDT.dominates(VPBB, UI->getParent())) {
227+
if (!VPDT.dominates(VPBB, UI->getParent())) {
230228
errs() << "Use before def!\n";
231229
return false;
232230
}

‎llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,17 @@ define i64 @same_exit_block_pre_inc_use1() #1 {
2424
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
2525
; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
2626
; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 16
27-
; CHECK-NEXT: [[INDEX_NEXT1:%.*]] = add i64 3, [[N_VEC]]
27+
; CHECK-NEXT: [[TMP6:%.*]] = add i64 3, [[N_VEC]]
2828
; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 16 x i64> @llvm.stepvector.nxv16i64()
2929
; CHECK-NEXT: [[TMP8:%.*]] = mul <vscale x 16 x i64> [[TMP7]], splat (i64 1)
3030
; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 16 x i64> splat (i64 3), [[TMP8]]
3131
; CHECK-NEXT: [[TMP9:%.*]] = mul i64 1, [[TMP5]]
3232
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP9]], i64 0
3333
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[DOTSPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
34-
; CHECK-NEXT: br label [[LOOP:%.*]]
34+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
3535
; CHECK: vector.body:
36-
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[LOOP]] ]
37-
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP]] ]
36+
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
37+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
3838
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
3939
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 0
4040
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP10]]
@@ -50,33 +50,33 @@ define i64 @same_exit_block_pre_inc_use1() #1 {
5050
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]]
5151
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i64> [[VEC_IND]], [[DOTSPLAT]]
5252
; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP17]], [[TMP18]]
53-
; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
53+
; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
5454
; CHECK: middle.split:
55-
; CHECK-NEXT: br i1 [[TMP17]], label [[VECTOR_EARLY_EXIT:%.*]], label [[LOOP_INC:%.*]]
55+
; CHECK-NEXT: br i1 [[TMP17]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
5656
; CHECK: vector.early.exit:
57-
; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP16]], i1 true)
58-
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <vscale x 16 x i64> [[VEC_IND]], i64 [[TMP20]]
57+
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP16]], i1 true)
58+
; CHECK-NEXT: [[TMP20:%.*]] = extractelement <vscale x 16 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
5959
; CHECK-NEXT: br label [[LOOP_END:%.*]]
6060
; CHECK: middle.block:
6161
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]
6262
; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOP_END]], label [[SCALAR_PH]]
6363
; CHECK: scalar.ph:
64-
; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ [[INDEX_NEXT1]], [[LOOP_INC]] ], [ 3, [[ENTRY:%.*]] ]
65-
; CHECK-NEXT: br label [[LOOP1:%.*]]
64+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
65+
; CHECK-NEXT: br label [[LOOP:%.*]]
6666
; CHECK: loop:
67-
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC1:%.*]] ], [ [[INDEX2]], [[SCALAR_PH]] ]
67+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
6868
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
6969
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
7070
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
7171
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
7272
; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
73-
; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC1]], label [[LOOP_END]]
73+
; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
7474
; CHECK: loop.inc:
7575
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
7676
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
77-
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
77+
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
7878
; CHECK: loop.end:
79-
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ 67, [[LOOP_INC1]] ], [ 67, [[LOOP_INC]] ], [ [[TMP21]], [[VECTOR_EARLY_EXIT]] ]
79+
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[TMP20]], [[VECTOR_EARLY_EXIT]] ]
8080
; CHECK-NEXT: ret i64 [[RETVAL]]
8181
;
8282
entry:
@@ -117,10 +117,10 @@ define i64 @same_exit_block_pre_inc_use4() {
117117
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
118118
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
119119
; CHECK: vector.ph:
120-
; CHECK-NEXT: br label [[LOOP:%.*]]
120+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
121121
; CHECK: vector.body:
122-
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[LOOP]] ]
123-
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP]] ]
122+
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
123+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
124124
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
125125
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
126126
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[TMP0]]
@@ -133,18 +133,18 @@ define i64 @same_exit_block_pre_inc_use4() {
133133
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
134134
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
135135
; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
136-
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
136+
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
137137
; CHECK: middle.split:
138138
; CHECK-NEXT: br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
139139
; CHECK: vector.early.exit:
140-
; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP4]], i1 true)
141-
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i64> [[VEC_IND]], i64 [[TMP8]]
140+
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP4]], i1 true)
141+
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
142142
; CHECK-NEXT: br label [[LOOP_END:%.*]]
143143
; CHECK: middle.block:
144144
; CHECK-NEXT: br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
145145
; CHECK: scalar.ph:
146146
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
147-
; CHECK-NEXT: br label [[LOOP1:%.*]]
147+
; CHECK-NEXT: br label [[LOOP:%.*]]
148148
; CHECK: loop:
149149
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
150150
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[INDEX]]
@@ -154,9 +154,9 @@ define i64 @same_exit_block_pre_inc_use4() {
154154
; CHECK: loop.inc:
155155
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
156156
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
157-
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]], !llvm.loop [[LOOP5:![0-9]+]]
157+
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP5:![0-9]+]]
158158
; CHECK: loop.end:
159-
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[TMP9]], [[VECTOR_EARLY_EXIT]] ]
159+
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[TMP8]], [[VECTOR_EARLY_EXIT]] ]
160160
; CHECK-NEXT: ret i64 [[RETVAL]]
161161
;
162162
entry:
@@ -215,8 +215,8 @@ define i64 @loop_contains_safe_call() #1 {
215215
; CHECK: middle.split:
216216
; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
217217
; CHECK: vector.early.exit:
218-
; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 true)
219-
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[TMP9]]
218+
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 true)
219+
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
220220
; CHECK-NEXT: br label [[LOOP_END:%.*]]
221221
; CHECK: middle.block:
222222
; CHECK-NEXT: br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
@@ -235,7 +235,7 @@ define i64 @loop_contains_safe_call() #1 {
235235
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
236236
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP7:![0-9]+]]
237237
; CHECK: loop.end:
238-
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[TMP10]], [[VECTOR_EARLY_EXIT]] ]
238+
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[TMP9]], [[VECTOR_EARLY_EXIT]] ]
239239
; CHECK-NEXT: ret i64 [[RETVAL]]
240240
;
241241
entry:
@@ -274,10 +274,10 @@ define i64 @loop_contains_safe_div() #1 {
274274
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
275275
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
276276
; CHECK: vector.ph:
277-
; CHECK-NEXT: br label [[LOOP:%.*]]
277+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
278278
; CHECK: vector.body:
279-
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[LOOP]] ]
280-
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP]] ]
279+
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
280+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
281281
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
282282
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
283283
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP0]]
@@ -291,18 +291,18 @@ define i64 @loop_contains_safe_div() #1 {
291291
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
292292
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
293293
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
294-
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
294+
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
295295
; CHECK: middle.split:
296296
; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
297297
; CHECK: vector.early.exit:
298-
; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP5]], i1 true)
299-
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[VEC_IND]], i64 [[TMP9]]
298+
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP5]], i1 true)
299+
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
300300
; CHECK-NEXT: br label [[LOOP_END:%.*]]
301301
; CHECK: middle.block:
302302
; CHECK-NEXT: br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
303303
; CHECK: scalar.ph:
304304
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
305-
; CHECK-NEXT: br label [[LOOP1:%.*]]
305+
; CHECK-NEXT: br label [[LOOP:%.*]]
306306
; CHECK: loop:
307307
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
308308
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
@@ -313,9 +313,9 @@ define i64 @loop_contains_safe_div() #1 {
313313
; CHECK: loop.inc:
314314
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
315315
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
316-
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]], !llvm.loop [[LOOP9:![0-9]+]]
316+
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP9:![0-9]+]]
317317
; CHECK: loop.end:
318-
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[TMP10]], [[VECTOR_EARLY_EXIT]] ]
318+
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[TMP9]], [[VECTOR_EARLY_EXIT]] ]
319319
; CHECK-NEXT: ret i64 [[RETVAL]]
320320
;
321321
entry:
@@ -356,10 +356,10 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
356356
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
357357
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
358358
; CHECK: vector.ph:
359-
; CHECK-NEXT: br label [[LOOP:%.*]]
359+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
360360
; CHECK: vector.body:
361-
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[LOOP]] ]
362-
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP]] ]
361+
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]
362+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 3, i64 4, i64 5, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
363363
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
364364
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
365365
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[TMP0]]
@@ -375,19 +375,19 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
375375
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
376376
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
377377
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
378-
; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
378+
; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
379379
; CHECK: middle.split:
380380
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_LOAD2]], i32 3
381381
; CHECK-NEXT: br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
382382
; CHECK: vector.early.exit:
383-
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
384-
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[TMP11]]
383+
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
384+
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[FIRST_ACTIVE_LANE]]
385385
; CHECK-NEXT: br label [[LOOP_END:%.*]]
386386
; CHECK: middle.block:
387387
; CHECK-NEXT: br i1 true, label [[LOOP_END]], label [[SCALAR_PH]]
388388
; CHECK: scalar.ph:
389389
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
390-
; CHECK-NEXT: br label [[LOOP1:%.*]]
390+
; CHECK-NEXT: br label [[LOOP:%.*]]
391391
; CHECK: loop:
392392
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
393393
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
@@ -399,9 +399,9 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
399399
; CHECK-NEXT: [[LD2:%.*]] = load i64, ptr [[ARRAYIDX2]], align 8
400400
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
401401
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
402-
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]], !llvm.loop [[LOOP11:![0-9]+]]
402+
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP11:![0-9]+]]
403403
; CHECK: loop.end:
404-
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ [[LD2]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[TMP12]], [[VECTOR_EARLY_EXIT]] ]
404+
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ [[LD2]], [[LOOP_INC]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[TMP11]], [[VECTOR_EARLY_EXIT]] ]
405405
; CHECK-NEXT: ret i64 [[RETVAL]]
406406
;
407407
entry:

0 commit comments

Comments
 (0)