Skip to content

Commit 93dec84

Browse files
artagnonartagnon
committed
[VPlan] Recursively match operands of interleave group
When narrowing interleave groups, we use the canNarrowLoad check, which bails out when there any recipe that is not a VPWidenLoad, VPWidenInterleave, or live-in feeding the interleave: a lot of potential narrowing opportunities are missed as a result. Correctly identify that these three cases are the leaf cases, and match the recursive operands instead. Fixes #128062.
1 parent 456948d commit 93dec84

File tree

5 files changed

+201
-201
lines changed

5 files changed

+201
-201
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 51 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,14 @@ static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
628628
return Users.takeVector();
629629
}
630630

631+
static SmallVector<VPValue *> collectOperandsRecursively(VPRecipeBase *R) {
632+
SetVector<VPValue *> Operands(llvm::from_range, R->operands());
633+
for (unsigned I = 0; I != Operands.size(); ++I)
634+
if (auto *Cur = Operands[I]->getDefiningRecipe())
635+
Operands.insert_range(Cur->operands());
636+
return Operands.takeVector();
637+
}
638+
631639
/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
632640
/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
633641
/// VPWidenPointerInductionRecipe will generate vectors only. If some users
@@ -4054,25 +4062,42 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
40544062
return ExpandedSCEVs;
40554063
}
40564064

4057-
/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
4058-
/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
4059-
/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
4060-
/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
4061-
/// an index-independent load if it feeds all wide ops at all indices (\p OpV
4062-
/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
4063-
/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
4064-
/// is defined at \p Idx of a load interleave group.
4065-
static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx,
4066-
VPValue *OpV, unsigned Idx) {
4067-
auto *DefR = OpV->getDefiningRecipe();
4068-
if (!DefR)
4069-
return WideMember0->getOperand(OpIdx) == OpV;
4070-
if (auto *W = dyn_cast<VPWidenLoadRecipe>(DefR))
4071-
return !W->getMask() && WideMember0->getOperand(OpIdx) == OpV;
4072-
4073-
if (auto *IR = dyn_cast<VPInterleaveRecipe>(DefR))
4074-
return IR->getInterleaveGroup()->isFull() && IR->getVPValue(Idx) == OpV;
4075-
return false;
4065+
/// Returns true if the \p StoredValues of an interleave group match. It does
4066+
/// this by going through operands recursively until it hits the leaf cases:
4067+
/// VPWidenLoadRecipe, VPInterleaveRecipe, and live-ins.
4068+
static bool interleaveStoredValuesMatch(ArrayRef<VPValue *> StoredValues) {
4069+
auto *WideMember0 =
4070+
dyn_cast_or_null<VPWidenRecipe>(StoredValues[0]->getDefiningRecipe());
4071+
if (!WideMember0)
4072+
return false;
4073+
SmallVector<VPValue *> Ops0 = collectOperandsRecursively(WideMember0);
4074+
for (VPValue *ValI : StoredValues) {
4075+
auto *WideMemberI =
4076+
dyn_cast_or_null<VPWidenRecipe>(ValI->getDefiningRecipe());
4077+
if (!WideMemberI || WideMemberI->getOpcode() != WideMember0->getOpcode())
4078+
return false;
4079+
SmallVector<VPValue *> OpsI = collectOperandsRecursively(WideMemberI);
4080+
if (Ops0.size() != OpsI.size())
4081+
return false;
4082+
for (const auto &[Op0, OpI] : zip(Ops0, OpsI)) {
4083+
auto *Def0 = Op0->getDefiningRecipe();
4084+
auto *DefI = OpI->getDefiningRecipe();
4085+
if (!Def0 || !DefI) {
4086+
if (Op0 != OpI)
4087+
return false;
4088+
} else if (Def0->getVPDefID() != DefI->getVPDefID()) {
4089+
return false;
4090+
} else if (auto *W = dyn_cast<VPWidenLoadRecipe>(DefI)) {
4091+
if (W->isMasked() || Op0 != OpI)
4092+
return false;
4093+
} else if (auto *IR = dyn_cast<VPInterleaveRecipe>(DefI)) {
4094+
if (!IR->getInterleaveGroup()->isFull() ||
4095+
!equal(DefI->definedValues(), Def0->definedValues()))
4096+
return false;
4097+
}
4098+
}
4099+
}
4100+
return true;
40764101
}
40774102

40784103
/// Returns true if \p IR is a full interleave group with factor and number of
@@ -4191,24 +4216,9 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
41914216
continue;
41924217
}
41934218

4194-
// Check if all values feeding InterleaveR are matching wide recipes, which
4195-
// operands that can be narrowed.
4196-
auto *WideMember0 = dyn_cast_or_null<VPWidenRecipe>(
4197-
InterleaveR->getStoredValues()[0]->getDefiningRecipe());
4198-
if (!WideMember0)
4219+
// Check if all values feeding InterleaveR match.
4220+
if (!interleaveStoredValuesMatch(InterleaveR->getStoredValues()))
41994221
return;
4200-
for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues())) {
4201-
auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe());
4202-
if (!R || R->getOpcode() != WideMember0->getOpcode() ||
4203-
R->getNumOperands() > 2)
4204-
return;
4205-
if (any_of(enumerate(R->operands()),
4206-
[WideMember0, Idx = I](const auto &P) {
4207-
const auto &[OpIdx, OpV] = P;
4208-
return !canNarrowLoad(WideMember0, OpIdx, OpV, Idx);
4209-
}))
4210-
return;
4211-
}
42124222
StoreGroups.push_back(InterleaveR);
42134223
}
42144224

@@ -4240,7 +4250,11 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
42404250
NarrowedOps.insert(RepR);
42414251
return RepR;
42424252
}
4243-
auto *WideLoad = cast<VPWidenLoadRecipe>(R);
4253+
auto *WideLoad = dyn_cast<VPWidenLoadRecipe>(R);
4254+
if (!WideLoad) {
4255+
NarrowedOps.insert(V);
4256+
return V;
4257+
}
42444258

42454259
VPValue *PtrOp = WideLoad->getAddr();
42464260
if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(PtrOp))

‎llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll‎

Lines changed: 43 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ define void @test_add_double_same_const_args_1(ptr %res, ptr noalias %A, ptr noa
2828
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
2929
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3030
; CHECK: [[MIDDLE_BLOCK]]:
31-
; CHECK-NEXT: br [[EXIT:label %.*]]
32-
; CHECK: [[SCALAR_PH:.*:]]
31+
; CHECK-NEXT: br label %[[EXIT:.*]]
32+
; CHECK: [[EXIT]]:
33+
; CHECK-NEXT: ret void
3334
;
3435
entry:
3536
br label %loop
@@ -76,10 +77,11 @@ define void @test_add_double_same_const_args_2(ptr %res, ptr noalias %A, ptr noa
7677
; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[TMP9]], align 4
7778
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
7879
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
79-
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
80+
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
8081
; CHECK: [[MIDDLE_BLOCK]]:
81-
; CHECK-NEXT: br [[EXIT:label %.*]]
82-
; CHECK: [[SCALAR_PH:.*:]]
82+
; CHECK-NEXT: br label %[[EXIT:.*]]
83+
; CHECK: [[EXIT]]:
84+
; CHECK-NEXT: ret void
8385
;
8486
entry:
8587
br label %loop
@@ -136,10 +138,11 @@ define void @test_add_double_mixed_const_args(ptr %res, ptr noalias %A, ptr noal
136138
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP9]], align 4
137139
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
138140
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
139-
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
141+
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
140142
; CHECK: [[MIDDLE_BLOCK]]:
141-
; CHECK-NEXT: br [[EXIT:label %.*]]
142-
; CHECK: [[SCALAR_PH:.*:]]
143+
; CHECK-NEXT: br label %[[EXIT:.*]]
144+
; CHECK: [[EXIT]]:
145+
; CHECK-NEXT: ret void
143146
;
144147
entry:
145148
br label %loop
@@ -175,33 +178,24 @@ define void @test_add_double_same_var_args_1(ptr %res, ptr noalias %A, ptr noali
175178
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
176179
; CHECK: [[VECTOR_BODY]]:
177180
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
178-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
181+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
179182
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
180183
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
181-
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
182-
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
183-
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
184-
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
185-
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
186-
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
187-
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
188-
; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[STRIDED_VEC3]], [[BROADCAST_SPLAT]]
184+
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP1]], align 4
185+
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
189186
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[STRIDED_VEC1]], [[BROADCAST_SPLAT]]
190187
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[STRIDED_VEC4]], [[BROADCAST_SPLAT]]
191188
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
192189
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
193-
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
194-
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
195-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
196-
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
197-
; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
198-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
199-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
190+
; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4
191+
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
192+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
200193
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
201-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
194+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
202195
; CHECK: [[MIDDLE_BLOCK]]:
203-
; CHECK-NEXT: br [[EXIT:label %.*]]
204-
; CHECK: [[SCALAR_PH:.*:]]
196+
; CHECK-NEXT: br label %[[EXIT:.*]]
197+
; CHECK: [[EXIT]]:
198+
; CHECK-NEXT: ret void
205199
;
206200
entry:
207201
br label %loop
@@ -237,33 +231,24 @@ define void @test_add_double_same_var_args_2(ptr %res, ptr noalias %A, ptr noali
237231
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
238232
; CHECK: [[VECTOR_BODY]]:
239233
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
240-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
234+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
241235
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
242236
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
243-
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
244-
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
245-
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
246-
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
247-
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
248-
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
249-
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
250-
; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC3]]
237+
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP1]], align 4
238+
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
251239
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC1]]
252240
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC4]]
253241
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
254242
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
255-
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
256-
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
257-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
258-
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
259-
; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
260-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
261-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
243+
; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4
244+
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
245+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
262246
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
263-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
247+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
264248
; CHECK: [[MIDDLE_BLOCK]]:
265-
; CHECK-NEXT: br [[EXIT:label %.*]]
266-
; CHECK: [[SCALAR_PH:.*:]]
249+
; CHECK-NEXT: br label %[[EXIT:.*]]
250+
; CHECK: [[EXIT]]:
251+
; CHECK-NEXT: ret void
267252
;
268253
entry:
269254
br label %loop
@@ -322,10 +307,11 @@ define void @test_add_double_same_var_args_at_different_positions(ptr %res, ptr
322307
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
323308
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
324309
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
325-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
310+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
326311
; CHECK: [[MIDDLE_BLOCK]]:
327-
; CHECK-NEXT: br [[EXIT:label %.*]]
328-
; CHECK: [[SCALAR_PH:.*:]]
312+
; CHECK-NEXT: br label %[[EXIT:.*]]
313+
; CHECK: [[EXIT]]:
314+
; CHECK-NEXT: ret void
329315
;
330316
entry:
331317
br label %loop
@@ -386,10 +372,11 @@ define void @test_add_double_different_var_args_1(ptr %res, ptr noalias %A, ptr
386372
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP8]], align 4
387373
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
388374
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
389-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
375+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
390376
; CHECK: [[MIDDLE_BLOCK]]:
391-
; CHECK-NEXT: br [[EXIT:label %.*]]
392-
; CHECK: [[SCALAR_PH:.*:]]
377+
; CHECK-NEXT: br label %[[EXIT:.*]]
378+
; CHECK: [[EXIT]]:
379+
; CHECK-NEXT: ret void
393380
;
394381
entry:
395382
br label %loop
@@ -450,10 +437,11 @@ define void @test_add_double_different_var_args_2(ptr %res, ptr noalias %A, ptr
450437
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP8]], align 4
451438
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
452439
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
453-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
440+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
454441
; CHECK: [[MIDDLE_BLOCK]]:
455-
; CHECK-NEXT: br [[EXIT:label %.*]]
456-
; CHECK: [[SCALAR_PH:.*:]]
442+
; CHECK-NEXT: br label %[[EXIT:.*]]
443+
; CHECK: [[EXIT]]:
444+
; CHECK-NEXT: ret void
457445
;
458446
entry:
459447
br label %loop

0 commit comments

Comments
 (0)