Skip to content

Commit 6f11c8f

Browse files
artagnonartagnon
committed
[VPlan] Recursively match operands of interleave group
When narrowing interleave groups, we use the canNarrowLoad check, which bails out when there any recipe that is not a VPWidenLoad, VPWidenInterleave, or live-in feeding the interleave: a lot of potential narrowing opportunities are missed as a result. Correctly identify that these three cases are the leaf cases, and match the recursive operands instead. Fixes #128062.
1 parent 17ddd88 commit 6f11c8f

File tree

5 files changed

+201
-201
lines changed

5 files changed

+201
-201
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 51 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -647,6 +647,14 @@ static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
647647
return Users.takeVector();
648648
}
649649

650+
static SmallVector<VPValue *> collectOperandsRecursively(VPRecipeBase *R) {
651+
SetVector<VPValue *> Operands(llvm::from_range, R->operands());
652+
for (unsigned I = 0; I != Operands.size(); ++I)
653+
if (auto *Cur = Operands[I]->getDefiningRecipe())
654+
Operands.insert_range(Cur->operands());
655+
return Operands.takeVector();
656+
}
657+
650658
/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
651659
/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
652660
/// VPWidenPointerInductionRecipe will generate vectors only. If some users
@@ -4060,25 +4068,42 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
40604068
return ExpandedSCEVs;
40614069
}
40624070

4063-
/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
4064-
/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
4065-
/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
4066-
/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
4067-
/// an index-independent load if it feeds all wide ops at all indices (\p OpV
4068-
/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
4069-
/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
4070-
/// is defined at \p Idx of a load interleave group.
4071-
static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx,
4072-
VPValue *OpV, unsigned Idx) {
4073-
auto *DefR = OpV->getDefiningRecipe();
4074-
if (!DefR)
4075-
return WideMember0->getOperand(OpIdx) == OpV;
4076-
if (auto *W = dyn_cast<VPWidenLoadRecipe>(DefR))
4077-
return !W->getMask() && WideMember0->getOperand(OpIdx) == OpV;
4078-
4079-
if (auto *IR = dyn_cast<VPInterleaveRecipe>(DefR))
4080-
return IR->getInterleaveGroup()->isFull() && IR->getVPValue(Idx) == OpV;
4081-
return false;
4071+
/// Returns true if the \p StoredValues of an interleave group match. It does
4072+
/// this by going through operands recursively until it hits the leaf cases:
4073+
/// VPWidenLoadRecipe, VPInterleaveRecipe, and live-ins.
4074+
static bool interleaveStoredValuesMatch(ArrayRef<VPValue *> StoredValues) {
4075+
auto *WideMember0 =
4076+
dyn_cast_or_null<VPWidenRecipe>(StoredValues[0]->getDefiningRecipe());
4077+
if (!WideMember0)
4078+
return false;
4079+
SmallVector<VPValue *> Ops0 = collectOperandsRecursively(WideMember0);
4080+
for (VPValue *ValI : StoredValues) {
4081+
auto *WideMemberI =
4082+
dyn_cast_or_null<VPWidenRecipe>(ValI->getDefiningRecipe());
4083+
if (!WideMemberI || WideMemberI->getOpcode() != WideMember0->getOpcode())
4084+
return false;
4085+
SmallVector<VPValue *> OpsI = collectOperandsRecursively(WideMemberI);
4086+
if (Ops0.size() != OpsI.size())
4087+
return false;
4088+
for (const auto &[Op0, OpI] : zip(Ops0, OpsI)) {
4089+
auto *Def0 = Op0->getDefiningRecipe();
4090+
auto *DefI = OpI->getDefiningRecipe();
4091+
if (!Def0 || !DefI) {
4092+
if (Op0 != OpI)
4093+
return false;
4094+
} else if (Def0->getVPDefID() != DefI->getVPDefID()) {
4095+
return false;
4096+
} else if (auto *W = dyn_cast<VPWidenLoadRecipe>(DefI)) {
4097+
if (W->isMasked() || Op0 != OpI)
4098+
return false;
4099+
} else if (auto *IR = dyn_cast<VPInterleaveRecipe>(DefI)) {
4100+
if (!IR->getInterleaveGroup()->isFull() ||
4101+
!equal(DefI->definedValues(), Def0->definedValues()))
4102+
return false;
4103+
}
4104+
}
4105+
}
4106+
return true;
40824107
}
40834108

40844109
/// Returns true if \p IR is a full interleave group with factor and number of
@@ -4197,24 +4222,9 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
41974222
continue;
41984223
}
41994224

4200-
// Check if all values feeding InterleaveR are matching wide recipes, which
4201-
// operands that can be narrowed.
4202-
auto *WideMember0 = dyn_cast_or_null<VPWidenRecipe>(
4203-
InterleaveR->getStoredValues()[0]->getDefiningRecipe());
4204-
if (!WideMember0)
4225+
// Check if all values feeding InterleaveR match.
4226+
if (!interleaveStoredValuesMatch(InterleaveR->getStoredValues()))
42054227
return;
4206-
for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues())) {
4207-
auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe());
4208-
if (!R || R->getOpcode() != WideMember0->getOpcode() ||
4209-
R->getNumOperands() > 2)
4210-
return;
4211-
if (any_of(enumerate(R->operands()),
4212-
[WideMember0, Idx = I](const auto &P) {
4213-
const auto &[OpIdx, OpV] = P;
4214-
return !canNarrowLoad(WideMember0, OpIdx, OpV, Idx);
4215-
}))
4216-
return;
4217-
}
42184228
StoreGroups.push_back(InterleaveR);
42194229
}
42204230

@@ -4247,7 +4257,11 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
42474257
NarrowedOps.insert(RepR);
42484258
return RepR;
42494259
}
4250-
auto *WideLoad = cast<VPWidenLoadRecipe>(R);
4260+
auto *WideLoad = dyn_cast<VPWidenLoadRecipe>(R);
4261+
if (!WideLoad) {
4262+
NarrowedOps.insert(V);
4263+
return V;
4264+
}
42514265

42524266
VPValue *PtrOp = WideLoad->getAddr();
42534267
if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(PtrOp))

‎llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll‎

Lines changed: 43 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ define void @test_add_double_same_const_args_1(ptr %res, ptr noalias %A, ptr noa
2828
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
2929
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3030
; CHECK: [[MIDDLE_BLOCK]]:
31-
; CHECK-NEXT: br [[EXIT:label %.*]]
32-
; CHECK: [[SCALAR_PH:.*:]]
31+
; CHECK-NEXT: br label %[[EXIT:.*]]
32+
; CHECK: [[EXIT]]:
33+
; CHECK-NEXT: ret void
3334
;
3435
entry:
3536
br label %loop
@@ -76,10 +77,11 @@ define void @test_add_double_same_const_args_2(ptr %res, ptr noalias %A, ptr noa
7677
; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[TMP9]], align 4
7778
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
7879
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
79-
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
80+
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
8081
; CHECK: [[MIDDLE_BLOCK]]:
81-
; CHECK-NEXT: br [[EXIT:label %.*]]
82-
; CHECK: [[SCALAR_PH:.*:]]
82+
; CHECK-NEXT: br label %[[EXIT:.*]]
83+
; CHECK: [[EXIT]]:
84+
; CHECK-NEXT: ret void
8385
;
8486
entry:
8587
br label %loop
@@ -136,10 +138,11 @@ define void @test_add_double_mixed_const_args(ptr %res, ptr noalias %A, ptr noal
136138
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP9]], align 4
137139
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
138140
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
139-
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
141+
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
140142
; CHECK: [[MIDDLE_BLOCK]]:
141-
; CHECK-NEXT: br [[EXIT:label %.*]]
142-
; CHECK: [[SCALAR_PH:.*:]]
143+
; CHECK-NEXT: br label %[[EXIT:.*]]
144+
; CHECK: [[EXIT]]:
145+
; CHECK-NEXT: ret void
143146
;
144147
entry:
145148
br label %loop
@@ -175,33 +178,24 @@ define void @test_add_double_same_var_args_1(ptr %res, ptr noalias %A, ptr noali
175178
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
176179
; CHECK: [[VECTOR_BODY]]:
177180
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
178-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
181+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
179182
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
180183
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
181-
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
182-
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
183-
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
184-
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
185-
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
186-
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
187-
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
188-
; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[STRIDED_VEC3]], [[BROADCAST_SPLAT]]
184+
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP1]], align 4
185+
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
189186
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[STRIDED_VEC1]], [[BROADCAST_SPLAT]]
190187
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[STRIDED_VEC4]], [[BROADCAST_SPLAT]]
191188
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
192189
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
193-
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
194-
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
195-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
196-
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
197-
; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
198-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
199-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
190+
; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4
191+
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
192+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
200193
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
201-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
194+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
202195
; CHECK: [[MIDDLE_BLOCK]]:
203-
; CHECK-NEXT: br [[EXIT:label %.*]]
204-
; CHECK: [[SCALAR_PH:.*:]]
196+
; CHECK-NEXT: br label %[[EXIT:.*]]
197+
; CHECK: [[EXIT]]:
198+
; CHECK-NEXT: ret void
205199
;
206200
entry:
207201
br label %loop
@@ -237,33 +231,24 @@ define void @test_add_double_same_var_args_2(ptr %res, ptr noalias %A, ptr noali
237231
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
238232
; CHECK: [[VECTOR_BODY]]:
239233
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
240-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
234+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
241235
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
242236
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
243-
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
244-
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
245-
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
246-
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
247-
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
248-
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
249-
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
250-
; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC3]]
237+
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP1]], align 4
238+
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
251239
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC1]]
252240
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC4]]
253241
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
254242
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
255-
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
256-
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
257-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
258-
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
259-
; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
260-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
261-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
243+
; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4
244+
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
245+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
262246
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
263-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
247+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
264248
; CHECK: [[MIDDLE_BLOCK]]:
265-
; CHECK-NEXT: br [[EXIT:label %.*]]
266-
; CHECK: [[SCALAR_PH:.*:]]
249+
; CHECK-NEXT: br label %[[EXIT:.*]]
250+
; CHECK: [[EXIT]]:
251+
; CHECK-NEXT: ret void
267252
;
268253
entry:
269254
br label %loop
@@ -322,10 +307,11 @@ define void @test_add_double_same_var_args_at_different_positions(ptr %res, ptr
322307
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
323308
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
324309
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
325-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
310+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
326311
; CHECK: [[MIDDLE_BLOCK]]:
327-
; CHECK-NEXT: br [[EXIT:label %.*]]
328-
; CHECK: [[SCALAR_PH:.*:]]
312+
; CHECK-NEXT: br label %[[EXIT:.*]]
313+
; CHECK: [[EXIT]]:
314+
; CHECK-NEXT: ret void
329315
;
330316
entry:
331317
br label %loop
@@ -386,10 +372,11 @@ define void @test_add_double_different_var_args_1(ptr %res, ptr noalias %A, ptr
386372
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP8]], align 4
387373
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
388374
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
389-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
375+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
390376
; CHECK: [[MIDDLE_BLOCK]]:
391-
; CHECK-NEXT: br [[EXIT:label %.*]]
392-
; CHECK: [[SCALAR_PH:.*:]]
377+
; CHECK-NEXT: br label %[[EXIT:.*]]
378+
; CHECK: [[EXIT]]:
379+
; CHECK-NEXT: ret void
393380
;
394381
entry:
395382
br label %loop
@@ -450,10 +437,11 @@ define void @test_add_double_different_var_args_2(ptr %res, ptr noalias %A, ptr
450437
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP8]], align 4
451438
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
452439
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
453-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
440+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
454441
; CHECK: [[MIDDLE_BLOCK]]:
455-
; CHECK-NEXT: br [[EXIT:label %.*]]
456-
; CHECK: [[SCALAR_PH:.*:]]
442+
; CHECK-NEXT: br label %[[EXIT:.*]]
443+
; CHECK: [[EXIT]]:
444+
; CHECK-NEXT: ret void
457445
;
458446
entry:
459447
br label %loop

0 commit comments

Comments
 (0)