Skip to content

Commit fafb462

Browse files
artagnonartagnon
committed
[VPlan] Recursively match operands of interleave group
When narrowing interleave groups, we use the canNarrowLoad check, which bails out when there any recipe that is not a VPWidenLoad, VPWidenInterleave, or live-in feeding the interleave: a lot of potential narrowing opportunities are missed as a result. Correctly identify that these three cases are the leaf cases, and match the recursive operands instead. Fixes #128062.
1 parent 0b4c451 commit fafb462

File tree

5 files changed

+201
-201
lines changed

5 files changed

+201
-201
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 51 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,14 @@ static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
628628
return Users.takeVector();
629629
}
630630

631+
static SmallVector<VPValue *> collectOperandsRecursively(VPRecipeBase *R) {
632+
SetVector<VPValue *> Operands(llvm::from_range, R->operands());
633+
for (unsigned I = 0; I != Operands.size(); ++I)
634+
if (auto *Cur = Operands[I]->getDefiningRecipe())
635+
Operands.insert_range(Cur->operands());
636+
return Operands.takeVector();
637+
}
638+
631639
/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
632640
/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
633641
/// VPWidenPointerInductionRecipe will generate vectors only. If some users
@@ -4064,25 +4072,42 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
40644072
return ExpandedSCEVs;
40654073
}
40664074

4067-
/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
4068-
/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
4069-
/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
4070-
/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
4071-
/// an index-independent load if it feeds all wide ops at all indices (\p OpV
4072-
/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
4073-
/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
4074-
/// is defined at \p Idx of a load interleave group.
4075-
static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx,
4076-
VPValue *OpV, unsigned Idx) {
4077-
auto *DefR = OpV->getDefiningRecipe();
4078-
if (!DefR)
4079-
return WideMember0->getOperand(OpIdx) == OpV;
4080-
if (auto *W = dyn_cast<VPWidenLoadRecipe>(DefR))
4081-
return !W->getMask() && WideMember0->getOperand(OpIdx) == OpV;
4082-
4083-
if (auto *IR = dyn_cast<VPInterleaveRecipe>(DefR))
4084-
return IR->getInterleaveGroup()->isFull() && IR->getVPValue(Idx) == OpV;
4085-
return false;
4075+
/// Returns true if the \p StoredValues of an interleave group match. It does
4076+
/// this by going through operands recursively until it hits the leaf cases:
4077+
/// VPWidenLoadRecipe, VPInterleaveRecipe, and live-ins.
4078+
static bool interleaveStoredValuesMatch(ArrayRef<VPValue *> StoredValues) {
4079+
auto *WideMember0 =
4080+
dyn_cast_or_null<VPWidenRecipe>(StoredValues[0]->getDefiningRecipe());
4081+
if (!WideMember0)
4082+
return false;
4083+
SmallVector<VPValue *> Ops0 = collectOperandsRecursively(WideMember0);
4084+
for (VPValue *ValI : StoredValues) {
4085+
auto *WideMemberI =
4086+
dyn_cast_or_null<VPWidenRecipe>(ValI->getDefiningRecipe());
4087+
if (!WideMemberI || WideMemberI->getOpcode() != WideMember0->getOpcode())
4088+
return false;
4089+
SmallVector<VPValue *> OpsI = collectOperandsRecursively(WideMemberI);
4090+
if (Ops0.size() != OpsI.size())
4091+
return false;
4092+
for (const auto &[Op0, OpI] : zip(Ops0, OpsI)) {
4093+
auto *Def0 = Op0->getDefiningRecipe();
4094+
auto *DefI = OpI->getDefiningRecipe();
4095+
if (!Def0 || !DefI) {
4096+
if (Op0 != OpI)
4097+
return false;
4098+
} else if (Def0->getVPDefID() != DefI->getVPDefID()) {
4099+
return false;
4100+
} else if (auto *W = dyn_cast<VPWidenLoadRecipe>(DefI)) {
4101+
if (W->isMasked() || Op0 != OpI)
4102+
return false;
4103+
} else if (auto *IR = dyn_cast<VPInterleaveRecipe>(DefI)) {
4104+
if (!IR->getInterleaveGroup()->isFull() ||
4105+
!equal(DefI->definedValues(), Def0->definedValues()))
4106+
return false;
4107+
}
4108+
}
4109+
}
4110+
return true;
40864111
}
40874112

40884113
/// Returns true if \p IR is a full interleave group with factor and number of
@@ -4201,24 +4226,9 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
42014226
continue;
42024227
}
42034228

4204-
// Check if all values feeding InterleaveR are matching wide recipes, which
4205-
// operands that can be narrowed.
4206-
auto *WideMember0 = dyn_cast_or_null<VPWidenRecipe>(
4207-
InterleaveR->getStoredValues()[0]->getDefiningRecipe());
4208-
if (!WideMember0)
4229+
// Check if all values feeding InterleaveR match.
4230+
if (!interleaveStoredValuesMatch(InterleaveR->getStoredValues()))
42094231
return;
4210-
for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues())) {
4211-
auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe());
4212-
if (!R || R->getOpcode() != WideMember0->getOpcode() ||
4213-
R->getNumOperands() > 2)
4214-
return;
4215-
if (any_of(enumerate(R->operands()),
4216-
[WideMember0, Idx = I](const auto &P) {
4217-
const auto &[OpIdx, OpV] = P;
4218-
return !canNarrowLoad(WideMember0, OpIdx, OpV, Idx);
4219-
}))
4220-
return;
4221-
}
42224232
StoreGroups.push_back(InterleaveR);
42234233
}
42244234

@@ -4250,7 +4260,11 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
42504260
NarrowedOps.insert(RepR);
42514261
return RepR;
42524262
}
4253-
auto *WideLoad = cast<VPWidenLoadRecipe>(R);
4263+
auto *WideLoad = dyn_cast<VPWidenLoadRecipe>(R);
4264+
if (!WideLoad) {
4265+
NarrowedOps.insert(V);
4266+
return V;
4267+
}
42544268

42554269
VPValue *PtrOp = WideLoad->getAddr();
42564270
if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(PtrOp))

‎llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-constant-ops.ll‎

Lines changed: 43 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ define void @test_add_double_same_const_args_1(ptr %res, ptr noalias %A, ptr noa
2828
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
2929
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3030
; CHECK: [[MIDDLE_BLOCK]]:
31-
; CHECK-NEXT: br [[EXIT:label %.*]]
32-
; CHECK: [[SCALAR_PH:.*:]]
31+
; CHECK-NEXT: br label %[[EXIT:.*]]
32+
; CHECK: [[EXIT]]:
33+
; CHECK-NEXT: ret void
3334
;
3435
entry:
3536
br label %loop
@@ -76,10 +77,11 @@ define void @test_add_double_same_const_args_2(ptr %res, ptr noalias %A, ptr noa
7677
; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[TMP9]], align 4
7778
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
7879
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
79-
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
80+
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
8081
; CHECK: [[MIDDLE_BLOCK]]:
81-
; CHECK-NEXT: br [[EXIT:label %.*]]
82-
; CHECK: [[SCALAR_PH:.*:]]
82+
; CHECK-NEXT: br label %[[EXIT:.*]]
83+
; CHECK: [[EXIT]]:
84+
; CHECK-NEXT: ret void
8385
;
8486
entry:
8587
br label %loop
@@ -136,10 +138,11 @@ define void @test_add_double_mixed_const_args(ptr %res, ptr noalias %A, ptr noal
136138
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP9]], align 4
137139
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
138140
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
139-
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
141+
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
140142
; CHECK: [[MIDDLE_BLOCK]]:
141-
; CHECK-NEXT: br [[EXIT:label %.*]]
142-
; CHECK: [[SCALAR_PH:.*:]]
143+
; CHECK-NEXT: br label %[[EXIT:.*]]
144+
; CHECK: [[EXIT]]:
145+
; CHECK-NEXT: ret void
143146
;
144147
entry:
145148
br label %loop
@@ -175,33 +178,24 @@ define void @test_add_double_same_var_args_1(ptr %res, ptr noalias %A, ptr noali
175178
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
176179
; CHECK: [[VECTOR_BODY]]:
177180
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
178-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
181+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
179182
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
180183
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
181-
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
182-
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
183-
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
184-
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
185-
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
186-
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
187-
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
188-
; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[STRIDED_VEC3]], [[BROADCAST_SPLAT]]
184+
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP1]], align 4
185+
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
189186
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[STRIDED_VEC1]], [[BROADCAST_SPLAT]]
190187
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[STRIDED_VEC4]], [[BROADCAST_SPLAT]]
191188
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
192189
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
193-
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
194-
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
195-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
196-
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
197-
; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
198-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
199-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
190+
; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4
191+
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
192+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
200193
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
201-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
194+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
202195
; CHECK: [[MIDDLE_BLOCK]]:
203-
; CHECK-NEXT: br [[EXIT:label %.*]]
204-
; CHECK: [[SCALAR_PH:.*:]]
196+
; CHECK-NEXT: br label %[[EXIT:.*]]
197+
; CHECK: [[EXIT]]:
198+
; CHECK-NEXT: ret void
205199
;
206200
entry:
207201
br label %loop
@@ -237,33 +231,24 @@ define void @test_add_double_same_var_args_2(ptr %res, ptr noalias %A, ptr noali
237231
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
238232
; CHECK: [[VECTOR_BODY]]:
239233
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
240-
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 2
234+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
241235
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[INDEX]]
242236
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP0]]
243-
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP1]], align 4
244-
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
245-
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
246-
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
247-
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
248-
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
249-
; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
250-
; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC3]]
237+
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x double>, ptr [[TMP1]], align 4
238+
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
251239
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC1]]
252240
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC4]]
253241
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[INDEX]]
254242
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
255-
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
256-
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
257-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
258-
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
259-
; CHECK-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
260-
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
261-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
243+
; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP7]], align 4
244+
; CHECK-NEXT: store <2 x double> [[TMP6]], ptr [[TMP8]], align 4
245+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
262246
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
263-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
247+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
264248
; CHECK: [[MIDDLE_BLOCK]]:
265-
; CHECK-NEXT: br [[EXIT:label %.*]]
266-
; CHECK: [[SCALAR_PH:.*:]]
249+
; CHECK-NEXT: br label %[[EXIT:.*]]
250+
; CHECK: [[EXIT]]:
251+
; CHECK-NEXT: ret void
267252
;
268253
entry:
269254
br label %loop
@@ -322,10 +307,11 @@ define void @test_add_double_same_var_args_at_different_positions(ptr %res, ptr
322307
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP8]], align 4
323308
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
324309
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
325-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
310+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
326311
; CHECK: [[MIDDLE_BLOCK]]:
327-
; CHECK-NEXT: br [[EXIT:label %.*]]
328-
; CHECK: [[SCALAR_PH:.*:]]
312+
; CHECK-NEXT: br label %[[EXIT:.*]]
313+
; CHECK: [[EXIT]]:
314+
; CHECK-NEXT: ret void
329315
;
330316
entry:
331317
br label %loop
@@ -386,10 +372,11 @@ define void @test_add_double_different_var_args_1(ptr %res, ptr noalias %A, ptr
386372
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP8]], align 4
387373
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
388374
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
389-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
375+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
390376
; CHECK: [[MIDDLE_BLOCK]]:
391-
; CHECK-NEXT: br [[EXIT:label %.*]]
392-
; CHECK: [[SCALAR_PH:.*:]]
377+
; CHECK-NEXT: br label %[[EXIT:.*]]
378+
; CHECK: [[EXIT]]:
379+
; CHECK-NEXT: ret void
393380
;
394381
entry:
395382
br label %loop
@@ -450,10 +437,11 @@ define void @test_add_double_different_var_args_2(ptr %res, ptr noalias %A, ptr
450437
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC7]], ptr [[TMP8]], align 4
451438
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
452439
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
453-
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
440+
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
454441
; CHECK: [[MIDDLE_BLOCK]]:
455-
; CHECK-NEXT: br [[EXIT:label %.*]]
456-
; CHECK: [[SCALAR_PH:.*:]]
442+
; CHECK-NEXT: br label %[[EXIT:.*]]
443+
; CHECK: [[EXIT]]:
444+
; CHECK-NEXT: ret void
457445
;
458446
entry:
459447
br label %loop

0 commit comments

Comments
 (0)