Skip to content

Commit a773d07

Browse files
fhahnfhahn
committed
[LoopPeel] Support peeling last iteration with multiple exits.
Generalize the logic to peel from end to work for multi-exit loops, by checking the exit count of the latch instead of the backedge-taken count (of all exits). This allows peeling quite a few more loops (e.g. 250 loops peeled with the change vs 47 peeled w/o on a IR corpus including SPEC, llvm-test-suite and a few proprietary workloads).
1 parent 3c9812e commit a773d07

File tree

3 files changed

+214
-63
lines changed

3 files changed

+214
-63
lines changed

‎llvm/lib/Transforms/Utils/LoopPeel.cpp‎

Lines changed: 33 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -327,26 +327,27 @@ static unsigned peelToTurnInvariantLoadsDerefencebale(Loop &L,
327327
}
328328

329329
bool llvm::canPeelLastIteration(const Loop &L, ScalarEvolution &SE) {
330-
const SCEV *BTC = SE.getBackedgeTakenCount(&L);
331330
Value *Inc;
332331
CmpPredicate Pred;
333332
BasicBlock *Succ1;
334333
BasicBlock *Succ2;
335-
// The loop must execute at least 2 iterations to guarantee that peeled
336-
// iteration executes.
334+
BasicBlock *Latch = L.getLoopLatch();
335+
// The loop must exit via the latch and additional exits are fine.
336+
if (!Latch || !L.isLoopExiting(Latch))
337+
return false;
338+
339+
// The loop's exit count via the latch must be least 1 to guarantee that
340+
// peeled iteration executes.
337341
// TODO: Add checks during codegen.
338-
if (isa<SCEVCouldNotCompute>(BTC) ||
339-
!SE.isKnownPredicate(CmpInst::ICMP_UGT, BTC, SE.getZero(BTC->getType())))
342+
const SCEV *EC = SE.getExitCount(&L, Latch);
343+
if (isa<SCEVCouldNotCompute>(EC) ||
344+
!SE.isKnownPredicate(CmpInst::ICMP_NE, EC, SE.getZero(EC->getType())))
340345
return false;
341346

342347
// Check if the exit condition of the loop can be adjusted by the peeling
343-
// codegen. For now, it must
344-
// * exit via the latch,
345-
// * the exit condition must be a NE/EQ compare of an induction with step
346-
// of 1 and must only be used by the exiting branch.
347-
BasicBlock *Latch = L.getLoopLatch();
348-
return Latch && Latch == L.getExitingBlock() &&
349-
match(Latch->getTerminator(),
348+
// codegen. For now the exit condition of the latch must be a NE/EQ compare of
349+
// an induction with step of 1 and must only be used by the exiting branch.
350+
return match(Latch->getTerminator(),
350351
m_Br(m_OneUse(m_ICmp(Pred, m_Value(Inc), m_Value())),
351352
m_BasicBlock(Succ1), m_BasicBlock(Succ2))) &&
352353
((Pred == CmpInst::ICMP_EQ && Succ2 == L.getHeader()) ||
@@ -365,10 +366,10 @@ static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
365366
if (!canPeelLastIteration(L, SE))
366367
return false;
367368

368-
const SCEV *BTC = SE.getBackedgeTakenCount(&L);
369-
const SCEV *ValAtLastIter = LeftAR->evaluateAtIteration(BTC, SE);
369+
const SCEV *EC = SE.getExitCount(&L, L.getLoopLatch());
370+
const SCEV *ValAtLastIter = LeftAR->evaluateAtIteration(EC, SE);
370371
const SCEV *ValAtSecondToLastIter = LeftAR->evaluateAtIteration(
371-
SE.getMinusSCEV(BTC, SE.getOne(BTC->getType())), SE);
372+
SE.getMinusSCEV(EC, SE.getOne(EC->getType())), SE);
372373

373374
return SE.isKnownPredicate(ICmpInst::getInversePredicate(Pred), ValAtLastIter,
374375
RightSCEV) &&
@@ -944,6 +945,8 @@ static void cloneLoopBlocks(
944945
// a value coming into the header.
945946
for (auto Edge : ExitEdges)
946947
for (PHINode &PHI : Edge.second->phis()) {
948+
if (PeelLast && Edge.first == Latch)
949+
continue;
947950
Value *LatchVal = PHI.getIncomingValueForBlock(Edge.first);
948951
Instruction *LatchInst = dyn_cast<Instruction>(LatchVal);
949952
if (LatchInst && L->contains(LatchInst))
@@ -1020,7 +1023,6 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
10201023
BasicBlock *PreHeader = L->getLoopPreheader();
10211024
BasicBlock *Latch = L->getLoopLatch();
10221025
SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> ExitEdges;
1023-
L->getExitEdges(ExitEdges);
10241026

10251027
// Remember dominators of blocks we might reach through exits to change them
10261028
// later. Immediate dominator of such block might change, because we add more
@@ -1076,12 +1078,16 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
10761078
// InsertBot:
10771079
// Exit:
10781080
// ...
1079-
BasicBlock *Exit = L->getExitBlock();
1081+
auto *LatchBr = cast<BranchInst>(Latch->getTerminator());
1082+
BasicBlock *Exit = L->contains(LatchBr->getSuccessor(0))
1083+
? LatchBr->getSuccessor(1)
1084+
: LatchBr->getSuccessor(0);
10801085
for (PHINode &P : Exit->phis())
10811086
ExitValues[&P] = P.getIncomingValueForBlock(Latch);
10821087

10831088
InsertTop = SplitEdge(Latch, Exit, &DT, LI);
10841089
InsertBot = SplitBlock(InsertTop, InsertTop->getTerminator(), &DT, LI);
1090+
L->getExitEdges(ExitEdges);
10851091

10861092
InsertTop->setName(Exit->getName() + ".peel.begin");
10871093
InsertBot->setName(Exit->getName() + ".peel.next");
@@ -1138,6 +1144,7 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
11381144
InsertTop->setName(Header->getName() + ".peel.begin");
11391145
InsertBot->setName(Header->getName() + ".peel.next");
11401146
NewPreHeader->setName(PreHeader->getName() + ".peel.newph");
1147+
L->getExitEdges(ExitEdges);
11411148
}
11421149

11431150
Instruction *LatchTerm =
@@ -1211,10 +1218,15 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
12111218
}
12121219

12131220
if (PeelLast) {
1214-
// Now adjust users of the original exit values by replacing them with the
1215-
// exit value from the peeled iteration.
1216-
for (const auto &[P, E] : ExitValues)
1217-
P->replaceAllUsesWith(isa<Constant>(E) ? E : &*VMap.lookup(E));
1221+
if (ExitEdges.size() == 1) {
1222+
// If we have a single existing edge, adjust users of the original exit
1223+
// values by replacing them with the exit value from the peeled iteration.
1224+
// If there are multiple exiting edges, all users outside the loop are
1225+
// served by a common exit block with LCSSA phis that will get updated to
1226+
// use the value from the peeled iteration separately.
1227+
for (const auto &[P, E] : ExitValues)
1228+
P->replaceAllUsesWith(isa<Constant>(E) ? E : &*VMap.lookup(E));
1229+
}
12181230
formLCSSA(*L, DT, LI, SE);
12191231
} else {
12201232
// Now adjust the phi nodes in the loop header to get their initial values

‎llvm/test/Transforms/LoopUnroll/peel-last-iteration-multi-exit.ll‎

Lines changed: 150 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -62,16 +62,35 @@ define void @peel_last_multi_exit_btc_computable_no_exit_values(i32 %n) {
6262
; CHECK-NEXT: [[ENTRY:.*]]:
6363
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
6464
; CHECK: [[LOOP_HEADER]]:
65-
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT_PEEL:%.*]], %[[LOOP_LATCH:.*]] ]
66-
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], [[N]]
67-
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
65+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
66+
; CHECK-NEXT: [[EC_0:%.*]] = icmp eq i32 [[IV]], [[N]]
67+
; CHECK-NEXT: br i1 [[EC_0]], label %[[EXITSPLIT_LOOPEXIT:.*]], label %[[LOOP_LATCH]]
6868
; CHECK: [[LOOP_LATCH]]:
69+
; CHECK-NEXT: call void @foo(i32 20)
70+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
71+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 16
72+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_PEEL_BEGIN:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
73+
; CHECK: [[EXIT_PEEL_BEGIN]]:
74+
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP_LATCH]] ]
75+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL:.*]]
76+
; CHECK: [[LOOP_HEADER_PEEL]]:
77+
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], [[N]]
78+
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[EXITSPLIT:.*]], label %[[LOOP_LATCH_PEEL:.*]]
79+
; CHECK: [[LOOP_LATCH_PEEL]]:
6980
; CHECK-NEXT: [[C_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], 16
7081
; CHECK-NEXT: [[COND_PEEL:%.*]] = select i1 [[C_PEEL]], i32 10, i32 20
7182
; CHECK-NEXT: call void @foo(i32 [[COND_PEEL]])
72-
; CHECK-NEXT: [[IV_NEXT_PEEL]] = add i32 [[IV_NEXT_LCSSA]], 1
83+
; CHECK-NEXT: [[IV_NEXT_PEEL:%.*]] = add i32 [[IV_NEXT_LCSSA]], 1
7384
; CHECK-NEXT: [[EC_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_PEEL]], 17
74-
; CHECK-NEXT: br i1 [[EC_PEEL]], label %[[EXIT]], label %[[LOOP_HEADER]]
85+
; CHECK-NEXT: br i1 [[EC_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
86+
; CHECK: [[EXIT_PEEL_NEXT]]:
87+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL_NEXT:.*]]
88+
; CHECK: [[LOOP_HEADER_PEEL_NEXT]]:
89+
; CHECK-NEXT: br label %[[EXIT:.*]]
90+
; CHECK: [[EXITSPLIT_LOOPEXIT]]:
91+
; CHECK-NEXT: br label %[[EXITSPLIT]]
92+
; CHECK: [[EXITSPLIT]]:
93+
; CHECK-NEXT: br label %[[EXIT]]
7594
; CHECK: [[EXIT]]:
7695
; CHECK-NEXT: ret void
7796
;
@@ -101,18 +120,40 @@ define i32 @peel_last_multi_exit_btc_computable_exit_constant_values(i32 %n) {
101120
; CHECK-NEXT: [[ENTRY:.*]]:
102121
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
103122
; CHECK: [[LOOP_HEADER]]:
104-
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT_PEEL:%.*]], %[[LOOP_LATCH:.*]] ]
105-
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], [[N]]
106-
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
123+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
124+
; CHECK-NEXT: [[EC_0:%.*]] = icmp eq i32 [[IV]], [[N]]
125+
; CHECK-NEXT: br i1 [[EC_0]], label %[[EXITSPLIT_LOOPEXIT:.*]], label %[[LOOP_LATCH]]
107126
; CHECK: [[LOOP_LATCH]]:
127+
; CHECK-NEXT: call void @foo(i32 20)
128+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
129+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 16
130+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_PEEL_BEGIN:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
131+
; CHECK: [[EXIT_PEEL_BEGIN]]:
132+
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP_LATCH]] ]
133+
; CHECK-NEXT: [[SPLIT:%.*]] = phi i32 [ 2, %[[LOOP_LATCH]] ]
134+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL:.*]]
135+
; CHECK: [[LOOP_HEADER_PEEL]]:
136+
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], [[N]]
137+
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[EXITSPLIT:.*]], label %[[LOOP_LATCH_PEEL:.*]]
138+
; CHECK: [[LOOP_LATCH_PEEL]]:
108139
; CHECK-NEXT: [[C_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], 16
109140
; CHECK-NEXT: [[COND_PEEL:%.*]] = select i1 [[C_PEEL]], i32 10, i32 20
110141
; CHECK-NEXT: call void @foo(i32 [[COND_PEEL]])
111-
; CHECK-NEXT: [[IV_NEXT_PEEL]] = add i32 [[IV_NEXT_LCSSA]], 1
142+
; CHECK-NEXT: [[IV_NEXT_PEEL:%.*]] = add i32 [[IV_NEXT_LCSSA]], 1
112143
; CHECK-NEXT: [[EC_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_PEEL]], 17
113-
; CHECK-NEXT: br i1 [[EC_PEEL]], label %[[EXIT]], label %[[LOOP_HEADER]]
144+
; CHECK-NEXT: br i1 [[EC_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
145+
; CHECK: [[EXIT_PEEL_NEXT]]:
146+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL_NEXT:.*]]
147+
; CHECK: [[LOOP_HEADER_PEEL_NEXT]]:
148+
; CHECK-NEXT: br label %[[EXIT:.*]]
149+
; CHECK: [[EXITSPLIT_LOOPEXIT]]:
150+
; CHECK-NEXT: [[RES_PH_PH:%.*]] = phi i32 [ 1, %[[LOOP_HEADER]] ]
151+
; CHECK-NEXT: br label %[[EXITSPLIT]]
152+
; CHECK: [[EXITSPLIT]]:
153+
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ 1, %[[LOOP_HEADER_PEEL]] ], [ [[RES_PH_PH]], %[[EXITSPLIT_LOOPEXIT]] ]
154+
; CHECK-NEXT: br label %[[EXIT]]
114155
; CHECK: [[EXIT]]:
115-
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 1, %[[LOOP_HEADER]] ], [ 2, %[[LOOP_LATCH]] ]
156+
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[SPLIT]], %[[LOOP_HEADER_PEEL_NEXT]] ], [ [[RES_PH]], %[[EXITSPLIT]] ]
116157
; CHECK-NEXT: ret i32 [[RES]]
117158
;
118159
entry:
@@ -142,18 +183,40 @@ define i32 @peel_last_multi_exit_btc_computable_exit_values_from_loop(i32 %n) {
142183
; CHECK-NEXT: [[ENTRY:.*]]:
143184
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
144185
; CHECK: [[LOOP_HEADER]]:
145-
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT_PEEL:%.*]], %[[LOOP_LATCH:.*]] ]
146-
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], [[N]]
147-
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
186+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
187+
; CHECK-NEXT: [[EC_0:%.*]] = icmp eq i32 [[IV]], [[N]]
188+
; CHECK-NEXT: br i1 [[EC_0]], label %[[EXITSPLIT_LOOPEXIT:.*]], label %[[LOOP_LATCH]]
148189
; CHECK: [[LOOP_LATCH]]:
190+
; CHECK-NEXT: call void @foo(i32 20)
191+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
192+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 16
193+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_PEEL_BEGIN:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
194+
; CHECK: [[EXIT_PEEL_BEGIN]]:
195+
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP_LATCH]] ]
196+
; CHECK-NEXT: [[SPLIT:%.*]] = phi i32 [ 20, %[[LOOP_LATCH]] ]
197+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL:.*]]
198+
; CHECK: [[LOOP_HEADER_PEEL]]:
199+
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], [[N]]
200+
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[EXITSPLIT:.*]], label %[[LOOP_LATCH_PEEL:.*]]
201+
; CHECK: [[LOOP_LATCH_PEEL]]:
149202
; CHECK-NEXT: [[C_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], 16
150203
; CHECK-NEXT: [[COND_PEEL:%.*]] = select i1 [[C_PEEL]], i32 10, i32 20
151204
; CHECK-NEXT: call void @foo(i32 [[COND_PEEL]])
152-
; CHECK-NEXT: [[IV_NEXT_PEEL]] = add i32 [[IV_NEXT_LCSSA]], 1
205+
; CHECK-NEXT: [[IV_NEXT_PEEL:%.*]] = add i32 [[IV_NEXT_LCSSA]], 1
153206
; CHECK-NEXT: [[EC_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_PEEL]], 17
154-
; CHECK-NEXT: br i1 [[EC_PEEL]], label %[[EXIT]], label %[[LOOP_HEADER]]
207+
; CHECK-NEXT: br i1 [[EC_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
208+
; CHECK: [[EXIT_PEEL_NEXT]]:
209+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL_NEXT:.*]]
210+
; CHECK: [[LOOP_HEADER_PEEL_NEXT]]:
211+
; CHECK-NEXT: br label %[[EXIT:.*]]
212+
; CHECK: [[EXITSPLIT_LOOPEXIT]]:
213+
; CHECK-NEXT: [[RES_PH_PH:%.*]] = phi i32 [ [[IV]], %[[LOOP_HEADER]] ]
214+
; CHECK-NEXT: br label %[[EXITSPLIT]]
215+
; CHECK: [[EXITSPLIT]]:
216+
; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[IV_NEXT_LCSSA]], %[[LOOP_HEADER_PEEL]] ], [ [[RES_PH_PH]], %[[EXITSPLIT_LOOPEXIT]] ]
217+
; CHECK-NEXT: br label %[[EXIT]]
155218
; CHECK: [[EXIT]]:
156-
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[IV_NEXT_LCSSA]], %[[LOOP_HEADER]] ], [ [[COND_PEEL]], %[[LOOP_LATCH]] ]
219+
; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[SPLIT]], %[[LOOP_HEADER_PEEL_NEXT]] ], [ [[RES_PH]], %[[EXITSPLIT]] ]
157220
; CHECK-NEXT: ret i32 [[RES]]
158221
;
159222
entry:
@@ -183,21 +246,39 @@ define i32 @peel_last_multi_exit_btc_computable_exit_values_from_loop_multiple_e
183246
; CHECK-NEXT: [[ENTRY:.*]]:
184247
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
185248
; CHECK: [[LOOP_HEADER]]:
186-
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
249+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT1:%.*]], %[[LOOP_LATCH:.*]] ]
187250
; CHECK-NEXT: [[EC_0:%.*]] = icmp eq i32 [[IV]], [[N]]
188-
; CHECK-NEXT: br i1 [[EC_0]], label %[[EXIT_0:.*]], label %[[LOOP_LATCH]]
251+
; CHECK-NEXT: br i1 [[EC_0]], label %[[EXIT_0_LOOPEXIT:.*]], label %[[LOOP_LATCH]]
189252
; CHECK: [[LOOP_LATCH]]:
190-
; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[IV]], 16
253+
; CHECK-NEXT: call void @foo(i32 20)
254+
; CHECK-NEXT: [[IV_NEXT1]] = add nuw nsw i32 [[IV]], 1
255+
; CHECK-NEXT: [[EC1:%.*]] = icmp eq i32 [[IV_NEXT1]], 16
256+
; CHECK-NEXT: br i1 [[EC1]], label %[[EXIT_1_PEEL_BEGIN:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
257+
; CHECK: [[EXIT_0_LOOPEXIT]]:
258+
; CHECK-NEXT: [[RES_0_PH:%.*]] = phi i32 [ [[IV]], %[[LOOP_HEADER]] ]
259+
; CHECK-NEXT: br label %[[EXIT_0:.*]]
260+
; CHECK: [[EXIT_0]]:
261+
; CHECK-NEXT: [[RES_0:%.*]] = phi i32 [ [[IV_NEXT_LCSSA:%.*]], %[[LOOP_HEADER_PEEL:.*]] ], [ [[RES_0_PH]], %[[EXIT_0_LOOPEXIT]] ]
262+
; CHECK-NEXT: ret i32 [[RES_0]]
263+
; CHECK: [[EXIT_1_PEEL_BEGIN]]:
264+
; CHECK-NEXT: [[IV_NEXT_LCSSA]] = phi i32 [ [[IV_NEXT1]], %[[LOOP_LATCH]] ]
265+
; CHECK-NEXT: [[RES_1:%.*]] = phi i32 [ 20, %[[LOOP_LATCH]] ]
266+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL]]
267+
; CHECK: [[LOOP_HEADER_PEEL]]:
268+
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], [[N]]
269+
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[EXIT_0]], label %[[LOOP_LATCH_PEEL:.*]]
270+
; CHECK: [[LOOP_LATCH_PEEL]]:
271+
; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[IV_NEXT_LCSSA]], 16
191272
; CHECK-NEXT: [[COND:%.*]] = select i1 [[C]], i32 10, i32 20
192273
; CHECK-NEXT: call void @foo(i32 [[COND]])
193-
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
274+
; CHECK-NEXT: [[IV_NEXT:%.*]] = add i32 [[IV_NEXT_LCSSA]], 1
194275
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 17
195-
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_1:.*]], label %[[LOOP_HEADER]]
196-
; CHECK: [[EXIT_0]]:
197-
; CHECK-NEXT: [[RES_0:%.*]] = phi i32 [ [[IV]], %[[LOOP_HEADER]] ]
198-
; CHECK-NEXT: ret i32 [[RES_0]]
276+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_1_PEEL_NEXT:.*]], label %[[EXIT_1_PEEL_NEXT]]
277+
; CHECK: [[EXIT_1_PEEL_NEXT]]:
278+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL_NEXT:.*]]
279+
; CHECK: [[LOOP_HEADER_PEEL_NEXT]]:
280+
; CHECK-NEXT: br label %[[EXIT_1:.*]]
199281
; CHECK: [[EXIT_1]]:
200-
; CHECK-NEXT: [[RES_1:%.*]] = phi i32 [ [[COND]], %[[LOOP_LATCH]] ]
201282
; CHECK-NEXT: ret i32 [[RES_1]]
202283
;
203284
entry:
@@ -230,22 +311,49 @@ define i64 @peel_last_btc_not_computable() {
230311
; CHECK-NEXT: [[ENTRY:.*]]:
231312
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
232313
; CHECK: [[LOOP_HEADER]]:
233-
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT_PEEL:%.*]], %[[LOOP_LATCH:.*]] ]
234-
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = call i1 @cond()
235-
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[THEN_1:.*]], label %[[EXIT:.*]]
314+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
315+
; CHECK-NEXT: [[EC_0:%.*]] = call i1 @cond()
316+
; CHECK-NEXT: br i1 [[EC_0]], label %[[THEN_1:.*]], label %[[EXITSPLIT_LOOPEXIT:.*]]
236317
; CHECK: [[THEN_1]]:
237318
; CHECK-NEXT: call void @foo(i32 1)
238-
; CHECK-NEXT: [[C_PEEL:%.*]] = icmp eq i64 [[IV_NEXT_LCSSA]], 7
239-
; CHECK-NEXT: br i1 [[C_PEEL]], label %[[LOOP_LATCH]], label %[[THEN_2:.*]]
319+
; CHECK-NEXT: br i1 false, label %[[LOOP_LATCH]], label %[[THEN_2:.*]]
240320
; CHECK: [[THEN_2]]:
241321
; CHECK-NEXT: call void @foo(i32 2)
242322
; CHECK-NEXT: br label %[[LOOP_LATCH]]
243323
; CHECK: [[LOOP_LATCH]]:
244-
; CHECK-NEXT: [[IV_NEXT_PEEL]] = add i64 [[IV_NEXT_LCSSA]], 1
324+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
325+
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], 7
326+
; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT_PEEL_BEGIN:.*]], !llvm.loop [[LOOP5:![0-9]+]]
327+
; CHECK: [[EXIT_PEEL_BEGIN]]:
328+
; CHECK-NEXT: [[IV_NEXT_LCSSA:%.*]] = phi i64 [ [[IV_NEXT]], %[[LOOP_LATCH]] ]
329+
; CHECK-NEXT: [[SPLIT:%.*]] = phi i64 [ 1, %[[LOOP_LATCH]] ]
330+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL:.*]]
331+
; CHECK: [[LOOP_HEADER_PEEL]]:
332+
; CHECK-NEXT: [[EC_0_PEEL:%.*]] = call i1 @cond()
333+
; CHECK-NEXT: br i1 [[EC_0_PEEL]], label %[[THEN_1_PEEL:.*]], label %[[EXITSPLIT:.*]]
334+
; CHECK: [[THEN_1_PEEL]]:
335+
; CHECK-NEXT: call void @foo(i32 1)
336+
; CHECK-NEXT: [[C_PEEL:%.*]] = icmp eq i64 [[IV_NEXT_LCSSA]], 7
337+
; CHECK-NEXT: br i1 [[C_PEEL]], label %[[LOOP_LATCH_PEEL:.*]], label %[[THEN_2_PEEL:.*]]
338+
; CHECK: [[THEN_2_PEEL]]:
339+
; CHECK-NEXT: call void @foo(i32 2)
340+
; CHECK-NEXT: br label %[[LOOP_LATCH_PEEL]]
341+
; CHECK: [[LOOP_LATCH_PEEL]]:
342+
; CHECK-NEXT: [[IV_NEXT_PEEL:%.*]] = add i64 [[IV_NEXT_LCSSA]], 1
245343
; CHECK-NEXT: [[EXITCOND_PEEL:%.*]] = icmp ne i64 [[IV_NEXT_PEEL]], 8
246-
; CHECK-NEXT: br i1 [[EXITCOND_PEEL]], label %[[LOOP_HEADER]], label %[[EXIT]]
344+
; CHECK-NEXT: br i1 [[EXITCOND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
345+
; CHECK: [[EXIT_PEEL_NEXT]]:
346+
; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL_NEXT:.*]]
347+
; CHECK: [[LOOP_HEADER_PEEL_NEXT]]:
348+
; CHECK-NEXT: br label %[[EXIT:.*]]
349+
; CHECK: [[EXITSPLIT_LOOPEXIT]]:
350+
; CHECK-NEXT: [[RES_PH_PH:%.*]] = phi i64 [ 2, %[[LOOP_HEADER]] ]
351+
; CHECK-NEXT: br label %[[EXITSPLIT]]
352+
; CHECK: [[EXITSPLIT]]:
353+
; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ 2, %[[LOOP_HEADER_PEEL]] ], [ [[RES_PH_PH]], %[[EXITSPLIT_LOOPEXIT]] ]
354+
; CHECK-NEXT: br label %[[EXIT]]
247355
; CHECK: [[EXIT]]:
248-
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ 1, %[[LOOP_LATCH]] ], [ 2, %[[LOOP_HEADER]] ]
356+
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[SPLIT]], %[[LOOP_HEADER_PEEL_NEXT]] ], [ [[RES_PH]], %[[EXITSPLIT]] ]
249357
; CHECK-NEXT: ret i64 [[RES]]
250358
;
251359
entry:
@@ -313,3 +421,11 @@ loop.latch:
313421
exit:
314422
ret void
315423
}
424+
;.
425+
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
426+
; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
427+
; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
428+
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
429+
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
430+
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
431+
;.

0 commit comments

Comments
 (0)