diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp index bf882d7406853..6312831cf0ee0 100644 --- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp @@ -201,18 +201,27 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count, /// unroll count is non-zero. /// /// This function performs the following: -/// - Update PHI nodes at the unrolling loop exit and epilog loop exit -/// - Create PHI nodes at the unrolling loop exit to combine -/// values that exit the unrolling loop code and jump around it. +/// - Update PHI nodes at the epilog loop exit +/// - Create PHI nodes at the unrolling loop exit and epilog preheader to +/// combine values that exit the unrolling loop code and jump around it. /// - Update PHI operands in the epilog loop by the new PHI nodes -/// - Branch around the epilog loop if extra iters (ModVal) is zero. +/// - At the unrolling loop exit, branch around the epilog loop if extra iters +// (ModVal) is zero. +/// - At the epilog preheader, add an llvm.assume call that extra iters is +/// non-zero. If the unrolling loop exit is the predecessor, the above new +/// branch guarantees that assumption. If the unrolling loop preheader is the +/// predecessor, then the required first iteration from the original loop has +/// yet to be executed, so it must be executed in the epilog loop. If we +/// later unroll the epilog loop, that llvm.assume call somehow enables +/// ScalarEvolution to compute a epilog loop maximum trip count, which enables +/// eliminating the branch at the end of the final unrolled epilog iteration. /// static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, BasicBlock *Exit, BasicBlock *PreHeader, BasicBlock *EpilogPreHeader, BasicBlock *NewPreHeader, ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA, ScalarEvolution &SE, - unsigned Count) { + unsigned Count, AssumptionCache &AC) { BasicBlock *Latch = L->getLoopLatch(); assert(Latch && "Loop must have a latch"); BasicBlock *EpilogLatch = cast(VMap[Latch]); @@ -231,7 +240,7 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, // EpilogLatch // Exit (EpilogPN) - // Update PHI nodes at NewExit and Exit. + // Update PHI nodes at Exit. for (PHINode &PN : NewExit->phis()) { // PN should be used in another PHI located in Exit block as // Exit was split by SplitBlockPredecessors into Exit and NewExit @@ -246,15 +255,11 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, // epilogue edges have already been added. // // There is EpilogPreHeader incoming block instead of NewExit as - // NewExit was spilt 1 more time to get EpilogPreHeader. + // NewExit was split 1 more time to get EpilogPreHeader. assert(PN.hasOneUse() && "The phi should have 1 use"); PHINode *EpilogPN = cast(PN.use_begin()->getUser()); assert(EpilogPN->getParent() == Exit && "EpilogPN should be in Exit block"); - // Add incoming PreHeader from branch around the Loop - PN.addIncoming(PoisonValue::get(PN.getType()), PreHeader); - SE.forgetValue(&PN); - Value *V = PN.getIncomingValueForBlock(Latch); Instruction *I = dyn_cast(V); if (I && L->contains(I)) @@ -271,35 +276,52 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, NewExit); // Now PHIs should look like: // NewExit: - // PN = PHI [I, Latch], [poison, PreHeader] + // PN = PHI [I, Latch] // ... // Exit: // EpilogPN = PHI [PN, NewExit], [VMap[I], EpilogLatch] } - // Create PHI nodes at NewExit (from the unrolling loop Latch and PreHeader). - // Update corresponding PHI nodes in epilog loop. + // Create PHI nodes at NewExit (from the unrolling loop Latch) and at + // EpilogPreHeader (from PreHeader and NewExit). Update corresponding PHI + // nodes in epilog loop. for (BasicBlock *Succ : successors(Latch)) { // Skip this as we already updated phis in exit blocks. if (!L->contains(Succ)) continue; + + // Succ here appears to always be just L->getHeader(). Otherwise, how do we + // know its corresponding epilog block (from VMap) is EpilogHeader and thus + // EpilogPreHeader is the right incoming block for VPN, as set below? + // TODO: Can we thus avoid the enclosing loop over successors? + assert(Succ == L->getHeader() && + "Expect the only in-loop successor of latch to be the loop header"); + for (PHINode &PN : Succ->phis()) { - // Add new PHI nodes to the loop exit block and update epilog - // PHIs with the new PHI values. - PHINode *NewPN = PHINode::Create(PN.getType(), 2, PN.getName() + ".unr"); - NewPN->insertBefore(NewExit->getFirstNonPHIIt()); - // Adding a value to the new PHI node from the unrolling loop preheader. - NewPN->addIncoming(PN.getIncomingValueForBlock(NewPreHeader), PreHeader); - // Adding a value to the new PHI node from the unrolling loop latch. - NewPN->addIncoming(PN.getIncomingValueForBlock(Latch), Latch); + // Add new PHI nodes to the loop exit block. + PHINode *NewPN0 = PHINode::Create(PN.getType(), /*NumReservedValues=*/1, + PN.getName() + ".unr"); + NewPN0->insertBefore(NewExit->getFirstNonPHIIt()); + // Add value to the new PHI node from the unrolling loop latch. + NewPN0->addIncoming(PN.getIncomingValueForBlock(Latch), Latch); + + // Add new PHI nodes to EpilogPreHeader. + PHINode *NewPN1 = PHINode::Create(PN.getType(), /*NumReservedValues=*/2, + PN.getName() + ".epil.init"); + NewPN1->insertBefore(EpilogPreHeader->getFirstNonPHIIt()); + // Add value to the new PHI node from the unrolling loop preheader. + NewPN1->addIncoming(PN.getIncomingValueForBlock(NewPreHeader), PreHeader); + // Add value to the new PHI node from the epilog loop guard. + NewPN1->addIncoming(NewPN0, NewExit); // Update the existing PHI node operand with the value from the new PHI // node. Corresponding instruction in epilog loop should be PHI. PHINode *VPN = cast(VMap[&PN]); - VPN->setIncomingValueForBlock(EpilogPreHeader, NewPN); + VPN->setIncomingValueForBlock(EpilogPreHeader, NewPN1); } } + // In NewExit, branch around the epilog loop if no extra iters. Instruction *InsertPt = NewExit->getTerminator(); IRBuilder<> B(InsertPt); Value *BrLoopExit = B.CreateIsNotNull(ModVal, "lcmp.mod"); @@ -308,7 +330,7 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, SmallVector Preds(predecessors(Exit)); SplitBlockPredecessors(Exit, Preds, ".epilog-lcssa", DT, LI, nullptr, PreserveLCSSA); - // Add the branch to the exit block (around the unrolling loop) + // Add the branch to the exit block (around the epilog loop) MDNode *BranchWeights = nullptr; if (hasBranchWeightMD(*Latch->getTerminator())) { // Assume equal distribution in interval [0, Count). @@ -322,10 +344,11 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, DT->changeImmediateDominator(Exit, NewDom); } - // Split the main loop exit to maintain canonicalization guarantees. - SmallVector NewExitPreds{Latch}; - SplitBlockPredecessors(NewExit, NewExitPreds, ".loopexit", DT, LI, nullptr, - PreserveLCSSA); + // In EpilogPreHeader, assume extra iters is non-zero. + IRBuilder<> B2(EpilogPreHeader, EpilogPreHeader->getFirstNonPHIIt()); + Value *ModIsNotNull = B2.CreateIsNotNull(ModVal, "lcmp.mod"); + AssumeInst *AI = cast(B2.CreateAssumption(ModIsNotNull)); + AC.registerAssumption(AI); } /// Create a clone of the blocks in a loop and connect them together. A new @@ -795,7 +818,8 @@ bool llvm::UnrollRuntimeLoopRemainder( ConstantInt::get(BECount->getType(), Count - 1)) : B.CreateIsNotNull(ModVal, "lcmp.mod"); - BasicBlock *RemainderLoop = UseEpilogRemainder ? NewExit : PrologPreHeader; + BasicBlock *RemainderLoop = + UseEpilogRemainder ? EpilogPreHeader : PrologPreHeader; BasicBlock *UnrollingLoop = UseEpilogRemainder ? NewPreHeader : PrologExit; // Branch to either remainder (extra iterations) loop or unrolling loop. MDNode *BranchWeights = nullptr; @@ -808,7 +832,7 @@ bool llvm::UnrollRuntimeLoopRemainder( PreHeaderBR->eraseFromParent(); if (DT) { if (UseEpilogRemainder) - DT->changeImmediateDominator(NewExit, PreHeader); + DT->changeImmediateDominator(EpilogPreHeader, PreHeader); else DT->changeImmediateDominator(PrologExit, PreHeader); } @@ -880,7 +904,8 @@ bool llvm::UnrollRuntimeLoopRemainder( // from both the original loop and the remainder code reaching the exit // blocks. While the IDom of these exit blocks were from the original loop, // now the IDom is the preheader (which decides whether the original loop or - // remainder code should run). + // remainder code should run) unless the block still has just the original + // predecessor (such as NewExit in the case of an epilog remainder). if (DT && !L->getExitingBlock()) { SmallVector ChildrenToUpdate; // NB! We have to examine the dom children of all loop blocks, not just @@ -891,7 +916,8 @@ bool llvm::UnrollRuntimeLoopRemainder( auto *DomNodeBB = DT->getNode(BB); for (auto *DomChild : DomNodeBB->children()) { auto *DomChildBB = DomChild->getBlock(); - if (!L->contains(LI->getLoopFor(DomChildBB))) + if (!L->contains(LI->getLoopFor(DomChildBB)) && + DomChildBB->getUniquePredecessor() != BB) ChildrenToUpdate.push_back(DomChildBB); } } @@ -930,7 +956,7 @@ bool llvm::UnrollRuntimeLoopRemainder( // Connect the epilog code to the original loop and update the // PHI functions. ConnectEpilog(L, ModVal, NewExit, LatchExit, PreHeader, EpilogPreHeader, - NewPreHeader, VMap, DT, LI, PreserveLCSSA, *SE, Count); + NewPreHeader, VMap, DT, LI, PreserveLCSSA, *SE, Count, *AC); // Update counter in loop for unrolling. // Use an incrementing IV. Pre-incr/post-incr is backedge/trip count. diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll index d23afaed9a047..abcc566ec5633 100644 --- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll +++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll @@ -5,7 +5,7 @@ ;; Check atoms are remapped for runtime unrolling. ; CHECK: for.body.epil: -; CHECK-NEXT: store i64 %indvars.iv.unr, ptr %p, align 4, !dbg [[G2R1:!.*]] +; CHECK-NEXT: store i64 %indvars.iv.epil.init, ptr %p, align 4, !dbg [[G2R1:!.*]] ; CHECK: for.body.epil.1: ; CHECK-NEXT: store i64 %indvars.iv.next.epil, ptr %p, align 4, !dbg [[G3R1:!.*]] diff --git a/llvm/test/Transforms/HardwareLoops/ARM/structure.ll b/llvm/test/Transforms/HardwareLoops/ARM/structure.ll index cb66fefbfcc85..6993fd16dbad5 100644 --- a/llvm/test/Transforms/HardwareLoops/ARM/structure.ll +++ b/llvm/test/Transforms/HardwareLoops/ARM/structure.ll @@ -321,10 +321,10 @@ for.inc: ; preds = %sw.bb, %sw.bb1, %fo ; CHECK-UNROLL-NOT: dls ; CHECK-UNROLL: [[LOOP:.LBB[0-9_]+]]: @ %for.body ; CHECK-UNROLL: le lr, [[LOOP]] -; CHECK-UNROLL: wls lr, r12, [[EXIT:.LBB[0-9_]+]] +; CHECK-UNROLL: dls lr, r12 ; CHECK-UNROLL: [[EPIL:.LBB[0-9_]+]]: ; CHECK-UNROLL: le lr, [[EPIL]] -; CHECK-UNROLL-NEXT: [[EXIT]] +; CHECK-UNROLL-NEXT: {{\.LBB[0-9_]+}}: @ %for.cond.cleanup define void @unroll_inc_int(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %N) { entry: @@ -357,10 +357,10 @@ for.body: ; CHECK-UNROLL-NOT: dls ; CHECK-UNROLL: [[LOOP:.LBB[0-9_]+]]: @ %for.body ; CHECK-UNROLL: le lr, [[LOOP]] -; CHECK-UNROLL: wls lr, r12, [[EPIL_EXIT:.LBB[0-9_]+]] +; CHECK-UNROLL: dls lr, r12 ; CHECK-UNROLL: [[EPIL:.LBB[0-9_]+]]: ; CHECK-UNROLL: le lr, [[EPIL]] -; CHECK-UNROLL: [[EPIL_EXIT]]: +; CHECK-UNROLL: {{\.LBB[0-9_]+}}: @ %for.cond.cleanup ; CHECK-UNROLL: pop define void @unroll_inc_unsigned(ptr nocapture %a, ptr nocapture readonly %b, ptr nocapture readonly %c, i32 %N) { entry: diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll b/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll index 3b69527cbea19..2e4fc55a8f16d 100644 --- a/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll +++ b/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll @@ -15,7 +15,7 @@ define void @small_load_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale) { ; APPLE-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; APPLE-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 7 ; APPLE-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7 -; APPLE-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; APPLE-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; APPLE: [[ENTRY_NEW]]: ; APPLE-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; APPLE-NEXT: br label %[[LOOP:.*]] @@ -72,18 +72,18 @@ define void @small_load_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale) { ; APPLE-NEXT: [[IV_NEXT_7]] = add nuw nsw i64 [[IV_EPIL]], 8 ; APPLE-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; APPLE-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; APPLE-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]] -; APPLE: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; APPLE-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[LOOP]] ] -; APPLE-NEXT: br label %[[EXIT_UNR_LCSSA]] +; APPLE-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]] ; APPLE: [[EXIT_UNR_LCSSA]]: -; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[LOOP]] ] ; APPLE-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; APPLE: [[LOOP_EPIL_PREHEADER]]: +; APPLE-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ] +; APPLE-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; APPLE-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; APPLE-NEXT: br label %[[LOOP_EPIL:.*]] ; APPLE: [[LOOP_EPIL]]: -; APPLE-NEXT: [[IV_EPIL1:%.*]] = phi i64 [ [[IV_UNR]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL1:%.*]], %[[LOOP_EPIL]] ] +; APPLE-NEXT: [[IV_EPIL1:%.*]] = phi i64 [ [[IV_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL1:%.*]], %[[LOOP_EPIL]] ] ; APPLE-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, %[[LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_EPIL]] ] ; APPLE-NEXT: [[SCALED_IV_EPIL1:%.*]] = mul nuw nsw i64 [[IV_EPIL1]], [[SCALE]] ; APPLE-NEXT: [[GEP_SRC_EPIL1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_EPIL1]] @@ -106,7 +106,7 @@ define void @small_load_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale) { ; OTHER-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; OTHER-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; OTHER-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; OTHER-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; OTHER-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; OTHER: [[ENTRY_NEW]]: ; OTHER-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; OTHER-NEXT: br label %[[LOOP:.*]] @@ -127,15 +127,15 @@ define void @small_load_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale) { ; OTHER-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; OTHER-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; OTHER-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; OTHER-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]] -; OTHER: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; OTHER-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ] -; OTHER-NEXT: br label %[[EXIT_UNR_LCSSA]] +; OTHER-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]] ; OTHER: [[EXIT_UNR_LCSSA]]: -; OTHER-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; OTHER-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ] ; OTHER-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; OTHER-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; OTHER-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; OTHER: [[LOOP_EPIL_PREHEADER]]: +; OTHER-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR1]], %[[EXIT_UNR_LCSSA]] ] +; OTHER-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; OTHER-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; OTHER-NEXT: br label %[[LOOP_EPIL:.*]] ; OTHER: [[LOOP_EPIL]]: ; OTHER-NEXT: [[SCALED_IV_EPIL:%.*]] = mul nuw nsw i64 [[IV_UNR]], [[SCALE]] @@ -172,7 +172,7 @@ define void @load_op_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale, float %k ; APPLE-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; APPLE-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; APPLE-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; APPLE-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; APPLE-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; APPLE: [[ENTRY_NEW]]: ; APPLE-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; APPLE-NEXT: br label %[[LOOP:.*]] @@ -195,15 +195,15 @@ define void @load_op_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale, float %k ; APPLE-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; APPLE-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; APPLE-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; APPLE-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]] -; APPLE: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; APPLE-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ] -; APPLE-NEXT: br label %[[EXIT_UNR_LCSSA]] +; APPLE-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]] ; APPLE: [[EXIT_UNR_LCSSA]]: -; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; APPLE-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ] ; APPLE-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; APPLE: [[LOOP_EPIL_PREHEADER]]: +; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR1]], %[[EXIT_UNR_LCSSA]] ] +; APPLE-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; APPLE-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; APPLE-NEXT: br label %[[LOOP_EPIL:.*]] ; APPLE: [[LOOP_EPIL]]: ; APPLE-NEXT: [[SCALED_IV_EPIL:%.*]] = mul nuw nsw i64 [[IV_UNR]], [[SCALE]] @@ -222,7 +222,7 @@ define void @load_op_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale, float %k ; OTHER-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; OTHER-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; OTHER-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; OTHER-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; OTHER-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; OTHER: [[ENTRY_NEW]]: ; OTHER-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; OTHER-NEXT: br label %[[LOOP:.*]] @@ -245,15 +245,15 @@ define void @load_op_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale, float %k ; OTHER-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; OTHER-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; OTHER-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; OTHER-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]] -; OTHER: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; OTHER-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ] -; OTHER-NEXT: br label %[[EXIT_UNR_LCSSA]] +; OTHER-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]] ; OTHER: [[EXIT_UNR_LCSSA]]: -; OTHER-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; OTHER-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ] ; OTHER-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; OTHER-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; OTHER-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; OTHER: [[LOOP_EPIL_PREHEADER]]: +; OTHER-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR1]], %[[EXIT_UNR_LCSSA]] ] +; OTHER-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; OTHER-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; OTHER-NEXT: br label %[[LOOP_EPIL:.*]] ; OTHER: [[LOOP_EPIL]]: ; OTHER-NEXT: [[SCALED_IV_EPIL:%.*]] = mul nuw nsw i64 [[IV_UNR]], [[SCALE]] @@ -375,7 +375,7 @@ define void @early_continue_dep_on_load_large(ptr %p.1, ptr %p.2, i64 %N, i32 %x ; APPLE-NEXT: [[TMP1:%.*]] = add i64 [[N]], -2 ; APPLE-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 3 ; APPLE-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 3 -; APPLE-NEXT: br i1 [[TMP2]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; APPLE-NEXT: br i1 [[TMP2]], label %[[LOOP_HEADER_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; APPLE: [[ENTRY_NEW]]: ; APPLE-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]] ; APPLE-NEXT: br label %[[LOOP_HEADER:.*]] @@ -439,7 +439,7 @@ define void @early_continue_dep_on_load_large(ptr %p.1, ptr %p.2, i64 %N, i32 %x ; APPLE-NEXT: [[GEP_4_1:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_1]], i64 4 ; APPLE-NEXT: [[L_2_1:%.*]] = load i8, ptr [[GEP_4_1]], align 4 ; APPLE-NEXT: [[C_2_1:%.*]] = icmp ugt i8 [[L_2_1]], 7 -; APPLE-NEXT: br i1 [[C_2_1]], label %[[MERGE_11:.*]], label %[[ELSE_1:.*]] +; APPLE-NEXT: br i1 [[C_2_1]], label %[[MERGE_12:.*]], label %[[ELSE_1:.*]] ; APPLE: [[ELSE_1]]: ; APPLE-NEXT: [[CONV_I_1:%.*]] = zext nneg i8 [[L_2_1]] to i64 ; APPLE-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds [9 x i8], ptr @A, i64 0, i64 [[CONV_I_1]] @@ -449,8 +449,8 @@ define void @early_continue_dep_on_load_large(ptr %p.1, ptr %p.2, i64 %N, i32 %x ; APPLE-NEXT: [[L_4_1:%.*]] = load i32, ptr [[GEP_B_1]], align 4 ; APPLE-NEXT: [[GEP_C_1:%.*]] = getelementptr inbounds [8 x i32], ptr @C, i64 0, i64 [[IDXPROM_I_1]] ; APPLE-NEXT: [[L_5_1:%.*]] = load i32, ptr [[GEP_C_1]], align 4 -; APPLE-NEXT: br label %[[MERGE_11]] -; APPLE: [[MERGE_11]]: +; APPLE-NEXT: br label %[[MERGE_12]] +; APPLE: [[MERGE_12]]: ; APPLE-NEXT: [[MERGE_1_1:%.*]] = phi i32 [ 0, %[[THEN_1]] ], [ [[L_4_1]], %[[ELSE_1]] ] ; APPLE-NEXT: [[MERGE_2_1:%.*]] = phi i32 [ 0, %[[THEN_1]] ], [ [[L_5_1]], %[[ELSE_1]] ] ; APPLE-NEXT: [[ADD14_1:%.*]] = add nsw i32 [[MERGE_2_1]], [[X]] @@ -488,7 +488,7 @@ define void @early_continue_dep_on_load_large(ptr %p.1, ptr %p.2, i64 %N, i32 %x ; APPLE-NEXT: [[GEP_4_2:%.*]] = getelementptr inbounds nuw i8, ptr [[GEP_2]], i64 4 ; APPLE-NEXT: [[L_2_2:%.*]] = load i8, ptr [[GEP_4_2]], align 4 ; APPLE-NEXT: [[C_2_2:%.*]] = icmp ugt i8 [[L_2_2]], 7 -; APPLE-NEXT: br i1 [[C_2_2]], label %[[MERGE_22:.*]], label %[[ELSE_2:.*]] +; APPLE-NEXT: br i1 [[C_2_2]], label %[[MERGE_23:.*]], label %[[ELSE_2:.*]] ; APPLE: [[ELSE_2]]: ; APPLE-NEXT: [[CONV_I_2:%.*]] = zext nneg i8 [[L_2_2]] to i64 ; APPLE-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds [9 x i8], ptr @A, i64 0, i64 [[CONV_I_2]] @@ -498,8 +498,8 @@ define void @early_continue_dep_on_load_large(ptr %p.1, ptr %p.2, i64 %N, i32 %x ; APPLE-NEXT: [[L_4_2:%.*]] = load i32, ptr [[GEP_B_2]], align 4 ; APPLE-NEXT: [[GEP_C_2:%.*]] = getelementptr inbounds [8 x i32], ptr @C, i64 0, i64 [[IDXPROM_I_2]] ; APPLE-NEXT: [[L_5_2:%.*]] = load i32, ptr [[GEP_C_2]], align 4 -; APPLE-NEXT: br label %[[MERGE_22]] -; APPLE: [[MERGE_22]]: +; APPLE-NEXT: br label %[[MERGE_23]] +; APPLE: [[MERGE_23]]: ; APPLE-NEXT: [[MERGE_1_2:%.*]] = phi i32 [ 0, %[[THEN_2]] ], [ [[L_4_2]], %[[ELSE_2]] ] ; APPLE-NEXT: [[MERGE_2_2:%.*]] = phi i32 [ 0, %[[THEN_2]] ], [ [[L_5_2]], %[[ELSE_2]] ] ; APPLE-NEXT: [[ADD14_2:%.*]] = add nsw i32 [[MERGE_2_2]], [[X]] @@ -580,18 +580,18 @@ define void @early_continue_dep_on_load_large(ptr %p.1, ptr %p.2, i64 %N, i32 %x ; APPLE-NEXT: [[IV_NEXT_3]] = add nuw nsw i64 [[IV_EPIL]], 4 ; APPLE-NEXT: [[NITER_NEXT_3]] = add i64 [[NITER]], 4 ; APPLE-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; APPLE-NEXT: br i1 [[NITER_NCMP_3]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP_HEADER]] -; APPLE: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; APPLE-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_3]], %[[LOOP_LATCH_3]] ] -; APPLE-NEXT: br label %[[EXIT_UNR_LCSSA]] +; APPLE-NEXT: br i1 [[NITER_NCMP_3]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP_HEADER]] ; APPLE: [[EXIT_UNR_LCSSA]]: -; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_3]], %[[LOOP_LATCH_3]] ] ; APPLE-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_HEADER_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_HEADER_EPIL_PREHEADER]], label %[[EXIT:.*]] ; APPLE: [[LOOP_HEADER_EPIL_PREHEADER]]: +; APPLE-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ] +; APPLE-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; APPLE-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; APPLE-NEXT: br label %[[LOOP_HEADER_EPIL:.*]] ; APPLE: [[LOOP_HEADER_EPIL]]: -; APPLE-NEXT: [[IV_EPIL1:%.*]] = phi i64 [ [[IV_UNR]], %[[LOOP_HEADER_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL1:%.*]], %[[LOOP_LATCH_EPIL:.*]] ] +; APPLE-NEXT: [[IV_EPIL1:%.*]] = phi i64 [ [[IV_EPIL_INIT]], %[[LOOP_HEADER_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL1:%.*]], %[[LOOP_LATCH_EPIL:.*]] ] ; APPLE-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, %[[LOOP_HEADER_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_LATCH_EPIL]] ] ; APPLE-NEXT: [[GEP_EPIL1:%.*]] = getelementptr { i32, i8, i8, [2 x i8] }, ptr [[P_1]], i64 [[IV_EPIL1]] ; APPLE-NEXT: [[L_1_EPIL1:%.*]] = load i32, ptr [[GEP_EPIL1]], align 4 @@ -1034,7 +1034,7 @@ define i32 @test_add_reduction_runtime(ptr %a, i64 noundef %n) { ; APPLE-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; APPLE-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 3 ; APPLE-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 3 -; APPLE-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; APPLE-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; APPLE: [[ENTRY_NEW]]: ; APPLE-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; APPLE-NEXT: br label %[[LOOP:.*]] @@ -1063,26 +1063,25 @@ define i32 @test_add_reduction_runtime(ptr %a, i64 noundef %n) { ; APPLE-NEXT: [[IV_NEXT_3]] = add nuw nsw i64 [[IV_EPIL]], 4 ; APPLE-NEXT: [[NITER_NEXT_3]] = add nuw i64 [[NITER]], 4 ; APPLE-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; APPLE-NEXT: br i1 [[NITER_NCMP_3]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]] -; APPLE: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; APPLE-NEXT: [[RES_PH_PH:%.*]] = phi i32 [ [[RDX_NEXT_3]], %[[LOOP]] ] -; APPLE-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_3]], %[[LOOP]] ] -; APPLE-NEXT: [[RDX_UNR_PH:%.*]] = phi i32 [ [[RDX_NEXT_3]], %[[LOOP]] ] +; APPLE-NEXT: br i1 [[NITER_NCMP_3]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]] +; APPLE: [[EXIT_UNR_LCSSA]]: +; APPLE-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_3]], %[[LOOP]] ] +; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_3]], %[[LOOP]] ] +; APPLE-NEXT: [[RDX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_3]], %[[LOOP]] ] ; APPLE-NEXT: [[BIN_RDX:%.*]] = add i32 [[RDX_NEXT_1]], [[RDX_NEXT]] ; APPLE-NEXT: [[BIN_RDX2:%.*]] = add i32 [[RDX_NEXT_2]], [[BIN_RDX]] ; APPLE-NEXT: [[BIN_RDX3:%.*]] = add i32 [[RDX_NEXT_3]], [[BIN_RDX2]] -; APPLE-NEXT: br label %[[EXIT_UNR_LCSSA]] -; APPLE: [[EXIT_UNR_LCSSA]]: -; APPLE-NEXT: [[RES_PH:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[BIN_RDX3]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; APPLE-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[BIN_RDX3]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] ; APPLE-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; APPLE: [[LOOP_EPIL_PREHEADER]]: +; APPLE-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ] +; APPLE-NEXT: [[RDX_EPIL_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[BIN_RDX3]], %[[EXIT_UNR_LCSSA]] ] +; APPLE-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; APPLE-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; APPLE-NEXT: br label %[[LOOP_EPIL:.*]] ; APPLE: [[LOOP_EPIL]]: -; APPLE-NEXT: [[IV_EPIL1:%.*]] = phi i64 [ [[IV_UNR]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] -; APPLE-NEXT: [[RDX_EPIL:%.*]] = phi i32 [ [[RDX_UNR]], %[[LOOP_EPIL_PREHEADER]] ], [ [[RDX_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] +; APPLE-NEXT: [[IV_EPIL1:%.*]] = phi i64 [ [[IV_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] +; APPLE-NEXT: [[RDX_EPIL:%.*]] = phi i32 [ [[RDX_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[RDX_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] ; APPLE-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, %[[LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_EPIL]] ] ; APPLE-NEXT: [[GEP_A_EPIL1:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_EPIL1]] ; APPLE-NEXT: [[TMP7:%.*]] = load i32, ptr [[GEP_A_EPIL1]], align 2 @@ -1096,7 +1095,7 @@ define i32 @test_add_reduction_runtime(ptr %a, i64 noundef %n) { ; APPLE-NEXT: [[RES_PH1:%.*]] = phi i32 [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ] ; APPLE-NEXT: br label %[[EXIT]] ; APPLE: [[EXIT]]: -; APPLE-NEXT: [[RES:%.*]] = phi i32 [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RES_PH1]], %[[EXIT_EPILOG_LCSSA]] ] +; APPLE-NEXT: [[RES:%.*]] = phi i32 [ [[BIN_RDX3]], %[[EXIT_UNR_LCSSA]] ], [ [[RES_PH1]], %[[EXIT_EPILOG_LCSSA]] ] ; APPLE-NEXT: ret i32 [[RES]] ; ; OTHER-LABEL: define i32 @test_add_reduction_runtime( @@ -1105,7 +1104,7 @@ define i32 @test_add_reduction_runtime(ptr %a, i64 noundef %n) { ; OTHER-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; OTHER-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 3 ; OTHER-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 3 -; OTHER-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; OTHER-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; OTHER: [[ENTRY_NEW]]: ; OTHER-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; OTHER-NEXT: br label %[[LOOP:.*]] @@ -1131,23 +1130,22 @@ define i32 @test_add_reduction_runtime(ptr %a, i64 noundef %n) { ; OTHER-NEXT: [[IV_NEXT_3]] = add nuw nsw i64 [[IV]], 4 ; OTHER-NEXT: [[NITER_NEXT_3]] = add i64 [[NITER]], 4 ; OTHER-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; OTHER-NEXT: br i1 [[NITER_NCMP_3]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]] -; OTHER: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; OTHER-NEXT: [[RES_PH_PH:%.*]] = phi i32 [ [[RDX_NEXT_3]], %[[LOOP]] ] -; OTHER-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_3]], %[[LOOP]] ] -; OTHER-NEXT: [[RDX_UNR_PH:%.*]] = phi i32 [ [[RDX_NEXT_3]], %[[LOOP]] ] -; OTHER-NEXT: br label %[[EXIT_UNR_LCSSA]] +; OTHER-NEXT: br i1 [[NITER_NCMP_3]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]] ; OTHER: [[EXIT_UNR_LCSSA]]: -; OTHER-NEXT: [[RES_PH:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[RES_PH_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; OTHER-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; OTHER-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; OTHER-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_3]], %[[LOOP]] ] +; OTHER-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_3]], %[[LOOP]] ] +; OTHER-NEXT: [[RDX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_3]], %[[LOOP]] ] ; OTHER-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; OTHER-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; OTHER-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; OTHER: [[LOOP_EPIL_PREHEADER]]: +; OTHER-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ] +; OTHER-NEXT: [[RDX_EPIL_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_UNR]], %[[EXIT_UNR_LCSSA]] ] +; OTHER-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; OTHER-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; OTHER-NEXT: br label %[[LOOP_EPIL:.*]] ; OTHER: [[LOOP_EPIL]]: -; OTHER-NEXT: [[IV_EPIL:%.*]] = phi i64 [ [[IV_UNR]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] -; OTHER-NEXT: [[RDX_EPIL:%.*]] = phi i32 [ [[RDX_UNR]], %[[LOOP_EPIL_PREHEADER]] ], [ [[RDX_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] +; OTHER-NEXT: [[IV_EPIL:%.*]] = phi i64 [ [[IV_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] +; OTHER-NEXT: [[RDX_EPIL:%.*]] = phi i32 [ [[RDX_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[RDX_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] ; OTHER-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, %[[LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_EPIL]] ] ; OTHER-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_EPIL]] ; OTHER-NEXT: [[TMP6:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 2 diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll b/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll index b8215d9b6f33c..66c55f20e533f 100644 --- a/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll +++ b/llvm/test/Transforms/LoopUnroll/AArch64/runtime-unroll-generic.ll @@ -69,12 +69,14 @@ define void @runtime_unroll_generic(i32 %arg_0, ptr %arg_1, ptr %arg_2, ptr %arg ; CHECK-A55-NEXT: [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4 ; CHECK-A55-NEXT: [[NITER_NEXT_3]] = add i64 [[NITER]], 4 ; CHECK-A55-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-A55-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_END_LOOPEXIT_UNR_LCSSA]], label [[FOR_BODY6]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-A55-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_END_LOOPEXIT_UNR_LCSSA1:%.*]], label [[FOR_BODY6]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-A55: for.end.loopexit.unr-lcssa: -; CHECK-A55-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[FOR_BODY6_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3]], [[FOR_BODY6]] ] ; CHECK-A55-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 -; CHECK-A55-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_END]], label [[FOR_BODY6_EPIL:%.*]] -; CHECK-A55: for.body6.epil: +; CHECK-A55-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_END]], label [[FOR_END_LOOPEXIT_UNR_LCSSA]] +; CHECK-A55: for.body6.epil.preheader: +; CHECK-A55-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[FOR_BODY6_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3]], [[FOR_END_LOOPEXIT_UNR_LCSSA1]] ] +; CHECK-A55-NEXT: [[LCMP_MOD5:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-A55-NEXT: tail call void @llvm.assume(i1 [[LCMP_MOD5]]) ; CHECK-A55-NEXT: [[ARRAYIDX10_EPIL:%.*]] = getelementptr inbounds nuw i16, ptr [[ARG_2]], i64 [[INDVARS_IV_UNR]] ; CHECK-A55-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX10_EPIL]], align 2 ; CHECK-A55-NEXT: [[CONV_EPIL:%.*]] = sext i16 [[TMP13]] to i32 diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/vector.ll b/llvm/test/Transforms/LoopUnroll/AArch64/vector.ll index 38d559f86587e..2bafa08a81c38 100644 --- a/llvm/test/Transforms/LoopUnroll/AArch64/vector.ll +++ b/llvm/test/Transforms/LoopUnroll/AArch64/vector.ll @@ -9,7 +9,7 @@ define void @reverse(ptr %dst, ptr %src, i64 %len) { ; APPLE-NEXT: [[TMP0:%.*]] = add i64 [[LEN]], -1 ; APPLE-NEXT: [[XTRAITER:%.*]] = and i64 [[LEN]], 7 ; APPLE-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7 -; APPLE-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; APPLE-NEXT: br i1 [[TMP1]], label %[[FOR_BODY_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; APPLE: [[ENTRY_NEW]]: ; APPLE-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[LEN]], [[XTRAITER]] ; APPLE-NEXT: br label %[[FOR_BODY:.*]] @@ -66,18 +66,18 @@ define void @reverse(ptr %dst, ptr %src, i64 %len) { ; APPLE-NEXT: [[IV_NEXT_7]] = add nuw nsw i64 [[IV]], 8 ; APPLE-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; APPLE-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; APPLE-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[FOR_BODY]] -; APPLE: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; APPLE-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[FOR_BODY]] ] -; APPLE-NEXT: br label %[[EXIT_UNR_LCSSA]] +; APPLE-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA:.*]], label %[[FOR_BODY]] ; APPLE: [[EXIT_UNR_LCSSA]]: -; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; APPLE-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[FOR_BODY]] ] ; APPLE-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_BODY_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; APPLE-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_BODY_EPIL_PREHEADER]], label %[[EXIT:.*]] ; APPLE: [[FOR_BODY_EPIL_PREHEADER]]: +; APPLE-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ] +; APPLE-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; APPLE-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; APPLE-NEXT: br label %[[FOR_BODY_EPIL:.*]] ; APPLE: [[FOR_BODY_EPIL]]: -; APPLE-NEXT: [[IV_EPIL:%.*]] = phi i64 [ [[IV_UNR]], %[[FOR_BODY_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[FOR_BODY_EPIL]] ] +; APPLE-NEXT: [[IV_EPIL:%.*]] = phi i64 [ [[IV_EPIL_INIT]], %[[FOR_BODY_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[FOR_BODY_EPIL]] ] ; APPLE-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, %[[FOR_BODY_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[FOR_BODY_EPIL]] ] ; APPLE-NEXT: [[TMP18:%.*]] = sub nsw i64 [[LEN]], [[IV_EPIL]] ; APPLE-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds <4 x float>, ptr [[SRC]], i64 [[TMP18]] @@ -100,7 +100,7 @@ define void @reverse(ptr %dst, ptr %src, i64 %len) { ; CORTEXA55-NEXT: [[TMP0:%.*]] = add i64 [[LEN]], -1 ; CORTEXA55-NEXT: [[XTRAITER:%.*]] = and i64 [[LEN]], 3 ; CORTEXA55-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 3 -; CORTEXA55-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; CORTEXA55-NEXT: br i1 [[TMP1]], label %[[FOR_BODY_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; CORTEXA55: [[ENTRY_NEW]]: ; CORTEXA55-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[LEN]], [[XTRAITER]] ; CORTEXA55-NEXT: br label %[[FOR_BODY:.*]] @@ -133,15 +133,15 @@ define void @reverse(ptr %dst, ptr %src, i64 %len) { ; CORTEXA55-NEXT: [[IV_NEXT_3]] = add nuw nsw i64 [[IV]], 4 ; CORTEXA55-NEXT: [[NITER_NEXT_3]] = add i64 [[NITER]], 4 ; CORTEXA55-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CORTEXA55-NEXT: br i1 [[NITER_NCMP_3]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[FOR_BODY]] -; CORTEXA55: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; CORTEXA55-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_3]], %[[FOR_BODY]] ] -; CORTEXA55-NEXT: br label %[[EXIT_UNR_LCSSA]] +; CORTEXA55-NEXT: br i1 [[NITER_NCMP_3]], label %[[EXIT_UNR_LCSSA:.*]], label %[[FOR_BODY]] ; CORTEXA55: [[EXIT_UNR_LCSSA]]: -; CORTEXA55-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CORTEXA55-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_3]], %[[FOR_BODY]] ] ; CORTEXA55-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CORTEXA55-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_BODY_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; CORTEXA55-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_BODY_EPIL_PREHEADER]], label %[[EXIT:.*]] ; CORTEXA55: [[FOR_BODY_EPIL_PREHEADER]]: +; CORTEXA55-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR1]], %[[EXIT_UNR_LCSSA]] ] +; CORTEXA55-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CORTEXA55-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CORTEXA55-NEXT: br label %[[FOR_BODY_EPIL:.*]] ; CORTEXA55: [[FOR_BODY_EPIL]]: ; CORTEXA55-NEXT: [[TMP10:%.*]] = sub nsw i64 [[LEN]], [[IV_UNR]] diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll index 2486b8039803e..adf1e2117a0b5 100644 --- a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll +++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll @@ -14,7 +14,7 @@ define amdgpu_kernel void @unroll_when_cascaded_gep(i32 %arg) { ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[ARG:%.*]], 1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP0]], 7 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[ARG]], 7 -; CHECK-NEXT: br i1 [[TMP1]], label [[BB2_UNR_LCSSA:%.*]], label [[BB_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[BB1_EPIL_PREHEADER:%.*]], label [[BB_NEW:%.*]] ; CHECK: bb.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[TMP0]], [[XTRAITER]] ; CHECK-NEXT: br label [[BB1:%.*]] @@ -24,18 +24,18 @@ define amdgpu_kernel void @unroll_when_cascaded_gep(i32 %arg) { ; CHECK-NEXT: [[ADD_7]] = add i32 [[PHI]], 8 ; CHECK-NEXT: [[NITER_NEXT_7]] = add i32 [[NITER]], 8 ; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i32 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[BB2_UNR_LCSSA_LOOPEXIT:%.*]], label [[BB1]] -; CHECK: bb2.unr-lcssa.loopexit: -; CHECK-NEXT: [[PHI_UNR_PH:%.*]] = phi i32 [ [[ADD_7]], [[BB1]] ] -; CHECK-NEXT: br label [[BB2_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[BB2_UNR_LCSSA:%.*]], label [[BB1]] ; CHECK: bb2.unr-lcssa: -; CHECK-NEXT: [[PHI_UNR:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[PHI_UNR_PH]], [[BB2_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[PHI_UNR:%.*]] = phi i32 [ [[ADD_7]], [[BB1]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[BB1_EPIL_PREHEADER:%.*]], label [[BB2:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[BB1_EPIL_PREHEADER]], label [[BB2:%.*]] ; CHECK: bb1.epil.preheader: +; CHECK-NEXT: [[PHI_EPIL_INIT:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[PHI_UNR]], [[BB2_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[BB1_EPIL:%.*]] ; CHECK: bb1.epil: -; CHECK-NEXT: [[PHI_EPIL:%.*]] = phi i32 [ [[PHI_UNR]], [[BB1_EPIL_PREHEADER]] ], [ [[ADD_EPIL:%.*]], [[BB1_EPIL]] ] +; CHECK-NEXT: [[PHI_EPIL:%.*]] = phi i32 [ [[PHI_EPIL_INIT]], [[BB1_EPIL_PREHEADER]] ], [ [[ADD_EPIL:%.*]], [[BB1_EPIL]] ] ; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i32 [ 0, [[BB1_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[BB1_EPIL]] ] ; CHECK-NEXT: [[GETELEMENTPTR_EPIL:%.*]] = getelementptr [1024 x i32], ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @global, i32 8), i32 0, i32 0 ; CHECK-NEXT: [[ADD_EPIL]] = add i32 [[PHI_EPIL]], 1 diff --git a/llvm/test/Transforms/LoopUnroll/ARM/multi-blocks.ll b/llvm/test/Transforms/LoopUnroll/ARM/multi-blocks.ll index d2911a14d8485..7dacbf6124cb6 100644 --- a/llvm/test/Transforms/LoopUnroll/ARM/multi-blocks.ll +++ b/llvm/test/Transforms/LoopUnroll/ARM/multi-blocks.ll @@ -11,22 +11,21 @@ define void @test_three_blocks(ptr nocapture %Output, ptr nocapture readonly %Co ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[MAXJ]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[MAXJ]], 3 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 3 -; CHECK-NEXT: br i1 [[TMP1]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] ; CHECK: for.body.preheader.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[MAXJ]], [[XTRAITER]] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.cond.cleanup.loopexit.unr-lcssa.loopexit: +; CHECK: for.cond.cleanup.loopexit.unr-lcssa: ; CHECK-NEXT: [[TEMP_1_LCSSA_PH_PH:%.*]] = phi i32 [ [[TEMP_1_3:%.*]], [[FOR_INC_3:%.*]] ] ; CHECK-NEXT: [[J_010_UNR_PH:%.*]] = phi i32 [ [[INC_3:%.*]], [[FOR_INC_3]] ] ; CHECK-NEXT: [[TEMP_09_UNR_PH:%.*]] = phi i32 [ [[TEMP_1_3]], [[FOR_INC_3]] ] -; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] -; CHECK: for.cond.cleanup.loopexit.unr-lcssa: -; CHECK-NEXT: [[TEMP_1_LCSSA_PH:%.*]] = phi i32 [ poison, [[FOR_BODY_PREHEADER]] ], [ [[TEMP_1_LCSSA_PH_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT:%.*]] ] -; CHECK-NEXT: [[J_010_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[J_010_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[TEMP_09_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TEMP_09_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: br i1 [[LCMP_MOD1]], label [[FOR_BODY_EPIL_PREHEADER]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ; CHECK: for.body.epil.preheader: +; CHECK-NEXT: [[J_010_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[J_010_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]] ] +; CHECK-NEXT: [[TEMP_09_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TEMP_09_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD]]) ; CHECK-NEXT: br label [[FOR_BODY_EPIL:%.*]] ; CHECK: for.body.epil: ; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[CONDITION:%.*]], i32 [[J_010_UNR]] @@ -75,7 +74,7 @@ define void @test_three_blocks(ptr nocapture %Output, ptr nocapture readonly %Co ; CHECK-NEXT: [[TEMP_1_LCSSA_PH1:%.*]] = phi i32 [ [[TEMP_1_EPIL]], [[FOR_INC_EPIL]] ], [ [[TEMP_1_EPIL_1]], [[FOR_INC_EPIL_1]] ], [ [[TEMP_1_EPIL_2]], [[FOR_INC_EPIL_2]] ] ; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT]] ; CHECK: for.cond.cleanup.loopexit: -; CHECK-NEXT: [[TEMP_1_LCSSA:%.*]] = phi i32 [ [[TEMP_1_LCSSA_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ], [ [[TEMP_1_LCSSA_PH1]], [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA]] ] +; CHECK-NEXT: [[TEMP_1_LCSSA:%.*]] = phi i32 [ [[TEMP_1_LCSSA_PH_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ], [ [[TEMP_1_LCSSA_PH1]], [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA]] ] ; CHECK-NEXT: br label [[FOR_COND_CLEANUP]] ; CHECK: for.cond.cleanup: ; CHECK-NEXT: [[TEMP_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TEMP_1_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] @@ -135,7 +134,7 @@ define void @test_three_blocks(ptr nocapture %Output, ptr nocapture readonly %Co ; CHECK-NEXT: [[INC_3]] = add nuw i32 [[J_010]], 4 ; CHECK-NEXT: [[NITER_NEXT_3]] = add i32 [[NITER]], 4 ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i32 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]], label [[FOR_BODY]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]], label [[FOR_BODY]] ; entry: %cmp8 = icmp eq i32 %MaxJ, 0 @@ -354,24 +353,23 @@ define void @test_four_blocks(ptr nocapture %Output, ptr nocapture readonly %Con ; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[MAXJ]], -2 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP0]], 3 ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 3 -; CHECK-NEXT: br i1 [[TMP2]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY_LR_PH_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP2]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_BODY_LR_PH_NEW:%.*]] ; CHECK: for.body.lr.ph.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[TMP0]], [[XTRAITER]] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.cond.cleanup.loopexit.unr-lcssa.loopexit: +; CHECK: for.cond.cleanup.loopexit.unr-lcssa: ; CHECK-NEXT: [[TEMP_1_LCSSA_PH_PH:%.*]] = phi i32 [ [[TEMP_1_3:%.*]], [[FOR_INC_3:%.*]] ] ; CHECK-NEXT: [[I_UNR_PH:%.*]] = phi i32 [ [[I2_3:%.*]], [[FOR_INC_3]] ] ; CHECK-NEXT: [[J_027_UNR_PH:%.*]] = phi i32 [ [[INC_3:%.*]], [[FOR_INC_3]] ] ; CHECK-NEXT: [[TEMP_026_UNR_PH:%.*]] = phi i32 [ [[TEMP_1_3]], [[FOR_INC_3]] ] -; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] -; CHECK: for.cond.cleanup.loopexit.unr-lcssa: -; CHECK-NEXT: [[TEMP_1_LCSSA_PH:%.*]] = phi i32 [ poison, [[FOR_BODY_LR_PH]] ], [ [[TEMP_1_LCSSA_PH_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT:%.*]] ] -; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_BODY_LR_PH]] ], [ [[I_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[J_027_UNR:%.*]] = phi i32 [ 1, [[FOR_BODY_LR_PH]] ], [ [[J_027_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[TEMP_026_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[TEMP_026_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: br i1 [[LCMP_MOD1]], label [[FOR_BODY_EPIL_PREHEADER]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ; CHECK: for.body.epil.preheader: +; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ [[DOTPRE]], [[FOR_BODY_LR_PH]] ], [ [[I_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]] ] +; CHECK-NEXT: [[J_027_UNR:%.*]] = phi i32 [ 1, [[FOR_BODY_LR_PH]] ], [ [[J_027_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[TEMP_026_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[TEMP_026_UNR_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD]]) ; CHECK-NEXT: br label [[FOR_BODY_EPIL:%.*]] ; CHECK: for.body.epil: ; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[CONDITION:%.*]], i32 [[J_027_UNR]] @@ -450,7 +448,7 @@ define void @test_four_blocks(ptr nocapture %Output, ptr nocapture readonly %Con ; CHECK-NEXT: [[TEMP_1_LCSSA_PH1:%.*]] = phi i32 [ [[TEMP_1_EPIL]], [[FOR_INC_EPIL]] ], [ [[TEMP_1_EPIL_1]], [[FOR_INC_EPIL_1]] ], [ [[TEMP_1_EPIL_2]], [[FOR_INC_EPIL_2]] ] ; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT]] ; CHECK: for.cond.cleanup.loopexit: -; CHECK-NEXT: [[TEMP_1_LCSSA:%.*]] = phi i32 [ [[TEMP_1_LCSSA_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ], [ [[TEMP_1_LCSSA_PH1]], [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA]] ] +; CHECK-NEXT: [[TEMP_1_LCSSA:%.*]] = phi i32 [ [[TEMP_1_LCSSA_PH_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ], [ [[TEMP_1_LCSSA_PH1]], [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA]] ] ; CHECK-NEXT: br label [[FOR_COND_CLEANUP]] ; CHECK: for.cond.cleanup: ; CHECK-NEXT: [[TEMP_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TEMP_1_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] @@ -551,7 +549,7 @@ define void @test_four_blocks(ptr nocapture %Output, ptr nocapture readonly %Con ; CHECK-NEXT: [[INC_3]] = add nuw i32 [[J_027]], 4 ; CHECK-NEXT: [[NITER_NEXT_3]] = add i32 [[NITER]], 4 ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i32 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]], label [[FOR_BODY]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]], label [[FOR_BODY]] ; entry: %cmp25 = icmp ugt i32 %MaxJ, 1 diff --git a/llvm/test/Transforms/LoopUnroll/Hexagon/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopUnroll/Hexagon/reuse-lcssa-phi-scev-expansion.ll index f74fb14e397f3..8edc133f0d008 100644 --- a/llvm/test/Transforms/LoopUnroll/Hexagon/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopUnroll/Hexagon/reuse-lcssa-phi-scev-expansion.ll @@ -29,7 +29,7 @@ define void @preserve_lcssa_when_reusing_existing_phi() { ; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP1]], 7 ; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 7 -; CHECK-NEXT: br i1 [[TMP3]], label %[[LOOP_1_LATCH_UNR_LCSSA:.*]], label %[[LOOP_4_PREHEADER_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP3]], label %[[LOOP_4_EPIL_PREHEADER:.*]], label %[[LOOP_4_PREHEADER_NEW:.*]] ; CHECK: [[LOOP_4_PREHEADER_NEW]]: ; CHECK-NEXT: br label %[[LOOP_4:.*]] ; CHECK: [[LOOP_2_LATCH]]: @@ -47,18 +47,18 @@ define void @preserve_lcssa_when_reusing_existing_phi() { ; CHECK-NEXT: call void @foo() ; CHECK-NEXT: [[INC_I_7]] = add nuw nsw i32 [[IV_4]], 8 ; CHECK-NEXT: [[NITER_NEXT_7]] = add nuw nsw i32 [[NITER]], 8 -; CHECK-NEXT: br i1 true, label %[[LOOP_1_LATCH_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP_4]] -; CHECK: [[LOOP_1_LATCH_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[IV_4_UNR_PH:%.*]] = phi i32 [ [[INC_I_7]], %[[LOOP_4]] ] -; CHECK-NEXT: br label %[[LOOP_1_LATCH_UNR_LCSSA]] +; CHECK-NEXT: br i1 true, label %[[LOOP_1_LATCH_UNR_LCSSA:.*]], label %[[LOOP_4]] ; CHECK: [[LOOP_1_LATCH_UNR_LCSSA]]: -; CHECK-NEXT: [[IV_4_UNR:%.*]] = phi i32 [ 0, %[[LOOP_4_PREHEADER]] ], [ [[IV_4_UNR_PH]], %[[LOOP_1_LATCH_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[IV_4_UNR:%.*]] = phi i32 [ [[INC_I_7]], %[[LOOP_4]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_4_EPIL_PREHEADER:.*]], label %[[LOOP_1_LATCH:.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_4_EPIL_PREHEADER]], label %[[LOOP_1_LATCH:.*]] ; CHECK: [[LOOP_4_EPIL_PREHEADER]]: +; CHECK-NEXT: [[IV_4_EPIL_INIT:%.*]] = phi i32 [ 0, %[[LOOP_4_PREHEADER]] ], [ [[IV_4_UNR]], %[[LOOP_1_LATCH_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; CHECK-NEXT: br label %[[LOOP_4_EPIL:.*]] ; CHECK: [[LOOP_4_EPIL]]: -; CHECK-NEXT: [[IV_4_EPIL:%.*]] = phi i32 [ [[INC_I_EPIL:%.*]], %[[LOOP_4_EPIL]] ], [ [[IV_4_UNR]], %[[LOOP_4_EPIL_PREHEADER]] ] +; CHECK-NEXT: [[IV_4_EPIL:%.*]] = phi i32 [ [[INC_I_EPIL:%.*]], %[[LOOP_4_EPIL]] ], [ [[IV_4_EPIL_INIT]], %[[LOOP_4_EPIL_PREHEADER]] ] ; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i32 [ 0, %[[LOOP_4_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_4_EPIL]] ] ; CHECK-NEXT: call void @foo() ; CHECK-NEXT: [[INC_I_EPIL]] = add i32 [[IV_4_EPIL]], 1 diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors-inseltpoison.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors-inseltpoison.ll index 456875ecb7d1f..5d08e9dea0889 100644 --- a/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors-inseltpoison.ll +++ b/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors-inseltpoison.ll @@ -51,16 +51,16 @@ define ptr @f(ptr returned %s, i32 zeroext %x, i32 signext %k) local_unnamed_add ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] ; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT:%.*]], label [[VECTOR_BODY]] -; CHECK: middle.block.unr-lcssa.loopexit: +; CHECK: middle.block.unr-lcssa: ; CHECK-NEXT: [[INDEX_UNR_PH:%.*]] = phi i64 [ [[INDEX_NEXT_1]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_IND12_UNR_PH:%.*]] = phi <16 x i32> [ [[VEC_IND_NEXT13_1]], [[VECTOR_BODY]] ] -; CHECK-NEXT: br label [[MIDDLE_BLOCK_UNR_LCSSA]] -; CHECK: middle.block.unr-lcssa: -; CHECK-NEXT: [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[VEC_IND12_UNR:%.*]] = phi <16 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND12_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[VECTOR_BODY_EPIL_PREHEADER:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[MIDDLE_BLOCK:%.*]] ; CHECK: vector.body.epil.preheader: +; CHECK-NEXT: [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[VEC_IND12_UNR:%.*]] = phi <16 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND12_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[VECTOR_BODY_EPIL:%.*]] ; CHECK: vector.body.epil: ; CHECK-NEXT: [[TMP14:%.*]] = shl <16 x i32> splat (i32 1), [[VEC_IND12_UNR]] diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors.ll index cd4198f8160f7..03277fc8bb644 100644 --- a/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors.ll +++ b/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors.ll @@ -51,16 +51,16 @@ define ptr @f(ptr returned %s, i32 zeroext %x, i32 signext %k) local_unnamed_add ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] ; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT:%.*]], label [[VECTOR_BODY]] -; CHECK: middle.block.unr-lcssa.loopexit: +; CHECK: middle.block.unr-lcssa: ; CHECK-NEXT: [[INDEX_UNR_PH:%.*]] = phi i64 [ [[INDEX_NEXT_1]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_IND12_UNR_PH:%.*]] = phi <16 x i32> [ [[VEC_IND_NEXT13_1]], [[VECTOR_BODY]] ] -; CHECK-NEXT: br label [[MIDDLE_BLOCK_UNR_LCSSA]] -; CHECK: middle.block.unr-lcssa: -; CHECK-NEXT: [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[VEC_IND12_UNR:%.*]] = phi <16 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND12_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[VECTOR_BODY_EPIL_PREHEADER:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[MIDDLE_BLOCK:%.*]] ; CHECK: vector.body.epil.preheader: +; CHECK-NEXT: [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[VEC_IND12_UNR:%.*]] = phi <16 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND12_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[VECTOR_BODY_EPIL:%.*]] ; CHECK: vector.body.epil: ; CHECK-NEXT: [[TMP14:%.*]] = shl <16 x i32> splat (i32 1), [[VEC_IND12_UNR]] diff --git a/llvm/test/Transforms/LoopUnroll/RISCV/vector.ll b/llvm/test/Transforms/LoopUnroll/RISCV/vector.ll index 811d055cb4c45..b575057ff6d15 100644 --- a/llvm/test/Transforms/LoopUnroll/RISCV/vector.ll +++ b/llvm/test/Transforms/LoopUnroll/RISCV/vector.ll @@ -26,7 +26,7 @@ define void @reverse(ptr %dst, ptr %src, i64 %len) { ; SIFIVE-NEXT: [[TMP2:%.*]] = add i64 [[LEN]], -1 ; SIFIVE-NEXT: [[XTRAITER:%.*]] = and i64 [[LEN]], 7 ; SIFIVE-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], 7 -; SIFIVE-NEXT: br i1 [[TMP3]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; SIFIVE-NEXT: br i1 [[TMP3]], label %[[FOR_BODY_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; SIFIVE: [[ENTRY_NEW]]: ; SIFIVE-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[LEN]], [[XTRAITER]] ; SIFIVE-NEXT: br label %[[FOR_BODY:.*]] @@ -83,15 +83,15 @@ define void @reverse(ptr %dst, ptr %src, i64 %len) { ; SIFIVE-NEXT: [[IV_NEXT_7]] = add nuw nsw i64 [[IV]], 8 ; SIFIVE-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; SIFIVE-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; SIFIVE-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[FOR_BODY]] -; SIFIVE: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; SIFIVE-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[FOR_BODY]] ] -; SIFIVE-NEXT: br label %[[EXIT_UNR_LCSSA]] +; SIFIVE-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA:.*]], label %[[FOR_BODY]] ; SIFIVE: [[EXIT_UNR_LCSSA]]: -; SIFIVE-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; SIFIVE-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[FOR_BODY]] ] ; SIFIVE-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; SIFIVE-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_BODY_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; SIFIVE-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_BODY_EPIL_PREHEADER]], label %[[EXIT:.*]] ; SIFIVE: [[FOR_BODY_EPIL_PREHEADER]]: +; SIFIVE-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR1]], %[[EXIT_UNR_LCSSA]] ] +; SIFIVE-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; SIFIVE-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; SIFIVE-NEXT: br label %[[FOR_BODY_EPIL:.*]] ; SIFIVE: [[FOR_BODY_EPIL]]: ; SIFIVE-NEXT: [[TMP18:%.*]] = sub nsw i64 [[LEN]], [[IV_UNR]] diff --git a/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll b/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll index ea499e548934f..b456ad8224638 100644 --- a/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll +++ b/llvm/test/Transforms/LoopUnroll/WebAssembly/basic-unrolling.ll @@ -124,15 +124,17 @@ define hidden void @runtime(ptr nocapture %a, ptr nocapture readonly %b, ptr noc ; CHECK: for.body.preheader: ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[N]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[N]], 1 -; CHECK-NEXT: br i1 [[TMP0]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP0]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] ; CHECK: for.body.preheader.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i32 [[N]], -2 ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.cond.cleanup.loopexit.unr-lcssa: -; CHECK-NEXT: [[I_09_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INC_1:%.*]], [[FOR_BODY]] ] ; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_EPIL:%.*]] -; CHECK: for.body.epil: +; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_EPIL_PREHEADER]] +; CHECK: for.body.epil.preheader: +; CHECK-NEXT: [[I_09_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INC_1:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 [[I_09_UNR]] ; CHECK-NEXT: [[I_EPIL:%.*]] = load i32, ptr [[ARRAYIDX_EPIL]], align 4 ; CHECK-NEXT: [[ARRAYIDX1_EPIL:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i32 [[I_09_UNR]] diff --git a/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll b/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll index 7fd4eb18f16eb..6e600d2a659da 100644 --- a/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll +++ b/llvm/test/Transforms/LoopUnroll/convergent.controlled.ll @@ -302,7 +302,7 @@ define i32 @pragma_unroll_with_remainder(i32 %n) { ; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP0]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 1 -; CHECK-NEXT: br i1 [[TMP2]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP2]], label [[L3_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[TMP0]], [[XTRAITER]] ; CHECK-NEXT: br label [[L3:%.*]], !llvm.loop [[LOOP4]] @@ -316,13 +316,13 @@ define i32 @pragma_unroll_with_remainder(i32 %n) { ; CHECK-NEXT: [[INC_1]] = add nsw i32 [[X_0]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i32 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i32 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[L3]], !llvm.loop [[LOOP8:![0-9]+]] -; CHECK: exit.unr-lcssa.loopexit: -; CHECK-NEXT: br label [[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA:%.*]], label [[L3]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: exit.unr-lcssa: ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[L3_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[L3_EPIL_PREHEADER]], label [[EXIT:%.*]] ; CHECK: l3.epil.preheader: +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[L3_EPIL:%.*]] ; CHECK: l3.epil: ; CHECK-NEXT: [[TOK_LOOP_EPIL:%.*]] = call token @llvm.experimental.convergence.anchor() diff --git a/llvm/test/Transforms/LoopUnroll/followup.ll b/llvm/test/Transforms/LoopUnroll/followup.ll index e4ae7b6356d38..051e43d52b3be 100644 --- a/llvm/test/Transforms/LoopUnroll/followup.ll +++ b/llvm/test/Transforms/LoopUnroll/followup.ll @@ -43,7 +43,7 @@ for.end: ; preds = %for.body, %entry ; COUNT: ![[LOOP]] = distinct !{![[LOOP]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_UNROLLED]]} -; EPILOG: br i1 %niter.ncmp.7, label %for.end.loopexit.unr-lcssa.loopexit, label %for.body, !llvm.loop ![[LOOP_0:[0-9]+]] +; EPILOG: br i1 %niter.ncmp.7, label %for.end.loopexit.unr-lcssa, label %for.body, !llvm.loop ![[LOOP_0:[0-9]+]] ; EPILOG: br i1 %epil.iter.cmp, label %for.body.epil, label %for.end.loopexit.epilog-lcssa, !llvm.loop ![[LOOP_2:[0-9]+]] ; EPILOG: ![[LOOP_0]] = distinct !{![[LOOP_0]], ![[FOLLOWUP_ALL:[0-9]+]], ![[FOLLOWUP_UNROLLED:[0-9]+]]} diff --git a/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll b/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll index 835fc2feb2f41..ee28aa1b0008b 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll @@ -3,9 +3,7 @@ ; Test that epilogue is tagged with the same debug information as original loop body rather than original loop exit. ; CHECK: for.body.i: -; CHECK: br i1 {{.*}}, label %lee1.exit.loopexit.unr-lcssa.loopexit, label %for.body.i, !dbg ![[LOOP_LOC:[0-9]+]] -; CHECK: lee1.exit.loopexit.unr-lcssa.loopexit: -; CHECK: br label %lee1.exit.loopexit.unr-lcssa, !dbg ![[LOOP_LOC]] +; CHECK: br i1 {{.*}}, label %lee1.exit.loopexit.unr-lcssa, label %for.body.i, !dbg ![[LOOP_LOC:[0-9]+]] ; CHECK: lee1.exit.loopexit.unr-lcssa: ; CHECK: %lcmp.mod = icmp ne i32 %xtraiter, 0, !dbg ![[LOOP_LOC]] ; CHECK: br i1 %lcmp.mod, label %for.body.i.epil.preheader, label %lee1.exit.loopexit, !dbg ![[LOOP_LOC]] diff --git a/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll b/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll index a97b39494b2ef..0c52b5a0edef8 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll @@ -20,7 +20,7 @@ define void @pr56282() { ; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP1]], 7 ; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP2]], 7 -; CHECK-NEXT: br i1 [[TMP3]], label [[OUTER_MIDDLE_UNR_LCSSA:%.*]], label [[OUTER_HEADER_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP3]], label [[INNER_1_HEADER_EPIL_PREHEADER:%.*]], label [[OUTER_HEADER_NEW:%.*]] ; CHECK: outer.header.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP1]], [[XTRAITER]] ; CHECK-NEXT: br label [[INNER_1_HEADER:%.*]] @@ -62,17 +62,16 @@ define void @pr56282() { ; CHECK: inner.1.latch.7: ; CHECK-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp ne i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[INNER_1_HEADER]], label [[OUTER_MIDDLE_UNR_LCSSA_LOOPEXIT:%.*]] -; CHECK: outer.middle.unr-lcssa.loopexit: +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[INNER_1_HEADER]], label [[OUTER_MIDDLE_UNR_LCSSA:%.*]] +; CHECK: outer.middle.unr-lcssa: ; CHECK-NEXT: [[V_LCSSA1_PH_PH:%.*]] = phi i32 [ [[V_7]], [[INNER_1_LATCH_7]] ] ; CHECK-NEXT: [[INNER_1_IV_UNR_PH:%.*]] = phi i64 [ [[INNER_1_IV_NEXT_7]], [[INNER_1_LATCH_7]] ] -; CHECK-NEXT: br label [[OUTER_MIDDLE_UNR_LCSSA]] -; CHECK: outer.middle.unr-lcssa: -; CHECK-NEXT: [[V_LCSSA1_PH:%.*]] = phi i32 [ poison, [[OUTER_HEADER]] ], [ [[V_LCSSA1_PH_PH]], [[OUTER_MIDDLE_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[INNER_1_IV_UNR:%.*]] = phi i64 [ 0, [[OUTER_HEADER]] ], [ [[INNER_1_IV_UNR_PH]], [[OUTER_MIDDLE_UNR_LCSSA_LOOPEXIT]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[INNER_1_HEADER_EPIL_PREHEADER:%.*]], label [[OUTER_MIDDLE:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[INNER_1_HEADER_EPIL_PREHEADER]], label [[OUTER_MIDDLE:%.*]] ; CHECK: inner.1.header.epil.preheader: +; CHECK-NEXT: [[INNER_1_IV_UNR:%.*]] = phi i64 [ 0, [[OUTER_HEADER]] ], [ [[INNER_1_IV_UNR_PH]], [[OUTER_MIDDLE_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD3:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD3]]) ; CHECK-NEXT: br label [[INNER_1_HEADER_EPIL:%.*]] ; CHECK: inner.1.header.epil: ; CHECK-NEXT: [[INNER_1_IV_EPIL:%.*]] = phi i64 [ [[INNER_1_IV_UNR]], [[INNER_1_HEADER_EPIL_PREHEADER]] ], [ [[INNER_1_IV_NEXT_EPIL:%.*]], [[INNER_1_LATCH_EPIL:%.*]] ] @@ -90,7 +89,7 @@ define void @pr56282() { ; CHECK-NEXT: [[V_LCSSA1_PH2:%.*]] = phi i32 [ [[V_EPIL]], [[INNER_1_LATCH_EPIL]] ] ; CHECK-NEXT: br label [[OUTER_MIDDLE]] ; CHECK: outer.middle: -; CHECK-NEXT: [[V_LCSSA1:%.*]] = phi i32 [ [[V_LCSSA1_PH]], [[OUTER_MIDDLE_UNR_LCSSA]] ], [ [[V_LCSSA1_PH2]], [[OUTER_MIDDLE_EPILOG_LCSSA]] ] +; CHECK-NEXT: [[V_LCSSA1:%.*]] = phi i32 [ [[V_LCSSA1_PH_PH]], [[OUTER_MIDDLE_UNR_LCSSA]] ], [ [[V_LCSSA1_PH2]], [[OUTER_MIDDLE_EPILOG_LCSSA]] ] ; CHECK-NEXT: [[C_3:%.*]] = icmp ugt i32 [[V_LCSSA1]], 0 ; CHECK-NEXT: br i1 [[C_3]], label [[INNER_2_PREHEADER:%.*]], label [[EXIT:%.*]] ; CHECK: inner.2.preheader: @@ -102,7 +101,7 @@ define void @pr56282() { ; CHECK-NEXT: ret void ; CHECK: exit.deopt.loopexit: ; CHECK-NEXT: br label [[EXIT_DEOPT:%.*]] -; CHECK: exit.deopt.loopexit3: +; CHECK: exit.deopt.loopexit4: ; CHECK-NEXT: br label [[EXIT_DEOPT]] ; CHECK: exit.deopt: ; CHECK-NEXT: call void (...) @llvm.experimental.deoptimize.isVoid(i32 0) [ "deopt"() ] @@ -233,7 +232,7 @@ define void @pr56286(i64 %x, ptr %src, ptr %dst, ptr %ptr.src) !prof !0 { ; CHECK-NEXT: store i32 [[L_1_7]], ptr [[DST]], align 8 ; CHECK-NEXT: [[INNER_1_IV_NEXT_7]] = add i64 [[INNER_1_IV]], 8 ; CHECK-NEXT: [[CMP_2_7:%.*]] = icmp sgt i64 [[INNER_1_IV_NEXT_6]], 0 -; CHECK-NEXT: br i1 [[CMP_2_7]], label [[OUTER_MIDDLE_UNR_LCSSA:%.*]], label [[INNER_1_HEADER]], !prof [[PROF6:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP_2_7]], label [[OUTER_MIDDLE_UNR_LCSSA:%.*]], label [[INNER_1_HEADER]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: outer.middle.unr-lcssa: ; CHECK-NEXT: [[L_1_LCSSA_PH:%.*]] = phi i32 [ [[L_1_7]], [[INNER_1_LATCH_7]] ] ; CHECK-NEXT: br label [[OUTER_MIDDLE]] diff --git a/llvm/test/Transforms/LoopUnroll/runtime-i128.ll b/llvm/test/Transforms/LoopUnroll/runtime-i128.ll index 4cd8e7ca5d16f..fec8626e8c59c 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-i128.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-i128.ll @@ -11,7 +11,7 @@ define void @test(i128 %n, i128 %m) { ; CHECK-NEXT: [[TMP1:%.*]] = add i128 [[TMP0]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i128 [[TMP0]], 7 ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i128 [[TMP1]], 7 -; CHECK-NEXT: br i1 [[TMP2]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP2]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i128 [[TMP0]], [[XTRAITER]] ; CHECK-NEXT: br label [[LOOP:%.*]] @@ -29,18 +29,18 @@ define void @test(i128 %n, i128 %m) { ; CHECK-NEXT: [[IV_NEXT_7]] = add i128 [[IV]], 8 ; CHECK-NEXT: [[NITER_NEXT_7]] = add i128 [[NITER]], 8 ; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp ne i128 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[LOOP]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]] -; CHECK: exit.unr-lcssa.loopexit: -; CHECK-NEXT: [[IV_UNR_PH:%.*]] = phi i128 [ [[IV_NEXT_7]], [[LOOP]] ] -; CHECK-NEXT: br label [[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[LOOP]], label [[EXIT_UNR_LCSSA:%.*]] ; CHECK: exit.unr-lcssa: -; CHECK-NEXT: [[IV_UNR:%.*]] = phi i128 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR_PH]], [[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[IV_UNR:%.*]] = phi i128 [ [[IV_NEXT_7]], [[LOOP]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i128 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER]], label [[EXIT:%.*]] ; CHECK: loop.epil.preheader: +; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i128 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR]], [[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i128 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[LOOP_EPIL:%.*]] ; CHECK: loop.epil: -; CHECK-NEXT: [[IV_EPIL:%.*]] = phi i128 [ [[IV_UNR]], [[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], [[LOOP_EPIL]] ] +; CHECK-NEXT: [[IV_EPIL:%.*]] = phi i128 [ [[IV_EPIL_INIT]], [[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], [[LOOP_EPIL]] ] ; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i128 [ 0, [[LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[LOOP_EPIL]] ] ; CHECK-NEXT: call void @foo() ; CHECK-NEXT: [[IV_NEXT_EPIL]] = add i128 [[IV_EPIL]], 1 diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-at-most-two-exits.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-at-most-two-exits.ll index 8472a8c9762b3..85de29de750fa 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-loop-at-most-two-exits.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-at-most-two-exits.ll @@ -9,7 +9,7 @@ define i32 @test(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -1 ; ENABLED-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 ; ENABLED-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 7 -; ENABLED-NEXT: br i1 [[TMP2]], label [[FOR_END_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; ENABLED-NEXT: br i1 [[TMP2]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; ENABLED: entry.new: ; ENABLED-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]] ; ENABLED-NEXT: br label [[HEADER:%.*]] @@ -71,23 +71,22 @@ define i32 @test(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[INDVARS_IV_NEXT_7]] = add i64 [[INDVARS_IV]], 8 ; ENABLED-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; ENABLED-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; ENABLED-NEXT: br i1 [[NITER_NCMP_7]], label [[FOR_END_UNR_LCSSA_LOOPEXIT:%.*]], label [[HEADER]] -; ENABLED: for.end.unr-lcssa.loopexit: +; ENABLED-NEXT: br i1 [[NITER_NCMP_7]], label [[FOR_END_UNR_LCSSA:%.*]], label [[HEADER]] +; ENABLED: for.end.unr-lcssa: ; ENABLED-NEXT: [[SUM_0_LCSSA_PH_PH:%.*]] = phi i32 [ [[ADD_7]], [[FOR_BODY_7]] ] ; ENABLED-NEXT: [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_7]], [[FOR_BODY_7]] ] ; ENABLED-NEXT: [[SUM_02_UNR_PH:%.*]] = phi i32 [ [[ADD_7]], [[FOR_BODY_7]] ] -; ENABLED-NEXT: br label [[FOR_END_UNR_LCSSA]] -; ENABLED: for.end.unr-lcssa: -; ENABLED-NEXT: [[SUM_0_LCSSA_PH:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[SUM_0_LCSSA_PH_PH]], [[FOR_END_UNR_LCSSA_LOOPEXIT]] ] -; ENABLED-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_UNR_PH]], [[FOR_END_UNR_LCSSA_LOOPEXIT]] ] -; ENABLED-NEXT: [[SUM_02_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[FOR_END_UNR_LCSSA_LOOPEXIT]] ] ; ENABLED-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[FOR_END:%.*]] +; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER]], label [[FOR_END:%.*]] ; ENABLED: header.epil.preheader: +; ENABLED-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_UNR_PH]], [[FOR_END_UNR_LCSSA]] ] +; ENABLED-NEXT: [[SUM_02_EPIL_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[FOR_END_UNR_LCSSA]] ] +; ENABLED-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; ENABLED-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; ENABLED-NEXT: br label [[HEADER_EPIL:%.*]] ; ENABLED: header.epil: -; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL:%.*]] ], [ [[INDVARS_IV_UNR]], [[HEADER_EPIL_PREHEADER]] ] -; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[SUM_02_UNR]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL:%.*]] ], [ [[INDVARS_IV_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[SUM_02_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] ; ENABLED-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, [[HEADER_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[FOR_BODY_EPIL]] ] ; ENABLED-NEXT: [[CMP_EPIL:%.*]] = icmp eq i64 [[N]], 42 ; ENABLED-NEXT: br i1 [[CMP_EPIL]], label [[FOR_EXIT2_LOOPEXIT2:%.*]], label [[FOR_BODY_EPIL]] @@ -104,12 +103,12 @@ define i32 @test(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[SUM_0_LCSSA_PH1:%.*]] = phi i32 [ [[ADD_EPIL]], [[FOR_BODY_EPIL]] ] ; ENABLED-NEXT: br label [[FOR_END]] ; ENABLED: for.end: -; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[FOR_END_UNR_LCSSA]] ], [ [[SUM_0_LCSSA_PH1]], [[FOR_END_EPILOG_LCSSA]] ] +; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH_PH]], [[FOR_END_UNR_LCSSA]] ], [ [[SUM_0_LCSSA_PH1]], [[FOR_END_EPILOG_LCSSA]] ] ; ENABLED-NEXT: ret i32 [[SUM_0_LCSSA]] ; ENABLED: for.exit2.loopexit: ; ENABLED-NEXT: [[RETVAL_PH:%.*]] = phi i32 [ [[SUM_02]], [[HEADER]] ], [ [[ADD]], [[FOR_BODY]] ], [ [[ADD_1]], [[FOR_BODY_1]] ], [ [[ADD_2]], [[FOR_BODY_2]] ], [ [[ADD_3]], [[FOR_BODY_3]] ], [ [[ADD_4]], [[FOR_BODY_4]] ], [ [[ADD_5]], [[FOR_BODY_5]] ], [ [[ADD_6]], [[FOR_BODY_6]] ] ; ENABLED-NEXT: br label [[FOR_EXIT2:%.*]] -; ENABLED: for.exit2.loopexit2: +; ENABLED: for.exit2.loopexit3: ; ENABLED-NEXT: [[RETVAL_PH3:%.*]] = phi i32 [ [[SUM_02_EPIL]], [[HEADER_EPIL]] ] ; ENABLED-NEXT: br label [[FOR_EXIT2]] ; ENABLED: for.exit2: diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-branchweight.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-branchweight.ll index 6e3bbe16167e7..26171990a2592 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-loop-branchweight.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-branchweight.ll @@ -3,7 +3,7 @@ ;; Check that the remainder loop is properly assigned a branch weight for its latch branch. ; CHECK-LABEL: @test( ; CHECK-LABEL: for.body: -; CHECK: br i1 [[COND1:%.*]], label %for.end.loopexit.unr-lcssa.loopexit, label %for.body, !prof ![[#PROF:]], !llvm.loop ![[#LOOP:]] +; CHECK: br i1 [[COND1:%.*]], label %for.end.loopexit.unr-lcssa, label %for.body, !prof ![[#PROF:]], !llvm.loop ![[#LOOP:]] ; CHECK-LABEL: for.body.epil: ; CHECK: br i1 [[COND2:%.*]], label %for.body.epil, label %for.end.loopexit.epilog-lcssa, !prof ![[#PROF2:]], !llvm.loop ![[#LOOP2:]] ; CHECK: ![[#PROF]] = !{!"branch_weights", i32 1, i32 2499} diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll index 5f6e66e344625..6835e9b1d6f8c 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll @@ -15,7 +15,7 @@ define void @test1(i64 %trip, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %exit2.loopexit.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %loop_header @@ -29,7 +29,7 @@ define void @test1(i64 %trip, i1 %cond) { ; EPILOG-NEXT: br i1 false, label %loop_latch, label %exit3.loopexit ; EPILOG: exit3.loopexit: ; EPILOG-NEXT: br label %exit3 -; EPILOG: exit3.loopexit2: +; EPILOG: exit3.loopexit3: ; EPILOG-NEXT: br label %exit3 ; EPILOG: exit3: ; EPILOG-NEXT: ret void @@ -79,30 +79,30 @@ define void @test1(i64 %trip, i1 %cond) { ; EPILOG-NEXT: %iv_next.7 = add i64 %iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit2.loopexit.unr-lcssa.loopexit +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit2.loopexit.unr-lcssa ; EPILOG: exit1.loopexit: ; EPILOG-NEXT: br label %exit1 -; EPILOG: exit1.loopexit1: +; EPILOG: exit1.loopexit2: ; EPILOG-NEXT: br label %exit1 ; EPILOG: exit1: ; EPILOG-NEXT: ret void -; EPILOG: exit2.loopexit.unr-lcssa.loopexit: -; EPILOG-NEXT: %iv.unr.ph = phi i64 [ %iv_next.7, %loop_latch.7 ] -; EPILOG-NEXT: br label %exit2.loopexit.unr-lcssa ; EPILOG: exit2.loopexit.unr-lcssa: -; EPILOG-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit2.loopexit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %iv.unr = phi i64 [ %iv_next.7, %loop_latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit2.loopexit ; EPILOG: loop_header.epil.preheader: +; EPILOG-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit2.loopexit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %loop_header.epil ; EPILOG: loop_header.epil: -; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.unr, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.epil.init, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %loop_header.epil.preheader ], [ %epil.iter.next, %loop_latch.epil ] ; EPILOG-NEXT: br i1 %cond, label %loop_latch.epil, label %loop_exiting_bb1.epil ; EPILOG: loop_exiting_bb1.epil: -; EPILOG-NEXT: br i1 false, label %loop_exiting_bb2.epil, label %exit1.loopexit1 +; EPILOG-NEXT: br i1 false, label %loop_exiting_bb2.epil, label %exit1.loopexit2 ; EPILOG: loop_exiting_bb2.epil: -; EPILOG-NEXT: br i1 false, label %loop_latch.epil, label %exit3.loopexit2 +; EPILOG-NEXT: br i1 false, label %loop_latch.epil, label %exit3.loopexit3 ; EPILOG: loop_latch.epil: ; EPILOG-NEXT: %iv_next.epil = add i64 %iv.epil, 1 ; EPILOG-NEXT: %cmp.epil = icmp ne i64 %iv_next.epil, %trip @@ -120,7 +120,7 @@ define void @test1(i64 %trip, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %exit2.loopexit.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %loop_header @@ -146,17 +146,17 @@ define void @test1(i64 %trip, i1 %cond) { ; EPILOG-BLOCK-NEXT: %iv_next.1 = add i64 %iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit2.loopexit.unr-lcssa.loopexit, !llvm.loop !0 +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit2.loopexit.unr-lcssa, !llvm.loop !0 ; EPILOG-BLOCK: exit1.loopexit: ; EPILOG-BLOCK-NEXT: br label %exit1 ; EPILOG-BLOCK: exit1: ; EPILOG-BLOCK-NEXT: ret void -; EPILOG-BLOCK: exit2.loopexit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: br label %exit2.loopexit.unr-lcssa ; EPILOG-BLOCK: exit2.loopexit.unr-lcssa: ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit2.loopexit ; EPILOG-BLOCK: loop_header.epil.preheader: +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %loop_header.epil ; EPILOG-BLOCK: loop_header.epil: ; EPILOG-BLOCK-NEXT: br i1 %cond, label %loop_latch.epil, label %loop_exiting_bb1.epil @@ -366,7 +366,7 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %for.end.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %header @@ -448,28 +448,27 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; EPILOG-NEXT: %indvars.iv.next.7 = add i64 %indvars.iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp eq i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %for.end.unr-lcssa.loopexit, label %header -; EPILOG: for.end.unr-lcssa.loopexit: -; EPILOG-NEXT: %sum.0.lcssa.ph.ph = phi i32 [ %add.7, %for.body.7 ] -; EPILOG-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.7, %for.body.7 ] -; EPILOG-NEXT: %sum.02.unr.ph = phi i32 [ %add.7, %for.body.7 ] -; EPILOG-NEXT: br label %for.end.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %for.end.unr-lcssa, label %header ; EPILOG: for.end.unr-lcssa: -; EPILOG-NEXT: %sum.0.lcssa.ph = phi i32 [ poison, %entry ], [ %sum.0.lcssa.ph.ph, %for.end.unr-lcssa.loopexit ] -; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %for.end.unr-lcssa.loopexit ] -; EPILOG-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %for.end.unr-lcssa.loopexit ] +; EPILOG-NEXT: %sum.0.lcssa.ph = phi i32 [ %add.7, %for.body.7 ] +; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.7, %for.body.7 ] +; EPILOG-NEXT: %sum.02.unr = phi i32 [ %add.7, %for.body.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %for.end ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %for.end.unr-lcssa ] +; EPILOG-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %for.end.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %for.body.epil ], [ %indvars.iv.unr, %header.epil.preheader ] -; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.02.unr, %header.epil.preheader ] +; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %for.body.epil ], [ %indvars.iv.epil.init, %header.epil.preheader ] +; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.02.epil.init, %header.epil.preheader ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %header.epil.preheader ], [ %epil.iter.next, %for.body.epil ] -; EPILOG-NEXT: br i1 false, label %for.exit2.loopexit2, label %for.exiting_block.epil +; EPILOG-NEXT: br i1 false, label %for.exit2.loopexit3, label %for.exiting_block.epil ; EPILOG: for.exiting_block.epil: ; EPILOG-NEXT: %cmp.epil = icmp eq i64 %n, 42 -; EPILOG-NEXT: br i1 %cmp.epil, label %for.exit2.loopexit2, label %for.body.epil +; EPILOG-NEXT: br i1 %cmp.epil, label %for.exit2.loopexit3, label %for.body.epil ; EPILOG: for.body.epil: ; EPILOG-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil ; EPILOG-NEXT: %11 = load i32, ptr %arrayidx.epil, align 4 @@ -488,11 +487,11 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; EPILOG: for.exit2.loopexit: ; EPILOG-NEXT: %retval.ph = phi i32 [ 42, %for.exiting_block ], [ %sum.02, %header ], [ %add, %for.body ], [ 42, %for.exiting_block.1 ], [ %add.1, %for.body.1 ], [ 42, %for.exiting_block.2 ], [ %add.2, %for.body.2 ], [ 42, %for.exiting_block.3 ], [ %add.3, %for.body.3 ], [ 42, %for.exiting_block.4 ], [ %add.4, %for.body.4 ], [ 42, %for.exiting_block.5 ], [ %add.5, %for.body.5 ], [ 42, %for.exiting_block.6 ], [ %add.6, %for.body.6 ], [ 42, %for.exiting_block.7 ] ; EPILOG-NEXT: br label %for.exit2 -; EPILOG: for.exit2.loopexit2: -; EPILOG-NEXT: %retval.ph3 = phi i32 [ 42, %for.exiting_block.epil ], [ %sum.02.epil, %header.epil ] +; EPILOG: for.exit2.loopexit3: +; EPILOG-NEXT: %retval.ph4 = phi i32 [ 42, %for.exiting_block.epil ], [ %sum.02.epil, %header.epil ] ; EPILOG-NEXT: br label %for.exit2 ; EPILOG: for.exit2: -; EPILOG-NEXT: %retval = phi i32 [ %retval.ph, %for.exit2.loopexit ], [ %retval.ph3, %for.exit2.loopexit2 ] +; EPILOG-NEXT: %retval = phi i32 [ %retval.ph, %for.exit2.loopexit ], [ %retval.ph4, %for.exit2.loopexit3 ] ; EPILOG-NEXT: ret i32 %retval ; ; EPILOG-BLOCK-LABEL: @test2( @@ -501,7 +500,7 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %for.end.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -529,19 +528,18 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; EPILOG-BLOCK-NEXT: %indvars.iv.next.1 = add i64 %indvars.iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp eq i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %for.end.unr-lcssa.loopexit, label %header, !llvm.loop !2 -; EPILOG-BLOCK: for.end.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %sum.0.lcssa.ph.ph = phi i32 [ %add.1, %for.body.1 ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.1, %for.body.1 ] -; EPILOG-BLOCK-NEXT: %sum.02.unr.ph = phi i32 [ %add.1, %for.body.1 ] -; EPILOG-BLOCK-NEXT: br label %for.end.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %for.end.unr-lcssa, label %header, !llvm.loop !2 ; EPILOG-BLOCK: for.end.unr-lcssa: -; EPILOG-BLOCK-NEXT: %sum.0.lcssa.ph = phi i32 [ poison, %entry ], [ %sum.0.lcssa.ph.ph, %for.end.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %for.end.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %for.end.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %sum.0.lcssa.ph = phi i32 [ %add.1, %for.body.1 ] +; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.1, %for.body.1 ] +; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ %add.1, %for.body.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %for.end ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %for.end.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %for.end.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: ; EPILOG-BLOCK-NEXT: br i1 false, label %for.exit2, label %for.exiting_block.epil @@ -549,9 +547,9 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; EPILOG-BLOCK-NEXT: %cmp.epil = icmp eq i64 %n, 42 ; EPILOG-BLOCK-NEXT: br i1 %cmp.epil, label %for.exit2, label %for.body.epil ; EPILOG-BLOCK: for.body.epil: -; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.unr +; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil.init ; EPILOG-BLOCK-NEXT: %5 = load i32, ptr %arrayidx.epil, align 4 -; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.unr +; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.epil.init ; EPILOG-BLOCK-NEXT: br label %for.end ; EPILOG-BLOCK: for.end: ; EPILOG-BLOCK-NEXT: %sum.0.lcssa = phi i32 [ %sum.0.lcssa.ph, %for.end.unr-lcssa ], [ %add.epil, %for.body.epil ] @@ -560,7 +558,7 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; EPILOG-BLOCK-NEXT: %retval.ph = phi i32 [ 42, %for.exiting_block ], [ %sum.02, %header ], [ %add, %for.body ], [ 42, %for.exiting_block.1 ] ; EPILOG-BLOCK-NEXT: br label %for.exit2 ; EPILOG-BLOCK: for.exit2: -; EPILOG-BLOCK-NEXT: %retval = phi i32 [ %sum.02.unr, %header.epil ], [ 42, %for.exiting_block.epil ], [ %retval.ph, %for.exit2.loopexit ] +; EPILOG-BLOCK-NEXT: %retval = phi i32 [ %sum.02.epil.init, %header.epil ], [ 42, %for.exiting_block.epil ], [ %retval.ph, %for.exit2.loopexit ] ; EPILOG-BLOCK-NEXT: ret i32 %retval ; ; PROLOG-LABEL: @test2( @@ -796,7 +794,7 @@ define void @test3(i64 %trip, i64 %add, i1 %arg) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %exit2.loopexit.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %loop_header @@ -812,7 +810,7 @@ define void @test3(i64 %trip, i64 %add, i1 %arg) { ; EPILOG-NEXT: ] ; EPILOG: exit3.loopexit: ; EPILOG-NEXT: br label %exit3 -; EPILOG: exit3.loopexit2: +; EPILOG: exit3.loopexit3: ; EPILOG-NEXT: br label %exit3 ; EPILOG: exit3: ; EPILOG-NEXT: ret void @@ -877,33 +875,33 @@ define void @test3(i64 %trip, i64 %add, i1 %arg) { ; EPILOG-NEXT: %sum.next.7 = add i64 %sum.next.6, %add ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit2.loopexit.unr-lcssa.loopexit +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit2.loopexit.unr-lcssa ; EPILOG: exit1.loopexit: ; EPILOG-NEXT: br label %exit1 -; EPILOG: exit1.loopexit1: +; EPILOG: exit1.loopexit2: ; EPILOG-NEXT: br label %exit1 ; EPILOG: exit1: ; EPILOG-NEXT: ret void -; EPILOG: exit2.loopexit.unr-lcssa.loopexit: -; EPILOG-NEXT: %iv.unr.ph = phi i64 [ %iv_next.7, %loop_latch.7 ] -; EPILOG-NEXT: %sum.unr.ph = phi i64 [ %sum.next.7, %loop_latch.7 ] -; EPILOG-NEXT: br label %exit2.loopexit.unr-lcssa ; EPILOG: exit2.loopexit.unr-lcssa: -; EPILOG-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit2.loopexit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %sum.unr = phi i64 [ 0, %entry ], [ %sum.unr.ph, %exit2.loopexit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %iv.unr = phi i64 [ %iv_next.7, %loop_latch.7 ] +; EPILOG-NEXT: %sum.unr = phi i64 [ %sum.next.7, %loop_latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit2.loopexit ; EPILOG: loop_header.epil.preheader: +; EPILOG-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit2.loopexit.unr-lcssa ] +; EPILOG-NEXT: %sum.epil.init = phi i64 [ 0, %entry ], [ %sum.unr, %exit2.loopexit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %loop_header.epil ; EPILOG: loop_header.epil: -; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.unr, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] -; EPILOG-NEXT: %sum.epil = phi i64 [ %sum.unr, %loop_header.epil.preheader ], [ %sum.next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.epil.init, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %sum.epil = phi i64 [ %sum.epil.init, %loop_header.epil.preheader ], [ %sum.next.epil, %loop_latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %loop_header.epil.preheader ], [ %epil.iter.next, %loop_latch.epil ] ; EPILOG-NEXT: br i1 %arg, label %loop_latch.epil, label %loop_exiting_bb1.epil ; EPILOG: loop_exiting_bb1.epil: ; EPILOG-NEXT: switch i64 %sum.epil, label %loop_latch.epil [ -; EPILOG-NEXT: i64 24, label %exit1.loopexit1 -; EPILOG-NEXT: i64 42, label %exit3.loopexit2 +; EPILOG-NEXT: i64 24, label %exit1.loopexit2 +; EPILOG-NEXT: i64 42, label %exit3.loopexit3 ; EPILOG-NEXT: ] ; EPILOG: loop_latch.epil: ; EPILOG-NEXT: %iv_next.epil = add nuw nsw i64 %iv.epil, 1 @@ -923,7 +921,7 @@ define void @test3(i64 %trip, i64 %add, i1 %arg) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %exit2.loopexit.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %loop_header @@ -954,24 +952,24 @@ define void @test3(i64 %trip, i64 %add, i1 %arg) { ; EPILOG-BLOCK-NEXT: %sum.next.1 = add i64 %sum.next, %add ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit2.loopexit.unr-lcssa.loopexit, !llvm.loop !3 +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit2.loopexit.unr-lcssa, !llvm.loop !3 ; EPILOG-BLOCK: exit1.loopexit: ; EPILOG-BLOCK-NEXT: br label %exit1 ; EPILOG-BLOCK: exit1: ; EPILOG-BLOCK-NEXT: ret void -; EPILOG-BLOCK: exit2.loopexit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %sum.unr.ph = phi i64 [ %sum.next.1, %loop_latch.1 ] -; EPILOG-BLOCK-NEXT: br label %exit2.loopexit.unr-lcssa ; EPILOG-BLOCK: exit2.loopexit.unr-lcssa: -; EPILOG-BLOCK-NEXT: %sum.unr = phi i64 [ 0, %entry ], [ %sum.unr.ph, %exit2.loopexit.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %sum.unr = phi i64 [ %sum.next.1, %loop_latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit2.loopexit ; EPILOG-BLOCK: loop_header.epil.preheader: +; EPILOG-BLOCK-NEXT: %sum.epil.init = phi i64 [ 0, %entry ], [ %sum.unr, %exit2.loopexit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %loop_header.epil ; EPILOG-BLOCK: loop_header.epil: ; EPILOG-BLOCK-NEXT: br i1 %arg, label %loop_latch.epil, label %loop_exiting_bb1.epil ; EPILOG-BLOCK: loop_exiting_bb1.epil: -; EPILOG-BLOCK-NEXT: switch i64 %sum.unr, label %loop_latch.epil [ +; EPILOG-BLOCK-NEXT: switch i64 %sum.epil.init, label %loop_latch.epil [ ; EPILOG-BLOCK-NEXT: i64 24, label %exit1 ; EPILOG-BLOCK-NEXT: i64 42, label %exit3 ; EPILOG-BLOCK-NEXT: ] @@ -1204,7 +1202,7 @@ define i32 @hdr_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %header @@ -1286,28 +1284,27 @@ define i32 @hdr_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %indvars.iv.next.7 = add i64 %indvars.iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp eq i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latchExit.unr-lcssa.loopexit, label %header -; EPILOG: latchExit.unr-lcssa.loopexit: -; EPILOG-NEXT: %result.ph.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.7, %latch.7 ] -; EPILOG-NEXT: %sum.02.unr.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: br label %latchExit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latchExit.unr-lcssa, label %header ; EPILOG: latchExit.unr-lcssa: -; EPILOG-NEXT: %result.ph = phi i32 [ poison, %entry ], [ %result.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %result.ph = phi i32 [ %add.7, %latch.7 ] +; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.7, %latch.7 ] +; EPILOG-NEXT: %sum.02.unr = phi i32 [ %add.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.unr, %header.epil.preheader ] -; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.unr, %header.epil.preheader ] +; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.epil.init, %header.epil.preheader ] +; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.epil.init, %header.epil.preheader ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ] -; EPILOG-NEXT: br i1 %cond, label %latchExit.epilog-lcssa.loopexit2, label %for.exiting_block.epil +; EPILOG-NEXT: br i1 %cond, label %latchExit.epilog-lcssa.loopexit3, label %for.exiting_block.epil ; EPILOG: for.exiting_block.epil: ; EPILOG-NEXT: %cmp.epil = icmp eq i64 %n, 42 -; EPILOG-NEXT: br i1 %cmp.epil, label %for.exit2.loopexit4, label %latch.epil +; EPILOG-NEXT: br i1 %cmp.epil, label %for.exit2.loopexit5, label %latch.epil ; EPILOG: latch.epil: ; EPILOG-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil ; EPILOG-NEXT: %11 = load i32, ptr %arrayidx.epil, align 4 @@ -1316,22 +1313,22 @@ define i32 @hdr_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %exitcond.epil = icmp eq i64 %indvars.iv.next.epil, %n ; EPILOG-NEXT: %epil.iter.next = add i64 %epil.iter, 1 ; EPILOG-NEXT: %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter -; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit2, !llvm.loop !4 +; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit3, !llvm.loop !4 ; EPILOG: latchExit.epilog-lcssa.loopexit: ; EPILOG-NEXT: %result.ph1.ph = phi i32 [ 0, %header ], [ 0, %latch ], [ 0, %latch.1 ], [ 0, %latch.2 ], [ 0, %latch.3 ], [ 0, %latch.4 ], [ 0, %latch.5 ], [ 0, %latch.6 ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa -; EPILOG: latchExit.epilog-lcssa.loopexit2: -; EPILOG-NEXT: %result.ph1.ph3 = phi i32 [ 0, %header.epil ], [ %add.epil, %latch.epil ] +; EPILOG: latchExit.epilog-lcssa.loopexit3: +; EPILOG-NEXT: %result.ph1.ph4 = phi i32 [ 0, %header.epil ], [ %add.epil, %latch.epil ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa ; EPILOG: latchExit.epilog-lcssa: -; EPILOG-NEXT: %result.ph1 = phi i32 [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %result.ph1.ph3, %latchExit.epilog-lcssa.loopexit2 ] +; EPILOG-NEXT: %result.ph1 = phi i32 [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %result.ph1.ph4, %latchExit.epilog-lcssa.loopexit3 ] ; EPILOG-NEXT: br label %latchExit ; EPILOG: latchExit: ; EPILOG-NEXT: %result = phi i32 [ %result.ph, %latchExit.unr-lcssa ], [ %result.ph1, %latchExit.epilog-lcssa ] ; EPILOG-NEXT: ret i32 %result ; EPILOG: for.exit2.loopexit: ; EPILOG-NEXT: br label %for.exit2 -; EPILOG: for.exit2.loopexit4: +; EPILOG: for.exit2.loopexit5: ; EPILOG-NEXT: br label %for.exit2 ; EPILOG: for.exit2: ; EPILOG-NEXT: ret i32 42 @@ -1342,7 +1339,7 @@ define i32 @hdr_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -1370,19 +1367,18 @@ define i32 @hdr_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %indvars.iv.next.1 = add i64 %indvars.iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp eq i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latchExit.unr-lcssa.loopexit, label %header, !llvm.loop !4 -; EPILOG-BLOCK: latchExit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %result.ph.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %sum.02.unr.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: br label %latchExit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latchExit.unr-lcssa, label %header, !llvm.loop !4 ; EPILOG-BLOCK: latchExit.unr-lcssa: -; EPILOG-BLOCK-NEXT: %result.ph = phi i32 [ poison, %entry ], [ %result.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %result.ph = phi i32 [ %add.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ %add.1, %latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: ; EPILOG-BLOCK-NEXT: br i1 %cond, label %latchExit.epilog-lcssa, label %for.exiting_block.epil @@ -1390,9 +1386,9 @@ define i32 @hdr_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %cmp.epil = icmp eq i64 %n, 42 ; EPILOG-BLOCK-NEXT: br i1 %cmp.epil, label %for.exit2, label %latch.epil ; EPILOG-BLOCK: latch.epil: -; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.unr +; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil.init ; EPILOG-BLOCK-NEXT: %5 = load i32, ptr %arrayidx.epil, align 4 -; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.unr +; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.epil.init ; EPILOG-BLOCK-NEXT: br label %latchExit.epilog-lcssa ; EPILOG-BLOCK: latchExit.epilog-lcssa.loopexit: ; EPILOG-BLOCK-NEXT: %result.ph1.ph = phi i32 [ 0, %header ], [ 0, %latch ] @@ -1644,7 +1640,7 @@ define i32 @otherblock_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %header @@ -1726,28 +1722,27 @@ define i32 @otherblock_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %indvars.iv.next.7 = add i64 %indvars.iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp eq i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latchExit.unr-lcssa.loopexit, label %header -; EPILOG: latchExit.unr-lcssa.loopexit: -; EPILOG-NEXT: %result.ph.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.7, %latch.7 ] -; EPILOG-NEXT: %sum.02.unr.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: br label %latchExit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latchExit.unr-lcssa, label %header ; EPILOG: latchExit.unr-lcssa: -; EPILOG-NEXT: %result.ph = phi i32 [ poison, %entry ], [ %result.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %result.ph = phi i32 [ %add.7, %latch.7 ] +; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.7, %latch.7 ] +; EPILOG-NEXT: %sum.02.unr = phi i32 [ %add.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.unr, %header.epil.preheader ] -; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.unr, %header.epil.preheader ] +; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.epil.init, %header.epil.preheader ] +; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.epil.init, %header.epil.preheader ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ] -; EPILOG-NEXT: br i1 %cond, label %for.exit2.loopexit2, label %for.exiting_block.epil +; EPILOG-NEXT: br i1 %cond, label %for.exit2.loopexit3, label %for.exiting_block.epil ; EPILOG: for.exiting_block.epil: ; EPILOG-NEXT: %cmp.epil = icmp eq i64 %n, 42 -; EPILOG-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa.loopexit3, label %latch.epil +; EPILOG-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa.loopexit4, label %latch.epil ; EPILOG: latch.epil: ; EPILOG-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil ; EPILOG-NEXT: %11 = load i32, ptr %arrayidx.epil, align 4 @@ -1756,22 +1751,22 @@ define i32 @otherblock_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %exitcond.epil = icmp eq i64 %indvars.iv.next.epil, %n ; EPILOG-NEXT: %epil.iter.next = add i64 %epil.iter, 1 ; EPILOG-NEXT: %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter -; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit3, !llvm.loop !5 +; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit4, !llvm.loop !5 ; EPILOG: latchExit.epilog-lcssa.loopexit: ; EPILOG-NEXT: %result.ph1.ph = phi i32 [ 2, %for.exiting_block ], [ 2, %for.exiting_block.1 ], [ 2, %for.exiting_block.2 ], [ 2, %for.exiting_block.3 ], [ 2, %for.exiting_block.4 ], [ 2, %for.exiting_block.5 ], [ 2, %for.exiting_block.6 ], [ 2, %for.exiting_block.7 ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa -; EPILOG: latchExit.epilog-lcssa.loopexit3: -; EPILOG-NEXT: %result.ph1.ph4 = phi i32 [ 2, %for.exiting_block.epil ], [ %add.epil, %latch.epil ] +; EPILOG: latchExit.epilog-lcssa.loopexit4: +; EPILOG-NEXT: %result.ph1.ph5 = phi i32 [ 2, %for.exiting_block.epil ], [ %add.epil, %latch.epil ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa ; EPILOG: latchExit.epilog-lcssa: -; EPILOG-NEXT: %result.ph1 = phi i32 [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %result.ph1.ph4, %latchExit.epilog-lcssa.loopexit3 ] +; EPILOG-NEXT: %result.ph1 = phi i32 [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %result.ph1.ph5, %latchExit.epilog-lcssa.loopexit4 ] ; EPILOG-NEXT: br label %latchExit ; EPILOG: latchExit: ; EPILOG-NEXT: %result = phi i32 [ %result.ph, %latchExit.unr-lcssa ], [ %result.ph1, %latchExit.epilog-lcssa ] ; EPILOG-NEXT: ret i32 %result ; EPILOG: for.exit2.loopexit: ; EPILOG-NEXT: br label %for.exit2 -; EPILOG: for.exit2.loopexit2: +; EPILOG: for.exit2.loopexit3: ; EPILOG-NEXT: br label %for.exit2 ; EPILOG: for.exit2: ; EPILOG-NEXT: ret i32 42 @@ -1782,7 +1777,7 @@ define i32 @otherblock_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -1810,19 +1805,18 @@ define i32 @otherblock_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %indvars.iv.next.1 = add i64 %indvars.iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp eq i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latchExit.unr-lcssa.loopexit, label %header, !llvm.loop !5 -; EPILOG-BLOCK: latchExit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %result.ph.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %sum.02.unr.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: br label %latchExit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latchExit.unr-lcssa, label %header, !llvm.loop !5 ; EPILOG-BLOCK: latchExit.unr-lcssa: -; EPILOG-BLOCK-NEXT: %result.ph = phi i32 [ poison, %entry ], [ %result.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %result.ph = phi i32 [ %add.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ %add.1, %latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: ; EPILOG-BLOCK-NEXT: br i1 %cond, label %for.exit2, label %for.exiting_block.epil @@ -1830,9 +1824,9 @@ define i32 @otherblock_latch_same_exit(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %cmp.epil = icmp eq i64 %n, 42 ; EPILOG-BLOCK-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa, label %latch.epil ; EPILOG-BLOCK: latch.epil: -; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.unr +; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil.init ; EPILOG-BLOCK-NEXT: %5 = load i32, ptr %arrayidx.epil, align 4 -; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.unr +; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.epil.init ; EPILOG-BLOCK-NEXT: br label %latchExit.epilog-lcssa ; EPILOG-BLOCK: latchExit.epilog-lcssa.loopexit: ; EPILOG-BLOCK-NEXT: %result.ph1.ph = phi i32 [ 2, %for.exiting_block ], [ 2, %for.exiting_block.1 ] @@ -2085,7 +2079,7 @@ define i32 @otherblock_latch_same_exit2(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %header @@ -2167,28 +2161,27 @@ define i32 @otherblock_latch_same_exit2(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %indvars.iv.next.7 = add i64 %indvars.iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp eq i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latchExit.unr-lcssa.loopexit, label %header -; EPILOG: latchExit.unr-lcssa.loopexit: -; EPILOG-NEXT: %result.ph.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.7, %latch.7 ] -; EPILOG-NEXT: %sum.02.unr.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: br label %latchExit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latchExit.unr-lcssa, label %header ; EPILOG: latchExit.unr-lcssa: -; EPILOG-NEXT: %result.ph = phi i32 [ poison, %entry ], [ %result.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %result.ph = phi i32 [ %add.7, %latch.7 ] +; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.7, %latch.7 ] +; EPILOG-NEXT: %sum.02.unr = phi i32 [ %add.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.unr, %header.epil.preheader ] -; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.unr, %header.epil.preheader ] +; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.epil.init, %header.epil.preheader ] +; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.epil.init, %header.epil.preheader ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ] -; EPILOG-NEXT: br i1 %cond, label %for.exit2.loopexit2, label %for.exiting_block.epil +; EPILOG-NEXT: br i1 %cond, label %for.exit2.loopexit3, label %for.exiting_block.epil ; EPILOG: for.exiting_block.epil: ; EPILOG-NEXT: %cmp.epil = icmp eq i64 %n, 42 -; EPILOG-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa.loopexit3, label %latch.epil +; EPILOG-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa.loopexit4, label %latch.epil ; EPILOG: latch.epil: ; EPILOG-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil ; EPILOG-NEXT: %11 = load i32, ptr %arrayidx.epil, align 4 @@ -2197,22 +2190,22 @@ define i32 @otherblock_latch_same_exit2(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %exitcond.epil = icmp eq i64 %indvars.iv.next.epil, %n ; EPILOG-NEXT: %epil.iter.next = add i64 %epil.iter, 1 ; EPILOG-NEXT: %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter -; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit3, !llvm.loop !6 +; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit4, !llvm.loop !6 ; EPILOG: latchExit.epilog-lcssa.loopexit: ; EPILOG-NEXT: %result.ph1.ph = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %for.exiting_block.1 ], [ %add.1, %for.exiting_block.2 ], [ %add.2, %for.exiting_block.3 ], [ %add.3, %for.exiting_block.4 ], [ %add.4, %for.exiting_block.5 ], [ %add.5, %for.exiting_block.6 ], [ %add.6, %for.exiting_block.7 ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa -; EPILOG: latchExit.epilog-lcssa.loopexit3: -; EPILOG-NEXT: %result.ph1.ph4 = phi i32 [ %sum.02.epil, %for.exiting_block.epil ], [ %add.epil, %latch.epil ] +; EPILOG: latchExit.epilog-lcssa.loopexit4: +; EPILOG-NEXT: %result.ph1.ph5 = phi i32 [ %sum.02.epil, %for.exiting_block.epil ], [ %add.epil, %latch.epil ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa ; EPILOG: latchExit.epilog-lcssa: -; EPILOG-NEXT: %result.ph1 = phi i32 [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %result.ph1.ph4, %latchExit.epilog-lcssa.loopexit3 ] +; EPILOG-NEXT: %result.ph1 = phi i32 [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %result.ph1.ph5, %latchExit.epilog-lcssa.loopexit4 ] ; EPILOG-NEXT: br label %latchExit ; EPILOG: latchExit: ; EPILOG-NEXT: %result = phi i32 [ %result.ph, %latchExit.unr-lcssa ], [ %result.ph1, %latchExit.epilog-lcssa ] ; EPILOG-NEXT: ret i32 %result ; EPILOG: for.exit2.loopexit: ; EPILOG-NEXT: br label %for.exit2 -; EPILOG: for.exit2.loopexit2: +; EPILOG: for.exit2.loopexit3: ; EPILOG-NEXT: br label %for.exit2 ; EPILOG: for.exit2: ; EPILOG-NEXT: ret i32 42 @@ -2223,7 +2216,7 @@ define i32 @otherblock_latch_same_exit2(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -2251,19 +2244,18 @@ define i32 @otherblock_latch_same_exit2(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %indvars.iv.next.1 = add i64 %indvars.iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp eq i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latchExit.unr-lcssa.loopexit, label %header, !llvm.loop !6 -; EPILOG-BLOCK: latchExit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %result.ph.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %sum.02.unr.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: br label %latchExit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latchExit.unr-lcssa, label %header, !llvm.loop !6 ; EPILOG-BLOCK: latchExit.unr-lcssa: -; EPILOG-BLOCK-NEXT: %result.ph = phi i32 [ poison, %entry ], [ %result.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %result.ph = phi i32 [ %add.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ %add.1, %latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: ; EPILOG-BLOCK-NEXT: br i1 %cond, label %for.exit2, label %for.exiting_block.epil @@ -2271,15 +2263,15 @@ define i32 @otherblock_latch_same_exit2(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %cmp.epil = icmp eq i64 %n, 42 ; EPILOG-BLOCK-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa, label %latch.epil ; EPILOG-BLOCK: latch.epil: -; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.unr +; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil.init ; EPILOG-BLOCK-NEXT: %5 = load i32, ptr %arrayidx.epil, align 4 -; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.unr +; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.epil.init ; EPILOG-BLOCK-NEXT: br label %latchExit.epilog-lcssa ; EPILOG-BLOCK: latchExit.epilog-lcssa.loopexit: ; EPILOG-BLOCK-NEXT: %result.ph1.ph = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %for.exiting_block.1 ] ; EPILOG-BLOCK-NEXT: br label %latchExit.epilog-lcssa ; EPILOG-BLOCK: latchExit.epilog-lcssa: -; EPILOG-BLOCK-NEXT: %result.ph1 = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.unr, %for.exiting_block.epil ], [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %result.ph1 = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.epil.init, %for.exiting_block.epil ], [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ] ; EPILOG-BLOCK-NEXT: br label %latchExit ; EPILOG-BLOCK: latchExit: ; EPILOG-BLOCK-NEXT: %result = phi i32 [ %result.ph, %latchExit.unr-lcssa ], [ %result.ph1, %latchExit.epilog-lcssa ] @@ -2527,7 +2519,7 @@ define i32 @otherblock_latch_same_exit3(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %header @@ -2609,52 +2601,51 @@ define i32 @otherblock_latch_same_exit3(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-NEXT: %indvars.iv.next.7 = add i64 %indvars.iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp eq i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latchExit.unr-lcssa.loopexit, label %header -; EPILOG: latchExit.unr-lcssa.loopexit: -; EPILOG-NEXT: %result.ph.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.7, %latch.7 ] -; EPILOG-NEXT: %sum.02.unr.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: br label %latchExit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latchExit.unr-lcssa, label %header ; EPILOG: latchExit.unr-lcssa: -; EPILOG-NEXT: %result.ph = phi i32 [ poison, %entry ], [ %result.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %result.ph = phi i32 [ %add.7, %latch.7 ] +; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.7, %latch.7 ] +; EPILOG-NEXT: %sum.02.unr = phi i32 [ %add.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.unr, %header.epil.preheader ] -; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.unr, %header.epil.preheader ] +; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.epil.init, %header.epil.preheader ] +; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.epil.init, %header.epil.preheader ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ] -; EPILOG-NEXT: br i1 %cond, label %for.exit2.loopexit2, label %for.exiting_block.epil +; EPILOG-NEXT: br i1 %cond, label %for.exit2.loopexit3, label %for.exiting_block.epil ; EPILOG: for.exiting_block.epil: ; EPILOG-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil ; EPILOG-NEXT: %11 = load i32, ptr %arrayidx.epil, align 4 ; EPILOG-NEXT: %add.epil = add nsw i32 %11, %sum.02.epil ; EPILOG-NEXT: %cmp.epil = icmp eq i64 %n, 42 -; EPILOG-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa.loopexit3, label %latch.epil +; EPILOG-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa.loopexit4, label %latch.epil ; EPILOG: latch.epil: ; EPILOG-NEXT: %indvars.iv.next.epil = add i64 %indvars.iv.epil, 1 ; EPILOG-NEXT: %exitcond.epil = icmp eq i64 %indvars.iv.next.epil, %n ; EPILOG-NEXT: %epil.iter.next = add i64 %epil.iter, 1 ; EPILOG-NEXT: %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter -; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit3, !llvm.loop !7 +; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit4, !llvm.loop !7 ; EPILOG: latchExit.epilog-lcssa.loopexit: ; EPILOG-NEXT: %result.ph1.ph = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %for.exiting_block.1 ], [ %add.1, %for.exiting_block.2 ], [ %add.2, %for.exiting_block.3 ], [ %add.3, %for.exiting_block.4 ], [ %add.4, %for.exiting_block.5 ], [ %add.5, %for.exiting_block.6 ], [ %add.6, %for.exiting_block.7 ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa -; EPILOG: latchExit.epilog-lcssa.loopexit3: -; EPILOG-NEXT: %result.ph1.ph4 = phi i32 [ %sum.02.epil, %for.exiting_block.epil ], [ %add.epil, %latch.epil ] +; EPILOG: latchExit.epilog-lcssa.loopexit4: +; EPILOG-NEXT: %result.ph1.ph5 = phi i32 [ %sum.02.epil, %for.exiting_block.epil ], [ %add.epil, %latch.epil ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa ; EPILOG: latchExit.epilog-lcssa: -; EPILOG-NEXT: %result.ph1 = phi i32 [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %result.ph1.ph4, %latchExit.epilog-lcssa.loopexit3 ] +; EPILOG-NEXT: %result.ph1 = phi i32 [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %result.ph1.ph5, %latchExit.epilog-lcssa.loopexit4 ] ; EPILOG-NEXT: br label %latchExit ; EPILOG: latchExit: ; EPILOG-NEXT: %result = phi i32 [ %result.ph, %latchExit.unr-lcssa ], [ %result.ph1, %latchExit.epilog-lcssa ] ; EPILOG-NEXT: ret i32 %result ; EPILOG: for.exit2.loopexit: ; EPILOG-NEXT: br label %for.exit2 -; EPILOG: for.exit2.loopexit2: +; EPILOG: for.exit2.loopexit3: ; EPILOG-NEXT: br label %for.exit2 ; EPILOG: for.exit2: ; EPILOG-NEXT: ret i32 42 @@ -2665,7 +2656,7 @@ define i32 @otherblock_latch_same_exit3(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -2693,26 +2684,25 @@ define i32 @otherblock_latch_same_exit3(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %indvars.iv.next.1 = add i64 %indvars.iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp eq i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latchExit.unr-lcssa.loopexit, label %header, !llvm.loop !7 -; EPILOG-BLOCK: latchExit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %result.ph.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %sum.02.unr.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: br label %latchExit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latchExit.unr-lcssa, label %header, !llvm.loop !7 ; EPILOG-BLOCK: latchExit.unr-lcssa: -; EPILOG-BLOCK-NEXT: %result.ph = phi i32 [ poison, %entry ], [ %result.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %result.ph = phi i32 [ %add.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ %add.1, %latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: ; EPILOG-BLOCK-NEXT: br i1 %cond, label %for.exit2, label %for.exiting_block.epil ; EPILOG-BLOCK: for.exiting_block.epil: -; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.unr +; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil.init ; EPILOG-BLOCK-NEXT: %5 = load i32, ptr %arrayidx.epil, align 4 -; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.unr +; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %5, %sum.02.epil.init ; EPILOG-BLOCK-NEXT: %cmp.epil = icmp eq i64 %n, 42 ; EPILOG-BLOCK-NEXT: br i1 %cmp.epil, label %latchExit.epilog-lcssa, label %latch.epil ; EPILOG-BLOCK: latch.epil: @@ -2721,7 +2711,7 @@ define i32 @otherblock_latch_same_exit3(ptr nocapture %a, i64 %n, i1 %cond) { ; EPILOG-BLOCK-NEXT: %result.ph1.ph = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %for.exiting_block.1 ] ; EPILOG-BLOCK-NEXT: br label %latchExit.epilog-lcssa ; EPILOG-BLOCK: latchExit.epilog-lcssa: -; EPILOG-BLOCK-NEXT: %result.ph1 = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.unr, %for.exiting_block.epil ], [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %result.ph1 = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.epil.init, %for.exiting_block.epil ], [ %result.ph1.ph, %latchExit.epilog-lcssa.loopexit ] ; EPILOG-BLOCK-NEXT: br label %latchExit ; EPILOG-BLOCK: latchExit: ; EPILOG-BLOCK-NEXT: %result = phi i32 [ %result.ph, %latchExit.unr-lcssa ], [ %result.ph1, %latchExit.epilog-lcssa ] @@ -3013,7 +3003,7 @@ define void @unique_exit(i32 %N, i32 %M) { ; EPILOG-NEXT: %1 = add i32 %0, -1 ; EPILOG-NEXT: %xtraiter = and i32 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i32 %1, 7 -; EPILOG-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %preheader.new +; EPILOG-NEXT: br i1 %2, label %header.epil.preheader, label %preheader.new ; EPILOG: preheader.new: ; EPILOG-NEXT: %unroll_iter = sub i32 %0, %xtraiter ; EPILOG-NEXT: br label %header @@ -3054,37 +3044,36 @@ define void @unique_exit(i32 %N, i32 %M) { ; EPILOG: latch.7: ; EPILOG-NEXT: %niter.next.7 = add nuw i32 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i32 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %header, label %latchExit.unr-lcssa.loopexit -; EPILOG: latchExit.unr-lcssa.loopexit: -; EPILOG-NEXT: %i2.ph.ph.ph = phi i32 [ -1, %latch.7 ] -; EPILOG-NEXT: %i4.unr.ph = phi i32 [ %inc.7, %latch.7 ] -; EPILOG-NEXT: br label %latchExit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %header, label %latchExit.unr-lcssa ; EPILOG: latchExit.unr-lcssa: -; EPILOG-NEXT: %i2.ph.ph = phi i32 [ poison, %preheader ], [ %i2.ph.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %i4.unr = phi i32 [ 0, %preheader ], [ %i4.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %i2.ph.ph = phi i32 [ -1, %latch.7 ] +; EPILOG-NEXT: %i4.unr = phi i32 [ %inc.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i32 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %i4.epil.init = phi i32 [ 0, %preheader ], [ %i4.unr, %latchExit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod2 = icmp ne i32 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %i4.epil = phi i32 [ %inc.epil, %latch.epil ], [ %i4.unr, %header.epil.preheader ] +; EPILOG-NEXT: %i4.epil = phi i32 [ %inc.epil, %latch.epil ], [ %i4.epil.init, %header.epil.preheader ] ; EPILOG-NEXT: %epil.iter = phi i32 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ] ; EPILOG-NEXT: %inc.epil = add nuw i32 %i4.epil, 1 ; EPILOG-NEXT: %cmp1.epil = icmp ult i32 %inc.epil, %N -; EPILOG-NEXT: br i1 %cmp1.epil, label %latch.epil, label %latchExit.epilog-lcssa.loopexit2 +; EPILOG-NEXT: br i1 %cmp1.epil, label %latch.epil, label %latchExit.epilog-lcssa.loopexit3 ; EPILOG: latch.epil: ; EPILOG-NEXT: %cmp.epil = icmp ult i32 %inc.epil, %M.shifted ; EPILOG-NEXT: %epil.iter.next = add i32 %epil.iter, 1 ; EPILOG-NEXT: %epil.iter.cmp = icmp ne i32 %epil.iter.next, %xtraiter -; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit2, !llvm.loop !8 +; EPILOG-NEXT: br i1 %epil.iter.cmp, label %header.epil, label %latchExit.epilog-lcssa.loopexit3, !llvm.loop !8 ; EPILOG: latchExit.epilog-lcssa.loopexit: ; EPILOG-NEXT: %i2.ph.ph1.ph = phi i32 [ %i4, %header ], [ %inc, %latch ], [ %inc.1, %latch.1 ], [ %inc.2, %latch.2 ], [ %inc.3, %latch.3 ], [ %inc.4, %latch.4 ], [ %inc.5, %latch.5 ], [ %inc.6, %latch.6 ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa -; EPILOG: latchExit.epilog-lcssa.loopexit2: -; EPILOG-NEXT: %i2.ph.ph1.ph3 = phi i32 [ %i4.epil, %header.epil ], [ -1, %latch.epil ] +; EPILOG: latchExit.epilog-lcssa.loopexit3: +; EPILOG-NEXT: %i2.ph.ph1.ph4 = phi i32 [ %i4.epil, %header.epil ], [ -1, %latch.epil ] ; EPILOG-NEXT: br label %latchExit.epilog-lcssa ; EPILOG: latchExit.epilog-lcssa: -; EPILOG-NEXT: %i2.ph.ph1 = phi i32 [ %i2.ph.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %i2.ph.ph1.ph3, %latchExit.epilog-lcssa.loopexit2 ] +; EPILOG-NEXT: %i2.ph.ph1 = phi i32 [ %i2.ph.ph1.ph, %latchExit.epilog-lcssa.loopexit ], [ %i2.ph.ph1.ph4, %latchExit.epilog-lcssa.loopexit3 ] ; EPILOG-NEXT: br label %latchExit ; EPILOG: latchExit: ; EPILOG-NEXT: %i2.ph = phi i32 [ %i2.ph.ph, %latchExit.unr-lcssa ], [ %i2.ph.ph1, %latchExit.epilog-lcssa ] @@ -3098,7 +3087,7 @@ define void @unique_exit(i32 %N, i32 %M) { ; EPILOG-BLOCK-NEXT: %1 = add i32 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i32 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i32 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %latchExit.unr-lcssa, label %preheader.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %header.epil.preheader, label %preheader.new ; EPILOG-BLOCK: preheader.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i32 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -3115,20 +3104,19 @@ define void @unique_exit(i32 %N, i32 %M) { ; EPILOG-BLOCK: latch.1: ; EPILOG-BLOCK-NEXT: %niter.next.1 = add nuw i32 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i32 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %header, label %latchExit.unr-lcssa.loopexit, !llvm.loop !8 -; EPILOG-BLOCK: latchExit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %i2.ph.ph.ph = phi i32 [ -1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %i4.unr.ph = phi i32 [ %inc.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: br label %latchExit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %header, label %latchExit.unr-lcssa, !llvm.loop !8 ; EPILOG-BLOCK: latchExit.unr-lcssa: -; EPILOG-BLOCK-NEXT: %i2.ph.ph = phi i32 [ poison, %preheader ], [ %i2.ph.ph.ph, %latchExit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %i4.unr = phi i32 [ 0, %preheader ], [ %i4.unr.ph, %latchExit.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %i2.ph.ph = phi i32 [ -1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %i4.unr = phi i32 [ %inc.1, %latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i32 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchExit ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %i4.epil.init = phi i32 [ 0, %preheader ], [ %i4.unr, %latchExit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod2 = icmp ne i32 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: -; EPILOG-BLOCK-NEXT: %inc.epil = add nuw i32 %i4.unr, 1 +; EPILOG-BLOCK-NEXT: %inc.epil = add nuw i32 %i4.epil.init, 1 ; EPILOG-BLOCK-NEXT: %cmp1.epil = icmp ult i32 %inc.epil, %N ; EPILOG-BLOCK-NEXT: br i1 %cmp1.epil, label %latch.epil, label %latchExit.epilog-lcssa ; EPILOG-BLOCK: latch.epil: @@ -3137,7 +3125,7 @@ define void @unique_exit(i32 %N, i32 %M) { ; EPILOG-BLOCK-NEXT: %i2.ph.ph1.ph = phi i32 [ %i4, %header ], [ %inc, %latch ] ; EPILOG-BLOCK-NEXT: br label %latchExit.epilog-lcssa ; EPILOG-BLOCK: latchExit.epilog-lcssa: -; EPILOG-BLOCK-NEXT: %i2.ph.ph1 = phi i32 [ -1, %latch.epil ], [ %i4.unr, %header.epil ], [ %i2.ph.ph1.ph, %latchExit.epilog-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %i2.ph.ph1 = phi i32 [ -1, %latch.epil ], [ %i4.epil.init, %header.epil ], [ %i2.ph.ph1.ph, %latchExit.epilog-lcssa.loopexit ] ; EPILOG-BLOCK-NEXT: br label %latchExit ; EPILOG-BLOCK: latchExit: ; EPILOG-BLOCK-NEXT: %i2.ph = phi i32 [ %i2.ph.ph, %latchExit.unr-lcssa ], [ %i2.ph.ph1, %latchExit.epilog-lcssa ] @@ -3300,7 +3288,7 @@ define i64 @test5(i64 %trip, i64 %add, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %latchexit.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %loop_header @@ -3390,39 +3378,38 @@ define i64 @test5(i64 %trip, i64 %add, i1 %cond) { ; EPILOG-NEXT: %sum.next.7 = add i64 %sum.next.6, %add ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %latchexit.unr-lcssa.loopexit +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %latchexit.unr-lcssa ; EPILOG: exit1.loopexit: ; EPILOG-NEXT: %result.ph = phi i64 [ %ivy, %loop_exiting ], [ %ivy, %loop_exiting ], [ %ivy.1, %loop_exiting.1 ], [ %ivy.1, %loop_exiting.1 ], [ %ivy.2, %loop_exiting.2 ], [ %ivy.2, %loop_exiting.2 ], [ %ivy.3, %loop_exiting.3 ], [ %ivy.3, %loop_exiting.3 ], [ %ivy.4, %loop_exiting.4 ], [ %ivy.4, %loop_exiting.4 ], [ %ivy.5, %loop_exiting.5 ], [ %ivy.5, %loop_exiting.5 ], [ %ivy.6, %loop_exiting.6 ], [ %ivy.6, %loop_exiting.6 ], [ %ivy.7, %loop_exiting.7 ], [ %ivy.7, %loop_exiting.7 ] ; EPILOG-NEXT: br label %exit1 -; EPILOG: exit1.loopexit2: -; EPILOG-NEXT: %result.ph3 = phi i64 [ %ivy.epil, %loop_exiting.epil ], [ %ivy.epil, %loop_exiting.epil ] +; EPILOG: exit1.loopexit3: +; EPILOG-NEXT: %result.ph4 = phi i64 [ %ivy.epil, %loop_exiting.epil ], [ %ivy.epil, %loop_exiting.epil ] ; EPILOG-NEXT: br label %exit1 ; EPILOG: exit1: -; EPILOG-NEXT: %result = phi i64 [ %result.ph, %exit1.loopexit ], [ %result.ph3, %exit1.loopexit2 ] +; EPILOG-NEXT: %result = phi i64 [ %result.ph, %exit1.loopexit ], [ %result.ph4, %exit1.loopexit3 ] ; EPILOG-NEXT: ret i64 %result -; EPILOG: latchexit.unr-lcssa.loopexit: -; EPILOG-NEXT: %sum.next.lcssa.ph.ph = phi i64 [ %sum.next.7, %loop_latch.7 ] -; EPILOG-NEXT: %iv.unr.ph = phi i64 [ %iv_next.7, %loop_latch.7 ] -; EPILOG-NEXT: %sum.unr.ph = phi i64 [ %sum.next.7, %loop_latch.7 ] -; EPILOG-NEXT: br label %latchexit.unr-lcssa ; EPILOG: latchexit.unr-lcssa: -; EPILOG-NEXT: %sum.next.lcssa.ph = phi i64 [ poison, %entry ], [ %sum.next.lcssa.ph.ph, %latchexit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %latchexit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %sum.unr = phi i64 [ 0, %entry ], [ %sum.unr.ph, %latchexit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %sum.next.lcssa.ph = phi i64 [ %sum.next.7, %loop_latch.7 ] +; EPILOG-NEXT: %iv.unr = phi i64 [ %iv_next.7, %loop_latch.7 ] +; EPILOG-NEXT: %sum.unr = phi i64 [ %sum.next.7, %loop_latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %latchexit ; EPILOG: loop_header.epil.preheader: +; EPILOG-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %latchexit.unr-lcssa ] +; EPILOG-NEXT: %sum.epil.init = phi i64 [ 0, %entry ], [ %sum.unr, %latchexit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-NEXT: br label %loop_header.epil ; EPILOG: loop_header.epil: -; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.unr, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] -; EPILOG-NEXT: %sum.epil = phi i64 [ %sum.unr, %loop_header.epil.preheader ], [ %sum.next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.epil.init, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %sum.epil = phi i64 [ %sum.epil.init, %loop_header.epil.preheader ], [ %sum.next.epil, %loop_latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %loop_header.epil.preheader ], [ %epil.iter.next, %loop_latch.epil ] ; EPILOG-NEXT: br i1 %cond, label %loop_latch.epil, label %loop_exiting.epil ; EPILOG: loop_exiting.epil: ; EPILOG-NEXT: %ivy.epil = add i64 %iv.epil, %add ; EPILOG-NEXT: switch i64 %sum.epil, label %loop_latch.epil [ -; EPILOG-NEXT: i64 24, label %exit1.loopexit2 -; EPILOG-NEXT: i64 42, label %exit1.loopexit2 +; EPILOG-NEXT: i64 24, label %exit1.loopexit3 +; EPILOG-NEXT: i64 42, label %exit1.loopexit3 ; EPILOG-NEXT: ] ; EPILOG: loop_latch.epil: ; EPILOG-NEXT: %iv_next.epil = add nuw nsw i64 %iv.epil, 1 @@ -3444,7 +3431,7 @@ define i64 @test5(i64 %trip, i64 %add, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %latchexit.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %loop_header @@ -3474,36 +3461,35 @@ define i64 @test5(i64 %trip, i64 %add, i1 %cond) { ; EPILOG-BLOCK-NEXT: %sum.next.1 = add i64 %sum.next, %add ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %latchexit.unr-lcssa.loopexit, !llvm.loop !9 +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %latchexit.unr-lcssa, !llvm.loop !9 ; EPILOG-BLOCK: exit1.loopexit: ; EPILOG-BLOCK-NEXT: %result.ph = phi i64 [ %ivy, %loop_exiting ], [ %ivy, %loop_exiting ], [ %ivy.1, %loop_exiting.1 ], [ %ivy.1, %loop_exiting.1 ] ; EPILOG-BLOCK-NEXT: br label %exit1 ; EPILOG-BLOCK: exit1: ; EPILOG-BLOCK-NEXT: %result = phi i64 [ %ivy.epil, %loop_exiting.epil ], [ %ivy.epil, %loop_exiting.epil ], [ %result.ph, %exit1.loopexit ] ; EPILOG-BLOCK-NEXT: ret i64 %result -; EPILOG-BLOCK: latchexit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %sum.next.lcssa.ph.ph = phi i64 [ %sum.next.1, %loop_latch.1 ] -; EPILOG-BLOCK-NEXT: %iv.unr.ph = phi i64 [ %iv_next.1, %loop_latch.1 ] -; EPILOG-BLOCK-NEXT: %sum.unr.ph = phi i64 [ %sum.next.1, %loop_latch.1 ] -; EPILOG-BLOCK-NEXT: br label %latchexit.unr-lcssa ; EPILOG-BLOCK: latchexit.unr-lcssa: -; EPILOG-BLOCK-NEXT: %sum.next.lcssa.ph = phi i64 [ poison, %entry ], [ %sum.next.lcssa.ph.ph, %latchexit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %latchexit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %sum.unr = phi i64 [ 0, %entry ], [ %sum.unr.ph, %latchexit.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %sum.next.lcssa.ph = phi i64 [ %sum.next.1, %loop_latch.1 ] +; EPILOG-BLOCK-NEXT: %iv.unr = phi i64 [ %iv_next.1, %loop_latch.1 ] +; EPILOG-BLOCK-NEXT: %sum.unr = phi i64 [ %sum.next.1, %loop_latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %latchexit ; EPILOG-BLOCK: loop_header.epil.preheader: +; EPILOG-BLOCK-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %latchexit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %sum.epil.init = phi i64 [ 0, %entry ], [ %sum.unr, %latchexit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-BLOCK-NEXT: br label %loop_header.epil ; EPILOG-BLOCK: loop_header.epil: ; EPILOG-BLOCK-NEXT: br i1 %cond, label %loop_latch.epil, label %loop_exiting.epil ; EPILOG-BLOCK: loop_exiting.epil: -; EPILOG-BLOCK-NEXT: %ivy.epil = add i64 %iv.unr, %add -; EPILOG-BLOCK-NEXT: switch i64 %sum.unr, label %loop_latch.epil [ +; EPILOG-BLOCK-NEXT: %ivy.epil = add i64 %iv.epil.init, %add +; EPILOG-BLOCK-NEXT: switch i64 %sum.epil.init, label %loop_latch.epil [ ; EPILOG-BLOCK-NEXT: i64 24, label %exit1 ; EPILOG-BLOCK-NEXT: i64 42, label %exit1 ; EPILOG-BLOCK-NEXT: ] ; EPILOG-BLOCK: loop_latch.epil: -; EPILOG-BLOCK-NEXT: %sum.next.epil = add i64 %sum.unr, %add +; EPILOG-BLOCK-NEXT: %sum.next.epil = add i64 %sum.epil.init, %add ; EPILOG-BLOCK-NEXT: br label %latchexit ; EPILOG-BLOCK: latchexit: ; EPILOG-BLOCK-NEXT: %sum.next.lcssa = phi i64 [ %sum.next.lcssa.ph, %latchexit.unr-lcssa ], [ %sum.next.epil, %loop_latch.epil ] @@ -3752,7 +3738,7 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %latch_exit.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %header @@ -3834,28 +3820,27 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) { ; EPILOG-NEXT: %indvars.iv.next.7 = add i64 %indvars.iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp eq i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latch_exit.unr-lcssa.loopexit, label %header -; EPILOG: latch_exit.unr-lcssa.loopexit: -; EPILOG-NEXT: %sum.0.lcssa.ph.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.7, %latch.7 ] -; EPILOG-NEXT: %sum.02.unr.ph = phi i32 [ %add.7, %latch.7 ] -; EPILOG-NEXT: br label %latch_exit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %latch_exit.unr-lcssa, label %header ; EPILOG: latch_exit.unr-lcssa: -; EPILOG-NEXT: %sum.0.lcssa.ph = phi i32 [ poison, %entry ], [ %sum.0.lcssa.ph.ph, %latch_exit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latch_exit.unr-lcssa.loopexit ] -; EPILOG-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latch_exit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %sum.0.lcssa.ph = phi i32 [ %add.7, %latch.7 ] +; EPILOG-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.7, %latch.7 ] +; EPILOG-NEXT: %sum.02.unr = phi i32 [ %add.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latch_exit ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latch_exit.unr-lcssa ] +; EPILOG-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latch_exit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.unr, %header.epil.preheader ] -; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.unr, %header.epil.preheader ] +; EPILOG-NEXT: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.epil.init, %header.epil.preheader ] +; EPILOG-NEXT: %sum.02.epil = phi i32 [ %add.epil, %latch.epil ], [ %sum.02.epil.init, %header.epil.preheader ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ] -; EPILOG-NEXT: br i1 false, label %for.exit2.loopexit2, label %for.exiting_block.epil +; EPILOG-NEXT: br i1 false, label %for.exit2.loopexit3, label %for.exiting_block.epil ; EPILOG: for.exiting_block.epil: ; EPILOG-NEXT: %cmp.epil = icmp eq i64 %n, 42 -; EPILOG-NEXT: br i1 %cmp.epil, label %for.exit2.loopexit2, label %latch.epil +; EPILOG-NEXT: br i1 %cmp.epil, label %for.exit2.loopexit3, label %latch.epil ; EPILOG: latch.epil: ; EPILOG-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil ; EPILOG-NEXT: %load.epil = load i32, ptr %arrayidx.epil, align 4 @@ -3874,11 +3859,11 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) { ; EPILOG: for.exit2.loopexit: ; EPILOG-NEXT: %retval.ph = phi i32 [ 42, %for.exiting_block ], [ %sum.02, %header ], [ %add, %latch ], [ 42, %for.exiting_block.1 ], [ %add.1, %latch.1 ], [ 42, %for.exiting_block.2 ], [ %add.2, %latch.2 ], [ 42, %for.exiting_block.3 ], [ %add.3, %latch.3 ], [ 42, %for.exiting_block.4 ], [ %add.4, %latch.4 ], [ 42, %for.exiting_block.5 ], [ %add.5, %latch.5 ], [ 42, %for.exiting_block.6 ], [ %add.6, %latch.6 ], [ 42, %for.exiting_block.7 ] ; EPILOG-NEXT: br label %for.exit2 -; EPILOG: for.exit2.loopexit2: -; EPILOG-NEXT: %retval.ph3 = phi i32 [ 42, %for.exiting_block.epil ], [ %sum.02.epil, %header.epil ] +; EPILOG: for.exit2.loopexit3: +; EPILOG-NEXT: %retval.ph4 = phi i32 [ 42, %for.exiting_block.epil ], [ %sum.02.epil, %header.epil ] ; EPILOG-NEXT: br label %for.exit2 ; EPILOG: for.exit2: -; EPILOG-NEXT: %retval = phi i32 [ %retval.ph, %for.exit2.loopexit ], [ %retval.ph3, %for.exit2.loopexit2 ] +; EPILOG-NEXT: %retval = phi i32 [ %retval.ph, %for.exit2.loopexit ], [ %retval.ph4, %for.exit2.loopexit3 ] ; EPILOG-NEXT: %addx = add i32 %retval, %x ; EPILOG-NEXT: br i1 %cond, label %exit_true, label %exit_false ; EPILOG: exit_true: @@ -3892,7 +3877,7 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %latch_exit.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -3920,19 +3905,18 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) { ; EPILOG-BLOCK-NEXT: %indvars.iv.next.1 = add i64 %indvars.iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp eq i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latch_exit.unr-lcssa.loopexit, label %header, !llvm.loop !10 -; EPILOG-BLOCK: latch_exit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %sum.0.lcssa.ph.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: %sum.02.unr.ph = phi i32 [ %add.1, %latch.1 ] -; EPILOG-BLOCK-NEXT: br label %latch_exit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %latch_exit.unr-lcssa, label %header, !llvm.loop !10 ; EPILOG-BLOCK: latch_exit.unr-lcssa: -; EPILOG-BLOCK-NEXT: %sum.0.lcssa.ph = phi i32 [ poison, %entry ], [ %sum.0.lcssa.ph.ph, %latch_exit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ 0, %entry ], [ %indvars.iv.unr.ph, %latch_exit.unr-lcssa.loopexit ] -; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ 0, %entry ], [ %sum.02.unr.ph, %latch_exit.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %sum.0.lcssa.ph = phi i32 [ %add.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %indvars.iv.unr = phi i64 [ %indvars.iv.next.1, %latch.1 ] +; EPILOG-BLOCK-NEXT: %sum.02.unr = phi i32 [ %add.1, %latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latch_exit ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %indvars.iv.epil.init = phi i64 [ 0, %entry ], [ %indvars.iv.unr, %latch_exit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %sum.02.epil.init = phi i32 [ 0, %entry ], [ %sum.02.unr, %latch_exit.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod2 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod2) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: ; EPILOG-BLOCK-NEXT: br i1 false, label %for.exit2, label %for.exiting_block.epil @@ -3940,9 +3924,9 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) { ; EPILOG-BLOCK-NEXT: %cmp.epil = icmp eq i64 %n, 42 ; EPILOG-BLOCK-NEXT: br i1 %cmp.epil, label %for.exit2, label %latch.epil ; EPILOG-BLOCK: latch.epil: -; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.unr +; EPILOG-BLOCK-NEXT: %arrayidx.epil = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.epil.init ; EPILOG-BLOCK-NEXT: %load.epil = load i32, ptr %arrayidx.epil, align 4 -; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %load.epil, %sum.02.unr +; EPILOG-BLOCK-NEXT: %add.epil = add nsw i32 %load.epil, %sum.02.epil.init ; EPILOG-BLOCK-NEXT: br label %latch_exit ; EPILOG-BLOCK: latch_exit: ; EPILOG-BLOCK-NEXT: %sum.0.lcssa = phi i32 [ %sum.0.lcssa.ph, %latch_exit.unr-lcssa ], [ %add.epil, %latch.epil ] @@ -3951,7 +3935,7 @@ define i32 @test6(ptr nocapture %a, i64 %n, i1 %cond, i32 %x) { ; EPILOG-BLOCK-NEXT: %retval.ph = phi i32 [ 42, %for.exiting_block ], [ %sum.02, %header ], [ %add, %latch ], [ 42, %for.exiting_block.1 ] ; EPILOG-BLOCK-NEXT: br label %for.exit2 ; EPILOG-BLOCK: for.exit2: -; EPILOG-BLOCK-NEXT: %retval = phi i32 [ %sum.02.unr, %header.epil ], [ 42, %for.exiting_block.epil ], [ %retval.ph, %for.exit2.loopexit ] +; EPILOG-BLOCK-NEXT: %retval = phi i32 [ %sum.02.epil.init, %header.epil ], [ 42, %for.exiting_block.epil ], [ %retval.ph, %for.exit2.loopexit ] ; EPILOG-BLOCK-NEXT: %addx = add i32 %retval, %x ; EPILOG-BLOCK-NEXT: br i1 %cond, label %exit_true, label %exit_false ; EPILOG-BLOCK: exit_true: @@ -4213,7 +4197,7 @@ define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) { ; EPILOG-NEXT: %2 = add i64 %1, -1 ; EPILOG-NEXT: %xtraiter = and i64 %1, 7 ; EPILOG-NEXT: %3 = icmp ult i64 %2, 7 -; EPILOG-NEXT: br i1 %3, label %latchexit.unr-lcssa, label %preheader.new +; EPILOG-NEXT: br i1 %3, label %header.epil.preheader, label %preheader.new ; EPILOG: preheader.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %1, %xtraiter ; EPILOG-NEXT: br label %header @@ -4239,20 +4223,20 @@ define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) { ; EPILOG-NEXT: %add.7 = add nuw nsw i64 %i6, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %header, label %latchexit.unr-lcssa.loopexit -; EPILOG: latchexit.unr-lcssa.loopexit: -; EPILOG-NEXT: %i6.unr.ph = phi i64 [ %add.7, %latch.7 ] -; EPILOG-NEXT: br label %latchexit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %header, label %latchexit.unr-lcssa ; EPILOG: latchexit.unr-lcssa: -; EPILOG-NEXT: %i6.unr = phi i64 [ 1, %preheader ], [ %i6.unr.ph, %latchexit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %i6.unr = phi i64 [ %add.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchexit ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %i6.epil.init = phi i64 [ 1, %preheader ], [ %i6.unr, %latchexit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %i6.epil = phi i64 [ %i6.unr, %header.epil.preheader ], [ %add.epil, %latch.epil ] +; EPILOG-NEXT: %i6.epil = phi i64 [ %i6.epil.init, %header.epil.preheader ], [ %add.epil, %latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ] -; EPILOG-NEXT: br i1 false, label %loopexit1.loopexit1, label %latch.epil +; EPILOG-NEXT: br i1 false, label %loopexit1.loopexit2, label %latch.epil ; EPILOG: latch.epil: ; EPILOG-NEXT: %add.epil = add nuw nsw i64 %i6.epil, 1 ; EPILOG-NEXT: %i9.epil = icmp slt i64 %add.epil, %sext @@ -4268,11 +4252,11 @@ define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) { ; EPILOG: loopexit1.loopexit: ; EPILOG-NEXT: %sext3.ph = phi i32 [ %shft, %header ], [ %shft, %latch ], [ %shft, %latch.1 ], [ %shft, %latch.2 ], [ %shft, %latch.3 ], [ %shft, %latch.4 ], [ %shft, %latch.5 ], [ %shft, %latch.6 ] ; EPILOG-NEXT: br label %loopexit1 -; EPILOG: loopexit1.loopexit1: -; EPILOG-NEXT: %sext3.ph2 = phi i32 [ %shft, %header.epil ] +; EPILOG: loopexit1.loopexit2: +; EPILOG-NEXT: %sext3.ph3 = phi i32 [ %shft, %header.epil ] ; EPILOG-NEXT: br label %loopexit1 ; EPILOG: loopexit1: -; EPILOG-NEXT: %sext3 = phi i32 [ %sext3.ph, %loopexit1.loopexit ], [ %sext3.ph2, %loopexit1.loopexit1 ] +; EPILOG-NEXT: %sext3 = phi i32 [ %sext3.ph, %loopexit1.loopexit ], [ %sext3.ph3, %loopexit1.loopexit2 ] ; EPILOG-NEXT: ret i32 %sext3 ; ; EPILOG-BLOCK-LABEL: @test7( @@ -4287,7 +4271,7 @@ define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) { ; EPILOG-BLOCK-NEXT: %2 = add i64 %1, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %1, 1 ; EPILOG-BLOCK-NEXT: %3 = icmp ult i64 %2, 1 -; EPILOG-BLOCK-NEXT: br i1 %3, label %latchexit.unr-lcssa, label %preheader.new +; EPILOG-BLOCK-NEXT: br i1 %3, label %header.epil.preheader, label %preheader.new ; EPILOG-BLOCK: preheader.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %1, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -4301,13 +4285,13 @@ define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) { ; EPILOG-BLOCK-NEXT: %add.1 = add nuw nsw i64 %i6, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %header, label %latchexit.unr-lcssa.loopexit, !llvm.loop !11 -; EPILOG-BLOCK: latchexit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: br label %latchexit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %header, label %latchexit.unr-lcssa, !llvm.loop !11 ; EPILOG-BLOCK: latchexit.unr-lcssa: ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %latchexit ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: ; EPILOG-BLOCK-NEXT: br i1 false, label %loopexit1, label %latch.epil @@ -4480,7 +4464,7 @@ define void @test8() { ; EPILOG-NEXT: br label %outerloop ; EPILOG: outerloop.loopexit.loopexit: ; EPILOG-NEXT: br label %outerloop.loopexit -; EPILOG: outerloop.loopexit.loopexit1: +; EPILOG: outerloop.loopexit.loopexit2: ; EPILOG-NEXT: br label %outerloop.loopexit ; EPILOG: outerloop.loopexit: ; EPILOG-NEXT: br label %outerloop @@ -4490,7 +4474,7 @@ define void @test8() { ; EPILOG-NEXT: %1 = sub i64 99, %i ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %exit.unr-lcssa, label %outerloop.new +; EPILOG-NEXT: br i1 %2, label %innerH.epil.preheader, label %outerloop.new ; EPILOG: outerloop.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %innerH @@ -4516,21 +4500,21 @@ define void @test8() { ; EPILOG: latch.7: ; EPILOG-NEXT: %niter.next.7 = add nuw nsw i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %innerH, label %exit.unr-lcssa.loopexit -; EPILOG: exit.unr-lcssa.loopexit: -; EPILOG-NEXT: %i3.unr.ph = phi i64 [ %i4.7, %latch.7 ] -; EPILOG-NEXT: br label %exit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %innerH, label %exit.unr-lcssa ; EPILOG: exit.unr-lcssa: -; EPILOG-NEXT: %i3.unr = phi i64 [ %i, %outerloop ], [ %i3.unr.ph, %exit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %i3.unr = phi i64 [ %i4.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %innerH.epil.preheader, label %exit.loopexit ; EPILOG: innerH.epil.preheader: +; EPILOG-NEXT: %i3.epil.init = phi i64 [ %i, %outerloop ], [ %i3.unr, %exit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %innerH.epil ; EPILOG: innerH.epil: -; EPILOG-NEXT: %i3.epil = phi i64 [ %i4.epil, %latch.epil ], [ %i3.unr, %innerH.epil.preheader ] +; EPILOG-NEXT: %i3.epil = phi i64 [ %i4.epil, %latch.epil ], [ %i3.epil.init, %innerH.epil.preheader ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %innerH.epil.preheader ], [ %epil.iter.next, %latch.epil ] ; EPILOG-NEXT: %i4.epil = add nuw nsw i64 %i3.epil, 1 -; EPILOG-NEXT: br i1 false, label %outerloop.loopexit.loopexit1, label %latch.epil +; EPILOG-NEXT: br i1 false, label %outerloop.loopexit.loopexit2, label %latch.epil ; EPILOG: latch.epil: ; EPILOG-NEXT: %i6.epil = icmp ult i64 %i4.epil, 100 ; EPILOG-NEXT: %epil.iter.next = add i64 %epil.iter, 1 @@ -4549,27 +4533,26 @@ define void @test8() { ; EPILOG-BLOCK: outerloop.loopexit.loopexit: ; EPILOG-BLOCK-NEXT: br label %outerloop.loopexit ; EPILOG-BLOCK: outerloop.loopexit: -; EPILOG-BLOCK-NEXT: br i1 false, label %exit.unr-lcssa.1, label %outerloop.new.1 +; EPILOG-BLOCK-NEXT: br i1 false, label %innerH.epil.preheader.1, label %outerloop.new.1 ; EPILOG-BLOCK: outerloop.new.1: ; EPILOG-BLOCK-NEXT: br label %innerH.1 ; EPILOG-BLOCK: innerH.1: ; EPILOG-BLOCK-NEXT: %i3.1 = phi i64 [ 0, %outerloop.new.1 ], [ %i4.1.1, %latch.1.1 ] ; EPILOG-BLOCK-NEXT: %niter.1 = phi i64 [ 0, %outerloop.new.1 ], [ %niter.next.1.1, %latch.1.1 ] -; EPILOG-BLOCK-NEXT: br i1 false, label %outerloop.loopexit.loopexit.1, label %latch.12 -; EPILOG-BLOCK: latch.12: +; EPILOG-BLOCK-NEXT: br i1 false, label %outerloop.loopexit.loopexit.1, label %latch.13 +; EPILOG-BLOCK: latch.13: ; EPILOG-BLOCK-NEXT: %i4.1.1 = add nuw nsw i64 %i3.1, 2 ; EPILOG-BLOCK-NEXT: br i1 false, label %outerloop.loopexit.loopexit.1, label %latch.1.1 ; EPILOG-BLOCK: latch.1.1: ; EPILOG-BLOCK-NEXT: %niter.next.1.1 = add i64 %niter.1, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1.1 = icmp ne i64 %niter.next.1.1, 100 -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1.1, label %innerH.1, label %exit.unr-lcssa.loopexit.1, !llvm.loop !12 -; EPILOG-BLOCK: exit.unr-lcssa.loopexit.1: -; EPILOG-BLOCK-NEXT: br label %exit.unr-lcssa.1 -; EPILOG-BLOCK: outerloop.loopexit.loopexit.1: -; EPILOG-BLOCK-NEXT: br label %outerloop.loopexit.1 +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1.1, label %innerH.1, label %exit.unr-lcssa.1, !llvm.loop !12 ; EPILOG-BLOCK: exit.unr-lcssa.1: ; EPILOG-BLOCK-NEXT: br i1 false, label %innerH.epil.preheader.1, label %exit.loopexit +; EPILOG-BLOCK: outerloop.loopexit.loopexit.1: +; EPILOG-BLOCK-NEXT: br label %outerloop.loopexit.1 ; EPILOG-BLOCK: innerH.epil.preheader.1: +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 false) ; EPILOG-BLOCK-NEXT: br label %innerH.epil.1 ; EPILOG-BLOCK: innerH.epil.1: ; EPILOG-BLOCK-NEXT: br i1 false, label %outerloop.loopexit.1, label %latch.epil @@ -4581,7 +4564,7 @@ define void @test8() { ; EPILOG-BLOCK-NEXT: %1 = sub i64 99, %i ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %exit.unr-lcssa, label %outerloop.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %innerH.epil.preheader, label %outerloop.new ; EPILOG-BLOCK: outerloop.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %innerH @@ -4595,13 +4578,13 @@ define void @test8() { ; EPILOG-BLOCK: latch.1: ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %innerH, label %exit.unr-lcssa.loopexit, !llvm.loop !12 -; EPILOG-BLOCK: exit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: br label %exit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %innerH, label %exit.unr-lcssa, !llvm.loop !12 ; EPILOG-BLOCK: exit.unr-lcssa: ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %innerH.epil.preheader, label %exit.loopexit ; EPILOG-BLOCK: innerH.epil.preheader: +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %innerH.epil ; EPILOG-BLOCK: innerH.epil: ; EPILOG-BLOCK-NEXT: br i1 false, label %outerloop.loopexit, label %latch.epil @@ -4788,7 +4771,7 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) { ; EPILOG-NEXT: %2 = add i32 %1, -1 ; EPILOG-NEXT: %xtraiter = and i32 %1, 7 ; EPILOG-NEXT: %3 = icmp ult i32 %2, 7 -; EPILOG-NEXT: br i1 %3, label %outerLatch.loopexit.unr-lcssa, label %preheader.new +; EPILOG-NEXT: br i1 %3, label %header.epil.preheader, label %preheader.new ; EPILOG: preheader.new: ; EPILOG-NEXT: %unroll_iter = sub i32 %1, %xtraiter ; EPILOG-NEXT: br label %header @@ -4799,11 +4782,11 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) { ; EPILOG: innerexit.loopexit: ; EPILOG-NEXT: %trip.lcssa.ph = phi i32 [ %trip, %header ], [ %trip, %latch ], [ %trip, %latch.1 ], [ %trip, %latch.2 ], [ %trip, %latch.3 ], [ %trip, %latch.4 ], [ %trip, %latch.5 ], [ %trip, %latch.6 ] ; EPILOG-NEXT: br label %innerexit -; EPILOG: innerexit.loopexit1: -; EPILOG-NEXT: %trip.lcssa.ph2 = phi i32 [ %trip, %header.epil ] +; EPILOG: innerexit.loopexit2: +; EPILOG-NEXT: %trip.lcssa.ph3 = phi i32 [ %trip, %header.epil ] ; EPILOG-NEXT: br label %innerexit ; EPILOG: innerexit: -; EPILOG-NEXT: %trip.lcssa = phi i32 [ %trip.lcssa.ph, %innerexit.loopexit ], [ %trip.lcssa.ph2, %innerexit.loopexit1 ] +; EPILOG-NEXT: %trip.lcssa = phi i32 [ %trip.lcssa.ph, %innerexit.loopexit ], [ %trip.lcssa.ph3, %innerexit.loopexit2 ] ; EPILOG-NEXT: %i9 = call ptr addrspace(1) @foo(i32 %trip.lcssa) ; EPILOG-NEXT: ret ptr addrspace(1) %i9 ; EPILOG: latch: @@ -4824,21 +4807,21 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) { ; EPILOG-NEXT: %iv.next.7 = add nuw nsw i64 %phi, 8 ; EPILOG-NEXT: %niter.next.7 = add i32 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i32 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %header, label %outerLatch.loopexit.unr-lcssa.loopexit -; EPILOG: outerLatch.loopexit.unr-lcssa.loopexit: -; EPILOG-NEXT: %phi.unr.ph = phi i64 [ %iv.next.7, %latch.7 ] -; EPILOG-NEXT: br label %outerLatch.loopexit.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %header, label %outerLatch.loopexit.unr-lcssa ; EPILOG: outerLatch.loopexit.unr-lcssa: -; EPILOG-NEXT: %phi.unr = phi i64 [ %i4, %preheader ], [ %phi.unr.ph, %outerLatch.loopexit.unr-lcssa.loopexit ] +; EPILOG-NEXT: %phi.unr = phi i64 [ %iv.next.7, %latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i32 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %outerLatch.loopexit ; EPILOG: header.epil.preheader: +; EPILOG-NEXT: %phi.epil.init = phi i64 [ %i4, %preheader ], [ %phi.unr, %outerLatch.loopexit.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i32 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %header.epil ; EPILOG: header.epil: -; EPILOG-NEXT: %phi.epil = phi i64 [ %phi.unr, %header.epil.preheader ], [ %iv.next.epil, %latch.epil ] +; EPILOG-NEXT: %phi.epil = phi i64 [ %phi.epil.init, %header.epil.preheader ], [ %iv.next.epil, %latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i32 [ 0, %header.epil.preheader ], [ %epil.iter.next, %latch.epil ] ; EPILOG-NEXT: %i7.epil = trunc i64 %phi.epil to i32 -; EPILOG-NEXT: br i1 true, label %latch.epil, label %innerexit.loopexit1 +; EPILOG-NEXT: br i1 true, label %latch.epil, label %innerexit.loopexit2 ; EPILOG: latch.epil: ; EPILOG-NEXT: %i11.epil = add nsw i32 %i7.epil, 1 ; EPILOG-NEXT: %innercnd.epil = icmp slt i32 %i11.epil, %trip @@ -4866,7 +4849,7 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) { ; EPILOG-BLOCK-NEXT: %2 = add i32 %1, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i32 %1, 1 ; EPILOG-BLOCK-NEXT: %3 = icmp ult i32 %2, 1 -; EPILOG-BLOCK-NEXT: br i1 %3, label %outerLatch.loopexit.unr-lcssa, label %preheader.new +; EPILOG-BLOCK-NEXT: br i1 %3, label %header.epil.preheader, label %preheader.new ; EPILOG-BLOCK: preheader.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i32 %1, %xtraiter ; EPILOG-BLOCK-NEXT: br label %header @@ -4877,17 +4860,17 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) { ; EPILOG-BLOCK: innerexit.loopexit.loopexit: ; EPILOG-BLOCK-NEXT: %trip.lcssa.ph.ph = phi i32 [ %trip, %latch ], [ %trip, %header ] ; EPILOG-BLOCK-NEXT: br label %innerexit.loopexit -; EPILOG-BLOCK: innerexit.loopexit.loopexit4: -; EPILOG-BLOCK-NEXT: %trip.lcssa.ph.ph5 = phi i32 [ %trip.1, %latch.13 ], [ %trip.1, %header.1 ] +; EPILOG-BLOCK: innerexit.loopexit.loopexit5: +; EPILOG-BLOCK-NEXT: %trip.lcssa.ph.ph6 = phi i32 [ %trip.1, %latch.14 ], [ %trip.1, %header.1 ] ; EPILOG-BLOCK-NEXT: br label %innerexit.loopexit ; EPILOG-BLOCK: innerexit.loopexit: -; EPILOG-BLOCK-NEXT: %trip.lcssa.ph = phi i32 [ %trip.lcssa.ph.ph, %innerexit.loopexit.loopexit ], [ %trip.lcssa.ph.ph5, %innerexit.loopexit.loopexit4 ] +; EPILOG-BLOCK-NEXT: %trip.lcssa.ph = phi i32 [ %trip.lcssa.ph.ph, %innerexit.loopexit.loopexit ], [ %trip.lcssa.ph.ph6, %innerexit.loopexit.loopexit5 ] ; EPILOG-BLOCK-NEXT: br label %innerexit -; EPILOG-BLOCK: innerexit.loopexit1: -; EPILOG-BLOCK-NEXT: %trip.lcssa.ph2 = phi i32 [ %trip, %header.epil ], [ %trip.1, %header.epil.1 ] +; EPILOG-BLOCK: innerexit.loopexit2: +; EPILOG-BLOCK-NEXT: %trip.lcssa.ph3 = phi i32 [ %trip, %header.epil ], [ %trip.1, %header.epil.1 ] ; EPILOG-BLOCK-NEXT: br label %innerexit ; EPILOG-BLOCK: innerexit: -; EPILOG-BLOCK-NEXT: %trip.lcssa = phi i32 [ %trip.lcssa.ph, %innerexit.loopexit ], [ %trip.lcssa.ph2, %innerexit.loopexit1 ] +; EPILOG-BLOCK-NEXT: %trip.lcssa = phi i32 [ %trip.lcssa.ph, %innerexit.loopexit ], [ %trip.lcssa.ph3, %innerexit.loopexit2 ] ; EPILOG-BLOCK-NEXT: %i9 = call ptr addrspace(1) @foo(i32 %trip.lcssa) ; EPILOG-BLOCK-NEXT: ret ptr addrspace(1) %i9 ; EPILOG-BLOCK: latch: @@ -4896,16 +4879,16 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) { ; EPILOG-BLOCK-NEXT: %iv.next.1 = add nuw nsw i64 %phi, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i32 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i32 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %header, label %outerLatch.loopexit.unr-lcssa.loopexit, !llvm.loop !14 -; EPILOG-BLOCK: outerLatch.loopexit.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: br label %outerLatch.loopexit.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %header, label %outerLatch.loopexit.unr-lcssa, !llvm.loop !14 ; EPILOG-BLOCK: outerLatch.loopexit.unr-lcssa: ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i32 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %header.epil.preheader, label %outerLatch.loopexit ; EPILOG-BLOCK: header.epil.preheader: +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i32 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %header.epil ; EPILOG-BLOCK: header.epil: -; EPILOG-BLOCK-NEXT: br i1 true, label %latch.epil, label %innerexit.loopexit1 +; EPILOG-BLOCK-NEXT: br i1 true, label %latch.epil, label %innerexit.loopexit2 ; EPILOG-BLOCK: latch.epil: ; EPILOG-BLOCK-NEXT: br label %outerLatch.loopexit ; EPILOG-BLOCK: outerLatch.loopexit: @@ -4919,30 +4902,30 @@ define ptr addrspace(1) @test9(ptr nocapture readonly %arg, i32 %n) { ; EPILOG-BLOCK-NEXT: %5 = add i32 %4, -1 ; EPILOG-BLOCK-NEXT: %xtraiter.1 = and i32 %4, 1 ; EPILOG-BLOCK-NEXT: %6 = icmp ult i32 %5, 1 -; EPILOG-BLOCK-NEXT: br i1 %6, label %outerLatch.loopexit.unr-lcssa.1, label %preheader.new.1 +; EPILOG-BLOCK-NEXT: br i1 %6, label %header.epil.preheader.1, label %preheader.new.1 ; EPILOG-BLOCK: preheader.new.1: ; EPILOG-BLOCK-NEXT: %unroll_iter.1 = sub i32 %4, %xtraiter.1 ; EPILOG-BLOCK-NEXT: br label %header.1 ; EPILOG-BLOCK: header.1: ; EPILOG-BLOCK-NEXT: %phi.1 = phi i64 [ 0, %preheader.new.1 ], [ %iv.next.1.1, %latch.1.1 ] ; EPILOG-BLOCK-NEXT: %niter.1 = phi i32 [ 0, %preheader.new.1 ], [ %niter.next.1.1, %latch.1.1 ] -; EPILOG-BLOCK-NEXT: br i1 true, label %latch.13, label %innerexit.loopexit.loopexit4 -; EPILOG-BLOCK: latch.13: -; EPILOG-BLOCK-NEXT: br i1 true, label %latch.1.1, label %innerexit.loopexit.loopexit4 +; EPILOG-BLOCK-NEXT: br i1 true, label %latch.14, label %innerexit.loopexit.loopexit5 +; EPILOG-BLOCK: latch.14: +; EPILOG-BLOCK-NEXT: br i1 true, label %latch.1.1, label %innerexit.loopexit.loopexit5 ; EPILOG-BLOCK: latch.1.1: ; EPILOG-BLOCK-NEXT: %iv.next.1.1 = add nuw nsw i64 %phi.1, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1.1 = add i32 %niter.1, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1.1 = icmp ne i32 %niter.next.1.1, %unroll_iter.1 -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1.1, label %header.1, label %outerLatch.loopexit.unr-lcssa.loopexit.1, !llvm.loop !14 -; EPILOG-BLOCK: outerLatch.loopexit.unr-lcssa.loopexit.1: -; EPILOG-BLOCK-NEXT: br label %outerLatch.loopexit.unr-lcssa.1 +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1.1, label %header.1, label %outerLatch.loopexit.unr-lcssa.1, !llvm.loop !14 ; EPILOG-BLOCK: outerLatch.loopexit.unr-lcssa.1: ; EPILOG-BLOCK-NEXT: %lcmp.mod.1 = icmp ne i32 %xtraiter.1, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod.1, label %header.epil.preheader.1, label %outerLatch.loopexit.1 ; EPILOG-BLOCK: header.epil.preheader.1: +; EPILOG-BLOCK-NEXT: %lcmp.mod1.1 = icmp ne i32 %xtraiter.1, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1.1) ; EPILOG-BLOCK-NEXT: br label %header.epil.1 ; EPILOG-BLOCK: header.epil.1: -; EPILOG-BLOCK-NEXT: br i1 true, label %latch.epil.1, label %innerexit.loopexit1 +; EPILOG-BLOCK-NEXT: br i1 true, label %latch.epil.1, label %innerexit.loopexit2 ; EPILOG-BLOCK: latch.epil.1: ; EPILOG-BLOCK-NEXT: br label %outerLatch.loopexit.1 ; EPILOG-BLOCK: outerLatch.loopexit.1: @@ -5171,7 +5154,7 @@ define void @test10(i64 %trip, i64 %trip2) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %exit2.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %loop_header @@ -5220,28 +5203,28 @@ define void @test10(i64 %trip, i64 %trip2) { ; EPILOG-NEXT: %iv_next.7 = add i64 %iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit2.unr-lcssa.loopexit +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit2.unr-lcssa ; EPILOG: exit1.loopexit: ; EPILOG-NEXT: br label %exit1 -; EPILOG: exit1.loopexit1: +; EPILOG: exit1.loopexit2: ; EPILOG-NEXT: br label %exit1 ; EPILOG: exit1: ; EPILOG-NEXT: ret void -; EPILOG: exit2.unr-lcssa.loopexit: -; EPILOG-NEXT: %iv.unr.ph = phi i64 [ %iv_next.7, %loop_latch.7 ] -; EPILOG-NEXT: br label %exit2.unr-lcssa ; EPILOG: exit2.unr-lcssa: -; EPILOG-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit2.unr-lcssa.loopexit ] +; EPILOG-NEXT: %iv.unr = phi i64 [ %iv_next.7, %loop_latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit2 ; EPILOG: loop_header.epil.preheader: +; EPILOG-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit2.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %loop_header.epil ; EPILOG: loop_header.epil: -; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.unr, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.epil.init, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %loop_header.epil.preheader ], [ %epil.iter.next, %loop_latch.epil ] ; EPILOG-NEXT: call void @bar() ; EPILOG-NEXT: %cmp_early.epil = icmp ne i64 %iv.epil, %trip2 -; EPILOG-NEXT: br i1 %cmp_early.epil, label %loop_latch.epil, label %exit1.loopexit1 +; EPILOG-NEXT: br i1 %cmp_early.epil, label %loop_latch.epil, label %exit1.loopexit2 ; EPILOG: loop_latch.epil: ; EPILOG-NEXT: %iv_next.epil = add i64 %iv.epil, 1 ; EPILOG-NEXT: %cmp.epil = icmp ne i64 %iv_next.epil, %trip @@ -5259,7 +5242,7 @@ define void @test10(i64 %trip, i64 %trip2) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %exit2.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %loop_header @@ -5278,23 +5261,23 @@ define void @test10(i64 %trip, i64 %trip2) { ; EPILOG-BLOCK-NEXT: %iv_next.1 = add i64 %iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit2.unr-lcssa.loopexit, !llvm.loop !16 +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit2.unr-lcssa, !llvm.loop !16 ; EPILOG-BLOCK: exit1.loopexit: ; EPILOG-BLOCK-NEXT: br label %exit1 ; EPILOG-BLOCK: exit1: ; EPILOG-BLOCK-NEXT: ret void -; EPILOG-BLOCK: exit2.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %iv.unr.ph = phi i64 [ %iv_next.1, %loop_latch.1 ] -; EPILOG-BLOCK-NEXT: br label %exit2.unr-lcssa ; EPILOG-BLOCK: exit2.unr-lcssa: -; EPILOG-BLOCK-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit2.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %iv.unr = phi i64 [ %iv_next.1, %loop_latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit2 ; EPILOG-BLOCK: loop_header.epil.preheader: +; EPILOG-BLOCK-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit2.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %loop_header.epil ; EPILOG-BLOCK: loop_header.epil: ; EPILOG-BLOCK-NEXT: call void @bar() -; EPILOG-BLOCK-NEXT: %cmp_early.epil = icmp ne i64 %iv.unr, %trip2 +; EPILOG-BLOCK-NEXT: %cmp_early.epil = icmp ne i64 %iv.epil.init, %trip2 ; EPILOG-BLOCK-NEXT: br i1 %cmp_early.epil, label %loop_latch.epil, label %exit1 ; EPILOG-BLOCK: loop_latch.epil: ; EPILOG-BLOCK-NEXT: br label %exit2 @@ -5460,7 +5443,7 @@ define void @test11(i64 %trip, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %exit2.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %loop_header @@ -5494,27 +5477,27 @@ define void @test11(i64 %trip, i1 %cond) { ; EPILOG-NEXT: %iv_next.7 = add i64 %iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit2.unr-lcssa.loopexit +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit2.unr-lcssa ; EPILOG: exit1.loopexit: ; EPILOG-NEXT: br label %exit1 -; EPILOG: exit1.loopexit1: +; EPILOG: exit1.loopexit2: ; EPILOG-NEXT: br label %exit1 ; EPILOG: exit1: ; EPILOG-NEXT: ret void -; EPILOG: exit2.unr-lcssa.loopexit: -; EPILOG-NEXT: %iv.unr.ph = phi i64 [ %iv_next.7, %loop_latch.7 ] -; EPILOG-NEXT: br label %exit2.unr-lcssa ; EPILOG: exit2.unr-lcssa: -; EPILOG-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit2.unr-lcssa.loopexit ] +; EPILOG-NEXT: %iv.unr = phi i64 [ %iv_next.7, %loop_latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit2 ; EPILOG: loop_header.epil.preheader: +; EPILOG-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit2.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %loop_header.epil ; EPILOG: loop_header.epil: -; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.unr, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.epil.init, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %loop_header.epil.preheader ], [ %epil.iter.next, %loop_latch.epil ] ; EPILOG-NEXT: call void @bar() -; EPILOG-NEXT: br i1 %cond, label %loop_latch.epil, label %exit1.loopexit1 +; EPILOG-NEXT: br i1 %cond, label %loop_latch.epil, label %exit1.loopexit2 ; EPILOG: loop_latch.epil: ; EPILOG-NEXT: %iv_next.epil = add i64 %iv.epil, 1 ; EPILOG-NEXT: %cmp.epil = icmp ne i64 %iv_next.epil, %trip @@ -5532,7 +5515,7 @@ define void @test11(i64 %trip, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %exit2.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %loop_header @@ -5548,17 +5531,17 @@ define void @test11(i64 %trip, i1 %cond) { ; EPILOG-BLOCK-NEXT: %iv_next.1 = add i64 %iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit2.unr-lcssa.loopexit, !llvm.loop !17 +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit2.unr-lcssa, !llvm.loop !17 ; EPILOG-BLOCK: exit1.loopexit: ; EPILOG-BLOCK-NEXT: br label %exit1 ; EPILOG-BLOCK: exit1: ; EPILOG-BLOCK-NEXT: ret void -; EPILOG-BLOCK: exit2.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: br label %exit2.unr-lcssa ; EPILOG-BLOCK: exit2.unr-lcssa: ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit2 ; EPILOG-BLOCK: loop_header.epil.preheader: +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %loop_header.epil ; EPILOG-BLOCK: loop_header.epil: ; EPILOG-BLOCK-NEXT: call void @bar() @@ -5706,7 +5689,7 @@ define void @test12(i64 %trip, i64 %trip2, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %exit1.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %loop_header @@ -5771,33 +5754,33 @@ define void @test12(i64 %trip, i64 %trip2, i1 %cond) { ; EPILOG-NEXT: %iv_next.7 = add i64 %iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit1.unr-lcssa.loopexit -; EPILOG: exit1.unr-lcssa.loopexit: -; EPILOG-NEXT: %iv.unr.ph = phi i64 [ %iv_next.7, %loop_latch.7 ] -; EPILOG-NEXT: br label %exit1.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit1.unr-lcssa ; EPILOG: exit1.unr-lcssa: -; EPILOG-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit1.unr-lcssa.loopexit ] +; EPILOG-NEXT: %iv.unr = phi i64 [ %iv_next.7, %loop_latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit1 ; EPILOG: loop_header.epil.preheader: +; EPILOG-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit1.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %loop_header.epil ; EPILOG: loop_header.epil: -; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.unr, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.epil.init, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %loop_header.epil.preheader ], [ %epil.iter.next, %loop_latch.epil ] ; EPILOG-NEXT: call void @bar() ; EPILOG-NEXT: %cmp_early.epil = icmp ne i64 %iv.epil, %trip2 -; EPILOG-NEXT: br i1 %cmp_early.epil, label %loop_exiting_bb2.epil, label %exit1.epilog-lcssa.loopexit1 +; EPILOG-NEXT: br i1 %cmp_early.epil, label %loop_exiting_bb2.epil, label %exit1.epilog-lcssa.loopexit2 ; EPILOG: loop_exiting_bb2.epil: -; EPILOG-NEXT: br i1 %cond, label %loop_latch.epil, label %exit1.epilog-lcssa.loopexit1 +; EPILOG-NEXT: br i1 %cond, label %loop_latch.epil, label %exit1.epilog-lcssa.loopexit2 ; EPILOG: loop_latch.epil: ; EPILOG-NEXT: %iv_next.epil = add i64 %iv.epil, 1 ; EPILOG-NEXT: %cmp.epil = icmp ne i64 %iv_next.epil, %trip ; EPILOG-NEXT: %epil.iter.next = add i64 %epil.iter, 1 ; EPILOG-NEXT: %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter -; EPILOG-NEXT: br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !16 +; EPILOG-NEXT: br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit2, !llvm.loop !16 ; EPILOG: exit1.epilog-lcssa.loopexit: ; EPILOG-NEXT: br label %exit1.epilog-lcssa -; EPILOG: exit1.epilog-lcssa.loopexit1: +; EPILOG: exit1.epilog-lcssa.loopexit2: ; EPILOG-NEXT: br label %exit1.epilog-lcssa ; EPILOG: exit1.epilog-lcssa: ; EPILOG-NEXT: br label %exit1 @@ -5810,7 +5793,7 @@ define void @test12(i64 %trip, i64 %trip2, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %exit1.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %loop_header @@ -5833,19 +5816,19 @@ define void @test12(i64 %trip, i64 %trip2, i1 %cond) { ; EPILOG-BLOCK-NEXT: %iv_next.1 = add i64 %iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit1.unr-lcssa.loopexit, !llvm.loop !18 -; EPILOG-BLOCK: exit1.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %iv.unr.ph = phi i64 [ %iv_next.1, %loop_latch.1 ] -; EPILOG-BLOCK-NEXT: br label %exit1.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit1.unr-lcssa, !llvm.loop !18 ; EPILOG-BLOCK: exit1.unr-lcssa: -; EPILOG-BLOCK-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit1.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %iv.unr = phi i64 [ %iv_next.1, %loop_latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit1 ; EPILOG-BLOCK: loop_header.epil.preheader: +; EPILOG-BLOCK-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit1.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %loop_header.epil ; EPILOG-BLOCK: loop_header.epil: ; EPILOG-BLOCK-NEXT: call void @bar() -; EPILOG-BLOCK-NEXT: %cmp_early.epil = icmp ne i64 %iv.unr, %trip2 +; EPILOG-BLOCK-NEXT: %cmp_early.epil = icmp ne i64 %iv.epil.init, %trip2 ; EPILOG-BLOCK-NEXT: br i1 %cmp_early.epil, label %loop_exiting_bb2.epil, label %exit1.epilog-lcssa ; EPILOG-BLOCK: loop_exiting_bb2.epil: ; EPILOG-BLOCK-NEXT: br i1 %cond, label %loop_latch.epil, label %exit1.epilog-lcssa @@ -6038,7 +6021,7 @@ define void @test13(i64 %trip, i64 %trip2) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %exit1.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %loop_header @@ -6111,34 +6094,34 @@ define void @test13(i64 %trip, i64 %trip2) { ; EPILOG-NEXT: %iv_next.7 = add i64 %iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit1.unr-lcssa.loopexit -; EPILOG: exit1.unr-lcssa.loopexit: -; EPILOG-NEXT: %iv.unr.ph = phi i64 [ %iv_next.7, %loop_latch.7 ] -; EPILOG-NEXT: br label %exit1.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit1.unr-lcssa ; EPILOG: exit1.unr-lcssa: -; EPILOG-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit1.unr-lcssa.loopexit ] +; EPILOG-NEXT: %iv.unr = phi i64 [ %iv_next.7, %loop_latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit1 ; EPILOG: loop_header.epil.preheader: +; EPILOG-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit1.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %loop_header.epil ; EPILOG: loop_header.epil: -; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.unr, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.epil.init, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %loop_header.epil.preheader ], [ %epil.iter.next, %loop_latch.epil ] ; EPILOG-NEXT: call void @bar() ; EPILOG-NEXT: %cmp_early.epil = icmp ne i64 %iv.epil, %trip2 -; EPILOG-NEXT: br i1 %cmp_early.epil, label %loop_exiting_bb2.epil, label %exit1.epilog-lcssa.loopexit1 +; EPILOG-NEXT: br i1 %cmp_early.epil, label %loop_exiting_bb2.epil, label %exit1.epilog-lcssa.loopexit2 ; EPILOG: loop_exiting_bb2.epil: ; EPILOG-NEXT: %unknown.epil = call i1 @unknown_cond() -; EPILOG-NEXT: br i1 %unknown.epil, label %loop_latch.epil, label %exit1.epilog-lcssa.loopexit1 +; EPILOG-NEXT: br i1 %unknown.epil, label %loop_latch.epil, label %exit1.epilog-lcssa.loopexit2 ; EPILOG: loop_latch.epil: ; EPILOG-NEXT: %iv_next.epil = add i64 %iv.epil, 1 ; EPILOG-NEXT: %cmp.epil = icmp ne i64 %iv_next.epil, %trip ; EPILOG-NEXT: %epil.iter.next = add i64 %epil.iter, 1 ; EPILOG-NEXT: %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter -; EPILOG-NEXT: br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !17 +; EPILOG-NEXT: br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit2, !llvm.loop !17 ; EPILOG: exit1.epilog-lcssa.loopexit: ; EPILOG-NEXT: br label %exit1.epilog-lcssa -; EPILOG: exit1.epilog-lcssa.loopexit1: +; EPILOG: exit1.epilog-lcssa.loopexit2: ; EPILOG-NEXT: br label %exit1.epilog-lcssa ; EPILOG: exit1.epilog-lcssa: ; EPILOG-NEXT: br label %exit1 @@ -6151,7 +6134,7 @@ define void @test13(i64 %trip, i64 %trip2) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %exit1.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %loop_header @@ -6176,19 +6159,19 @@ define void @test13(i64 %trip, i64 %trip2) { ; EPILOG-BLOCK-NEXT: %iv_next.1 = add i64 %iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit1.unr-lcssa.loopexit, !llvm.loop !19 -; EPILOG-BLOCK: exit1.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: %iv.unr.ph = phi i64 [ %iv_next.1, %loop_latch.1 ] -; EPILOG-BLOCK-NEXT: br label %exit1.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit1.unr-lcssa, !llvm.loop !19 ; EPILOG-BLOCK: exit1.unr-lcssa: -; EPILOG-BLOCK-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit1.unr-lcssa.loopexit ] +; EPILOG-BLOCK-NEXT: %iv.unr = phi i64 [ %iv_next.1, %loop_latch.1 ] ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit1 ; EPILOG-BLOCK: loop_header.epil.preheader: +; EPILOG-BLOCK-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit1.unr-lcssa ] +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %loop_header.epil ; EPILOG-BLOCK: loop_header.epil: ; EPILOG-BLOCK-NEXT: call void @bar() -; EPILOG-BLOCK-NEXT: %cmp_early.epil = icmp ne i64 %iv.unr, %trip2 +; EPILOG-BLOCK-NEXT: %cmp_early.epil = icmp ne i64 %iv.epil.init, %trip2 ; EPILOG-BLOCK-NEXT: br i1 %cmp_early.epil, label %loop_exiting_bb2.epil, label %exit1.epilog-lcssa ; EPILOG-BLOCK: loop_exiting_bb2.epil: ; EPILOG-BLOCK-NEXT: %unknown.epil = call i1 @unknown_cond() @@ -6393,7 +6376,7 @@ define void @test14(i64 %trip, i1 %cond) { ; EPILOG-NEXT: %1 = add i64 %0, -1 ; EPILOG-NEXT: %xtraiter = and i64 %0, 7 ; EPILOG-NEXT: %2 = icmp ult i64 %1, 7 -; EPILOG-NEXT: br i1 %2, label %exit1.unr-lcssa, label %entry.new +; EPILOG-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG: entry.new: ; EPILOG-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-NEXT: br label %loop_header @@ -6451,33 +6434,33 @@ define void @test14(i64 %trip, i1 %cond) { ; EPILOG-NEXT: %iv_next.7 = add i64 %iv, 8 ; EPILOG-NEXT: %niter.next.7 = add i64 %niter, 8 ; EPILOG-NEXT: %niter.ncmp.7 = icmp ne i64 %niter.next.7, %unroll_iter -; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit1.unr-lcssa.loopexit -; EPILOG: exit1.unr-lcssa.loopexit: -; EPILOG-NEXT: %iv.unr.ph = phi i64 [ %iv_next.7, %loop_latch.7 ] -; EPILOG-NEXT: br label %exit1.unr-lcssa +; EPILOG-NEXT: br i1 %niter.ncmp.7, label %loop_header, label %exit1.unr-lcssa ; EPILOG: exit1.unr-lcssa: -; EPILOG-NEXT: %iv.unr = phi i64 [ 0, %entry ], [ %iv.unr.ph, %exit1.unr-lcssa.loopexit ] +; EPILOG-NEXT: %iv.unr = phi i64 [ %iv_next.7, %loop_latch.7 ] ; EPILOG-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit1 ; EPILOG: loop_header.epil.preheader: +; EPILOG-NEXT: %iv.epil.init = phi i64 [ 0, %entry ], [ %iv.unr, %exit1.unr-lcssa ] +; EPILOG-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-NEXT: br label %loop_header.epil ; EPILOG: loop_header.epil: -; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.unr, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] +; EPILOG-NEXT: %iv.epil = phi i64 [ %iv.epil.init, %loop_header.epil.preheader ], [ %iv_next.epil, %loop_latch.epil ] ; EPILOG-NEXT: %epil.iter = phi i64 [ 0, %loop_header.epil.preheader ], [ %epil.iter.next, %loop_latch.epil ] ; EPILOG-NEXT: call void @bar() -; EPILOG-NEXT: br i1 %cond, label %loop_exiting_bb2.epil, label %exit1.epilog-lcssa.loopexit1 +; EPILOG-NEXT: br i1 %cond, label %loop_exiting_bb2.epil, label %exit1.epilog-lcssa.loopexit2 ; EPILOG: loop_exiting_bb2.epil: ; EPILOG-NEXT: %unknown.epil = call i1 @unknown_cond() -; EPILOG-NEXT: br i1 %unknown.epil, label %loop_latch.epil, label %exit1.epilog-lcssa.loopexit1 +; EPILOG-NEXT: br i1 %unknown.epil, label %loop_latch.epil, label %exit1.epilog-lcssa.loopexit2 ; EPILOG: loop_latch.epil: ; EPILOG-NEXT: %iv_next.epil = add i64 %iv.epil, 1 ; EPILOG-NEXT: %cmp.epil = icmp ne i64 %iv_next.epil, %trip ; EPILOG-NEXT: %epil.iter.next = add i64 %epil.iter, 1 ; EPILOG-NEXT: %epil.iter.cmp = icmp ne i64 %epil.iter.next, %xtraiter -; EPILOG-NEXT: br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit1, !llvm.loop !18 +; EPILOG-NEXT: br i1 %epil.iter.cmp, label %loop_header.epil, label %exit1.epilog-lcssa.loopexit2, !llvm.loop !18 ; EPILOG: exit1.epilog-lcssa.loopexit: ; EPILOG-NEXT: br label %exit1.epilog-lcssa -; EPILOG: exit1.epilog-lcssa.loopexit1: +; EPILOG: exit1.epilog-lcssa.loopexit2: ; EPILOG-NEXT: br label %exit1.epilog-lcssa ; EPILOG: exit1.epilog-lcssa: ; EPILOG-NEXT: br label %exit1 @@ -6490,7 +6473,7 @@ define void @test14(i64 %trip, i1 %cond) { ; EPILOG-BLOCK-NEXT: %1 = add i64 %0, -1 ; EPILOG-BLOCK-NEXT: %xtraiter = and i64 %0, 1 ; EPILOG-BLOCK-NEXT: %2 = icmp ult i64 %1, 1 -; EPILOG-BLOCK-NEXT: br i1 %2, label %exit1.unr-lcssa, label %entry.new +; EPILOG-BLOCK-NEXT: br i1 %2, label %loop_header.epil.preheader, label %entry.new ; EPILOG-BLOCK: entry.new: ; EPILOG-BLOCK-NEXT: %unroll_iter = sub i64 %0, %xtraiter ; EPILOG-BLOCK-NEXT: br label %loop_header @@ -6512,13 +6495,13 @@ define void @test14(i64 %trip, i1 %cond) { ; EPILOG-BLOCK-NEXT: %iv_next.1 = add i64 %iv, 2 ; EPILOG-BLOCK-NEXT: %niter.next.1 = add i64 %niter, 2 ; EPILOG-BLOCK-NEXT: %niter.ncmp.1 = icmp ne i64 %niter.next.1, %unroll_iter -; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit1.unr-lcssa.loopexit, !llvm.loop !20 -; EPILOG-BLOCK: exit1.unr-lcssa.loopexit: -; EPILOG-BLOCK-NEXT: br label %exit1.unr-lcssa +; EPILOG-BLOCK-NEXT: br i1 %niter.ncmp.1, label %loop_header, label %exit1.unr-lcssa, !llvm.loop !20 ; EPILOG-BLOCK: exit1.unr-lcssa: ; EPILOG-BLOCK-NEXT: %lcmp.mod = icmp ne i64 %xtraiter, 0 ; EPILOG-BLOCK-NEXT: br i1 %lcmp.mod, label %loop_header.epil.preheader, label %exit1 ; EPILOG-BLOCK: loop_header.epil.preheader: +; EPILOG-BLOCK-NEXT: %lcmp.mod1 = icmp ne i64 %xtraiter, 0 +; EPILOG-BLOCK-NEXT: call void @llvm.assume(i1 %lcmp.mod1) ; EPILOG-BLOCK-NEXT: br label %loop_header.epil ; EPILOG-BLOCK: loop_header.epil: ; EPILOG-BLOCK-NEXT: call void @bar() diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop.ll index 8acf74a84d2ba..492de063573be 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-loop.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop.ll @@ -22,7 +22,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; EPILOG: br i1 %cmp1, label %for.end, label %for.body.preheader, !prof [[EPILOG_PROF_0:![0-9]+]] ; EPILOG: for.body.preheader: ; EPILOG: %xtraiter = and i32 %n -; EPILOG: br i1 %1, label %for.end.loopexit.unr-lcssa, label %for.body.preheader.new, !prof [[EPILOG_PROF_1:![0-9]+]] +; EPILOG: br i1 %1, label %for.body.epil.preheader, label %for.body.preheader.new, !prof [[EPILOG_PROF_1:![0-9]+]] ; EPILOG: for.end.loopexit.unr-lcssa: ; EPILOG: %lcmp.mod = icmp ne i32 %xtraiter, 0 @@ -41,7 +41,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; NOPROLOG-NOT: %xtraiter = and i32 %n ; EPILOG: for.body.epil: -; EPILOG: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %for.body.epil ], [ %indvars.iv.unr, %for.body.epil.preheader ] +; EPILOG: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %for.body.epil ], [ %indvars.iv.epil.init, %for.body.epil.preheader ] ; EPILOG: %epil.iter.next = add i32 %epil.iter, 1 ; EPILOG: %epil.iter.cmp = icmp ne i32 %epil.iter.next, %xtraiter ; EPILOG: br i1 %epil.iter.cmp, label %for.body.epil, label %for.end.loopexit.epilog-lcssa, !prof [[EPILOG_PROF_3:![0-9]+]], !llvm.loop [[EPILOG_LOOP:![0-9]+]] diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll index 492ddd130356d..0eeb3ad77c49e 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll @@ -8,9 +8,9 @@ ; EPILOG: for.body.preheader: -; EPILOG: br i1 %1, label %for.end.loopexit.unr-lcssa, label %for.body.preheader.new, !dbg [[PH_LOC:![0-9]+]] +; EPILOG: br i1 %1, label %for.body.epil.preheader, label %for.body.preheader.new, !dbg [[PH_LOC:![0-9]+]] ; EPILOG: for.body: -; EPILOG: br i1 %niter.ncmp.1, label %for.end.loopexit.unr-lcssa.loopexit, label %for.body, !dbg [[PH_LOC]] +; EPILOG: br i1 %niter.ncmp.1, label %for.end.loopexit.unr-lcssa, label %for.body, !dbg [[PH_LOC]] ; EPILOG-NOT: br i1 %niter.ncmp.2, label %for.end.loopexit{{.*}}, label %for.body ; EPILOG: for.body.epil.preheader: ; EPILOG: br label %for.body.epil, !dbg [[PH_LOC]] diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll index 0e11fff9f0208..a573de2d104a6 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll @@ -8,8 +8,8 @@ ; This test makes sure we're not unrolling 'odd' counts ; EPILOG: for.body: -; EPILOG: br i1 %niter.ncmp.3, label %for.end.loopexit.unr-lcssa.loopexit{{.*}}, label %for.body -; EPILOG-NOT: br i1 %niter.ncmp.4, label %for.end.loopexit.unr-lcssa.loopexit{{.*}}, label %for.body +; EPILOG: br i1 %niter.ncmp.3, label %for.end.loopexit.unr-lcssa{{.*}}, label %for.body +; EPILOG-NOT: br i1 %niter.ncmp.4, label %for.end.loopexit.unr-lcssa{{.*}}, label %for.body ; EPILOG: for.body.epil: ; PROLOG: for.body.prol: diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll index fa9f9026d524c..0cee4e26d3d5d 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll @@ -69,7 +69,7 @@ define i3 @test(ptr %a, i3 %n) { ; UNROLL-4-NEXT: [[TMP0:%.*]] = add i3 [[N]], -1 ; UNROLL-4-NEXT: [[XTRAITER:%.*]] = and i3 [[N]], 3 ; UNROLL-4-NEXT: [[TMP1:%.*]] = icmp ult i3 [[TMP0]], 3 -; UNROLL-4-NEXT: br i1 [[TMP1]], label [[FOR_END_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] +; UNROLL-4-NEXT: br i1 [[TMP1]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] ; UNROLL-4: for.body.preheader.new: ; UNROLL-4-NEXT: [[UNROLL_ITER:%.*]] = sub i3 [[N]], [[XTRAITER]] ; UNROLL-4-NEXT: br label [[FOR_BODY:%.*]] @@ -95,23 +95,22 @@ define i3 @test(ptr %a, i3 %n) { ; UNROLL-4-NEXT: [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4 ; UNROLL-4-NEXT: [[NITER_NEXT_3]] = add i3 [[NITER]], -4 ; UNROLL-4-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i3 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; UNROLL-4-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; UNROLL-4: for.end.loopexit.unr-lcssa.loopexit: +; UNROLL-4-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_END_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; UNROLL-4: for.end.loopexit.unr-lcssa: ; UNROLL-4-NEXT: [[ADD_LCSSA_PH_PH:%.*]] = phi i3 [ [[ADD_3]], [[FOR_BODY]] ] ; UNROLL-4-NEXT: [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3]], [[FOR_BODY]] ] ; UNROLL-4-NEXT: [[SUM_02_UNR_PH:%.*]] = phi i3 [ [[ADD_3]], [[FOR_BODY]] ] -; UNROLL-4-NEXT: br label [[FOR_END_LOOPEXIT_UNR_LCSSA]] -; UNROLL-4: for.end.loopexit.unr-lcssa: -; UNROLL-4-NEXT: [[ADD_LCSSA_PH:%.*]] = phi i3 [ poison, [[FOR_BODY_PREHEADER]] ], [ [[ADD_LCSSA_PH_PH]], [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] -; UNROLL-4-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_UNR_PH]], [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] -; UNROLL-4-NEXT: [[SUM_02_UNR:%.*]] = phi i3 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[SUM_02_UNR_PH]], [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] ; UNROLL-4-NEXT: [[LCMP_MOD:%.*]] = icmp ne i3 [[XTRAITER]], 0 -; UNROLL-4-NEXT: br i1 [[LCMP_MOD]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_END_LOOPEXIT:%.*]] +; UNROLL-4-NEXT: br i1 [[LCMP_MOD]], label [[FOR_BODY_EPIL_PREHEADER]], label [[FOR_END_LOOPEXIT:%.*]] ; UNROLL-4: for.body.epil.preheader: +; UNROLL-4-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_UNR_PH]], [[FOR_END_LOOPEXIT_UNR_LCSSA]] ] +; UNROLL-4-NEXT: [[SUM_02_EPIL_INIT:%.*]] = phi i3 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[SUM_02_UNR_PH]], [[FOR_END_LOOPEXIT_UNR_LCSSA]] ] +; UNROLL-4-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i3 [[XTRAITER]], 0 +; UNROLL-4-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; UNROLL-4-NEXT: br label [[FOR_BODY_EPIL:%.*]] ; UNROLL-4: for.body.epil: -; UNROLL-4-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[INDVARS_IV_UNR]], [[FOR_BODY_EPIL_PREHEADER]] ] -; UNROLL-4-NEXT: [[SUM_02_EPIL:%.*]] = phi i3 [ [[ADD_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[SUM_02_UNR]], [[FOR_BODY_EPIL_PREHEADER]] ] +; UNROLL-4-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[INDVARS_IV_EPIL_INIT]], [[FOR_BODY_EPIL_PREHEADER]] ] +; UNROLL-4-NEXT: [[SUM_02_EPIL:%.*]] = phi i3 [ [[ADD_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[SUM_02_EPIL_INIT]], [[FOR_BODY_EPIL_PREHEADER]] ] ; UNROLL-4-NEXT: [[EPIL_ITER:%.*]] = phi i3 [ 0, [[FOR_BODY_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[FOR_BODY_EPIL]] ] ; UNROLL-4-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i3, ptr [[A]], i64 [[INDVARS_IV_EPIL]] ; UNROLL-4-NEXT: [[TMP6:%.*]] = load i3, ptr [[ARRAYIDX_EPIL]], align 1 @@ -126,7 +125,7 @@ define i3 @test(ptr %a, i3 %n) { ; UNROLL-4-NEXT: [[ADD_LCSSA_PH1:%.*]] = phi i3 [ [[ADD_EPIL]], [[FOR_BODY_EPIL]] ] ; UNROLL-4-NEXT: br label [[FOR_END_LOOPEXIT]] ; UNROLL-4: for.end.loopexit: -; UNROLL-4-NEXT: [[ADD_LCSSA:%.*]] = phi i3 [ [[ADD_LCSSA_PH]], [[FOR_END_LOOPEXIT_UNR_LCSSA]] ], [ [[ADD_LCSSA_PH1]], [[FOR_END_LOOPEXIT_EPILOG_LCSSA]] ] +; UNROLL-4-NEXT: [[ADD_LCSSA:%.*]] = phi i3 [ [[ADD_LCSSA_PH_PH]], [[FOR_END_LOOPEXIT_UNR_LCSSA]] ], [ [[ADD_LCSSA_PH1]], [[FOR_END_LOOPEXIT_EPILOG_LCSSA]] ] ; UNROLL-4-NEXT: br label [[FOR_END]] ; UNROLL-4: for.end: ; UNROLL-4-NEXT: [[SUM_0_LCSSA:%.*]] = phi i3 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_END_LOOPEXIT]] ] diff --git a/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll b/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll index d3e5e0b0b6980..65ef3e40ec229 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll @@ -19,7 +19,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) { ; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 7 -; CHECK-NEXT: br i1 [[TMP2]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP2]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[TMP0]], -8 ; CHECK-NEXT: br label [[HEADER:%.*]] @@ -94,20 +94,19 @@ define i32 @test1(ptr nocapture %a, i64 %n) { ; CHECK-NEXT: [[INDVARS_IV_NEXT_7]] = add i64 [[INDVARS_IV]], 8 ; CHECK-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[HEADER]] -; CHECK: latchexit.unr-lcssa.loopexit: -; CHECK-NEXT: br label [[LATCHEXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[HEADER]] ; CHECK: latchexit.unr-lcssa: -; CHECK-NEXT: [[SUM_0_LCSSA_PH:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[ADD_7]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT_7]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[SUM_02_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD_7]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] ; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[LATCHEXIT:%.*]], label [[HEADER_EPIL_PREHEADER:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[LATCHEXIT:%.*]], label [[HEADER_EPIL_PREHEADER]] ; CHECK: header.epil.preheader: +; CHECK-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_7]], [[LATCHEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[SUM_02_EPIL_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD_7]], [[LATCHEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD3:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD3]]) ; CHECK-NEXT: br label [[HEADER_EPIL:%.*]] ; CHECK: header.epil: -; CHECK-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_UNR]], [[HEADER_EPIL_PREHEADER]] ] -; CHECK-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_UNR]], [[HEADER_EPIL_PREHEADER]] ] +; CHECK-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] +; CHECK-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] ; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ [[EPIL_ITER_NEXT:%.*]], [[LATCH_EPIL]] ], [ 0, [[HEADER_EPIL_PREHEADER]] ] ; CHECK-NEXT: br label [[FOR_EXITING_BLOCK_EPIL:%.*]] ; CHECK: for.exiting_block.epil: @@ -124,11 +123,11 @@ define i32 @test1(ptr nocapture %a, i64 %n) { ; CHECK: latchexit.epilog-lcssa: ; CHECK-NEXT: br label [[LATCHEXIT]] ; CHECK: latchexit: -; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[ADD_EPIL]], [[LATCHEXIT_EPILOG_LCSSA]] ] +; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[ADD_7]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[ADD_EPIL]], [[LATCHEXIT_EPILOG_LCSSA]] ] ; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] ; CHECK: otherexit.loopexit: ; CHECK-NEXT: br label [[OTHEREXIT:%.*]] -; CHECK: otherexit.loopexit3: +; CHECK: otherexit.loopexit4: ; CHECK-NEXT: br label [[OTHEREXIT]] ; CHECK: otherexit: ; CHECK-NEXT: [[SUM_02_LCSSA:%.*]] = phi i32 [ [[SUM_02]], [[OTHEREXIT_LOOPEXIT]] ], [ [[SUM_02_EPIL]], [[OTHEREXIT_LOOPEXIT3]] ] @@ -166,7 +165,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -1 ; ENABLED-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 ; ENABLED-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 7 -; ENABLED-NEXT: br i1 [[TMP2]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; ENABLED-NEXT: br i1 [[TMP2]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; ENABLED: entry.new: ; ENABLED-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]] ; ENABLED-NEXT: br label [[HEADER:%.*]] @@ -248,23 +247,22 @@ define i32 @test1(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[INDVARS_IV_NEXT_7]] = add i64 [[INDVARS_IV]], 8 ; ENABLED-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; ENABLED-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; ENABLED-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[HEADER]] -; ENABLED: latchexit.unr-lcssa.loopexit: +; ENABLED-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[HEADER]] +; ENABLED: latchexit.unr-lcssa: ; ENABLED-NEXT: [[SUM_0_LCSSA_PH_PH:%.*]] = phi i32 [ [[ADD_7]], [[LATCH_7]] ] ; ENABLED-NEXT: [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_7]], [[LATCH_7]] ] ; ENABLED-NEXT: [[SUM_02_UNR_PH:%.*]] = phi i32 [ [[ADD_7]], [[LATCH_7]] ] -; ENABLED-NEXT: br label [[LATCHEXIT_UNR_LCSSA]] -; ENABLED: latchexit.unr-lcssa: -; ENABLED-NEXT: [[SUM_0_LCSSA_PH:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[SUM_0_LCSSA_PH_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; ENABLED-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_UNR_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; ENABLED-NEXT: [[SUM_02_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] ; ENABLED-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[LATCHEXIT:%.*]] +; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER]], label [[LATCHEXIT:%.*]] ; ENABLED: header.epil.preheader: +; ENABLED-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_UNR_PH]], [[LATCHEXIT_UNR_LCSSA]] ] +; ENABLED-NEXT: [[SUM_02_EPIL_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[LATCHEXIT_UNR_LCSSA]] ] +; ENABLED-NEXT: [[LCMP_MOD3:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; ENABLED-NEXT: call void @llvm.assume(i1 [[LCMP_MOD3]]) ; ENABLED-NEXT: br label [[HEADER_EPIL:%.*]] ; ENABLED: header.epil: -; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_UNR]], [[HEADER_EPIL_PREHEADER]] ] -; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_UNR]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] ; ENABLED-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, [[HEADER_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[LATCH_EPIL]] ] ; ENABLED-NEXT: br label [[FOR_EXITING_BLOCK_EPIL:%.*]] ; ENABLED: for.exiting_block.epil: @@ -283,12 +281,12 @@ define i32 @test1(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[SUM_0_LCSSA_PH2:%.*]] = phi i32 [ [[ADD_EPIL]], [[LATCH_EPIL]] ] ; ENABLED-NEXT: br label [[LATCHEXIT]] ; ENABLED: latchexit: -; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[SUM_0_LCSSA_PH2]], [[LATCHEXIT_EPILOG_LCSSA]] ] +; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[SUM_0_LCSSA_PH2]], [[LATCHEXIT_EPILOG_LCSSA]] ] ; ENABLED-NEXT: ret i32 [[SUM_0_LCSSA]] ; ENABLED: otherexit.loopexit: ; ENABLED-NEXT: [[SUM_02_LCSSA_PH:%.*]] = phi i32 [ [[SUM_02]], [[FOR_EXITING_BLOCK]] ], [ [[ADD]], [[FOR_EXITING_BLOCK_1]] ], [ [[ADD_1]], [[FOR_EXITING_BLOCK_2]] ], [ [[ADD_2]], [[FOR_EXITING_BLOCK_3]] ], [ [[ADD_3]], [[FOR_EXITING_BLOCK_4]] ], [ [[ADD_4]], [[FOR_EXITING_BLOCK_5]] ], [ [[ADD_5]], [[FOR_EXITING_BLOCK_6]] ], [ [[ADD_6]], [[FOR_EXITING_BLOCK_7]] ] ; ENABLED-NEXT: br label [[OTHEREXIT:%.*]] -; ENABLED: otherexit.loopexit3: +; ENABLED: otherexit.loopexit4: ; ENABLED-NEXT: [[SUM_02_LCSSA_PH4:%.*]] = phi i32 [ [[SUM_02_EPIL]], [[FOR_EXITING_BLOCK_EPIL]] ] ; ENABLED-NEXT: br label [[OTHEREXIT]] ; ENABLED: otherexit: @@ -380,7 +378,7 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -1 ; ENABLED-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 ; ENABLED-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 7 -; ENABLED-NEXT: br i1 [[TMP2]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; ENABLED-NEXT: br i1 [[TMP2]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; ENABLED: entry.new: ; ENABLED-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]] ; ENABLED-NEXT: br label [[HEADER:%.*]] @@ -462,23 +460,22 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[INDVARS_IV_NEXT_7]] = add i64 [[INDVARS_IV]], 8 ; ENABLED-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; ENABLED-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; ENABLED-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[HEADER]] -; ENABLED: latchexit.unr-lcssa.loopexit: +; ENABLED-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[HEADER]] +; ENABLED: latchexit.unr-lcssa: ; ENABLED-NEXT: [[SUM_0_LCSSA_PH_PH:%.*]] = phi i32 [ [[ADD_7]], [[LATCH_7]] ] ; ENABLED-NEXT: [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_7]], [[LATCH_7]] ] ; ENABLED-NEXT: [[SUM_02_UNR_PH:%.*]] = phi i32 [ [[ADD_7]], [[LATCH_7]] ] -; ENABLED-NEXT: br label [[LATCHEXIT_UNR_LCSSA]] -; ENABLED: latchexit.unr-lcssa: -; ENABLED-NEXT: [[SUM_0_LCSSA_PH:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[SUM_0_LCSSA_PH_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; ENABLED-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_UNR_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; ENABLED-NEXT: [[SUM_02_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] ; ENABLED-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[LATCHEXIT:%.*]] +; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER]], label [[LATCHEXIT:%.*]] ; ENABLED: header.epil.preheader: +; ENABLED-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_UNR_PH]], [[LATCHEXIT_UNR_LCSSA]] ] +; ENABLED-NEXT: [[SUM_02_EPIL_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[LATCHEXIT_UNR_LCSSA]] ] +; ENABLED-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; ENABLED-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; ENABLED-NEXT: br label [[HEADER_EPIL:%.*]] ; ENABLED: header.epil: -; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_UNR]], [[HEADER_EPIL_PREHEADER]] ] -; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_UNR]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] ; ENABLED-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, [[HEADER_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[LATCH_EPIL]] ] ; ENABLED-NEXT: br label [[FOR_EXITING_BLOCK_EPIL:%.*]] ; ENABLED: for.exiting_block.epil: @@ -497,12 +494,12 @@ define i32 @test2(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[SUM_0_LCSSA_PH1:%.*]] = phi i32 [ [[ADD_EPIL]], [[LATCH_EPIL]] ] ; ENABLED-NEXT: br label [[LATCHEXIT]] ; ENABLED: latchexit: -; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[SUM_0_LCSSA_PH1]], [[LATCHEXIT_EPILOG_LCSSA]] ] +; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[SUM_0_LCSSA_PH1]], [[LATCHEXIT_EPILOG_LCSSA]] ] ; ENABLED-NEXT: ret i32 [[SUM_0_LCSSA]] ; ENABLED: otherexit.loopexit: ; ENABLED-NEXT: [[RVAL_PH:%.*]] = phi i32 [ [[SUM_02]], [[FOR_EXITING_BLOCK]] ], [ [[ADD]], [[FOR_EXITING_BLOCK_1]] ], [ [[ADD_1]], [[FOR_EXITING_BLOCK_2]] ], [ [[ADD_2]], [[FOR_EXITING_BLOCK_3]] ], [ [[ADD_3]], [[FOR_EXITING_BLOCK_4]] ], [ [[ADD_4]], [[FOR_EXITING_BLOCK_5]] ], [ [[ADD_5]], [[FOR_EXITING_BLOCK_6]] ], [ [[ADD_6]], [[FOR_EXITING_BLOCK_7]] ] ; ENABLED-NEXT: br label [[OTHEREXIT:%.*]] -; ENABLED: otherexit.loopexit2: +; ENABLED: otherexit.loopexit3: ; ENABLED-NEXT: [[RVAL_PH3:%.*]] = phi i32 [ [[SUM_02_EPIL]], [[FOR_EXITING_BLOCK_EPIL]] ] ; ENABLED-NEXT: br label [[OTHEREXIT]] ; ENABLED: otherexit: @@ -747,7 +744,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) { ; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 ; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 7 -; CHECK-NEXT: br i1 [[TMP2]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP2]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[TMP0]], -8 ; CHECK-NEXT: br label [[HEADER:%.*]] @@ -822,20 +819,19 @@ define i32 @test5(ptr nocapture %a, i64 %n) { ; CHECK-NEXT: [[INDVARS_IV_NEXT_7]] = add i64 [[INDVARS_IV]], 8 ; CHECK-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[HEADER]] -; CHECK: latchexit.unr-lcssa.loopexit: -; CHECK-NEXT: br label [[LATCHEXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[HEADER]] ; CHECK: latchexit.unr-lcssa: -; CHECK-NEXT: [[SUM_0_LCSSA_PH:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[ADD_7]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT_7]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[SUM_02_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD_7]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] ; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[LATCHEXIT:%.*]], label [[HEADER_EPIL_PREHEADER:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[LATCHEXIT:%.*]], label [[HEADER_EPIL_PREHEADER]] ; CHECK: header.epil.preheader: +; CHECK-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_7]], [[LATCHEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[SUM_02_EPIL_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD_7]], [[LATCHEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD3:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD3]]) ; CHECK-NEXT: br label [[HEADER_EPIL:%.*]] ; CHECK: header.epil: -; CHECK-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_UNR]], [[HEADER_EPIL_PREHEADER]] ] -; CHECK-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_UNR]], [[HEADER_EPIL_PREHEADER]] ] +; CHECK-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] +; CHECK-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] ; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ [[EPIL_ITER_NEXT:%.*]], [[LATCH_EPIL]] ], [ 0, [[HEADER_EPIL_PREHEADER]] ] ; CHECK-NEXT: br label [[FOR_EXITING_BLOCK_EPIL:%.*]] ; CHECK: for.exiting_block.epil: @@ -852,11 +848,11 @@ define i32 @test5(ptr nocapture %a, i64 %n) { ; CHECK: latchexit.epilog-lcssa: ; CHECK-NEXT: br label [[LATCHEXIT]] ; CHECK: latchexit: -; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[ADD_EPIL]], [[LATCHEXIT_EPILOG_LCSSA]] ] +; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[ADD_7]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[ADD_EPIL]], [[LATCHEXIT_EPILOG_LCSSA]] ] ; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] ; CHECK: otherexit.loopexit: ; CHECK-NEXT: br label [[OTHEREXIT:%.*]] -; CHECK: otherexit.loopexit3: +; CHECK: otherexit.loopexit4: ; CHECK-NEXT: br label [[OTHEREXIT]] ; CHECK: otherexit: ; CHECK-NEXT: [[SUM_02_LCSSA:%.*]] = phi i32 [ [[SUM_02]], [[OTHEREXIT_LOOPEXIT]] ], [ [[SUM_02_EPIL]], [[OTHEREXIT_LOOPEXIT3]] ] @@ -899,7 +895,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -1 ; ENABLED-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 ; ENABLED-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 7 -; ENABLED-NEXT: br i1 [[TMP2]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; ENABLED-NEXT: br i1 [[TMP2]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; ENABLED: entry.new: ; ENABLED-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]] ; ENABLED-NEXT: br label [[HEADER:%.*]] @@ -981,23 +977,22 @@ define i32 @test5(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[INDVARS_IV_NEXT_7]] = add i64 [[INDVARS_IV]], 8 ; ENABLED-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; ENABLED-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; ENABLED-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[HEADER]] -; ENABLED: latchexit.unr-lcssa.loopexit: +; ENABLED-NEXT: br i1 [[NITER_NCMP_7]], label [[LATCHEXIT_UNR_LCSSA:%.*]], label [[HEADER]] +; ENABLED: latchexit.unr-lcssa: ; ENABLED-NEXT: [[SUM_0_LCSSA_PH_PH:%.*]] = phi i32 [ [[ADD_7]], [[LATCH_7]] ] ; ENABLED-NEXT: [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_7]], [[LATCH_7]] ] ; ENABLED-NEXT: [[SUM_02_UNR_PH:%.*]] = phi i32 [ [[ADD_7]], [[LATCH_7]] ] -; ENABLED-NEXT: br label [[LATCHEXIT_UNR_LCSSA]] -; ENABLED: latchexit.unr-lcssa: -; ENABLED-NEXT: [[SUM_0_LCSSA_PH:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[SUM_0_LCSSA_PH_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; ENABLED-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_UNR_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] -; ENABLED-NEXT: [[SUM_02_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[LATCHEXIT_UNR_LCSSA_LOOPEXIT]] ] ; ENABLED-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER:%.*]], label [[LATCHEXIT:%.*]] +; ENABLED-NEXT: br i1 [[LCMP_MOD]], label [[HEADER_EPIL_PREHEADER]], label [[LATCHEXIT:%.*]] ; ENABLED: header.epil.preheader: +; ENABLED-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_UNR_PH]], [[LATCHEXIT_UNR_LCSSA]] ] +; ENABLED-NEXT: [[SUM_02_EPIL_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[SUM_02_UNR_PH]], [[LATCHEXIT_UNR_LCSSA]] ] +; ENABLED-NEXT: [[LCMP_MOD3:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; ENABLED-NEXT: call void @llvm.assume(i1 [[LCMP_MOD3]]) ; ENABLED-NEXT: br label [[HEADER_EPIL:%.*]] ; ENABLED: header.epil: -; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_UNR]], [[HEADER_EPIL_PREHEADER]] ] -; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_UNR]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[LATCH_EPIL:%.*]] ], [ [[INDVARS_IV_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] +; ENABLED-NEXT: [[SUM_02_EPIL:%.*]] = phi i32 [ [[ADD_EPIL:%.*]], [[LATCH_EPIL]] ], [ [[SUM_02_EPIL_INIT]], [[HEADER_EPIL_PREHEADER]] ] ; ENABLED-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, [[HEADER_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[LATCH_EPIL]] ] ; ENABLED-NEXT: br label [[FOR_EXITING_BLOCK_EPIL:%.*]] ; ENABLED: for.exiting_block.epil: @@ -1016,13 +1011,13 @@ define i32 @test5(ptr nocapture %a, i64 %n) { ; ENABLED-NEXT: [[SUM_0_LCSSA_PH2:%.*]] = phi i32 [ [[ADD_EPIL]], [[LATCH_EPIL]] ] ; ENABLED-NEXT: br label [[LATCHEXIT]] ; ENABLED: latchexit: -; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[SUM_0_LCSSA_PH2]], [[LATCHEXIT_EPILOG_LCSSA]] ] +; ENABLED-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[SUM_0_LCSSA_PH2]], [[LATCHEXIT_EPILOG_LCSSA]] ] ; ENABLED-NEXT: ret i32 [[SUM_0_LCSSA]] ; ENABLED: otherexit.loopexit: ; ENABLED-NEXT: [[SUM_02_LCSSA_PH:%.*]] = phi i32 [ [[SUM_02]], [[FOR_EXITING_BLOCK]] ], [ [[ADD]], [[FOR_EXITING_BLOCK_1]] ], [ [[ADD_1]], [[FOR_EXITING_BLOCK_2]] ], [ [[ADD_2]], [[FOR_EXITING_BLOCK_3]] ], [ [[ADD_3]], [[FOR_EXITING_BLOCK_4]] ], [ [[ADD_4]], [[FOR_EXITING_BLOCK_5]] ], [ [[ADD_5]], [[FOR_EXITING_BLOCK_6]] ], [ [[ADD_6]], [[FOR_EXITING_BLOCK_7]] ] ; ENABLED-NEXT: [[RVAL_PH:%.*]] = phi i32 [ [[SUM_02]], [[FOR_EXITING_BLOCK]] ], [ [[ADD]], [[FOR_EXITING_BLOCK_1]] ], [ [[ADD_1]], [[FOR_EXITING_BLOCK_2]] ], [ [[ADD_2]], [[FOR_EXITING_BLOCK_3]] ], [ [[ADD_3]], [[FOR_EXITING_BLOCK_4]] ], [ [[ADD_4]], [[FOR_EXITING_BLOCK_5]] ], [ [[ADD_5]], [[FOR_EXITING_BLOCK_6]] ], [ [[ADD_6]], [[FOR_EXITING_BLOCK_7]] ] ; ENABLED-NEXT: br label [[OTHEREXIT:%.*]] -; ENABLED: otherexit.loopexit3: +; ENABLED: otherexit.loopexit4: ; ENABLED-NEXT: [[SUM_02_LCSSA_PH4:%.*]] = phi i32 [ [[SUM_02_EPIL]], [[FOR_EXITING_BLOCK_EPIL]] ] ; ENABLED-NEXT: [[RVAL_PH5:%.*]] = phi i32 [ [[SUM_02_EPIL]], [[FOR_EXITING_BLOCK_EPIL]] ] ; ENABLED-NEXT: br label [[OTHEREXIT]] diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-assume-no-remainder.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-assume-no-remainder.ll index 81fceb61a2450..73f7fd37a0099 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-assume-no-remainder.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-assume-no-remainder.ll @@ -91,7 +91,7 @@ define dso_local void @cannotProveDivisibleTC(ptr noalias nocapture %a, ptr noal ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[N]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 1 -; CHECK-NEXT: br i1 [[TMP1]], label [[EXIT_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] ; CHECK: for.body.preheader.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] @@ -112,15 +112,15 @@ define dso_local void @cannotProveDivisibleTC(ptr noalias nocapture %a, ptr noal ; CHECK-NEXT: [[INC_1]] = add nuw nsw i32 [[I_011]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i32 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp ne i32 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]] -; CHECK: exit.loopexit.unr-lcssa.loopexit: -; CHECK-NEXT: [[I_011_UNR_PH:%.*]] = phi i32 [ [[INC_1]], [[FOR_BODY]] ] -; CHECK-NEXT: br label [[EXIT_LOOPEXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT_UNR_LCSSA:%.*]], !llvm.loop [[LOOP2:![0-9]+]] ; CHECK: exit.loopexit.unr-lcssa: -; CHECK-NEXT: [[I_011_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[I_011_UNR_PH]], [[EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[I_011_UNR1:%.*]] = phi i32 [ [[INC_1]], [[FOR_BODY]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[FOR_BODY_EPIL_PREHEADER]], label [[EXIT_LOOPEXIT:%.*]] ; CHECK: for.body.epil.preheader: +; CHECK-NEXT: [[I_011_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[I_011_UNR1]], [[EXIT_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[FOR_BODY_EPIL:%.*]] ; CHECK: for.body.epil: ; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[I_011_UNR]] diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll index 0b9c6ac1d324b..a5ac2cf46653d 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-reductions.ll @@ -8,7 +8,7 @@ define i32 @test_add_reduction(ptr %a, i64 %n) { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; CHECK-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; CHECK: [[ENTRY_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label %[[LOOP:.*]] @@ -27,28 +27,27 @@ define i32 @test_add_reduction(ptr %a, i64 %n) { ; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]] -; CHECK: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[RES_PH_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ] -; CHECK-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ] -; CHECK-NEXT: [[RDX_UNR_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ] -; CHECK-NEXT: [[BIN_RDX:%.*]] = add i32 [[RDX_NEXT_1]], [[RDX_NEXT]] -; CHECK-NEXT: br label %[[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[EXIT_UNR_LCSSA]]: -; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ] +; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ] +; CHECK-NEXT: [[RDX_UNR1:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ] +; CHECK-NEXT: [[BIN_RDX:%.*]] = add i32 [[RDX_NEXT_1]], [[RDX_NEXT]] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; CHECK: [[LOOP_EPIL_PREHEADER]]: +; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; CHECK-NEXT: br label %[[LOOP_EPIL:.*]] ; CHECK: [[LOOP_EPIL]]: -; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_UNR]] +; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_EPIL_INIT]] ; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 2 ; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = add nuw nsw i32 [[RDX_UNR]], [[TMP4]] ; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: -; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RES_PH]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ] +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[BIN_RDX]], %[[EXIT_UNR_LCSSA]] ], [ [[RDX_NEXT_EPIL]], %[[LOOP_EPIL]] ] ; CHECK-NEXT: ret i32 [[RES]] ; entry: @@ -76,7 +75,7 @@ define i32 @test_add_reduction_constant_op(ptr %a, i64 %n) { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; CHECK-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; CHECK: [[ENTRY_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label %[[LOOP:.*]] @@ -88,17 +87,16 @@ define i32 @test_add_reduction_constant_op(ptr %a, i64 %n) { ; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP2:![0-9]+]] -; CHECK: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[RES_PH_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ] -; CHECK-NEXT: [[RDX_UNR_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ] -; CHECK-NEXT: br label %[[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP2:![0-9]+]] ; CHECK: [[EXIT_UNR_LCSSA]]: -; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[RES_PH_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ] +; CHECK-NEXT: [[RDX_UNR1:%.*]] = phi i32 [ [[RDX_NEXT_1]], %[[LOOP]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; CHECK: [[LOOP_EPIL_PREHEADER]]: +; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_UNR1]], %[[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; CHECK-NEXT: br label %[[LOOP_EPIL:.*]] ; CHECK: [[LOOP_EPIL]]: ; CHECK-NEXT: [[RDX_NEXT_EPIL:%.*]] = add nuw nsw i32 [[RDX_UNR]], 1 @@ -130,7 +128,7 @@ define i32 @test_add_reduction_8x_unroll(ptr %a, i64 %n) { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 7 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7 -; CHECK-NEXT: br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; CHECK: [[ENTRY_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label %[[LOOP:.*]] @@ -172,23 +170,22 @@ define i32 @test_add_reduction_8x_unroll(ptr %a, i64 %n) { ; CHECK-NEXT: [[IV_NEXT_7]] = add nuw nsw i64 [[IV]], 8 ; CHECK-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]] -; CHECK: [[EXIT_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[RES_PH_PH:%.*]] = phi i32 [ [[RDX_NEXT_7]], %[[LOOP]] ] -; CHECK-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[LOOP]] ] -; CHECK-NEXT: [[RDX_UNR_PH:%.*]] = phi i32 [ [[RDX_NEXT_7]], %[[LOOP]] ] -; CHECK-NEXT: br label %[[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: [[EXIT_UNR_LCSSA]]: -; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[RES_PH_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[RES_PH:%.*]] = phi i32 [ [[RDX_NEXT_7]], %[[LOOP]] ] +; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ [[IV_NEXT_7]], %[[LOOP]] ] +; CHECK-NEXT: [[RDX_UNR:%.*]] = phi i32 [ [[RDX_NEXT_7]], %[[LOOP]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER]], label %[[EXIT:.*]] ; CHECK: [[LOOP_EPIL_PREHEADER]]: +; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[RDX_EPIL_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[RDX_UNR]], %[[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; CHECK-NEXT: br label %[[LOOP_EPIL:.*]] ; CHECK: [[LOOP_EPIL]]: -; CHECK-NEXT: [[IV_EPIL:%.*]] = phi i64 [ [[IV_UNR]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] -; CHECK-NEXT: [[RDX_EPIL:%.*]] = phi i32 [ [[RDX_UNR]], %[[LOOP_EPIL_PREHEADER]] ], [ [[RDX_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] +; CHECK-NEXT: [[IV_EPIL:%.*]] = phi i64 [ [[IV_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] +; CHECK-NEXT: [[RDX_EPIL:%.*]] = phi i32 [ [[RDX_EPIL_INIT]], %[[LOOP_EPIL_PREHEADER]] ], [ [[RDX_NEXT_EPIL:%.*]], %[[LOOP_EPIL]] ] ; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, %[[LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_EPIL]] ] ; CHECK-NEXT: [[GEP_A_EPIL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[IV_EPIL]] ; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[GEP_A_EPIL]], align 2 diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll index a3cfeac1f86be..5f4bbf15f01af 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll @@ -11,31 +11,30 @@ define i32 @unroll(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %N) ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 3 ; CHECK-NEXT: [[TMP0:%.*]] = icmp ult i32 [[N]], 4 -; CHECK-NEXT: br i1 [[TMP0]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY_LR_PH_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP0]], label [[FOR_BODY_EPIL_PREHEADER:%.*]], label [[FOR_BODY_LR_PH_NEW:%.*]] ; CHECK: for.body.lr.ph.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292 ; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.cond.cleanup.loopexit.unr-lcssa.loopexit: -; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ; CHECK: for.cond.cleanup.loopexit.unr-lcssa: -; CHECK-NEXT: [[ADD_LCSSA_PH:%.*]] = phi i32 [ poison, [[FOR_BODY_LR_PH]] ], [ [[ADD_3:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT:%.*]] ] -; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[C_010_UNR:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[ADD_3]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] ; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY_EPIL_PREHEADER:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[FOR_BODY_EPIL_PREHEADER]] ; CHECK: for.body.epil.preheader: +; CHECK-NEXT: [[INDVARS_IV_EPIL_INIT:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]] ] +; CHECK-NEXT: [[C_010_EPIL_INIT:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[ADD_3:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; CHECK-NEXT: br label [[FOR_BODY_EPIL:%.*]] ; CHECK: for.body.epil: -; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV_UNR]] +; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV_EPIL_INIT]] ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX_EPIL]], align 4 -; CHECK-NEXT: [[ARRAYIDX2_EPIL:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV_UNR]] +; CHECK-NEXT: [[ARRAYIDX2_EPIL:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV_EPIL_INIT]] ; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX2_EPIL]], align 4 ; CHECK-NEXT: [[MUL_EPIL:%.*]] = mul nsw i32 [[TMP2]], [[TMP1]] -; CHECK-NEXT: [[ADD_EPIL:%.*]] = add nsw i32 [[MUL_EPIL]], [[C_010_UNR]] +; CHECK-NEXT: [[ADD_EPIL:%.*]] = add nsw i32 [[MUL_EPIL]], [[C_010_EPIL_INIT]] ; CHECK-NEXT: [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 1 ; CHECK-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA:%.*]], label [[FOR_BODY_EPIL_1:%.*]] ; CHECK: for.body.epil.1: -; CHECK-NEXT: [[INDVARS_IV_NEXT_EPIL:%.*]] = add nuw nsw i64 [[INDVARS_IV_UNR]], 1 +; CHECK-NEXT: [[INDVARS_IV_NEXT_EPIL:%.*]] = add nuw nsw i64 [[INDVARS_IV_EPIL_INIT]], 1 ; CHECK-NEXT: [[ARRAYIDX_EPIL_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_EPIL]] ; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX_EPIL_1]], align 4 ; CHECK-NEXT: [[ARRAYIDX2_EPIL_1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV_NEXT_EPIL]] @@ -45,7 +44,7 @@ define i32 @unroll(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %N) ; CHECK-NEXT: [[EPIL_ITER_CMP_1_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 2 ; CHECK-NEXT: br i1 [[EPIL_ITER_CMP_1_NOT]], label [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA]], label [[FOR_BODY_EPIL_2:%.*]] ; CHECK: for.body.epil.2: -; CHECK-NEXT: [[INDVARS_IV_NEXT_EPIL_1:%.*]] = add nuw nsw i64 [[INDVARS_IV_UNR]], 2 +; CHECK-NEXT: [[INDVARS_IV_NEXT_EPIL_1:%.*]] = add nuw nsw i64 [[INDVARS_IV_EPIL_INIT]], 2 ; CHECK-NEXT: [[ARRAYIDX_EPIL_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_EPIL_1]] ; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX_EPIL_2]], align 4 ; CHECK-NEXT: [[ARRAYIDX2_EPIL_2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV_NEXT_EPIL_1]] @@ -57,7 +56,7 @@ define i32 @unroll(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %N) ; CHECK-NEXT: [[ADD_LCSSA_PH1:%.*]] = phi i32 [ [[ADD_EPIL]], [[FOR_BODY_EPIL]] ], [ [[ADD_EPIL_1]], [[FOR_BODY_EPIL_1]] ], [ [[ADD_EPIL_2]], [[FOR_BODY_EPIL_2]] ] ; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT]] ; CHECK: for.cond.cleanup.loopexit: -; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD_LCSSA_PH]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ], [ [[ADD_LCSSA_PH1]], [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA]] ] +; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD_3]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ], [ [[ADD_LCSSA_PH1]], [[FOR_COND_CLEANUP_LOOPEXIT_EPILOG_LCSSA]] ] ; CHECK-NEXT: br label [[FOR_COND_CLEANUP]] ; CHECK: for.cond.cleanup: ; CHECK-NEXT: [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] @@ -96,7 +95,7 @@ define i32 @unroll(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %N) ; CHECK-NEXT: [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4 ; CHECK-NEXT: [[NITER_NEXT_3]] = add i64 [[NITER]], 4 ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; entry: %cmp9 = icmp eq i32 %N, 0 diff --git a/llvm/test/Transforms/LoopUnroll/scev-invalidation-lcssa.ll b/llvm/test/Transforms/LoopUnroll/scev-invalidation-lcssa.ll index 0a3d201e617de..fd07238b16421 100644 --- a/llvm/test/Transforms/LoopUnroll/scev-invalidation-lcssa.ll +++ b/llvm/test/Transforms/LoopUnroll/scev-invalidation-lcssa.ll @@ -30,7 +30,7 @@ define i32 @f(i1 %cond1) #0 !prof !0 { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[LD_LCSSA]], 1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[LD_LCSSA]], 7 -; CHECK-NEXT: br i1 [[TMP1]], label [[EXIT2_UNR_LCSSA:%.*]], label [[ENTRY2_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP2_EPIL_PREHEADER:%.*]], label [[ENTRY2_NEW:%.*]] ; CHECK: entry2.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP0]], [[XTRAITER]] ; CHECK-NEXT: br label [[LOOP2:%.*]] @@ -40,18 +40,18 @@ define i32 @f(i1 %cond1) #0 !prof !0 { ; CHECK-NEXT: [[INC_7]] = add i64 [[PHI]], 8 ; CHECK-NEXT: [[NITER_NEXT_7]] = add i64 [[NITER]], 8 ; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NEXT_7]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[EXIT2_UNR_LCSSA_LOOPEXIT:%.*]], label [[LOOP2]] -; CHECK: exit2.unr-lcssa.loopexit: -; CHECK-NEXT: [[PHI_UNR_PH:%.*]] = phi i64 [ [[INC_7]], [[LOOP2]] ] -; CHECK-NEXT: br label [[EXIT2_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[EXIT2_UNR_LCSSA:%.*]], label [[LOOP2]] ; CHECK: exit2.unr-lcssa: -; CHECK-NEXT: [[PHI_UNR:%.*]] = phi i64 [ 0, [[ENTRY2]] ], [ [[PHI_UNR_PH]], [[EXIT2_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[PHI_UNR:%.*]] = phi i64 [ [[INC_7]], [[LOOP2]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP2_EPIL_PREHEADER:%.*]], label [[EXIT2:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP2_EPIL_PREHEADER]], label [[EXIT2:%.*]] ; CHECK: loop2.epil.preheader: +; CHECK-NEXT: [[PHI_EPIL_INIT:%.*]] = phi i64 [ 0, [[ENTRY2]] ], [ [[PHI_UNR]], [[EXIT2_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; CHECK-NEXT: br label [[LOOP2_EPIL:%.*]] ; CHECK: loop2.epil: -; CHECK-NEXT: [[PHI_EPIL:%.*]] = phi i64 [ [[PHI_UNR]], [[LOOP2_EPIL_PREHEADER]] ], [ [[INC_EPIL:%.*]], [[LOOP2_EPIL]] ] +; CHECK-NEXT: [[PHI_EPIL:%.*]] = phi i64 [ [[PHI_EPIL_INIT]], [[LOOP2_EPIL_PREHEADER]] ], [ [[INC_EPIL:%.*]], [[LOOP2_EPIL]] ] ; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ 0, [[LOOP2_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[LOOP2_EPIL]] ] ; CHECK-NEXT: [[INC_EPIL]] = add i64 [[PHI_EPIL]], 1 ; CHECK-NEXT: [[COND2_EPIL:%.*]] = icmp eq i64 [[LD_LCSSA]], [[PHI_EPIL]] diff --git a/llvm/test/Transforms/LoopUnroll/tripcount-overflow.ll b/llvm/test/Transforms/LoopUnroll/tripcount-overflow.ll index 14812865fd558..f839c88b93e54 100644 --- a/llvm/test/Transforms/LoopUnroll/tripcount-overflow.ll +++ b/llvm/test/Transforms/LoopUnroll/tripcount-overflow.ll @@ -17,7 +17,7 @@ define i32 @foo(i32 %N) { ; EPILOG-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], 1 ; EPILOG-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP0]], 1 ; EPILOG-NEXT: [[TMP1:%.*]] = icmp ult i32 [[N]], 1 -; EPILOG-NEXT: br i1 [[TMP1]], label [[WHILE_END_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; EPILOG-NEXT: br i1 [[TMP1]], label [[WHILE_BODY_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; EPILOG: entry.new: ; EPILOG-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[TMP0]], [[XTRAITER]] ; EPILOG-NEXT: br label [[WHILE_BODY:%.*]] @@ -28,22 +28,21 @@ define i32 @foo(i32 %N) { ; EPILOG-NEXT: [[INC_1]] = add i32 [[I]], 2 ; EPILOG-NEXT: [[NITER_NEXT_1]] = add i32 [[NITER]], 2 ; EPILOG-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i32 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; EPILOG-NEXT: br i1 [[NITER_NCMP_1]], label [[WHILE_END_UNR_LCSSA_LOOPEXIT:%.*]], label [[WHILE_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; EPILOG: while.end.unr-lcssa.loopexit: +; EPILOG-NEXT: br i1 [[NITER_NCMP_1]], label [[WHILE_END_UNR_LCSSA:%.*]], label [[WHILE_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; EPILOG: while.end.unr-lcssa: ; EPILOG-NEXT: [[I_LCSSA_PH_PH:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ] ; EPILOG-NEXT: [[I_UNR_PH:%.*]] = phi i32 [ [[INC_1]], [[WHILE_BODY]] ] -; EPILOG-NEXT: br label [[WHILE_END_UNR_LCSSA]] -; EPILOG: while.end.unr-lcssa: -; EPILOG-NEXT: [[I_LCSSA_PH:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[I_LCSSA_PH_PH]], [[WHILE_END_UNR_LCSSA_LOOPEXIT]] ] -; EPILOG-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[I_UNR_PH]], [[WHILE_END_UNR_LCSSA_LOOPEXIT]] ] ; EPILOG-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; EPILOG-NEXT: br i1 [[LCMP_MOD]], label [[WHILE_BODY_EPIL_PREHEADER:%.*]], label [[WHILE_END:%.*]] +; EPILOG-NEXT: br i1 [[LCMP_MOD]], label [[WHILE_BODY_EPIL_PREHEADER]], label [[WHILE_END:%.*]] ; EPILOG: while.body.epil.preheader: +; EPILOG-NEXT: [[I_EPIL_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_UNR_PH]], [[WHILE_END_UNR_LCSSA]] ] +; EPILOG-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; EPILOG-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]]) ; EPILOG-NEXT: br label [[WHILE_BODY_EPIL:%.*]] ; EPILOG: while.body.epil: ; EPILOG-NEXT: br label [[WHILE_END]] ; EPILOG: while.end: -; EPILOG-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I_LCSSA_PH]], [[WHILE_END_UNR_LCSSA]] ], [ [[I_UNR]], [[WHILE_BODY_EPIL]] ] +; EPILOG-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I_LCSSA_PH_PH]], [[WHILE_END_UNR_LCSSA]] ], [ [[I_EPIL_INIT]], [[WHILE_BODY_EPIL]] ] ; EPILOG-NEXT: ret i32 [[I_LCSSA]] ; ; PROLOG-LABEL: @foo( diff --git a/llvm/test/Transforms/LoopUnroll/unroll-heuristics-pgo.ll b/llvm/test/Transforms/LoopUnroll/unroll-heuristics-pgo.ll index 20a247f3e7490..611ee5fb5807e 100644 --- a/llvm/test/Transforms/LoopUnroll/unroll-heuristics-pgo.ll +++ b/llvm/test/Transforms/LoopUnroll/unroll-heuristics-pgo.ll @@ -8,7 +8,7 @@ ; CHECK: %mul.1 = mul ; CHECK: %mul.2 = mul ; CHECK: %mul.3 = mul -; CHECK: br i1 %niter.ncmp.7, label %loop.end.unr-lcssa.loopexit, label %loop, !prof [[PROF0:![0-9]+]] +; CHECK: br i1 %niter.ncmp.7, label %loop.end.unr-lcssa, label %loop, !prof [[PROF0:![0-9]+]] ; CHECK: loop.epil: ; CHECK: br i1 %epil.iter.cmp, label %loop.epil, label %loop.end.epilog-lcssa, !prof [[PROF1:![0-9]+]], !llvm.loop {{![0-9]+}} define i32 @bar_prof(ptr noalias nocapture readonly %src, i64 %c) !prof !1 { diff --git a/llvm/test/Transforms/LoopUnroll/unroll-loads-cse.ll b/llvm/test/Transforms/LoopUnroll/unroll-loads-cse.ll index d4105254e5314..f85aac75539c6 100644 --- a/llvm/test/Transforms/LoopUnroll/unroll-loads-cse.ll +++ b/llvm/test/Transforms/LoopUnroll/unroll-loads-cse.ll @@ -12,7 +12,7 @@ define void @cse_matching_load_from_previous_unrolled_iteration(ptr %src, ptr no ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; CHECK-NEXT: br i1 [[TMP1]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label [[LOOP:%.*]] @@ -35,15 +35,15 @@ define void @cse_matching_load_from_previous_unrolled_iteration(ptr %src, ptr no ; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]] -; CHECK: exit.unr-lcssa.loopexit: -; CHECK-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] -; CHECK-NEXT: br label [[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: exit.unr-lcssa: -; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR_PH]], [[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER]], label [[EXIT:%.*]] ; CHECK: loop.epil.preheader: +; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR1]], [[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[LOOP_EPIL:%.*]] ; CHECK: loop.epil: ; CHECK-NEXT: [[GEP_SRC_12_EPIL:%.*]] = getelementptr i64, ptr [[SRC_12]], i64 [[IV_UNR]] @@ -88,7 +88,7 @@ define void @cse_different_load_types(ptr %src, ptr noalias %dst, i64 %N) { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; CHECK-NEXT: br i1 [[TMP1]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label [[LOOP:%.*]] @@ -115,15 +115,15 @@ define void @cse_different_load_types(ptr %src, ptr noalias %dst, i64 %N) { ; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]] -; CHECK: exit.unr-lcssa.loopexit: -; CHECK-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] -; CHECK-NEXT: br label [[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA:%.*]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: exit.unr-lcssa: -; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR_PH]], [[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER]], label [[EXIT:%.*]] ; CHECK: loop.epil.preheader: +; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR1]], [[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[LOOP_EPIL:%.*]] ; CHECK: loop.epil: ; CHECK-NEXT: [[GEP_SRC_12_EPIL:%.*]] = getelementptr i64, ptr [[SRC_12]], i64 [[IV_UNR]] @@ -170,7 +170,7 @@ define void @cse_volatile_loads(ptr %src, ptr noalias %dst, i64 %N) { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; CHECK-NEXT: br i1 [[TMP1]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label [[LOOP:%.*]] @@ -195,15 +195,15 @@ define void @cse_volatile_loads(ptr %src, ptr noalias %dst, i64 %N) { ; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]] -; CHECK: exit.unr-lcssa.loopexit: -; CHECK-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] -; CHECK-NEXT: br label [[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA:%.*]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: exit.unr-lcssa: -; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR_PH]], [[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER]], label [[EXIT:%.*]] ; CHECK: loop.epil.preheader: +; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR1]], [[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[LOOP_EPIL:%.*]] ; CHECK: loop.epil: ; CHECK-NEXT: [[GEP_SRC_12_EPIL:%.*]] = getelementptr i64, ptr [[SRC_12]], i64 [[IV_UNR]] @@ -248,7 +248,7 @@ define void @cse_atomic_loads(ptr %src, ptr noalias %dst, i64 %N) { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; CHECK-NEXT: br i1 [[TMP1]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label [[LOOP:%.*]] @@ -273,15 +273,15 @@ define void @cse_atomic_loads(ptr %src, ptr noalias %dst, i64 %N) { ; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]] -; CHECK: exit.unr-lcssa.loopexit: -; CHECK-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] -; CHECK-NEXT: br label [[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA:%.*]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: exit.unr-lcssa: -; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR_PH]], [[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER]], label [[EXIT:%.*]] ; CHECK: loop.epil.preheader: +; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR1]], [[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[LOOP_EPIL:%.*]] ; CHECK: loop.epil: ; CHECK-NEXT: [[GEP_SRC_12_EPIL:%.*]] = getelementptr i64, ptr [[SRC_12]], i64 [[IV_UNR]] @@ -326,7 +326,7 @@ define void @cse_load_may_be_clobbered(ptr %src, ptr %dst, i64 %N) { ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[N]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1 -; CHECK-NEXT: br i1 [[TMP1]], label [[EXIT_UNR_LCSSA:%.*]], label [[ENTRY_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP1]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[ENTRY_NEW:%.*]] ; CHECK: entry.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]] ; CHECK-NEXT: br label [[LOOP:%.*]] @@ -351,15 +351,15 @@ define void @cse_load_may_be_clobbered(ptr %src, ptr %dst, i64 %N) { ; CHECK-NEXT: [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]] -; CHECK: exit.unr-lcssa.loopexit: -; CHECK-NEXT: [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] -; CHECK-NEXT: br label [[EXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_UNR_LCSSA:%.*]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: exit.unr-lcssa: -; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR_PH]], [[EXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[IV_UNR1:%.*]] = phi i64 [ [[IV_NEXT_1]], [[LOOP]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER:%.*]], label [[EXIT:%.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[LOOP_EPIL_PREHEADER]], label [[EXIT:%.*]] ; CHECK: loop.epil.preheader: +; CHECK-NEXT: [[IV_UNR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_UNR1]], [[EXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label [[LOOP_EPIL:%.*]] ; CHECK: loop.epil: ; CHECK-NEXT: [[GEP_SRC_12_EPIL:%.*]] = getelementptr i64, ptr [[SRC_12]], i64 [[IV_UNR]] diff --git a/llvm/test/Transforms/LoopUnrollAndJam/dependencies_visit_order.ll b/llvm/test/Transforms/LoopUnrollAndJam/dependencies_visit_order.ll index f1a5adf096a55..351065080119d 100644 --- a/llvm/test/Transforms/LoopUnrollAndJam/dependencies_visit_order.ll +++ b/llvm/test/Transforms/LoopUnrollAndJam/dependencies_visit_order.ll @@ -6,7 +6,7 @@ target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" define void @test1() { ; CHECK-LABEL: @test1( ; CHECK-NEXT: bb: -; CHECK-NEXT: br i1 false, label [[BB1_BB43_CRIT_EDGE_UNR_LCSSA:%.*]], label [[BB_NEW:%.*]] +; CHECK-NEXT: br i1 false, label [[BB5_PREHEADER_EPIL_PREHEADER:%.*]], label [[BB_NEW:%.*]] ; CHECK: bb.new: ; CHECK-NEXT: br label [[BB5_PREHEADER:%.*]] ; CHECK: bb5.preheader: @@ -30,17 +30,16 @@ define void @test1() { ; CHECK-NEXT: br i1 true, label [[BB38]], label [[BB10_PREHEADER]] ; CHECK: bb38: ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i16 [[NITER_NEXT_3]], -28 -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label [[BB1_BB43_CRIT_EDGE_UNR_LCSSA_LOOPEXIT:%.*]], label [[BB5_PREHEADER]], !llvm.loop [[LOOP0:![0-9]+]] -; CHECK: bb1.bb43_crit_edge.unr-lcssa.loopexit: -; CHECK-NEXT: [[I10_UNR_PH:%.*]] = phi i16 [ [[I42_3]], [[BB38]] ] -; CHECK-NEXT: br label [[BB1_BB43_CRIT_EDGE_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label [[BB1_BB43_CRIT_EDGE_UNR_LCSSA:%.*]], label [[BB5_PREHEADER]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: bb1.bb43_crit_edge.unr-lcssa: -; CHECK-NEXT: [[I10_UNR:%.*]] = phi i16 [ 0, [[BB:%.*]] ], [ [[I10_UNR_PH]], [[BB1_BB43_CRIT_EDGE_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: br i1 true, label [[BB5_PREHEADER_EPIL_PREHEADER:%.*]], label [[BB1_BB43_CRIT_EDGE:%.*]] +; CHECK-NEXT: [[I10_UNR:%.*]] = phi i16 [ [[I42_3]], [[BB38]] ] +; CHECK-NEXT: br i1 true, label [[BB5_PREHEADER_EPIL_PREHEADER]], label [[BB1_BB43_CRIT_EDGE:%.*]] ; CHECK: bb5.preheader.epil.preheader: +; CHECK-NEXT: [[I10_EPIL_INIT:%.*]] = phi i16 [ 0, [[BB:%.*]] ], [ [[I10_UNR]], [[BB1_BB43_CRIT_EDGE_UNR_LCSSA]] ] +; CHECK-NEXT: call void @llvm.assume(i1 true) ; CHECK-NEXT: br label [[BB5_PREHEADER_EPIL:%.*]] ; CHECK: bb5.preheader.epil: -; CHECK-NEXT: [[I10_EPIL:%.*]] = phi i16 [ [[I10_UNR]], [[BB5_PREHEADER_EPIL_PREHEADER]] ], [ [[I42_EPIL:%.*]], [[BB38_EPIL:%.*]] ] +; CHECK-NEXT: [[I10_EPIL:%.*]] = phi i16 [ [[I10_EPIL_INIT]], [[BB5_PREHEADER_EPIL_PREHEADER]] ], [ [[I42_EPIL:%.*]], [[BB38_EPIL:%.*]] ] ; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i16 [ 0, [[BB5_PREHEADER_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[BB38_EPIL]] ] ; CHECK-NEXT: br label [[BB10_PREHEADER_EPIL:%.*]] ; CHECK: bb10.preheader.epil: diff --git a/llvm/test/Transforms/LoopUnrollAndJam/followup.ll b/llvm/test/Transforms/LoopUnrollAndJam/followup.ll index 5186f7720e3f4..c8be48bfb0bbc 100644 --- a/llvm/test/Transforms/LoopUnrollAndJam/followup.ll +++ b/llvm/test/Transforms/LoopUnrollAndJam/followup.ll @@ -52,7 +52,7 @@ for.end: ; CHECK: br i1 %exitcond.3, label %for.latch, label %for.inner, !llvm.loop ![[LOOP_INNER:[0-9]+]] -; CHECK: br i1 %niter.ncmp.3, label %for.end.loopexit.unr-lcssa.loopexit, label %for.outer, !llvm.loop ![[LOOP_OUTER:[0-9]+]] +; CHECK: br i1 %niter.ncmp.3, label %for.end.loopexit.unr-lcssa, label %for.outer, !llvm.loop ![[LOOP_OUTER:[0-9]+]] ; CHECK: br i1 %exitcond.epil, label %for.latch.epil, label %for.inner.epil, !llvm.loop ![[LOOP_REMAINDER_INNER:[0-9]+]] ; CHECK: br i1 %exitcond.epil.1, label %for.latch.epil.1, label %for.inner.epil.1, !llvm.loop ![[LOOP_REMAINDER_INNER]] ; CHECK: br i1 %exitcond.epil.2, label %for.latch.epil.2, label %for.inner.epil.2, !llvm.loop ![[LOOP_REMAINDER_INNER]] diff --git a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll index 6f48c41a2ad06..9ee51cfbcb590 100644 --- a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll +++ b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll @@ -17,7 +17,7 @@ define void @test1(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[I]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[I]], 3 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 3 -; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA:.*]], label %[[FOR_OUTER_PREHEADER_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_OUTER_PREHEADER_NEW:.*]] ; CHECK: [[FOR_OUTER_PREHEADER_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[I]], [[XTRAITER]] ; CHECK-NEXT: br label %[[FOR_OUTER:.*]] @@ -71,15 +71,15 @@ define void @test1(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: [[ARRAYIDX6_3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[ADD8_2]] ; CHECK-NEXT: store i32 [[ADD_LCSSA_3]], ptr [[ARRAYIDX6_3]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i32 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP4:![0-9]+]] -; CHECK: [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[I_UNR_PH:%.*]] = phi i32 [ [[ADD8_3]], %[[FOR_LATCH]] ] -; CHECK-NEXT: br label %[[FOR_END_LOOPEXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: [[FOR_END_LOOPEXIT_UNR_LCSSA]]: -; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_OUTER_PREHEADER]] ], [ [[I_UNR_PH]], %[[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[I_UNR1:%.*]] = phi i32 [ [[ADD8_3]], %[[FOR_LATCH]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_END_LOOPEXIT:.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER]], label %[[FOR_END_LOOPEXIT:.*]] ; CHECK: [[FOR_OUTER_EPIL_PREHEADER]]: +; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_OUTER_PREHEADER]] ], [ [[I_UNR1]], %[[FOR_END_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label %[[FOR_OUTER_EPIL:.*]] ; CHECK: [[FOR_OUTER_EPIL]]: ; CHECK-NEXT: br label %[[FOR_INNER_EPIL:.*]] @@ -193,7 +193,7 @@ define void @test2(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[I]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[I]], 3 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 3 -; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_END10_LOOPEXIT_UNR_LCSSA:.*]], label %[[FOR_OUTER_PREHEADER_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_OUTER_PREHEADER_NEW:.*]] ; CHECK: [[FOR_OUTER_PREHEADER_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[I]], [[XTRAITER]] ; CHECK-NEXT: br label %[[FOR_OUTER:.*]] @@ -251,15 +251,15 @@ define void @test2(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: store i32 [[ADD_LCSSA_2]], ptr [[ARRAYIDX_2]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: store i32 [[ADD_LCSSA_3]], ptr [[ARRAYIDX_3]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i32 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_END10_LOOPEXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP6:![0-9]+]] -; CHECK: [[FOR_END10_LOOPEXIT_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[I_UNR_PH:%.*]] = phi i32 [ [[ADD9_3]], %[[FOR_LATCH]] ] -; CHECK-NEXT: br label %[[FOR_END10_LOOPEXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_END10_LOOPEXIT_UNR_LCSSA:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: [[FOR_END10_LOOPEXIT_UNR_LCSSA]]: -; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_OUTER_PREHEADER]] ], [ [[I_UNR_PH]], %[[FOR_END10_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[I_UNR1:%.*]] = phi i32 [ [[ADD9_3]], %[[FOR_LATCH]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_END10_LOOPEXIT:.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER]], label %[[FOR_END10_LOOPEXIT:.*]] ; CHECK: [[FOR_OUTER_EPIL_PREHEADER]]: +; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_OUTER_PREHEADER]] ], [ [[I_UNR1]], %[[FOR_END10_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label %[[FOR_OUTER_EPIL:.*]] ; CHECK: [[FOR_OUTER_EPIL]]: ; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[I_UNR]] @@ -615,7 +615,7 @@ define i32 @test6() #0 { ; CHECK-LABEL: define i32 @test6() { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[F_PROMOTED10:%.*]] = load i32, ptr @f, align 4, !tbaa [[INT_TBAA0]] -; CHECK-NEXT: br i1 false, label %[[FOR_END_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; CHECK-NEXT: br i1 false, label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; CHECK: [[ENTRY_NEW]]: ; CHECK-NEXT: br label %[[FOR_OUTER:.*]] ; CHECK: [[FOR_OUTER]]: @@ -636,18 +636,15 @@ define i32 @test6() #0 { ; CHECK-NEXT: [[EXITCOND_3:%.*]] = icmp ne i32 [[INC_3]], 7 ; CHECK-NEXT: br i1 [[EXITCOND_3]], label %[[FOR_INNER]], label %[[FOR_LATCH]] ; CHECK: [[FOR_LATCH]]: -; CHECK-NEXT: br i1 false, label %[[FOR_OUTER]], label %[[FOR_END_UNR_LCSSA_LOOPEXIT:.*]], !llvm.loop [[LOOP7:![0-9]+]] -; CHECK: [[FOR_END_UNR_LCSSA_LOOPEXIT]]: +; CHECK-NEXT: br i1 false, label %[[FOR_OUTER]], label %[[FOR_END_UNR_LCSSA:.*]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: [[FOR_END_UNR_LCSSA]]: ; CHECK-NEXT: [[DOTLCSSA_LCSSA_PH_PH:%.*]] = phi i32 [ 2, %[[FOR_LATCH]] ] ; CHECK-NEXT: [[INC_LCSSA_LCSSA_PH_PH:%.*]] = phi i32 [ 7, %[[FOR_LATCH]] ] ; CHECK-NEXT: [[P0_UNR_PH:%.*]] = phi i32 [ 2, %[[FOR_LATCH]] ] -; CHECK-NEXT: br label %[[FOR_END_UNR_LCSSA]] -; CHECK: [[FOR_END_UNR_LCSSA]]: -; CHECK-NEXT: [[DOTLCSSA_LCSSA_PH:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[DOTLCSSA_LCSSA_PH_PH]], %[[FOR_END_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[INC_LCSSA_LCSSA_PH:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[INC_LCSSA_LCSSA_PH_PH]], %[[FOR_END_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[P0_UNR:%.*]] = phi i32 [ [[F_PROMOTED10]], %[[ENTRY]] ], [ [[P0_UNR_PH]], %[[FOR_END_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: br i1 true, label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_END:.*]] +; CHECK-NEXT: br i1 true, label %[[FOR_OUTER_EPIL_PREHEADER]], label %[[FOR_END:.*]] ; CHECK: [[FOR_OUTER_EPIL_PREHEADER]]: +; CHECK-NEXT: [[P0_UNR:%.*]] = phi i32 [ [[F_PROMOTED10]], %[[ENTRY]] ], [ [[P0_UNR_PH]], %[[FOR_END_UNR_LCSSA]] ] +; CHECK-NEXT: call void @llvm.assume(i1 true) ; CHECK-NEXT: br label %[[FOR_OUTER_EPIL:.*]] ; CHECK: [[FOR_OUTER_EPIL]]: ; CHECK-NEXT: br label %[[FOR_INNER_EPIL:.*]] @@ -661,8 +658,8 @@ define i32 @test6() #0 { ; CHECK-NEXT: [[DOTLCSSA_EPIL:%.*]] = phi i32 [ [[P1_EPIL]], %[[FOR_INNER_EPIL]] ] ; CHECK-NEXT: br label %[[FOR_END]] ; CHECK: [[FOR_END]]: -; CHECK-NEXT: [[DOTLCSSA_LCSSA:%.*]] = phi i32 [ [[DOTLCSSA_LCSSA_PH]], %[[FOR_END_UNR_LCSSA]] ], [ [[DOTLCSSA_EPIL]], %[[FOR_LATCH_EPIL]] ] -; CHECK-NEXT: [[INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[INC_LCSSA_LCSSA_PH]], %[[FOR_END_UNR_LCSSA]] ], [ 7, %[[FOR_LATCH_EPIL]] ] +; CHECK-NEXT: [[DOTLCSSA_LCSSA:%.*]] = phi i32 [ [[DOTLCSSA_LCSSA_PH_PH]], %[[FOR_END_UNR_LCSSA]] ], [ [[DOTLCSSA_EPIL]], %[[FOR_LATCH_EPIL]] ] +; CHECK-NEXT: [[INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[INC_LCSSA_LCSSA_PH_PH]], %[[FOR_END_UNR_LCSSA]] ], [ 7, %[[FOR_LATCH_EPIL]] ] ; CHECK-NEXT: ret i32 0 ; entry: @@ -708,7 +705,7 @@ define void @test7(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[I]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[I]], 3 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 3 -; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA:.*]], label %[[FOR_PREHEADER_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_PREHEADER_NEW:.*]] ; CHECK: [[FOR_PREHEADER_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[I]], [[XTRAITER]] ; CHECK-NEXT: br label %[[FOR_OUTER:.*]] @@ -747,7 +744,7 @@ define void @test7(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: store i32 [[ADD9_LCSSA_2]], ptr [[ARRAYIDX_2]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: store i32 [[ADD9_LCSSA_3]], ptr [[ARRAYIDX_3]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i32 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: [[FOR_INNER]]: ; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[FOR_OUTER]] ], [ [[ADD9]], %[[FOR_INNER]] ] ; CHECK-NEXT: [[J:%.*]] = phi i32 [ 0, %[[FOR_OUTER]] ], [ [[ADD10:%.*]], %[[FOR_INNER]] ] @@ -775,14 +772,14 @@ define void @test7(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: [[ADD10_3]] = add nuw i32 [[J_3]], 1 ; CHECK-NEXT: [[EXITCOND_3:%.*]] = icmp eq i32 [[ADD10_3]], [[E]] ; CHECK-NEXT: br i1 [[EXITCOND_3]], label %[[FOR_LATCH]], label %[[FOR_INNER]] -; CHECK: [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[I_UNR_PH:%.*]] = phi i32 [ [[ADD_3]], %[[FOR_LATCH]] ] -; CHECK-NEXT: br label %[[FOR_END_LOOPEXIT_UNR_LCSSA]] ; CHECK: [[FOR_END_LOOPEXIT_UNR_LCSSA]]: -; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_PREHEADER]] ], [ [[I_UNR_PH]], %[[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[I_UNR1:%.*]] = phi i32 [ [[ADD_3]], %[[FOR_LATCH]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_END_LOOPEXIT:.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER]], label %[[FOR_END_LOOPEXIT:.*]] ; CHECK: [[FOR_OUTER_EPIL_PREHEADER]]: +; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_PREHEADER]] ], [ [[I_UNR1]], %[[FOR_END_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label %[[FOR_OUTER_EPIL:.*]] ; CHECK: [[FOR_OUTER_EPIL]]: ; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[I_UNR]] @@ -907,7 +904,7 @@ define void @test8(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: [[X_038:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_CLEANUP:.*]] ], [ 0, %[[FOR_PREHEADER]] ] ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[I]], 3 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 3 -; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_CLEANUP_UNR_LCSSA:.*]], label %[[FOR_OUTEST_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_OUTEST_NEW:.*]] ; CHECK: [[FOR_OUTEST_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[I]], [[XTRAITER]] ; CHECK-NEXT: br label %[[FOR_OUTER:.*]] @@ -922,10 +919,10 @@ define void @test8(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[ADD]] ; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX_1]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: [[ADD_1:%.*]] = add nuw nsw i32 [[I]], 2 -; CHECK-NEXT: [[ARRAYIDX6_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[ADD_1]] -; CHECK-NEXT: store i32 2, ptr [[ARRAYIDX6_1]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[ADD_1]] -; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX_2]], align 4, !tbaa [[INT_TBAA0]] +; CHECK-NEXT: store i32 2, ptr [[ARRAYIDX_2]], align 4, !tbaa [[INT_TBAA0]] +; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[ADD_1]] +; CHECK-NEXT: store i32 0, ptr [[ARRAYIDX_4]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: [[ADD_2:%.*]] = add nuw nsw i32 [[I]], 3 ; CHECK-NEXT: [[ARRAYIDX6_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[ADD_2]] ; CHECK-NEXT: store i32 2, ptr [[ARRAYIDX6_2]], align 4, !tbaa [[INT_TBAA0]] @@ -970,18 +967,18 @@ define void @test8(i32 %I, i32 %E, ptr noalias nocapture %A, ptr noalias nocaptu ; CHECK-NEXT: [[ADD9_LCSSA_3:%.*]] = phi i32 [ [[ADD9_3]], %[[FOR_INNER]] ] ; CHECK-NEXT: store i32 [[ADD9_LCSSA]], ptr [[ARRAYIDX]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: store i32 [[ADD9_LCSSA_1]], ptr [[ARRAYIDX_1]], align 4, !tbaa [[INT_TBAA0]] -; CHECK-NEXT: store i32 [[ADD9_LCSSA_2]], ptr [[ARRAYIDX_2]], align 4, !tbaa [[INT_TBAA0]] +; CHECK-NEXT: store i32 [[ADD9_LCSSA_2]], ptr [[ARRAYIDX_4]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: store i32 [[ADD9_LCSSA_3]], ptr [[ARRAYIDX_3]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i32 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_CLEANUP_UNR_LCSSA_LOOPEXIT:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP9:![0-9]+]] -; CHECK: [[FOR_CLEANUP_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[I_UNR_PH:%.*]] = phi i32 [ [[ADD_3]], %[[FOR_LATCH]] ] -; CHECK-NEXT: br label %[[FOR_CLEANUP_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_CLEANUP_UNR_LCSSA:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: [[FOR_CLEANUP_UNR_LCSSA]]: -; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_OUTEST]] ], [ [[I_UNR_PH]], %[[FOR_CLEANUP_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[I_UNR1:%.*]] = phi i32 [ [[ADD_3]], %[[FOR_LATCH]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_CLEANUP]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER]], label %[[FOR_CLEANUP]] ; CHECK: [[FOR_OUTER_EPIL_PREHEADER]]: +; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_OUTEST]] ], [ [[I_UNR1]], %[[FOR_CLEANUP_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label %[[FOR_OUTER_EPIL:.*]] ; CHECK: [[FOR_OUTER_EPIL]]: ; CHECK-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[I_UNR]] @@ -1116,7 +1113,7 @@ define void @test9(i32 %I, i32 %E, ptr nocapture %A, ptr nocapture readonly %B) ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[I]], -1 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[I]], 3 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 3 -; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA:.*]], label %[[FOR_OUTER_PREHEADER_NEW:.*]] +; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_OUTER_PREHEADER_NEW:.*]] ; CHECK: [[FOR_OUTER_PREHEADER_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[I]], [[XTRAITER]] ; CHECK-NEXT: br label %[[FOR_OUTER:.*]] @@ -1174,15 +1171,15 @@ define void @test9(i32 %I, i32 %E, ptr nocapture %A, ptr nocapture readonly %B) ; CHECK-NEXT: [[ARRAYIDX6_3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[ADD8_2]] ; CHECK-NEXT: store i32 [[ADD_LCSSA_3]], ptr [[ARRAYIDX6_3]], align 4, !tbaa [[INT_TBAA0]] ; CHECK-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i32 [[NITER_NEXT_3]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP12:![0-9]+]] -; CHECK: [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]]: -; CHECK-NEXT: [[I_UNR_PH:%.*]] = phi i32 [ [[ADD8_3]], %[[FOR_LATCH]] ] -; CHECK-NEXT: br label %[[FOR_END_LOOPEXIT_UNR_LCSSA]] +; CHECK-NEXT: br i1 [[NITER_NCMP_3]], label %[[FOR_END_LOOPEXIT_UNR_LCSSA:.*]], label %[[FOR_OUTER]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: [[FOR_END_LOOPEXIT_UNR_LCSSA]]: -; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_OUTER_PREHEADER]] ], [ [[I_UNR_PH]], %[[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[I_UNR1:%.*]] = phi i32 [ [[ADD8_3]], %[[FOR_LATCH]] ] ; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER:.*]], label %[[FOR_END_LOOPEXIT:.*]] +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[FOR_OUTER_EPIL_PREHEADER]], label %[[FOR_END_LOOPEXIT:.*]] ; CHECK: [[FOR_OUTER_EPIL_PREHEADER]]: +; CHECK-NEXT: [[I_UNR:%.*]] = phi i32 [ 0, %[[FOR_OUTER_PREHEADER]] ], [ [[I_UNR1]], %[[FOR_END_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]]) ; CHECK-NEXT: br label %[[FOR_OUTER_EPIL:.*]] ; CHECK: [[FOR_OUTER_EPIL]]: ; CHECK-NEXT: br label %[[FOR_INNER_EPIL:.*]] @@ -1293,11 +1290,11 @@ for.end: define signext i16 @test10(i32 %k) #0 { ; CHECK-LABEL: define signext i16 @test10( ; CHECK-SAME: i32 [[K:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr @c, align 1 ; CHECK-NEXT: [[TOBOOL9:%.*]] = icmp eq i8 [[TMP0]], 0 ; CHECK-NEXT: [[TOBOOL13:%.*]] = icmp ne i32 [[K]], 0 -; CHECK-NEXT: br i1 false, label %[[FOR_END26_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]] +; CHECK-NEXT: br i1 false, label %[[FOR_BODY_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]] ; CHECK: [[ENTRY_NEW]]: ; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_BODY]]: @@ -1325,18 +1322,14 @@ define signext i16 @test10(i32 %k) #0 { ; CHECK-NEXT: br i1 [[TOBOOL9]], label %[[FOR_BODY2_SPLIT_1:.*]], label %[[FOR_BODY2_SPLIT2_1:.*]] ; CHECK: [[FOR_INC24]]: ; CHECK-NEXT: [[STOREMERGE_4_LCSSA_3:%.*]] = phi i64 [ [[STOREMERGE_4_3:%.*]], %[[FOR_INC21_3]] ] -; CHECK-NEXT: br i1 false, label %[[FOR_BODY]], label %[[FOR_END26_UNR_LCSSA_LOOPEXIT:.*]], !llvm.loop [[LOOP13:![0-9]+]] -; CHECK: [[FOR_END26_UNR_LCSSA_LOOPEXIT]]: +; CHECK-NEXT: br i1 false, label %[[FOR_BODY]], label %[[FOR_END26_UNR_LCSSA:.*]], !llvm.loop [[LOOP13:![0-9]+]] +; CHECK: [[FOR_END26_UNR_LCSSA]]: ; CHECK-NEXT: [[DEC_LCSSA_LCSSA_PH_PH:%.*]] = phi i64 [ 0, %[[FOR_INC24]] ] ; CHECK-NEXT: [[STOREMERGE_4_LCSSA_LCSSA_PH_PH:%.*]] = phi i64 [ [[STOREMERGE_4_LCSSA_3]], %[[FOR_INC24]] ] ; CHECK-NEXT: [[STOREMERGE_5_LCSSA_LCSSA_PH_PH:%.*]] = phi i32 [ 0, %[[FOR_INC24]] ] -; CHECK-NEXT: br label %[[FOR_END26_UNR_LCSSA]] -; CHECK: [[FOR_END26_UNR_LCSSA]]: -; CHECK-NEXT: [[DEC_LCSSA_LCSSA_PH:%.*]] = phi i64 [ poison, %[[ENTRY]] ], [ [[DEC_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[STOREMERGE_4_LCSSA_LCSSA_PH:%.*]] = phi i64 [ poison, %[[ENTRY]] ], [ [[STOREMERGE_4_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: [[STOREMERGE_5_LCSSA_LCSSA_PH:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[STOREMERGE_5_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA_LOOPEXIT]] ] -; CHECK-NEXT: br i1 true, label %[[FOR_BODY_EPIL_PREHEADER:.*]], label %[[FOR_END26:.*]] +; CHECK-NEXT: br i1 true, label %[[FOR_BODY_EPIL_PREHEADER]], label %[[FOR_END26:.*]] ; CHECK: [[FOR_BODY_EPIL_PREHEADER]]: +; CHECK-NEXT: call void @llvm.assume(i1 true) ; CHECK-NEXT: br label %[[FOR_BODY_EPIL:.*]] ; CHECK: [[FOR_BODY_EPIL]]: ; CHECK-NEXT: br label %[[FOR_BODY2_EPIL:.*]] @@ -1360,9 +1353,9 @@ define signext i16 @test10(i32 %k) #0 { ; CHECK-NEXT: [[STOREMERGE_4_LCSSA_EPIL:%.*]] = phi i64 [ [[STOREMERGE_4_EPIL]], %[[FOR_INC21_EPIL]] ] ; CHECK-NEXT: br label %[[FOR_END26]] ; CHECK: [[FOR_END26]]: -; CHECK-NEXT: [[DEC_LCSSA_LCSSA:%.*]] = phi i64 [ [[DEC_LCSSA_LCSSA_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ 0, %[[FOR_INC24_EPIL]] ] -; CHECK-NEXT: [[STOREMERGE_4_LCSSA_LCSSA:%.*]] = phi i64 [ [[STOREMERGE_4_LCSSA_LCSSA_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ [[STOREMERGE_4_LCSSA_EPIL]], %[[FOR_INC24_EPIL]] ] -; CHECK-NEXT: [[STOREMERGE_5_LCSSA_LCSSA:%.*]] = phi i32 [ [[STOREMERGE_5_LCSSA_LCSSA_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ 0, %[[FOR_INC24_EPIL]] ] +; CHECK-NEXT: [[DEC_LCSSA_LCSSA:%.*]] = phi i64 [ [[DEC_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ 0, %[[FOR_INC24_EPIL]] ] +; CHECK-NEXT: [[STOREMERGE_4_LCSSA_LCSSA:%.*]] = phi i64 [ [[STOREMERGE_4_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ [[STOREMERGE_4_LCSSA_EPIL]], %[[FOR_INC24_EPIL]] ] +; CHECK-NEXT: [[STOREMERGE_5_LCSSA_LCSSA:%.*]] = phi i32 [ [[STOREMERGE_5_LCSSA_LCSSA_PH_PH]], %[[FOR_END26_UNR_LCSSA]] ], [ 0, %[[FOR_INC24_EPIL]] ] ; CHECK-NEXT: store i64 [[DEC_LCSSA_LCSSA]], ptr @g, align 8 ; CHECK-NEXT: ret i16 0 ; CHECK: [[FOR_BODY2_SPLIT2_1]]: diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll index 3b0ad73d91338..39217e51ab117 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll @@ -23,7 +23,7 @@ define void @fp_iv_loop1(ptr noalias nocapture %A, i32 %N) #0 { ; AUTO_VEC: [[ITER_CHECK]]: ; AUTO_VEC-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64 ; AUTO_VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4 -; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] +; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] ; AUTO_VEC: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: ; AUTO_VEC-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32 ; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] @@ -60,7 +60,7 @@ define void @fp_iv_loop1(ptr noalias nocapture %A, i32 %N) #0 { ; AUTO_VEC-NEXT: [[TMP11:%.*]] = fmul fast float 5.000000e-01, [[DOTCAST12]] ; AUTO_VEC-NEXT: [[IND_END1:%.*]] = fadd fast float 1.000000e+00, [[TMP11]] ; AUTO_VEC-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 -; AUTO_VEC-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[FOR_BODY]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]] +; AUTO_VEC-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]] ; AUTO_VEC: [[VEC_EPILOG_PH]]: ; AUTO_VEC-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] ; AUTO_VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi float [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] @@ -84,14 +84,14 @@ define void @fp_iv_loop1(ptr noalias nocapture %A, i32 %N) #0 { ; AUTO_VEC-NEXT: br i1 [[TMP9]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; AUTO_VEC: [[VEC_EPILOG_MIDDLE_BLOCK]]: ; AUTO_VEC-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]] -; AUTO_VEC-NEXT: br i1 [[CMP_N9]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]] -; AUTO_VEC: [[FOR_BODY]]: +; AUTO_VEC-NEXT: br i1 [[CMP_N9]], label %[[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] +; AUTO_VEC: [[VEC_EPILOG_SCALAR_PH]]: ; AUTO_VEC-NEXT: [[BC_RESUME_VAL10:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] ; AUTO_VEC-NEXT: [[BC_RESUME_VAL11:%.*]] = phi float [ [[TMP10]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, %[[ITER_CHECK]] ] ; AUTO_VEC-NEXT: br label %[[LOOP:.*]] ; AUTO_VEC: [[LOOP]]: -; AUTO_VEC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL10]], %[[FOR_BODY]] ] -; AUTO_VEC-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL11]], %[[FOR_BODY]] ] +; AUTO_VEC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL10]], %[[VEC_EPILOG_SCALAR_PH]] ] +; AUTO_VEC-NEXT: [[X_06:%.*]] = phi float [ [[CONV1:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL11]], %[[VEC_EPILOG_SCALAR_PH]] ] ; AUTO_VEC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]] ; AUTO_VEC-NEXT: store float [[X_06]], ptr [[ARRAYIDX]], align 4 ; AUTO_VEC-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01 @@ -144,19 +144,19 @@ define void @fp_iv_loop2(ptr noalias nocapture %A, i32 %N) { ; AUTO_VEC-SAME: ptr noalias captures(none) [[A:%.*]], i32 [[N:%.*]]) #[[ATTR0]] { ; AUTO_VEC-NEXT: [[ENTRY:.*:]] ; AUTO_VEC-NEXT: [[CMP4:%.*]] = icmp sgt i32 [[N]], 0 -; AUTO_VEC-NEXT: br i1 [[CMP4]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]] -; AUTO_VEC: [[FOR_BODY_PREHEADER]]: -; AUTO_VEC-NEXT: br label %[[FOR_BODY:.*]] -; AUTO_VEC: [[FOR_BODY]]: -; AUTO_VEC-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ] -; AUTO_VEC-NEXT: [[X_06_EPIL:%.*]] = phi float [ [[CONV1_EPIL:%.*]], %[[FOR_BODY]] ], [ 1.000000e+00, %[[FOR_BODY_PREHEADER]] ] +; AUTO_VEC-NEXT: br i1 [[CMP4]], label %[[LOOP_PREHEADER:.*]], label %[[FOR_END:.*]] +; AUTO_VEC: [[LOOP_PREHEADER]]: +; AUTO_VEC-NEXT: br label %[[LOOP:.*]] +; AUTO_VEC: [[LOOP]]: +; AUTO_VEC-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ] +; AUTO_VEC-NEXT: [[X_06_EPIL:%.*]] = phi float [ [[CONV1_EPIL:%.*]], %[[LOOP]] ], [ 1.000000e+00, %[[LOOP_PREHEADER]] ] ; AUTO_VEC-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV_EPIL]] ; AUTO_VEC-NEXT: store float [[X_06_EPIL]], ptr [[ARRAYIDX_EPIL]], align 4 ; AUTO_VEC-NEXT: [[CONV1_EPIL]] = fadd float [[X_06_EPIL]], 5.000000e-01 ; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1 ; AUTO_VEC-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT_EPIL]] to i32 ; AUTO_VEC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]] -; AUTO_VEC-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[FOR_BODY]] +; AUTO_VEC-NEXT: br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT:.*]], label %[[LOOP]] ; AUTO_VEC: [[FOR_END_LOOPEXIT]]: ; AUTO_VEC-NEXT: br label %[[FOR_END]] ; AUTO_VEC: [[FOR_END]]: @@ -193,7 +193,7 @@ define double @external_use_with_fast_math(ptr %a, i64 %n) { ; AUTO_VEC-NEXT: [[ENTRY:.*]]: ; AUTO_VEC-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) ; AUTO_VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 16 -; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]] +; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; AUTO_VEC: [[VECTOR_PH]]: ; AUTO_VEC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 16 ; AUTO_VEC-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]] @@ -222,14 +222,14 @@ define double @external_use_with_fast_math(ptr %a, i64 %n) { ; AUTO_VEC: [[MIDDLE_BLOCK]]: ; AUTO_VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] ; AUTO_VEC-NEXT: [[TMP7:%.*]] = fsub fast double [[TMP6]], 3.000000e+00 -; AUTO_VEC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[FOR_BODY]] -; AUTO_VEC: [[FOR_BODY]]: +; AUTO_VEC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] +; AUTO_VEC: [[SCALAR_PH]]: ; AUTO_VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] ; AUTO_VEC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi double [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ] ; AUTO_VEC-NEXT: br label %[[LOOP:.*]] ; AUTO_VEC: [[LOOP]]: -; AUTO_VEC-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[FOR_BODY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] -; AUTO_VEC-NEXT: [[J:%.*]] = phi double [ [[BC_RESUME_VAL1]], %[[FOR_BODY]] ], [ [[J_NEXT:%.*]], %[[LOOP]] ] +; AUTO_VEC-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; AUTO_VEC-NEXT: [[J:%.*]] = phi double [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[LOOP]] ] ; AUTO_VEC-NEXT: [[T0:%.*]] = getelementptr double, ptr [[A]], i64 [[I]] ; AUTO_VEC-NEXT: store double [[J]], ptr [[T0]], align 8 ; AUTO_VEC-NEXT: [[I_NEXT]] = add i64 [[I]], 1 @@ -261,19 +261,19 @@ for.end: define double @external_use_without_fast_math(ptr %a, i64 %n) { ; AUTO_VEC-LABEL: define double @external_use_without_fast_math( ; AUTO_VEC-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; AUTO_VEC-NEXT: [[ENTRY_NEW:.*]]: -; AUTO_VEC-NEXT: br label %[[FOR_BODY:.*]] -; AUTO_VEC: [[FOR_BODY]]: -; AUTO_VEC-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[I_NEXT_7:%.*]], %[[FOR_BODY]] ] -; AUTO_VEC-NEXT: [[J:%.*]] = phi double [ 0.000000e+00, %[[ENTRY_NEW]] ], [ [[J_NEXT_7:%.*]], %[[FOR_BODY]] ] +; AUTO_VEC-NEXT: [[ENTRY:.*]]: +; AUTO_VEC-NEXT: br label %[[LOOP:.*]] +; AUTO_VEC: [[LOOP]]: +; AUTO_VEC-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT_7:%.*]], %[[LOOP]] ] +; AUTO_VEC-NEXT: [[J:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[J_NEXT_7:%.*]], %[[LOOP]] ] ; AUTO_VEC-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[A]], i64 [[I]] ; AUTO_VEC-NEXT: store double [[J]], ptr [[TMP7]], align 8 ; AUTO_VEC-NEXT: [[I_NEXT_7]] = add i64 [[I]], 1 ; AUTO_VEC-NEXT: [[J_NEXT_7]] = fadd double [[J]], 3.000000e+00 ; AUTO_VEC-NEXT: [[COND:%.*]] = icmp slt i64 [[I_NEXT_7]], [[N]] -; AUTO_VEC-NEXT: br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END:.*]] +; AUTO_VEC-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[FOR_END:.*]] ; AUTO_VEC: [[FOR_END]]: -; AUTO_VEC-NEXT: [[J_LCSSA:%.*]] = phi double [ [[J]], %[[FOR_BODY]] ] +; AUTO_VEC-NEXT: [[J_LCSSA:%.*]] = phi double [ [[J]], %[[LOOP]] ] ; AUTO_VEC-NEXT: ret double [[J_LCSSA]] ; entry: @@ -308,7 +308,7 @@ define void @fadd_reassoc_FMF(ptr nocapture %p, i32 %N) { ; AUTO_VEC-NEXT: [[ITER_CHECK:.*]]: ; AUTO_VEC-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64 ; AUTO_VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4 -; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[FOR_BODY:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] +; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]] ; AUTO_VEC: [[VECTOR_MAIN_LOOP_ITER_CHECK]]: ; AUTO_VEC-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 32 ; AUTO_VEC-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]] @@ -353,7 +353,7 @@ define void @fadd_reassoc_FMF(ptr nocapture %p, i32 %N) { ; AUTO_VEC-NEXT: [[TMP12:%.*]] = fmul reassoc float 4.200000e+01, [[DOTCAST16]] ; AUTO_VEC-NEXT: [[IND_END1:%.*]] = fadd reassoc float 1.000000e+00, [[TMP12]] ; AUTO_VEC-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 -; AUTO_VEC-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[FOR_BODY]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]] +; AUTO_VEC-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]] ; AUTO_VEC: [[VEC_EPILOG_PH]]: ; AUTO_VEC-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] ; AUTO_VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi float [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] @@ -379,14 +379,14 @@ define void @fadd_reassoc_FMF(ptr nocapture %p, i32 %N) { ; AUTO_VEC-NEXT: br i1 [[TMP15]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] ; AUTO_VEC: [[VEC_EPILOG_MIDDLE_BLOCK]]: ; AUTO_VEC-NEXT: [[CMP_N18:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC6]] -; AUTO_VEC-NEXT: br i1 [[CMP_N18]], label %[[EXIT]], label %[[FOR_BODY]] -; AUTO_VEC: [[FOR_BODY]]: +; AUTO_VEC-NEXT: br i1 [[CMP_N18]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]] +; AUTO_VEC: [[VEC_EPILOG_SCALAR_PH]]: ; AUTO_VEC-NEXT: [[BC_RESUME_VAL14:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ] ; AUTO_VEC-NEXT: [[BC_RESUME_VAL15:%.*]] = phi float [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 1.000000e+00, %[[ITER_CHECK]] ] ; AUTO_VEC-NEXT: br label %[[LOOP:.*]] ; AUTO_VEC: [[LOOP]]: -; AUTO_VEC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL14]], %[[FOR_BODY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ] -; AUTO_VEC-NEXT: [[X_012:%.*]] = phi float [ [[BC_RESUME_VAL15]], %[[FOR_BODY]] ], [ [[ADD3:%.*]], %[[LOOP]] ] +; AUTO_VEC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL14]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ] +; AUTO_VEC-NEXT: [[X_012:%.*]] = phi float [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[ADD3:%.*]], %[[LOOP]] ] ; AUTO_VEC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[P]], i64 [[INDVARS_IV]] ; AUTO_VEC-NEXT: [[TMP16:%.*]] = load float, ptr [[ARRAYIDX]], align 4 ; AUTO_VEC-NEXT: [[ADD:%.*]] = fadd reassoc float [[X_012]], [[TMP16]] diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll index d8fc42bfaaec2..57dacd455482e 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/extra-unroll-simplifications.ll @@ -14,7 +14,7 @@ define void @partial_unroll_forced(i32 %N, ptr %src, ptr noalias %dst) { ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[N]], 1 -; CHECK-NEXT: br i1 [[TMP0]], label [[EXIT_LOOPEXIT_UNR_LCSSA:%.*]], label [[LOOP_LATCH_PREHEADER_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP0]], label [[LOOP_LATCH_EPIL_PREHEADER:%.*]], label [[LOOP_LATCH_PREHEADER_NEW:%.*]] ; CHECK: loop.latch.preheader.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483646 ; CHECK-NEXT: br label [[LOOP_LATCH:%.*]] @@ -35,12 +35,14 @@ define void @partial_unroll_forced(i32 %N, ptr %src, ptr noalias %dst) { ; CHECK-NEXT: [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_LOOPEXIT_UNR_LCSSA]], label [[LOOP_LATCH]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_LOOPEXIT_UNR_LCSSA:%.*]], label [[LOOP_LATCH]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: exit.loopexit.unr-lcssa: -; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[LOOP_LATCH_PREHEADER]] ], [ [[INDVARS_IV_NEXT_1]], [[LOOP_LATCH]] ] ; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[EXIT]], label [[LOOP_LATCH_EPIL:%.*]] -; CHECK: loop.latch.epil: +; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[EXIT]], label [[LOOP_LATCH_EPIL_PREHEADER]] +; CHECK: loop.latch.epil.preheader: +; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[LOOP_LATCH_PREHEADER]] ], [ [[INDVARS_IV_NEXT_1]], [[EXIT_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD4:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: tail call void @llvm.assume(i1 [[LCMP_MOD4]]) ; CHECK-NEXT: [[SRC_IDX_EPIL:%.*]] = getelementptr <8 x half>, ptr [[SRC]], i64 [[INDVARS_IV_UNR]] ; CHECK-NEXT: [[L_EPIL:%.*]] = load <8 x half>, ptr [[SRC_IDX_EPIL]], align 16 ; CHECK-NEXT: [[DST_IDX_EPIL:%.*]] = getelementptr <8 x half>, ptr [[DST]], i64 [[INDVARS_IV_UNR]] @@ -84,7 +86,7 @@ define void @cse_matching_load_from_previous_unrolled_iteration(i32 %N, ptr %src ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[N]] to i64 ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[N]], 1 -; CHECK-NEXT: br i1 [[TMP0]], label [[EXIT_LOOPEXIT_UNR_LCSSA:%.*]], label [[LOOP_LATCH_PREHEADER_NEW:%.*]] +; CHECK-NEXT: br i1 [[TMP0]], label [[LOOP_LATCH_EPIL_PREHEADER:%.*]], label [[LOOP_LATCH_PREHEADER_NEW:%.*]] ; CHECK: loop.latch.preheader.new: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 2147483646 ; CHECK-NEXT: br label [[LOOP_LATCH:%.*]] @@ -107,12 +109,14 @@ define void @cse_matching_load_from_previous_unrolled_iteration(i32 %N, ptr %src ; CHECK-NEXT: [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2 ; CHECK-NEXT: [[NITER_NEXT_1]] = add i64 [[NITER]], 2 ; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_LOOPEXIT_UNR_LCSSA]], label [[LOOP_LATCH]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[EXIT_LOOPEXIT_UNR_LCSSA:%.*]], label [[LOOP_LATCH]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: exit.loopexit.unr-lcssa: -; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[LOOP_LATCH_PREHEADER]] ], [ [[INDVARS_IV_NEXT_1]], [[LOOP_LATCH]] ] ; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 -; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[EXIT]], label [[LOOP_LATCH_EPIL:%.*]] -; CHECK: loop.latch.epil: +; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[EXIT]], label [[LOOP_LATCH_EPIL_PREHEADER]] +; CHECK: loop.latch.epil.preheader: +; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[LOOP_LATCH_PREHEADER]] ], [ [[INDVARS_IV_NEXT_1]], [[EXIT_LOOPEXIT_UNR_LCSSA]] ] +; CHECK-NEXT: [[LCMP_MOD4:%.*]] = icmp ne i64 [[XTRAITER]], 0 +; CHECK-NEXT: tail call void @llvm.assume(i1 [[LCMP_MOD4]]) ; CHECK-NEXT: [[GEP_SRC_12_EPIL:%.*]] = getelementptr <2 x i32>, ptr [[SRC_12]], i64 [[INDVARS_IV_UNR]] ; CHECK-NEXT: [[L_12_EPIL:%.*]] = load <2 x i32>, ptr [[GEP_SRC_12_EPIL]], align 8 ; CHECK-NEXT: [[GEP_SRC_4_EPIL:%.*]] = getelementptr <2 x i32>, ptr [[SRC_4]], i64 [[INDVARS_IV_UNR]]