[SimplifyCFG] Emit SelectInst when folding branches to common dest with different PHI incoming values

llvm/include/llvm/Transforms/Utils/LoopConstrainer.h

@@ -10,6 +10,7 @@
 #define LLVM_TRANSFORMS_UTILS_LOOP_CONSTRAINER_H
 
 #include "llvm/Support/Casting.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <optional>
 
@@ -190,6 +191,7 @@ class LoopConstrainer {
   Function &F;
   LLVMContext &Ctx;
   ScalarEvolution &SE;
+  TargetTransformInfo &TTI;
   DominatorTree &DT;
   LoopInfo &LI;
   function_ref<void(Loop *, bool)> LPMAddNewLoop;
@@ -216,7 +218,8 @@ class LoopConstrainer {
   LoopConstrainer(Loop &L, LoopInfo &LI,
                   function_ref<void(Loop *, bool)> LPMAddNewLoop,
                   const LoopStructure &LS, ScalarEvolution &SE,
-                  DominatorTree &DT, Type *T, SubRanges SR);
+                  TargetTransformInfo &TTI, DominatorTree &DT, Type *T,
+                  SubRanges SR);
 
   // Entry point for the algorithm.  Returns true on success.
   bool run();

llvm/include/llvm/Transforms/Utils/LoopPeel.h

@@ -31,8 +31,9 @@ bool canPeelLastIteration(const Loop &L, ScalarEvolution &SE);
 /// off the last \p PeelCount iterations from \p L (canPeelLastIteration must be
 /// true for \p L), otherwise peel off the first \p PeelCount iterations.
 bool peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
-              ScalarEvolution *SE, DominatorTree &DT, AssumptionCache *AC,
-              bool PreserveLCSSA, ValueToValueMapTy &VMap);
+              ScalarEvolution *SE, const TargetTransformInfo *TTI,
+              DominatorTree &DT, AssumptionCache *AC, bool PreserveLCSSA,
+              ValueToValueMapTy &VMap);
 
 TargetTransformInfo::PeelingPreferences
 gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,

llvm/include/llvm/Transforms/Utils/LoopSimplify.h

@@ -40,6 +40,7 @@
 
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Transforms/Utils/Local.h"
 
 namespace llvm {
 
@@ -64,7 +65,8 @@ class LoopSimplifyPass : public PassInfoMixin<LoopSimplifyPass> {
 /// analyses if they're non-null, and LCSSA if \c PreserveLCSSA is true.
 LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
                            ScalarEvolution *SE, AssumptionCache *AC,
-                           MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
+                           MemorySSAUpdater *MSSAU,
+                           const TargetTransformInfo *TTI, bool PreserveLCSSA);
 
 } // end namespace llvm
 

llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp

@@ -234,6 +234,7 @@ class InductiveRangeCheck {
 class InductiveRangeCheckElimination {
   ScalarEvolution &SE;
   BranchProbabilityInfo *BPI;
+  TargetTransformInfo &TTI;
   DominatorTree &DT;
   LoopInfo &LI;
 
@@ -248,9 +249,10 @@ class InductiveRangeCheckElimination {
 
 public:
   InductiveRangeCheckElimination(ScalarEvolution &SE,
-                                 BranchProbabilityInfo *BPI, DominatorTree &DT,
+                                 BranchProbabilityInfo *BPI,
+                                 TargetTransformInfo &TTI, DominatorTree &DT,
                                  LoopInfo &LI, GetBFIFunc GetBFI = std::nullopt)
-      : SE(SE), BPI(BPI), DT(DT), LI(LI), GetBFI(GetBFI) {}
+      : SE(SE), BPI(BPI), TTI(TTI), DT(DT), LI(LI), GetBFI(GetBFI) {}
 
   bool run(Loop *L, function_ref<void(Loop *, bool)> LPMAddNewLoop);
 };
@@ -900,6 +902,7 @@ IntersectUnsignedRange(ScalarEvolution &SE,
 
 PreservedAnalyses IRCEPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
   LoopInfo &LI = AM.getResult<LoopAnalysis>(F);
   // There are no loops in the function. Return before computing other expensive
   // analyses.
@@ -913,13 +916,13 @@ PreservedAnalyses IRCEPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto getBFI = [&F, &AM ]()->BlockFrequencyInfo & {
     return AM.getResult<BlockFrequencyAnalysis>(F);
   };
-  InductiveRangeCheckElimination IRCE(SE, &BPI, DT, LI, { getBFI });
+  InductiveRangeCheckElimination IRCE(SE, &BPI, TTI, DT, LI, {getBFI});
 
   bool Changed = false;
   {
     bool CFGChanged = false;
     for (const auto &L : LI) {
-      CFGChanged |= simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr,
+      CFGChanged |= simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr, &TTI,
                                  /*PreserveLCSSA=*/false);
       Changed |= formLCSSARecursively(*L, DT, &LI, &SE);
     }
@@ -1080,7 +1083,7 @@ bool InductiveRangeCheckElimination::run(
     return false;
   }
 
-  LoopConstrainer LC(*L, LI, LPMAddNewLoop, LS, SE, DT,
+  LoopConstrainer LC(*L, LI, LPMAddNewLoop, LS, SE, TTI, DT,
                      SafeIterRange->getBegin()->getType(), *MaybeSR);
 
   if (LC.run()) {

llvm/lib/Transforms/Scalar/LoopBoundSplit.cpp

@@ -285,7 +285,8 @@ static BranchInst *findSplitCandidate(const Loop &L, ScalarEvolution &SE,
 }
 
 static bool splitLoopBound(Loop &L, DominatorTree &DT, LoopInfo &LI,
-                           ScalarEvolution &SE, LPMUpdater &U) {
+                           ScalarEvolution &SE, LPMUpdater &U,
+                           const TargetTransformInfo *TTI) {
   ConditionInfo SplitCandidateCond;
   ConditionInfo ExitingCond;
 
@@ -460,8 +461,8 @@ static bool splitLoopBound(Loop &L, DominatorTree &DT, LoopInfo &LI,
   SE.forgetLoop(&L);
 
   // Canonicalize loops.
-  simplifyLoop(&L, &DT, &LI, &SE, nullptr, nullptr, true);
-  simplifyLoop(PostLoop, &DT, &LI, &SE, nullptr, nullptr, true);
+  simplifyLoop(&L, &DT, &LI, &SE, nullptr, nullptr, TTI, true);
+  simplifyLoop(PostLoop, &DT, &LI, &SE, nullptr, nullptr, TTI, true);
 
   // Add new post-loop to loop pass manager.
   U.addSiblingLoops(PostLoop);
@@ -478,7 +479,9 @@ PreservedAnalyses LoopBoundSplitPass::run(Loop &L, LoopAnalysisManager &AM,
   LLVM_DEBUG(dbgs() << "Spliting bound of loop in " << F.getName() << ": " << L
                     << "\n");
 
-  if (!splitLoopBound(L, AR.DT, AR.LI, AR.SE, U))
+  auto &FAM = AM.getResult<FunctionAnalysisManagerLoopProxy>(L, AR);
+  auto TTI = FAM.getCachedResult<TargetIRAnalysis>(F);
+  if (!splitLoopBound(L, AR.DT, AR.LI, AR.SE, U, TTI))
     return PreservedAnalyses::all();
 
   assert(AR.DT.verify(DominatorTree::VerificationLevel::Fast));

llvm/lib/Transforms/Scalar/LoopFuse.cpp

@@ -791,7 +791,7 @@ struct LoopFuser {
 
     ValueToValueMapTy VMap;
     FC0.Peeled =
-        peelLoop(FC0.L, PeelCount, false, &LI, &SE, DT, &AC, true, VMap);
+        peelLoop(FC0.L, PeelCount, false, &LI, &SE, &TTI, DT, &AC, true, VMap);
     if (FC0.Peeled) {
       LLVM_DEBUG(dbgs() << "Done Peeling\n");
 
@@ -2083,8 +2083,8 @@ PreservedAnalyses LoopFusePass::run(Function &F, FunctionAnalysisManager &AM) {
   // LoopSimplify pass as a dependency.
   bool Changed = false;
   for (auto &L : LI) {
-    Changed |=
-        simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, false /* PreserveLCSSA */);
+    Changed |= simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, &TTI,
+                            false /* PreserveLCSSA */);
   }
   if (Changed)
     PDT.recalculate(F);

llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp

@@ -640,12 +640,10 @@ class LoadEliminationForLoop {
 
 } // end anonymous namespace
 
-static bool eliminateLoadsAcrossLoops(Function &F, LoopInfo &LI,
-                                      DominatorTree &DT,
-                                      BlockFrequencyInfo *BFI,
-                                      ProfileSummaryInfo *PSI,
-                                      ScalarEvolution *SE, AssumptionCache *AC,
-                                      LoopAccessInfoManager &LAIs) {
+static bool eliminateLoadsAcrossLoops(
+    Function &F, LoopInfo &LI, DominatorTree &DT, BlockFrequencyInfo *BFI,
+    ProfileSummaryInfo *PSI, ScalarEvolution *SE, AssumptionCache *AC,
+    LoopAccessInfoManager &LAIs, const TargetTransformInfo *TTI) {
   // Build up a worklist of inner-loops to transform to avoid iterator
   // invalidation.
   // FIXME: This logic comes from other passes that actually change the loop
@@ -657,7 +655,8 @@ static bool eliminateLoadsAcrossLoops(Function &F, LoopInfo &LI,
 
   for (Loop *TopLevelLoop : LI)
     for (Loop *L : depth_first(TopLevelLoop)) {
-      Changed |= simplifyLoop(L, &DT, &LI, SE, AC, /*MSSAU*/ nullptr, false);
+      Changed |=
+          simplifyLoop(L, &DT, &LI, SE, AC, /*MSSAU*/ nullptr, TTI, false);
       // We only handle inner-most loops.
       if (L->isInnermost())
         Worklist.push_back(L);
@@ -692,8 +691,10 @@ PreservedAnalyses LoopLoadEliminationPass::run(Function &F,
   auto *BFI = (PSI && PSI->hasProfileSummary()) ?
       &AM.getResult<BlockFrequencyAnalysis>(F) : nullptr;
   LoopAccessInfoManager &LAIs = AM.getResult<LoopAccessAnalysis>(F);
+  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
 
-  bool Changed = eliminateLoadsAcrossLoops(F, LI, DT, BFI, PSI, &SE, &AC, LAIs);
+  bool Changed =
+      eliminateLoadsAcrossLoops(F, LI, DT, BFI, PSI, &SE, &AC, LAIs, &TTI);
 
   if (!Changed)
     return PreservedAnalyses::all();

llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp

@@ -1314,8 +1314,8 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
     });
 
     ValueToValueMapTy VMap;
-    if (peelLoop(L, PP.PeelCount, PP.PeelLast, LI, &SE, DT, &AC, PreserveLCSSA,
-                 VMap)) {
+    if (peelLoop(L, PP.PeelCount, PP.PeelLast, LI, &SE, &TTI, DT, &AC,
+                 PreserveLCSSA, VMap)) {
       simplifyLoopAfterUnroll(L, true, LI, &SE, &DT, &AC, &TTI, nullptr);
       // If the loop was peeled, we already "used up" the profile information
       // we had, so we don't want to unroll or peel again.
@@ -1623,8 +1623,8 @@ PreservedAnalyses LoopUnrollPass::run(Function &F,
   // will simplify all loops, regardless of whether anything end up being
   // unrolled.
   for (const auto &L : LI) {
-    Changed |=
-        simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, false /* PreserveLCSSA */);
+    Changed |= simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, &TTI,
+                            false /* PreserveLCSSA */);
     Changed |= formLCSSARecursively(*L, DT, &LI, &SE);
   }
 

llvm/lib/Transforms/Utils/LoopConstrainer.cpp

@@ -460,10 +460,11 @@ static void DisableAllLoopOptsOnLoop(Loop &L) {
 LoopConstrainer::LoopConstrainer(Loop &L, LoopInfo &LI,
                                  function_ref<void(Loop *, bool)> LPMAddNewLoop,
                                  const LoopStructure &LS, ScalarEvolution &SE,
-                                 DominatorTree &DT, Type *T, SubRanges SR)
+                                 TargetTransformInfo &TTI, DominatorTree &DT,
+                                 Type *T, SubRanges SR)
     : F(*L.getHeader()->getParent()), Ctx(L.getHeader()->getContext()), SE(SE),
-      DT(DT), LI(LI), LPMAddNewLoop(LPMAddNewLoop), OriginalLoop(L), RangeTy(T),
-      MainLoopStructure(LS), SR(SR) {}
+      TTI(TTI), DT(DT), LI(LI), LPMAddNewLoop(LPMAddNewLoop), OriginalLoop(L),
+      RangeTy(T), MainLoopStructure(LS), SR(SR) {}
 
 void LoopConstrainer::cloneLoop(LoopConstrainer::ClonedLoop &Result,
                                 const char *Tag) const {
@@ -872,7 +873,7 @@ bool LoopConstrainer::run() {
   // This function canonicalizes the loop into Loop-Simplify and LCSSA forms.
   auto CanonicalizeLoop = [&](Loop *L, bool IsOriginalLoop) {
     formLCSSARecursively(*L, DT, &LI, &SE);
-    simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr, true);
+    simplifyLoop(L, &DT, &LI, &SE, nullptr, nullptr, &TTI, true);
     // Pre/post loops are slow paths, we do not need to perform any loop
     // optimizations on them.
     if (!IsOriginalLoop)

llvm/lib/Transforms/Utils/LoopPeel.cpp

@@ -351,7 +351,7 @@ bool llvm::canPeelLastIteration(const Loop &L, ScalarEvolution &SE) {
                     m_BasicBlock(Succ1), m_BasicBlock(Succ2))) &&
          ((Pred == CmpInst::ICMP_EQ && Succ2 == L.getHeader()) ||
           (Pred == CmpInst::ICMP_NE && Succ1 == L.getHeader())) &&
-         Bound->getType()->isIntegerTy() && 
+         Bound->getType()->isIntegerTy() &&
          SE.isLoopInvariant(SE.getSCEV(Bound), &L) &&
          match(SE.getSCEV(Inc),
                m_scev_AffineAddRec(m_SCEV(), m_scev_One(), m_SpecificLoop(&L)));
@@ -1030,8 +1030,9 @@ llvm::gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
 /// for the bulk of dynamic execution, can be further simplified by scalar
 /// optimizations.
 bool llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
-                    ScalarEvolution *SE, DominatorTree &DT, AssumptionCache *AC,
-                    bool PreserveLCSSA, ValueToValueMapTy &LVMap) {
+                    ScalarEvolution *SE, const TargetTransformInfo *TTI,
+                    DominatorTree &DT, AssumptionCache *AC, bool PreserveLCSSA,
+                    ValueToValueMapTy &LVMap) {
   assert(PeelCount > 0 && "Attempt to peel out zero iterations?");
   assert(canPeel(L) && "Attempt to peel a loop which is not peelable?");
   assert((!PeelLast || (canPeelLastIteration(*L, *SE) && PeelCount == 1)) &&
@@ -1308,7 +1309,7 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, bool PeelLast, LoopInfo *LI,
 #endif
 
   // FIXME: Incrementally update loop-simplify
-  simplifyLoop(L, &DT, LI, SE, AC, nullptr, PreserveLCSSA);
+  simplifyLoop(L, &DT, LI, SE, AC, nullptr, TTI, PreserveLCSSA);
 
   NumPeeled++;
   NumPeeledEnd += PeelLast;

llvm/lib/Transforms/Utils/LoopSimplify.cpp

@@ -66,7 +66,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 using namespace llvm;
 
@@ -475,7 +474,9 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
 static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
                             DominatorTree *DT, LoopInfo *LI,
                             ScalarEvolution *SE, AssumptionCache *AC,
-                            MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+                            MemorySSAUpdater *MSSAU,
+                            const TargetTransformInfo *TTI,
+                            bool PreserveLCSSA) {
   bool Changed = false;
   if (MSSAU && VerifyMemorySSA)
     MSSAU->getMemorySSA()->verifyMemorySSA();
@@ -656,7 +657,7 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
       // The block has now been cleared of all instructions except for
       // a comparison and a conditional branch. SimplifyCFG may be able
       // to fold it now.
-      if (!foldBranchToCommonDest(BI, /*DTU=*/nullptr, MSSAU))
+      if (!foldBranchToCommonDest(BI, /*DTU=*/nullptr, MSSAU, TTI))
         continue;
 
       // Success. The block is now dead, so remove it from the loop,
@@ -696,7 +697,8 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
 
 bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
                         ScalarEvolution *SE, AssumptionCache *AC,
-                        MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+                        MemorySSAUpdater *MSSAU, const TargetTransformInfo *TTI,
+                        bool PreserveLCSSA) {
   bool Changed = false;
 
 #ifndef NDEBUG
@@ -724,7 +726,7 @@ bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
 
   while (!Worklist.empty())
     Changed |= simplifyOneLoop(Worklist.pop_back_val(), Worklist, DT, LI, SE,
-                               AC, MSSAU, PreserveLCSSA);
+                               AC, MSSAU, TTI, PreserveLCSSA);
 
   // Changing exit conditions for blocks may affect exit counts of this loop and
   // any of its parents, so we must invalidate the entire subtree if we've made
@@ -747,6 +749,7 @@ namespace {
 
     void getAnalysisUsage(AnalysisUsage &AU) const override {
       AU.addRequired<AssumptionCacheTracker>();
+      AU.addRequired<TargetTransformInfoWrapperPass>();
 
       // We need loop information to identify the loops...
       AU.addRequired<DominatorTreeWrapperPass>();
@@ -777,6 +780,7 @@ INITIALIZE_PASS_BEGIN(LoopSimplify, "loop-simplify",
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_END(LoopSimplify, "loop-simplify", "Canonicalize natural loops",
                     false, false)
 
@@ -791,6 +795,7 @@ bool LoopSimplify::runOnFunction(Function &F) {
   bool Changed = false;
   LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
   auto *SEWP = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
   ScalarEvolution *SE = SEWP ? &SEWP->getSE() : nullptr;
   AssumptionCache *AC =
@@ -807,7 +812,8 @@ bool LoopSimplify::runOnFunction(Function &F) {
 
   // Simplify each loop nest in the function.
   for (auto *L : *LI)
-    Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), PreserveLCSSA);
+    Changed |=
+        simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), &TTI, PreserveLCSSA);
 
 #ifndef NDEBUG
   if (PreserveLCSSA) {
@@ -832,13 +838,13 @@ PreservedAnalyses LoopSimplifyPass::run(Function &F,
     auto *MSSA = &MSSAAnalysis->getMSSA();
     MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
   }
-
+  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
 
   // Note that we don't preserve LCSSA in the new PM, if you need it run LCSSA
   // after simplifying the loops. MemorySSA is preserved if it exists.
   for (auto *L : *LI)
-    Changed |=
-        simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), /*PreserveLCSSA*/ false);
+    Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), &TTI,
+                            /*PreserveLCSSA*/ false);
 
   if (!Changed)
     return PreservedAnalyses::all();

llvm/lib/Transforms/Utils/LoopUnroll.cpp

@@ -1078,11 +1078,11 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
 
     // TODO: That potentially might be compile-time expensive. We should try
     // to fix the loop-simplified form incrementally.
-    simplifyLoop(OuterL, DT, LI, SE, AC, nullptr, PreserveLCSSA);
+    simplifyLoop(OuterL, DT, LI, SE, AC, nullptr, TTI, PreserveLCSSA);
   } else {
     // Simplify loops for which we might've broken loop-simplify form.
     for (Loop *SubLoop : LoopsToSimplify)
-      simplifyLoop(SubLoop, DT, LI, SE, AC, nullptr, PreserveLCSSA);
+      simplifyLoop(SubLoop, DT, LI, SE, AC, nullptr, TTI, PreserveLCSSA);
   }
 
   return CompletelyUnroll ? LoopUnrollResult::FullyUnrolled