[CodeGen][NewPM] Port MachineSink to NPM (#115434)

Targets can set the EnableSinkAndFold option in CGPassBuilderOptions for
the NPM pipeline in buildCodeGenPipeline(... &Opts, ...)

Author: optimisan

llvm/include/llvm/CodeGen/MachineSink.h

@@ -0,0 +1,30 @@
+//===- MachineSink.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINESINK_H
+#define LLVM_CODEGEN_MACHINESINK_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+
+class MachineSinkingPass : public PassInfoMixin<MachineSinkingPass> {
+  bool EnableSinkAndFold;
+
+public:
+  MachineSinkingPass(bool EnableSinkAndFold = false)
+      : EnableSinkAndFold(EnableSinkAndFold) {}
+
+  PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &);
+
+  void printPipeline(raw_ostream &OS,
+                     function_ref<StringRef(StringRef)> MapClassName2PassName);
+};
+
+} // namespace llvm
+#endif // LLVM_CODEGEN_MACHINESINK_H

llvm/include/llvm/CodeGen/Passes.h

@@ -353,7 +353,7 @@ namespace llvm {
   extern char &EarlyMachineLICMID;
 
   /// MachineSinking - This pass performs sinking on machine instructions.
-  extern char &MachineSinkingID;
+  extern char &MachineSinkingLegacyID;
 
   /// MachineCopyPropagation - This pass performs copy propagation on
   /// machine instructions.

llvm/include/llvm/InitializePasses.h

@@ -208,7 +208,7 @@ void initializeMachinePostDominatorTreeWrapperPassPass(PassRegistry &);
 void initializeMachineRegionInfoPassPass(PassRegistry &);
 void initializeMachineSanitizerBinaryMetadataPass(PassRegistry &);
 void initializeMachineSchedulerLegacyPass(PassRegistry &);
-void initializeMachineSinkingPass(PassRegistry &);
+void initializeMachineSinkingLegacyPass(PassRegistry &);
 void initializeMachineTraceMetricsWrapperPassPass(PassRegistry &);
 void initializeMachineUniformityInfoPrinterPassPass(PassRegistry &);
 void initializeMachineUniformityAnalysisPassPass(PassRegistry &);

llvm/include/llvm/Passes/CodeGenPassBuilder.h

@@ -52,6 +52,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/MachineScheduler.h"
+#include "llvm/CodeGen/MachineSink.h"
 #include "llvm/CodeGen/MachineVerifier.h"
 #include "llvm/CodeGen/OptimizePHIs.h"
 #include "llvm/CodeGen/PHIElimination.h"
@@ -1044,7 +1045,7 @@ void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineSSAOptimization(
   addPass(EarlyMachineLICMPass());
   addPass(MachineCSEPass());
 
-  addPass(MachineSinkingPass());
+  addPass(MachineSinkingPass(Opt.EnableSinkAndFold));
 
   addPass(PeepholeOptimizerPass());
   // Clean-up the dead code that may have been generated by peephole

llvm/include/llvm/Passes/MachinePassRegistry.def

@@ -190,6 +190,13 @@ MACHINE_FUNCTION_PASS("verify<machine-trace-metrics>", MachineTraceMetricsVerifi
 #define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER,    \
                                           PARAMS)
 #endif
+MACHINE_FUNCTION_PASS_WITH_PARAMS(
+    "machine-sink", "MachineSinkingPass",
+    [](bool EnableSinkAndFold) {
+      return MachineSinkingPass(EnableSinkAndFold);
+    },
+    parseMachineSinkingPassOptions, "enable-sink-fold")
+
 MACHINE_FUNCTION_PASS_WITH_PARAMS(
     "regallocfast", "RegAllocFastPass",
     [](RegAllocFastPass::Options Opts) { return RegAllocFastPass(Opts); },
@@ -258,7 +265,6 @@ DUMMY_MACHINE_FUNCTION_PASS("machine-combiner", MachineCombinerPass)
 DUMMY_MACHINE_FUNCTION_PASS("static-data-splitter", StaticDataSplitter)
 DUMMY_MACHINE_FUNCTION_PASS("machine-function-splitter", MachineFunctionSplitterPass)
 DUMMY_MACHINE_FUNCTION_PASS("machine-sanmd", MachineSanitizerBinaryMetadata)
-DUMMY_MACHINE_FUNCTION_PASS("machine-sink", MachineSinkingPass)
 DUMMY_MACHINE_FUNCTION_PASS("machine-uniformity", MachineUniformityInfoWrapperPass)
 DUMMY_MACHINE_FUNCTION_PASS("machineinstr-printer", MachineFunctionPrinterPass)
 DUMMY_MACHINE_FUNCTION_PASS("mirfs-discriminators", MIRAddFSDiscriminatorsPass)

llvm/include/llvm/Target/CGPassBuilderOption.h

@@ -49,6 +49,7 @@ struct CGPassBuilderOption {
   bool EnableBlockPlacementStats = false;
   bool EnableGlobalMergeFunc = false;
   bool EnableMachineFunctionSplitter = false;
+  bool EnableSinkAndFold = false;
   bool MISchedPostRA = false;
   bool EarlyLiveIntervals = false;
   bool GCEmptyBlocks = false;

llvm/lib/CodeGen/CodeGen.cpp

@@ -95,7 +95,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeMachinePostDominatorTreeWrapperPassPass(Registry);
   initializeMachineRegionInfoPassPass(Registry);
   initializeMachineSchedulerLegacyPass(Registry);
-  initializeMachineSinkingPass(Registry);
+  initializeMachineSinkingLegacyPass(Registry);
   initializeMachineUniformityAnalysisPassPass(Registry);
   initializeMachineUniformityInfoPrinterPassPass(Registry);
   initializeMachineVerifierLegacyPassPass(Registry);

llvm/lib/CodeGen/MachineSink.cpp

@@ -15,6 +15,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/MachineSink.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/MapVector.h"
@@ -26,6 +27,8 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
@@ -42,6 +45,7 @@
 #include "llvm/CodeGen/MachineSizeOpts.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/RegisterPressure.h"
+#include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -118,7 +122,7 @@ using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
 
 namespace {
 
-class MachineSinking : public MachineFunctionPass {
+class MachineSinking {
   const TargetSubtargetInfo *STI = nullptr;
   const TargetInstrInfo *TII = nullptr;
   const TargetRegisterInfo *TRI = nullptr;
@@ -132,6 +136,11 @@ class MachineSinking : public MachineFunctionPass {
   AliasAnalysis *AA = nullptr;
   RegisterClassInfo RegClassInfo;
   TargetSchedModel SchedModel;
+  // Required for split critical edge
+  LiveIntervals *LIS;
+  SlotIndexes *SI;
+  LiveVariables *LV;
+  MachineLoopInfo *MLI;
 
   // Remember which edges have been considered for breaking.
   SmallSet<std::pair<MachineBasicBlock *, MachineBasicBlock *>, 8>
@@ -189,30 +198,19 @@ class MachineSinking : public MachineFunctionPass {
   bool EnableSinkAndFold;
 
 public:
-  static char ID; // Pass identification
-
-  MachineSinking() : MachineFunctionPass(ID) {
-    initializeMachineSinkingPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    MachineFunctionPass::getAnalysisUsage(AU);
-    AU.addRequired<AAResultsWrapperPass>();
-    AU.addRequired<MachineDominatorTreeWrapperPass>();
-    AU.addRequired<MachinePostDominatorTreeWrapperPass>();
-    AU.addRequired<MachineCycleInfoWrapperPass>();
-    AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
-    AU.addPreserved<MachineCycleInfoWrapperPass>();
-    AU.addPreserved<MachineLoopInfoWrapperPass>();
-    AU.addRequired<ProfileSummaryInfoWrapperPass>();
-    if (UseBlockFreqInfo)
-      AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
-    AU.addRequired<TargetPassConfig>();
-  }
-
-  void releaseMemory() override {
+  MachineSinking(bool EnableSinkAndFold, MachineDominatorTree *DT,
+                 MachinePostDominatorTree *PDT, LiveVariables *LV,
+                 MachineLoopInfo *MLI, SlotIndexes *SI, LiveIntervals *LIS,
+                 MachineCycleInfo *CI, ProfileSummaryInfo *PSI,
+                 MachineBlockFrequencyInfo *MBFI,
+                 const MachineBranchProbabilityInfo *MBPI, AliasAnalysis *AA)
+      : DT(DT), PDT(PDT), CI(CI), PSI(PSI), MBFI(MBFI), MBPI(MBPI), AA(AA),
+        LIS(LIS), SI(SI), LV(LV), MLI(MLI),
+        EnableSinkAndFold(EnableSinkAndFold) {}
+
+  bool run(MachineFunction &MF);
+
+  void releaseMemory() {
     CEBCandidates.clear();
     CEMergeCandidates.clear();
   }
@@ -290,21 +288,47 @@ class MachineSinking : public MachineFunctionPass {
   bool registerPressureExceedsLimit(const MachineBasicBlock &MBB);
 };
 
+class MachineSinkingLegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  MachineSinkingLegacy() : MachineFunctionPass(ID) {
+    initializeMachineSinkingLegacyPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    MachineFunctionPass::getAnalysisUsage(AU);
+    AU.addRequired<AAResultsWrapperPass>();
+    AU.addRequired<MachineDominatorTreeWrapperPass>();
+    AU.addRequired<MachinePostDominatorTreeWrapperPass>();
+    AU.addRequired<MachineCycleInfoWrapperPass>();
+    AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
+    AU.addPreserved<MachineCycleInfoWrapperPass>();
+    AU.addPreserved<MachineLoopInfoWrapperPass>();
+    AU.addRequired<ProfileSummaryInfoWrapperPass>();
+    if (UseBlockFreqInfo)
+      AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
+    AU.addRequired<TargetPassConfig>();
+  }
+};
+
 } // end anonymous namespace
 
-char MachineSinking::ID = 0;
+char MachineSinkingLegacy::ID = 0;
 
-char &llvm::MachineSinkingID = MachineSinking::ID;
+char &llvm::MachineSinkingLegacyID = MachineSinkingLegacy::ID;
 
-INITIALIZE_PASS_BEGIN(MachineSinking, DEBUG_TYPE, "Machine code sinking", false,
-                      false)
+INITIALIZE_PASS_BEGIN(MachineSinkingLegacy, DEBUG_TYPE, "Machine code sinking",
+                      false, false)
 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
-INITIALIZE_PASS_END(MachineSinking, DEBUG_TYPE, "Machine code sinking", false,
-                    false)
+INITIALIZE_PASS_END(MachineSinkingLegacy, DEBUG_TYPE, "Machine code sinking",
+                    false, false)
 
 /// Return true if a target defined block prologue instruction interferes
 /// with a sink candidate.
@@ -728,28 +752,87 @@ void MachineSinking::FindCycleSinkCandidates(
   }
 }
 
-bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
+PreservedAnalyses
+MachineSinkingPass::run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM) {
+  auto *DT = &MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
+  auto *PDT = &MFAM.getResult<MachinePostDominatorTreeAnalysis>(MF);
+  auto *CI = &MFAM.getResult<MachineCycleAnalysis>(MF);
+  auto *PSI = MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(MF)
+                  .getCachedResult<ProfileSummaryAnalysis>(
+                      *MF.getFunction().getParent());
+  auto *MBFI = UseBlockFreqInfo
+                   ? &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF)
+                   : nullptr;
+  auto *MBPI = &MFAM.getResult<MachineBranchProbabilityAnalysis>(MF);
+  auto *AA = &MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF)
+                  .getManager()
+                  .getResult<AAManager>(MF.getFunction());
+  auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF);
+  auto *SI = MFAM.getCachedResult<SlotIndexesAnalysis>(MF);
+  auto *LV = MFAM.getCachedResult<LiveVariablesAnalysis>(MF);
+  auto *MLI = MFAM.getCachedResult<MachineLoopAnalysis>(MF);
+  MachineSinking Impl(EnableSinkAndFold, DT, PDT, LV, MLI, SI, LIS, CI, PSI,
+                      MBFI, MBPI, AA);
+  bool Changed = Impl.run(MF);
+  if (!Changed)
+    return PreservedAnalyses::all();
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserve<MachineCycleAnalysis>();
+  PA.preserve<MachineLoopAnalysis>();
+  return PA;
+}
+
+void MachineSinkingPass::printPipeline(
+    raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
+  OS << MapClassName2PassName(name()); // ideally machine-sink
+  if (EnableSinkAndFold)
+    OS << "<enable-sink-fold>";
+}
+
+bool MachineSinkingLegacy::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
 
+  TargetPassConfig *PassConfig = &getAnalysis<TargetPassConfig>();
+  bool EnableSinkAndFold = PassConfig->getEnableSinkAndFold();
+
+  auto *DT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+  auto *PDT =
+      &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
+  auto *CI = &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo();
+  auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
+  auto *MBFI =
+      UseBlockFreqInfo
+          ? &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI()
+          : nullptr;
+  auto *MBPI =
+      &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
+  auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+  // Get analyses for split critical edge.
+  auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
+  auto *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
+  auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
+  auto *SI = SIWrapper ? &SIWrapper->getSI() : nullptr;
+  auto *LVWrapper = getAnalysisIfAvailable<LiveVariablesWrapperPass>();
+  auto *LV = LVWrapper ? &LVWrapper->getLV() : nullptr;
+  auto *MLIWrapper = getAnalysisIfAvailable<MachineLoopInfoWrapperPass>();
+  auto *MLI = MLIWrapper ? &MLIWrapper->getLI() : nullptr;
+
+  MachineSinking Impl(EnableSinkAndFold, DT, PDT, LV, MLI, SI, LIS, CI, PSI,
+                      MBFI, MBPI, AA);
+  return Impl.run(MF);
+}
+
+bool MachineSinking::run(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "******** Machine Sinking ********\n");
 
   STI = &MF.getSubtarget();
   TII = STI->getInstrInfo();
   TRI = STI->getRegisterInfo();
   MRI = &MF.getRegInfo();
-  DT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
-  PDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
-  CI = &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo();
-  PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
-  MBFI = UseBlockFreqInfo
-             ? &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI()
-             : nullptr;
-  MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
-  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+
   RegClassInfo.runOnMachineFunction(MF);
-  TargetPassConfig *PassConfig = &getAnalysis<TargetPassConfig>();
-  EnableSinkAndFold = PassConfig->getEnableSinkAndFold();
 
   bool EverMadeChange = false;
 
@@ -767,8 +850,8 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
     MachineDomTreeUpdater MDTU(DT, PDT,
                                MachineDomTreeUpdater::UpdateStrategy::Lazy);
     for (const auto &Pair : ToSplit) {
-      auto NewSucc =
-          Pair.first->SplitCriticalEdge(Pair.second, *this, nullptr, &MDTU);
+      auto NewSucc = Pair.first->SplitCriticalEdge(
+          Pair.second, {LIS, SI, LV, MLI}, nullptr, &MDTU);
       if (NewSucc != nullptr) {
         LLVM_DEBUG(dbgs() << " *** Splitting critical edge: "
                           << printMBBReference(*Pair.first) << " -- "
@@ -858,6 +941,7 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
     MRI->clearKillFlags(I);
   RegsToClearKillFlags.clear();
 
+  releaseMemory();
   return EverMadeChange;
 }
 

llvm/lib/CodeGen/TargetPassConfig.cpp

@@ -324,7 +324,7 @@ static IdentifyingPassPtr overridePass(AnalysisID StandardID,
   if (StandardID == &MachineLICMID)
     return applyDisable(TargetID, DisablePostRAMachineLICM);
 
-  if (StandardID == &MachineSinkingID)
+  if (StandardID == &MachineSinkingLegacyID)
     return applyDisable(TargetID, DisableMachineSink);
 
   if (StandardID == &PostRAMachineSinkingID)
@@ -1316,7 +1316,7 @@ void TargetPassConfig::addMachineSSAOptimization() {
   addPass(&EarlyMachineLICMID);
   addPass(&MachineCSELegacyID);
 
-  addPass(&MachineSinkingID);
+  addPass(&MachineSinkingLegacyID);
 
   addPass(&PeepholeOptimizerLegacyID);
   // Clean-up the dead code that may have been generated by peephole