[MachinePipeliner] Add an abstract layer to manipulate Data Dependenc… (#109918)

…e Graph

In MachinePipeliner, a DAG class is used to represent the Data
Dependence Graph. Data Dependence Graph generally contains cycles, so
it's not appropriate to use DAG classes. In fact, some "hacks" are used
to express back-edges in the current implementation. This patch adds a
new class to provide a better interface for manipulating dependencies.
Our approach is as follows:

- To build the graph, we use the ScheduleDAGInstrs class as it is,
because it has powerful functions and the current implementation depends
heavily on it.
- After the graph construction is finished (i.e., during scheduling), we
use the new class DataDependenceGraph to manipulate the dependencies.

Since we don't change the dependencies during scheduling, the new class
only provides functions to read them. Also, this patch is just a
refactoring, i.e., scheduling results should not change with or without
this patch.

Author: kasuga-fj

llvm/include/llvm/CodeGen/MachinePipeliner.h

@@ -40,6 +40,7 @@
 #ifndef LLVM_CODEGEN_MACHINEPIPELINER_H
 #define LLVM_CODEGEN_MACHINEPIPELINER_H
 
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/CodeGen/DFAPacketizer.h"
 #include "llvm/CodeGen/MachineDominators.h"
@@ -114,10 +115,123 @@ class MachinePipeliner : public MachineFunctionPass {
   bool useWindowScheduler(bool Changed);
 };
 
+/// Represents a dependence between two instruction.
+class SwingSchedulerDDGEdge {
+  SUnit *Dst = nullptr;
+  SDep Pred;
+  unsigned Distance = 0;
+
+public:
+  /// Creates an edge corresponding to an edge represented by \p PredOrSucc and
+  /// \p Dep in the original DAG. This pair has no information about the
+  /// direction of the edge, so we need to pass an additional argument \p
+  /// IsSucc.
+  SwingSchedulerDDGEdge(SUnit *PredOrSucc, const SDep &Dep, bool IsSucc)
+      : Dst(PredOrSucc), Pred(Dep), Distance(0u) {
+    SUnit *Src = Dep.getSUnit();
+
+    if (IsSucc) {
+      std::swap(Src, Dst);
+      Pred.setSUnit(Src);
+    }
+
+    // An anti-dependence to PHI means loop-carried dependence.
+    if (Pred.getKind() == SDep::Anti && Src->getInstr()->isPHI()) {
+      Distance = 1;
+      std::swap(Src, Dst);
+      auto Reg = Pred.getReg();
+      Pred = SDep(Src, SDep::Kind::Data, Reg);
+    }
+  }
+
+  /// Returns the SUnit from which the edge comes (source node).
+  SUnit *getSrc() const { return Pred.getSUnit(); }
+
+  /// Returns the SUnit to which the edge points (destination node).
+  SUnit *getDst() const { return Dst; }
+
+  /// Returns the latency value for the edge.
+  unsigned getLatency() const { return Pred.getLatency(); }
+
+  /// Sets the latency for the edge.
+  void setLatency(unsigned Latency) { Pred.setLatency(Latency); }
+
+  /// Returns the distance value for the edge.
+  unsigned getDistance() const { return Distance; }
+
+  /// Sets the distance value for the edge.
+  void setDistance(unsigned D) { Distance = D; }
+
+  /// Returns the register associated with the edge.
+  Register getReg() const { return Pred.getReg(); }
+
+  /// Returns true if the edge represents anti dependence.
+  bool isAntiDep() const { return Pred.getKind() == SDep::Kind::Anti; }
+
+  /// Returns true if the edge represents output dependence.
+  bool isOutputDep() const { return Pred.getKind() == SDep::Kind::Output; }
+
+  /// Returns true if the edge represents a dependence that is not data, anti or
+  /// output dependence.
+  bool isOrderDep() const { return Pred.getKind() == SDep::Kind::Order; }
+
+  /// Returns true if the edge represents unknown scheduling barrier.
+  bool isBarrier() const { return Pred.isBarrier(); }
+
+  /// Returns true if the edge represents an artificial dependence.
+  bool isArtificial() const { return Pred.isArtificial(); }
+
+  /// Tests if this is a Data dependence that is associated with a register.
+  bool isAssignedRegDep() const { return Pred.isAssignedRegDep(); }
+
+  /// Returns true for DDG nodes that we ignore when computing the cost
+  /// functions. We ignore the back-edge recurrence in order to avoid unbounded
+  /// recursion in the calculation of the ASAP, ALAP, etc functions.
+  bool ignoreDependence(bool IgnoreAnti) const;
+};
+
+/// Represents dependencies between instructions. This class is a wrapper of
+/// `SUnits` and its dependencies to manipulate back-edges in a natural way.
+/// Currently it only supports back-edges via PHI, which are expressed as
+/// anti-dependencies in the original DAG.
+/// FIXME: Support any other loop-carried dependencies
+class SwingSchedulerDDG {
+  using EdgesType = SmallVector<SwingSchedulerDDGEdge, 4>;
+
+  struct SwingSchedulerDDGEdges {
+    EdgesType Preds;
+    EdgesType Succs;
+  };
+
+  void initEdges(SUnit *SU);
+
+  SUnit *EntrySU;
+  SUnit *ExitSU;
+
+  std::vector<SwingSchedulerDDGEdges> EdgesVec;
+  SwingSchedulerDDGEdges EntrySUEdges;
+  SwingSchedulerDDGEdges ExitSUEdges;
+
+  void addEdge(const SUnit *SU, const SwingSchedulerDDGEdge &Edge);
+
+  SwingSchedulerDDGEdges &getEdges(const SUnit *SU);
+  const SwingSchedulerDDGEdges &getEdges(const SUnit *SU) const;
+
+public:
+  SwingSchedulerDDG(std::vector<SUnit> &SUnits, SUnit *EntrySU, SUnit *ExitSU);
+
+  const EdgesType &getInEdges(const SUnit *SU) const;
+
+  const EdgesType &getOutEdges(const SUnit *SU) const;
+};
+
 /// This class builds the dependence graph for the instructions in a loop,
 /// and attempts to schedule the instructions using the SMS algorithm.
 class SwingSchedulerDAG : public ScheduleDAGInstrs {
   MachinePipeliner &Pass;
+
+  std::unique_ptr<SwingSchedulerDDG> DDG;
+
   /// The minimum initiation interval between iterations for this schedule.
   unsigned MII = 0;
   /// The maximum initiation interval between iterations for this schedule.
@@ -130,7 +244,7 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
   unsigned II_setByPragma = 0;
   TargetInstrInfo::PipelinerLoopInfo *LoopPipelinerInfo = nullptr;
 
-  /// A toplogical ordering of the SUnits, which is needed for changing
+  /// A topological ordering of the SUnits, which is needed for changing
   /// dependences and iterating over the SUnits.
   ScheduleDAGTopologicalSort Topo;
 
@@ -252,27 +366,7 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
     return ScheduleInfo[Node->NodeNum].ZeroLatencyHeight;
   }
 
-  /// Return true if the dependence is a back-edge in the data dependence graph.
-  /// Since the DAG doesn't contain cycles, we represent a cycle in the graph
-  /// using an anti dependence from a Phi to an instruction.
-  bool isBackedge(SUnit *Source, const SDep &Dep) {
-    if (Dep.getKind() != SDep::Anti)
-      return false;
-    return Source->getInstr()->isPHI() || Dep.getSUnit()->getInstr()->isPHI();
-  }
-
-  bool isLoopCarriedDep(SUnit *Source, const SDep &Dep,
-                        bool isSucc = true) const;
-
-  /// The distance function, which indicates that operation V of iteration I
-  /// depends on operations U of iteration I-distance.
-  unsigned getDistance(SUnit *U, SUnit *V, const SDep &Dep) {
-    // Instructions that feed a Phi have a distance of 1. Computing larger
-    // values for arrays requires data dependence information.
-    if (V->getInstr()->isPHI() && Dep.getKind() == SDep::Anti)
-      return 1;
-    return 0;
-  }
+  bool isLoopCarriedDep(const SwingSchedulerDDGEdge &Edge) const;
 
   void applyInstrChange(MachineInstr *MI, SMSchedule &Schedule);
 
@@ -294,6 +388,8 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs {
 
   static bool classof(const ScheduleDAGInstrs *DAG) { return true; }
 
+  const SwingSchedulerDDG *getDDG() const { return DDG.get(); }
+
 private:
   void addLoopCarriedDependences(AAResults *AA);
   void updatePhiDependences();
@@ -357,15 +453,16 @@ class NodeSet {
     //
     // Hold a map from each SUnit in the circle to the maximum distance from the
     // source node by only considering the nodes.
+    const SwingSchedulerDDG *DDG = DAG->getDDG();
     DenseMap<SUnit *, unsigned> SUnitToDistance;
     for (auto *Node : Nodes)
       SUnitToDistance[Node] = 0;
 
     for (unsigned I = 1, E = Nodes.size(); I <= E; ++I) {
       SUnit *U = Nodes[I - 1];
       SUnit *V = Nodes[I % Nodes.size()];
-      for (const SDep &Succ : U->Succs) {
-        SUnit *SuccSUnit = Succ.getSUnit();
+      for (const SwingSchedulerDDGEdge &Succ : DDG->getOutEdges(U)) {
+        SUnit *SuccSUnit = Succ.getDst();
         if (V != SuccSUnit)
           continue;
         if (SUnitToDistance[U] + Succ.getLatency() > SUnitToDistance[V]) {
@@ -377,13 +474,13 @@ class NodeSet {
     SUnit *FirstNode = Nodes[0];
     SUnit *LastNode = Nodes[Nodes.size() - 1];
 
-    for (auto &PI : LastNode->Preds) {
+    for (auto &PI : DDG->getInEdges(LastNode)) {
       // If we have an order dep that is potentially loop carried then a
       // back-edge exists between the last node and the first node that isn't
       // modeled in the DAG. Handle it manually by adding 1 to the distance of
       // the last node.
-      if (PI.getSUnit() != FirstNode || PI.getKind() != SDep::Order ||
-          !DAG->isLoopCarriedDep(LastNode, PI, false))
+      if (PI.getSrc() != FirstNode || !PI.isOrderDep() ||
+          !DAG->isLoopCarriedDep(PI))
         continue;
       SUnitToDistance[FirstNode] =
           std::max(SUnitToDistance[FirstNode], SUnitToDistance[LastNode] + 1);
@@ -627,11 +724,13 @@ class SMSchedule {
 
   /// Return the cycle of the earliest scheduled instruction in the dependence
   /// chain.
-  int earliestCycleInChain(const SDep &Dep);
+  int earliestCycleInChain(const SwingSchedulerDDGEdge &Dep,
+                           const SwingSchedulerDDG *DDG);
 
   /// Return the cycle of the latest scheduled instruction in the dependence
   /// chain.
-  int latestCycleInChain(const SDep &Dep);
+  int latestCycleInChain(const SwingSchedulerDDGEdge &Dep,
+                         const SwingSchedulerDDG *DDG);
 
   void computeStart(SUnit *SU, int *MaxEarlyStart, int *MinLateStart, int II,
                     SwingSchedulerDAG *DAG);
@@ -694,7 +793,7 @@ class SMSchedule {
                              MachineOperand &MO) const;
 
   bool onlyHasLoopCarriedOutputOrOrderPreds(SUnit *SU,
-                                            SwingSchedulerDAG *DAG) const;
+                                            const SwingSchedulerDDG *DDG) const;
   void print(raw_ostream &os) const;
   void dump() const;
 };