Automerge: [DependenceAnalysis] Extending SIV to handle fusable loops (#128782)

When there is a dependency between two memory instructions in separate loops that have the same iteration space and depth, SIV will be able to test them and compute the direction and the distance of the dependency.

Author: 1997alireza

llvm/include/llvm/Analysis/DependenceAnalysis.h

@@ -82,6 +82,16 @@ class LLVM_ABI Dependence {
   /// Dependence::DVEntry - Each level in the distance/direction vector
   /// has a direction (or perhaps a union of several directions), and
   /// perhaps a distance.
+  /// The dependency information could be across a single loop level or across
+  /// two separate levels that have the same trip count and nesting depth,
+  /// which helps to provide information for loop fusion candidation.
+  /// For example, loops b and c have the same iteration count and depth:
+  ///    for (a = ...) {
+  ///      for (b = 0; b < 10; b++) {
+  ///      }
+  ///      for (c = 0; c < 10; c++) {
+  ///      }
+  ///    }
   struct DVEntry {
     enum : unsigned char {
       NONE = 0,
@@ -144,12 +154,25 @@ class LLVM_ABI Dependence {
   /// source and destination of the dependence.
   virtual unsigned getLevels() const { return 0; }
 
-  /// getDirection - Returns the direction associated with a particular level.
-  virtual unsigned getDirection(unsigned Level) const { return DVEntry::ALL; }
+  /// getSameSDLevels - Returns the number of separate SameSD loops surrounding
+  /// the source and destination of the dependence.
+  virtual unsigned getSameSDLevels() const { return 0; }
 
-  /// getDistance - Returns the distance (or NULL) associated with a particular
-  /// level.
-  virtual const SCEV *getDistance(unsigned Level) const { return nullptr; }
+  /// getDVEntry - Returns the DV entry associated with a regular or a
+  /// SameSD level
+  DVEntry getDVEntry(unsigned Level, bool isSameSD) const;
+
+  /// getDirection - Returns the direction associated with a particular
+  /// common or SameSD level.
+  virtual unsigned getDirection(unsigned Level, bool SameSD = false) const {
+    return DVEntry::ALL;
+  }
+
+  /// getDistance - Returns the distance (or NULL) associated with a
+  /// particular common or SameSD level.
+  virtual const SCEV *getDistance(unsigned Level, bool SameSD = false) const {
+    return nullptr;
+  }
 
   /// Check if the direction vector is negative. A negative direction
   /// vector means Src and Dst are reversed in the actual program.
@@ -162,21 +185,32 @@ class LLVM_ABI Dependence {
   virtual bool normalize(ScalarEvolution *SE) { return false; }
 
   /// isPeelFirst - Returns true if peeling the first iteration from
-  /// this loop will break this dependence.
-  virtual bool isPeelFirst(unsigned Level) const { return false; }
+  /// this regular or SameSD loop level will break this dependence.
+  virtual bool isPeelFirst(unsigned Level, bool SameSD = false) const {
+    return false;
+  }
 
   /// isPeelLast - Returns true if peeling the last iteration from
-  /// this loop will break this dependence.
-  virtual bool isPeelLast(unsigned Level) const { return false; }
+  /// this regular or SameSD loop level will break this dependence.
+  virtual bool isPeelLast(unsigned Level, bool SameSD = false) const {
+    return false;
+  }
 
-  /// isSplitable - Returns true if splitting this loop will break the
-  /// dependence.
-  virtual bool isSplitable(unsigned Level) const { return false; }
+  /// isSplitable - Returns true if splitting the loop will break
+  /// the dependence.
+  virtual bool isSplitable(unsigned Level, bool SameSD = false) const {
+    return false;
+  }
 
-  /// isScalar - Returns true if a particular level is scalar; that is,
-  /// if no subscript in the source or destination mention the induction
-  /// variable associated with the loop at this level.
-  virtual bool isScalar(unsigned Level) const;
+  /// inSameSDLoops - Returns true if this level is an SameSD level, i.e.,
+  /// performed across two separate loop nests that have the Same Iteration and
+  /// Depth.
+  virtual bool inSameSDLoops(unsigned Level) const { return false; }
+
+  /// isScalar - Returns true if a particular regular or SameSD level is
+  /// scalar; that is, if no subscript in the source or destination mention
+  /// the induction variable associated with the loop at this level.
+  virtual bool isScalar(unsigned Level, bool SameSD = false) const;
 
   /// getNextPredecessor - Returns the value of the NextPredecessor field.
   const Dependence *getNextPredecessor() const { return NextPredecessor; }
@@ -198,6 +232,10 @@ class LLVM_ABI Dependence {
   /// dump - For debugging purposes, dumps a dependence to OS.
   void dump(raw_ostream &OS) const;
 
+  /// dumpImp - For debugging purposes. Dumps a dependence to OS with or
+  /// without considering the SameSD levels.
+  void dumpImp(raw_ostream &OS, bool SameSD = false) const;
+
 protected:
   Instruction *Src, *Dst;
 
@@ -238,13 +276,30 @@ class LLVM_ABI FullDependence final : public Dependence {
   /// source and destination of the dependence.
   unsigned getLevels() const override { return Levels; }
 
+  /// getSameSDLevels - Returns the number of separate SameSD loops surrounding
+  /// the source and destination of the dependence.
+  unsigned getSameSDLevels() const override { return SameSDLevels; }
+
+  /// getDVEntry - Returns the DV entry associated with a regular or a
+  /// SameSD level.
+  DVEntry getDVEntry(unsigned Level, bool isSameSD) const {
+    if (!isSameSD) {
+      assert(0 < Level && Level <= Levels && "Level out of range");
+      return DV[Level - 1];
+    } else {
+      assert(Levels < Level && Level <= Levels + SameSDLevels &&
+             "isSameSD level out of range");
+      return DVSameSD[Level - Levels - 1];
+    }
+  }
+
   /// getDirection - Returns the direction associated with a particular
-  /// level.
-  unsigned getDirection(unsigned Level) const override;
+  /// common or SameSD level.
+  unsigned getDirection(unsigned Level, bool SameSD = false) const override;
 
   /// getDistance - Returns the distance (or NULL) associated with a
-  /// particular level.
-  const SCEV *getDistance(unsigned Level) const override;
+  /// particular common or SameSD level.
+  const SCEV *getDistance(unsigned Level, bool SameSD = false) const override;
 
   /// Check if the direction vector is negative. A negative direction
   /// vector means Src and Dst are reversed in the actual program.
@@ -257,27 +312,34 @@ class LLVM_ABI FullDependence final : public Dependence {
   bool normalize(ScalarEvolution *SE) override;
 
   /// isPeelFirst - Returns true if peeling the first iteration from
-  /// this loop will break this dependence.
-  bool isPeelFirst(unsigned Level) const override;
+  /// this regular or SameSD loop level will break this dependence.
+  bool isPeelFirst(unsigned Level, bool SameSD = false) const override;
 
   /// isPeelLast - Returns true if peeling the last iteration from
-  /// this loop will break this dependence.
-  bool isPeelLast(unsigned Level) const override;
+  /// this regular or SameSD loop level will break this dependence.
+  bool isPeelLast(unsigned Level, bool SameSD = false) const override;
 
   /// isSplitable - Returns true if splitting the loop will break
   /// the dependence.
-  bool isSplitable(unsigned Level) const override;
+  bool isSplitable(unsigned Level, bool SameSD = false) const override;
+
+  /// inSameSDLoops - Returns true if this level is an SameSD level, i.e.,
+  /// performed across two separate loop nests that have the Same Iteration and
+  /// Depth.
+  bool inSameSDLoops(unsigned Level) const override;
 
-  /// isScalar - Returns true if a particular level is scalar; that is,
-  /// if no subscript in the source or destination mention the induction
-  /// variable associated with the loop at this level.
-  bool isScalar(unsigned Level) const override;
+  /// isScalar - Returns true if a particular regular or SameSD level is
+  /// scalar; that is, if no subscript in the source or destination mention
+  /// the induction variable associated with the loop at this level.
+  bool isScalar(unsigned Level, bool SameSD = false) const override;
 
 private:
   unsigned short Levels;
+  unsigned short SameSDLevels;
   bool LoopIndependent;
   bool Consistent; // Init to true, then refine.
   std::unique_ptr<DVEntry[]> DV;
+  std::unique_ptr<DVEntry[]> DVSameSD; // DV entries on SameSD levels
   friend class DependenceInfo;
 };
 
@@ -406,7 +468,8 @@ class DependenceInfo {
     const SCEV *A;
     const SCEV *B;
     const SCEV *C;
-    const Loop *AssociatedLoop;
+    const Loop *AssociatedSrcLoop;
+    const Loop *AssociatedDstLoop;
 
   public:
     /// isEmpty - Return true if the constraint is of kind Empty.
@@ -450,19 +513,27 @@ class DependenceInfo {
     /// Otherwise assert.
     LLVM_ABI const SCEV *getD() const;
 
-    /// getAssociatedLoop - Returns the loop associated with this constraint.
-    LLVM_ABI const Loop *getAssociatedLoop() const;
+    /// getAssociatedSrcLoop - Returns the source loop associated with this
+    /// constraint.
+    LLVM_ABI const Loop *getAssociatedSrcLoop() const;
+
+    /// getAssociatedDstLoop - Returns the destination loop associated with
+    /// this constraint.
+    LLVM_ABI const Loop *getAssociatedDstLoop() const;
 
     /// setPoint - Change a constraint to Point.
     LLVM_ABI void setPoint(const SCEV *X, const SCEV *Y,
-                           const Loop *CurrentLoop);
+                           const Loop *CurrentSrcLoop,
+                           const Loop *CurrentDstLoop);
 
     /// setLine - Change a constraint to Line.
     LLVM_ABI void setLine(const SCEV *A, const SCEV *B, const SCEV *C,
-                          const Loop *CurrentLoop);
+                          const Loop *CurrentSrcLoop,
+                          const Loop *CurrentDstLoop);
 
     /// setDistance - Change a constraint to Distance.
-    LLVM_ABI void setDistance(const SCEV *D, const Loop *CurrentLoop);
+    LLVM_ABI void setDistance(const SCEV *D, const Loop *CurrentSrcLoop,
+                              const Loop *CurrentDstLoop);
 
     /// setEmpty - Change a constraint to Empty.
     LLVM_ABI void setEmpty();
@@ -475,6 +546,12 @@ class DependenceInfo {
     LLVM_ABI void dump(raw_ostream &OS) const;
   };
 
+  /// Returns true if two loops have the Same iteration Space and Depth. To be
+  /// more specific, two loops have SameSD if they are in the same nesting
+  /// depth and have the same backedge count. SameSD stands for Same iteration
+  /// Space and Depth.
+  bool haveSameSD(const Loop *SrcLoop, const Loop *DstLoop) const;
+
   /// establishNestingLevels - Examines the loop nesting of the Src and Dst
   /// instructions and establishes their shared loops. Sets the variables
   /// CommonLevels, SrcLevels, and MaxLevels.
@@ -525,9 +602,21 @@ class DependenceInfo {
   ///     e - 5
   ///     f - 6
   ///     g - 7 = MaxLevels
+  /// SameSDLevels counts the number of levels after common levels that are
+  /// not common but have the same iteration space and depth. Internally this
+  /// is checked using haveSameSD. Assume that in this code fragment, levels c
+  /// and e have the same iteration space and depth, but levels d and f does
+  /// not. Then SameSDLevels is set to 1. In that case the level numbers for the
+  /// previous code look like
+  ///     a   - 1
+  ///     b   - 2
+  ///     c,e - 3 = CommonLevels
+  ///     d   - 4 = SrcLevels
+  ///     f   - 5
+  ///     g   - 6 = MaxLevels
   void establishNestingLevels(const Instruction *Src, const Instruction *Dst);
 
-  unsigned CommonLevels, SrcLevels, MaxLevels;
+  unsigned CommonLevels, SrcLevels, MaxLevels, SameSDLevels;
 
   /// mapSrcLoop - Given one of the loops containing the source, return
   /// its level index in our numbering scheme.
@@ -652,9 +741,9 @@ class DependenceInfo {
   /// If there might be a dependence, returns false.
   /// Sets appropriate direction and distance.
   bool strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
-                     const SCEV *DstConst, const Loop *CurrentLoop,
-                     unsigned Level, FullDependence &Result,
-                     Constraint &NewConstraint) const;
+                     const SCEV *DstConst, const Loop *CurrentSrcLoop,
+                     const Loop *CurrentDstLoop, unsigned Level,
+                     FullDependence &Result, Constraint &NewConstraint) const;
 
   /// weakCrossingSIVtest - Tests the weak-crossing SIV subscript pair
   /// (Src and Dst) for dependence.
@@ -667,9 +756,9 @@ class DependenceInfo {
   /// Set consistent to false.
   /// Marks the dependence as splitable.
   bool weakCrossingSIVtest(const SCEV *SrcCoeff, const SCEV *SrcConst,
-                           const SCEV *DstConst, const Loop *CurrentLoop,
-                           unsigned Level, FullDependence &Result,
-                           Constraint &NewConstraint,
+                           const SCEV *DstConst, const Loop *CurrentSrcLoop,
+                           const Loop *CurrentDstLoop, unsigned Level,
+                           FullDependence &Result, Constraint &NewConstraint,
                            const SCEV *&SplitIter) const;
 
   /// ExactSIVtest - Tests the SIV subscript pair
@@ -683,8 +772,9 @@ class DependenceInfo {
   /// Set consistent to false.
   bool exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
                     const SCEV *SrcConst, const SCEV *DstConst,
-                    const Loop *CurrentLoop, unsigned Level,
-                    FullDependence &Result, Constraint &NewConstraint) const;
+                    const Loop *CurrentSrcLoop, const Loop *CurrentDstLoop,
+                    unsigned Level, FullDependence &Result,
+                    Constraint &NewConstraint) const;
 
   /// weakZeroSrcSIVtest - Tests the weak-zero SIV subscript pair
   /// (Src and Dst) for dependence.
@@ -697,8 +787,9 @@ class DependenceInfo {
   /// Set consistent to false.
   /// If loop peeling will break the dependence, mark appropriately.
   bool weakZeroSrcSIVtest(const SCEV *DstCoeff, const SCEV *SrcConst,
-                          const SCEV *DstConst, const Loop *CurrentLoop,
-                          unsigned Level, FullDependence &Result,
+                          const SCEV *DstConst, const Loop *CurrentSrcLoop,
+                          const Loop *CurrentDstLoop, unsigned Level,
+                          FullDependence &Result,
                           Constraint &NewConstraint) const;
 
   /// weakZeroDstSIVtest - Tests the weak-zero SIV subscript pair
@@ -712,8 +803,9 @@ class DependenceInfo {
   /// Set consistent to false.
   /// If loop peeling will break the dependence, mark appropriately.
   bool weakZeroDstSIVtest(const SCEV *SrcCoeff, const SCEV *SrcConst,
-                          const SCEV *DstConst, const Loop *CurrentLoop,
-                          unsigned Level, FullDependence &Result,
+                          const SCEV *DstConst, const Loop *CurrentSrcLoop,
+                          const Loop *CurrentDstLoop, unsigned Level,
+                          FullDependence &Result,
                           Constraint &NewConstraint) const;
 
   /// exactRDIVtest - Tests the RDIV subscript pair for dependence.