[Polly] Implement user-directed loop distribution/fission.

This is a simple version without the possibility to define distribute
points or followup-transformations. However, it is the first
transformation that has to check whether the transformation is correct.

It interprets the same metadata as the LoopDistribute pass.

Re-apply after revert in c7bcd72 with
fix: Take isBand out of #ifndef NDEBUG since it now is used
unconditionally.

Author: Meinersbur

polly/include/polly/DependenceInfo.h

@@ -124,6 +124,10 @@ struct Dependences {
   ///         dependences.
   bool isValidSchedule(Scop &S, const StatementToIslMapTy &NewSchedules) const;
 
+  /// Return true of the schedule @p NewSched is a schedule for @S that does not
+  /// violate any dependences.
+  bool isValidSchedule(Scop &S, isl::schedule NewSched) const;
+
   /// Print the stored dependence information.
   void print(llvm::raw_ostream &OS) const;
 

polly/include/polly/ManualOptimizer.h

@@ -15,8 +15,13 @@
 
 #include "isl/isl-noexceptions.h"
 
+namespace llvm {
+class OptimizationRemarkEmitter;
+}
+
 namespace polly {
 class Scop;
+struct Dependences;
 
 /// Apply loop-transformation metadata.
 ///
@@ -30,7 +35,9 @@ class Scop;
 /// @return The transformed schedule with all mark-nodes with loop
 ///         transformations applied. Returns NULL in case of an error or @p
 ///         Sched itself if no transformation has been applied.
-isl::schedule applyManualTransformations(Scop *S, isl::schedule Sched);
+isl::schedule applyManualTransformations(Scop *S, isl::schedule Sched,
+                                         const Dependences &D,
+                                         llvm::OptimizationRemarkEmitter *ORE);
 } // namespace polly
 
 #endif /* POLLY_MANUALOPTIMIZER_H */

polly/include/polly/ScheduleTreeTransform.h

@@ -178,6 +178,9 @@ isl::schedule applyFullUnroll(isl::schedule_node BandToUnroll);
 /// Replace the AST band @p BandToUnroll by a partially unrolled equivalent.
 isl::schedule applyPartialUnroll(isl::schedule_node BandToUnroll, int Factor);
 
+/// Loop-distribute the band @p BandToFission as much as possible.
+isl::schedule applyMaxFission(isl::schedule_node BandToFission);
+
 /// Build the desired set of partial tile prefixes.
 ///
 /// We build a set of partial tile prefixes, which are prefixes of the vector

polly/lib/Analysis/DependenceInfo.cpp

@@ -636,6 +636,19 @@ void Dependences::calculateDependences(Scop &S) {
   LLVM_DEBUG(dump());
 }
 
+bool Dependences::isValidSchedule(Scop &S, isl::schedule NewSched) const {
+  // TODO: Also check permutable/coincident flags as well.
+
+  StatementToIslMapTy NewSchedules;
+  for (auto NewMap : NewSched.get_map().get_map_list()) {
+    auto Stmt = reinterpret_cast<ScopStmt *>(
+        NewMap.get_tuple_id(isl::dim::in).get_user());
+    NewSchedules[Stmt] = NewMap;
+  }
+
+  return isValidSchedule(S, NewSchedules);
+}
+
 bool Dependences::isValidSchedule(
     Scop &S, const StatementToIslMapTy &NewSchedule) const {
   if (LegalityCheckDisabled)

polly/lib/Transform/ManualOptimizer.cpp

@@ -11,11 +11,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "polly/ManualOptimizer.h"
+#include "polly/DependenceInfo.h"
+#include "polly/Options.h"
 #include "polly/ScheduleTreeTransform.h"
 #include "polly/Support/ScopHelper.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 
@@ -25,6 +28,12 @@ using namespace polly;
 using namespace llvm;
 
 namespace {
+
+static cl::opt<bool> IgnoreDepcheck(
+    "polly-pragma-ignore-depcheck",
+    cl::desc("Skip the dependency check for pragma-based transformations"),
+    cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
+
 /// Same as llvm::hasUnrollTransformation(), but takes a LoopID as argument
 /// instead of a Loop.
 static TransformationMode hasUnrollTransformation(MDNode *LoopID) {
@@ -48,6 +57,31 @@ static TransformationMode hasUnrollTransformation(MDNode *LoopID) {
   return TM_Unspecified;
 }
 
+// Return the first DebugLoc in the list.
+static DebugLoc findFirstDebugLoc(MDNode *MD) {
+  if (MD) {
+    for (const MDOperand &X : drop_begin(MD->operands(), 1)) {
+      Metadata *A = X.get();
+      if (!isa<DILocation>(A))
+        continue;
+      return cast<DILocation>(A);
+    }
+  }
+
+  return {};
+}
+
+static DebugLoc findTransformationDebugLoc(MDNode *LoopMD, StringRef Name) {
+  // First find dedicated transformation location
+  // (such as the location of #pragma clang loop)
+  MDNode *MD = findOptionMDForLoopID(LoopMD, Name);
+  if (DebugLoc K = findFirstDebugLoc(MD))
+    return K;
+
+  // Otherwise, fall back to the location of the loop itself
+  return findFirstDebugLoc(LoopMD);
+}
+
 /// Apply full or partial unrolling.
 static isl::schedule applyLoopUnroll(MDNode *LoopMD,
                                      isl::schedule_node BandToUnroll) {
@@ -78,6 +112,15 @@ static isl::schedule applyLoopUnroll(MDNode *LoopMD,
   return {};
 }
 
+static isl::schedule applyLoopFission(MDNode *LoopMD,
+                                      isl::schedule_node BandToFission) {
+  // TODO: Make it possible to selectively fission substatements.
+  // TODO: Apply followup loop properties.
+  // TODO: Instead of fission every statement, find the maximum set that does
+  // not cause a dependency violation.
+  return applyMaxFission(BandToFission);
+}
+
 // Return the properties from a LoopID. Scalar properties are ignored.
 static auto getLoopMDProps(MDNode *LoopMD) {
   return map_range(
@@ -96,14 +139,76 @@ class SearchTransformVisitor
   BaseTy &getBase() { return *this; }
   const BaseTy &getBase() const { return *this; }
 
+  polly::Scop *S;
+  const Dependences *D;
+  OptimizationRemarkEmitter *ORE;
+
   // Set after a transformation is applied. Recursive search must be aborted
   // once this happens to ensure that any new followup transformation is
   // transformed in innermost-first order.
   isl::schedule Result;
 
+  /// Check wether a schedule after a  transformation is legal. Return the old
+  /// schedule without the transformation.
+  isl::schedule
+  checkDependencyViolation(llvm::MDNode *LoopMD, llvm::Value *CodeRegion,
+                           const isl::schedule_node &OrigBand,
+                           StringRef DebugLocAttr, StringRef TransPrefix,
+                           StringRef RemarkName, StringRef TransformationName) {
+    if (D->isValidSchedule(*S, Result))
+      return Result;
+
+    LLVMContext &Ctx = LoopMD->getContext();
+    LLVM_DEBUG(dbgs() << "Dependency violation detected\n");
+
+    DebugLoc TransformLoc = findTransformationDebugLoc(LoopMD, DebugLocAttr);
+
+    if (IgnoreDepcheck) {
+      LLVM_DEBUG(dbgs() << "Still accepting transformation due to "
+                           "-polly-pragma-ignore-depcheck\n");
+      if (ORE) {
+        ORE->emit(
+            OptimizationRemark(DEBUG_TYPE, RemarkName, TransformLoc, CodeRegion)
+            << (Twine("Could not verify dependencies for ") +
+                TransformationName +
+                "; still applying because of -polly-pragma-ignore-depcheck")
+                   .str());
+      }
+      return Result;
+    }
+
+    LLVM_DEBUG(dbgs() << "Rolling back transformation\n");
+
+    if (ORE) {
+      ORE->emit(DiagnosticInfoOptimizationFailure(DEBUG_TYPE, RemarkName,
+                                                  TransformLoc, CodeRegion)
+                << (Twine("not applying ") + TransformationName +
+                    ": cannot ensure semantic equivalence due to possible "
+                    "dependency violations")
+                       .str());
+    }
+
+    // If illegal, revert and remove the transformation to not risk re-trying
+    // indefintely.
+    MDNode *NewLoopMD =
+        makePostTransformationMetadata(Ctx, LoopMD, {TransPrefix}, {});
+    BandAttr *Attr = getBandAttr(OrigBand);
+    Attr->Metadata = NewLoopMD;
+
+    // Roll back old schedule.
+    return OrigBand.get_schedule();
+  }
+
 public:
-  static isl::schedule applyOneTransformation(const isl::schedule &Sched) {
-    SearchTransformVisitor Transformer;
+  SearchTransformVisitor(polly::Scop *S, const Dependences *D,
+                         OptimizationRemarkEmitter *ORE)
+      : S(S), D(D), ORE(ORE) {}
+
+  static isl::schedule applyOneTransformation(polly::Scop *S,
+                                              const Dependences *D,
+                                              OptimizationRemarkEmitter *ORE,
+                                              const isl::schedule &Sched) {
+    SearchTransformVisitor Transformer(S, D, ORE);
     Transformer.visit(Sched);
     return Transformer.Result;
   }
@@ -125,6 +230,14 @@ class SearchTransformVisitor
       return;
     }
 
+    // CodeRegion used but ORE to determine code hotness.
+    // TODO: Works only for original loop; for transformed loops, should track
+    // where the loop's body code comes from.
+    Loop *Loop = Attr->OriginalLoop;
+    Value *CodeRegion = nullptr;
+    if (Loop)
+      CodeRegion = Loop->getHeader();
+
     MDNode *LoopMD = Attr->Metadata;
     if (!LoopMD)
       return;
@@ -146,6 +259,15 @@ class SearchTransformVisitor
         Result = applyLoopUnroll(LoopMD, Band);
         if (!Result.is_null())
           return;
+      } else if (AttrName == "llvm.loop.distribute.enable") {
+        Result = applyLoopFission(LoopMD, Band);
+        if (!Result.is_null())
+          Result = checkDependencyViolation(
+              LoopMD, CodeRegion, Band, "llvm.loop.distribute.loc",
+              "llvm.loop.distribute.", "FailedRequestedFission",
+              "loop fission/distribution");
+        if (!Result.is_null())
+          return;
       }
 
       // not a loop transformation; look for next property
@@ -162,11 +284,14 @@ class SearchTransformVisitor
 
 } // namespace
 
-isl::schedule polly::applyManualTransformations(Scop *S, isl::schedule Sched) {
+isl::schedule
+polly::applyManualTransformations(Scop *S, isl::schedule Sched,
+                                  const Dependences &D,
+                                  OptimizationRemarkEmitter *ORE) {
   // Search the loop nest for transformations until fixpoint.
   while (true) {
     isl::schedule Result =
-        SearchTransformVisitor::applyOneTransformation(Sched);
+        SearchTransformVisitor::applyOneTransformation(S, &D, ORE, Sched);
     if (Result.is_null()) {
       // No (more) transformation has been found.
       break;

polly/lib/Transform/ScheduleOptimizer.cpp

@@ -55,6 +55,7 @@
 #include "polly/Support/ISLOStream.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/CommandLine.h"
 #include "isl/options.h"
@@ -668,7 +669,9 @@ static void walkScheduleTreeForStatistics(isl::schedule Schedule, int Version) {
 static bool runIslScheduleOptimizer(
     Scop &S,
     function_ref<const Dependences &(Dependences::AnalysisLevel)> GetDeps,
-    TargetTransformInfo *TTI, isl::schedule &LastSchedule) {
+    TargetTransformInfo *TTI, OptimizationRemarkEmitter *ORE,
+    isl::schedule &LastSchedule) {
+
   // Skip SCoPs in case they're already optimised by PPCGCodeGeneration
   if (S.isToBeSkipped())
     return false;
@@ -689,8 +692,8 @@ static bool runIslScheduleOptimizer(
 
   bool HasUserTransformation = false;
   if (PragmaBasedOpts) {
-    isl::schedule ManuallyTransformed =
-        applyManualTransformations(&S, Schedule);
+    isl::schedule ManuallyTransformed = applyManualTransformations(
+        &S, Schedule, GetDeps(Dependences::AL_Statement), ORE);
     if (ManuallyTransformed.is_null()) {
       LLVM_DEBUG(dbgs() << "Error during manual optimization\n");
       return false;
@@ -849,7 +852,9 @@ static bool runIslScheduleOptimizer(
     walkScheduleTreeForStatistics(Schedule, 2);
   }
 
-  if (!ScheduleTreeOptimizer::isProfitableSchedule(S, Schedule))
+  // Skip profitability check if user transformation(s) have been applied.
+  if (!HasUserTransformation &&
+      !ScheduleTreeOptimizer::isProfitableSchedule(S, Schedule))
     return false;
 
   auto ScopStats = S.getStatistics();
@@ -878,9 +883,11 @@ bool IslScheduleOptimizerWrapperPass::runOnScop(Scop &S) {
     return getAnalysis<DependenceInfo>().getDependences(
         Dependences::AL_Statement);
   };
+  OptimizationRemarkEmitter &ORE =
+      getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
   TargetTransformInfo *TTI =
       &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
-  return runIslScheduleOptimizer(S, getDependences, TTI, LastSchedule);
+  return runIslScheduleOptimizer(S, getDependences, TTI, &ORE, LastSchedule);
 }
 
 static void runScheduleOptimizerPrinter(raw_ostream &OS,
@@ -915,8 +922,10 @@ void IslScheduleOptimizerWrapperPass::getAnalysisUsage(
   ScopPass::getAnalysisUsage(AU);
   AU.addRequired<DependenceInfo>();
   AU.addRequired<TargetTransformInfoWrapperPass>();
+  AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
 
   AU.addPreserved<DependenceInfo>();
+  AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
 }
 
 } // namespace
@@ -930,6 +939,7 @@ INITIALIZE_PASS_BEGIN(IslScheduleOptimizerWrapperPass, "polly-opt-isl",
 INITIALIZE_PASS_DEPENDENCY(DependenceInfo);
 INITIALIZE_PASS_DEPENDENCY(ScopInfoRegionPass);
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass);
+INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass);
 INITIALIZE_PASS_END(IslScheduleOptimizerWrapperPass, "polly-opt-isl",
                     "Polly - Optimize schedule of SCoP", false, false)
 
@@ -941,9 +951,10 @@ runIslScheduleOptimizerUsingNPM(Scop &S, ScopAnalysisManager &SAM,
   auto GetDeps = [&Deps](Dependences::AnalysisLevel) -> const Dependences & {
     return Deps.getDependences(Dependences::AL_Statement);
   };
+  OptimizationRemarkEmitter ORE(&S.getFunction());
   TargetTransformInfo *TTI = &SAR.TTI;
   isl::schedule LastSchedule;
-  bool Modified = runIslScheduleOptimizer(S, GetDeps, TTI, LastSchedule);
+  bool Modified = runIslScheduleOptimizer(S, GetDeps, TTI, &ORE, LastSchedule);
   if (OS) {
     *OS << "Printing analysis 'Polly - Optimize schedule of SCoP' for region: '"
         << S.getName() << "' in function '" << S.getFunction().getName()