Merge pull request #74109 from nate-chandler/rdar113142446

[ConsumeObjectChecker] End lifetimes at consumes.

Author: nate-chandler

include/swift/SIL/ParseTestSpecification.h

@@ -186,7 +186,15 @@ struct Arguments {
   }
   Operand *takeOperand() { return takeInstance<OperandArgument>("operand"); }
   SILInstruction *takeInstruction() {
-    return takeInstance<InstructionArgument>("instruction");
+    auto argument = takeArgument();
+    if (isa<ValueArgument>(argument)) {
+      auto value = cast<ValueArgument>(argument).getValue();
+      auto *definingInstruction = value.getDefiningInstruction();
+      assert(definingInstruction &&
+             "selected instruction via argument value!?");
+      return definingInstruction;
+    }
+    return getInstance<InstructionArgument>("instruction", argument);
   }
   SILArgument *takeBlockArgument() {
     return takeInstance<BlockArgumentArgument>("block argument");

include/swift/SIL/PrunedLiveness.h

@@ -613,9 +613,10 @@ class PrunedLiveRange : public PrunedLiveness {
                                            ValueSet &visited,
                                            SILValue value);
 
-  void updateForUse(SILInstruction *user, LifetimeEnding lifetimeEnding);
-
 public:
+  /// Add \p inst to liveness which uses the def as indicated by \p usage.
+  void updateForUse(SILInstruction *inst, LifetimeEnding usage);
+
   /// For flexibility, \p lifetimeEnding is provided by the
   /// caller. PrunedLiveness makes no assumptions about the def-use
   /// relationships that generate liveness. For example, use->isLifetimeEnding()

include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h

@@ -180,6 +180,16 @@ class CanonicalOSSAConsumeInfo final {
   SWIFT_ASSERT_ONLY_DECL(void dump() const LLVM_ATTRIBUTE_USED);
 };
 
+enum PruneDebugInsts_t : bool {
+  DontPruneDebugInsts = false,
+  PruneDebugInsts = true,
+};
+
+enum MaximizeLifetime_t : bool {
+  DontMaximizeLifetime = false,
+  MaximizeLifetime = true,
+};
+
 /// Canonicalize OSSA lifetimes.
 ///
 /// Allows the allocation of analysis state to be reused across calls to
@@ -221,11 +231,11 @@ class CanonicalizeOSSALifetime final {
 private:
   /// If true, then debug_value instructions outside of non-debug
   /// liveness may be pruned during canonicalization.
-  bool pruneDebugMode;
+  const PruneDebugInsts_t pruneDebugMode;
 
   /// If true, lifetimes will not be shortened except when necessary to avoid
   /// copies.
-  bool maximizeLifetime;
+  const MaximizeLifetime_t maximizeLifetime;
 
   // If present, will be used to ensure that the lifetime is not shortened to
   // end inside an access scope which it previously enclosed.  (Note that ending
@@ -246,6 +256,9 @@ class CanonicalizeOSSALifetime final {
   /// The SILValue to canonicalize.
   SILValue currentDef;
 
+  /// Instructions beyond which liveness is not extended by destroy uses.
+  ArrayRef<SILInstruction *> currentLexicalLifetimeEnds;
+
   /// Original points in the CFG where the current value's lifetime is consumed
   /// or destroyed.  Each block either contains a consuming instruction (e.g.
   /// `destroy_value`) or is on the availability boundary of the value in a
@@ -306,13 +319,15 @@ class CanonicalizeOSSALifetime final {
   struct LivenessState {
     BitfieldRef<SSAPrunedLiveness>::StackState state;
 
-    LivenessState(CanonicalizeOSSALifetime &parent, SILValue def)
+    LivenessState(CanonicalizeOSSALifetime &parent, SILValue def,
+                  ArrayRef<SILInstruction *> lexicalLifetimeEnds)
         : state(parent.liveness, def->getFunction()) {
-      parent.initializeLiveness(def);
+      parent.initializeLiveness(def, lexicalLifetimeEnds);
     }
   };
 
-  CanonicalizeOSSALifetime(bool pruneDebugMode, bool maximizeLifetime,
+  CanonicalizeOSSALifetime(PruneDebugInsts_t pruneDebugMode,
+                           MaximizeLifetime_t maximizeLifetime,
                            SILFunction *function,
                            NonLocalAccessBlockAnalysis *accessBlockAnalysis,
                            DominanceInfo *domTree,
@@ -324,7 +339,8 @@ class CanonicalizeOSSALifetime final {
 
   SILValue getCurrentDef() const { return currentDef; }
 
-  void initializeLiveness(SILValue def) {
+  void initializeLiveness(SILValue def,
+                          ArrayRef<SILInstruction *> lexicalLifetimeEnds) {
     assert(consumingBlocks.empty() && debugValues.empty());
     // Clear the cached analysis pointer just in case the client invalidates the
     // analysis, freeing its memory.
@@ -333,6 +349,7 @@ class CanonicalizeOSSALifetime final {
     destroys.clear();
 
     currentDef = def;
+    currentLexicalLifetimeEnds = lexicalLifetimeEnds;
 
     if (maximizeLifetime || respectsDeinitBarriers()) {
       liveness->initializeDiscoveredBlocks(&discoveredBlocks);
@@ -347,8 +364,18 @@ class CanonicalizeOSSALifetime final {
   }
 
   /// Top-Level API: rewrites copies and destroys within \p def's extended
-  /// lifetime. \p lifetime caches transient analysis state across multiple
-  /// calls.
+  /// lifetime.
+  ///
+  /// For lexical values, canonicalization respects deinit barriers, introducing
+  /// copies as needed to maintain lifetimes beyond final consuming uses to the
+  /// original lexical lifetime end.  When a lexical value is explicitly
+  /// consumed (via the `consume` keyword), however, the lifetime does not
+  /// extend to original destroys beyond that consume--the value must be dead
+  /// after the corresponding marker instructions (`move_value`); to support
+  /// this shortening, the marker instructions must be provided as \p
+  /// lexicalLifetimeEnds. When provided, deinit barriers will be respected
+  /// except to the extent doing so would result in the value being live after
+  /// the marker instructions.
   ///
   /// Return true if any change was made to \p def's extended lifetime. \p def
   /// itself will not be deleted and no instructions outside of \p def's
@@ -358,7 +385,8 @@ class CanonicalizeOSSALifetime final {
   /// This only deletes instructions within \p def's extended lifetime. Use
   /// InstructionDeleter::cleanUpDeadInstructions() to recursively delete dead
   /// operands.
-  bool canonicalizeValueLifetime(SILValue def);
+  bool canonicalizeValueLifetime(
+      SILValue def, ArrayRef<SILInstruction *> lexicalLifetimeEnds = {});
 
   /// Compute the liveness information for \p def. But do not do any rewriting
   /// or computation of boundaries.
@@ -448,6 +476,10 @@ class CanonicalizeOSSALifetime final {
   void extendLivenessToDeinitBarriers();
 
   void extendUnconsumedLiveness(PrunedLivenessBoundary const &boundary);
+  void visitExtendedUnconsumedBoundary(
+      ArrayRef<SILInstruction *> ends,
+      llvm::function_ref<void(SILInstruction *, PrunedLiveness::LifetimeEnding)>
+          visitor);
 
   void insertDestroysOnBoundary(PrunedLivenessBoundary const &boundary);
 

lib/SILOptimizer/Mandatory/ConsumeOperatorCopyableValuesChecker.cpp

@@ -25,6 +25,7 @@
 #include "swift/SIL/SILInstruction.h"
 #include "swift/SIL/SILUndef.h"
 #include "swift/SILOptimizer/Analysis/Analysis.h"
+#include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h"
 #include "swift/SILOptimizer/Analysis/ClosureScope.h"
 #include "swift/SILOptimizer/Analysis/DominanceAnalysis.h"
 #include "swift/SILOptimizer/Analysis/LoopAnalysis.h"
@@ -216,17 +217,18 @@ namespace {
 struct ConsumeOperatorCopyableValuesChecker {
   SILFunction *fn;
   CheckerLivenessInfo livenessInfo;
-  DominanceInfo *dominanceToUpdate;
-  SILLoopInfo *loopInfoToUpdate;
-
-  ConsumeOperatorCopyableValuesChecker(SILFunction *fn)
-      : fn(fn), dominanceToUpdate(nullptr), loopInfoToUpdate(nullptr) {}
-
-  void setDominanceToUpdate(DominanceInfo *newDFI) {
-    dominanceToUpdate = newDFI;
-  }
-
-  void setLoopInfoToUpdate(SILLoopInfo *newLFI) { loopInfoToUpdate = newLFI; }
+  DominanceInfo *dominance;
+  InstructionDeleter deleter;
+  CanonicalizeOSSALifetime canonicalizer;
+
+  ConsumeOperatorCopyableValuesChecker(SILFunction *fn,
+                                       DominanceInfo *dominance,
+                                       BasicCalleeAnalysis *calleeAnalysis)
+      : fn(fn), dominance(dominance),
+        canonicalizer(DontPruneDebugInsts,
+                      MaximizeLifetime_t(!fn->shouldOptimize()), fn,
+                      /*accessBlockAnalysis=*/nullptr, dominance,
+                      calleeAnalysis, deleter) {}
 
   bool check();
 
@@ -480,6 +482,7 @@ bool ConsumeOperatorCopyableValuesChecker::check() {
     // Then look at all of our found consuming uses. See if any of these are
     // _move that are within the boundary.
     bool foundMove = false;
+    SmallVector<SILInstruction *, 2> validMoves;
     auto dbgVarInst = DebugVarCarryingInst::getFromValue(lexicalValue);
     StringRef varName = DebugVarCarryingInst::getName(dbgVarInst);
     for (auto *use : livenessInfo.consumingUse) {
@@ -510,6 +513,7 @@ bool ConsumeOperatorCopyableValuesChecker::check() {
                 dbgVarInst->getLoc(), SILUndef::get(mvi->getOperand()),
                 *varInfo, false /*poison*/, UsesMoveableValueDebugInfo);
           }
+          validMoves.push_back(mvi);
         }
         foundMove = true;
       }
@@ -520,6 +524,11 @@ bool ConsumeOperatorCopyableValuesChecker::check() {
     if (foundMove) {
       dbgVarInst.markAsMoved();
     }
+
+    if (validMoves.size() > 0) {
+      canonicalizer.clear();
+      canonicalizer.canonicalizeValueLifetime(lexicalValue, validMoves);
+    }
   }
 
   return false;
@@ -575,18 +584,16 @@ class ConsumeOperatorCopyableValuesCheckerPass : public SILFunctionTransform {
     LLVM_DEBUG(llvm::dbgs() << "*** Checking moved values in fn: "
                             << getFunction()->getName() << '\n');
 
-    ConsumeOperatorCopyableValuesChecker checker(getFunction());
-
-    // If we already had dominance or loop info generated, update them when
-    // splitting blocks.
     auto *dominanceAnalysis = getAnalysis<DominanceAnalysis>();
-    if (dominanceAnalysis->hasFunctionInfo(fn))
-      checker.setDominanceToUpdate(dominanceAnalysis->get(fn));
+    auto *dominance = dominanceAnalysis->get(fn);
+    auto *calleeAnalysis = getAnalysis<BasicCalleeAnalysis>();
+    ConsumeOperatorCopyableValuesChecker checker(getFunction(), dominance,
+                                                 calleeAnalysis);
     auto *loopAnalysis = getAnalysis<SILLoopAnalysis>();
-    if (loopAnalysis->hasFunctionInfo(fn))
-      checker.setLoopInfoToUpdate(loopAnalysis->get(fn));
 
     if (checker.check()) {
+      // If we already had dominance or loop info generated, update them when
+      // splitting blocks.
       AnalysisPreserver preserveDominance(dominanceAnalysis);
       AnalysisPreserver preserveLoop(loopAnalysis);
       invalidateAnalysis(

lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.h

@@ -48,8 +48,8 @@ struct OSSACanonicalizer {
 
   OSSACanonicalizer(SILFunction *fn, DominanceInfo *domTree,
                     InstructionDeleter &deleter)
-      : canonicalizer(false /*pruneDebugMode*/,
-                      !fn->shouldOptimize() /*maximizeLifetime*/, fn,
+      : canonicalizer(DontPruneDebugInsts,
+                      MaximizeLifetime_t(!fn->shouldOptimize()), fn,
                       nullptr /*accessBlockAnalysis*/, domTree,
                       nullptr /*calleeAnalysis*/, deleter) {}
 
@@ -64,7 +64,7 @@ struct OSSACanonicalizer {
     CanonicalizeOSSALifetime::LivenessState canonicalizerState;
 
     LivenessState(OSSACanonicalizer &parent, SILValue def)
-        : parent(parent), canonicalizerState(parent.canonicalizer, def) {}
+        : parent(parent), canonicalizerState(parent.canonicalizer, def, {}) {}
 
     ~LivenessState() { parent.clear(); }
   };

lib/SILOptimizer/SILCombiner/SILCombine.cpp

@@ -351,8 +351,8 @@ void SILCombiner::canonicalizeOSSALifetimes(SILInstruction *currentInst) {
 
   DominanceInfo *domTree = DA->get(&Builder.getFunction());
   CanonicalizeOSSALifetime canonicalizer(
-      false /*prune debug*/,
-      !parentTransform->getFunction()->shouldOptimize() /*maximize lifetime*/,
+      DontPruneDebugInsts,
+      MaximizeLifetime_t(!parentTransform->getFunction()->shouldOptimize()),
       parentTransform->getFunction(), NLABA, domTree, CA, deleter);
   CanonicalizeBorrowScope borrowCanonicalizer(parentTransform->getFunction(),
                                               deleter);

lib/SILOptimizer/Transforms/CopyPropagation.cpp

@@ -425,15 +425,15 @@ namespace {
 
 class CopyPropagation : public SILFunctionTransform {
   /// If true, debug_value instructions should be pruned.
-  bool pruneDebug;
+  PruneDebugInsts_t pruneDebug;
   /// If true, all values will be canonicalized.
   bool canonicalizeAll;
   /// If true, then borrow scopes will be canonicalized, allowing copies of
   /// guaranteed values to be optimized. Does *not* shrink the borrow scope.
   bool canonicalizeBorrows;
 
 public:
-  CopyPropagation(bool pruneDebug, bool canonicalizeAll,
+  CopyPropagation(PruneDebugInsts_t pruneDebug, bool canonicalizeAll,
                   bool canonicalizeBorrows)
       : pruneDebug(pruneDebug), canonicalizeAll(canonicalizeAll),
         canonicalizeBorrows(canonicalizeBorrows) {}
@@ -493,7 +493,7 @@ void CopyPropagation::run() {
   // canonicalizer performs all modifications through deleter's callbacks, so we
   // don't need to explicitly check for changes.
   CanonicalizeOSSALifetime canonicalizer(
-      pruneDebug, /*maximizeLifetime=*/!getFunction()->shouldOptimize(),
+      pruneDebug, MaximizeLifetime_t(!getFunction()->shouldOptimize()),
       getFunction(), accessBlockAnalysis, domTree, calleeAnalysis, deleter);
   // NOTE: We assume that the function is in reverse post order so visiting the
   //       blocks and pushing begin_borrows as we see them and then popping them
@@ -645,11 +645,11 @@ void CopyPropagation::run() {
 // MandatoryCopyPropagation is not currently enabled in the -Onone pipeline
 // because it may negatively affect the debugging experience.
 SILTransform *swift::createMandatoryCopyPropagation() {
-  return new CopyPropagation(/*pruneDebug*/ true, /*canonicalizeAll*/ true,
+  return new CopyPropagation(PruneDebugInsts, /*canonicalizeAll*/ true,
                              /*canonicalizeBorrows*/ false);
 }
 
 SILTransform *swift::createCopyPropagation() {
-  return new CopyPropagation(/*pruneDebug*/ true, /*canonicalizeAll*/ true,
+  return new CopyPropagation(PruneDebugInsts, /*canonicalizeAll*/ true,
                              /*canonicalizeBorrows*/ EnableRewriteBorrows);
 }

lib/SILOptimizer/Transforms/SILMem2Reg.cpp

@@ -2071,7 +2071,7 @@ void MemoryToRegisters::canonicalizeValueLifetimes(
     }
   }
   CanonicalizeOSSALifetime canonicalizer(
-      /*pruneDebug=*/true, /*maximizeLifetime=*/!f.shouldOptimize(), &f,
+      PruneDebugInsts, MaximizeLifetime_t(!f.shouldOptimize()), &f,
       accessBlockAnalysis, domInfo, calleeAnalysis, deleter);
   for (auto value : owned) {
     if (isa<SILUndef>(value) || value->isMarkedAsDeleted())