[CopyPropagation] Add ShrinkBorrowScope.

During copy propagation (for which -enable-copy-propagation must still
be passed), also try to shrink borrow scopes by hoisting end_borrows
using the newly added ShrinkBorrowScope utility.

Allow end_borrow instructions to be hoisted over instructions that are
not deinit barriers for the value which is borrowed.  Deinit barriers
include uses of the value, loads of memory, loads of weak references
that may be zeroed during deinit, and "synchronization points".

rdar://79149830

Author: nate-chandler

include/swift/SIL/OwnershipUtils.h

@@ -31,6 +31,7 @@ class SILModule;
 class SILValue;
 class DeadEndBlocks;
 class PrunedLiveness;
+struct BorrowedValue;
 
 /// Returns true if v is an address or trivial.
 bool isValueAddressOrTrivial(SILValue v);
@@ -103,6 +104,15 @@ inline bool isForwardingConsume(SILValue value) {
 bool findInnerTransitiveGuaranteedUses(
     SILValue guaranteedValue, SmallVectorImpl<Operand *> *usePoints = nullptr);
 
+/// Like findInnerTransitiveGuaranteedUses except that rather than it being a
+/// precondition that the provided value not be a BorrowedValue, it is a [type-
+/// system-enforced] precondition that the provided value be a BorrowedValue.
+///
+/// TODO: Merge with findInnerTransitiveGuaranteedUses.
+bool findInnerTransitiveGuaranteedUsesOfBorrowedValue(
+    BorrowedValue borrowedValue,
+    SmallVectorImpl<Operand *> *usePoints = nullptr);
+
 /// Find leaf "use points" of a guaranteed value within its enclosing borrow
 /// scope (without looking through reborrows). To find the use points of the
 /// extended borrow scope, after looking through reborrows, use

include/swift/SIL/SILInstruction.h

@@ -630,6 +630,12 @@ class SILInstruction : public llvm::ilist_node<SILInstruction> {
   /// Can this instruction abort the program in some manner?
   bool mayTrap() const;
 
+  /// Involves a synchronization point like a memory barrier, lock or syscall.
+  ///
+  /// TODO: We need side-effect analysis and library annotation for this to be
+  ///       a reasonable API.  For now, this is just a placeholder.
+  bool maySynchronize() const;
+
   /// Returns true if the given instruction is completely identical to RHS.
   bool isIdenticalTo(const SILInstruction *RHS) const {
     return isIdenticalTo(RHS,

include/swift/SILOptimizer/Utils/CanonicalizeBorrowScope.h

@@ -16,11 +16,8 @@
 /// deleted, which in turn allows canonicalization of the outer owned values
 /// (via CanonicalizeOSSALifetime).
 ///
-/// This does not shrink borrow scopes; it does not rewrite end_borrows.
-///
-/// TODO: A separate utility to shrink borrow scopes should eventually run
-/// before this utility. It should hoist end_borrow up to the latest "destroy
-/// barrier" whenever the scope does not contain a PointerEscape.
+/// This does not shrink borrow scopes; it does not rewrite end_borrows.  For
+/// that, see ShrinkBorrowScope.
 ///
 //===----------------------------------------------------------------------===//
 
@@ -150,6 +147,8 @@ class CanonicalizeBorrowScope {
   bool consolidateBorrowScope();
 };
 
+bool shrinkBorrowScope(BeginBorrowInst *bbi, InstructionDeleter &deleter);
+
 } // namespace swift
 
 #endif // SWIFT_SILOPTIMIZER_UTILS_CANONICALIZEBORROWSCOPES_H

lib/SIL/IR/SILInstruction.cpp

@@ -1370,6 +1370,12 @@ bool SILInstruction::mayTrap() const {
   }
 }
 
+bool SILInstruction::maySynchronize() const {
+  // TODO: We need side-effect analysis and library annotation for this to be
+  //       a reasonable API.  For now, this is just a placeholder.
+  return isa<FullApplySite>(this);
+}
+
 bool SILInstruction::isMetaInstruction() const {
   // Every instruction that implements getVarInfo() should be in this list.
   switch (getKind()) {

lib/SIL/Utils/OwnershipUtils.cpp

@@ -188,6 +188,103 @@ bool swift::findInnerTransitiveGuaranteedUses(
   return true;
 }
 
+/// Like findInnerTransitiveGuaranteedUses except that rather than it being a
+/// precondition that the provided value not be a BorrowedValue, it is a [type-
+/// system-enforced] precondition that the provided value be a BorrowedValue.
+///
+/// TODO: Merge with findInnerTransitiveGuaranteedUses.  Note that at the moment
+///       the two are _almost_ identical, but not quite because the other has a
+///       #if 0 and not just leaf uses but ALL uses are recorded.
+bool swift::findInnerTransitiveGuaranteedUsesOfBorrowedValue(
+    BorrowedValue borrowedValue, SmallVectorImpl<Operand *> *usePoints) {
+
+  auto recordUse = [&](Operand *use) {
+    if (usePoints && use->getOperandOwnership() != OperandOwnership::NonUse) {
+      usePoints->push_back(use);
+    }
+  };
+
+  // Push the value's immediate uses.
+  //
+  // TODO: The worklist can be a simple vector without any a membership check if
+  // destructures are changed to be represented as reborrows. Currently a
+  // destructure forwards multiple results! This means that the worklist could
+  // grow exponentially without the membership check. It's fine to do this
+  // membership check locally in this function (within a borrow scope) because
+  // it isn't needed for the immediate uses, only the transitive uses.
+  GraphNodeWorklist<Operand *, 8> worklist;
+  for (Operand *use : borrowedValue.value->getUses()) {
+    if (use->getOperandOwnership() != OperandOwnership::NonUse)
+      worklist.insert(use);
+  }
+
+  // --- Transitively follow forwarded uses and look for escapes.
+
+  // usePoints grows in this loop.
+  while (Operand *use = worklist.pop()) {
+    switch (use->getOperandOwnership()) {
+    case OperandOwnership::NonUse:
+    case OperandOwnership::TrivialUse:
+    case OperandOwnership::ForwardingConsume:
+    case OperandOwnership::DestroyingConsume:
+      llvm_unreachable("this operand cannot handle an inner guaranteed use");
+
+    case OperandOwnership::ForwardingUnowned:
+    case OperandOwnership::PointerEscape:
+      return false;
+
+    case OperandOwnership::InstantaneousUse:
+    case OperandOwnership::UnownedInstantaneousUse:
+    case OperandOwnership::BitwiseEscape:
+    // Reborrow only happens when this is called on a value that creates a
+    // borrow scope.
+    case OperandOwnership::Reborrow:
+    // EndBorrow either happens when this is called on a value that creates a
+    // borrow scope, or when it is pushed as a use when processing a nested
+    // borrow.
+    case OperandOwnership::EndBorrow:
+      recordUse(use);
+      break;
+
+    case OperandOwnership::InteriorPointer:
+      if (InteriorPointerOperandKind::get(use) ==
+          InteriorPointerOperandKind::Invalid)
+        return false;
+      // If our base guaranteed value does not have any consuming uses (consider
+      // function arguments), we need to be sure to include interior pointer
+      // operands since we may not get a use from a end_scope instruction.
+      if (InteriorPointerOperand(use).findTransitiveUses(usePoints) !=
+          AddressUseKind::NonEscaping) {
+        return false;
+      }
+      break;
+
+    case OperandOwnership::ForwardingBorrow: {
+      ForwardingOperand(use).visitForwardedValues([&](SILValue result) {
+        // Do not include transitive uses with 'none' ownership
+        if (result.getOwnershipKind() == OwnershipKind::None)
+          return true;
+        for (auto *resultUse : result->getUses()) {
+          if (resultUse->getOperandOwnership() != OperandOwnership::NonUse) {
+            worklist.insert(resultUse);
+          }
+        }
+        return true;
+      });
+      recordUse(use);
+      break;
+    }
+    case OperandOwnership::Borrow:
+      BorrowingOperand(use).visitExtendedScopeEndingUses([&](Operand *endUse) {
+        recordUse(endUse);
+        return true;
+      });
+      break;
+    }
+  }
+  return true;
+}
+
 // Find all use points of \p guaranteedValue within its borrow scope. All use
 // points will be dominated by \p guaranteedValue.
 //

lib/SILOptimizer/Transforms/CopyPropagation.cpp

@@ -75,6 +75,9 @@ struct CanonicalDefWorklist {
   CanonicalDefWorklist(bool canonicalizeBorrows)
       : canonicalizeBorrows(canonicalizeBorrows) {}
 
+  // Update the worklist for the def corresponding to \p bbi, a BeginBorrow.
+  void updateForBorrow(BeginBorrowInst *bbi) { borrowedValues.insert(bbi); }
+
   // Update the worklist for the def corresponding to \p copy, which is usually
   // a CopyValue, but may be any owned value such as the operand of a
   // DestroyValue (to force lifetime shortening).
@@ -414,18 +417,30 @@ void CopyPropagation::run() {
   InstructionDeleter deleter(std::move(callbacks));
   bool changed = false;
 
-  // Driver: Find all copied defs.
+  GraphNodeWorklist<BeginBorrowInst *, 16> beginBorrowsToShrink;
+
+  // Driver: Find all copied or borrowed defs.
   for (auto &bb : *f) {
     for (auto &i : bb) {
       if (auto *copy = dyn_cast<CopyValueInst>(&i)) {
         defWorklist.updateForCopy(copy);
+      } else if (auto *borrow = dyn_cast<BeginBorrowInst>(&i)) {
+        beginBorrowsToShrink.insert(borrow);
       } else if (canonicalizeAll) {
         if (auto *destroy = dyn_cast<DestroyValueInst>(&i)) {
           defWorklist.updateForCopy(destroy->getOperand());
         }
       }
     }
   }
+
+  // NOTE: We assume that the function is in reverse post order so visiting the
+  //       blocks and pushing begin_borrows as we see them and then popping them
+  //       off the end will result in shrinking inner borrow scopes first.
+  while (auto *bbi = beginBorrowsToShrink.pop()) {
+    shrinkBorrowScope(bbi, deleter);
+  }
+
   // canonicalizer performs all modifications through deleter's callbacks, so we
   // don't need to explicitly check for changes.
   CanonicalizeOSSALifetime canonicalizer(pruneDebug, poisonRefs,

lib/SILOptimizer/Utils/CMakeLists.txt

@@ -22,6 +22,7 @@ target_sources(swiftSILOptimizer PRIVATE
   OptimizerStatsUtils.cpp
   PartialApplyCombiner.cpp
   PerformanceInlinerUtils.cpp
+  ShrinkBorrowScope.cpp
   SILInliner.cpp
   SILSSAUpdater.cpp
   SpecializationMangler.cpp