Merge pull request #65417 from nate-chandler/rdar108514447

[SemanticARCOpts] Promote moved load [copy]s to load_borrows.

Author: nate-chandler

include/swift/SIL/OwnershipUtils.h

@@ -1104,6 +1104,9 @@ class OwnedValueIntroducerKind {
     /// memory location.
     LoadTake,
 
+    /// An owned value produced by moving from another owned value.
+    Move,
+
     /// An owned value that is a result of a true phi argument.
     ///
     /// A true phi argument here is defined as an SIL phi argument that only has
@@ -1178,6 +1181,8 @@ class OwnedValueIntroducerKind {
         return Kind::LoadCopy;
       return Kind::Invalid;
     }
+    case ValueKind::MoveValueInst:
+      return Kind::Move;
     case ValueKind::PartialApplyInst:
       return Kind::PartialApplyInit;
     case ValueKind::AllocBoxInst:
@@ -1250,6 +1255,7 @@ struct OwnedValueIntroducer {
     case OwnedValueIntroducerKind::BeginApply:
     case OwnedValueIntroducerKind::TryApply:
     case OwnedValueIntroducerKind::LoadTake:
+    case OwnedValueIntroducerKind::Move:
     case OwnedValueIntroducerKind::Phi:
     case OwnedValueIntroducerKind::Struct:
     case OwnedValueIntroducerKind::Tuple:
@@ -1280,6 +1286,7 @@ struct OwnedValueIntroducer {
     case OwnedValueIntroducerKind::BeginApply:
     case OwnedValueIntroducerKind::TryApply:
     case OwnedValueIntroducerKind::LoadTake:
+    case OwnedValueIntroducerKind::Move:
     case OwnedValueIntroducerKind::FunctionArgument:
     case OwnedValueIntroducerKind::PartialApplyInit:
     case OwnedValueIntroducerKind::AllocBoxInit:

lib/SIL/Utils/OwnershipUtils.cpp

@@ -1073,6 +1073,9 @@ void OwnedValueIntroducerKind::print(llvm::raw_ostream &os) const {
   case OwnedValueIntroducerKind::LoadTake:
     os << "LoadTake";
     return;
+  case OwnedValueIntroducerKind::Move:
+    os << "Move";
+    return;
   case OwnedValueIntroducerKind::Phi:
     os << "Phi";
     return;

lib/SILOptimizer/SemanticARC/LoadCopyToLoadBorrowOpt.cpp

@@ -319,8 +319,6 @@ static bool isWrittenTo(Context &ctx, LoadInst *load,
 //                            Top Level Entrypoint
 //===----------------------------------------------------------------------===//
 
-// Convert a load [copy] from unique storage [read] that has all uses that can
-// accept a guaranteed parameter to a load_borrow.
 bool SemanticARCOptVisitor::visitLoadInst(LoadInst *li) {
   // This optimization can use more complex analysis. We should do some
   // experiments before enabling this by default as a guaranteed optimization.
@@ -334,14 +332,36 @@ bool SemanticARCOptVisitor::visitLoadInst(LoadInst *li) {
   if (li->getOwnershipQualifier() != LoadOwnershipQualifier::Copy)
     return false;
 
-  // Ok, we have our load [copy]. Make sure its value is truly a dead live range
-  // implying it is only ever consumed by destroy_value instructions. If it is
-  // consumed, we need to pass off a +1 value, so bail.
+  // Ok, we have our load [copy].  Try to optimize considering its live range.
+  if (performLoadCopyToLoadBorrowOptimization(li, li))
+    return true;
+  // Check whether the load [copy]'s only use is as an operand to a move_value.
+  auto *use = li->getSingleUse();
+  if (!use)
+    return false;
+  auto *mvi = dyn_cast<MoveValueInst>(use->getUser());
+  if (!mvi)
+    return false;
+  // Try to optimize considering the move_value's live range.
+  return performLoadCopyToLoadBorrowOptimization(li, mvi);
+}
+
+// Convert a load [copy] from unique storage [read] whose representative
+// (either the load [copy] itself or a move from it) has all uses that can
+// accept a guaranteed parameter to a load_borrow.
+bool SemanticARCOptVisitor::performLoadCopyToLoadBorrowOptimization(
+    LoadInst *li, SILValue original) {
+  assert(li->getOwnershipQualifier() == LoadOwnershipQualifier::Copy);
+  assert(li == original || li->getSingleUse()->getUser() ==
+                               cast<SingleValueInstruction>(original));
+  // Make sure its value is truly a dead live range implying it is only ever
+  // consumed by destroy_value instructions. If it is consumed, we need to pass
+  // off a +1 value, so bail.
   //
   // FIXME: We should consider if it is worth promoting a load [copy]
   // -> load_borrow if we can put a copy_value on a cold path and thus
   // eliminate RR traffic on a hot path.
-  OwnershipLiveRange lr(li);
+  OwnershipLiveRange lr(original);
   if (bool(lr.hasUnknownConsumingUse()))
     return false;
 
@@ -355,8 +375,17 @@ bool SemanticARCOptVisitor::visitLoadInst(LoadInst *li) {
   // load_borrow.
   auto *lbi =
       SILBuilderWithScope(li).createLoadBorrow(li->getLoc(), li->getOperand());
-
   lr.insertEndBorrowsAtDestroys(lbi, getDeadEndBlocks(), ctx.lifetimeFrontier);
-  std::move(lr).convertToGuaranteedAndRAUW(lbi, getCallbacks());
+  SILValue replacement = lbi;
+  if (original != li) {
+    getCallbacks().eraseAndRAUWSingleValueInst(li, lbi);
+    auto *bbi =
+        SILBuilderWithScope(cast<SingleValueInstruction>(original))
+            .createBeginBorrow(li->getLoc(), lbi, original->isLexical());
+    replacement = bbi;
+    lr.insertEndBorrowsAtDestroys(bbi, getDeadEndBlocks(),
+                                  ctx.lifetimeFrontier);
+  }
+  std::move(lr).convertToGuaranteedAndRAUW(replacement, getCallbacks());
   return true;
 }

lib/SILOptimizer/SemanticARC/OwnershipLiveRange.cpp

@@ -296,6 +296,7 @@ static SILValue convertIntroducerToGuaranteed(OwnedValueIntroducer introducer) {
   case OwnedValueIntroducerKind::BeginApply:
   case OwnedValueIntroducerKind::TryApply:
   case OwnedValueIntroducerKind::LoadTake:
+  case OwnedValueIntroducerKind::Move:
   case OwnedValueIntroducerKind::FunctionArgument:
   case OwnedValueIntroducerKind::PartialApplyInit:
   case OwnedValueIntroducerKind::AllocBoxInit:

lib/SILOptimizer/SemanticARC/SemanticARCOptVisitor.h

@@ -202,6 +202,7 @@ struct LLVM_LIBRARY_VISIBILITY SemanticARCOptVisitor
   bool optimize();
   bool optimizeWithoutFixedPoint();
 
+  bool performLoadCopyToLoadBorrowOptimization(LoadInst *li, SILValue original);
   bool performGuaranteedCopyValueOptimization(CopyValueInst *cvi);
   bool eliminateDeadLiveRangeCopyValue(CopyValueInst *cvi);
   bool tryJoiningCopyValueLiveRangeWithOperand(CopyValueInst *cvi);

test/SILOptimizer/semantic-arc-opts-loadcopy-to-loadborrow.sil

@@ -54,6 +54,10 @@ final class Klass {
   let baseLet: Klass
 }
 
+class C {}
+
+sil @getC : $@convention(thin) () -> (@owned C)
+
 extension Klass : MyFakeAnyObject {
   func myFakeMethod()
 }
@@ -1552,3 +1556,48 @@ bb2(%7 : @owned $NonTrivialStruct):
 bb3:
  unreachable
 }
+
+// CHECK-LABEL: sil [ossa] @promote_moved_from_load_copy : {{.*}} {
+// CHECK:         [[ADDR:%[^,]+]] = alloc_stack $C
+// CHECK:         [[GET_C:%[^,]+]] = function_ref @getC
+// CHECK:         [[STORED:%[^,]+]] = apply [[GET_C]]()
+// CHECK:         store [[STORED]] to [init] [[ADDR]]
+// CHECK:         [[LOADED:%[^,]+]] = load_borrow [[ADDR]]
+// CHECK:         [[LIFETIME:%[^,]+]] = begin_borrow [lexical] [[LOADED]]
+// CHECK:         end_borrow [[LIFETIME]]
+// CHECK:         end_borrow [[LOADED]]
+// CHECK:         destroy_addr [[ADDR]]
+// CHECK:         dealloc_stack [[ADDR]]
+// CHECK-LABEL: } // end sil function 'promote_moved_from_load_copy'
+sil [ossa] @promote_moved_from_load_copy : $@convention(thin) () -> () {
+bb0:
+  %addr = alloc_stack $C
+  %getC = function_ref @getC : $@convention(thin) () -> (@owned C)
+  %stored = apply %getC() : $@convention(thin) () -> (@owned C)
+  store %stored to [init] %addr : $*C
+  %loaded = load [copy] %addr : $*C
+  %lifetime = move_value [lexical] %loaded : $C
+  destroy_value %lifetime : $C
+  destroy_addr %addr : $*C
+  dealloc_stack %addr : $*C
+  %retval = tuple ()
+  return %retval : $()
+}
+
+// CHECK-LABEL: sil [ossa] @dont_promote_moved_from_load_copy_with_destroy_addr_above_range : {{.*}} {
+// CHECK:         load [copy]
+// CHECK-LABEL: } // end sil function 'dont_promote_moved_from_load_copy_with_destroy_addr_above_range'
+sil [ossa] @dont_promote_moved_from_load_copy_with_destroy_addr_above_range : $@convention(thin) () -> () {
+bb0:
+  %addr = alloc_stack $C
+  %getC = function_ref @getC : $@convention(thin) () -> (@owned C)
+  %stored = apply %getC() : $@convention(thin) () -> (@owned C)
+  store %stored to [init] %addr : $*C
+  %loaded = load [copy] %addr : $*C
+  destroy_addr %addr : $*C
+  %lifetime = move_value [lexical] %loaded : $C
+  destroy_value %lifetime : $C
+  dealloc_stack %addr : $*C
+  %retval = tuple ()
+  return %retval : $()
+}