@@ -314,8 +314,14 @@ static bool memInstMustConsume(Operand *memOper) {
314314 return applySite.getArgumentOperandConvention (*memOper).isOwnedConvention ();
315315 }
316316 case SILInstructionKind::PartialApplyInst: {
317- // If we are on the stack, we do not consume. Otherwise, we do.
318- return !cast<PartialApplyInst>(memInst)->
isOnStack ();
317+ // If we are on the stack or have an inout convention, we do not
318+ // consume. Otherwise, we do.
319+ auto *pai = cast<PartialApplyInst>(memInst);
320+ if (pai->isOnStack ())
321+ return false ;
322+ ApplySite applySite (pai);
323+ auto convention = applySite.getArgumentConvention (*memOper);
324+ return convention.isInoutConvention ();
319325 }
320326 case SILInstructionKind::DestroyAddrInst:
321327 return true ;
@@ -987,6 +993,141 @@ struct MoveOnlyAddressCheckerPImpl {
987993
988994} // namespace
989995
996+ // ===----------------------------------------------------------------------===//
997+ // MARK: CopiedLoadBorrowElimination
998+ // ===----------------------------------------------------------------------===//
999+
1000+ namespace {
1001+
1002+ // / An early transform that we run to convert any load_borrow that are copied
1003+ // / directly or that have any subelement that is copied to a load [copy]. This
1004+ // / lets the rest of the optimization handle these as appropriate.
1005+ struct CopiedLoadBorrowEliminationVisitor : public AccessUseVisitor {
1006+ SILFunction *fn;
1007+ StackList<LoadBorrowInst *> targets;
1008+
1009+ CopiedLoadBorrowEliminationVisitor (SILFunction *fn)
1010+ : AccessUseVisitor(AccessUseType::Overlapping,
1011+ NestedAccessType::IgnoreAccessBegin),
1012+ fn (fn), targets(fn) {}
1013+
1014+ bool visitUse (Operand *op, AccessUseType useTy) override {
1015+ LLVM_DEBUG (llvm::dbgs () << " CopiedLBElim. Visiting: " getUser ());
1016+ auto *lbi = dyn_cast<LoadBorrowInst>(op->getUser ());
1017+ if (!lbi)
1018+ return true ;
1019+
1020+ LLVM_DEBUG (llvm::dbgs () << " Found load_borrow: "
1021+
1022+ StackList<Operand *> useWorklist (lbi->getFunction ());
1023+ for (auto *use : lbi->getUses ())
1024+ useWorklist.push_back (use);
1025+
1026+ bool shouldConvertToLoadCopy = false ;
1027+ while (!useWorklist.empty ()) {
1028+ auto *nextUse = useWorklist.pop_back_val ();
1029+ switch (nextUse->getOperandOwnership ()) {
1030+ case OperandOwnership::NonUse:
1031+ case OperandOwnership::ForwardingUnowned:
1032+ case OperandOwnership::PointerEscape:
1033+ continue ;
1034+
1035+ // These might be uses that we need to perform a destructure or insert
1036+ // struct_extracts for.
1037+ case OperandOwnership::TrivialUse:
1038+ case OperandOwnership::InstantaneousUse:
1039+ case OperandOwnership::UnownedInstantaneousUse:
1040+ case OperandOwnership::InteriorPointer:
1041+ case OperandOwnership::BitwiseEscape: {
1042+ // Look through copy_value of a move only value. We treat copy_value of
1043+ // copyable values as normal uses.
1044+ if (auto *cvi = dyn_cast<CopyValueInst>(nextUse->getUser ())) {
1045+ if (cvi->getOperand ()->getType ().isMoveOnly ()) {
1046+ shouldConvertToLoadCopy = true ;
1047+ break ;
1048+ }
1049+ }
1050+ continue ;
1051+ }
1052+
1053+ case OperandOwnership::ForwardingConsume:
1054+ case OperandOwnership::DestroyingConsume:
1055+ // We can only hit this if our load_borrow was copied.
1056+ llvm_unreachable (" We should never hit this"
1057+
1058+ case OperandOwnership::GuaranteedForwarding:
1059+ // If we have a switch_enum, we always need to convert it to a load
1060+ // [copy] since we need to destructure through it.
1061+ shouldConvertToLoadCopy |= isa<SwitchEnumInst>(nextUse->getUser ());
1062+
1063+ ForwardingOperand (nextUse).visitForwardedValues ([&](SILValue value) {
1064+ for (auto *use : value->getUses ()) {
1065+ useWorklist.push_back (use);
1066+ }
1067+ return true ;
1068+ });
1069+ continue ;
1070+
1071+ case OperandOwnership::Borrow:
1072+ LLVM_DEBUG (llvm::dbgs () << " Found recursive borrow!\n "
1073+ // Look through borrows.
1074+ for (auto value : nextUse->getUser ()->getResults ()) {
1075+ for (auto *use : value->getUses ()) {
1076+ useWorklist.push_back (use);
1077+ }
1078+ }
1079+ continue ;
1080+ case OperandOwnership::EndBorrow:
1081+ LLVM_DEBUG (llvm::dbgs () << " Found end borrow!\n "
1082+ continue ;
1083+ case OperandOwnership::Reborrow:
1084+ llvm_unreachable (" Unsupported for now?!"
1085+ }
1086+
1087+ if (shouldConvertToLoadCopy)
1088+ break ;
1089+ }
1090+
1091+ LLVM_DEBUG (llvm::dbgs ()
1092+ << " Load Borrow was copied: "
1093+ << (shouldConvertToLoadCopy ? " true" " false" ' \n '
1094+ if (!shouldConvertToLoadCopy)
1095+ return true ;
1096+
1097+ targets.push_back (lbi);
1098+ return true ;
1099+ }
1100+
1101+ void process () {
1102+ if (targets.empty ())
1103+ return ;
1104+
1105+ while (!targets.empty ()) {
1106+ auto *lbi = targets.pop_back_val ();
1107+ SILBuilderWithScope builder (lbi);
1108+ SILValue li = builder.emitLoadValueOperation (
1109+ lbi->getLoc (), lbi->getOperand (), LoadOwnershipQualifier::Copy);
1110+ SILValue borrow = builder.createBeginBorrow (lbi->getLoc (), li);
1111+
1112+ for (auto *ebi : lbi->getEndBorrows ()) {
1113+ auto *next = ebi->getNextInstruction ();
1114+ SILBuilderWithScope builder (next);
1115+ auto loc = RegularLocation::getAutoGeneratedLocation ();
1116+ builder.emitDestroyValueOperation (loc, li);
1117+ }
1118+
1119+ lbi->replaceAllUsesWith (borrow);
1120+ lbi->eraseFromParent ();
1121+ }
1122+
1123+ LLVM_DEBUG (llvm::dbgs () << " After Load Borrow Elim. Func Dump Start! "
1124+ fn->print (llvm::dbgs ()));
1125+ LLVM_DEBUG (llvm::dbgs () << " After Load Borrow Elim. Func Dump End!\n "
1126+ }
1127+ };
1128+
1129+ } // namespace
1130+
9901131// ===----------------------------------------------------------------------===//
9911132// MARK: GatherLexicalLifetimeUseVisitor
9921133// ===----------------------------------------------------------------------===//
@@ -1892,6 +2033,19 @@ bool MoveOnlyAddressCheckerPImpl::performSingleCheck(
18922033 return false ;
18932034 }
18942035
2036+ // Before we do anything, convert any load_borrow + copy_value into load
2037+ // [copy] + begin_borrow for further processing.
2038+ {
2039+ CopiedLoadBorrowEliminationVisitor copiedLoadBorrowEliminator (fn);
2040+ if (!visitAccessPathBaseUses (copiedLoadBorrowEliminator, accessPathWithBase,
2041+ fn)) {
2042+ LLVM_DEBUG (llvm::dbgs () << " Failed copied load borrow eliminator visit: "
2043+ << *markedAddress);
2044+ return false ;
2045+ }
2046+ copiedLoadBorrowEliminator.process ();
2047+ }
2048+
18952049 // Then gather all uses of our address by walking from def->uses. We use this
18962050 // to categorize the uses of this address into their ownership behavior (e.x.:
18972051 // init, reinit, take, destroy, etc.).
0 commit comments