[move-only] Perform an exclusive borrow when passing a var to a consuming var.

Consider the following example:

```
class Klass {}

@_moveOnly struct Butt {
  var k = Klass()
}

func mixedUse(_: inout Butt, _: __owned Butt) {}

func foo() {
    var y = Butt()
    mixedUse(&y, y)
}
```

In this case, we want to have an exclusivity violation. Before this patch, we
did a by-value load [copy] of y and then performed the inout access. Since the
access scopes did not overlap, we would not get an exclusivity violation.
Additionally, since the checker assumes that exclusivity violations will be
caught in such a situation, we convert the load [copy] to a load [take] causing
a later memory lifetime violation as seen in the following SIL:

```
sil hidden [ossa] @$s4test3fooyyF : $@convention(thin) () -> () {
bb0:
  %0 = alloc_stack [lexical] $Butt, var, name "y" // users: %4, %5, %8, %12, %13
  %1 = metatype $@thin Butt.Type                  // user: %3
  // function_ref Butt.init()
  %2 = function_ref @$s4test4ButtVACycfC : $@convention(method) (@thin Butt.Type) -> @owned Butt // user: %3
  %3 = apply %2(%1) : $@convention(method) (@thin Butt.Type) -> @owned Butt // user: %4
  store %3 to [init] %0 : $*Butt                  // id: %4
  %5 = begin_access [modify] [static] %0 : $*Butt // users: %7, %6
  %6 = load [take] %5 : $*Butt                    // user: %10                // <————————— This was a load [copy].
  end_access %5 : $*Butt                          // id: %7
  %8 = begin_access [modify] [static] %0 : $*Butt // users: %11, %10
  // function_ref mixedUse2(_:_:)
  %9 = function_ref @$s4test9mixedUse2yyAA4ButtVz_ADntF : $@convention(thin) (@inout Butt, @owned Butt) -> () // user: %10
  %10 = apply %9(%8, %6) : $@convention(thin) (@inout Butt, @owned Butt) -> ()
  end_access %8 : $*Butt                          // id: %11
  destroy_addr %0 : $*Butt                        // id: %12
  dealloc_stack %0 : $*Butt                       // id: %13
  %14 = tuple ()                                  // user: %15
  return %14 : $()                                // id: %15
} // end sil function '$s4test3fooyyF'
```

Now, instead we create a [consume] access and get the nice exclusivity error we
are looking for.

NOTE: As part of this I needed to tweak the verifier so that [deinit] accesses
are now allowed to have any form of access enforcement before we are in
LoweredSIL. I left in the original verifier error in LoweredSIL and additionally
left in the original error in IRGen. The reason why I am doing this is that I
need the deinit access to represent semantically what consuming from a
ref_element_addr, global, or escaping mutable var look like at the SIL level so
that the move checker can error upon it. Since we will error upon such
consumptions in Canonical SIL, such code patterns will never actually hit
Lowered/IRGen SIL, so it is safe to do so (and the verifier/errors will help us
if we make any mistakes). In the case of a non-escaping var though, we will be
able to use deinit statically and the move checker will make sure that it is not
reused before it is reinitialized.

rdar://101767439

Author: gottesmm

include/swift/AST/StorageImpl.h

@@ -81,7 +81,7 @@ enum class AccessKind : uint8_t {
   Write,
 
   /// The access may require either reading or writing the current value.
-  ReadWrite
+  ReadWrite,
 };
 
 /// Produce the aggregate access kind of the combination of two accesses.

include/swift/AST/Types.h

@@ -3951,6 +3951,10 @@ class SILParameterInfo {
     return getConvention() == ParameterConvention::Indirect_In_Guaranteed;
   }
 
+  bool isIndirectIn() const {
+    return getConvention() == ParameterConvention::Indirect_In;
+  }
+
   bool isIndirectInOut() const {
     return getConvention() == ParameterConvention::Indirect_Inout;
   }

include/swift/SIL/SILBuilder.h

@@ -1777,6 +1777,24 @@ class SILBuilder {
   void emitDestructureAddressOperation(SILLocation loc, SILValue operand,
                                        SmallVectorImpl<SILValue> &result);
 
+  void emitDestructureAddressOperation(
+      SILLocation loc, SILValue operand,
+      function_ref<void(unsigned, SILValue)> result);
+
+  void emitDestructureOperation(SILLocation loc, SILValue operand,
+                                SmallVectorImpl<SILValue> &result) {
+    if (operand->getType().isAddress())
+      return emitDestructureAddressOperation(loc, operand, result);
+    return emitDestructureValueOperation(loc, operand, result);
+  }
+
+  void emitDestructureOperation(SILLocation loc, SILValue operand,
+                                function_ref<void(unsigned, SILValue)> result) {
+    if (operand->getType().isAddress())
+      return emitDestructureAddressOperation(loc, operand, result);
+    return emitDestructureValueOperation(loc, operand, result);
+  }
+
   ClassMethodInst *createClassMethod(SILLocation Loc, SILValue Operand,
                                      SILDeclRef Member, SILType MethodTy) {
     return insert(new (getModule()) ClassMethodInst(

lib/SIL/IR/SILBuilder.cpp

@@ -570,6 +570,25 @@ void SILBuilder::emitDestructureAddressOperation(
                   });
 }
 
+void SILBuilder::emitDestructureAddressOperation(
+    SILLocation loc, SILValue v,
+    function_ref<void(unsigned, SILValue)> results) {
+
+  // If we do not have a tuple or a struct, add to our results list.
+  SILType type = v->getType();
+  if (!(type.is<TupleType>() || type.getStructOrBoundGenericStruct())) {
+    return;
+  }
+
+  SmallVector<Projection, 16> projections;
+  Projection::getFirstLevelProjections(v->getType(), getModule(),
+                                       getTypeExpansionContext(), projections);
+  for (auto pair : llvm::enumerate(projections)) {
+    results(pair.index(),
+            pair.value().createAddressProjection(*this, loc, v).get());
+  }
+}
+
 void SILBuilder::emitDestructureValueOperation(
     SILLocation loc, SILValue operand,
     function_ref<void(unsigned, SILValue)> func) {

lib/SIL/Verifier/SILVerifier.cpp

@@ -2341,14 +2341,26 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
 
     switch (BAI->getAccessKind()) {
     case SILAccessKind::Init:
-    case SILAccessKind::Deinit:
       // A signed access preserves the access marker until IRGen
-      require(
-          BAI->getEnforcement() == SILAccessEnforcement::Static ||
-              BAI->getEnforcement() == SILAccessEnforcement::Signed,
-          "init/deinit accesses cannot use non-static/non-signed enforcement");
+      require(BAI->getEnforcement() == SILAccessEnforcement::Static ||
+                  BAI->getEnforcement() == SILAccessEnforcement::Signed,
+              "init accesses cannot use non-static/non-signed enforcement");
       break;
 
+    case SILAccessKind::Deinit:
+      // A signed access preserves the access marker until IRGen.
+      //
+      // NOTE: We allow for deinit to be non-static before Lowered SIL to allow
+      // for it to be used to represent deinit accesses for move only types
+      // stored in classes, globals, and escaping mutable captures. This is ok
+      // to do since even though we allow for them to be represented there, the
+      // move only checker passes will always emit an error for them implying
+      // that we will never get to LoweredSIL and codegen.
+      require(BAI->getEnforcement() == SILAccessEnforcement::Static ||
+                  BAI->getEnforcement() == SILAccessEnforcement::Signed ||
+                  BAI->getModule().getStage() != SILStage::Lowered,
+              "init accesses cannot use non-static/non-signed enforcement");
+      break;
     case SILAccessKind::Read:
     case SILAccessKind::Modify:
       break;
@@ -2407,10 +2419,22 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
 
     switch (BUAI->getAccessKind()) {
     case SILAccessKind::Init:
-    case SILAccessKind::Deinit:
       require(BUAI->getEnforcement() == SILAccessEnforcement::Static,
-              "init/deinit accesses cannot use non-static enforcement");
+              "init accesses cannot use non-static enforcement");
       break;
+    case SILAccessKind::Deinit:
+      // NOTE: We allow for deinit to be non-static before Lowered SIL to allow
+      // for it to be used to represent deinit accesses for move only types
+      // stored in classes, globals, and escaping mutable captures. This is ok
+      // to do since even though we allow for them to be represented there, the
+      // move only checker passes will always emit an error for them implying
+      // that we will never get to LoweredSIL and codegen.
+      require(
+          BUAI->getEnforcement() == SILAccessEnforcement::Static ||
+              BUAI->getModule().getStage() != SILStage::Lowered,
+          "deinit accesses cannot use non-static enforcement in Lowered SIL");
+      break;
+
     case SILAccessKind::Read:
     case SILAccessKind::Modify:
       break;

lib/SILGen/SILGenApply.cpp

@@ -2439,17 +2439,21 @@ class DelayedArgument {
 
     /// A string r-value needs to be converted to a pointer type.
     RValueStringToPointer,
-    
+
     /// A function conversion needs to occur.
     FunctionConversion,
-    
+
     LastRVKind = FunctionConversion,
-    
+
     /// This is an immutable borrow from an l-value.
     BorrowedLValue,
 
     /// A default argument that needs to be evaluated.
     DefaultArgument,
+
+    /// This is a consume of an l-value. It acts like a BorrowedLValue, but we
+    /// use a deinit access scope.
+    ConsumedLValue,
   };
 
 private:
@@ -2495,10 +2499,16 @@ class DelayedArgument {
     ClaimedParamsRef ParamsToEmit;
   };
 
+  struct ConsumedLValueStorage {
+    LValue LV;
+    SILLocation Loc;
+    AbstractionPattern OrigParamType;
+    ClaimedParamsRef ParamsToEmit;
+  };
+
   using ValueMembers =
-    ExternalUnionMembers<RValueStorage, LValueStorage,
-                         DefaultArgumentStorage,
-                         BorrowedLValueStorage>;
+      ExternalUnionMembers<RValueStorage, LValueStorage, DefaultArgumentStorage,
+                           BorrowedLValueStorage, ConsumedLValueStorage>;
   static ValueMembers::Index getValueMemberIndexForKind(KindTy kind) {
     switch (kind) {
     case InOut:
@@ -2513,6 +2523,8 @@ class DelayedArgument {
       return ValueMembers::indexOf<DefaultArgumentStorage>();
     case BorrowedLValue:
       return ValueMembers::indexOf<BorrowedLValueStorage>();
+    case ConsumedLValue:
+      return ValueMembers::indexOf<ConsumedLValueStorage>();
     }
     llvm_unreachable("bad kind");
   }
@@ -2586,13 +2598,18 @@ class DelayedArgument {
   }
 
   DelayedArgument(LValue &&lv, SILLocation loc,
-                  AbstractionPattern origResultType,
-                  ClaimedParamsRef params)
-    : Kind(BorrowedLValue) {
-    Value.emplaceAggregate<BorrowedLValueStorage>(Kind, std::move(lv), loc,
-                                                  origResultType, params);
+                  AbstractionPattern origResultType, ClaimedParamsRef params,
+                  bool isBorrowed = true)
+      : Kind(isBorrowed ? BorrowedLValue : ConsumedLValue) {
+    if (isBorrowed) {
+      Value.emplaceAggregate<BorrowedLValueStorage>(Kind, std::move(lv), loc,
+                                                    origResultType, params);
+    } else {
+      Value.emplaceAggregate<ConsumedLValueStorage>(Kind, std::move(lv), loc,
+                                                    origResultType, params);
+    }
   }
-  
+
   DelayedArgument(SILLocation loc,
                   ConcreteDeclRef defaultArgsOwner,
                   unsigned destIndex,
@@ -2654,6 +2671,10 @@ class DelayedArgument {
       emitBorrowedLValue(SGF, Value.get<BorrowedLValueStorage>(Kind),
                          args, argIndex);
       return;
+    case ConsumedLValue:
+      emitConsumedLValue(SGF, Value.get<ConsumedLValueStorage>(Kind), args,
+                         argIndex);
+      return;
     }
     llvm_unreachable("bad kind");
   }
@@ -2700,6 +2721,10 @@ class DelayedArgument {
                           SmallVectorImpl<ManagedValue> &args,
                           size_t &argIndex);
 
+  void emitConsumedLValue(SILGenFunction &SGF, ConsumedLValueStorage &info,
+                          SmallVectorImpl<ManagedValue> &args,
+                          size_t &argIndex);
+
   // (value, owner)
   std::pair<ManagedValue, ManagedValue>
   finishOriginalExpr(SILGenFunction &SGF, Expr *expr) {
@@ -2741,6 +2766,7 @@ class DelayedArgument {
     switch (Kind) {
     case InOut:
     case BorrowedLValue:
+    case ConsumedLValue:
     case DefaultArgument:
       llvm_unreachable("no original expr to finish in these cases");
 
@@ -3184,6 +3210,13 @@ class ArgEmitter {
         return;
     }
 
+    if (param.isConsumed()) {
+      if (tryEmitConsumedMoveOnly(std::move(arg), loweredSubstArgType,
+                                  loweredSubstParamType, origParamType,
+                                  paramSlice))
+        return;
+    }
+
     if (SGF.silConv.isSILIndirect(param)) {
       emitIndirect(std::move(arg), loweredSubstArgType, origParamType, param);
       return;
@@ -3426,6 +3459,55 @@ class ArgEmitter {
                         origParamType, claimedParams);
   }
 
+  bool tryEmitConsumedMoveOnly(ArgumentSource &&arg,
+                               SILType loweredSubstArgType,
+                               SILType loweredSubstParamType,
+                               AbstractionPattern origParamType,
+                               ClaimedParamsRef paramsSlice) {
+    assert(paramsSlice.size() == 1);
+
+    // Try to find an expression we can emit as a borrowed l-value.
+    auto lvExpr = std::move(arg).findStorageReferenceExprForMoveOnlyBorrow(SGF);
+    if (!lvExpr)
+      return false;
+
+    emitConsumed(lvExpr, loweredSubstArgType, loweredSubstParamType,
+                 origParamType, paramsSlice);
+
+    return true;
+  }
+
+  void emitConsumed(Expr *arg, SILType loweredSubstArgType,
+                    SILType loweredSubstParamType,
+                    AbstractionPattern origParamType,
+                    ClaimedParamsRef claimedParams) {
+    auto emissionKind = SGFAccessKind::OwnedAddressConsume;
+
+    LValue argLV = SGF.emitLValue(arg, emissionKind);
+
+    if (loweredSubstParamType.hasAbstractionDifference(Rep,
+                                                       loweredSubstArgType)) {
+      argLV.addSubstToOrigComponent(origParamType, loweredSubstParamType);
+    }
+
+    DelayedArguments.emplace_back(std::move(argLV), arg, origParamType,
+                                  claimedParams, false /*is borrowed*/);
+    Args.push_back(ManagedValue());
+  }
+
+  void emitExpandedConsumed(Expr *arg, AbstractionPattern origParamType) {
+    CanType substArgType = arg->getType()->getCanonicalType();
+    auto count = getFlattenedValueCount(origParamType, substArgType);
+    auto claimedParams = claimNextParameters(count);
+
+    SILType loweredSubstArgType = SGF.getLoweredType(substArgType);
+    SILType loweredSubstParamType =
+        SGF.getLoweredType(origParamType, substArgType);
+
+    return emitConsumed(arg, loweredSubstArgType, loweredSubstParamType,
+                        origParamType, claimedParams);
+  }
+
   void emitDirect(ArgumentSource &&arg, SILType loweredSubstArgType,
                   AbstractionPattern origParamType,
                   SILParameterInfo param) {
@@ -3806,6 +3888,66 @@ void DelayedArgument::emitBorrowedLValue(SILGenFunction &SGF,
   // That should drain all the parameters.
   assert(params.empty());
 }
+
+static void emitConsumedLValueRecursive(SILGenFunction &SGF, SILLocation loc,
+                                        ManagedValue value,
+                                        AbstractionPattern origParamType,
+                                        ClaimedParamsRef &params,
+                                        MutableArrayRef<ManagedValue> args,
+                                        size_t &argIndex) {
+  // Recurse into tuples.
+  if (origParamType.isTuple()) {
+    SGF.B.emitDestructureOperation(
+        loc, value, [&](unsigned eltIndex, ManagedValue eltValue) {
+          auto origEltType = origParamType.getTupleElementType(eltIndex);
+          // Recurse.
+          emitConsumedLValueRecursive(SGF, loc, eltValue, origEltType, params,
+                                      args, argIndex);
+        });
+    return;
+  }
+
+  // Claim the next parameter.
+  auto param = params.front();
+  params = params.slice(1);
+
+  // Load if necessary.
+  assert(param.isConsumed() && "Should have a consumed parameter?");
+  if (value.getType().isAddress()) {
+    if (!param.isIndirectIn() || !SGF.silConv.useLoweredAddresses()) {
+      value = SGF.B.createFormalAccessLoadTake(loc, value);
+    }
+  }
+
+  assert(param.getInterfaceType() == value.getType().getASTType());
+  args[argIndex++] = value;
+}
+
+void DelayedArgument::emitConsumedLValue(SILGenFunction &SGF,
+                                         ConsumedLValueStorage &info,
+                                         SmallVectorImpl<ManagedValue> &args,
+                                         size_t &argIndex) {
+  // Begin the access.
+  auto value = SGF.emitConsumedLValue(info.Loc, std::move(info.LV));
+  ClaimedParamsRef params = info.ParamsToEmit;
+
+  // We inserted exactly one space in the argument array, so fix that up
+  // to have the right number of spaces.
+  if (params.size() == 0) {
+    args.erase(args.begin() + argIndex);
+    return;
+  } else if (params.size() > 1) {
+    args.insert(args.begin() + argIndex + 1, params.size() - 1, ManagedValue());
+  }
+
+  // Recursively expand.
+  emitConsumedLValueRecursive(SGF, info.Loc, value, info.OrigParamType, params,
+                              args, argIndex);
+
+  // That should drain all the parameters.
+  assert(params.empty());
+}
+
 } // end anonymous namespace
 
 namespace {