[irgen] Implement support for alloc_stack non_nested.

SwiftCompilerSources/Sources/SIL/Utilities/Verifier.swift

@@ -22,7 +22,7 @@ private func require(_ condition: Bool, _ message: @autoclosure () -> String, at
   if !condition {
     let msg = message()
     msg._withBridgedStringRef { stringRef in
-      BridgedVerifier.verifierError(stringRef, atInstruction.bridged, Optional<Argument>.none.bridged)
+      BridgedVerifier.verifierError(stringRef, atInstruction.bridged)
     }
   }
 }

include/swift/SIL/SILBridging.h

@@ -1627,8 +1627,12 @@ struct BridgedVerifier {
   static void runSwiftFunctionVerification(swift::SILFunction * _Nonnull f, swift::SILContext * _Nonnull context);
 
   static void registerVerifier(VerifyFunctionFn verifyFunctionFn);
-  static void verifierError(BridgedStringRef message, OptionalBridgedInstruction atInstruction,
+  static void verifierError(BridgedStringRef message,
+                            OptionalBridgedInstruction atInstruction);
+  static void verifierError(BridgedStringRef message,
                             OptionalBridgedArgument atArgument);
+  static void verifierError(BridgedStringRef message,
+                            OptionalBridgedValue atValue);
 };
 
 struct BridgedUtilities {

include/swift/SIL/SILCloner.h

@@ -1062,6 +1062,7 @@ SILCloner<ImplClass>::visitAllocStackInst(AllocStackInst *Inst) {
       true
 #endif
   );
+  NewInst->setStackAllocationIsNested(Inst->isStackAllocationNested());
   recordClonedInstruction(Inst, NewInst);
 }
 

include/swift/SIL/SILInstruction.h

@@ -141,6 +141,24 @@ class SILPrintContext;
 
 template <typename ImplClass> class SILClonerWithScopes;
 
+/// An enum that describes whether or not a stack allocation may violate the
+/// stack discipline of swift.
+///
+/// DISCUSSION: In swift, we require that all allocations on the stack be
+/// strictly allocated in a LIFO order... that is the last stack allocation
+/// created must be the first stack allocation destroyed. In some cases, we
+/// cannot guarantee that behavior. In such cases we may need to use other
+/// strategies that do not involve stack memory (e.x.: using heap memory
+/// although we do not require heap memory to be used).
+enum StackAllocationIsNested_t : bool {
+  /// The instruction may not obey the LIFO rule of stack allocation.
+  StackAllocationIsNotNested = false,
+
+  /// The instruction obeys the LIFO rule of stack allocation and can allocate
+  /// memory on the stack normally.
+  StackAllocationIsNested = true,
+};
+
 enum class MemoryBehavior {
   None,
   /// The instruction may read memory.
@@ -858,6 +876,15 @@ class SILInstruction : public llvm::ilist_node<SILInstruction> {
   /// The stack allocation produced by the instruction, if any.
   SILValue getStackAllocation() const;
 
+  /// Returns the kind of stack memory that should be allocated. There are
+  /// certain (unfortunate) situations in which "stack" allocations may become
+  /// unnested and must use alternative allocation strategies. Rather than
+  /// requiring all of these to explicitly use heap allocation, which may be
+  /// to be significantly less efficient (e.g. )
+  StackAllocationIsNested_t isStackAllocationNested() const;
+
+  void setStackAllocationIsNested(StackAllocationIsNested_t isNested);
+
   /// Returns true if this is the deallocation of a stack allocating instruction.
   /// The first operand must be the allocating instruction.
   bool isDeallocatingStack() const;
@@ -2055,6 +2082,24 @@ class AllocStackInst final
     }
   }
 
+  StackAllocationIsNested_t isStackAllocationNested() const {
+    if (sharedUInt8().AllocStackInst.isNested) {
+      return StackAllocationIsNested;
+    }
+    return StackAllocationIsNotNested;
+  }
+
+  void setStackAllocationIsNested(StackAllocationIsNested_t isNested) {
+    switch (isNested) {
+    case StackAllocationIsNotNested:
+      sharedUInt8().AllocStackInst.isNested = false;
+      break;
+    case StackAllocationIsNested:
+      sharedUInt8().AllocStackInst.isNested = true;
+      break;
+    }
+  }
+
   void markUsesMoveableValueDebugInfo() {
     sharedUInt8().AllocStackInst.usesMoveableValueDebugInfo =
         (bool)UsesMoveableValueDebugInfo;

include/swift/SIL/SILModule.h

@@ -1161,10 +1161,18 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const SILModule &M){
   return OS;
 }
 
-void verificationFailure(const Twine &complaint,
-              const SILInstruction *atInstruction,
-              const SILArgument *atArgument,
-              llvm::function_ref<void(SILPrintContext &ctx)> extraContext);
+void verificationFailure(
+    const Twine &complaint, const SILFunction *fn,
+    llvm::function_ref<void(SILPrintContext &ctx)> extraContext);
+void verificationFailure(
+    const Twine &complaint, const SILInstruction *atInstruction,
+    llvm::function_ref<void(SILPrintContext &ctx)> extraContext);
+void verificationFailure(
+    const Twine &complaint, const SILArgument *atArgument,
+    llvm::function_ref<void(SILPrintContext &ctx)> extraContext);
+void verificationFailure(
+    const Twine &complaint, SILValue atValue,
+    llvm::function_ref<void(SILPrintContext &ctx)> extraContext);
 
 inline bool SILOptions::supportsLexicalLifetimes(const SILModule &mod) const {
   switch (mod.getStage()) {

include/swift/SIL/SILNode.h

@@ -229,7 +229,8 @@ class alignas(8) SILNode :
                  lexical : 1,
                  fromVarDecl : 1,
                  usesMoveableValueDebugInfo : 1,
-                 hasInvalidatedVarInfo : 1);
+                 hasInvalidatedVarInfo : 1,
+                 isNested : 1);
 
     SHARED_FIELD(AllocBoxInst, uint8_t
                  dynamicLifetime : 1,

lib/IRGen/FixedTypeInfo.h

@@ -82,8 +82,12 @@ class FixedTypeInfo : public TypeInfo {
   }
 
   StackAddress allocateStack(IRGenFunction &IGF, SILType T,
-                             const llvm::Twine &name) const override;
-  void deallocateStack(IRGenFunction &IGF, StackAddress addr, SILType T) const override;
+                             const llvm::Twine &name,
+                             StackAllocationIsNested_t isNested =
+                                 StackAllocationIsNested) const override;
+  void deallocateStack(IRGenFunction &IGF, StackAddress addr, SILType T,
+                       StackAllocationIsNested_t isNested =
+                           StackAllocationIsNested) const override;
   void destroyStack(IRGenFunction &IGF, StackAddress addr, SILType T,
                     bool isOutlined) const override;
 

lib/IRGen/GenArray.cpp

@@ -638,24 +638,26 @@ class NonFixedArrayTypeInfo final
     return nullptr;
   }
 
-  StackAddress allocateStack(IRGenFunction &IGF, SILType T,
-                             const llvm::Twine &name) const override {
+  StackAddress
+  allocateStack(IRGenFunction &IGF, SILType T, const llvm::Twine &name,
+                StackAllocationIsNested_t isNested) const override {
     // Allocate memory on the stack.
-    auto alloca = IGF.emitDynamicAlloca(T, name);
+    auto alloca = IGF.emitDynamicStackAllocation(T, isNested, name);
     IGF.Builder.CreateLifetimeStart(alloca.getAddressPointer());
     return alloca.withAddress(getAddressForPointer(alloca.getAddressPointer()));
   }
 
-  void deallocateStack(IRGenFunction &IGF, StackAddress stackAddress,
-                       SILType T) const override {
+  void deallocateStack(IRGenFunction &IGF, StackAddress stackAddress, SILType T,
+                       StackAllocationIsNested_t isNested) const override {
     IGF.Builder.CreateLifetimeEnd(stackAddress.getAddress().getAddress());
-    IGF.emitDeallocateDynamicAlloca(stackAddress);
+    IGF.emitDynamicStackDeallocation(stackAddress, isNested);
   }
 
   void destroyStack(IRGenFunction &IGF, StackAddress stackAddress, SILType T,
                     bool isOutlined) const override {
     emitDestroyCall(IGF, T, stackAddress.getAddress());
-    deallocateStack(IGF, stackAddress, T);
+    // For now, just always do nested.
+    deallocateStack(IGF, stackAddress, T, StackAllocationIsNested);
   }
 
   TypeLayoutEntry *

lib/IRGen/GenInit.cpp

@@ -59,8 +59,9 @@ void IRGenModule::emitSILGlobalVariable(SILGlobalVariable *var) {
                      var->isDefinition() ? ForDefinition : NotForDefinition);
 }
 
-StackAddress FixedTypeInfo::allocateStack(IRGenFunction &IGF, SILType T,
-                                          const Twine &name) const {
+StackAddress
+FixedTypeInfo::allocateStack(IRGenFunction &IGF, SILType T, const Twine &name,
+                             StackAllocationIsNested_t isNested) const {
   // If the type is known to be empty, don't actually allocate anything.
   if (isKnownEmpty(ResilienceExpansion::Maximal)) {
     auto addr = getUndefAddress();
@@ -81,7 +82,8 @@ void FixedTypeInfo::destroyStack(IRGenFunction &IGF, StackAddress addr,
 }
 
 void FixedTypeInfo::deallocateStack(IRGenFunction &IGF, StackAddress addr,
-                                    SILType T) const {
+                                    SILType T,
+                                    StackAllocationIsNested_t isNested) const {
   if (isKnownEmpty(ResilienceExpansion::Maximal))
     return;
   IGF.Builder.CreateLifetimeEnd(addr.getAddress(), getFixedSize());

lib/IRGen/GenOpaque.cpp

@@ -553,6 +553,38 @@ irgen::emitInitializeBufferWithCopyOfBufferCall(IRGenFunction &IGF,
   return call;
 }
 
+StackAddress IRGenFunction::emitDynamicStackAllocation(
+    SILType T, StackAllocationIsNested_t isNested, const llvm::Twine &name) {
+  if (isNested) {
+    return emitDynamicAlloca(T, name);
+  }
+
+  // First malloc the memory.
+  auto mallocFn = IGM.getMallocFunctionPointer();
+  auto *size = emitLoadOfSize(*this, T);
+  auto *call = Builder.CreateCall(mallocFn, {size});
+  call->setDoesNotThrow();
+  call->setCallingConv(IGM.C_CC);
+
+  // Then create the dynamic alloca to store the malloc pointer into.
+
+  // TODO: Alignment should be platform specific.
+  auto address = Address(call, IGM.Int8Ty, Alignment(16));
+  return StackAddress{address};
+}
+
+void IRGenFunction::emitDynamicStackDeallocation(
+    StackAddress address, StackAllocationIsNested_t isNested) {
+  if (isNested) {
+    return emitDeallocateDynamicAlloca(address);
+  }
+
+  auto *call = Builder.CreateCall(IGM.getFreeFunctionPointer(),
+                                  {address.getAddressPointer()});
+  call->setDoesNotThrow();
+  call->setCallingConv(IGM.C_CC);
+}
+
 /// Emit a dynamic alloca call to allocate enough memory to hold an object of
 /// type 'T' and an optional llvm.stackrestore point if 'isInEntryBlock' is
 /// false.

lib/IRGen/GenType.cpp

@@ -1373,12 +1373,13 @@ namespace {
     llvm::Constant *getStaticStride(IRGenModule &IGM) const override {
       return nullptr;
     }
-    StackAddress allocateStack(IRGenFunction &IGF, SILType T,
-                               const llvm::Twine &name) const override {
+    StackAddress
+    allocateStack(IRGenFunction &IGF, SILType T, const llvm::Twine &name,
+                  StackAllocationIsNested_t isNested) const override {
       llvm_unreachable("should not call on an immovable opaque type");
     }
-    void deallocateStack(IRGenFunction &IGF, StackAddress addr,
-                         SILType T) const override {
+    void deallocateStack(IRGenFunction &IGF, StackAddress addr, SILType T,
+                         StackAllocationIsNested_t isNested) const override {
       llvm_unreachable("should not call on an immovable opaque type");
     }
     void destroyStack(IRGenFunction &IGF, StackAddress addr, SILType T,

lib/IRGen/IRGenFunction.h

@@ -25,6 +25,7 @@
 #include "swift/AST/ReferenceCounting.h"
 #include "swift/AST/Type.h"
 #include "swift/Basic/LLVM.h"
+#include "swift/SIL/SILInstruction.h"
 #include "swift/SIL/SILLocation.h"
 #include "swift/SIL/SILType.h"
 #include "llvm/ADT/DenseMap.h"
@@ -317,6 +318,12 @@ class IRGenFunction {
                                    bool useTaskDeallocThrough = false,
                                    bool forCalleeCoroutineFrame = false);
 
+  StackAddress emitDynamicStackAllocation(SILType type,
+                                          StackAllocationIsNested_t isNested,
+                                          const llvm::Twine &name = "");
+  void emitDynamicStackDeallocation(StackAddress address,
+                                    StackAllocationIsNested_t isNested);
+
   llvm::BasicBlock *createBasicBlock(const llvm::Twine &Name);
   const TypeInfo &getTypeInfoForUnlowered(Type subst);
   const TypeInfo &getTypeInfoForUnlowered(AbstractionPattern orig, Type subst);

lib/IRGen/IRGenSIL.cpp

@@ -882,6 +882,14 @@ class IRGenSILFunction :
     return isTaskAlloc;
   }
 
+  static bool isCallToMalloc(llvm::Value *val) {
+    auto *call = dyn_cast<llvm::CallInst>(val);
+    if (!call)
+      return false;
+    auto *callee = call->getCalledFunction();
+    return callee && callee->getName() == "malloc";
+  }
+
   static bool isTaskAlloc(llvm::Value *Storage) {
     while (Storage) {
       if (auto *LdInst = dyn_cast<llvm::LoadInst>(Storage))
@@ -6518,7 +6526,8 @@ void IRGenSILFunction::emitDebugInfoAfterAllocStack(AllocStackInst *i,
 
   // At this point addr must be an alloca or an undef.
   assert(isa<llvm::AllocaInst>(addr) || isa<llvm::UndefValue>(addr) ||
-         isa<llvm::IntrinsicInst>(addr) || isCallToSwiftTaskAlloc(addr));
+         isa<llvm::IntrinsicInst>(addr) || isCallToSwiftTaskAlloc(addr) ||
+         isCallToMalloc(addr));
 
   auto Indirection = DirectValue;
   if (InCoroContext(*CurSILFn, *i))
@@ -6575,7 +6584,8 @@ void IRGenSILFunction::visitAllocStackInst(swift::AllocStackInst *i) {
   DebugTypeInfo DbgTy;
   emitDebugInfoBeforeAllocStack(i, type, DbgTy);
 
-  auto stackAddr = type.allocateStack(*this, i->getElementType(), dbgname);
+  auto stackAddr = type.allocateStack(*this, i->getElementType(), dbgname,
+                                      i->isStackAllocationNested());
   setLoweredStackAddress(i, stackAddr);
   Address addr = stackAddr.getAddress();
 
@@ -6668,23 +6678,27 @@ void IRGenSILFunction::visitDeallocStackInst(swift::DeallocStackInst *i) {
   if (auto *closure = dyn_cast<PartialApplyInst>(i->getOperand())) {
     assert(closure->isOnStack());
     auto stackAddr = LoweredPartialApplyAllocations[i->getOperand()];
-    emitDeallocateDynamicAlloca(stackAddr);
+    emitDeallocateDynamicAlloca(stackAddr, closure->isStackAllocationNested());
     return;
   }
   if (isaResultOf<BeginApplyInst>(i->getOperand())) {
     auto *mvi = getAsResultOf<BeginApplyInst>(i->getOperand());
     auto *bai = cast<BeginApplyInst>(mvi->getParent());
+    // FIXME: [non_nested]
     const auto &coroutine = getLoweredCoroutine(bai->getTokenResult());
     emitDeallocYieldOnce2CoroutineFrame(*this,
                                         coroutine.getCalleeAllocatedFrame());
     return;
   }
 
-  auto allocatedType = i->getOperand()->getType();
+  auto *asi = cast<AllocStackInst>(i->getOperand());
+
+  auto allocatedType = asi->getType();
   const TypeInfo &allocatedTI = getTypeInfo(allocatedType);
-  StackAddress stackAddr = getLoweredStackAddress(i->getOperand());
+  StackAddress stackAddr = getLoweredStackAddress(asi);
+  auto isNested = asi->isStackAllocationNested();
 
-  allocatedTI.deallocateStack(*this, stackAddr, allocatedType);
+  allocatedTI.deallocateStack(*this, stackAddr, allocatedType, isNested);
 }
 
 void IRGenSILFunction::visitDeallocStackRefInst(DeallocStackRefInst *i) {

lib/IRGen/NonFixedTypeInfo.h

@@ -63,18 +63,21 @@ class WitnessSizedTypeInfo : public IndirectTypeInfo<Impl, TypeInfo> {
   static bool isFixed() { return false; }
 
   StackAddress allocateStack(IRGenFunction &IGF, SILType T,
-                             const llvm::Twine &name) const override {
+                             const llvm::Twine &name,
+                             StackAllocationIsNested_t isNested =
+                                 StackAllocationIsNested) const override {
     // Allocate memory on the stack.
-    auto alloca = IGF.emitDynamicAlloca(T, name);
+    auto alloca = IGF.emitDynamicStackAllocation(T, isNested, name);
     IGF.Builder.CreateLifetimeStart(alloca.getAddressPointer());
     return alloca.withAddress(
              getAsBitCastAddress(IGF, alloca.getAddressPointer()));
   }
 
-  void deallocateStack(IRGenFunction &IGF, StackAddress stackAddress,
-                       SILType T) const override {
+  void deallocateStack(IRGenFunction &IGF, StackAddress stackAddress, SILType T,
+                       StackAllocationIsNested_t isNested =
+                           StackAllocationIsNested) const override {
     IGF.Builder.CreateLifetimeEnd(stackAddress.getAddress().getAddress());
-    IGF.emitDeallocateDynamicAlloca(stackAddress);
+    IGF.emitDynamicStackDeallocation(stackAddress, isNested);
   }
 
   void destroyStack(IRGenFunction &IGF, StackAddress stackAddress, SILType T,

lib/IRGen/TypeInfo.h

@@ -28,6 +28,7 @@
 #include "IRGen.h"
 #include "Outlining.h"
 #include "swift/AST/ReferenceCounting.h"
+#include "swift/SIL/SILInstruction.h"
 #include "llvm/ADT/MapVector.h"
 
 namespace llvm {
@@ -354,12 +355,14 @@ class TypeInfo {
                         bool useStructLayouts) const = 0;
 
   /// Allocate a variable of this type on the stack.
-  virtual StackAddress allocateStack(IRGenFunction &IGF, SILType T,
-                                     const llvm::Twine &name) const = 0;
+  virtual StackAddress allocateStack(
+      IRGenFunction &IGF, SILType T, const llvm::Twine &name,
+      StackAllocationIsNested_t isNested = StackAllocationIsNested) const = 0;
 
   /// Deallocate a variable of this type.
-  virtual void deallocateStack(IRGenFunction &IGF, StackAddress addr,
-                               SILType T) const = 0;
+  virtual void deallocateStack(
+      IRGenFunction &IGF, StackAddress addr, SILType T,
+      StackAllocationIsNested_t isNested = StackAllocationIsNested) const = 0;
 
   /// Destroy the value of a variable of this type, then deallocate its
   /// memory.