[CoroutineAccessors] Use retcon.once variant.

Allocate a coroutine frame in the caller based on the size in the
corresponding "function pointer" and pass it along with an allocator to
the callee.

Author: nate-chandler

include/swift/ABI/Coro.h

@@ -0,0 +1,80 @@
+//===---------- Coro.h - ABI structures for coroutines ----------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// Swift ABI describing the coroutine ABI.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_ABI_CORO_H
+#define SWIFT_ABI_CORO_H
+
+#include "swift/Basic/FlagSet.h"
+#include <cstddef>
+#include <cstdint>
+
+namespace swift {
+
+enum class CoroAllocatorKind : uint8_t {
+  // stacksave/stackrestore
+  Sync = 0,
+  // swift_task_alloc/swift_task_dealloc_through
+  Async = 1,
+  // malloc/free
+  Malloc = 2,
+};
+
+class CoroAllocatorFlags : public FlagSet<uint32_t> {
+public:
+  // clang-format off
+  enum {
+    Kind           = 0,
+    Kind_width     = 8,
+
+    // 24 reserved bits
+  };
+  // clang-format on
+
+  constexpr explicit CoroAllocatorFlags(uint32_t bits) : FlagSet(bits) {}
+  CoroAllocatorFlags(CoroAllocatorKind kind) { setKind(kind); }
+  constexpr CoroAllocatorFlags() {}
+
+  FLAGSET_DEFINE_FIELD_ACCESSORS(Kind, Kind_width, CoroAllocatorKind, getKind,
+                                 setKind)
+};
+
+using CoroAllocation = void *;
+using CoroAllocateFn = CoroAllocation (*)(size_t);
+using CoroDealllocateFn = void (*)(CoroAllocation);
+
+struct CoroAllocator {
+  CoroAllocatorFlags flags;
+  CoroAllocateFn allocate;
+  CoroDealllocateFn deallocate;
+
+  /// Whether the allocator should deallocate memory on calls to
+  /// swift_coro_dealloc.
+  constexpr bool shouldDeallocateImmediately() {
+    // Only the "mallocator" should immediately deallocate in
+    // swift_coro_dealloc.  It must because the ABI does not provide a means for
+    // the callee to return its allocations to the caller.
+    //
+    // Async allocations can be deferred until the first non-coroutine caller
+    // from where swift_task_dealloc_through can be called and passed the
+    // caller-allocated fixed-per-callee-sized-frame.
+    // Stack allocations just pop the stack.
+    return flags.getKind() == CoroAllocatorKind::Malloc;
+  }
+};
+
+} // end namespace swift
+
+#endif

include/swift/AST/IRGenOptions.h

@@ -386,6 +386,7 @@ class IRGenOptions {
   unsigned LazyInitializeClassMetadata : 1;
   unsigned LazyInitializeProtocolConformances : 1;
   unsigned IndirectAsyncFunctionPointer : 1;
+  unsigned IndirectCoroFunctionPointer : 1;
 
   /// Use absolute function references instead of relative ones.
   /// Mainly used on WebAssembly, that doesn't support relative references
@@ -580,10 +581,10 @@ class IRGenOptions {
         HasValueNamesSetting(false), ValueNames(false),
         ReflectionMetadata(ReflectionMetadataMode::Runtime),
         EnableReflectionNames(true), DisableLLVMMergeFunctions(false),
-        EnableAnonymousContextMangledNames(false),
-        ForcePublicLinkage(false), LazyInitializeClassMetadata(false),
+        EnableAnonymousContextMangledNames(false), ForcePublicLinkage(false),
+        LazyInitializeClassMetadata(false),
         LazyInitializeProtocolConformances(false),
-        IndirectAsyncFunctionPointer(false),
+        IndirectAsyncFunctionPointer(false), IndirectCoroFunctionPointer(false),
         CompactAbsoluteFunctionPointer(false), DisableLegacyTypeInfo(false),
         PrespecializeGenericMetadata(false), UseIncrementalLLVMCodeGen(true),
         UseTypeLayoutValueHandling(true), ForceStructTypeLayouts(false),
@@ -598,16 +599,15 @@ class IRGenOptions {
         EnableGlobalISel(false), VirtualFunctionElimination(false),
         WitnessMethodElimination(false), ConditionalRuntimeRecords(false),
         InternalizeAtLink(false), InternalizeSymbols(false),
-        MergeableSymbols(false),
-        EmitGenericRODatas(true), NoPreallocatedInstantiationCaches(false),
+        MergeableSymbols(false), EmitGenericRODatas(true),
+        NoPreallocatedInstantiationCaches(false),
         DisableReadonlyStaticObjects(false), CollocatedMetadataFunctions(false),
         ColocateTypeDescriptors(true), UseRelativeProtocolWitnessTables(false),
         UseFragileResilientProtocolWitnesses(false), EnableHotColdSplit(false),
         EmitAsyncFramePushPopMetadata(true), EmitTypeMallocForCoroFrame(false),
         EmitYieldOnce2AsYieldOnce(true), AsyncFramePointerAll(false),
-        UseProfilingMarkerThunks(false),
-        DebugInfoForProfiling(false), CmdArgs(),
-        SanitizeCoverage(llvm::SanitizerCoverageOptions()),
+        UseProfilingMarkerThunks(false), DebugInfoForProfiling(false),
+        CmdArgs(), SanitizeCoverage(llvm::SanitizerCoverageOptions()),
         TypeInfoFilter(TypeInfoDumpFilter::All),
         PlatformCCallingConvention(llvm::CallingConv::C), UseCASBackend(false),
         CASObjMode(llvm::CASBackendMode::Native) {

include/swift/IRGen/Linking.h

@@ -389,6 +389,11 @@ class LinkEntity {
     /// The pointer is an AbstractFunctionDecl*.
     AsyncFunctionPointerAST,
 
+    /// The same as CoroFunctionPointer but with a different stored value, for
+    /// use by TBDGen.
+    /// The pointer is an AbstractFunctionDecl*.
+    CoroFunctionPointerAST,
+
     /// The pointer is a SILFunction*.
     DynamicallyReplaceableFunctionKey,
 
@@ -558,11 +563,6 @@ class LinkEntity {
     /// The pointer is a SILFunction*.
     CoroFunctionPointer,
 
-    /// The same as CoroFunctionPointer but with a different stored value, for
-    /// use by TBDGen.
-    /// The pointer is an AbstractFunctionDecl*.
-    CoroFunctionPointerAST,
-
     /// An coro function pointer for a method dispatch thunk.  The pointer is
     /// a FuncDecl* inside a protocol or a class.
     DispatchThunkCoroFunctionPointer,
@@ -610,9 +610,7 @@ class LinkEntity {
     return !(LHS == RHS);
   }
 
-  static bool isDeclKind(Kind k) {
-    return k <= Kind::AsyncFunctionPointerAST;
-  }
+  static bool isDeclKind(Kind k) { return k <= Kind::CoroFunctionPointerAST; }
   static bool isTypeKind(Kind k) {
     return k >= Kind::ProtocolWitnessTableLazyAccessFunction;
   }
@@ -1544,6 +1542,17 @@ class LinkEntity {
     return entity;
   }
 
+  static LinkEntity forCoroFunctionPointer(SILDeclRef declRef) {
+    LinkEntity entity;
+    entity.setForDecl(declRef.isDistributedThunk()
+                          ? Kind::DistributedThunkCoroFunctionPointer
+                          : Kind::CoroFunctionPointerAST,
+                      declRef.getAbstractFunctionDecl());
+    entity.SecondaryPointer =
+        reinterpret_cast<void *>(static_cast<uintptr_t>(declRef.kind));
+    return entity;
+  }
+
   LinkEntity getUnderlyingEntityForCoroFunctionPointer() const {
     LinkEntity entity;
     entity.Pointer = Pointer;

include/swift/Runtime/Concurrency.h

@@ -18,6 +18,7 @@
 #define SWIFT_RUNTIME_CONCURRENCY_H
 
 #include "swift/ABI/AsyncLet.h"
+#include "swift/ABI/Coro.h"
 #include "swift/ABI/Task.h"
 #include "swift/ABI/TaskGroup.h"
 
@@ -119,6 +120,35 @@ void *swift_task_alloc(size_t size);
 SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift)
 void swift_task_dealloc(void *ptr);
 
+/// Deallocate multiple memory allocations in a task.
+///
+/// The pointer provided must be a pointer previously allocated on
+/// this task that has not yet been deallocated.  All allocations up to and
+/// including that allocation will be deallocated.
+SWIFT_EXPORT_FROM(swift_Concurrency)
+SWIFT_CC(swift) void swift_task_dealloc_through(void *ptr);
+
+// TODO: CoroutineAccessors: Eliminate this entry point and replace its uses
+//                           with direct references the globals.
+SWIFT_EXPORT_FROM(swift_Concurrency)
+SWIFT_CC(swift)
+CoroAllocator *swift_coro_getGlobalAllocator(CoroAllocatorFlags flags);
+
+// TODO: CoroutineAccessors: Mark the underlying struct const.
+SWIFT_EXPORT_FROM(swift_Concurrency)
+CoroAllocator *const _swift_coro_task_allocator;
+
+// TODO: CoroutineAccessors: Move these declarations back to swiftCore {{
+SWIFT_EXPORT_FROM(swift_Concurrency)
+SWIFT_CC(swift) void *swift_coro_alloc(CoroAllocator *allocator, size_t size);
+SWIFT_EXPORT_FROM(swift_Concurrency)
+SWIFT_CC(swift) void swift_coro_dealloc(CoroAllocator *allocator, void *ptr);
+
+// TODO: CoroutineAccessors: Mark the underlying struct const.
+SWIFT_EXPORT_FROM(swift_Concurrency)
+CoroAllocator *const _swift_coro_malloc_allocator;
+// }} TODO: CoroutineAccessors
+
 /// Cancel a task and all of its child tasks.
 ///
 /// This can be called from any thread.

include/swift/Runtime/Coro.h

@@ -0,0 +1,26 @@
+//===-------- Coro.h - Runtime interface for coroutines ---------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// The runtime interface for coroutines.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_RUNTIME_CORO_H
+#define SWIFT_RUNTIME_CORO_H
+
+#include "swift/ABI/Coro.h"
+#include "swift/Runtime/Config.h"
+#include <cstddef>
+
+namespace swift {} // end namespace swift
+
+#endif

include/swift/Runtime/RuntimeFunctions.def

@@ -2325,6 +2325,16 @@ FUNCTION(TaskDealloc,
          EFFECT(Concurrency),
          MEMEFFECTS(ArgMemOnly))
 
+// void swift_task_dealloc_through(void *ptr);
+FUNCTION(TaskDeallocThrough,
+         _Concurrency, swift_task_dealloc_through, SwiftCC,
+         CoroutineAccessorsAvailability,
+         RETURNS(VoidTy),
+         ARGS(Int8PtrTy),
+         ATTRS(NoUnwind),
+         EFFECT(Concurrency),
+         MEMEFFECTS(ArgMemOnly))
+
 // void swift_task_cancel(AsyncTask *task);
 FUNCTION(TaskCancel,
          _Concurrency, swift_task_cancel, SwiftCC,
@@ -2989,6 +2999,36 @@ FUNCTION(MemsetS, c, memset_s, C_CC, AlwaysAvailable,
          EFFECT(NoEffect),
          UNKNOWN_MEMEFFECTS)
 
+// void *swift_coro_alloc(CoroAllocator *, size_t size);
+FUNCTION(CoroAlloc,
+         _Concurrency, swift_coro_alloc, SwiftCC,
+         CoroutineAccessorsAvailability,
+         RETURNS(Int8PtrTy),
+         ARGS(CoroAllocatorPtrTy, SizeTy),
+         NO_ATTRS,
+         EFFECT(Allocating, Concurrency),
+         UNKNOWN_MEMEFFECTS)
+
+// void swift_coro_dealloc(CoroAllocator *, void *ptr);
+FUNCTION(CoroDealloc,
+         _Concurrency, swift_coro_dealloc, SwiftCC,
+         CoroutineAccessorsAvailability,
+         RETURNS(VoidTy),
+         ARGS(CoroAllocatorPtrTy, Int8PtrTy),
+         NO_ATTRS,
+         EFFECT(Deallocating, Concurrency),
+         UNKNOWN_MEMEFFECTS)
+
+// const CoroAllocator *swift_coro_getGlobalAllocator(CoroAllocatorFlags flags);
+FUNCTION(CoroGetGlobalAllocator, 
+         _Concurrency, swift_coro_getGlobalAllocator, SwiftCC, 
+         CoroutineAccessorsAvailability, 
+         RETURNS(CoroAllocatorPtrTy), 
+         ARGS(CoroAllocatorFlagsTy), 
+         ATTRS(NoUnwind, WillReturn), 
+         EFFECT(Allocating),
+         UNKNOWN_MEMEFFECTS)
+
 #undef RETURNS
 #undef ARGS
 #undef ATTRS

include/swift/SIL/SILFunction.h

@@ -768,6 +768,10 @@ class SILFunction
 
   bool isAsync() const { return LoweredType->isAsync(); }
 
+  bool isCalleeAllocatedCoroutine() const {
+    return LoweredType->isCalleeAllocatedCoroutine();
+  }
+
   /// Returns the calling convention used by this entry point.
   SILFunctionTypeRepresentation getRepresentation() const {
     return getLoweredFunctionType()->getRepresentation();

include/swift/SIL/SILSymbolVisitor.h

@@ -100,6 +100,7 @@ class SILSymbolVisitor {
   virtual void addAsyncFunctionPointer(SILDeclRef declRef) {}
   virtual void addBaseConformanceDescriptor(BaseConformance BC) {}
   virtual void addClassMetadataBaseOffset(ClassDecl *CD) {}
+  virtual void addCoroFunctionPointer(SILDeclRef declRef) {}
   virtual void addDispatchThunk(SILDeclRef declRef) {}
   virtual void addDynamicFunction(AbstractFunctionDecl *AFD,
                                   DynamicKind dynKind) {}

lib/Frontend/CompilerInvocation.cpp

@@ -3431,6 +3431,11 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
   // AsyncFunctionPointer access.
   Opts.IndirectAsyncFunctionPointer = Triple.isOSBinFormatCOFF();
 
+  // PE/COFF cannot deal with the cross-module reference to the
+  // CoroFunctionPointer data block.  Force the use of indirect
+  // CoroFunctionPointer access.
+  Opts.IndirectCoroFunctionPointer = Triple.isOSBinFormatCOFF();
+
   // On some Harvard architectures that allow sliding code and data address space
   // offsets independently, it's impossible to make direct relative reference to
   // code from data because the relative offset between them is not representable.

lib/IRGen/CallEmission.h

@@ -72,6 +72,10 @@ class CallEmission {
   /// Whether this is a coroutine invocation.
   bool IsCoroutine;
 
+  /// Whether this is a invocation for a coroutine that dynamically allocates
+  /// in the callee.
+  bool IsCalleeAllocatedCoroutine;
+
   /// Whether we've emitted the call for the current callee yet.  This
   /// is just for debugging purposes --- e.g. the destructor asserts
   /// that it's true --- but is otherwise derivable from
@@ -178,6 +182,9 @@ class CallEmission {
   virtual llvm::Value *getResumeFunctionPointer() = 0;
   virtual llvm::Value *getAsyncContext() = 0;
 
+  virtual StackAddress getCoroStaticFrame() = 0;
+  virtual llvm::Value *getCoroAllocator() = 0;
+
   void setIndirectTypedErrorResultSlot(llvm::Value *addr) {
     Args[IndirectTypedErrorArgIdx] = addr;
   }