Skip to content

Commit 454160b

Browse files
nate-chandlernate-chandler
authored
Merge pull request #84831 from nate-chandler/general-coro/20251009/1
[CoroutineAccessors] Sign de/allocation functions.
2 parents 5807cb7 + 603dc2e commit 454160b

File tree

11 files changed

+429
-222
lines changed

11 files changed

+429
-222
lines changed

‎include/swift/ABI/MetadataValues.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1793,6 +1793,10 @@ namespace SpecialPointerAuthDiscriminators {
17931793
/// IsCurrentGlobalActor function used between the Swift runtime and
17941794
/// concurrency runtime.
17951795
const uint16_t IsCurrentGlobalActorFunction = 0xd1b8; // = 53688
1796+
1797+
/// Function pointers stored in the coro allocator struct.
1798+
const uint16_t CoroAllocationFunction = 0x5f95; // = 24469
1799+
const uint16_t CoroDeallocationFunction = 0x9faf; // = 40879
17961800
}
17971801

17981802
/// The number of arguments that will be passed directly to a generic

‎include/swift/AST/IRGenOptions.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,12 @@ struct PointerAuthOptions : clang::PointerAuthOptions {
251251

252252
/// Like PartialApplyCapture but for use with CoroFunctionPointer values.
253253
PointerAuthSchema CoroPartialApplyCapture;
254+
255+
/// Stored in a coro allocator struct, the function used to allocate memory.
256+
PointerAuthSchema CoroAllocationFunction;
257+
258+
/// Stored in a coro allocator struct, the function used to deallocate memory.
259+
PointerAuthSchema CoroDeallocationFunction;
254260
};
255261

256262
enum class JITDebugArtifact : unsigned {

‎lib/IRGen/GenCall.cpp‎

Lines changed: 1 addition & 214 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
#include "EntryPointArgumentEmission.h"
4949
#include "Explosion.h"
5050
#include "GenCall.h"
51+
#include "GenCoro.h"
5152
#include "GenFunc.h"
5253
#include "GenHeap.h"
5354
#include "GenKeyPath.h"
@@ -5169,141 +5170,6 @@ void irgen::emitYieldManyCoroutineEntry(
51695170
allocFn, deallocFn, {});
51705171
}
51715172

5172-
static llvm::Constant *getCoroAllocFn(IRGenModule &IGM) {
5173-
auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
5174-
return IGM.getOrCreateHelperFunction(
5175-
"_swift_coro_alloc", IGM.Int8PtrTy, {IGM.CoroAllocatorPtrTy, IGM.SizeTy},
5176-
[isSwiftCoroCCAvailable](IRGenFunction &IGF) {
5177-
auto parameters = IGF.collectParameters();
5178-
auto *allocator = parameters.claimNext();
5179-
auto *size = parameters.claimNext();
5180-
if (isSwiftCoroCCAvailable) {
5181-
// swiftcorocc is available, so if there's no allocator pointer,
5182-
// allocate storage on the stack and return a pointer to it without
5183-
// popping the stack.
5184-
auto *nullAllocator = IGF.Builder.CreateCmp(
5185-
llvm::CmpInst::Predicate::ICMP_EQ, allocator,
5186-
llvm::ConstantPointerNull::get(
5187-
cast<llvm::PointerType>(allocator->getType())));
5188-
auto *poplessReturn = IGF.createBasicBlock("popless");
5189-
auto *normalReturn = IGF.createBasicBlock("normal");
5190-
IGF.Builder.CreateCondBr(nullAllocator, poplessReturn, normalReturn);
5191-
IGF.Builder.emitBlock(poplessReturn);
5192-
// Emit the dynamic alloca.
5193-
auto *alloca =
5194-
IGF.Builder.IRBuilderBase::CreateAlloca(IGF.IGM.Int8Ty, size);
5195-
alloca->setAlignment(llvm::Align(MaximumAlignment));
5196-
auto *retPopless = IGF.Builder.CreateIntrinsic(
5197-
IGF.IGM.VoidTy, llvm::Intrinsic::ret_popless, {});
5198-
retPopless->setTailCallKind(
5199-
llvm::CallInst::TailCallKind::TCK_MustTail);
5200-
IGF.Builder.CreateRet(alloca);
5201-
// Start emitting the "normal" block.
5202-
IGF.Builder.emitBlock(normalReturn);
5203-
}
5204-
auto *calleePtr = IGF.Builder.CreateInBoundsGEP(
5205-
IGF.IGM.CoroAllocatorTy, allocator,
5206-
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
5207-
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1)});
5208-
auto *callee = IGF.Builder.CreateLoad(
5209-
Address(calleePtr, IGF.IGM.PtrTy, IGF.IGM.getPointerAlignment()),
5210-
"allocate_fn");
5211-
auto fnPtr = FunctionPointer::createUnsigned(
5212-
FunctionPointer::Kind::Function, callee,
5213-
Signature(cast<llvm::FunctionType>(IGF.IGM.CoroAllocateFnTy), {},
5214-
IGF.IGM.SwiftCC));
5215-
auto *call = IGF.Builder.CreateCall(fnPtr, {size});
5216-
call->setDoesNotThrow();
5217-
call->setCallingConv(IGF.IGM.SwiftCC);
5218-
IGF.Builder.CreateRet(call);
5219-
},
5220-
/*setIsNoInline=*/true,
5221-
/*forPrologue=*/false,
5222-
/*isPerformanceConstraint=*/false,
5223-
/*optionalLinkageOverride=*/nullptr, IGM.SwiftCoroCC,
5224-
/*transformAttributes=*/
5225-
[&IGM](llvm::AttributeList &attrs) {
5226-
IGM.addSwiftCoroAttributes(attrs, 0);
5227-
});
5228-
}
5229-
5230-
static llvm::Constant *getCoroDeallocFn(IRGenModule &IGM) {
5231-
auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
5232-
return IGM.getOrCreateHelperFunction(
5233-
"_swift_coro_dealloc", IGM.VoidTy,
5234-
{IGM.CoroAllocatorPtrTy, IGM.Int8PtrTy},
5235-
[isSwiftCoroCCAvailable](IRGenFunction &IGF) {
5236-
auto parameters = IGF.collectParameters();
5237-
auto *allocator = parameters.claimNext();
5238-
auto *ptr = parameters.claimNext();
5239-
if (isSwiftCoroCCAvailable) {
5240-
// swiftcorocc is available, so if there's no allocator pointer,
5241-
// storage was allocated on the stack which will be naturally cleaned
5242-
// up when the coroutine's frame is "freed".
5243-
auto *nullAllocator = IGF.Builder.CreateCmp(
5244-
llvm::CmpInst::Predicate::ICMP_EQ, allocator,
5245-
llvm::ConstantPointerNull::get(
5246-
cast<llvm::PointerType>(allocator->getType())));
5247-
auto *bailBlock = IGF.createBasicBlock("null_allocator");
5248-
auto *normalBlock = IGF.createBasicBlock("nonnull_allocator");
5249-
IGF.Builder.CreateCondBr(nullAllocator, bailBlock, normalBlock);
5250-
IGF.Builder.emitBlock(bailBlock);
5251-
// Nothing to do here.
5252-
IGF.Builder.CreateRetVoid();
5253-
// Start emitting the "normal" block.
5254-
IGF.Builder.emitBlock(normalBlock);
5255-
}
5256-
auto shouldDeallocateImmediatelyFlag = CoroAllocatorFlags(0);
5257-
shouldDeallocateImmediatelyFlag.setShouldDeallocateImmediately(true);
5258-
auto *flagsPtr = IGF.Builder.CreateInBoundsGEP(
5259-
IGF.IGM.CoroAllocatorTy, allocator,
5260-
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
5261-
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0)});
5262-
auto *flags = IGF.Builder.CreateLoad(
5263-
Address(flagsPtr, IGF.IGM.Int32Ty, Alignment(4)), "");
5264-
auto *deallocDeferringAllocator = IGF.Builder.CreateAnd(
5265-
flags,
5266-
llvm::APInt(IGF.IGM.Int32Ty->getBitWidth(),
5267-
shouldDeallocateImmediatelyFlag.getOpaqueValue()));
5268-
auto *isDeallocDeferringAllocator = IGF.Builder.CreateICmpNE(
5269-
deallocDeferringAllocator,
5270-
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0));
5271-
auto *deferringAllocatorBlock =
5272-
IGF.createBasicBlock("deferring_allocator");
5273-
auto *normalBlock = IGF.createBasicBlock("normal");
5274-
IGF.Builder.CreateCondBr(isDeallocDeferringAllocator,
5275-
deferringAllocatorBlock, normalBlock);
5276-
IGF.Builder.emitBlock(deferringAllocatorBlock);
5277-
// Nothing to do here.
5278-
IGF.Builder.CreateRetVoid();
5279-
// Start emitting the "normal" block.
5280-
IGF.Builder.emitBlock(normalBlock);
5281-
auto *calleePtr = IGF.Builder.CreateInBoundsGEP(
5282-
IGF.IGM.CoroAllocatorTy, allocator,
5283-
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
5284-
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 2)});
5285-
auto *callee = IGF.Builder.CreateLoad(
5286-
Address(calleePtr, IGF.IGM.PtrTy, IGF.IGM.getPointerAlignment()),
5287-
"deallocate_fn");
5288-
auto fnPtr = FunctionPointer::createUnsigned(
5289-
FunctionPointer::Kind::Function, callee,
5290-
Signature(cast<llvm::FunctionType>(IGF.IGM.CoroDeallocateFnTy), {},
5291-
IGF.IGM.SwiftCC));
5292-
auto *call = IGF.Builder.CreateCall(fnPtr, {ptr});
5293-
call->setDoesNotThrow();
5294-
call->setCallingConv(IGF.IGM.SwiftCC);
5295-
IGF.Builder.CreateRetVoid();
5296-
},
5297-
/*setIsNoInline=*/true,
5298-
/*forPrologue=*/false,
5299-
/*isPerformanceConstraint=*/false,
5300-
/*optionalLinkageOverride=*/nullptr, IGM.SwiftCoroCC,
5301-
/*transformAttributes=*/
5302-
[&IGM](llvm::AttributeList &attrs) {
5303-
IGM.addSwiftCoroAttributes(attrs, 0);
5304-
});
5305-
}
5306-
53075173
void irgen::emitYieldOnce2CoroutineEntry(IRGenFunction &IGF,
53085174
CanSILFunctionType fnType,
53095175
llvm::Value *buffer,
@@ -5346,85 +5212,6 @@ Address irgen::emitAllocYieldManyCoroutineBuffer(IRGenFunction &IGF) {
53465212
getYieldManyCoroutineBufferAlignment(IGF.IGM));
53475213
}
53485214

5349-
static llvm::Constant *getAddrOfSwiftCCMalloc(IRGenModule &IGM) {
5350-
auto mallocFnPtr = IGM.getMallocFunctionPointer();
5351-
auto sig = mallocFnPtr.getSignature();
5352-
if (sig.getCallingConv() == IGM.SwiftCC) {
5353-
return IGM.getMallocFn();
5354-
}
5355-
return IGM.getOrCreateHelperFunction(
5356-
"_swift_malloc", sig.getType()->getReturnType(), sig.getType()->params(),
5357-
[](IRGenFunction &IGF) {
5358-
auto parameters = IGF.collectParameters();
5359-
auto *size = parameters.claimNext();
5360-
auto malloc = IGF.IGM.getMallocFunctionPointer();
5361-
auto *call = IGF.Builder.CreateCall(malloc, {size});
5362-
IGF.Builder.CreateRet(call);
5363-
});
5364-
}
5365-
5366-
static llvm::Constant *getAddrOfSwiftCCFree(IRGenModule &IGM) {
5367-
auto freeFnPtr = IGM.getFreeFunctionPointer();
5368-
auto sig = freeFnPtr.getSignature();
5369-
if (sig.getCallingConv() == IGM.SwiftCC) {
5370-
return IGM.getFreeFn();
5371-
}
5372-
return IGM.getOrCreateHelperFunction(
5373-
"_swift_free", sig.getType()->getReturnType(), sig.getType()->params(),
5374-
[](IRGenFunction &IGF) {
5375-
auto parameters = IGF.collectParameters();
5376-
auto *ptr = parameters.claimNext();
5377-
auto free = IGF.IGM.getFreeFunctionPointer();
5378-
IGF.Builder.CreateCall(free, {ptr});
5379-
IGF.Builder.CreateRetVoid();
5380-
});
5381-
}
5382-
5383-
static llvm::Constant *getAddrOfGlobalCoroAllocator(
5384-
IRGenModule &IGM, CoroAllocatorKind kind, bool shouldDeallocateImmediately,
5385-
llvm::Constant *allocFn, llvm::Constant *deallocFn) {
5386-
auto entity = LinkEntity::forCoroAllocator(kind);
5387-
auto taskAllocator = IGM.getOrCreateLazyGlobalVariable(
5388-
entity,
5389-
[&](ConstantInitBuilder &builder) -> ConstantInitFuture {
5390-
auto allocator = builder.beginStruct(IGM.CoroAllocatorTy);
5391-
auto flags = CoroAllocatorFlags(kind);
5392-
flags.setShouldDeallocateImmediately(shouldDeallocateImmediately);
5393-
allocator.addInt32(flags.getOpaqueValue());
5394-
allocator.add(allocFn);
5395-
allocator.add(deallocFn);
5396-
return allocator.finishAndCreateFuture();
5397-
},
5398-
[&](llvm::GlobalVariable *var) { var->setConstant(true); });
5399-
return taskAllocator;
5400-
}
5401-
llvm::Constant *IRGenModule::getAddrOfGlobalCoroMallocAllocator() {
5402-
return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Malloc,
5403-
/*shouldDeallocateImmediately=*/true,
5404-
getAddrOfSwiftCCMalloc(*this),
5405-
getAddrOfSwiftCCFree(*this));
5406-
}
5407-
llvm::Constant *IRGenModule::getAddrOfGlobalCoroAsyncTaskAllocator() {
5408-
return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Async,
5409-
/*shouldDeallocateImmediately=*/false,
5410-
getTaskAllocFn(), getTaskDeallocFn());
5411-
}
5412-
llvm::Value *
5413-
irgen::emitYieldOnce2CoroutineAllocator(IRGenFunction &IGF,
5414-
std::optional<CoroAllocatorKind> kind) {
5415-
if (!kind) {
5416-
return IGF.getCoroutineAllocator();
5417-
}
5418-
switch (*kind) {
5419-
case CoroAllocatorKind::Stack:
5420-
return llvm::ConstantPointerNull::get(IGF.IGM.CoroAllocatorPtrTy);
5421-
case CoroAllocatorKind::Async:
5422-
return IGF.IGM.getAddrOfGlobalCoroAsyncTaskAllocator();
5423-
case CoroAllocatorKind::Malloc:
5424-
return IGF.IGM.getAddrOfGlobalCoroMallocAllocator();
5425-
}
5426-
llvm_unreachable("unhandled case");
5427-
}
54285215
StackAddress irgen::emitAllocYieldOnce2CoroutineFrame(IRGenFunction &IGF,
54295216
llvm::Value *size) {
54305217
return emitAllocCoroStaticFrame(IGF, size);

‎lib/IRGen/GenCall.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,9 +220,6 @@ namespace irgen {
220220
CanSILFunctionType coroutineType,
221221
NativeCCEntryPointArgumentEmission &emission);
222222

223-
llvm::Value *
224-
emitYieldOnce2CoroutineAllocator(IRGenFunction &IGF,
225-
std::optional<CoroAllocatorKind> kind);
226223
StackAddress emitAllocYieldOnce2CoroutineFrame(IRGenFunction &IGF,
227224
llvm::Value *size);
228225
void emitDeallocYieldOnce2CoroutineFrame(IRGenFunction &IGF,

0 commit comments

Comments
 (0)