Skip to content

Commit 8c5fc07

Browse files
aschwaighofervarungandhi-apple
aschwaighofer
authored and
varungandhi-apple
committed
[IRGen] Use coro.end.async to return from an async function
The `coro.end.async` intrinsic allow specifying a function that is to be tail-called as the last thing before returning. LLVM lowering will inline the `must-tail-call` function argument to `coro.end.async`. This `must-tail-call` function can contain a `musttail` call. ``` define @my_must_tail_call_func(void (*)(i64) %fnptr, i64 %args) { musttail call void %fnptr(i64 %args) ret void } define @async_func() { ... coro.end.async(..., @my_must_tail_call_func, %return_continuation, i64 %args) unreachable } ```
1 parent cdc5ddf commit 8c5fc07

12 files changed

+62
-25
lines changed

‎lib/IRGen/GenCall.cpp

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4797,8 +4797,24 @@ void irgen::emitAsyncReturn(IRGenFunction &IGF, AsyncContextLayout &asyncLayout,
47974797
Args.push_back(IGF.getAsyncTask());
47984798
Args.push_back(IGF.getAsyncExecutor());
47994799
Args.push_back(IGF.getAsyncContext());
4800-
auto call = IGF.Builder.CreateCall(fnPtr, Args);
4801-
call->setTailCall();
4800+
4801+
// Setup the coro.end.async intrinsic call.
4802+
auto &Builder = IGF.Builder;
4803+
auto mustTailCallFn = IGF.createAsyncDispatchFn(fnPtr,Args);
4804+
auto handle = IGF.getCoroutineHandle();
4805+
auto rawFnPtr =
4806+
Builder.CreateBitOrPointerCast(fnPtr.getRawPointer(), IGF.IGM.Int8PtrTy);
4807+
4808+
SmallVector<llvm::Value*, 8> arguments;
4809+
arguments.push_back(handle);
4810+
arguments.push_back(/*is unwind*/Builder.getFalse());
4811+
arguments.push_back(mustTailCallFn);
4812+
arguments.push_back(rawFnPtr);
4813+
for (auto *arg: Args)
4814+
arguments.push_back(arg);
4815+
4816+
Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_end_async, arguments);
4817+
Builder.CreateUnreachable();
48024818
}
48034819

48044820
void irgen::emitAsyncReturn(IRGenFunction &IGF, AsyncContextLayout &asyncLayout,

‎lib/IRGen/GenFunc.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1286,6 +1286,7 @@ class AsyncPartialApplicationForwarderEmission
12861286
return subIGF.Builder.CreateCall(fnPtr.getAsFunction(subIGF),
12871287
asyncExplosion.claimAll());
12881288
}
1289+
// [FIXME: swiftasynccc] This call should be marked musttail.
12891290
void createReturn(llvm::CallInst *call) override {
12901291
subIGF.Builder.CreateRetVoid();
12911292
}
@@ -2478,7 +2479,7 @@ IRGenFunction::createAsyncDispatchFn(const FunctionPointer &fnPtr,
24782479
llvm::Function *dispatch =
24792480
llvm::Function::Create(dispatchFnTy, llvm::Function::InternalLinkage,
24802481
llvm::StringRef(name), &IGM.Module);
2481-
dispatch->setCallingConv(IGM.DefaultCC);
2482+
dispatch->setCallingConv(IGM.SwiftAsyncCC);
24822483
dispatch->setDoesNotThrow();
24832484
IRGenFunction dispatchIGF(IGM, dispatch);
24842485
if (IGM.DebugInfo)
@@ -2499,7 +2500,7 @@ IRGenFunction::createAsyncDispatchFn(const FunctionPointer &fnPtr,
24992500
auto callee = FunctionPointer(fnPtr.getKind(), fnPtrArg, newAuthInfo,
25002501
fnPtr.getSignature());
25012502
auto call = Builder.CreateCall(callee, callArgs);
2502-
call->setTailCall();
2503+
call->setTailCallKind(IGM.AsyncTailCallKind);
25032504
Builder.CreateRetVoid();
25042505
return dispatch;
25052506
}
@@ -2566,7 +2567,7 @@ llvm::Function *IRGenFunction::createAsyncSuspendFn() {
25662567
llvm::Function *suspendFn =
25672568
llvm::Function::Create(suspendFnTy, llvm::Function::InternalLinkage,
25682569
name, &IGM.Module);
2569-
suspendFn->setCallingConv(IGM.DefaultCC);
2570+
suspendFn->setCallingConv(IGM.SwiftAsyncCC);
25702571
suspendFn->setDoesNotThrow();
25712572
IRGenFunction suspendIGF(IGM, suspendFn);
25722573
if (IGM.DebugInfo)
@@ -2594,7 +2595,7 @@ llvm::Function *IRGenFunction::createAsyncSuspendFn() {
25942595
{ task, executor, targetExecutor });
25952596
suspendCall->setDoesNotThrow();
25962597
suspendCall->setCallingConv(IGM.SwiftAsyncCC);
2597-
suspendCall->setTailCall();
2598+
suspendCall->setTailCallKind(IGM.AsyncTailCallKind);
25982599
Builder.CreateRetVoid();
25992600
return suspendFn;
26002601
}

‎lib/IRGen/GenThunk.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,6 @@ void IRGenThunk::emit() {
354354

355355
if (isAsync) {
356356
emitAsyncReturn(IGF, *asyncLayout, origTy, result);
357-
IGF.emitCoroutineOrAsyncExit();
358357
return;
359358
}
360359

‎lib/IRGen/IRGenModule.cpp

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -548,12 +548,16 @@ IRGenModule::IRGenModule(IRGenerator &irgen,
548548
DefaultCC = SWIFT_DEFAULT_LLVM_CC;
549549
SwiftCC = llvm::CallingConv::Swift;
550550

551-
bool isAsynCCSupported =
551+
bool isAsyncCCSupported =
552552
clangASTContext.getTargetInfo().checkCallingConvention(clang::CC_SwiftAsync)
553553
== clang::TargetInfo::CCCR_OK;
554-
SwiftAsyncCC = (opts.UseAsyncLowering && isAsynCCSupported)
555-
? llvm::CallingConv::SwiftTail
556-
: SwiftCC;
554+
if (opts.UseAsyncLowering && isAsyncCCSupported) {
555+
SwiftAsyncCC = llvm::CallingConv::SwiftTail;
556+
AsyncTailCallKind = llvm::CallInst::TCK_MustTail;
557+
} else {
558+
SwiftAsyncCC = SwiftCC;
559+
AsyncTailCallKind = llvm::CallInst::TCK_Tail;
560+
}
557561

558562
if (opts.DebugInfoLevel > IRGenDebugInfoLevel::None)
559563
DebugInfo = IRGenDebugInfo::createIRGenDebugInfo(IRGen.Opts, *CI, *this,

‎lib/IRGen/IRGenModule.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
#include "llvm/IR/Attributes.h"
4545
#include "llvm/IR/CallingConv.h"
4646
#include "llvm/IR/Constant.h"
47+
#include "llvm/IR/Instructions.h"
4748
#include "llvm/IR/ValueHandle.h"
4849
#include "llvm/Target/TargetMachine.h"
4950

@@ -756,6 +757,9 @@ class IRGenModule {
756757
llvm::CallingConv::ID SwiftCC; /// swift calling convention
757758
llvm::CallingConv::ID SwiftAsyncCC; /// swift calling convention for async
758759

760+
/// What kind of tail call should be used for async->async calls.
761+
llvm::CallInst::TailCallKind AsyncTailCallKind;
762+
759763
Signature getAssociatedTypeWitnessTableAccessFunctionSignature();
760764

761765
/// Get the bit width of an integer type for the target platform.

‎lib/IRGen/IRGenSIL.cpp

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3434,9 +3434,14 @@ void IRGenFunction::emitCoroutineOrAsyncExit() {
34343434
// Emit the block.
34353435
Builder.emitBlock(coroEndBB);
34363436
auto handle = getCoroutineHandle();
3437-
Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_end,
3438-
{handle,
3439-
/*is unwind*/ Builder.getFalse()});
3437+
if (isAsync())
3438+
Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_end_async,
3439+
{handle,
3440+
/*is unwind*/ Builder.getFalse()});
3441+
else
3442+
Builder.CreateIntrinsicCall(llvm::Intrinsic::coro_end,
3443+
{handle,
3444+
/*is unwind*/ Builder.getFalse()});
34403445
Builder.CreateUnreachable();
34413446
}
34423447

@@ -3467,10 +3472,8 @@ static void emitReturnInst(IRGenSILFunction &IGF,
34673472
assert(!IGF.IndirectReturn.isValid() &&
34683473
"Formally direct results should stay direct results for async "
34693474
"functions");
3470-
34713475
auto asyncLayout = getAsyncContextLayout(IGF);
34723476
emitAsyncReturn(IGF, asyncLayout, fnType, result);
3473-
IGF.emitCoroutineOrAsyncExit();
34743477
} else {
34753478
auto funcLang = IGF.CurSILFn->getLoweredFunctionType()->getLanguage();
34763479
auto swiftCCReturn = funcLang == SILFunctionLanguage::Swift;
@@ -3513,7 +3516,6 @@ void IRGenSILFunction::visitThrowInst(swift::ThrowInst *i) {
35133516
if (isAsync()) {
35143517
auto layout = getAsyncContextLayout(*this);
35153518
emitAsyncReturn(*this, layout, i->getFunction()->getLoweredFunctionType());
3516-
emitCoroutineOrAsyncExit();
35173519
return;
35183520
}
35193521

‎test/IRGen/async.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public func task_future_wait(_ task: __owned SomeClass) async throws -> Int
1818

1919
// CHECK: define{{.*}} swift{{(tail)?}}cc void @"$s5async8testThisyyAA9SomeClassCnYF"(%swift.task* %0, %swift.executor* %1, %swift.context* swiftasync %2)
2020
// CHECK-64: call swiftcc i8* @swift_task_alloc(%swift.task* %{{[0-9]+}}, i64 64)
21-
// CHECK: tail call swift{{(tail)?}}cc void @swift_task_future_wait(
21+
// CHECK: {{(must)?}}tail call swift{{(tail)?}}cc void @swift_task_future_wait(
2222
public func testThis(_ task: __owned SomeClass) async {
2323
do {
2424
let _ = try await task_future_wait(task)

‎test/IRGen/async/get_async_continuation.sil

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ bb0:
8484
// CHECK: br label %coro.end
8585

8686
// CHECK: coro.end:
87-
// CHECK: call i1 @llvm.coro.end(
87+
// CHECK: call i1 (i8*, i1, ...) @llvm.coro.end.async(
8888
// CHECK: unreachable
8989

9090
// CHECK: await.async.maybe.resume:

‎test/IRGen/async/hop_to_executor.sil

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,18 @@ final actor MyActor {
2828
// CHECK: [[CAST_ACTOR:%[0-9]+]] = bitcast %T4test7MyActorC* [[ACTOR]] to %swift.executor*
2929
// CHECK-x86_64: call {{.*}} @llvm.coro.suspend.async{{.*}}(i32 2, i8* [[RESUME]], i8* bitcast (i8* (i8*)* @__swift_async_resume_get_context to i8*), i8* bitcast (void (i8*, %swift.executor*, %swift.task*, %swift.executor*, %swift.context*)* @__swift_suspend_point to i8*), i8* [[RESUME]], %swift.executor* [[CAST_ACTOR]], %swift.task* [[TASK]], %swift.executor* {{%[0-9]+}}, %swift.context* {{%[0-9]+}})
3030
// CHECK-arm64e: call {{.*}} @llvm.coro.suspend.async{{.*}}(i32 2, i8* [[RESUME]], i8* bitcast (i8* (i8*)* @__swift_async_resume_get_context to i8*), i8* bitcast (void (i8*, %swift.executor*, %swift.task*, %swift.executor*, %swift.context*)* @__swift_suspend_point to i8*), i8* [[SIGNED_RESUME]], %swift.executor* [[CAST_ACTOR]], %swift.task* [[TASK]], %swift.executor* {{%[0-9]+}}, %swift.context* {{%[0-9]+}})
31+
// CHECK: [[RET_CONTINUATION:%.*]] = bitcast void (%swift.task*, %swift.executor*, %swift.context*)* {{.*}} to i8*
32+
// CHECK: call i1 (i8*, i1, ...) @llvm.coro.end.async(i8* {{.*}}, i1 false, void (i8*, %swift.task*, %swift.executor*, %swift.context*)* @[[TAIL_CALL_FUNC:.*]], i8* [[RET_CONTINUATION]]
33+
// CHECK: unreachable
34+
3135
sil @test_simple : $@convention(method) @async (@guaranteed MyActor) -> () {
3236
bb0(%0 : $MyActor):
3337
hop_to_executor %0 : $MyActor
3438
%3 = tuple ()
3539
return %3 : $()
3640
}
3741

38-
// CHECK-LABEL: define internal void @__swift_suspend_point
42+
// CHECK-LABEL: define internal swifttailcc void @__swift_suspend_point
3943
// CHECK-SAME: (i8* %0, %swift.executor* %1, %swift.task* %2, %swift.executor* %3, %swift.context* [[CTXT:%[^,]+]])
4044
// CHECK: [[RESUME_ADDR:%[0-9]+]] = getelementptr inbounds %swift.task, %swift.task* %2, i32 0, i32 4
4145
// CHECK: store i8* %0, i8** [[RESUME_ADDR]]
@@ -46,8 +50,15 @@ bb0(%0 : $MyActor):
4650
// CHECK-arm64e: [[PTRAUTH_SIGN:%[^,]+]] = call i64 @llvm.ptrauth.sign.i64(i64 [[CTXT_INT]], i32 2, i64 [[PTRAUTH_BLEND]])
4751
// CHECK-arm64e: [[CTXT:%[^,]+]] = inttoptr i64 [[PTRAUTH_SIGN]] to %swift.context*
4852
// CHECK: store %swift.context* [[CTXT]], %swift.context** [[CTXT_ADDR]]
49-
// CHECK: tail call swift{{(tail)?}}cc void @swift_task_switch(%swift.task* %2, %swift.executor* %3, %swift.executor* %1)
53+
// CHECK: {{(must)?}}tail call swift{{(tail)?}}cc void @swift_task_switch(%swift.task* %2, %swift.executor* %3, %swift.executor* %1)
5054
// CHECK: ret void
5155

56+
// CHECK: define{{.*}} void @[[TAIL_CALL_FUNC]](i8* %0, %swift.task* %1, %swift.executor* %2, %swift.context* %3)
57+
// CHECK: %4 = bitcast i8* %0 to void (%swift.task*, %swift.executor*, %swift.context*)*
58+
// CHECK: {{(must)?}}tail call swift{{(tail)?}}cc void %4(%swift.task* %1, %swift.executor* %2, %swift.context* swiftasync %3)
59+
// CHECK: ret void
60+
// CHECK: }
61+
62+
5263
sil_vtable MyActor {
5364
}

‎test/IRGen/async/partial_apply.sil

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ bb0(%x : $*SwiftClassPair):
405405
sil public_external @use_closure2 : $@async @convention(thin) (@noescape @async @callee_guaranteed (Int) -> Int) -> ()
406406

407407
// CHECK-LABEL: define{{( dllexport)?}}{{( protected)?}} swift{{(tail)?}}cc void @partial_apply_stack_callee_guaranteed_indirect_guaranteed_class_pair_param(%swift.task* {{%[0-9]+}}, %swift.executor* {{%[0-9]+}}, %swift.context* swiftasync {{%[0-9]+}}) {{#[0-9]+}} {
408-
// CHECK-LABEL: define internal swift{{(tail)?}}cc void @"$s45indirect_guaranteed_captured_class_pair_paramTA.70"(%swift.task* {{%[0-9]+}}, %swift.executor* {{%[0-9]+}}, %swift.context* swiftasync {{%[0-9]+}}) {{#[0-9]+}} {
408+
// CHECK-LABEL: define internal swift{{(tail)?}}cc void @"$s45indirect_guaranteed_captured_class_pair_paramTA.{{[0-9]+}}"(%swift.task* {{%[0-9]+}}, %swift.executor* {{%[0-9]+}}, %swift.context* swiftasync {{%[0-9]+}}) {{#[0-9]+}} {
409409

410410
sil @partial_apply_stack_callee_guaranteed_indirect_guaranteed_class_pair_param : $@async @convention(thin) (@in_guaranteed SwiftClassPair) -> () {
411411
bb0(%x : $*SwiftClassPair):

0 commit comments

Comments
 (0)