Skip to content

Commit 335a188

Browse files
NewSigmaNewSigma
committed
Promote point of GRO
1 parent ca84e9e commit 335a188

File tree

4 files changed

+160
-41
lines changed

4 files changed

+160
-41
lines changed

‎clang/lib/CodeGen/CGCoroutine.cpp‎

Lines changed: 105 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ struct clang::CodeGen::CGCoroData {
5353
// handler, this is null.
5454
llvm::Value *ResumeEHVar = nullptr;
5555

56+
// Stores the i1 that identify if control flows off the end of the coroutine
57+
// body
58+
llvm::PHINode *BodyDone = nullptr;
59+
5660
// Stores the jump destination just before the coroutine memory is freed.
5761
// This is the destination that every suspend point jumps to for the cleanup
5862
// branch.
@@ -599,14 +603,18 @@ namespace {
599603
struct CallCoroDelete final : public EHScopeStack::Cleanup {
600604
Stmt *Deallocate;
601605

602-
// Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;"
606+
void Emit(CodeGenFunction &CGF, Flags F) override {
607+
// Cleanup always happens on unwind. No need for pre-cleanup block.
608+
if (F.isForNormalCleanup())
609+
EmitPreCleanupBB(CGF);
610+
// Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;"
611+
612+
// Note: That deallocation will be emitted twice: once for a normal exit and
613+
// once for exceptional exit. This usage is safe because Deallocate does not
614+
// contain any declarations. The SubStmtBuilder::makeNewAndDeleteExpr()
615+
// builds a single call to a deallocation function which is safe to emit
616+
// multiple times.
603617

604-
// Note: That deallocation will be emitted twice: once for a normal exit and
605-
// once for exceptional exit. This usage is safe because Deallocate does not
606-
// contain any declarations. The SubStmtBuilder::makeNewAndDeleteExpr()
607-
// builds a single call to a deallocation function which is safe to emit
608-
// multiple times.
609-
void Emit(CodeGenFunction &CGF, Flags) override {
610618
// Remember the current point, as we are going to emit deallocation code
611619
// first to get to coro.free instruction that is an argument to a delete
612620
// call.
@@ -642,6 +650,37 @@ struct CallCoroDelete final : public EHScopeStack::Cleanup {
642650
CGF.Builder.SetInsertPoint(AfterFreeBB);
643651
}
644652
explicit CallCoroDelete(Stmt *DeallocStmt) : Deallocate(DeallocStmt) {}
653+
654+
private:
655+
// [dcl.fct.def.coroutine]: The coroutine state is destroyed when control
656+
// flows off the end of the coroutine or the destroy member function of a
657+
// coroutine handle that refers to the coroutine is invoked.
658+
void EmitPreCleanupBB(CodeGenFunction &CGF) {
659+
auto &Builder = CGF.Builder;
660+
auto &Data = *CGF.CurCoro.Data;
661+
auto *BodyDone = Data.BodyDone;
662+
BasicBlock *SaveInsertBlock = Builder.GetInsertBlock();
663+
BasicBlock *PreConvBB = BodyDone->getParent();
664+
BasicBlock *AfterConvBB =
665+
cast<llvm::BranchInst>(PreConvBB->getTerminator())->getSuccessor(1);
666+
BasicBlock *CleanupBB = AfterConvBB->getSingleSuccessor();
667+
BasicBlock *PreCleanupBB = CleanupBB->splitBasicBlock(
668+
CleanupBB->getFirstNonPHIIt(), "pre.coro.cleanup", true);
669+
Builder.SetInsertPoint(PreCleanupBB);
670+
PreCleanupBB->getTerminator()->eraseFromParent();
671+
672+
auto *NeedCleanup = Builder.CreatePHI(
673+
Builder.getInt1Ty(), llvm::pred_size(PreCleanupBB), "is.completed");
674+
for (auto *Pred : llvm::predecessors(PreCleanupBB)) {
675+
auto *V = (Pred == AfterConvBB) ? cast<Value>(BodyDone)
676+
: cast<Value>(Builder.getTrue());
677+
NeedCleanup->addIncoming(V, Pred);
678+
}
679+
680+
auto *RetBB = Data.CleanupJD.getBlock();
681+
Builder.CreateCondBr(NeedCleanup, CleanupBB, RetBB);
682+
Builder.SetInsertPoint(SaveInsertBlock);
683+
}
645684
};
646685
}
647686

@@ -691,7 +730,7 @@ struct GetReturnObjectManager {
691730
// split its emission in two parts. EmitGroAlloca emits an alloca and sets up
692731
// cleanups. Later when coroutine promise is available we initialize the gro
693732
// and sets the flag that the cleanup is now active.
694-
void EmitGroAlloca() {
733+
void EmitGroActive() {
695734
if (DirectEmit)
696735
return;
697736

@@ -701,12 +740,23 @@ struct GetReturnObjectManager {
701740
return;
702741
}
703742

704-
auto *GroVarDecl = cast<VarDecl>(GroDeclStmt->getSingleDecl());
705-
706743
// Set GRO flag that it is not initialized yet
707744
GroActiveFlag = CGF.CreateTempAlloca(Builder.getInt1Ty(), CharUnits::One(),
708745
"gro.active");
709746
Builder.CreateStore(Builder.getFalse(), GroActiveFlag);
747+
}
748+
749+
void EmitGroAlloca() {
750+
if (DirectEmit)
751+
return;
752+
753+
auto *GroDeclStmt = dyn_cast_or_null<DeclStmt>(S.getResultDecl());
754+
if (!GroDeclStmt) {
755+
// If get_return_object returns void, no need to do an alloca.
756+
return;
757+
}
758+
759+
auto *GroVarDecl = cast<VarDecl>(GroDeclStmt->getSingleDecl());
710760

711761
GroEmission = CGF.EmitAutoVarAlloca(*GroVarDecl);
712762

@@ -768,6 +818,38 @@ struct GetReturnObjectManager {
768818
CGF.EmitAutoVarInit(GroEmission);
769819
Builder.CreateStore(Builder.getTrue(), GroActiveFlag);
770820
}
821+
822+
void EmitGroConv() {
823+
auto *InsertPt = Builder.GetInsertBlock();
824+
auto *PreConvBB = CGF.CurCoro.Data->SuspendBB;
825+
auto *AfterConvBB = CGF.createBasicBlock("after.gro.conv", CGF.CurFn, InsertPt);
826+
Builder.SetInsertPoint(AfterConvBB);
827+
BasicBlock *AfterFinalBB = nullptr;
828+
if (InsertPt) {
829+
AfterFinalBB = InsertPt->getSinglePredecessor();
830+
InsertPt->replaceAllUsesWith(PreConvBB);
831+
Builder.CreateBr(InsertPt);
832+
}
833+
834+
auto *ConvBB = CGF.createBasicBlock("gro.conv", CGF.CurFn, AfterConvBB);
835+
Builder.SetInsertPoint(ConvBB);
836+
Builder.CreateBr(AfterConvBB);
837+
838+
CGF.EmitBlock(PreConvBB);
839+
PreConvBB->moveBefore(ConvBB);
840+
auto *BodyDone =
841+
Builder.CreatePHI(Builder.getInt1Ty(), llvm::pred_size(PreConvBB));
842+
for (auto *Pred : llvm::predecessors(PreConvBB)) {
843+
auto *V = (Pred == AfterFinalBB) ? Builder.getTrue() : Builder.getFalse();
844+
BodyDone->addIncoming(V, Pred);
845+
}
846+
CGF.CurCoro.Data->BodyDone = BodyDone;
847+
auto *InRampFn = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_is_in_ramp);
848+
auto *InRamp = Builder.CreateCall(InRampFn, {}, "InRamp");
849+
Builder.CreateCondBr(InRamp, ConvBB, AfterConvBB);
850+
851+
Builder.SetInsertPoint(InsertPt == nullptr ? AfterConvBB : InsertPt);
852+
}
771853
};
772854
} // namespace
773855

@@ -790,12 +872,13 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
790872
auto *InitBB = createBasicBlock("coro.init");
791873
auto *FinalBB = createBasicBlock("coro.final");
792874
auto *RetBB = createBasicBlock("coro.ret");
875+
auto *PreConvBB = createBasicBlock("pre.gvo.conv");
793876

794877
auto *CoroId = Builder.CreateCall(
795878
CGM.getIntrinsic(llvm::Intrinsic::coro_id),
796879
{Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr});
797880
createCoroData(*this, CurCoro, CoroId);
798-
CurCoro.Data->SuspendBB = RetBB;
881+
CurCoro.Data->SuspendBB = PreConvBB;
799882
assert(ShouldEmitLifetimeMarkers &&
800883
"Must emit lifetime intrinsics for coroutines");
801884

@@ -840,7 +923,6 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
840923
CurCoro.Data->CoroBegin = CoroBegin;
841924

842925
GetReturnObjectManager GroManager(*this, S);
843-
GroManager.EmitGroAlloca();
844926

845927
CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB);
846928
{
@@ -884,6 +966,7 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
884966
// not needed.
885967
}
886968

969+
GroManager.EmitGroActive();
887970
EmitStmt(S.getPromiseDeclStmt());
888971

889972
Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl());
@@ -895,6 +978,7 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
895978
CoroId->setArgOperand(1, PromiseAddrVoidPtr);
896979

897980
// Now we have the promise, initialize the GRO
981+
GroManager.EmitGroAlloca();
898982
GroManager.EmitGroInit();
899983

900984
EHStack.pushCleanup<CallCoroEnd>(EHCleanup);
@@ -950,11 +1034,12 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
9501034
// We don't need FinalBB. Emit it to make sure the block is deleted.
9511035
EmitBlock(FinalBB, /*IsFinished=*/true);
9521036
}
1037+
GroManager.EmitGroConv();
9531038
}
9541039

9551040
EmitBlock(RetBB);
956-
// Emit coro.end before getReturnStmt (and parameter destructors), since
957-
// resume and destroy parts of the coroutine should not include them.
1041+
// Emit coro.end before ret instruction, since resume and destroy parts of the
1042+
// coroutine should return void.
9581043
llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end);
9591044
Builder.CreateCall(CoroEnd,
9601045
{NullPtr, Builder.getFalse(),
@@ -973,8 +1058,13 @@ void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) {
9731058
// shouldn't change the AST.
9741059
if (PreviousRetValue)
9751060
cast<ReturnStmt>(Ret)->setRetValue(PreviousRetValue);
1061+
// Send GRO conversion to ConvBB
1062+
auto *ConvBB =
1063+
cast<llvm::BranchInst>(PreConvBB->getTerminator())->getSuccessor(0);
1064+
auto FromIt = ++RetBB->getFirstInsertionPt();
1065+
auto ToIt = RetBB->getTerminator()->getIterator();
1066+
ConvBB->splice(ConvBB->getFirstNonPHIIt(), RetBB, FromIt, ToIt);
9761067
}
977-
9781068
// LLVM require the frontend to mark the coroutine.
9791069
CurFn->setPresplitCoroutine();
9801070

‎clang/test/CodeGenCoroutines/coro-dest-slot.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ extern "C" coro f(int) { co_return; }
2525
// CHECK: %[[CLEANUP_DEST0:.+]] = phi i32 [ 0, %[[INIT_READY]] ], [ 2, %[[INIT_CLEANUP]] ]
2626

2727
// CHECK: %[[FINAL_SUSPEND:.+]] = call i8 @llvm.coro.suspend(
28-
// CHECK-NEXT: switch i8 %{{.*}}, label %coro.ret [
28+
// CHECK-NEXT: switch i8 %{{.*}}, label %pre.gvo.conv [
2929
// CHECK-NEXT: i8 0, label %[[FINAL_READY:.+]]
3030
// CHECK-NEXT: i8 1, label %[[FINAL_CLEANUP:.+]]
3131
// CHECK-NEXT: ]

‎clang/test/CodeGenCoroutines/coro-gro.cpp‎

Lines changed: 52 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -29,47 +29,76 @@ void doSomething() noexcept;
2929
// CHECK: define{{.*}} i32 @_Z1fv(
3030
int f() {
3131
// CHECK: %[[RetVal:.+]] = alloca i32
32-
// CHECK: %[[GroActive:.+]] = alloca i1
33-
// CHECK: %[[CoroGro:.+]] = alloca %struct.GroType, {{.*}} !coro.outside.frame ![[OutFrameMetadata:.+]]
32+
// CHECK-NEXT: %[[GroActive:.+]] = alloca i1
33+
// CHECK-NEXT: %[[Promise:.+]] = alloca %"struct.std::coroutine_traits<int>::promise_type", align 1
34+
// CHECK-NEXT: %[[CoroGro:.+]] = alloca %struct.GroType, {{.*}} !coro.outside.frame ![[OutFrameMetadata:.+]]
3435

3536
// CHECK: %[[Size:.+]] = call i64 @llvm.coro.size.i64()
36-
// CHECK: call noalias noundef nonnull ptr @_Znwm(i64 noundef %[[Size]])
37+
// CHECK-NEXT: call noalias noundef nonnull ptr @_Znwm(i64 noundef %[[Size]])
38+
3739
// CHECK: store i1 false, ptr %[[GroActive]]
38-
// CHECK: call void @_ZNSt16coroutine_traitsIiJEE12promise_typeC1Ev(
39-
// CHECK: call void @_ZNSt16coroutine_traitsIiJEE12promise_type17get_return_objectEv({{.*}} %[[CoroGro]]
40-
// CHECK: store i1 true, ptr %[[GroActive]]
40+
// CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr %[[Promise]])
41+
// CHECK-NEXT: call void @_ZNSt16coroutine_traitsIiJEE12promise_typeC1Ev(
42+
// CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr %[[CoroGro]])
43+
// CHECK-NEXT: call void @_ZNSt16coroutine_traitsIiJEE12promise_type17get_return_objectEv({{.*}} %[[CoroGro]]
44+
// CHECK-NEXT: store i1 true, ptr %[[GroActive]]
4145

4246
Cleanup cleanup;
4347
doSomething();
4448
co_return;
4549

4650
// CHECK: call void @_Z11doSomethingv(
47-
// CHECK: call void @_ZNSt16coroutine_traitsIiJEE12promise_type11return_voidEv(
48-
// CHECK: call void @_ZN7CleanupD1Ev(
49-
50-
// Destroy promise and free the memory.
51-
52-
// CHECK: call void @_ZNSt16coroutine_traitsIiJEE12promise_typeD1Ev(
53-
// CHECK: %[[Mem:.+]] = call ptr @llvm.coro.free(
54-
// CHECK: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
55-
// CHECK: call void @_ZdlPvm(ptr noundef %[[Mem]], i64 noundef %[[SIZE]])
51+
// CHECK-NEXT: call void @_ZNSt16coroutine_traitsIiJEE12promise_type11return_voidEv(
52+
// CHECK-NEXT: call void @_ZN7CleanupD1Ev(
5653

5754
// Initialize retval from Gro and destroy Gro
5855
// Note this also tests delaying initialization when Gro and function return
5956
// types mismatch (see cwg2563).
6057

61-
// CHECK: %[[Conv:.+]] = call noundef i32 @_ZN7GroTypecviEv(
62-
// CHECK: store i32 %[[Conv]], ptr %[[RetVal]]
63-
// CHECK: %[[IsActive:.+]] = load i1, ptr %[[GroActive]]
64-
// CHECK: br i1 %[[IsActive]], label %[[CleanupGro:.+]], label %[[Done:.+]]
58+
// CHECK: pre.gvo.conv:
59+
// CHECK-NEXT: %10 = phi i1 [ true, %cleanup8 ], [ false, %final.suspend ], [ false, %init.suspend ]
60+
// CHECK-NEXT: %InRamp = call i1 @llvm.coro.is_in_ramp()
61+
// CHECK-NEXT: br i1 %InRamp, label %[[GroConv:.+]], label %[[AfterGroConv:.+]]
62+
63+
// CHECK: [[GroConv]]:
64+
// CHECK-NEXT: %[[Conv:.+]] = call noundef i32 @_ZN7GroTypecviEv(
65+
// CHECK-NEXT: store i32 %[[Conv]], ptr %[[RetVal]]
66+
// CHECK-NEXT: br label %[[AfterGroConv]]
67+
68+
// CHECK: [[AfterGroConv]]:
69+
// CHECK-NEXT: br label %[[PreCleanup:.+]]
70+
71+
// CHECK: [[PreCleanup]]:
72+
// CHECK-NEXT: %[[NeedCleanup:.+]] = phi i1 [ %10, %after.gro.conv ]
73+
// CHECK-NEXT: br i1 %[[NeedCleanup]], label %cleanup.cont10, label %[[CoroRet:.+]]
74+
75+
// CHECK: cleanup.cont10:
76+
// CHECK-NEXT: br label %[[Cleanup:.+]]
77+
78+
// CHECK: [[Cleanup]]:
79+
// CHECK-NEXT: %{{.*}} = phi i32
80+
// CHECK-NEXT: %[[IsActive:.+]] = load i1, ptr %[[GroActive]]
81+
// CHECK-NEXT: br i1 %[[IsActive]], label %[[CleanupGro:.+]], label %[[Done:.+]]
6582

6683
// CHECK: [[CleanupGro]]:
67-
// CHECK: call void @_ZN7GroTypeD1Ev(
68-
// CHECK: br label %[[Done]]
84+
// CHECK-NEXT: call void @_ZN7GroTypeD1Ev(
85+
// CHECK-NEXT: br label %[[Done]]
86+
87+
// Destroy promise and free the memory.
6988

7089
// CHECK: [[Done]]:
71-
// CHECK: %[[LoadRet:.+]] = load i32, ptr %[[RetVal]]
72-
// CHECK: ret i32 %[[LoadRet]]
90+
// CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr %[[CoroGro]])
91+
// CHECK-NEXT: call void @_ZNSt16coroutine_traitsIiJEE12promise_typeD1Ev(
92+
// CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr %[[Promise]])
93+
// CHECK-NEXT: %[[Mem:.+]] = call ptr @llvm.coro.free(
94+
95+
// CHECK: %[[SIZE:.+]] = call i64 @llvm.coro.size.i64()
96+
// CHECK-NEXT: call void @_ZdlPvm(ptr noundef %[[Mem]], i64 noundef %[[SIZE]])
97+
98+
// CHECK: [[CoroRet]]:
99+
// CHECK-NEXT: call void @llvm.coro.end(
100+
// CHECK-NEXT: %[[LoadRet:.+]] = load i32, ptr %[[RetVal]]
101+
// CHECK-NEXT: ret i32 %[[LoadRet]]
73102
}
74103

75104
class invoker {

‎clang/test/CodeGenCoroutines/coro-symmetric-transfer-02.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ Task bar() {
9191
// CHECK-NEXT: %{{.+}} = call token @llvm.coro.save(ptr null)
9292
// CHECK-NEXT: call void @llvm.coro.await.suspend.handle
9393
// CHECK-NEXT: %{{.+}} = call i8 @llvm.coro.suspend
94-
// CHECK-NEXT: switch i8 %{{.+}}, label %coro.ret [
94+
// CHECK-NEXT: switch i8 %{{.+}}, label %pre.gvo.conv [
9595
// CHECK-NEXT: i8 0, label %[[CASE1_AWAIT_READY]]
9696
// CHECK-NEXT: i8 1, label %[[CASE1_AWAIT_CLEANUP:.+]]
9797
// CHECK-NEXT: ]
@@ -105,7 +105,7 @@ Task bar() {
105105
// CHECK-NEXT: %{{.+}} = call token @llvm.coro.save(ptr null)
106106
// CHECK-NEXT: call void @llvm.coro.await.suspend.handle
107107
// CHECK-NEXT: %{{.+}} = call i8 @llvm.coro.suspend
108-
// CHECK-NEXT: switch i8 %{{.+}}, label %coro.ret [
108+
// CHECK-NEXT: switch i8 %{{.+}}, label %pre.gvo.conv [
109109
// CHECK-NEXT: i8 0, label %[[CASE2_AWAIT_READY]]
110110
// CHECK-NEXT: i8 1, label %[[CASE2_AWAIT_CLEANUP:.+]]
111111
// CHECK-NEXT: ]

0 commit comments

Comments
 (0)