Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
6ccbdcd
[Coroutines] added 5 tests that fail on the validations errors in #14…
tzuralon Jul 19, 2025
1d303ca
[Coroutines] fixed a bug when coro.end had an associated bundle, and …
tzuralon Jul 20, 2025
887bcb2
[Coroutines] fixed a bug when insertSpills performed spilling of Coro…
tzuralon Jul 20, 2025
7f00eeb
[Coroutines] added more 3 tests that are still failing due to complex…
tzuralon Jul 27, 2025
5899a93
[Coroutines] fixed the bugs introduced in the last 3 tests, improved …
tzuralon Jul 27, 2025
0b8e37a
Merge branch 'main' into users/tzuralon/coro-async-exceptions-fixes
tzuralon Jul 28, 2025
428d6b4
[Coroutines] removed 3 non-reduced tests that are redundant due to th…
tzuralon Jul 31, 2025
ffe4587
[Coroutines] reduced a test IR that introduces an unhandled edge-case…
tzuralon Jul 31, 2025
c8b944c
[Coroutines] reduced the IR tests a bit more by reducing unneccesary …
tzuralon Aug 3, 2025
8fa7b44
[Coroutines] applied formatter comments
tzuralon Aug 10, 2025
9ec807f
Merge branch 'main' into users/tzuralon/coro-async-exceptions-fixes
tzuralon Aug 18, 2025
3c5929a
[Coroutines] Reduce storage size for SmallSet, enhance naming
tzuralon Aug 22, 2025
3551cc8
[Coroutines] make comments more coherent
tzuralon Aug 22, 2025
c4795ae
[Coroutines] some formatting enhancements, improved performance by no…
tzuralon Aug 22, 2025
dbafc22
[Coroutines] - Remove redundant function finalizeBasicBlockCloneAndTr…
tzuralon Aug 22, 2025
0e1afdf
[Coroutines] - Make variable names more meaningful
tzuralon Sep 2, 2025
380a643
[Coroutines] - aligned the IR tests to have basic verification
tzuralon Sep 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 110 additions & 0 deletions llvm/lib/Transforms/Coroutines/CoroFrame.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "CoroInternal.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Analysis/StackLifetime.h"
#include "llvm/IR/DIBuilder.h"
Expand All @@ -35,6 +36,7 @@
#include "llvm/Transforms/Coroutines/SpillUtils.h"
#include "llvm/Transforms/Coroutines/SuspendCrossingInfo.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
#include <algorithm>
Expand Down Expand Up @@ -973,6 +975,112 @@ static StructType *buildFrameType(Function &F, coro::Shape &Shape,
return FrameTy;
}

// Dominance issue fixer for each predecessor satisfying predicate function
static void
remapPredecessorIfPredicate(BasicBlock *InitialBlock,
BasicBlock *ReplacementBlock,
std::function<bool(BasicBlock *)> Predicate) {

SmallVector<BasicBlock *> InitialBlockDominanceViolatingPredecessors;

auto Predecessors = predecessors(InitialBlock);
std::copy_if(Predecessors.begin(), Predecessors.end(),
std::back_inserter(InitialBlockDominanceViolatingPredecessors),
Predicate);

// Tailor the predecessor terminator - redirect it to ReplacementBlock.
for (auto *InitialBlockDominanceViolatingPredecessor :
InitialBlockDominanceViolatingPredecessors) {
auto *Terminator =
InitialBlockDominanceViolatingPredecessor->getTerminator();
if (auto *TerminatorIsInvoke = dyn_cast<InvokeInst>(Terminator)) {
if (InitialBlock == TerminatorIsInvoke->getUnwindDest()) {
TerminatorIsInvoke->setUnwindDest(ReplacementBlock);
} else {
TerminatorIsInvoke->setNormalDest(ReplacementBlock);
}
} else if (auto *TerminatorIsBranch = dyn_cast<BranchInst>(Terminator)) {
for (unsigned int SuccessorIdx = 0;
SuccessorIdx < TerminatorIsBranch->getNumSuccessors();
++SuccessorIdx) {
if (InitialBlock == TerminatorIsBranch->getSuccessor(SuccessorIdx)) {
TerminatorIsBranch->setSuccessor(SuccessorIdx, ReplacementBlock);
}
}
} else if (auto *TerminatorIsCleanupReturn =
dyn_cast<CleanupReturnInst>(Terminator)) {
TerminatorIsCleanupReturn->setUnwindDest(ReplacementBlock);
} else {
Terminator->print(dbgs());
report_fatal_error("Terminator is not implemented");
}
}
}

// Fixer for the "Instruction does not dominate all uses!" bug
// The fix consists of mapping dominance violating paths (where CoroBegin does
// not dominate cleanup nodes), duplicating them and setup 2 flows - the one
// that insertSpills intended to create (using the spill) and another one,
// preserving the logics of pre-splitting, which would be executed if unwinding
// happened before CoroBegin
static void enforceDominationByCoroBegin(const FrameDataInfo &FrameData,
const coro::Shape &Shape, Function *F,
DominatorTree &DT) {
SmallPtrSet<BasicBlock *, 3> SpillUserDominanceViolatingBlocksSet;

// Prepare the node set, logics will be run only on those nodes
for (const auto &E : FrameData.Spills) {
for (auto *U : E.second) {
auto *CurrentBlock = U->getParent();
if (!DT.dominates(Shape.CoroBegin, CurrentBlock)) {
SpillUserDominanceViolatingBlocksSet.insert(CurrentBlock);
}
}
}

if (SpillUserDominanceViolatingBlocksSet.empty()) {
return;
}

// Run is in reversed post order, to enforce visiting predecessors before
// successors
for (BasicBlock *CurrentBlock : ReversePostOrderTraversal<Function *>(F)) {
if (!SpillUserDominanceViolatingBlocksSet.contains(CurrentBlock)) {
continue;
}
SpillUserDominanceViolatingBlocksSet.erase(CurrentBlock);

// The duplicate will become the unspilled alternative
ValueToValueMapTy VMap;
auto *UnspilledAlternativeBlock =
CloneBasicBlock(CurrentBlock, VMap, ".unspilled_alternative", F);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we clear VMap before reuse it?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought it's mandatory to keep it across traverse (when tested it previously), but I don't find an actual example to prove it, so I will clear it for now.


// Remap node instructions
for (Instruction &UnspilledAlternativeBlockInstruction :
*UnspilledAlternativeBlock) {
RemapInstruction(&UnspilledAlternativeBlockInstruction, VMap,
RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
}

// Clone only if predecessor breaks dominance against CoroBegin
remapPredecessorIfPredicate(CurrentBlock, UnspilledAlternativeBlock,
[&DT, &Shape](BasicBlock *PredecessorNode) {
return !DT.dominates(Shape.CoroBegin,
PredecessorNode);
});

// We changed dominance tree, so recalculate it
DT.recalculate(*F);

if (SpillUserDominanceViolatingBlocksSet.empty()) {
return;
}
}

assert(SpillUserDominanceViolatingBlocksSet.empty() &&
"Graph is corrupted by SpillUserBlocksSet");
}

// Replace all alloca and SSA values that are accessed across suspend points
// with GetElementPointer from coroutine frame + loads and stores. Create an
// AllocaSpillBB that will become the new entry block for the resume parts of
Expand Down Expand Up @@ -1052,6 +1160,8 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
return GEP;
};

enforceDominationByCoroBegin(FrameData, Shape, F, DT);

for (auto const &E : FrameData.Spills) {
Value *Def = E.first;
auto SpillAlignment = Align(FrameData.getAlign(Def));
Expand Down
13 changes: 12 additions & 1 deletion llvm/lib/Transforms/Coroutines/CoroSplit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,18 @@ static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
// If coro.end has an associated bundle, add cleanupret instruction.
if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);

// If the terminator is an invoke,
// set the cleanupret unwind destination the same as the other edges, to
// preserve the invariant of "Unwind edges out of a funclet pad must have
// the same unwind dest"
BasicBlock *UnwindDest = nullptr;
if (auto *Invoke =
dyn_cast<InvokeInst>(FromPad->getParent()->getTerminator())) {
UnwindDest = Invoke->getUnwindDest();
}

auto *CleanupRet = Builder.CreateCleanupRet(FromPad, UnwindDest);
End->getParent()->splitBasicBlock(End);
CleanupRet->getParent()->getTerminator()->eraseFromParent();
}
Expand Down
44 changes: 44 additions & 0 deletions llvm/test/Transforms/Coroutines/pr148035_inst_does_not_dominate.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
; In coro-split, this coroutine code reduced IR, produced using clang with async-exceptions
; crashed before fix because of the validation mismatch of Instruction does not dominate all uses!
; RUN: opt < %s -passes='coro-split' -S

; Function Attrs: presplitcoroutine
define i8 @"?resuming_on_new_thread@@YA?AUtask@@Vunique_ptr@@@Z"(ptr %0) #0 personality ptr null {
invoke void @llvm.seh.scope.begin()
to label %2 unwind label %14

2: ; preds = %1
%3 = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
%4 = load volatile ptr, ptr null, align 8
%5 = call ptr @llvm.coro.begin(token %3, ptr %4)
%6 = call token @llvm.coro.save(ptr null)
%7 = call i8 @llvm.coro.suspend(token none, i1 false)
invoke void @llvm.seh.try.begin()
to label %common.ret unwind label %8

common.ret: ; preds = %12, %10, %2
ret i8 0

8: ; preds = %2
%9 = catchswitch within none [label %10] unwind label %12

10: ; preds = %8
%11 = catchpad within %9 [ptr null, i32 0, ptr null]
br label %common.ret

12: ; preds = %8
%13 = cleanuppad within none []
invoke void @llvm.seh.scope.end()
to label %common.ret unwind label %14

14: ; preds = %12, %1
%15 = cleanuppad within none []
store i32 0, ptr %0, align 4
Copy link
Contributor

@NewSigma NewSigma Sep 17, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems you are clearing pointer in unhandled_exception(). Could you please confirm it?

Copy link
Author

@tzuralon tzuralon Sep 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The pr148035_inst_does_not_dominate.ll is generated using llvm-reduce of a basic cpp file provided in the Issue.
It should only be an input test file to reproduce the potential bug, the fix I implemented takes this instruction as given.

Anyway, I think that this store is a product of the llvm-reduce, reduction of the default_delete of the std::unique_ptr

  %108 = cleanuppad within none []
  call void @"??1?$unique_ptr@HU?$default_delete@H@std@@@std@@QEAA@XZ"(ptr noundef nonnull align 8 dereferenceable(8) %0) #4 [ "funclet"(token %108) ]
  cleanupret from %108 unwind to caller

cleanupret from %15 unwind to caller
}

attributes #0 = { presplitcoroutine }

!llvm.module.flags = !{!0}

!0 = !{i32 2, !"eh-asynch", i32 1}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
; In coro-split, this coroutine code reduced IR, produced using clang with async-exceptions
; crashed before fix because of the validation mismatch of Instruction does not dominate all uses!
; RUN: opt < %s -passes='coro-split' -S

; Function Attrs: presplitcoroutine
define i8 @"?resuming_on_new_thread@@YA?AUtask@@Vunique_ptr@@@Z"(ptr %0) #0 personality ptr null {
invoke void @llvm.seh.scope.begin()
to label %2 unwind label %14

2: ; preds = %1
%3 = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
%4 = load volatile ptr, ptr null, align 8
%5 = call ptr @llvm.coro.begin(token %3, ptr %4)
%6 = call token @llvm.coro.save(ptr null)
%7 = call i8 @llvm.coro.suspend(token none, i1 false)
invoke void @llvm.seh.try.begin()
to label %common.ret unwind label %8

common.ret: ; preds = %12, %10, %2
ret i8 0

8: ; preds = %2
%9 = catchswitch within none [label %10] unwind label %12

10: ; preds = %8
%11 = catchpad within %9 [ptr null, i32 0, ptr null]
br label %common.ret

12: ; preds = %8
%13 = cleanuppad within none []
invoke void @llvm.seh.scope.end()
to label %common.ret unwind label %14

14: ; preds = %12, %1
%15 = cleanuppad within none []
store i32 0, ptr %0, align 4
br label %common.ret
}

attributes #0 = { presplitcoroutine }

!llvm.module.flags = !{!0}

!0 = !{i32 2, !"eh-asynch", i32 1}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
; In coro-split, this coroutine code reduced IR, produced using clang with async-exceptions
; crashed after first phase of fix because the terminator cleanupret was not implemented on predecessor fixer at the time
; RUN: opt < %s -passes='coro-split' -S

; Function Attrs: presplitcoroutine
define i8 @"?resuming_on_new_thread@@YA?AUtask@@V?$unique_ptr@HU?$default_delete@H@std@@@std@@0@Z"(ptr %0) #0 personality ptr null {
invoke void @llvm.seh.scope.begin()
to label %2 unwind label %15

2: ; preds = %1
%3 = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
%4 = call ptr @llvm.coro.begin(token %3, ptr null)
%5 = call token @llvm.coro.save(ptr null)
%6 = call i8 @llvm.coro.suspend(token none, i1 false)
invoke void @llvm.seh.try.begin()
to label %7 unwind label %8

7: ; preds = %2
ret i8 0

8: ; preds = %2
%9 = catchswitch within none [label %10] unwind label %12

10: ; preds = %8
%11 = catchpad within %9 [ptr null, i32 0, ptr null]
ret i8 0

12: ; preds = %8
%13 = cleanuppad within none []
invoke void @llvm.seh.scope.end()
to label %14 unwind label %15

14: ; preds = %12
ret i8 0

15: ; preds = %12, %1
%16 = cleanuppad within none []
cleanupret from %16 unwind label %17

17: ; preds = %15
%18 = cleanuppad within none []
store i32 0, ptr %0, align 4
cleanupret from %18 unwind to caller
}

attributes #0 = { presplitcoroutine }

!llvm.module.flags = !{!0}

!0 = !{i32 2, !"eh-asynch", i32 1}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
; In coro-split, this coroutine standard code reduced IR, produced using clang with async-exceptions
; crashed before fix because of the validation mismatch of Unwind edges out of a funclet pad must have the same unwind dest
; RUN: opt < %s -passes='coro-split' -S

; Function Attrs: presplitcoroutine
define i8 @"?resuming_on_new_thread@@YA?AUtask@@AEAVjthread@std@@@Z"() #0 personality ptr null {
%1 = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
%2 = call ptr @llvm.coro.begin(token %1, ptr null)
%3 = call token @llvm.coro.save(ptr null)
%4 = call i8 @llvm.coro.suspend(token none, i1 false)
invoke void @llvm.seh.try.begin()
to label %common.ret unwind label %5

common.ret: ; preds = %13, %7, %0
ret i8 0

5: ; preds = %0
%6 = catchswitch within none [label %7] unwind label %9

7: ; preds = %5
%8 = catchpad within %6 [ptr null, i32 0, ptr null]
br label %common.ret

9: ; preds = %5
%10 = cleanuppad within none []
invoke void @llvm.seh.scope.end() [ "funclet"(token %10) ]
to label %11 unwind label %13

11: ; preds = %9
%12 = call i1 @llvm.coro.end(ptr null, i1 true, token none) [ "funclet"(token %10) ]
cleanupret from %10 unwind label %13

13: ; preds = %11, %9
%14 = cleanuppad within none []
br label %common.ret
}

attributes #0 = { presplitcoroutine }

!llvm.module.flags = !{!0}

!0 = !{i32 2, !"eh-asynch", i32 1}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
; In coro-split, this coroutine standard code reduced IR, produced using clang with async-exceptions
; crashed before fix because of the validation mismatch of Unwind edges out of a funclet pad must have the same unwind dest
; RUN: opt < %s -passes='coro-split' -S

; Function Attrs: presplitcoroutine
define i1 @"?resuming_on_new_thread@@YA?AUtask@@AEAVjthread@std@@@Z"() #0 personality ptr null {
%1 = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
%2 = call ptr @llvm.coro.begin(token %1, ptr null)
%3 = call token @llvm.coro.save(ptr null)
%4 = call i8 @llvm.coro.suspend(token none, i1 false)
invoke void @llvm.seh.try.begin()
to label %common.ret unwind label %5

common.ret: ; preds = %11, %13, %7, %0
%common.ret.op = phi i1 [ false, %11], [false, %13 ], [false, %7], [false, %0]
ret i1 %common.ret.op

5: ; preds = %0
%6 = catchswitch within none [label %7] unwind label %9

7: ; preds = %5
%8 = catchpad within %6 [ptr null, i32 0, ptr null]
br label %common.ret

9: ; preds = %5
%10 = cleanuppad within none []
invoke void @llvm.seh.scope.end() [ "funclet"(token %10) ]
to label %11 unwind label %13

11: ; preds = %9
%12 = call i1 @llvm.coro.end(ptr null, i1 true, token none) [ "funclet"(token %10) ]
br label %common.ret

13: ; preds = %9
%14 = cleanuppad within none []
br label %common.ret
}

attributes #0 = { presplitcoroutine }

!llvm.module.flags = !{!0}

!0 = !{i32 2, !"eh-asynch", i32 1}