diff --git a/runtime-light/coroutine/async-stack-structs.h b/runtime-light/coroutine/async-stack-structs.h new file mode 100644 index 0000000000..578060ed8f --- /dev/null +++ b/runtime-light/coroutine/async-stack-structs.h @@ -0,0 +1,102 @@ +// Compiler for PHP (aka KPHP) +// Copyright (c) 2025 LLC «V Kontakte» +// Distributed under the GPL v3 License, see LICENSE.notice.txt + +#pragma once + +/** + * This header defines the data structures used to represent a coroutine asynchronous stack. + * + * Overview: + * The asynchronous stack is used to manage the execution state of coroutines, allowing for + * efficient context switching and stack management. + * + * Diagram: Normal and Asynchronous Stacks + * + * Base Pointer (%rbp) async_stack_root + * | | + * V V + * stack_frame async_stack_frame + * | | + * V V + * stack_frame async_stack_frame + * ... ... + * | | + * V V + * stack_frame async_stack_frame + * | | + * V V + * + * In the diagram above, the left side represents a typical call stack with stack frames linked by + * the base pointer (%rbp). The right side illustrates an asynchronous stack where `async_stack_frame` + * structures are linked by `async_stack_root`. + * + * Diagram: Backtrace Mechanism + * + * Base Pointer (%rbp) + * | + * V + * stack_frame + * | + * V + * stack_frame (stop_sync_frame) <- async_stack_root + * | + * V + * async_stack_frame (top_async_frame) + * | + * V + * async_stack_frame + * ... + * | + * V + * async_stack_frame + * + * The backtrace mechanism involves traversing the stack frames to capture the call stack. + * The `stop_sync_frame` serves as a marker where the transition to the asynchronous stack occurs, + * allowing the backtrace to continue through the `async_stack_frame` structures. + */ + +#define STACK_RETURN_ADDRESS __builtin_return_address(0) + +#define STACK_FRAME_ADDRESS __builtin_frame_address(0) + +namespace kphp::coro { + +struct stack_frame { + stack_frame* caller_stack_frame{}; + void* return_address{}; +}; + +struct async_stack_root; + +struct async_stack_frame { + async_stack_frame* caller_async_stack_frame{}; + async_stack_root* async_stack_root{}; + void* return_address{}; +}; + +struct async_stack_root { + async_stack_frame* top_async_stack_frame{}; + stack_frame* stop_sync_stack_frame{}; + async_stack_root* next_async_stack_root{}; +}; + +/** + * The async_stack_element class facilitates working with asynchronous traces in templated code. + * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. + */ +struct async_stack_element { + async_stack_frame& get_async_stack_frame() noexcept { + return m_async_stack_frame; + } + +private: + async_stack_frame m_async_stack_frame; +}; + +/** + * The async_stack_element class facilitates working with asynchronous traces in templated code. + * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. + */ + +} // namespace kphp::coro diff --git a/runtime-light/coroutine/async-stack.h b/runtime-light/coroutine/async-stack.h index 85ab351a5f..f3e248e2df 100644 --- a/runtime-light/coroutine/async-stack.h +++ b/runtime-light/coroutine/async-stack.h @@ -4,107 +4,49 @@ #pragma once +#include "runtime-light/coroutine/async-stack-structs.h" + #include #include -/** - * This header defines the data structures used to represent a coroutine asynchronous stack. - * - * Overview: - * The asynchronous stack is used to manage the execution state of coroutines, allowing for - * efficient context switching and stack management. - * - * Diagram: Normal and Asynchronous Stacks - * - * Base Pointer (%rbp) async_stack_root - * | | - * V V - * stack_frame async_stack_frame - * | | - * V V - * stack_frame async_stack_frame - * ... ... - * | | - * V V - * stack_frame async_stack_frame - * | | - * V V - * - * In the diagram above, the left side represents a typical call stack with stack frames linked by - * the base pointer (%rbp). The right side illustrates an asynchronous stack where `async_stack_frame` - * structures are linked by `async_stack_root`. - * - * Diagram: Backtrace Mechanism - * - * Base Pointer (%rbp) - * | - * V - * stack_frame - * | - * V - * stack_frame (stop_sync_frame) <- async_stack_root - * | - * V - * async_stack_frame (top_async_frame) - * | - * V - * async_stack_frame - * ... - * | - * V - * async_stack_frame - * - * The backtrace mechanism involves traversing the stack frames to capture the call stack. - * The `stop_sync_frame` serves as a marker where the transition to the asynchronous stack occurs, - * allowing the backtrace to continue through the `async_stack_frame` structures. - */ - -#define STACK_RETURN_ADDRESS __builtin_return_address(0) - -#define STACK_FRAME_ADDRESS __builtin_frame_address(0) +#include "runtime-light/coroutine/coroutine-state.h" +#include "runtime-light/stdlib/diagnostics/logs.h" namespace kphp::coro { -struct stack_frame { - stack_frame* caller_stack_frame{}; - void* return_address{}; -}; - -struct async_stack_root; +struct async_stack_root_wrapper { + async_stack_root root; -struct async_stack_frame { - async_stack_frame* caller_async_stack_frame{}; - async_stack_root* async_stack_root{}; - void* return_address{}; -}; - -struct async_stack_root { - async_stack_frame* top_async_stack_frame{}; - stack_frame* stop_sync_stack_frame{}; + async_stack_root_wrapper() = default; + async_stack_root_wrapper(const async_stack_root_wrapper& other) = delete; + async_stack_root_wrapper& operator=(const async_stack_root_wrapper& other) = delete; + async_stack_root_wrapper(async_stack_root_wrapper&& other) = default; + async_stack_root_wrapper& operator=(async_stack_root_wrapper&& other) = default; + ~async_stack_root_wrapper() { + CoroutineInstanceState::get().current_async_stack_root = root.next_async_stack_root; + } }; -/** - * The `resume` function is responsible for storing the current synchronous stack frame - * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows - * capturing one of the stack frames in the synchronous stack trace. - */ -inline void resume(std::coroutine_handle<> handle, async_stack_root& stack_root) noexcept { - auto* previous_stack_frame{std::exchange(stack_root.stop_sync_stack_frame, reinterpret_cast(STACK_FRAME_ADDRESS))}; - handle.resume(); - stack_root.stop_sync_stack_frame = previous_stack_frame; +inline void preparation_for_resume(kphp::coro::async_stack_root* root, void* stack_frame_addr, + CoroutineInstanceState& coroutine_state = CoroutineInstanceState::get()) { + root->stop_sync_stack_frame = reinterpret_cast(stack_frame_addr); + coroutine_state.current_async_stack_root = root; } /** - * The async_stack_element class facilitates working with asynchronous traces in templated code. - * This allows for uniform handling of any coroutines in places where async frames are pushed or popped. + * The `resume_with_new_root` function is responsible for storing the current synchronous stack frame + * in async_stack_root::stop_sync_frame before resuming the coroutine. This allows + * capturing one of the stack frames in the synchronous stack trace and jump between parts of the async_stack. */ -struct async_stack_element { - async_stack_frame& get_async_stack_frame() noexcept { - return m_async_stack_frame; - } +[[clang::noinline]] inline void resume_with_new_root(std::coroutine_handle<> handle, async_stack_root* new_async_stack_root) noexcept { + kphp::log::assertion(new_async_stack_root != nullptr); + auto& coroutine_st{CoroutineInstanceState::get()}; -private: - async_stack_frame m_async_stack_frame; -}; + new_async_stack_root->next_async_stack_root = coroutine_st.current_async_stack_root; + new_async_stack_root->stop_sync_stack_frame = reinterpret_cast(STACK_FRAME_ADDRESS); + coroutine_st.current_async_stack_root = new_async_stack_root; + handle.resume(); + new_async_stack_root->stop_sync_stack_frame = nullptr; +} } // namespace kphp::coro diff --git a/runtime-light/coroutine/await-set.h b/runtime-light/coroutine/await-set.h index 3d3bdd76f7..324091d48b 100644 --- a/runtime-light/coroutine/await-set.h +++ b/runtime-light/coroutine/await-set.h @@ -19,21 +19,17 @@ namespace kphp::coro { template class await_set { std::unique_ptr> m_await_broker; - kphp::coro::async_stack_root& m_coroutine_stack_root; public: await_set() noexcept - : m_await_broker(std::make_unique>()), - m_coroutine_stack_root(CoroutineInstanceState::get().coroutine_stack_root) {} + : m_await_broker(std::make_unique>()) {} await_set(await_set&& other) noexcept - : m_await_broker(std::move(other.m_await_broker)), - m_coroutine_stack_root(other.m_coroutine_stack_root) {} + : m_await_broker(std::move(other.m_await_broker)) {} await_set& operator=(await_set&& other) noexcept { if (this != std::addressof(other)) { m_await_broker = std::move(other.m_await_broker); - m_coroutine_stack_root = other.m_coroutine_stack_root; } return *this; } @@ -44,7 +40,7 @@ class await_set { template requires kphp::coro::concepts::awaitable && std::is_same_v::awaiter_return_type, return_type> void push(awaitable_type awaitable) noexcept { - m_await_broker->start_task(detail::await_set::make_await_set_task(std::move(awaitable)), m_coroutine_stack_root, STACK_RETURN_ADDRESS); + m_await_broker->start_task(detail::await_set::make_await_set_task(std::move(awaitable))); } auto next() noexcept { diff --git a/runtime-light/coroutine/coroutine-state.h b/runtime-light/coroutine/coroutine-state.h index b6d260c278..2a77c70977 100644 --- a/runtime-light/coroutine/coroutine-state.h +++ b/runtime-light/coroutine/coroutine-state.h @@ -6,7 +6,7 @@ #include "common/mixin/not_copyable.h" -#include "runtime-light/coroutine/async-stack.h" +#include "runtime-light/coroutine/async-stack-structs.h" struct CoroutineInstanceState final : private vk::not_copyable { @@ -14,5 +14,6 @@ struct CoroutineInstanceState final : private vk::not_copyable { static CoroutineInstanceState& get() noexcept; - kphp::coro::async_stack_root coroutine_stack_root; + kphp::coro::async_stack_root base_coroutine_stack_root; // used to reduce the number of accesses to thread-local state + kphp::coro::async_stack_root* current_async_stack_root{&base_coroutine_stack_root}; }; diff --git a/runtime-light/coroutine/detail/await-set.h b/runtime-light/coroutine/detail/await-set.h index 2edbd086e4..682447a809 100644 --- a/runtime-light/coroutine/detail/await-set.h +++ b/runtime-light/coroutine/detail/await-set.h @@ -30,6 +30,7 @@ struct await_set_awaiter { await_set_awaiter* m_next{}; await_set_awaiter* m_prev{}; std::coroutine_handle<> m_continuation; + kphp::coro::async_stack_root* m_async_stack_root{}; }; template @@ -65,9 +66,9 @@ class await_broker { kphp::memory::script::free(ptr); } - void start_task(await_set_task&& task, kphp::coro::async_stack_root& coroutine_stack_root, void* return_address) noexcept { + void start_task(await_set_task&& task) noexcept { auto task_iterator{m_tasks_storage.insert(m_tasks_storage.begin(), std::move(task))}; - task_iterator->start(*this, task_iterator, coroutine_stack_root, return_address); + task_iterator->start(*this, task_iterator); } void push_ready_task(await_set_ready_task_element& ready_task) noexcept { @@ -81,6 +82,7 @@ class await_broker { m_awaiters->m_prev = nullptr; } awaiter->m_continuation.resume(); + awaiter->m_async_stack_root->stop_sync_stack_frame = nullptr; } } @@ -139,8 +141,8 @@ class await_broker { if (awaiters_to_resume != nullptr) { awaiters_to_resume->m_prev = nullptr; } - waiter->m_continuation.resume(); + waiter->m_async_stack_root->stop_sync_stack_frame = nullptr; } } @@ -151,8 +153,6 @@ class await_broker { ~await_broker() { abort_all(); } - -private: }; template @@ -160,6 +160,8 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { std::optional>> m_await_broker{}; await_set_ready_task_element m_ready_task_element{}; + kphp::coro::async_stack_root_wrapper root_wrapper_{}; + public: await_set_task_promise_base() noexcept = default; @@ -199,9 +201,8 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { kphp::log::error("internal unhandled exception"); } - auto start(detail::await_set::await_broker& await_broker, - kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, - kphp::coro::async_stack_root& async_stack_root, void* return_address) noexcept { + [[clang::noinline]] auto start(detail::await_set::await_broker& await_broker, + kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location) noexcept { m_await_broker = await_broker; m_ready_task_element.m_storage_location = storage_location; @@ -212,11 +213,12 @@ class await_set_task_promise_base : public kphp::coro::async_stack_element { * */ auto& async_stack_frame{get_async_stack_frame()}; async_stack_frame.caller_async_stack_frame = nullptr; - async_stack_frame.async_stack_root = std::addressof(async_stack_root); - async_stack_frame.return_address = return_address; + async_stack_frame.async_stack_root = std::addressof(root_wrapper_.root); + async_stack_frame.return_address = STACK_RETURN_ADDRESS; // this is necessary to prevent double printing of a coroutine async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); + kphp::coro::resume_with_new_root(handle, std::addressof(root_wrapper_.root)); } }; @@ -272,9 +274,8 @@ class await_set_task { }; auto start(detail::await_set::await_broker& await_broker, - kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location, - kphp::coro::async_stack_root& async_stack_root, void* return_address) noexcept { - m_coroutine.promise().start(await_broker, storage_location, async_stack_root, return_address); + kphp::stl::list, kphp::memory::script_allocator>::iterator storage_location) noexcept { + m_coroutine.promise().start(await_broker, storage_location); } public: @@ -335,6 +336,7 @@ class await_set_awaitable { caller_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); m_awaiter.m_continuation = awaiting_coroutine; + m_awaiter.m_async_stack_root = caller_frame->async_stack_root; m_suspended = m_await_broker.suspend_awaiter(m_awaiter); return m_suspended; } @@ -346,6 +348,8 @@ class await_set_awaitable { kphp::log::assertion(async_stack_root != nullptr); async_stack_root->top_async_stack_frame = caller_frame; + kphp::coro::preparation_for_resume(async_stack_root, STACK_FRAME_ADDRESS); + m_suspended = false; return m_await_broker.try_get_result(); } diff --git a/runtime-light/coroutine/detail/task-self-deleting.h b/runtime-light/coroutine/detail/task-self-deleting.h index daa72cdd17..8cd10f4241 100644 --- a/runtime-light/coroutine/detail/task-self-deleting.h +++ b/runtime-light/coroutine/detail/task-self-deleting.h @@ -69,7 +69,7 @@ class task_self_deleting { // initialize task_self_deleting's frame as top async stack frame auto& async_stack_frame{m_promise.get_async_stack_frame()}; async_stack_frame.return_address = STACK_RETURN_ADDRESS; - async_stack_frame.async_stack_root = std::addressof(CoroutineInstanceState::get().coroutine_stack_root); + async_stack_frame.async_stack_root = CoroutineInstanceState::get().current_async_stack_root; } ~task_self_deleting() noexcept = default; diff --git a/runtime-light/coroutine/detail/when-all.h b/runtime-light/coroutine/detail/when-all.h index cc276f7996..fa751029b9 100644 --- a/runtime-light/coroutine/detail/when-all.h +++ b/runtime-light/coroutine/detail/when-all.h @@ -25,6 +25,7 @@ namespace kphp::coro::detail::when_all { class when_all_latch { size_t m_count{1}; std::coroutine_handle<> m_awaiting_coroutine; + kphp::coro::async_stack_root* m_async_stack_root{}; public: explicit when_all_latch(size_t count) noexcept @@ -51,14 +52,16 @@ class when_all_latch { return m_awaiting_coroutine != nullptr && m_count == 1; } - auto try_await(std::coroutine_handle<> awaiting_coroutine) noexcept -> bool { + auto try_await(std::coroutine_handle<> awaiting_coroutine, kphp::coro::async_stack_root* root) noexcept -> bool { m_awaiting_coroutine = awaiting_coroutine; + m_async_stack_root = root; return m_count != 1; } auto notify_awaitable_completed() noexcept -> void { if (--m_count == 1 && m_awaiting_coroutine != nullptr) { m_awaiting_coroutine.resume(); + m_async_stack_root->stop_sync_stack_frame = nullptr; } } }; @@ -104,20 +107,20 @@ class when_all_ready_awaitable> { template caller_promise_type> [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> bool { // async stack frame handling - void* const return_address{STACK_RETURN_ADDRESS}; m_caller_async_stack_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); - std::apply([&latch = m_awaitable.m_latch, &caller_async_stack_frame = *m_caller_async_stack_frame, - return_address](auto&... tasks) noexcept { (tasks.start(latch, caller_async_stack_frame, return_address), ...); }, - m_awaitable.m_tasks); - return m_awaitable.m_latch.try_await(awaiting_coroutine); + std::apply([&latch = m_awaitable.m_latch](auto&... tasks) noexcept { (tasks.start(latch), ...); }, m_awaitable.m_tasks); + return m_awaitable.m_latch.try_await(awaiting_coroutine, m_caller_async_stack_frame->async_stack_root); } auto await_resume() noexcept { // restore caller's async_stack_frame unless it's not set which could happen in case no suspension occured if (m_caller_async_stack_frame != nullptr) { - kphp::log::assertion(m_caller_async_stack_frame->async_stack_root != nullptr); - m_caller_async_stack_frame->async_stack_root->top_async_stack_frame = m_caller_async_stack_frame; + auto* root = m_caller_async_stack_frame->async_stack_root; + kphp::log::assertion(root != nullptr); + root->top_async_stack_frame = m_caller_async_stack_frame; + + kphp::coro::preparation_for_resume(root, STACK_FRAME_ADDRESS); } return std::apply([](task_types&&... tasks) noexcept { return std::make_tuple(std::move(tasks).result()...); }, std::move(m_awaitable.m_tasks)); } @@ -146,6 +149,7 @@ class when_all_ready_awaitable> { template class when_all_task_promise_base : public kphp::coro::async_stack_element { when_all_latch* m_latch{}; + kphp::coro::async_stack_root_wrapper root_wrapper_{}; public: when_all_task_promise_base() noexcept = default; @@ -182,17 +186,18 @@ class when_all_task_promise_base : public kphp::coro::async_stack_element { kphp::log::error("internal unhandled exception"); } - auto start(when_all_latch& latch, kphp::coro::async_stack_frame& caller_async_stack_frame, void* return_address) noexcept { + [[clang::noinline]] auto start(when_all_latch& latch) noexcept { m_latch = std::addressof(latch); auto& async_stack_frame{get_async_stack_frame()}; - kphp::log::assertion(caller_async_stack_frame.async_stack_root != nullptr); // initialize when_all_task's async stack frame and make it the top frame - async_stack_frame.caller_async_stack_frame = std::addressof(caller_async_stack_frame); - async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; - async_stack_frame.return_address = return_address; + async_stack_frame.caller_async_stack_frame = nullptr; + async_stack_frame.async_stack_root = std::addressof(root_wrapper_.root); + async_stack_frame.return_address = STACK_RETURN_ADDRESS; // this is necessary to prevent double printing of a coroutine async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + + decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); + kphp::coro::resume_with_new_root(handle, std::addressof(root_wrapper_.root)); } }; @@ -230,7 +235,7 @@ class when_all_task { } auto result() noexcept -> T { - kphp::log::assertion(m_result); + kphp::log::assertion(static_cast(m_result)); return std::move(*m_result); } @@ -247,8 +252,8 @@ class when_all_task { constexpr auto return_void() const noexcept -> void {} }; - auto start(when_all_latch& latch, kphp::coro::async_stack_frame& caller_async_stack_frame, void* return_address) noexcept -> void { - m_coroutine.promise().start(latch, caller_async_stack_frame, return_address); + auto start(when_all_latch& latch) noexcept -> void { + m_coroutine.promise().start(latch); } public: diff --git a/runtime-light/coroutine/detail/when-any.h b/runtime-light/coroutine/detail/when-any.h index f124f5928e..b4632b8468 100644 --- a/runtime-light/coroutine/detail/when-any.h +++ b/runtime-light/coroutine/detail/when-any.h @@ -12,6 +12,7 @@ #include #include +#include "runtime-light/coroutine/async-stack.h" #include "runtime-light/coroutine/concepts.h" #include "runtime-light/coroutine/type-traits.h" #include "runtime-light/coroutine/void-value.h" @@ -23,6 +24,7 @@ namespace kphp::coro::detail::when_any { class when_any_latch { bool m_toggled{}; std::coroutine_handle<> m_awaiting_coroutine; + kphp::coro::async_stack_root* m_async_stack_root{}; public: when_any_latch() noexcept = default; @@ -48,8 +50,9 @@ class when_any_latch { return m_toggled; } - auto try_await(std::coroutine_handle<> awaiting_coroutine) noexcept -> bool { + auto try_await(std::coroutine_handle<> awaiting_coroutine, kphp::coro::async_stack_root* root) noexcept -> bool { m_awaiting_coroutine = awaiting_coroutine; + m_async_stack_root = root; return !m_toggled; } @@ -57,6 +60,7 @@ class when_any_latch { m_toggled = true; if (m_awaiting_coroutine != nullptr) { m_awaiting_coroutine.resume(); + m_async_stack_root->stop_sync_stack_frame = nullptr; } } }; @@ -103,20 +107,20 @@ class when_any_ready_awaitable> { template caller_promise_type> [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> bool { // async stack frame handling - void* const return_address{STACK_RETURN_ADDRESS}; m_caller_async_stack_frame = std::addressof(awaiting_coroutine.promise().get_async_stack_frame()); - std::apply([&latch = m_awaitable.m_latch, &caller_async_stack_frame = *m_caller_async_stack_frame, - return_address](auto&... tasks) noexcept { (tasks.start(latch, caller_async_stack_frame, return_address), ...); }, - m_awaitable.m_tasks); - return m_awaitable.m_latch.try_await(awaiting_coroutine); + std::apply([&latch = m_awaitable.m_latch](auto&... tasks) noexcept { (tasks.start(latch), ...); }, m_awaitable.m_tasks); + return m_awaitable.m_latch.try_await(awaiting_coroutine, m_caller_async_stack_frame->async_stack_root); } auto await_resume() noexcept { // restore caller's async_stack_frame unless it's not set which could happen in case no suspension occured if (m_caller_async_stack_frame != nullptr) { - kphp::log::assertion(m_caller_async_stack_frame->async_stack_root != nullptr); - m_caller_async_stack_frame->async_stack_root->top_async_stack_frame = m_caller_async_stack_frame; + auto* root = m_caller_async_stack_frame->async_stack_root; + kphp::log::assertion(root != nullptr); + root->top_async_stack_frame = m_caller_async_stack_frame; + + kphp::coro::preparation_for_resume(root, STACK_FRAME_ADDRESS); } const auto task_result_processor{[&result = m_awaitable.m_result](auto&& task) noexcept { @@ -155,6 +159,7 @@ class when_any_ready_awaitable> { template class when_any_task_promise_base : public kphp::coro::async_stack_element { when_any_latch* m_latch{}; + kphp::coro::async_stack_root_wrapper root_wrapper_{}; public: when_any_task_promise_base() noexcept = default; @@ -191,17 +196,18 @@ class when_any_task_promise_base : public kphp::coro::async_stack_element { kphp::log::error("internal unhandled exception"); } - auto start(when_any_latch& latch, kphp::coro::async_stack_frame& caller_async_stack_frame, void* return_address) noexcept { + [[clang::noinline]] auto start(when_any_latch& latch) noexcept { m_latch = std::addressof(latch); auto& async_stack_frame{get_async_stack_frame()}; - kphp::log::assertion(caller_async_stack_frame.async_stack_root != nullptr); // initialize when_all_task's async stack frame and make it the top frame - async_stack_frame.caller_async_stack_frame = std::addressof(caller_async_stack_frame); - async_stack_frame.async_stack_root = caller_async_stack_frame.async_stack_root; - async_stack_frame.return_address = return_address; + async_stack_frame.caller_async_stack_frame = nullptr; + async_stack_frame.async_stack_root = std::addressof(root_wrapper_.root); + async_stack_frame.return_address = STACK_RETURN_ADDRESS; // this is necessary to prevent double printing of a coroutine async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); - std::coroutine_handle::from_promise(*static_cast(this)).resume(); + + decltype(auto) handle = std::coroutine_handle::from_promise(*static_cast(this)); + kphp::coro::resume_with_new_root(handle, std::addressof(root_wrapper_.root)); } }; @@ -260,9 +266,9 @@ class when_any_task { constexpr auto return_void() const noexcept -> void {} }; - auto start(when_any_latch& latch, kphp::coro::async_stack_frame& caller_async_stack_frame, void* return_address) noexcept -> void { + auto start(when_any_latch& latch) noexcept -> void { if (!latch.is_ready()) [[likely]] { - m_coroutine.promise().start(latch, caller_async_stack_frame, return_address); + m_coroutine.promise().start(latch); } } diff --git a/runtime-light/coroutine/event.h b/runtime-light/coroutine/event.h index 8e428b72ce..3cf9e3fe96 100644 --- a/runtime-light/coroutine/event.h +++ b/runtime-light/coroutine/event.h @@ -19,20 +19,20 @@ class event { // 2) awaiter* => linked list of awaiters waiting for the event to trigger // 3) this => The event is triggered and all awaiters are resumed void* m_state{}; + kphp::coro::async_stack_root* m_async_stack_root{}; struct awaiter { event& m_event; bool m_suspended{}; std::coroutine_handle<> m_awaiting_coroutine; - kphp::coro::async_stack_root& m_async_stack_root; - kphp::coro::async_stack_frame* m_caller_async_stack_frame{}; + CoroutineInstanceState& m_coroutine_state; awaiter* m_next{}; awaiter* m_prev{}; explicit awaiter(event& event) noexcept : m_event(event), - m_async_stack_root(CoroutineInstanceState::get().coroutine_stack_root) {} + m_coroutine_state(CoroutineInstanceState::get()) {} awaiter(const awaiter&) = delete; awaiter(awaiter&&) = delete; @@ -84,8 +84,7 @@ inline auto event::awaiter::await_ready() const noexcept -> bool { template caller_promise_type> auto event::awaiter::await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> void { - // save caller's async stack frame - m_caller_async_stack_frame = m_async_stack_root.top_async_stack_frame; + m_event.m_async_stack_root = m_coroutine_state.current_async_stack_root; m_suspended = true; m_awaiting_coroutine = awaiting_coroutine; @@ -103,9 +102,7 @@ auto event::awaiter::await_suspend(std::coroutine_handle aw inline auto event::awaiter::await_resume() noexcept -> void { if (std::exchange(m_suspended, false)) { - // restore caller's async stack frame if it was suspended - kphp::log::assertion(m_caller_async_stack_frame != nullptr); - m_async_stack_root.top_async_stack_frame = std::exchange(m_caller_async_stack_frame, nullptr); + kphp::coro::preparation_for_resume(m_event.m_async_stack_root, STACK_FRAME_ADDRESS, m_coroutine_state); } } @@ -118,6 +115,7 @@ inline auto event::set() noexcept -> void { while (awaiter != nullptr) { auto* next{awaiter->m_next}; awaiter->m_awaiting_coroutine.resume(); + m_async_stack_root->stop_sync_stack_frame = nullptr; awaiter = next; } } diff --git a/runtime-light/coroutine/io-scheduler.h b/runtime-light/coroutine/io-scheduler.h index 657dae1539..9e93b236f9 100644 --- a/runtime-light/coroutine/io-scheduler.h +++ b/runtime-light/coroutine/io-scheduler.h @@ -395,9 +395,7 @@ inline auto io_scheduler::process_events() noexcept -> k2::PollStatus { kphp::log::assertion(m_scheduled_tasks_tmp.empty()); std::swap(m_scheduled_tasks, m_scheduled_tasks_tmp); - std::ranges::for_each(m_scheduled_tasks_tmp, [&coroutine_stack_root = m_coroutine_instance_state.coroutine_stack_root](const auto& coroutine) noexcept { - kphp::coro::resume(coroutine, coroutine_stack_root); - }); + std::ranges::for_each(m_scheduled_tasks_tmp, [](const auto& coroutine) noexcept { coroutine.resume(); }); m_scheduled_tasks_tmp.clear(); } @@ -422,7 +420,7 @@ auto io_scheduler::start(coroutine_type coroutine) noexcept -> bool { if (!handle || handle.done()) [[unlikely]] { return false; } - kphp::coro::resume(handle, m_coroutine_instance_state.coroutine_stack_root); + kphp::coro::resume_with_new_root(handle, m_coroutine_instance_state.current_async_stack_root); return true; } @@ -430,11 +428,11 @@ inline auto io_scheduler::schedule() noexcept { class schedule_operation { friend class io_scheduler; io_scheduler& m_scheduler; - kphp::coro::async_stack_frame* const m_async_stack_frame{}; + kphp::coro::async_stack_root* const m_async_stack_root{}; explicit schedule_operation(io_scheduler& scheduler) noexcept : m_scheduler(scheduler), - m_async_stack_frame(m_scheduler.m_coroutine_instance_state.coroutine_stack_root.top_async_stack_frame) {} + m_async_stack_root(m_scheduler.m_coroutine_instance_state.current_async_stack_root) {} public: constexpr auto await_ready() const noexcept -> bool { @@ -446,7 +444,7 @@ inline auto io_scheduler::schedule() noexcept { } auto await_resume() const noexcept -> void { - m_scheduler.m_coroutine_instance_state.coroutine_stack_root.top_async_stack_frame = m_async_stack_frame; + kphp::coro::preparation_for_resume(m_async_stack_root, STACK_FRAME_ADDRESS, m_scheduler.m_coroutine_instance_state); } }; return schedule_operation{*this}; diff --git a/runtime-light/coroutine/shared-task.h b/runtime-light/coroutine/shared-task.h index 76957148d9..5188ebf35d 100644 --- a/runtime-light/coroutine/shared-task.h +++ b/runtime-light/coroutine/shared-task.h @@ -29,12 +29,23 @@ struct shared_task_awaiter final { template struct promise_base : kphp::coro::async_stack_element { +private: + mutable kphp::coro::async_stack_root_wrapper root_wrapper_{}; + +public: constexpr auto initial_suspend() const noexcept -> std::suspend_always { return {}; } constexpr auto final_suspend() const noexcept { struct awaiter { + private: + mutable kphp::coro::async_stack_root* root; + + public: + awaiter(kphp::coro::async_stack_root* prt) + : root{prt} {} + constexpr auto await_ready() const noexcept -> bool { return false; } @@ -51,8 +62,11 @@ struct promise_base : kphp::coro::async_stack_element { // read the m_next pointer before resuming the coroutine // since resuming the coroutine may destroy the shared_task_waiter value auto* next{awaiter->m_next}; - auto& async_stack_root{*promise.get_async_stack_frame().async_stack_root}; - kphp::coro::resume(awaiter->m_continuation, async_stack_root); + auto& async_stack_frame{promise.get_async_stack_frame()}; + + root->top_async_stack_frame = std::addressof(async_stack_frame); + async_stack_frame.async_stack_root = root; + kphp::coro::resume_with_new_root(awaiter->m_continuation, root); awaiter = next; } // return last awaiter's coroutine_handle to allow it to potentially be compiled as a tail-call @@ -61,7 +75,7 @@ struct promise_base : kphp::coro::async_stack_element { constexpr auto await_resume() const noexcept -> void {} }; - return awaiter{}; + return awaiter{std::addressof(root_wrapper_.root)}; } auto unhandled_exception() const noexcept -> void { @@ -99,8 +113,11 @@ struct promise_base : kphp::coro::async_stack_element { if (m_awaiters == NOT_STARTED_VAL) { m_awaiters = STARTED_NO_WAITERS_VAL; const auto& handle{std::coroutine_handle::from_promise(*static_cast(this))}; - auto& async_stack_root{*get_async_stack_frame().async_stack_root}; - kphp::coro::resume(handle, async_stack_root); + auto& async_stack_frame{get_async_stack_frame()}; + + async_stack_frame.async_stack_root = std::addressof(root_wrapper_.root); + async_stack_frame.async_stack_root->top_async_stack_frame = std::addressof(async_stack_frame); + kphp::coro::resume_with_new_root(handle, std::addressof(root_wrapper_.root)); } // coroutine already completed, don't suspend if (done()) { @@ -168,24 +185,13 @@ template class awaiter_base { bool m_suspended{}; - void set_async_top_frame(async_stack_frame& caller_frame, void* return_address) noexcept { + void set_async_top_frame(void* return_address) noexcept { /** * shared_task is the top of the stack for calls from it. * Therefore, it's awaiter doesn't store caller_frame, but it save `await_suspend()` return address * */ async_stack_frame& callee_frame{m_coro.promise().get_async_stack_frame()}; - callee_frame.return_address = return_address; - auto* async_stack_root{caller_frame.async_stack_root}; - kphp::log::assertion(async_stack_root != nullptr); - callee_frame.async_stack_root = async_stack_root; - async_stack_root->top_async_stack_frame = std::addressof(callee_frame); - } - - void reset_async_top_frame(async_stack_frame& caller_frame) noexcept { - auto* async_stack_root{caller_frame.async_stack_root}; - kphp::log::assertion(async_stack_root != nullptr); - async_stack_root->top_async_stack_frame = std::addressof(caller_frame); } protected: @@ -217,14 +223,15 @@ class awaiter_base { template caller_promise_type> [[clang::noinline]] auto await_suspend(std::coroutine_handle awaiting_coroutine) noexcept -> bool { - set_async_top_frame(awaiting_coroutine.promise().get_async_stack_frame(), STACK_RETURN_ADDRESS); + set_async_top_frame(STACK_RETURN_ADDRESS); m_awaiter.m_continuation = awaiting_coroutine; m_suspended = m_coro.promise().suspend_awaiter(m_awaiter); - reset_async_top_frame(awaiting_coroutine.promise().get_async_stack_frame()); return m_suspended; } auto await_resume() noexcept -> void { + auto& frame = m_coro.promise().get_async_stack_frame(); + kphp::coro::preparation_for_resume(frame.async_stack_root, STACK_FRAME_ADDRESS); m_suspended = false; } }; diff --git a/runtime-light/coroutine/task.h b/runtime-light/coroutine/task.h index 7ba3fb2041..93a70e61ca 100644 --- a/runtime-light/coroutine/task.h +++ b/runtime-light/coroutine/task.h @@ -133,6 +133,8 @@ class awaiter_base { } auto await_resume() noexcept -> void { + auto& frame = m_coro.promise().get_async_stack_frame(); + kphp::coro::preparation_for_resume(frame.async_stack_root, STACK_FRAME_ADDRESS); m_suspended = false; } }; diff --git a/runtime-light/state/instance-state.cpp b/runtime-light/state/instance-state.cpp index 64334b904d..11ae6facf2 100644 --- a/runtime-light/state/instance-state.cpp +++ b/runtime-light/state/instance-state.cpp @@ -76,8 +76,8 @@ void InstanceState::init_script_execution() noexcept { std::move(script_task))}; // initialize async stack auto& main_task_async_stack_frame{main_task.get_handle().promise().get_async_stack_frame()}; - main_task_async_stack_frame.async_stack_root = std::addressof(coroutine_instance_state.coroutine_stack_root); - coroutine_instance_state.coroutine_stack_root.top_async_stack_frame = std::addressof(main_task_async_stack_frame); + main_task_async_stack_frame.async_stack_root = coroutine_instance_state.current_async_stack_root; + coroutine_instance_state.current_async_stack_root->top_async_stack_frame = std::addressof(main_task_async_stack_frame); // spawn main task onto the scheduler kphp::log::assertion(io_scheduler.spawn(std::move(main_task))); } diff --git a/runtime-light/stdlib/diagnostics/backtrace.cpp b/runtime-light/stdlib/diagnostics/backtrace.cpp index 6ec787e497..490c585420 100644 --- a/runtime-light/stdlib/diagnostics/backtrace.cpp +++ b/runtime-light/stdlib/diagnostics/backtrace.cpp @@ -15,11 +15,11 @@ namespace { -size_t sync_frames(std::span addresses, kphp::coro::stack_frame* start_frame, kphp::coro::stack_frame* stop_frame) noexcept { +size_t sync_frames(std::span addresses, const kphp::coro::stack_frame* start_frame, const kphp::coro::stack_frame* stop_frame) noexcept { kphp::log::assertion(start_frame != nullptr && stop_frame != nullptr); auto address{addresses.begin()}; - for (auto* current_stack_frame{start_frame}; current_stack_frame != stop_frame && address != addresses.end(); + for (const auto* current_stack_frame{start_frame}; current_stack_frame != stop_frame && address != addresses.end(); current_stack_frame = current_stack_frame->caller_stack_frame) { *address = current_stack_frame->return_address; address = std::next(address); @@ -43,16 +43,27 @@ size_t async_frames(std::span addresses, kphp::coro::async_stack_frame* t namespace kphp::diagnostic::impl { -size_t async_backtrace(std::span addresses) noexcept { - if (const auto* instance_state{k2::instance_state()}; instance_state != nullptr) [[likely]] { - const auto& async_stack_root{instance_state->coroutine_instance_state.coroutine_stack_root}; +size_t async_backtrace_helper(std::span addresses, const kphp::coro::stack_frame* start_sync_frame, + const kphp::coro::async_stack_root* async_stack_root) noexcept { + const auto* const stop_sync_frame{async_stack_root->stop_sync_stack_frame}; + + size_t num_sync_frames{0}; + if (start_sync_frame && stop_sync_frame) { + num_sync_frames = sync_frames(addresses, start_sync_frame, stop_sync_frame); + } + const size_t num_async_frames{async_frames(addresses.subspan(num_sync_frames), async_stack_root->top_async_stack_frame)}; - auto* const start_sync_frame{reinterpret_cast(STACK_FRAME_ADDRESS)}; - auto* const stop_sync_frame{async_stack_root.stop_sync_stack_frame}; + const size_t result{num_sync_frames + num_async_frames}; + if (async_stack_root == async_stack_root->next_async_stack_root) { + return result; + } + return result + async_backtrace_helper(addresses.subspan(result), stop_sync_frame, async_stack_root->next_async_stack_root); +} - const size_t num_sync_frames{sync_frames(addresses, start_sync_frame, stop_sync_frame)}; - const size_t num_async_frames{async_frames(addresses.subspan(num_sync_frames), async_stack_root.top_async_stack_frame)}; - return num_sync_frames + num_async_frames; +size_t async_backtrace(std::span addresses) noexcept { + if (const auto* instance_state{k2::instance_state()}; instance_state != nullptr) [[likely]] { + return async_backtrace_helper(addresses, reinterpret_cast(STACK_FRAME_ADDRESS), + instance_state->coroutine_instance_state.current_async_stack_root); } return 0; }