From 6f5b38daac33fcc780cf39f25f4158edbd59981e Mon Sep 17 00:00:00 2001 From: b-pass Date: Wed, 14 May 2025 22:28:13 -0400 Subject: [PATCH 01/31] First draft a subinterpreter embedding API --- include/pybind11/gil.h | 2 +- include/pybind11/subinterpreter.h | 205 ++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 include/pybind11/subinterpreter.h diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index 1a9bfeaddf..e90ea41528 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -130,7 +130,7 @@ class gil_scoped_acquire { } /// This method will disable the PyThreadState_DeleteCurrent call and the - /// GIL won't be acquired. This method should be used if the interpreter + /// GIL won't be released. This method should be used if the interpreter /// could be shutting down when this is called, as thread deletion is not /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and /// protect subsequent code. diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h new file mode 100644 index 0000000000..0abbd7b3d2 --- /dev/null +++ b/include/pybind11/subinterpreter.h @@ -0,0 +1,205 @@ +#pragma once + +#include "detail/common.h" +#include "detail/internals.h" +#include "gil.h" + +#include + +#if !defined(PYBIND11_SUBINTERPRETER_SUPPORT) +# error "This platform does not support subinterpreters, do not include this file." +#endif + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +class subinterpreter; + +/// Activate the subinterpreter and acquire it's GIL, while also releasing any GIL and interpreter +/// currently held. Upon exiting the scope, the previous subinterpreter (if any) and it's +/// associated GIL are restored to their state as they were before the scope was entered. +class subinterpreter_scoped_activate { +public: + explicit subinterpreter_scoped_activate(subinterpreter const &si); + ~subinterpreter_scoped_activate(); + +private: + PyThreadState *old_tstate_ = nullptr; + PyThreadState *free_tstate_ = nullptr; + PyGILState_STATE gil_state_; + bool simple_gil_ = false; +}; + +class subinterpreter { +public: + subinterpreter() = default; + subinterpreter(subinterpreter const ©) = delete; + subinterpreter &operator=(subinterpreter const ©) = delete; + + subinterpreter(subinterpreter &&old) : tstate_(old.tstate_), istate_(old.istate_) { + old.tstate_ = nullptr; + old.istate_ = nullptr; + } + + subinterpreter &operator=(subinterpreter &&old) { + std::swap(old.tstate_, tstate_); + std::swap(old.istate_, istate_); + return *this; + } + + ~subinterpreter() { + if (tstate_) { + if (PyThread_get_thread_ident() != tstate_->native_thread_id) { + // Throwing from destructors is bad :( + // But if we don't throw, we either leak the interpreter or the code hangs because + // internal Python TSS values are wrong/missing + throw std::runtime_error( + "wrong thread called subinterpreter destruct. subinterpreters can only " + "destruct on the thread that created them!"); + } + + // has to be the active interpreter in order to call End on it + // switch into the expiring interpreter + auto old_tstate = PyThreadState_Swap(tstate_); + + // make sure we have the GIL + (void) PyGILState_Ensure(); + + // End it + Py_EndInterpreter(tstate_); + + // switch back to the old tstate and old GIL (if there was one) + if (old_tstate != tstate_) + PyThreadState_Swap(old_tstate); + + // do NOT decrease detail::get_num_interpreters_seen, because it can never decrease + // while other threads are running... + } + } + + /// abandon cleanup of this subinterpreter. might be needed during finalization + void disarm() { tstate_ = nullptr; } + + /// Get a handle to the main interpreter that can be used with subinterpreter_scoped_activate + /// Note that destructing the handle is a noop, the main interpreter can only be ended by + /// py::finalize_interpreter() + static subinterpreter_scoped_activate main_scoped_activate() { + subinterpreter m; + m.istate_ = PyInterpreterState_Main(); + m.disarm(); // make destruct a noop + return subinterpreter_scoped_activate(m); + } + + /// Create a new subinterpreter with the specified configuration + /// Note Well: + static inline subinterpreter create(PyInterpreterConfig const &cfg) { + error_scope err_scope; + auto main_guard = main_scoped_activate(); + subinterpreter result; + { + // we must hold the main GIL in order to create a subinterpreter + gil_scoped_acquire gil; + + auto prev_tstate = PyThreadState_Get(); + + auto status = Py_NewInterpreterFromConfig(&result.tstate_, &cfg); + + // this doesn't raise a normal Python exception, it provides an exit() status code. + if (PyStatus_Exception(status)) { + pybind11_fail("failed to create new sub-interpreter"); + } + + // upon success, the new interpreter is activated in this thread + result.istate_ = result.tstate_->interp; + detail::get_num_interpreters_seen() += 1; // there are now many interpreters + detail::get_internals(); // initialize internals.tstate, amongst other things... + + // we have to switch back to main, and then the scopes will handle cleanup + PyThreadState_Swap(prev_tstate); + } + return result; + } + + /// Call create() with a default configuration of an isolated interpreter that disallows fork, + /// exec, and Python threads. + static inline subinterpreter create() { + // same as the default config in the python docs + PyInterpreterConfig cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.check_multi_interp_extensions = 1; + cfg.gil = PyInterpreterConfig_OWN_GIL; + return create(cfg); + } + +private: + friend class subinterpreter_scoped_activate; + PyThreadState *tstate_ = nullptr; + PyInterpreterState *istate_ = nullptr; +}; + +subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpreter const &si) { +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + if (!si.istate_ || !si.tstate_) { + pybind11_fail("null subinterpreter"); + } +#endif + + auto cur_tstate = detail::get_thread_state_unchecked(); + if (cur_tstate && cur_tstate->interp == si.istate_) { + // we are already on this interpreter, make sure we hold the GIL + simple_gil_ = true; + gil_state_ = PyGILState_Ensure(); + return; + } + + PyThreadState *desired_tstate = nullptr; + + // get the state dict for the interpreter we want + dict idict = reinterpret_borrow(PyInterpreterState_GetDict(si.istate_)); + // and get the internals from it + auto *internals_pp = detail::get_internals_pp_from_capsule_in_state_dict( + idict, PYBIND11_INTERNALS_ID); + if (internals_pp && *internals_pp) { + // see if there is already a tstate for this thread + desired_tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE((*internals_pp)->tstate); + if (!desired_tstate) { + // nope, we have to create one. + desired_tstate = PyThreadState_New(si.istate_); + free_tstate_ = desired_tstate; +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + if (!desired_tstate) { + pybind11_fail("subinterpreter_scoped_activate: could not create thread state!"); + } +#endif + PYBIND11_TLS_REPLACE_VALUE((*internals_pp)->tstate, desired_tstate); + } + } else { + desired_tstate = PyThreadState_New(si.istate_); + free_tstate_ = desired_tstate; + } + + // make the interpreter active and acquire the GIL + old_tstate_ = PyThreadState_Swap(desired_tstate); +} + +subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { + if (simple_gil_) { + // We were on this interpreter already, so just make sure the GIL goes back as it was + PyGILState_Release(gil_state_); + } else { + if (free_tstate_) { +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + if (detail::get_thread_state_unchecked() != free_tstate_) { + pybind11_fail("~subinterpreter_scoped_activate: thread state must be current!"); + } +#endif + PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate); + PyThreadState_Clear(free_tstate_); + PyThreadState_DeleteCurrent(); + } + + // Go back the previous interpreter (if any) and acquire THAT gil + PyThreadState_Swap(old_tstate_); + } +} + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) From ee42fe5b7d0daea9f1ae46975b49cf8eda6f21b4 Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 18:09:15 -0400 Subject: [PATCH 02/31] Move subinterpreter tests to their own file --- tests/test_embed/CMakeLists.txt | 2 +- tests/test_embed/test_interpreter.cpp | 304 ++--------------------- tests/test_embed/test_subinterpreter.cpp | 288 +++++++++++++++++++++ 3 files changed, 305 insertions(+), 289 deletions(-) create mode 100644 tests/test_embed/test_subinterpreter.cpp diff --git a/tests/test_embed/CMakeLists.txt b/tests/test_embed/CMakeLists.txt index 1a537a6580..af3c848ad4 100644 --- a/tests/test_embed/CMakeLists.txt +++ b/tests/test_embed/CMakeLists.txt @@ -28,7 +28,7 @@ endif() find_package(Threads REQUIRED) -add_executable(test_embed catch.cpp test_interpreter.cpp) +add_executable(test_embed catch.cpp test_interpreter.cpp test_subinterpreter.cpp) pybind11_enable_warnings(test_embed) target_link_libraries(test_embed PRIVATE pybind11::embed Catch2::Catch2 Threads::Threads) diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 6e4be7378a..cbd29783b7 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -1,4 +1,5 @@ #include +#include // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). @@ -19,6 +20,21 @@ size_t get_sys_path_size() { return py::len(sys_path); } +bool has_state_dict_internals_obj() { + py::dict state = py::detail::get_python_state_dict(); + return state.contains(PYBIND11_INTERNALS_ID); +} + +bool has_pybind11_internals_static() { + auto *&ipp = py::detail::get_internals_pp(); + return (ipp != nullptr) && *ipp; +} + +uintptr_t get_details_as_uintptr() { + return reinterpret_cast( + py::detail::get_internals_pp()->get()); +} + class Widget { public: explicit Widget(std::string message) : message(std::move(message)) {} @@ -258,21 +274,6 @@ TEST_CASE("Add program dir to path using PyConfig") { } #endif -bool has_state_dict_internals_obj() { - py::dict state = py::detail::get_python_state_dict(); - return state.contains(PYBIND11_INTERNALS_ID); -} - -bool has_pybind11_internals_static() { - auto *&ipp = py::detail::get_internals_pp(); - return (ipp != nullptr) && *ipp; -} - -uintptr_t get_details_as_uintptr() { - return reinterpret_cast( - py::detail::get_internals_pp()->get()); -} - TEST_CASE("Restart the interpreter") { // Verify pre-restart state. REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast() == 3); @@ -336,279 +337,6 @@ TEST_CASE("Restart the interpreter") { REQUIRE(py_widget.attr("the_message").cast() == "Hello after restart"); } -#if defined(PYBIND11_SUBINTERPRETER_SUPPORT) -TEST_CASE("Subinterpreter") { - py::module_::import("external_module"); // in the main interpreter - - // Add tags to the modules in the main interpreter and test the basics. - py::module_::import("__main__").attr("main_tag") = "main interpreter"; - { - auto m = py::module_::import("widget_module"); - m.attr("extension_module_tag") = "added to module in main interpreter"; - - REQUIRE(m.attr("add")(1, 2).cast() == 3); - } - - auto main_int - = py::module_::import("external_module").attr("internals_at")().cast(); - - REQUIRE(has_state_dict_internals_obj()); - REQUIRE(has_pybind11_internals_static()); - - /// Create and switch to a subinterpreter. - auto *main_tstate = PyThreadState_Get(); - auto *sub_tstate = Py_NewInterpreter(); - - py::detail::get_num_interpreters_seen()++; - - // Subinterpreters get their own copy of builtins. - REQUIRE_FALSE(has_state_dict_internals_obj()); - - // internals hasn't been populated yet, but will be different for the subinterpreter - REQUIRE_FALSE(has_pybind11_internals_static()); - - py::list(py::module_::import("sys").attr("path")).append(py::str(".")); - - auto ext_int = py::module_::import("external_module").attr("internals_at")().cast(); - py::detail::get_internals(); - REQUIRE(has_pybind11_internals_static()); - REQUIRE(get_details_as_uintptr() == ext_int); - REQUIRE(main_int != ext_int); - - // Modules tags should be gone. - REQUIRE_FALSE(py::hasattr(py::module_::import("__main__"), "tag")); - { - REQUIRE_NOTHROW(py::module_::import("widget_module")); - auto m = py::module_::import("widget_module"); - REQUIRE_FALSE(py::hasattr(m, "extension_module_tag")); - - // Function bindings should still work. - REQUIRE(m.attr("add")(1, 2).cast() == 3); - } - - // The subinterpreter now has internals populated since we imported a pybind11 module - REQUIRE(has_pybind11_internals_static()); - - // Restore main interpreter. - Py_EndInterpreter(sub_tstate); - py::detail::get_num_interpreters_seen() = 1; - PyThreadState_Swap(main_tstate); - - REQUIRE(py::hasattr(py::module_::import("__main__"), "main_tag")); - REQUIRE(py::hasattr(py::module_::import("widget_module"), "extension_module_tag")); - REQUIRE(has_state_dict_internals_obj()); -} - -TEST_CASE("Multiple Subinterpreters") { - // Make sure the module is in the main interpreter and save its pointer - auto *main_ext = py::module_::import("external_module").ptr(); - auto main_int - = py::module_::import("external_module").attr("internals_at")().cast(); - py::module_::import("external_module").attr("multi_interp") = "1"; - - auto *main_tstate = PyThreadState_Get(); - - /// Create and switch to a subinterpreter. - auto *sub1_tstate = Py_NewInterpreter(); - py::detail::get_num_interpreters_seen()++; - - py::list(py::module_::import("sys").attr("path")).append(py::str(".")); - - // The subinterpreter has its own copy of this module which is completely separate from main - auto *sub1_ext = py::module_::import("external_module").ptr(); - REQUIRE(sub1_ext != main_ext); - REQUIRE_FALSE(py::hasattr(py::module_::import("external_module"), "multi_interp")); - py::module_::import("external_module").attr("multi_interp") = "2"; - // The subinterpreter also has its own internals - auto sub1_int - = py::module_::import("external_module").attr("internals_at")().cast(); - REQUIRE(sub1_int != main_int); - - // Create another interpreter - auto *sub2_tstate = Py_NewInterpreter(); - py::detail::get_num_interpreters_seen()++; - - py::list(py::module_::import("sys").attr("path")).append(py::str(".")); - - // The second subinterpreter is separate from both main and the other subinterpreter - auto *sub2_ext = py::module_::import("external_module").ptr(); - REQUIRE(sub2_ext != main_ext); - REQUIRE(sub2_ext != sub1_ext); - REQUIRE_FALSE(py::hasattr(py::module_::import("external_module"), "multi_interp")); - py::module_::import("external_module").attr("multi_interp") = "3"; - // The subinterpreter also has its own internals - auto sub2_int - = py::module_::import("external_module").attr("internals_at")().cast(); - REQUIRE(sub2_int != main_int); - REQUIRE(sub2_int != sub1_int); - - PyThreadState_Swap(sub1_tstate); // go back to sub1 - - REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) - == "2"); - - PyThreadState_Swap(main_tstate); // go back to main - - auto post_int - = py::module_::import("external_module").attr("internals_at")().cast(); - // Make sure internals went back the way it was before - REQUIRE(main_int == post_int); - - REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) - == "1"); - - PyThreadState_Swap(sub1_tstate); - Py_EndInterpreter(sub1_tstate); - PyThreadState_Swap(sub2_tstate); - Py_EndInterpreter(sub2_tstate); - - py::detail::get_num_interpreters_seen() = 1; - PyThreadState_Swap(main_tstate); -} -#endif - -#if defined(Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) && defined(PYBIND11_SUBINTERPRETER_SUPPORT) -TEST_CASE("Per-Subinterpreter GIL") { - auto main_int - = py::module_::import("external_module").attr("internals_at")().cast(); - - std::atomic started, sync, failure; - started = 0; - sync = 0; - failure = 0; - -// REQUIRE throws on failure, so we can't use it within the thread -# define T_REQUIRE(status) \ - do { \ - assert(status); \ - if (!(status)) \ - ++failure; \ - } while (0) - - auto &&thread_main = [&](int num) { - while (started == 0) - std::this_thread::sleep_for(std::chrono::microseconds(1)); - ++started; - - py::gil_scoped_acquire gil; - auto main_tstate = PyThreadState_Get(); - - // we have the GIL, we can access the main interpreter - auto t_int - = py::module_::import("external_module").attr("internals_at")().cast(); - T_REQUIRE(t_int == main_int); - py::module_::import("external_module").attr("multi_interp") = "1"; - - PyThreadState *sub = nullptr; - PyInterpreterConfig cfg; - memset(&cfg, 0, sizeof(cfg)); - cfg.check_multi_interp_extensions = 1; - cfg.gil = PyInterpreterConfig_OWN_GIL; - auto status = Py_NewInterpreterFromConfig(&sub, &cfg); - T_REQUIRE(!PyStatus_IsError(status)); - - py::detail::get_num_interpreters_seen()++; - - py::list(py::module_::import("sys").attr("path")).append(py::str(".")); - - // we have switched to the new interpreter and released the main gil - - // trampoline_module did not provide the per_interpreter_gil tag, so it cannot be - // imported - bool caught = false; - try { - py::module_::import("trampoline_module"); - } catch (pybind11::error_already_set &pe) { - T_REQUIRE(pe.matches(PyExc_ImportError)); - std::string msg(pe.what()); - T_REQUIRE(msg.find("does not support loading in subinterpreters") - != std::string::npos); - caught = true; - } - T_REQUIRE(caught); - - // widget_module did provide the per_interpreter_gil tag, so it this does not throw - py::module_::import("widget_module"); - - T_REQUIRE(!py::hasattr(py::module_::import("external_module"), "multi_interp")); - py::module_::import("external_module").attr("multi_interp") = std::to_string(num); - - // wait for something to set sync to our thread number - // we are holding our subinterpreter's GIL - while (sync != num) - std::this_thread::sleep_for(std::chrono::microseconds(1)); - - // now change it so the next thread can mvoe on - ++sync; - - // but keep holding the GIL until after the next thread moves on as well - while (sync == num + 1) - std::this_thread::sleep_for(std::chrono::microseconds(1)); - - // one last check before quitting the thread, the internals should be different - auto sub_int - = py::module_::import("external_module").attr("internals_at")().cast(); - T_REQUIRE(sub_int != main_int); - - Py_EndInterpreter(sub); - - // switch back so the scoped_acquire can release the GIL properly - PyThreadState_Swap(main_tstate); - }; - - std::thread t1(thread_main, 1); - std::thread t2(thread_main, 2); - - // we spawned two threads, at this point they are both waiting for started to increase - ++started; - - // ok now wait for the threads to start - while (started != 3) - std::this_thread::sleep_for(std::chrono::microseconds(1)); - - // we still hold the main GIL, at this point both threads are waiting on the main GIL - // IN THE CASE of free threading, the threads are waiting on sync (because there is no GIL) - - // IF the below code hangs in one of the wait loops, then the child thread GIL behavior did not - // function as expected. - { - // release the GIL and allow the threads to run - py::gil_scoped_release nogil; - - // the threads are now waiting on the sync - REQUIRE(sync == 0); - - // this will trigger thread 1 and then advance and trigger 2 and then advance - sync = 1; - - // wait for thread 2 to advance - while (sync != 3) - std::this_thread::sleep_for(std::chrono::microseconds(1)); - - // we know now that thread 1 has run and may be finishing - // and thread 2 is waiting for permission to advance - - // so we move sync so that thread 2 can finish executing - ++sync; - - // now wait for both threads to complete - t1.join(); - t2.join(); - } - - // now we have the gil again, sanity check - REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) - == "1"); - - // the threads are stopped. we can now lower this for the rest of the test - py::detail::get_num_interpreters_seen() = 1; - - // make sure nothing unexpected happened inside the threads, now that they are completed - REQUIRE(failure == 0); -# undef T_REQUIRE -} -#endif - TEST_CASE("Execution frame") { // When the interpreter is embedded, there is no execution frame, but `py::exec` // should still function by using reasonable globals: `__main__.__dict__`. diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp new file mode 100644 index 0000000000..f430bb5c04 --- /dev/null +++ b/tests/test_embed/test_subinterpreter.cpp @@ -0,0 +1,288 @@ +#include +#ifdef PYBIND11_SUBINTERPRETER_SUPPORT +#include + +// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to +// catch 2.0.1; this should be fixed in the next catch release after 2.0.1). +PYBIND11_WARNING_DISABLE_MSVC(4996) + +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace py::literals; + +bool has_state_dict_internals_obj(); +bool has_pybind11_internals_static(); +uintptr_t get_details_as_uintptr(); + +void unsafe_reset_internals_for_single_interpreter() { + // unsafe normally, but for subsequent tests, put this back.. we know there are no threads running and only 1 interpreter + py::detail::get_num_interpreters_seen() = 1; + py::detail::get_internals_pp() = nullptr; + py::detail::get_internals(); + py::detail::get_internals_pp() = nullptr; + py::detail::get_local_internals(); +} + +TEST_CASE("Single Subinterpreter") { + py::module_::import("external_module"); // in the main interpreter + + // Add tags to the modules in the main interpreter and test the basics. + py::module_::import("__main__").attr("main_tag") = "main interpreter"; + { + auto m = py::module_::import("widget_module"); + m.attr("extension_module_tag") = "added to module in main interpreter"; + + REQUIRE(m.attr("add")(1, 2).cast() == 3); + } + REQUIRE(has_state_dict_internals_obj()); + REQUIRE(has_pybind11_internals_static()); + + auto main_int = py::module_::import("external_module").attr("internals_at")().cast(); + + /// Create and switch to a subinterpreter. + { + py::scoped_subinterpreter ssi; + + // The subinterpreter has internals populated + REQUIRE(has_pybind11_internals_static()); + + py::list(py::module_::import("sys").attr("path")).append(py::str(".")); + + auto ext_int = py::module_::import("external_module").attr("internals_at")().cast(); + py::detail::get_internals(); + REQUIRE(has_pybind11_internals_static()); + REQUIRE(get_details_as_uintptr() == ext_int); + REQUIRE(ext_int != main_int); + + // Modules tags should be gone. + REQUIRE_FALSE(py::hasattr(py::module_::import("__main__"), "tag")); + { + auto m = py::module_::import("widget_module"); + REQUIRE_FALSE(py::hasattr(m, "extension_module_tag")); + + // Function bindings should still work. + REQUIRE(m.attr("add")(1, 2).cast() == 3); + } + } + + REQUIRE(py::hasattr(py::module_::import("__main__"), "main_tag")); + REQUIRE(py::hasattr(py::module_::import("widget_module"), "extension_module_tag")); + REQUIRE(has_state_dict_internals_obj()); + + unsafe_reset_internals_for_single_interpreter(); +} + +TEST_CASE("Multiple Subinterpreters") { + // Make sure the module is in the main interpreter and save its pointer + auto *main_ext = py::module_::import("external_module").ptr(); + auto main_int + = py::module_::import("external_module").attr("internals_at")().cast(); + py::module_::import("external_module").attr("multi_interp") = "1"; + + { + py::subinterpreter si1 = py::subinterpreter::create(); + std::unique_ptr psi2; + + PyObject *sub1_ext = nullptr; + PyObject *sub2_ext = nullptr; + uintptr_t sub1_int = 0; + uintptr_t sub2_int = 0; + + { + py::subinterpreter_scoped_activate scoped(si1); + py::list(py::module_::import("sys").attr("path")).append(py::str(".")); + + // The subinterpreter has its own copy of this module which is completely separate from main + sub1_ext = py::module_::import("external_module").ptr(); + REQUIRE(sub1_ext != main_ext); + REQUIRE_FALSE(py::hasattr(py::module_::import("external_module"), "multi_interp")); + py::module_::import("external_module").attr("multi_interp") = "2"; + // The subinterpreter also has its own internals + sub1_int + = py::module_::import("external_module").attr("internals_at")().cast(); + REQUIRE(sub1_int != main_int); + + // while the old one is active, create a new one + psi2 = std::make_unique(py::subinterpreter::create()); + } + + { + py::subinterpreter_scoped_activate scoped(*psi2); + py::list(py::module_::import("sys").attr("path")).append(py::str(".")); + + // The second subinterpreter is separate from both main and the other subinterpreter + sub2_ext = py::module_::import("external_module").ptr(); + REQUIRE(sub2_ext != main_ext); + REQUIRE(sub2_ext != sub1_ext); + REQUIRE_FALSE(py::hasattr(py::module_::import("external_module"), "multi_interp")); + py::module_::import("external_module").attr("multi_interp") = "3"; + // The subinterpreter also has its own internals + sub2_int + = py::module_::import("external_module").attr("internals_at")().cast(); + REQUIRE(sub2_int != main_int); + REQUIRE(sub2_int != sub1_int); + } + + { + py::subinterpreter_scoped_activate scoped(si1); + REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) + == "2"); + } + + // out here we should be in the main interpreter, with the GIL, with the other 2 still alive + + auto post_int + = py::module_::import("external_module").attr("internals_at")().cast(); + // Make sure internals went back the way it was before + REQUIRE(main_int == post_int); + + REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) + == "1"); + } + + // now back to just main + + auto post_int = py::module_::import("external_module").attr("internals_at")().cast(); + // Make sure internals went back the way it was before + REQUIRE(main_int == post_int); + + REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) + == "1"); + + unsafe_reset_internals_for_single_interpreter(); +} + +#ifdef Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +TEST_CASE("Per-Subinterpreter GIL") { + auto main_int + = py::module_::import("external_module").attr("internals_at")().cast(); + + std::atomic started, sync, failure; + started = 0; + sync = 0; + failure = 0; + +// REQUIRE throws on failure, so we can't use it within the thread +# define T_REQUIRE(status) \ + do { \ + assert(status); \ + if (!(status)) \ + ++failure; \ + } while (0) + + auto &&thread_main = [&](int num) { + while (started == 0) + std::this_thread::sleep_for(std::chrono::microseconds(1)); + ++started; + + py::gil_scoped_acquire gil; + + // we have the GIL, we can access the main interpreter + auto t_int + = py::module_::import("external_module").attr("internals_at")().cast(); + T_REQUIRE(t_int == main_int); + py::module_::import("external_module").attr("multi_interp") = "1"; + + auto sub = py::subinterpreter::create(); + + { + py::subinterpreter_scoped_activate sguard{sub}; + + py::list(py::module_::import("sys").attr("path")).append(py::str(".")); + + // we have switched to the new interpreter and released the main gil + + // trampoline_module did not provide the mod_per_interpreter_gil tag, so it cannot be + // imported + bool caught = false; + try { + py::module_::import("trampoline_module"); + } catch (pybind11::error_already_set &pe) { + caught = true; + } + T_REQUIRE(!caught); + + // widget_module did provide the per_interpreter_gil tag, so it this does not throw + py::module_::import("widget_module"); + + T_REQUIRE(!py::hasattr(py::module_::import("external_module"), "multi_interp")); + py::module_::import("external_module").attr("multi_interp") = std::to_string(num); + + // wait for something to set sync to our thread number + // we are holding our subinterpreter's GIL + while (sync != num) + std::this_thread::sleep_for(std::chrono::microseconds(1)); + + // now change it so the next thread can move on + ++sync; + + // but keep holding the GIL until after the next thread moves on as well + while (sync == num + 1) + std::this_thread::sleep_for(std::chrono::microseconds(1)); + + // one last check before quitting the thread, the internals should be different + auto sub_int + = py::module_::import("external_module").attr("internals_at")().cast(); + T_REQUIRE(sub_int != main_int); + } + }; +# undef T_REQUIRE + + std::thread t1(thread_main, 1); + std::thread t2(thread_main, 2); + + // we spawned two threads, at this point they are both waiting for started to increase + ++started; + + // ok now wait for the threads to start + while (started != 3) + std::this_thread::sleep_for(std::chrono::microseconds(1)); + + // we still hold the main GIL, at this point both threads are waiting on the main GIL + // IN THE CASE of free threading, the threads are waiting on sync (because there is no GIL) + + // IF the below code hangs in one of the wait loops, then the child thread GIL behavior did not + // function as expected. + { + // release the GIL and allow the threads to run + py::gil_scoped_release nogil; + + // the threads are now waiting on the sync + REQUIRE(sync == 0); + + // this will trigger thread 1 and then advance and trigger 2 and then advance + sync = 1; + + // wait for thread 2 to advance + while (sync != 3) + std::this_thread::sleep_for(std::chrono::microseconds(1)); + + // we know now that thread 1 has run and may be finishing + // and thread 2 is waiting for permission to advance + + // so we move sync so that thread 2 can finish executing + ++sync; + + // now wait for both threads to complete + t1.join(); + t2.join(); + } + + // now we have the gil again, sanity check + REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) + == "1"); + + unsafe_reset_internals_for_single_interpreter(); + + // make sure nothing unexpected happened inside the threads, now that they are completed + REQUIRE(failure == 0); +} +#endif//Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + +#endif// PYBIND11_SUBINTERPRETER_SUPPORT From 45159fd1d5d45665780c473969222e11dd50c473 Mon Sep 17 00:00:00 2001 From: b-pass Date: Thu, 15 May 2025 21:47:22 -0400 Subject: [PATCH 03/31] Migrate subinterpreter tests to use the new embedded class. --- CMakeLists.txt | 1 + include/pybind11/subinterpreter.h | 102 +++++++++++++++++++---- tests/test_embed/test_subinterpreter.cpp | 66 ++++++++------- 3 files changed, 121 insertions(+), 48 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 676fc4b66c..347f816354 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,6 +223,7 @@ set(PYBIND11_HEADERS include/pybind11/operators.h include/pybind11/pybind11.h include/pybind11/pytypes.h + include/pybind11/subinterpreter.h include/pybind11/stl.h include/pybind11/stl_bind.h include/pybind11/stl/filesystem.h diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 0abbd7b3d2..e15f93fd7a 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -22,6 +22,11 @@ class subinterpreter_scoped_activate { explicit subinterpreter_scoped_activate(subinterpreter const &si); ~subinterpreter_scoped_activate(); + subinterpreter_scoped_activate(subinterpreter_scoped_activate &&) = delete; + subinterpreter_scoped_activate(subinterpreter_scoped_activate const &) = delete; + subinterpreter_scoped_activate &operator=(subinterpreter_scoped_activate &) = delete; + subinterpreter_scoped_activate &operator=(subinterpreter_scoped_activate const &) = delete; + private: PyThreadState *old_tstate_ = nullptr; PyThreadState *free_tstate_ = nullptr; @@ -46,37 +51,86 @@ class subinterpreter { return *this; } - ~subinterpreter() { + /** + Because a subinterpreter must be destructed using the original PyThreadState returned when it + was created. However, because that state has TSS/TLS values (just like any PyThreadState) it + cannot be trivially moved to a different OS thread. If someone moves the subinterpreter and + destructs it, it may deadlock in Python cleanup. + + So we try to throw here instead, but if there is an active exception then we have to just leak + the interpreter. + */ + ~subinterpreter() noexcept(false) { if (tstate_) { - if (PyThread_get_thread_ident() != tstate_->native_thread_id) { - // Throwing from destructors is bad :( - // But if we don't throw, we either leak the interpreter or the code hangs because - // internal Python TSS values are wrong/missing - throw std::runtime_error( - "wrong thread called subinterpreter destruct. subinterpreters can only " - "destruct on the thread that created them!"); +#ifdef PY_HAVE_THREAD_NATIVE_ID + if (PyThread_get_thread_native_id() != tstate_->native_thread_id) { + auto cur_tstate = detail::get_thread_state_unchecked(); + if (cur_tstate && cur_tstate->interp == tstate_->interp) { + // the destructing subinterpreter was active, release the GIL + PyThreadState_Swap(nullptr); + } + + bool throwable; +# ifndef __cpp_lib_uncaught_exceptions + // std::uncaught_exception was removed in C++20 + throwable = !std::uncaught_exception(); +# else + // std::uncaught_exceptions was added in C++14 + throwable = !(std::uncaught_exceptions() > 0); +# endif + + if (throwable) { + throw std::runtime_error("Cannot destruct a subinterpreter on a different " + "thread from the one that created it!"); + } + return; } +#endif - // has to be the active interpreter in order to call End on it // switch into the expiring interpreter auto old_tstate = PyThreadState_Swap(tstate_); + bool switch_back = old_tstate && old_tstate->interp != tstate_->interp; - // make sure we have the GIL + // make sure we have the GIL for the interpreter we are ending (void) PyGILState_Ensure(); + // Get the internals pointer (without creating it if it doesn't exist). It's possible + // for the internals to be created during Py_EndInterpreter() (e.g. if a py::capsule + // calls `get_internals()` during destruction), so we get the pointer-pointer here and + // check it after. + auto *&internals_ptr_ptr = detail::get_internals_pp(); + auto *&local_internals_ptr_ptr = detail::get_internals_pp(); + { + dict state_dict = detail::get_python_state_dict(); + internals_ptr_ptr + = detail::get_internals_pp_from_capsule_in_state_dict( + state_dict, PYBIND11_INTERNALS_ID); + local_internals_ptr_ptr + = detail::get_internals_pp_from_capsule_in_state_dict( + state_dict, detail::get_local_internals_id()); + } + // End it Py_EndInterpreter(tstate_); - // switch back to the old tstate and old GIL (if there was one) - if (old_tstate != tstate_) - PyThreadState_Swap(old_tstate); - // do NOT decrease detail::get_num_interpreters_seen, because it can never decrease // while other threads are running... + + if (internals_ptr_ptr) { + internals_ptr_ptr->reset(); + } + if (local_internals_ptr_ptr) { + local_internals_ptr_ptr->reset(); + } + + // switch back to the old tstate and old GIL (if there was one) + if (switch_back) + PyThreadState_Swap(old_tstate); } } - /// abandon cleanup of this subinterpreter. might be needed during finalization + /// abandon cleanup of this subinterpreter (leak it). this might be needed during + /// finalization... void disarm() { tstate_ = nullptr; } /// Get a handle to the main interpreter that can be used with subinterpreter_scoped_activate @@ -136,9 +190,21 @@ class subinterpreter { PyInterpreterState *istate_ = nullptr; }; -subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpreter const &si) { +class scoped_subinterpreter { +public: + scoped_subinterpreter() : si_(subinterpreter::create()), scope_(si_) {} + + explicit scoped_subinterpreter(PyInterpreterConfig const &cfg) + : si_(subinterpreter::create(cfg)), scope_(si_) {} + +private: + subinterpreter si_; + subinterpreter_scoped_activate scope_; +}; + +inline subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpreter const &si) { #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) - if (!si.istate_ || !si.tstate_) { + if (!si.istate_) { pybind11_fail("null subinterpreter"); } #endif @@ -181,7 +247,7 @@ subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpreter co old_tstate_ = PyThreadState_Swap(desired_tstate); } -subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { +inline subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { if (simple_gil_) { // We were on this interpreter already, so just make sure the GIL goes back as it was PyGILState_Release(gil_state_); diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index f430bb5c04..dc1e4977b8 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -1,17 +1,17 @@ #include #ifdef PYBIND11_SUBINTERPRETER_SUPPORT -#include +# include // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). PYBIND11_WARNING_DISABLE_MSVC(4996) -#include -#include -#include -#include -#include -#include +# include +# include +# include +# include +# include +# include namespace py = pybind11; using namespace py::literals; @@ -21,7 +21,8 @@ bool has_pybind11_internals_static(); uintptr_t get_details_as_uintptr(); void unsafe_reset_internals_for_single_interpreter() { - // unsafe normally, but for subsequent tests, put this back.. we know there are no threads running and only 1 interpreter + // unsafe normally, but for subsequent tests, put this back.. we know there are no threads + // running and only 1 interpreter py::detail::get_num_interpreters_seen() = 1; py::detail::get_internals_pp() = nullptr; py::detail::get_internals(); @@ -49,12 +50,13 @@ TEST_CASE("Single Subinterpreter") { { py::scoped_subinterpreter ssi; - // The subinterpreter has internals populated + // The subinterpreter has internals populated REQUIRE(has_pybind11_internals_static()); py::list(py::module_::import("sys").attr("path")).append(py::str(".")); - auto ext_int = py::module_::import("external_module").attr("internals_at")().cast(); + auto ext_int + = py::module_::import("external_module").attr("internals_at")().cast(); py::detail::get_internals(); REQUIRE(has_pybind11_internals_static()); REQUIRE(get_details_as_uintptr() == ext_int); @@ -98,7 +100,8 @@ TEST_CASE("Multiple Subinterpreters") { py::subinterpreter_scoped_activate scoped(si1); py::list(py::module_::import("sys").attr("path")).append(py::str(".")); - // The subinterpreter has its own copy of this module which is completely separate from main + // The subinterpreter has its own copy of this module which is completely separate from + // main sub1_ext = py::module_::import("external_module").ptr(); REQUIRE(sub1_ext != main_ext); REQUIRE_FALSE(py::hasattr(py::module_::import("external_module"), "multi_interp")); @@ -111,7 +114,7 @@ TEST_CASE("Multiple Subinterpreters") { // while the old one is active, create a new one psi2 = std::make_unique(py::subinterpreter::create()); } - + { py::subinterpreter_scoped_activate scoped(*psi2); py::list(py::module_::import("sys").attr("path")).append(py::str(".")); @@ -128,15 +131,17 @@ TEST_CASE("Multiple Subinterpreters") { REQUIRE(sub2_int != main_int); REQUIRE(sub2_int != sub1_int); } - + { py::subinterpreter_scoped_activate scoped(si1); - REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) - == "2"); + REQUIRE( + py::cast(py::module_::import("external_module").attr("multi_interp")) + == "2"); } - // out here we should be in the main interpreter, with the GIL, with the other 2 still alive - + // out here we should be in the main interpreter, with the GIL, with the other 2 still + // alive + auto post_int = py::module_::import("external_module").attr("internals_at")().cast(); // Make sure internals went back the way it was before @@ -148,17 +153,18 @@ TEST_CASE("Multiple Subinterpreters") { // now back to just main - auto post_int = py::module_::import("external_module").attr("internals_at")().cast(); + auto post_int + = py::module_::import("external_module").attr("internals_at")().cast(); // Make sure internals went back the way it was before REQUIRE(main_int == post_int); REQUIRE(py::cast(py::module_::import("external_module").attr("multi_interp")) - == "1"); - + == "1"); + unsafe_reset_internals_for_single_interpreter(); } -#ifdef Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +# ifdef Py_MOD_PER_INTERPRETER_GIL_SUPPORTED TEST_CASE("Per-Subinterpreter GIL") { auto main_int = py::module_::import("external_module").attr("internals_at")().cast(); @@ -169,12 +175,12 @@ TEST_CASE("Per-Subinterpreter GIL") { failure = 0; // REQUIRE throws on failure, so we can't use it within the thread -# define T_REQUIRE(status) \ - do { \ - assert(status); \ - if (!(status)) \ - ++failure; \ - } while (0) +# define T_REQUIRE(status) \ + do { \ + assert(status); \ + if (!(status)) \ + ++failure; \ + } while (0) auto &&thread_main = [&](int num) { while (started == 0) @@ -232,7 +238,7 @@ TEST_CASE("Per-Subinterpreter GIL") { T_REQUIRE(sub_int != main_int); } }; -# undef T_REQUIRE +# undef T_REQUIRE std::thread t1(thread_main, 1); std::thread t2(thread_main, 2); @@ -283,6 +289,6 @@ TEST_CASE("Per-Subinterpreter GIL") { // make sure nothing unexpected happened inside the threads, now that they are completed REQUIRE(failure == 0); } -#endif//Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +# endif // Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#endif// PYBIND11_SUBINTERPRETER_SUPPORT +#endif // PYBIND11_SUBINTERPRETER_SUPPORT From 5b5a1e802ea1f089b229b0241b90dd8cbec0200e Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 18:10:01 -0400 Subject: [PATCH 04/31] Add a test for moving subinterpreters across threads for destruction And find a better way to make that work. --- include/pybind11/subinterpreter.h | 90 ++++++++---------------- tests/test_embed/test_interpreter.cpp | 1 - tests/test_embed/test_subinterpreter.cpp | 43 ++++++++++- 3 files changed, 69 insertions(+), 65 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index e15f93fd7a..20aeaddc38 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -51,44 +51,31 @@ class subinterpreter { return *this; } - /** - Because a subinterpreter must be destructed using the original PyThreadState returned when it - was created. However, because that state has TSS/TLS values (just like any PyThreadState) it - cannot be trivially moved to a different OS thread. If someone moves the subinterpreter and - destructs it, it may deadlock in Python cleanup. - - So we try to throw here instead, but if there is an active exception then we have to just leak - the interpreter. - */ - ~subinterpreter() noexcept(false) { + ~subinterpreter() { if (tstate_) { -#ifdef PY_HAVE_THREAD_NATIVE_ID - if (PyThread_get_thread_native_id() != tstate_->native_thread_id) { - auto cur_tstate = detail::get_thread_state_unchecked(); - if (cur_tstate && cur_tstate->interp == tstate_->interp) { - // the destructing subinterpreter was active, release the GIL - PyThreadState_Swap(nullptr); - } - - bool throwable; -# ifndef __cpp_lib_uncaught_exceptions - // std::uncaught_exception was removed in C++20 - throwable = !std::uncaught_exception(); -# else - // std::uncaught_exceptions was added in C++14 - throwable = !(std::uncaught_exceptions() > 0); -# endif - - if (throwable) { - throw std::runtime_error("Cannot destruct a subinterpreter on a different " - "thread from the one that created it!"); - } - return; + PyThreadState *old_tstate; + /* + If it is not, the interpreter destruction can hang inside Python threading cleanup. + So in that case, we make sure to create a new thread state to be used for cleanup + */ + bool wrong_thread = true; + #ifdef PY_HAVE_THREAD_NATIVE_ID + wrong_thread = PyThread_get_thread_native_id() != tstate_->native_thread_id; + #endif + if (wrong_thread) { + // The PyThreadState used in Py_EndInterpreter must be created on this OS thread. + // So create a new thread state here on the right thread + auto *temp = PyThreadState_New(tstate_->interp); + old_tstate = PyThreadState_Swap(temp); + // and make to clear the other one, because there must be only one at cleanup time + PyThreadState_Clear(tstate_); + PyThreadState_Delete(tstate_); + tstate_ = temp; + } + else { + old_tstate = PyThreadState_Swap(tstate_); } -#endif - // switch into the expiring interpreter - auto old_tstate = PyThreadState_Swap(tstate_); bool switch_back = old_tstate && old_tstate->interp != tstate_->interp; // make sure we have the GIL for the interpreter we are ending @@ -217,34 +204,15 @@ inline subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpr return; } - PyThreadState *desired_tstate = nullptr; - - // get the state dict for the interpreter we want - dict idict = reinterpret_borrow(PyInterpreterState_GetDict(si.istate_)); - // and get the internals from it - auto *internals_pp = detail::get_internals_pp_from_capsule_in_state_dict( - idict, PYBIND11_INTERNALS_ID); - if (internals_pp && *internals_pp) { - // see if there is already a tstate for this thread - desired_tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE((*internals_pp)->tstate); - if (!desired_tstate) { - // nope, we have to create one. - desired_tstate = PyThreadState_New(si.istate_); - free_tstate_ = desired_tstate; -#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) - if (!desired_tstate) { - pybind11_fail("subinterpreter_scoped_activate: could not create thread state!"); - } -#endif - PYBIND11_TLS_REPLACE_VALUE((*internals_pp)->tstate, desired_tstate); - } - } else { - desired_tstate = PyThreadState_New(si.istate_); - free_tstate_ = desired_tstate; - } + // we can't really innteract with the interpreter at all until we switch to it + // not even to, for example, look in it's state dict or touch its internals + free_tstate_ = PyThreadState_New(si.istate_); // make the interpreter active and acquire the GIL - old_tstate_ = PyThreadState_Swap(desired_tstate); + old_tstate_ = PyThreadState_Swap(free_tstate_); + + // save this in internals for scoped_gil calls + PYBIND11_TLS_REPLACE_VALUE(detail::get_internals().tstate, free_tstate_); } inline subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index cbd29783b7..e555c0d70c 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -1,5 +1,4 @@ #include -#include // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index dc1e4977b8..9fcd226c2d 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -80,6 +80,37 @@ TEST_CASE("Single Subinterpreter") { unsafe_reset_internals_for_single_interpreter(); } + +TEST_CASE("Move Subinterpreter") { + + auto ssi = std::make_unique(py::subinterpreter::create()); + + // on this thread, use the subinterpreter and import some non-trivial junk + { + py::subinterpreter_scoped_activate activate(*ssi); + + py::list(py::module_::import("sys").attr("path")).append(py::str(".")); + py::module_::import("datetime"); + py::module_::import("threading"); + py::module_::import("external_module"); + } + + std::thread temp([ssi=std::move(ssi)]() mutable { + + // Use it again + { + py::subinterpreter_scoped_activate activate(*ssi); + py::module_::import("external_module"); + } + + // free it + ssi.reset(); + }); + temp.join(); + REQUIRE(!ssi); + unsafe_reset_internals_for_single_interpreter(); +} + TEST_CASE("Multiple Subinterpreters") { // Make sure the module is in the main interpreter and save its pointer auto *main_ext = py::module_::import("external_module").ptr(); @@ -204,15 +235,21 @@ TEST_CASE("Per-Subinterpreter GIL") { // we have switched to the new interpreter and released the main gil - // trampoline_module did not provide the mod_per_interpreter_gil tag, so it cannot be - // imported + // trampoline_module did not provide the per_interpreter_gil tag, so it cannot be imported bool caught = false; try { py::module_::import("trampoline_module"); } catch (pybind11::error_already_set &pe) { + T_REQUIRE(pe.matches(PyExc_ImportError)); + std::string msg(pe.what()); + T_REQUIRE(msg.find("does not support loading in subinterpreters") + != std::string::npos); caught = true; } - T_REQUIRE(!caught); + T_REQUIRE(caught); + + // widget_module did provide the per_interpreter_gil tag, so it this does not throw + py::module_::import("widget_module"); // widget_module did provide the per_interpreter_gil tag, so it this does not throw py::module_::import("widget_module"); From 74080ebcb953068eb3953263eb90134e1b538115 Mon Sep 17 00:00:00 2001 From: b-pass Date: Thu, 15 May 2025 22:23:30 -0400 Subject: [PATCH 05/31] Code organization --- include/pybind11/subinterpreter.h | 114 ++++++++++++----------- tests/test_embed/test_subinterpreter.cpp | 7 +- 2 files changed, 64 insertions(+), 57 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 20aeaddc38..4a02b2c4ac 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -1,3 +1,12 @@ +/* + pybind11/subinterpreters.h: Support for creating and using subinterpreters + + Copyright (c) 2025 The Pybind Development Team. + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + #pragma once #include "detail/common.h" @@ -34,9 +43,11 @@ class subinterpreter_scoped_activate { bool simple_gil_ = false; }; +/// Holds a Python subinterpreter instance class subinterpreter { public: - subinterpreter() = default; + subinterpreter() = default; /// empty/unusable, but move-assignable. use create() to create a subinterpreter. + subinterpreter(subinterpreter const ©) = delete; subinterpreter &operator=(subinterpreter const ©) = delete; @@ -45,34 +56,72 @@ class subinterpreter { old.istate_ = nullptr; } - subinterpreter &operator=(subinterpreter &&old) { + subinterpreter &operator = (subinterpreter &&old) { std::swap(old.tstate_, tstate_); std::swap(old.istate_, istate_); return *this; } + /// Create a new subinterpreter with the specified configuration + /// Note Well: + static inline subinterpreter create(PyInterpreterConfig const &cfg) { + error_scope err_scope; + auto main_guard = main_scoped_activate(); + subinterpreter result; + { + // we must hold the main GIL in order to create a subinterpreter + gil_scoped_acquire gil; + + auto prev_tstate = PyThreadState_Get(); + + auto status = Py_NewInterpreterFromConfig(&result.tstate_, &cfg); + + // this doesn't raise a normal Python exception, it provides an exit() status code. + if (PyStatus_Exception(status)) { + pybind11_fail("failed to create new sub-interpreter"); + } + + // upon success, the new interpreter is activated in this thread + result.istate_ = result.tstate_->interp; + detail::get_num_interpreters_seen() += 1; // there are now many interpreters + detail::get_internals(); // initialize internals.tstate, amongst other things... + + // we have to switch back to main, and then the scopes will handle cleanup + PyThreadState_Swap(prev_tstate); + } + return result; + } + + /// Call create() with a default configuration of an isolated interpreter that disallows fork, + /// exec, and Python threads. + static inline subinterpreter create() { + // same as the default config in the python docs + PyInterpreterConfig cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.check_multi_interp_extensions = 1; + cfg.gil = PyInterpreterConfig_OWN_GIL; + return create(cfg); + } + ~subinterpreter() { if (tstate_) { PyThreadState *old_tstate; - /* - If it is not, the interpreter destruction can hang inside Python threading cleanup. - So in that case, we make sure to create a new thread state to be used for cleanup - */ + bool wrong_thread = true; - #ifdef PY_HAVE_THREAD_NATIVE_ID +#ifdef PY_HAVE_THREAD_NATIVE_ID wrong_thread = PyThread_get_thread_native_id() != tstate_->native_thread_id; - #endif +#endif if (wrong_thread) { // The PyThreadState used in Py_EndInterpreter must be created on this OS thread. - // So create a new thread state here on the right thread + // (If it is not, subinterpreter cleanup will hang within Python's threading + // module.) So create a new thread state here on the right OS thread. auto *temp = PyThreadState_New(tstate_->interp); old_tstate = PyThreadState_Swap(temp); - // and make to clear the other one, because there must be only one at cleanup time + // delete the other one because there must be only one at cleanup PyThreadState_Clear(tstate_); PyThreadState_Delete(tstate_); tstate_ = temp; - } - else { + } else { old_tstate = PyThreadState_Swap(tstate_); } @@ -130,47 +179,6 @@ class subinterpreter { return subinterpreter_scoped_activate(m); } - /// Create a new subinterpreter with the specified configuration - /// Note Well: - static inline subinterpreter create(PyInterpreterConfig const &cfg) { - error_scope err_scope; - auto main_guard = main_scoped_activate(); - subinterpreter result; - { - // we must hold the main GIL in order to create a subinterpreter - gil_scoped_acquire gil; - - auto prev_tstate = PyThreadState_Get(); - - auto status = Py_NewInterpreterFromConfig(&result.tstate_, &cfg); - - // this doesn't raise a normal Python exception, it provides an exit() status code. - if (PyStatus_Exception(status)) { - pybind11_fail("failed to create new sub-interpreter"); - } - - // upon success, the new interpreter is activated in this thread - result.istate_ = result.tstate_->interp; - detail::get_num_interpreters_seen() += 1; // there are now many interpreters - detail::get_internals(); // initialize internals.tstate, amongst other things... - - // we have to switch back to main, and then the scopes will handle cleanup - PyThreadState_Swap(prev_tstate); - } - return result; - } - - /// Call create() with a default configuration of an isolated interpreter that disallows fork, - /// exec, and Python threads. - static inline subinterpreter create() { - // same as the default config in the python docs - PyInterpreterConfig cfg; - memset(&cfg, 0, sizeof(cfg)); - cfg.check_multi_interp_extensions = 1; - cfg.gil = PyInterpreterConfig_OWN_GIL; - return create(cfg); - } - private: friend class subinterpreter_scoped_activate; PyThreadState *tstate_ = nullptr; diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index 9fcd226c2d..3b57816165 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -80,7 +80,6 @@ TEST_CASE("Single Subinterpreter") { unsafe_reset_internals_for_single_interpreter(); } - TEST_CASE("Move Subinterpreter") { auto ssi = std::make_unique(py::subinterpreter::create()); @@ -95,8 +94,7 @@ TEST_CASE("Move Subinterpreter") { py::module_::import("external_module"); } - std::thread temp([ssi=std::move(ssi)]() mutable { - + std::thread temp([ssi = std::move(ssi)]() mutable { // Use it again { py::subinterpreter_scoped_activate activate(*ssi); @@ -235,7 +233,8 @@ TEST_CASE("Per-Subinterpreter GIL") { // we have switched to the new interpreter and released the main gil - // trampoline_module did not provide the per_interpreter_gil tag, so it cannot be imported + // trampoline_module did not provide the per_interpreter_gil tag, so it cannot be + // imported bool caught = false; try { py::module_::import("trampoline_module"); From 51764f075683effc5d36004b672b48884c89b4e1 Mon Sep 17 00:00:00 2001 From: b-pass Date: Fri, 16 May 2025 18:36:50 -0400 Subject: [PATCH 06/31] Add a test which shows demostrates how gil_scoped interacts with sub-interpreters --- tests/test_embed/test_subinterpreter.cpp | 94 ++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index 3b57816165..b3df1f6811 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -109,6 +109,100 @@ TEST_CASE("Move Subinterpreter") { unsafe_reset_internals_for_single_interpreter(); } +TEST_CASE("GIL Subinterpreter") { + + PyInterpreterState *main_interp = PyInterpreterState_Get(); + + { + auto ssi = py::subinterpreter::create(); + + REQUIRE(main_interp == PyInterpreterState_Get()); + + PyInterpreterState *sub_interp = nullptr; + + { + py::subinterpreter_scoped_activate activate(ssi); + + sub_interp = PyInterpreterState_Get(); + REQUIRE(sub_interp != main_interp); + + py::list(py::module_::import("sys").attr("path")).append(py::str(".")); + py::module_::import("datetime"); + py::module_::import("threading"); + py::module_::import("external_module"); + + { + auto main_activate = py::subinterpreter::main_scoped_activate(); + REQUIRE(PyInterpreterState_Get() == main_interp); + + { + py::gil_scoped_release nogil{}; + { + py::gil_scoped_acquire yesgil{}; + REQUIRE(PyInterpreterState_Get() == main_interp); + } + } + + REQUIRE(PyInterpreterState_Get() == main_interp); + } + + REQUIRE(PyInterpreterState_Get() == sub_interp); + + { + py::gil_scoped_release nogil{}; + { + py::gil_scoped_acquire yesgil{}; + REQUIRE(PyInterpreterState_Get() == sub_interp); + } + } + + REQUIRE(PyInterpreterState_Get() == sub_interp); + } + + REQUIRE(PyInterpreterState_Get() == main_interp); + + { + py::gil_scoped_release nogil{}; + { + py::gil_scoped_acquire yesgil{}; + REQUIRE(PyInterpreterState_Get() == main_interp); + } + } + + REQUIRE(PyInterpreterState_Get() == main_interp); + + bool thread_result; + + { + thread_result = false; + py::gil_scoped_release nogil{}; + std::thread([&]() { + { + py::subinterpreter_scoped_activate ssa{ssi}; + } + { + py::gil_scoped_acquire gil{}; + thread_result = (PyInterpreterState_Get() == main_interp); + } + }).join(); + } + REQUIRE(thread_result); + + { + thread_result = false; + py::gil_scoped_release nogil{}; + std::thread([&]() { + py::gil_scoped_acquire gil{}; + thread_result = (PyInterpreterState_Get() == main_interp); + }).join(); + } + REQUIRE(thread_result); + } + + REQUIRE(PyInterpreterState_Get() == main_interp); + unsafe_reset_internals_for_single_interpreter(); +} + TEST_CASE("Multiple Subinterpreters") { // Make sure the module is in the main interpreter and save its pointer auto *main_ext = py::module_::import("external_module").ptr(); From 68543fa3f76135d4ef94a73378f5b920935aed08 Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 18:10:50 -0400 Subject: [PATCH 07/31] Add documentation for embeded sub-interpreters --- docs/advanced/embedding.rst | 173 +++++++++++++++++++++++++++--- include/pybind11/subinterpreter.h | 12 ++- 2 files changed, 164 insertions(+), 21 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index dec767aac9..2cbbe05913 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -237,31 +237,172 @@ global data. All the details can be found in the CPython documentation. Creating two concurrent ``scoped_interpreter`` guards is a fatal error. So is calling ``initialize_interpreter`` for a second time after the interpreter - has already been initialized. + has already been initialized. Use :class:`scoped_subinterpreter` to create + a sub-interpreter. See :ref:`subinterp` for important details on sub-interpreters. Do not use the raw CPython API functions ``Py_Initialize`` and ``Py_Finalize`` as these do not properly handle the lifetime of pybind11's internal data. +.. _subinterp: + Sub-interpreter support ======================= -Creating multiple copies of ``scoped_interpreter`` is not possible because it -represents the main Python interpreter. Sub-interpreters are something different -and they do permit the existence of multiple interpreters. This is an advanced -feature of the CPython API and should be handled with care. pybind11 does not -currently offer a C++ interface for sub-interpreters, so refer to the CPython -documentation for all the details regarding this feature. +A sub-interpreter is a separate interpreter instance which provides a +separate, isolated interpreter environment within the same process as the main +interpreter. Sub-interpreters are created and managed with a separate API from +the main interpreter. Beginning in Python 3.12, sub-interpreters each have +their own Global Interpreter Lock (GIL), which means that running a +sub-interpreter in a separate thread from the main interpreter can achieve true +concurrency. + +Managing multiple threads and the lifetimes of multiple interpreters and their +GILs can be challenging. Proceed with caution (and lots of testing)! + +The main interpreter must be initialized before creating a sub-interpreter, and +the main interpreter must outlive all sub-interpreters. Sub-interpreters are +managed through a different API than the main interpreter. + +The sub-interpreter API can be found in ``pybind11/subinterpreter.h``. + +The :class:`subinterpreter` class manages the lifetime of sub-interpreters. +Instances are movable, but not copyable. Default constructing this class does +*not* create a sub-interpreter (it creates an empty holder). To create a +sub-interpreter, call :func:`subinterpreter::create()`. + +.. warning:: + + Sub-interpreter creation acquires (and subsequently releases) the main + interpreter GIL. If another thread holds the main GIL, the function will + block until the main GIL can be acquired. + + Sub-interpreter destruction temporarily activates the sub-interpreter. The + sub-interpreter must not be active (on any threads) at the time the + :class:`subinterpreter` destructor is called. + + Both actions will re-acquire any interpreter's GIL that was held prior to + the call before returning (or return to no active interpreter if none was + active at the time of the call). + +Once a sub-interpreter is created, you can "activate" it on a thread (and +acquire it's GIL) by creating a :class:`subinterpreter_scoped_activate` +instance and passing it the sub-intepreter to be activated. The function +will acquire the sub-interpreter's GIL and make the sub-interpreter the +current active interpreter on the current thread for the lifetime of the +instance. When the :class:`subinterpreter_scoped_activate` instance goes out +of scope, the sub-interpreter GIL is released and the prior interpreter that +was active on the thread (if any) is reactivated and it's GIL is re-acquired. + +The :func:`subinterpreter::activate_main()` function activates the main +interpreter, acquiring it's GIL, and returns a +:class:`subinterpreter_scoped_activate` instance which will automatically +deactivate the main interpreter and release it's GIL when it goes out of +scope, just as :class:`subinterpreter_scoped_activate` also does for +sub-interpreters. + +:class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to +manage the GIL of a sub-interpreter just as they do for the main interpreter. +They both manage the GIL of the currently active interpreter, without the +programmer having to do anything special or different. There is one important +caveat: + +.. note:: + + When no interpreter is active through a + :class:`subinterpreter_scoped_activate` instance (such as on a new thread), + :class:`gil_scoped_acquire` will acquire the **main** GIL and + activate the **main** interpreter. + +Each sub-interpreter will import a separate copy of each ``PYBIND11_EMBEDDED_MODULE`` +when those modules specify a ``multiple_interpreters`` tag. If a module does not +specify a ``multiple_interpreters`` tag, then Python will report an ``ImportError`` +if it is imported in a sub-interpreter. + +Here is an example showing how to create and activate sub-interpreters: + +.. code-block:: cpp + + #include + #include + #include + + namespace py = pybind11; + + PYBIND11_EMBEDDED_MODULE(printer, m, py::multiple_interpreters::per_interpreter_gil()) { + m.def("which", [](const std::string& when) { + std::cout << when << "; Current Interpreter is " + << PyInterpreterState_GetID(PyInterpreterState_Get()) + << std::endl; + }); + } + + int main() { + py::scoped_interpreter main_int{}; + + py::module_::import("printer").attr("which")("First init"); + + { + py::subinterpreter sub = py::subinterpreter::create(); + + py::module_::import("printer").attr("which")("Created sub"); + + { + py::subinterpreter_scoped_activate ssa(sub); + py::module_::import("printer").attr("which")("Activated sub"); + } + + py::module_::import("printer").attr("which")("Deactivated sub"); + + { + py::gil_scoped_release nogil; + { + py::subinterpreter_scoped_activate ssa(sub); + { + auto main_sa = py::subinterpreter::main_scoped_activate(); + py::module_::import("printer").attr("which")("Main within sub"); + } + py::module_::import("printer").attr("which")("After Main, still within sub"); + } + } + } + + py::module_::import("printer").attr("which")("At end"); + + return 0; + } + +Expected output: + +.. code-block:: text + + First init; Current Interpreter is 0 + Created sub; Current Interpreter is 0 + Activated sub; Current Interpreter is 1 + Deactivated sub; Current Interpreter is 0 + Main within sub; Current Interpreter is 0 + After Main, still within sub; Current Interpreter is 1 + At end; Current Interpreter is 0 + +pybind11 also has a :class:`scoped_subinterpreter` class, which creates and +activates a sub-interpreter when it is constructed, and deactivates and deletes +it when it goes out of scope. + +Best Practices for sub-interpreter safety: + +- Never share Python objects across different interpreters. -We'll just mention a couple of caveats the sub-interpreters support in pybind11: +- Avoid global/static state whenever possible. Instead, keep state within each interpreter, + such as within the interpreter state dict, which can be accessed via + ``subinterpreter::current().state_dict()``, or within instance members and tied to + Python objects. - 1. Sub-interpreters will not receive independent copies of embedded modules. - Instead, these are shared and modifications in one interpreter may be - reflected in another. +- Avoid trying to "cache" Python objects in C++ variables across function calls (this is an easy + way to accidentally introduce sub-interpreter bugs). In the code example above, note that we + did not save the result of :func:`module_::import`, in order to avoid accidentally using the + resulting Python object when the wrong interpreter was active. - 2. Managing multiple threads, multiple interpreters and the GIL can be - challenging and there are several caveats here, even within the pure - CPython API (please refer to the Python docs for details). As for - pybind11, keep in mind that ``gil_scoped_release`` and ``gil_scoped_acquire`` - do not take sub-interpreters into account. +- While sub-interpreters each have their own GIL, there can now be multiple independent GILs in one + program, so your code needs to consider thread safety of within the C++ code, and the possibility + of deadlocks caused by multiple GILs and/or the interactions of the GIL(s) and C++'s own locking. diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 4a02b2c4ac..bf9096dfe7 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -46,8 +46,9 @@ class subinterpreter_scoped_activate { /// Holds a Python subinterpreter instance class subinterpreter { public: - subinterpreter() = default; /// empty/unusable, but move-assignable. use create() to create a subinterpreter. - + /// empty/unusable, but move-assignable. use create() to create a subinterpreter. + subinterpreter() = default; + subinterpreter(subinterpreter const ©) = delete; subinterpreter &operator=(subinterpreter const ©) = delete; @@ -56,14 +57,15 @@ class subinterpreter { old.istate_ = nullptr; } - subinterpreter &operator = (subinterpreter &&old) { + subinterpreter &operator=(subinterpreter &&old) { std::swap(old.tstate_, tstate_); std::swap(old.istate_, istate_); return *this; } /// Create a new subinterpreter with the specified configuration - /// Note Well: + /// @note This function acquires (and then releases) the main interpreter GIL, but the main + /// interpreter and its GIL are not required to be held prior to calling this function. static inline subinterpreter create(PyInterpreterConfig const &cfg) { error_scope err_scope; auto main_guard = main_scoped_activate(); @@ -92,7 +94,7 @@ class subinterpreter { return result; } - /// Call create() with a default configuration of an isolated interpreter that disallows fork, + /// Calls create() with a default configuration of an isolated interpreter that disallows fork, /// exec, and Python threads. static inline subinterpreter create() { // same as the default config in the python docs From 7a00f322a8dbda2c43d885ecde1929110da6e004 Mon Sep 17 00:00:00 2001 From: b-pass Date: Fri, 16 May 2025 19:43:45 -0400 Subject: [PATCH 08/31] Some additional docs work --- docs/advanced/embedding.rst | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 2cbbe05913..d1148d24d5 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -256,17 +256,19 @@ interpreter. Sub-interpreters are created and managed with a separate API from the main interpreter. Beginning in Python 3.12, sub-interpreters each have their own Global Interpreter Lock (GIL), which means that running a sub-interpreter in a separate thread from the main interpreter can achieve true -concurrency. +concurrency. -Managing multiple threads and the lifetimes of multiple interpreters and their -GILs can be challenging. Proceed with caution (and lots of testing)! +pybind11's sub-interpreter API can be found in ``pybind11/subinterpreter.h``. + +pybind11 :class:`subinterpreter` instances can be safely moved and shared between +threads as needed. However, managing multiple threads and the lifetimes of multiple +interpreters and their GILs can be challenging. +Proceed with caution (and lots of testing)! The main interpreter must be initialized before creating a sub-interpreter, and the main interpreter must outlive all sub-interpreters. Sub-interpreters are managed through a different API than the main interpreter. -The sub-interpreter API can be found in ``pybind11/subinterpreter.h``. - The :class:`subinterpreter` class manages the lifetime of sub-interpreters. Instances are movable, but not copyable. Default constructing this class does *not* create a sub-interpreter (it creates an empty holder). To create a @@ -391,6 +393,10 @@ it when it goes out of scope. Best Practices for sub-interpreter safety: +- Avoid moving or disarming RAII objects managing GIL and sub-interpreter lifetimes. Doing so can + lead to confusion about lifetimes. (For example, accidentally extending a + :class:`subinterpreter_scoped_activate` past the lifetime of it's :class:`subinterpreter`.) + - Never share Python objects across different interpreters. - Avoid global/static state whenever possible. Instead, keep state within each interpreter, @@ -404,5 +410,9 @@ Best Practices for sub-interpreter safety: resulting Python object when the wrong interpreter was active. - While sub-interpreters each have their own GIL, there can now be multiple independent GILs in one - program, so your code needs to consider thread safety of within the C++ code, and the possibility - of deadlocks caused by multiple GILs and/or the interactions of the GIL(s) and C++'s own locking. + program you need to consider the possibility of deadlocks caused by multiple GILs and/or the + interactions of the GIL(s) and your C++ code's own locking. + +- When using multiple threads to run independent sub-interpreters, the independent GILs allow + concurrent calls from different interpreters into the same C++ code from different threads. + So you must still consider the thread safety of your C++ code. From e9034065214d1ce8ebdafdda442fe78afe83449a Mon Sep 17 00:00:00 2001 From: b-pass Date: Fri, 16 May 2025 18:48:57 -0400 Subject: [PATCH 09/31] Add some convenience accessors --- include/pybind11/subinterpreter.h | 62 ++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index bf9096dfe7..c1fff29574 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -20,6 +20,15 @@ #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) +PyInterpreterState *get_interpreter_state_unchecked() { + auto cur_tstate = get_thread_state_unchecked(); + if (cur_tstate) + return cur_tstate->interp; + else + return nullptr; +} +PYBIND11_NAMESPACE_END() class subinterpreter; @@ -38,7 +47,7 @@ class subinterpreter_scoped_activate { private: PyThreadState *old_tstate_ = nullptr; - PyThreadState *free_tstate_ = nullptr; + PyThreadState *tstate_ = nullptr; PyGILState_STATE gil_state_; bool simple_gil_ = false; }; @@ -139,13 +148,13 @@ class subinterpreter { auto *&internals_ptr_ptr = detail::get_internals_pp(); auto *&local_internals_ptr_ptr = detail::get_internals_pp(); { - dict state_dict = detail::get_python_state_dict(); + dict sd = state_dict(); internals_ptr_ptr = detail::get_internals_pp_from_capsule_in_state_dict( - state_dict, PYBIND11_INTERNALS_ID); + sd, PYBIND11_INTERNALS_ID); local_internals_ptr_ptr = detail::get_internals_pp_from_capsule_in_state_dict( - state_dict, detail::get_local_internals_id()); + sd, detail::get_local_internals_id()); } // End it @@ -167,10 +176,6 @@ class subinterpreter { } } - /// abandon cleanup of this subinterpreter (leak it). this might be needed during - /// finalization... - void disarm() { tstate_ = nullptr; } - /// Get a handle to the main interpreter that can be used with subinterpreter_scoped_activate /// Note that destructing the handle is a noop, the main interpreter can only be ended by /// py::finalize_interpreter() @@ -181,6 +186,30 @@ class subinterpreter { return subinterpreter_scoped_activate(m); } + /// Get a non-owning wrapper of the currently active interpreter (if any) + static subinterpreter current() { + subinterpreter c; + c.istate_ = detail::get_interpreter_state_unchecked(); + c.disarm(); // make destruct a noop, we don't own this... + return c; + } + + /// Get the numerical identifier for the sub-interpreter + int64_t id() const { return PyInterpreterState_GetID(istate_); } + + /// Get the interpreter's state dict. This interpreter's GIL must be held before calling! + dict state_dict() { return reinterpret_borrow(PyInterpreterState_GetDict(istate_)); } + + /// abandon cleanup of this subinterpreter (leak it). this might be needed during + /// finalization... + void disarm() { tstate_ = nullptr; } + + /// An empty wrapper cannot be activated + bool empty() const { return istate_ == nullptr; } + + /// Is this wrapper non-empty + explicit operator bool() const { return !empty(); } + private: friend class subinterpreter_scoped_activate; PyThreadState *tstate_ = nullptr; @@ -200,14 +229,11 @@ class scoped_subinterpreter { }; inline subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpreter const &si) { -#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!si.istate_) { pybind11_fail("null subinterpreter"); } -#endif - auto cur_tstate = detail::get_thread_state_unchecked(); - if (cur_tstate && cur_tstate->interp == si.istate_) { + if (detail::get_interpreter_state_unchecked() == si.istate_) { // we are already on this interpreter, make sure we hold the GIL simple_gil_ = true; gil_state_ = PyGILState_Ensure(); @@ -216,13 +242,13 @@ inline subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpr // we can't really innteract with the interpreter at all until we switch to it // not even to, for example, look in it's state dict or touch its internals - free_tstate_ = PyThreadState_New(si.istate_); + tstate_ = PyThreadState_New(si.istate_); // make the interpreter active and acquire the GIL - old_tstate_ = PyThreadState_Swap(free_tstate_); + old_tstate_ = PyThreadState_Swap(tstate_); // save this in internals for scoped_gil calls - PYBIND11_TLS_REPLACE_VALUE(detail::get_internals().tstate, free_tstate_); + PYBIND11_TLS_REPLACE_VALUE(detail::get_internals().tstate, tstate_); } inline subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { @@ -230,14 +256,14 @@ inline subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { // We were on this interpreter already, so just make sure the GIL goes back as it was PyGILState_Release(gil_state_); } else { - if (free_tstate_) { + if (tstate_) { #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) - if (detail::get_thread_state_unchecked() != free_tstate_) { + if (detail::get_thread_state_unchecked() != tstate_) { pybind11_fail("~subinterpreter_scoped_activate: thread state must be current!"); } #endif PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate); - PyThreadState_Clear(free_tstate_); + PyThreadState_Clear(tstate_); PyThreadState_DeleteCurrent(); } From 70c24e8fdbb4431ab81e1086accd85c8b699d281 Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 18:11:12 -0400 Subject: [PATCH 10/31] Add some docs cross references --- docs/advanced/embedding.rst | 26 ++++++++++++++------------ docs/advanced/misc.rst | 2 ++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index d1148d24d5..92726a688c 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -247,8 +247,8 @@ global data. All the details can be found in the CPython documentation. .. _subinterp: -Sub-interpreter support -======================= +Embedding Sub-interpreters +========================== A sub-interpreter is a separate interpreter instance which provides a separate, isolated interpreter environment within the same process as the main @@ -256,13 +256,13 @@ interpreter. Sub-interpreters are created and managed with a separate API from the main interpreter. Beginning in Python 3.12, sub-interpreters each have their own Global Interpreter Lock (GIL), which means that running a sub-interpreter in a separate thread from the main interpreter can achieve true -concurrency. +concurrency. pybind11's sub-interpreter API can be found in ``pybind11/subinterpreter.h``. -pybind11 :class:`subinterpreter` instances can be safely moved and shared between -threads as needed. However, managing multiple threads and the lifetimes of multiple -interpreters and their GILs can be challenging. +pybind11 :class:`subinterpreter` instances can be safely moved and shared between +threads as needed. However, managing multiple threads and the lifetimes of multiple +interpreters and their GILs can be challenging. Proceed with caution (and lots of testing)! The main interpreter must be initialized before creating a sub-interpreter, and @@ -307,7 +307,7 @@ sub-interpreters. :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to manage the GIL of a sub-interpreter just as they do for the main interpreter. They both manage the GIL of the currently active interpreter, without the -programmer having to do anything special or different. There is one important +programmer having to do anything special or different. There is one important caveat: .. note:: @@ -319,7 +319,7 @@ caveat: Each sub-interpreter will import a separate copy of each ``PYBIND11_EMBEDDED_MODULE`` when those modules specify a ``multiple_interpreters`` tag. If a module does not -specify a ``multiple_interpreters`` tag, then Python will report an ``ImportError`` +specify a ``multiple_interpreters`` tag, then Python will report an ``ImportError`` if it is imported in a sub-interpreter. Here is an example showing how to create and activate sub-interpreters: @@ -393,8 +393,8 @@ it when it goes out of scope. Best Practices for sub-interpreter safety: -- Avoid moving or disarming RAII objects managing GIL and sub-interpreter lifetimes. Doing so can - lead to confusion about lifetimes. (For example, accidentally extending a +- Avoid moving or disarming RAII objects managing GIL and sub-interpreter lifetimes. Doing so can + lead to confusion about lifetimes. (For example, accidentally extending a :class:`subinterpreter_scoped_activate` past the lifetime of it's :class:`subinterpreter`.) - Never share Python objects across different interpreters. @@ -410,9 +410,11 @@ Best Practices for sub-interpreter safety: resulting Python object when the wrong interpreter was active. - While sub-interpreters each have their own GIL, there can now be multiple independent GILs in one - program you need to consider the possibility of deadlocks caused by multiple GILs and/or the + program you need to consider the possibility of deadlocks caused by multiple GILs and/or the interactions of the GIL(s) and your C++ code's own locking. - When using multiple threads to run independent sub-interpreters, the independent GILs allow - concurrent calls from different interpreters into the same C++ code from different threads. + concurrent calls from different interpreters into the same C++ code from different threads. So you must still consider the thread safety of your C++ code. + +- Familiarize yourself with :ref:`misc_concurrency`. diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 7d2a279585..b8cb1923e9 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -228,6 +228,8 @@ You can explicitly disable sub-interpreter support in your module by using the :func:`multiple_interpreters::not_supported()` tag. This is the default behavior if you do not specify a multiple_interpreters tag. +.. _misc_concurrency: + Concurrency and Parallelism in Python with pybind11 =================================================== From 3c78e4bde50e5845d1dfd754bff91cc97c9f7ed7 Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 18:11:39 -0400 Subject: [PATCH 11/31] Sync some things that were split out into #5665 --- docs/advanced/misc.rst | 4 ---- tests/test_embed/test_subinterpreter.cpp | 8 +++++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index b8cb1923e9..5bb14d3264 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -155,8 +155,6 @@ following checklist. within pybind11 that will throw exceptions on certain GIL handling errors (reference counting operations). -.. _misc_free_threading: - Free-threading support ================================================================== @@ -180,8 +178,6 @@ your code is thread safe. Modules must still be built against the Python free-t enable free-threading, even if they specify this tag. Adding this tag does not break compatibility with non-free-threaded Python. -.. _misc_subinterp: - Sub-interpreter support ================================================================== diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index b3df1f6811..6610352b7f 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -342,7 +342,13 @@ TEST_CASE("Per-Subinterpreter GIL") { T_REQUIRE(caught); // widget_module did provide the per_interpreter_gil tag, so it this does not throw - py::module_::import("widget_module"); + try { + py::module_::import("widget_module"); + caught = false; + } catch (pybind11::error_already_set &) { + caught = true; + } + T_REQUIRE(!caught); // widget_module did provide the per_interpreter_gil tag, so it this does not throw py::module_::import("widget_module"); From ca44bfe0fab2c6c27653a4524e438f3c7ebaa01a Mon Sep 17 00:00:00 2001 From: b-pass Date: Fri, 16 May 2025 21:09:52 -0400 Subject: [PATCH 12/31] Update subinterpreter docs example to not use the CPython api --- docs/advanced/embedding.rst | 2 +- include/pybind11/subinterpreter.h | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 92726a688c..9368804eda 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -335,7 +335,7 @@ Here is an example showing how to create and activate sub-interpreters: PYBIND11_EMBEDDED_MODULE(printer, m, py::multiple_interpreters::per_interpreter_gil()) { m.def("which", [](const std::string& when) { std::cout << when << "; Current Interpreter is " - << PyInterpreterState_GetID(PyInterpreterState_Get()) + << py::subinterpreter::current().id() << std::endl; }); } diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index c1fff29574..12d2a7ecd6 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -1,5 +1,5 @@ /* - pybind11/subinterpreters.h: Support for creating and using subinterpreters + pybind11/subinterpreter.h: Support for creating and using subinterpreters Copyright (c) 2025 The Pybind Development Team. @@ -195,7 +195,12 @@ class subinterpreter { } /// Get the numerical identifier for the sub-interpreter - int64_t id() const { return PyInterpreterState_GetID(istate_); } + int64_t id() const { + if (istate_ != nullptr) + return PyInterpreterState_GetID(istate_); + else + return -1; // CPython uses one-up numbers from 0, so negative should be safe to return here. + } /// Get the interpreter's state dict. This interpreter's GIL must be held before calling! dict state_dict() { return reinterpret_borrow(PyInterpreterState_GetDict(istate_)); } From e9b2a5792f04ddee2c84d2d687b212fde496a0dd Mon Sep 17 00:00:00 2001 From: b-pass Date: Fri, 16 May 2025 21:14:47 -0400 Subject: [PATCH 13/31] Fix pip test --- tests/extra_python_package/test_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 2919cf3ea3..9725bedae1 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -58,6 +58,7 @@ "include/pybind11/options.h", "include/pybind11/pybind11.h", "include/pybind11/pytypes.h", + "include/pybind11/subinterpreter.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", "include/pybind11/trampoline_self_life_support.h", From 096afd9e5a352179f867aacba2a1b0e8204d0fd5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 17 May 2025 01:15:10 +0000 Subject: [PATCH 14/31] style: pre-commit fixes --- include/pybind11/subinterpreter.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 12d2a7ecd6..16941ed438 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -195,11 +195,12 @@ class subinterpreter { } /// Get the numerical identifier for the sub-interpreter - int64_t id() const { + int64_t id() const { if (istate_ != nullptr) return PyInterpreterState_GetID(istate_); else - return -1; // CPython uses one-up numbers from 0, so negative should be safe to return here. + return -1; // CPython uses one-up numbers from 0, so negative should be safe to return + // here. } /// Get the interpreter's state dict. This interpreter's GIL must be held before calling! From 9db4f32205c9b53a2a537fa9ca390eb608ef73df Mon Sep 17 00:00:00 2001 From: b-pass Date: Fri, 16 May 2025 21:42:50 -0400 Subject: [PATCH 15/31] Fix MSVC warnings I am surprised other compilers allowed this code with a deleted move ctor. --- docs/advanced/embedding.rst | 8 ++++---- include/pybind11/subinterpreter.h | 8 ++++---- tests/test_embed/test_subinterpreter.cpp | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 9368804eda..dadfc1e58a 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -341,7 +341,7 @@ Here is an example showing how to create and activate sub-interpreters: } int main() { - py::scoped_interpreter main_int{}; + py::scoped_interpreter main_interp; py::module_::import("printer").attr("which")("First init"); @@ -351,7 +351,7 @@ Here is an example showing how to create and activate sub-interpreters: py::module_::import("printer").attr("which")("Created sub"); { - py::subinterpreter_scoped_activate ssa(sub); + py::subinterpreter_scoped_activate guard(sub); py::module_::import("printer").attr("which")("Activated sub"); } @@ -360,9 +360,9 @@ Here is an example showing how to create and activate sub-interpreters: { py::gil_scoped_release nogil; { - py::subinterpreter_scoped_activate ssa(sub); + py::subinterpreter_scoped_activate guard(sub); { - auto main_sa = py::subinterpreter::main_scoped_activate(); + py::subinterpreter_scoped_activate main_guard(py::subinterpreter::main()); py::module_::import("printer").attr("which")("Main within sub"); } py::module_::import("printer").attr("which")("After Main, still within sub"); diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 16941ed438..e1b4d505f3 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -28,7 +28,7 @@ PyInterpreterState *get_interpreter_state_unchecked() { else return nullptr; } -PYBIND11_NAMESPACE_END() +PYBIND11_NAMESPACE_END(detail) class subinterpreter; @@ -77,7 +77,7 @@ class subinterpreter { /// interpreter and its GIL are not required to be held prior to calling this function. static inline subinterpreter create(PyInterpreterConfig const &cfg) { error_scope err_scope; - auto main_guard = main_scoped_activate(); + subinterpreter_scoped_activate main_guard(main()); subinterpreter result; { // we must hold the main GIL in order to create a subinterpreter @@ -179,11 +179,11 @@ class subinterpreter { /// Get a handle to the main interpreter that can be used with subinterpreter_scoped_activate /// Note that destructing the handle is a noop, the main interpreter can only be ended by /// py::finalize_interpreter() - static subinterpreter_scoped_activate main_scoped_activate() { + static subinterpreter main() { subinterpreter m; m.istate_ = PyInterpreterState_Main(); m.disarm(); // make destruct a noop - return subinterpreter_scoped_activate(m); + return m; } /// Get a non-owning wrapper of the currently active interpreter (if any) diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index 6610352b7f..8a962240e1 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -132,7 +132,7 @@ TEST_CASE("GIL Subinterpreter") { py::module_::import("external_module"); { - auto main_activate = py::subinterpreter::main_scoped_activate(); + py::subinterpreter_scoped_activate main(py::subinterpreter::main()); REQUIRE(PyInterpreterState_Get() == main_interp); { From a536fdc3da79c6c600b50dff0dd4a91500e1d564 Mon Sep 17 00:00:00 2001 From: b-pass Date: Fri, 16 May 2025 22:03:32 -0400 Subject: [PATCH 16/31] Add some sub-headings to the docs --- docs/advanced/embedding.rst | 79 ++++++++++++++++++++++++++++--------- docs/advanced/misc.rst | 4 ++ 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index dadfc1e58a..6a7c2b767b 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -288,6 +288,18 @@ sub-interpreter, call :func:`subinterpreter::create()`. the call before returning (or return to no active interpreter if none was active at the time of the call). +Each sub-interpreter will import a separate copy of each ``PYBIND11_EMBEDDED_MODULE`` +when those modules specify a ``multiple_interpreters`` tag. If a module does not +specify a ``multiple_interpreters`` tag, then Python will report an ``ImportError`` +if it is imported in a sub-interpreter. + +pybind11 also has a :class:`scoped_subinterpreter` class, which creates and +activates a sub-interpreter when it is constructed, and deactivates and deletes +it when it goes out of scope. + +Activating a Sub-interpreter +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Once a sub-interpreter is created, you can "activate" it on a thread (and acquire it's GIL) by creating a :class:`subinterpreter_scoped_activate` instance and passing it the sub-intepreter to be activated. The function @@ -297,12 +309,44 @@ instance. When the :class:`subinterpreter_scoped_activate` instance goes out of scope, the sub-interpreter GIL is released and the prior interpreter that was active on the thread (if any) is reactivated and it's GIL is re-acquired. -The :func:`subinterpreter::activate_main()` function activates the main -interpreter, acquiring it's GIL, and returns a -:class:`subinterpreter_scoped_activate` instance which will automatically -deactivate the main interpreter and release it's GIL when it goes out of -scope, just as :class:`subinterpreter_scoped_activate` also does for -sub-interpreters. +When using ``subinterpreter_scoped_activate``: + +1. If the thread holds any interpreter's GIL: + - That GIL is released +2. The new sub-interpreter's GIL is acquired +3. The new sub-interpreter is made active. +4. When the scope ends: + - The sub-interpreter's GIL is released + - If there was a previous interpreter: + - The old interpreter's GIL is re-acquired + - The old interpreter is made active + - Otherwise, no interpreter is currently active and no GIL is held. + +Example: + +.. code-block:: cpp + + py::initialize_interpreter(); + // Main GIL is held + { + py::subinterpreter sub = py::subinterpreter::create(); + // Main interpreter is still active, main GIL re-acquired + { + py::subinterpreter_scoped_activate guard(sub); + // Sub-interpreter active, thread holds sub's GIL + { + py::subinterpreter_scoped_activate main_guard(py); + // Sub's GIL was automatically released + // Main interpreter active, thread holds main's GIL + } + // Back to sub-interpreter, thread holds sub's GIL again + } + // Main interpreter is active, main's GIL is held + } + + +GIL API for sub-interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to manage the GIL of a sub-interpreter just as they do for the main interpreter. @@ -317,10 +361,9 @@ caveat: :class:`gil_scoped_acquire` will acquire the **main** GIL and activate the **main** interpreter. -Each sub-interpreter will import a separate copy of each ``PYBIND11_EMBEDDED_MODULE`` -when those modules specify a ``multiple_interpreters`` tag. If a module does not -specify a ``multiple_interpreters`` tag, then Python will report an ``ImportError`` -if it is imported in a sub-interpreter. + +Full Sub-interpreter example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Here is an example showing how to create and activate sub-interpreters: @@ -387,15 +430,9 @@ Expected output: After Main, still within sub; Current Interpreter is 1 At end; Current Interpreter is 0 -pybind11 also has a :class:`scoped_subinterpreter` class, which creates and -activates a sub-interpreter when it is constructed, and deactivates and deletes -it when it goes out of scope. - -Best Practices for sub-interpreter safety: -- Avoid moving or disarming RAII objects managing GIL and sub-interpreter lifetimes. Doing so can - lead to confusion about lifetimes. (For example, accidentally extending a - :class:`subinterpreter_scoped_activate` past the lifetime of it's :class:`subinterpreter`.) +Best Practices for sub-interpreter safety +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Never share Python objects across different interpreters. @@ -409,8 +446,12 @@ Best Practices for sub-interpreter safety: did not save the result of :func:`module_::import`, in order to avoid accidentally using the resulting Python object when the wrong interpreter was active. +- Avoid moving or disarming RAII objects managing GIL and sub-interpreter lifetimes. Doing so can + lead to confusion about lifetimes. (For example, accidentally extending a + :class:`subinterpreter_scoped_activate` past the lifetime of it's :class:`subinterpreter`.) + - While sub-interpreters each have their own GIL, there can now be multiple independent GILs in one - program you need to consider the possibility of deadlocks caused by multiple GILs and/or the + program so you need to consider the possibility of deadlocks caused by multiple GILs and/or the interactions of the GIL(s) and your C++ code's own locking. - When using multiple threads to run independent sub-interpreters, the independent GILs allow diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index 5bb14d3264..b8cb1923e9 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -155,6 +155,8 @@ following checklist. within pybind11 that will throw exceptions on certain GIL handling errors (reference counting operations). +.. _misc_free_threading: + Free-threading support ================================================================== @@ -178,6 +180,8 @@ your code is thread safe. Modules must still be built against the Python free-t enable free-threading, even if they specify this tag. Adding this tag does not break compatibility with non-free-threaded Python. +.. _misc_subinterp: + Sub-interpreter support ================================================================== From 29e31715b932a47859a23996ba81527df85a61d4 Mon Sep 17 00:00:00 2001 From: b-pass Date: Fri, 16 May 2025 22:23:46 -0400 Subject: [PATCH 17/31] Oops, make_unique is C++14 so remove it from the tests. --- tests/test_embed/test_subinterpreter.cpp | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index 8a962240e1..4a052e9b5b 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -82,11 +82,11 @@ TEST_CASE("Single Subinterpreter") { TEST_CASE("Move Subinterpreter") { - auto ssi = std::make_unique(py::subinterpreter::create()); + std::unique_ptr sub(new py::subinterpreter(py::subinterpreter::create())); // on this thread, use the subinterpreter and import some non-trivial junk { - py::subinterpreter_scoped_activate activate(*ssi); + py::subinterpreter_scoped_activate activate(*sub); py::list(py::module_::import("sys").attr("path")).append(py::str(".")); py::module_::import("datetime"); @@ -94,18 +94,17 @@ TEST_CASE("Move Subinterpreter") { py::module_::import("external_module"); } - std::thread temp([ssi = std::move(ssi)]() mutable { + std::thread([&]() { // Use it again { - py::subinterpreter_scoped_activate activate(*ssi); + py::subinterpreter_scoped_activate activate(*sub); py::module_::import("external_module"); } + sub.reset(); + }).join(); + + REQUIRE(!sub); - // free it - ssi.reset(); - }); - temp.join(); - REQUIRE(!ssi); unsafe_reset_internals_for_single_interpreter(); } @@ -114,14 +113,14 @@ TEST_CASE("GIL Subinterpreter") { PyInterpreterState *main_interp = PyInterpreterState_Get(); { - auto ssi = py::subinterpreter::create(); + auto sub = py::subinterpreter::create(); REQUIRE(main_interp == PyInterpreterState_Get()); PyInterpreterState *sub_interp = nullptr; { - py::subinterpreter_scoped_activate activate(ssi); + py::subinterpreter_scoped_activate activate(sub); sub_interp = PyInterpreterState_Get(); REQUIRE(sub_interp != main_interp); @@ -178,7 +177,7 @@ TEST_CASE("GIL Subinterpreter") { py::gil_scoped_release nogil{}; std::thread([&]() { { - py::subinterpreter_scoped_activate ssa{ssi}; + py::subinterpreter_scoped_activate ssa{sub}; } { py::gil_scoped_acquire gil{}; @@ -235,7 +234,7 @@ TEST_CASE("Multiple Subinterpreters") { REQUIRE(sub1_int != main_int); // while the old one is active, create a new one - psi2 = std::make_unique(py::subinterpreter::create()); + psi2.reset(new py::subinterpreter(py::subinterpreter::create())); } { From d4b1c4435f612b0fe97e20845c207eb8e9f0ac51 Mon Sep 17 00:00:00 2001 From: b-pass Date: Sat, 17 May 2025 21:53:27 -0400 Subject: [PATCH 18/31] I think this fixes the EndInterpreter issues on all versions. It just has to be ifdef'd because it is slightly broken on 3.12, working well on 3.13, and kind of crashy on 3.14beta. These two verion ifdefs solve all the issues. --- include/pybind11/subinterpreter.h | 120 +++++++++++++++--------------- 1 file changed, 61 insertions(+), 59 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index e1b4d505f3..44d7e38ba7 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -61,14 +61,15 @@ class subinterpreter { subinterpreter(subinterpreter const ©) = delete; subinterpreter &operator=(subinterpreter const ©) = delete; - subinterpreter(subinterpreter &&old) : tstate_(old.tstate_), istate_(old.istate_) { - old.tstate_ = nullptr; + subinterpreter(subinterpreter &&old) + : istate_(old.istate_), creation_tstate_(old.creation_tstate_) { old.istate_ = nullptr; + old.creation_tstate_ = nullptr; } subinterpreter &operator=(subinterpreter &&old) { - std::swap(old.tstate_, tstate_); std::swap(old.istate_, istate_); + std::swap(old.creation_tstate_, creation_tstate_); return *this; } @@ -85,7 +86,7 @@ class subinterpreter { auto prev_tstate = PyThreadState_Get(); - auto status = Py_NewInterpreterFromConfig(&result.tstate_, &cfg); + auto status = Py_NewInterpreterFromConfig(&result.creation_tstate_, &cfg); // this doesn't raise a normal Python exception, it provides an exit() status code. if (PyStatus_Exception(status)) { @@ -93,10 +94,18 @@ class subinterpreter { } // upon success, the new interpreter is activated in this thread - result.istate_ = result.tstate_->interp; + result.istate_ = result.creation_tstate_->interp; detail::get_num_interpreters_seen() += 1; // there are now many interpreters detail::get_internals(); // initialize internals.tstate, amongst other things... + // In 3.13+ this state should be deleted right away, and the memory will be reused for + // the next threadstate on this interpreter. However, on 3.12 we cannot do that, we + // must keep it around (but not use it) ... see destructor. +#if PY_VERSION_HEX >= 0x030D0000 + PyThreadState_Clear(result.creation_tstate_); + PyThreadState_DeleteCurrent(); +#endif + // we have to switch back to main, and then the scopes will handle cleanup PyThreadState_Swap(prev_tstate); } @@ -115,65 +124,58 @@ class subinterpreter { } ~subinterpreter() { - if (tstate_) { - PyThreadState *old_tstate; + if (!creation_tstate_) { + // non-owning wrapper, do nothing. + return; + } - bool wrong_thread = true; -#ifdef PY_HAVE_THREAD_NATIVE_ID - wrong_thread = PyThread_get_thread_native_id() != tstate_->native_thread_id; -#endif - if (wrong_thread) { - // The PyThreadState used in Py_EndInterpreter must be created on this OS thread. - // (If it is not, subinterpreter cleanup will hang within Python's threading - // module.) So create a new thread state here on the right OS thread. - auto *temp = PyThreadState_New(tstate_->interp); - old_tstate = PyThreadState_Swap(temp); - // delete the other one because there must be only one at cleanup - PyThreadState_Clear(tstate_); - PyThreadState_Delete(tstate_); - tstate_ = temp; - } else { - old_tstate = PyThreadState_Swap(tstate_); - } + auto *temp = PyThreadState_New(istate_); + auto *old_tstate = PyThreadState_Swap(temp); - bool switch_back = old_tstate && old_tstate->interp != tstate_->interp; - - // make sure we have the GIL for the interpreter we are ending - (void) PyGILState_Ensure(); - - // Get the internals pointer (without creating it if it doesn't exist). It's possible - // for the internals to be created during Py_EndInterpreter() (e.g. if a py::capsule - // calls `get_internals()` during destruction), so we get the pointer-pointer here and - // check it after. - auto *&internals_ptr_ptr = detail::get_internals_pp(); - auto *&local_internals_ptr_ptr = detail::get_internals_pp(); - { - dict sd = state_dict(); - internals_ptr_ptr - = detail::get_internals_pp_from_capsule_in_state_dict( - sd, PYBIND11_INTERNALS_ID); - local_internals_ptr_ptr - = detail::get_internals_pp_from_capsule_in_state_dict( - sd, detail::get_local_internals_id()); - } + bool switch_back = old_tstate && old_tstate->interp != istate_; - // End it - Py_EndInterpreter(tstate_); +#if PY_VERSION_HEX < 0x030D0000 + // In 3.12 we can only delete this thread state when we know we are about to End the + // interpreter... because otherwise Python will try to reuse it, and fail, and abort. We + // cannot use this one in the call to EndInterpreter either because it may have been + // created on a different OS thread. So we have to make a new one first, switch to that + // one, and the delete this one finally. + PyThreadState_Clear(creation_tstate_); + PyThreadState_Delete(creation_tstate_); +#endif - // do NOT decrease detail::get_num_interpreters_seen, because it can never decrease - // while other threads are running... + // Get the internals pointer (without creating it if it doesn't exist). It's possible + // for the internals to be created during Py_EndInterpreter() (e.g. if a py::capsule + // calls `get_internals()` during destruction), so we get the pointer-pointer here and + // check it after. + auto *&internals_ptr_ptr = detail::get_internals_pp(); + auto *&local_internals_ptr_ptr = detail::get_internals_pp(); + { + dict sd = state_dict(); + internals_ptr_ptr + = detail::get_internals_pp_from_capsule_in_state_dict( + sd, PYBIND11_INTERNALS_ID); + local_internals_ptr_ptr + = detail::get_internals_pp_from_capsule_in_state_dict( + sd, detail::get_local_internals_id()); + } - if (internals_ptr_ptr) { - internals_ptr_ptr->reset(); - } - if (local_internals_ptr_ptr) { - local_internals_ptr_ptr->reset(); - } + // End it + Py_EndInterpreter(temp); - // switch back to the old tstate and old GIL (if there was one) - if (switch_back) - PyThreadState_Swap(old_tstate); + // do NOT decrease detail::get_num_interpreters_seen, because it can never decrease + // while other threads are running... + + if (internals_ptr_ptr) { + internals_ptr_ptr->reset(); + } + if (local_internals_ptr_ptr) { + local_internals_ptr_ptr->reset(); } + + // switch back to the old tstate and old GIL (if there was one) + if (switch_back) + PyThreadState_Swap(old_tstate); } /// Get a handle to the main interpreter that can be used with subinterpreter_scoped_activate @@ -208,7 +210,7 @@ class subinterpreter { /// abandon cleanup of this subinterpreter (leak it). this might be needed during /// finalization... - void disarm() { tstate_ = nullptr; } + void disarm() { creation_tstate_ = nullptr; } /// An empty wrapper cannot be activated bool empty() const { return istate_ == nullptr; } @@ -218,8 +220,8 @@ class subinterpreter { private: friend class subinterpreter_scoped_activate; - PyThreadState *tstate_ = nullptr; PyInterpreterState *istate_ = nullptr; + PyThreadState *creation_tstate_ = nullptr; }; class scoped_subinterpreter { From eda11c1d891734dd26e19e3cc11c08c00df903f6 Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 17:12:39 -0400 Subject: [PATCH 19/31] Add a note about exceptions. They contain Python object references and acquire the GIL, that means they are a danger with subinterpreters! --- docs/advanced/embedding.rst | 6 ++++++ include/pybind11/subinterpreter.h | 25 +++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 6a7c2b767b..cfef80eb29 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -436,6 +436,12 @@ Best Practices for sub-interpreter safety - Never share Python objects across different interpreters. +- :class:`error_already_set` objects contain a reference to the Python exception type, + and :func:`error_already_set::what()` acquires the GIL. So Python exceptions must + **never** be allowed to propoagate past the enclosing + :class:`subinterpreter_scoped_activate` instance! + (So your try/catch should be *just inside* the scope covered by the :class:`subinterpreter_scoped_activate`.) + - Avoid global/static state whenever possible. Instead, keep state within each interpreter, such as within the interpreter state dict, which can be accessed via ``subinterpreter::current().state_dict()``, or within instance members and tied to diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 44d7e38ba7..4e636ccc61 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -78,11 +78,10 @@ class subinterpreter { /// interpreter and its GIL are not required to be held prior to calling this function. static inline subinterpreter create(PyInterpreterConfig const &cfg) { error_scope err_scope; - subinterpreter_scoped_activate main_guard(main()); subinterpreter result; { // we must hold the main GIL in order to create a subinterpreter - gil_scoped_acquire gil; + subinterpreter_scoped_activate main_guard(main()); auto prev_tstate = PyThreadState_Get(); @@ -264,6 +263,28 @@ inline subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { // We were on this interpreter already, so just make sure the GIL goes back as it was PyGILState_Release(gil_state_); } else { +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + bool has_active_exception; +# if defined(__cpp_lib_uncaught_exceptions) + has_active_exception = std::uncaught_exceptions() > 0; +# else + // removed in C++20, replaced with uncaught_exceptions + has_active_exception = std::uncaught_exception(); +# endif + if (has_active_exception) { + try { + std::rethrow_exception(std::current_exception()); + } catch (error_already_set &e) { + // Because error_already_set holds python objects and what() acquires the GIL, it + // is basically never OK to let these exceptions propagate outside the current + // active interpreter. + pybind11_fail("~subinterpreter_scoped_activate: cannot propagate Python " + "exceptions outside of their owning interpreter"); + } catch (...) { + } + } +#endif + if (tstate_) { #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (detail::get_thread_state_unchecked() != tstate_) { From cd120191e50782ffb0f219db9beb6f8e5252a3ba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 18 May 2025 22:18:14 +0000 Subject: [PATCH 20/31] style: pre-commit fixes --- docs/advanced/embedding.rst | 6 +++--- tests/test_embed/test_subinterpreter.cpp | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index cfef80eb29..396a625ee6 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -436,9 +436,9 @@ Best Practices for sub-interpreter safety - Never share Python objects across different interpreters. -- :class:`error_already_set` objects contain a reference to the Python exception type, - and :func:`error_already_set::what()` acquires the GIL. So Python exceptions must - **never** be allowed to propoagate past the enclosing +- :class:`error_already_set` objects contain a reference to the Python exception type, + and :func:`error_already_set::what()` acquires the GIL. So Python exceptions must + **never** be allowed to propoagate past the enclosing :class:`subinterpreter_scoped_activate` instance! (So your try/catch should be *just inside* the scope covered by the :class:`subinterpreter_scoped_activate`.) diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index 4a052e9b5b..d042ae2ffe 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -44,7 +44,8 @@ TEST_CASE("Single Subinterpreter") { REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); - auto main_int = py::module_::import("external_module").attr("internals_at")().cast(); + auto main_int + = py::module_::import("external_module").attr("internals_at")().cast(); /// Create and switch to a subinterpreter. { From bddcfb5ce2b5db0b848518d217528e1234da721b Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 18:40:17 -0400 Subject: [PATCH 21/31] Add try/catch to docs examples to match the tips --- docs/advanced/embedding.rst | 30 ++++++++++++++++++++++++------ include/pybind11/subinterpreter.h | 2 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 396a625ee6..4a67c47958 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -395,7 +395,13 @@ Here is an example showing how to create and activate sub-interpreters: { py::subinterpreter_scoped_activate guard(sub); - py::module_::import("printer").attr("which")("Activated sub"); + try { + py::module_::import("printer").attr("which")("Activated sub"); + } + catch (py::error_already_set &e) { + std::cerr << "EXCEPTION " << e.what() << std::endl; + return 1; + } } py::module_::import("printer").attr("which")("Deactivated sub"); @@ -404,11 +410,23 @@ Here is an example showing how to create and activate sub-interpreters: py::gil_scoped_release nogil; { py::subinterpreter_scoped_activate guard(sub); - { - py::subinterpreter_scoped_activate main_guard(py::subinterpreter::main()); - py::module_::import("printer").attr("which")("Main within sub"); + try { + { + py::subinterpreter_scoped_activate main_guard(py::subinterpreter::main()); + try { + py::module_::import("printer").attr("which")("Main within sub"); + } + catch (py::error_already_set &e) { + std::cerr << "EXCEPTION " << e.what() << std::endl; + return 1; + } + } + py::module_::import("printer").attr("which")("After Main, still within sub"); + } + catch (py::error_already_set &e) { + std::cerr << "EXCEPTION " << e.what() << std::endl; + return 1; } - py::module_::import("printer").attr("which")("After Main, still within sub"); } } } @@ -438,7 +456,7 @@ Best Practices for sub-interpreter safety - :class:`error_already_set` objects contain a reference to the Python exception type, and :func:`error_already_set::what()` acquires the GIL. So Python exceptions must - **never** be allowed to propoagate past the enclosing + **never** be allowed to propagate past the enclosing :class:`subinterpreter_scoped_activate` instance! (So your try/catch should be *just inside* the scope covered by the :class:`subinterpreter_scoped_activate`.) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 4e636ccc61..a1ae26dcf5 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -274,7 +274,7 @@ inline subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { if (has_active_exception) { try { std::rethrow_exception(std::current_exception()); - } catch (error_already_set &e) { + } catch (error_already_set &) { // Because error_already_set holds python objects and what() acquires the GIL, it // is basically never OK to let these exceptions propagate outside the current // active interpreter. From 8770bfadb7f72552ee192771c81e72a7c6c074d0 Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 21:50:11 -0400 Subject: [PATCH 22/31] Python 3.12 is very picky about this first PyThreadState Try special casing the destruction on the same thread. --- include/pybind11/subinterpreter.h | 39 +++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index a1ae26dcf5..4c4b3febde 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -128,20 +128,35 @@ class subinterpreter { return; } - auto *temp = PyThreadState_New(istate_); - auto *old_tstate = PyThreadState_Swap(temp); - - bool switch_back = old_tstate && old_tstate->interp != istate_; + PyThreadState *destroy_tstate; + PyThreadState *old_tstate; + // Python 3.12 requires us to keep the original PyThreadState alive until we are ready to destroy the interpreter. We prefer to use that to destroy the interpreter. #if PY_VERSION_HEX < 0x030D0000 - // In 3.12 we can only delete this thread state when we know we are about to End the - // interpreter... because otherwise Python will try to reuse it, and fail, and abort. We - // cannot use this one in the call to EndInterpreter either because it may have been - // created on a different OS thread. So we have to make a new one first, switch to that - // one, and the delete this one finally. - PyThreadState_Clear(creation_tstate_); - PyThreadState_Delete(creation_tstate_); + // The tstate passed to Py_EndInterpreter MUST have been created on the current OS thread. + bool same_thread = false; + #ifdef PY_HAVE_THREAD_NATIVE_ID + same_thread = PyThread_get_thread_native_id() == creation_tstate_->native_thread_id; + #endif + if (same_thread) { + // OK it is safe to use the creation state here + destroy_tstate = creation_tstate_; + old_tstate = PyThreadState_Swap(destroy_tstate); + } else { + // We have to make a new tstate on this thread and use that. + destroy_tstate = PyThreadState_New(istate_); + old_tstate = PyThreadState_Swap(destroy_tstate); + + // We can use temp, the one we just created, and delete the creation state. + PyThreadState_Clear(creation_tstate_); + PyThreadState_Delete(creation_tstate_); + } +#else + destroy_state = PyThreadState_New(istate_); + old_tstate = PyThreadState_Swap(destroy_state); #endif + + bool switch_back = old_tstate && old_tstate->interp != istate_; // Get the internals pointer (without creating it if it doesn't exist). It's possible // for the internals to be created during Py_EndInterpreter() (e.g. if a py::capsule @@ -160,7 +175,7 @@ class subinterpreter { } // End it - Py_EndInterpreter(temp); + Py_EndInterpreter(destroy_tstate); // do NOT decrease detail::get_num_interpreters_seen, because it can never decrease // while other threads are running... From ae097c8d477ecac039c25f4cfa26840a679304ba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 01:51:10 +0000 Subject: [PATCH 23/31] style: pre-commit fixes --- include/pybind11/subinterpreter.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 4c4b3febde..e99e68c364 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -131,13 +131,14 @@ class subinterpreter { PyThreadState *destroy_tstate; PyThreadState *old_tstate; - // Python 3.12 requires us to keep the original PyThreadState alive until we are ready to destroy the interpreter. We prefer to use that to destroy the interpreter. + // Python 3.12 requires us to keep the original PyThreadState alive until we are ready to + // destroy the interpreter. We prefer to use that to destroy the interpreter. #if PY_VERSION_HEX < 0x030D0000 // The tstate passed to Py_EndInterpreter MUST have been created on the current OS thread. bool same_thread = false; - #ifdef PY_HAVE_THREAD_NATIVE_ID +# ifdef PY_HAVE_THREAD_NATIVE_ID same_thread = PyThread_get_thread_native_id() == creation_tstate_->native_thread_id; - #endif +# endif if (same_thread) { // OK it is safe to use the creation state here destroy_tstate = creation_tstate_; @@ -155,7 +156,7 @@ class subinterpreter { destroy_state = PyThreadState_New(istate_); old_tstate = PyThreadState_Swap(destroy_state); #endif - + bool switch_back = old_tstate && old_tstate->interp != istate_; // Get the internals pointer (without creating it if it doesn't exist). It's possible From 60622303540db27e400656764f8a59ab28ed8d12 Mon Sep 17 00:00:00 2001 From: b-pass Date: Sun, 18 May 2025 22:10:56 -0400 Subject: [PATCH 24/31] Missed a rename in a ifdef block --- include/pybind11/subinterpreter.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index e99e68c364..b9400cfa2e 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -148,13 +148,13 @@ class subinterpreter { destroy_tstate = PyThreadState_New(istate_); old_tstate = PyThreadState_Swap(destroy_tstate); - // We can use temp, the one we just created, and delete the creation state. + // We can use the one we just created, so we must delete the creation state. PyThreadState_Clear(creation_tstate_); PyThreadState_Delete(creation_tstate_); } #else - destroy_state = PyThreadState_New(istate_); - old_tstate = PyThreadState_Swap(destroy_state); + destroy_tstate = PyThreadState_New(istate_); + old_tstate = PyThreadState_Swap(destroy_tstate); #endif bool switch_back = old_tstate && old_tstate->interp != istate_; From 2aae3b85f71f9bfcc232f4dc29e203500f594db3 Mon Sep 17 00:00:00 2001 From: b-pass Date: Mon, 19 May 2025 16:44:21 -0400 Subject: [PATCH 25/31] I think this test is causing problems in 3.12, so try ifdefing it to see if the problems go away. --- tests/test_embed/test_subinterpreter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index d042ae2ffe..73ae3aa312 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -81,8 +81,8 @@ TEST_CASE("Single Subinterpreter") { unsafe_reset_internals_for_single_interpreter(); } +#if PY_VERSION_HEX >= 0x030D0000 TEST_CASE("Move Subinterpreter") { - std::unique_ptr sub(new py::subinterpreter(py::subinterpreter::create())); // on this thread, use the subinterpreter and import some non-trivial junk @@ -108,6 +108,7 @@ TEST_CASE("Move Subinterpreter") { unsafe_reset_internals_for_single_interpreter(); } +#endif TEST_CASE("GIL Subinterpreter") { From dc57729316b9b075c2da5072cfd9c9d22b337625 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 20:45:12 +0000 Subject: [PATCH 26/31] style: pre-commit fixes --- tests/test_embed/test_subinterpreter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_embed/test_subinterpreter.cpp b/tests/test_embed/test_subinterpreter.cpp index 73ae3aa312..9d7d88b8a0 100644 --- a/tests/test_embed/test_subinterpreter.cpp +++ b/tests/test_embed/test_subinterpreter.cpp @@ -81,7 +81,7 @@ TEST_CASE("Single Subinterpreter") { unsafe_reset_internals_for_single_interpreter(); } -#if PY_VERSION_HEX >= 0x030D0000 +# if PY_VERSION_HEX >= 0x030D0000 TEST_CASE("Move Subinterpreter") { std::unique_ptr sub(new py::subinterpreter(py::subinterpreter::create())); @@ -108,7 +108,7 @@ TEST_CASE("Move Subinterpreter") { unsafe_reset_internals_for_single_interpreter(); } -#endif +# endif TEST_CASE("GIL Subinterpreter") { From ae0da30f401d67cd8c8ea6489c9cc43a4a03a8d4 Mon Sep 17 00:00:00 2001 From: b-pass Date: Mon, 19 May 2025 18:03:46 -0400 Subject: [PATCH 27/31] Document the 3.12 constraints with a warning --- docs/advanced/embedding.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index 4a67c47958..aa574e976e 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -448,6 +448,14 @@ Expected output: After Main, still within sub; Current Interpreter is 1 At end; Current Interpreter is 0 +.. warning:: + + In Python 3.12 sub-interpreters must be destroyed in the same OS thread + that created them. Failure to follow this rule may result in deadlocks + or crashes when destroying the sub-interpreter on the wrong thread. + + This constraint is not present in Python 3.13+. + Best Practices for sub-interpreter safety ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -458,7 +466,8 @@ Best Practices for sub-interpreter safety and :func:`error_already_set::what()` acquires the GIL. So Python exceptions must **never** be allowed to propagate past the enclosing :class:`subinterpreter_scoped_activate` instance! - (So your try/catch should be *just inside* the scope covered by the :class:`subinterpreter_scoped_activate`.) + (So your try/catch should be *just inside* the scope covered by the + :class:`subinterpreter_scoped_activate`.) - Avoid global/static state whenever possible. Instead, keep state within each interpreter, such as within the interpreter state dict, which can be accessed via @@ -480,6 +489,7 @@ Best Practices for sub-interpreter safety - When using multiple threads to run independent sub-interpreters, the independent GILs allow concurrent calls from different interpreters into the same C++ code from different threads. - So you must still consider the thread safety of your C++ code. + So you must still consider the thread safety of your C++ code. Remember, in Python 3.12 + sub-interpreters must be destroyed on the same thread that they were created on. - Familiarize yourself with :ref:`misc_concurrency`. From 68c68d7d160612369a0c4b35644a1d83f2817ce5 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 20 May 2025 01:34:42 -0400 Subject: [PATCH 28/31] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/advanced/embedding.rst | 2 +- include/pybind11/subinterpreter.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index aa574e976e..3ac0579385 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -301,7 +301,7 @@ Activating a Sub-interpreter ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Once a sub-interpreter is created, you can "activate" it on a thread (and -acquire it's GIL) by creating a :class:`subinterpreter_scoped_activate` +acquire its GIL) by creating a :class:`subinterpreter_scoped_activate` instance and passing it the sub-intepreter to be activated. The function will acquire the sub-interpreter's GIL and make the sub-interpreter the current active interpreter on the current thread for the lifetime of the diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index b9400cfa2e..f77482875f 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -32,8 +32,8 @@ PYBIND11_NAMESPACE_END(detail) class subinterpreter; -/// Activate the subinterpreter and acquire it's GIL, while also releasing any GIL and interpreter -/// currently held. Upon exiting the scope, the previous subinterpreter (if any) and it's +/// Activate the subinterpreter and acquire its GIL, while also releasing any GIL and interpreter +/// currently held. Upon exiting the scope, the previous subinterpreter (if any) and its /// associated GIL are restored to their state as they were before the scope was entered. class subinterpreter_scoped_activate { public: @@ -263,8 +263,8 @@ inline subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpr return; } - // we can't really innteract with the interpreter at all until we switch to it - // not even to, for example, look in it's state dict or touch its internals + // we can't really interact with the interpreter at all until we switch to it + // not even to, for example, look in its state dict or touch its internals tstate_ = PyThreadState_New(si.istate_); // make the interpreter active and acquire the GIL From 30c520b47eac60ccd1de1e8c47200e2b8401f5c1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 20 May 2025 12:27:57 -0400 Subject: [PATCH 29/31] ci: add cpptest to the clang-tidy job Signed-off-by: Henry Schreiner --- .github/workflows/format.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 3d6d1e39fd..bc2db70ea2 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -55,3 +55,6 @@ jobs: - name: Build run: cmake --build build -j 2 -- --keep-going + + - name: Embedded + run: cmake --build build -t cpptest -j 2 -- --keep-going From babf12ad36a5d039957229b5f204ebab1e608089 Mon Sep 17 00:00:00 2001 From: b-pass Date: Tue, 20 May 2025 16:21:23 -0400 Subject: [PATCH 30/31] noexcept move operations --- include/pybind11/subinterpreter.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index f77482875f..3ce03de284 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -61,13 +61,13 @@ class subinterpreter { subinterpreter(subinterpreter const ©) = delete; subinterpreter &operator=(subinterpreter const ©) = delete; - subinterpreter(subinterpreter &&old) + subinterpreter(subinterpreter &&old) noexcept : istate_(old.istate_), creation_tstate_(old.creation_tstate_) { old.istate_ = nullptr; old.creation_tstate_ = nullptr; } - subinterpreter &operator=(subinterpreter &&old) { + subinterpreter &operator=(subinterpreter &&old) noexcept { std::swap(old.istate_, istate_); std::swap(old.creation_tstate_, creation_tstate_); return *this; From 59f82a2dc936fcb3d626372c781baeda200a9307 Mon Sep 17 00:00:00 2001 From: b-pass Date: Tue, 20 May 2025 16:22:26 -0400 Subject: [PATCH 31/31] Update include/pybind11/subinterpreter.h std::memset Co-authored-by: Aaron Gokaslan --- include/pybind11/subinterpreter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/subinterpreter.h b/include/pybind11/subinterpreter.h index 3ce03de284..306e47bc6a 100644 --- a/include/pybind11/subinterpreter.h +++ b/include/pybind11/subinterpreter.h @@ -116,7 +116,7 @@ class subinterpreter { static inline subinterpreter create() { // same as the default config in the python docs PyInterpreterConfig cfg; - memset(&cfg, 0, sizeof(cfg)); + std::memset(&cfg, 0, sizeof(cfg)); cfg.check_multi_interp_extensions = 1; cfg.gil = PyInterpreterConfig_OWN_GIL; return create(cfg);