diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9f7ae9617..ef554d4c2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,9 @@ jobs: python: - '3.8' - '3.13' + - '3.13t' - '3.14' + - '3.14t' - 'pypy-3.10' - 'pypy-3.11' - 'graalpy-24.2' diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index dc09161ec8..dec767aac9 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -212,6 +212,11 @@ naturally: assert(locals["message"].cast() == "1 + 2 = 3"); } +``PYBIND11_EMBEDDED_MODULE`` also accepts +:func:`py::mod_gil_not_used()`, +:func:`py::multiple_interpreters::per_interpreter_gil()`, and +:func:`py::multiple_interpreters::shared_gil()` tags just like ``PYBIND11_MODULE``. +See :ref:`misc_subinterp` and :ref:`misc_free_threading` for more information. Interpreter lifetime ==================== diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index a4b941f9aa..7d2a279585 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -155,6 +155,8 @@ following checklist. within pybind11 that will throw exceptions on certain GIL handling errors (reference counting operations). +.. _misc_free_threading: + Free-threading support ================================================================== @@ -178,6 +180,8 @@ your code is thread safe. Modules must still be built against the Python free-t enable free-threading, even if they specify this tag. Adding this tag does not break compatibility with non-free-threaded Python. +.. _misc_subinterp: + Sub-interpreter support ================================================================== diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index a456e80a6e..99a765ba16 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -38,24 +38,43 @@ }); } \endrst */ -#define PYBIND11_EMBEDDED_MODULE(name, variable) \ +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments") +#define PYBIND11_EMBEDDED_MODULE(name, variable, ...) \ static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name); \ + static ::pybind11::module_::slots_array PYBIND11_CONCAT(pybind11_module_slots_, name); \ + static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() { \ - auto m = ::pybind11::module_::create_extension_module( \ - PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name)); \ + static auto result = []() { \ + auto &slots = PYBIND11_CONCAT(pybind11_module_slots_, name); \ + slots[0] = {Py_mod_exec, \ + reinterpret_cast(&PYBIND11_CONCAT(pybind11_exec_, name))}; \ + slots[1] = {0, nullptr}; \ + return ::pybind11::module_::initialize_multiphase_module_def( \ + PYBIND11_TOSTRING(name), \ + nullptr, \ + &PYBIND11_CONCAT(pybind11_module_def_, name), \ + slots, \ + ##__VA_ARGS__); \ + }(); \ + return result.ptr(); \ + } \ + PYBIND11_EMBEDDED_MODULE_IMPL(name) \ + ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)( \ + PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name)); \ + int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject * pm) { \ try { \ + auto m = pybind11::reinterpret_borrow<::pybind11::module_>(pm); \ PYBIND11_CONCAT(pybind11_init_, name)(m); \ - return m.ptr(); \ + return 0; \ } \ PYBIND11_CATCH_INIT_EXCEPTIONS \ - return nullptr; \ + return -1; \ } \ - PYBIND11_EMBEDDED_MODULE_IMPL(name) \ - ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)( \ - PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name)); \ void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ \ & variable) // NOLINT(bugprone-macro-parentheses) +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 7108343f6f..3b09b765ee 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1484,7 +1484,7 @@ class module_ : public object { using slots_array = std::array; /** \rst - Initialized a module def for use with multi-phase module initialization. + Initialize a module def for use with multi-phase module initialization. ``def`` should point to a statically allocated module_def. ``slots`` must already contain a Py_mod_exec or Py_mod_create slot and will be filled with diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index cf9f86b521..6e4be7378a 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -55,7 +55,7 @@ class test_override_cache_helper_trampoline : public test_override_cache_helper int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); } }; -PYBIND11_EMBEDDED_MODULE(widget_module, m) { +PYBIND11_EMBEDDED_MODULE(widget_module, m, py::multiple_interpreters::per_interpreter_gil()) { py::class_(m, "Widget") .def(py::init()) .def_property_readonly("the_message", &Widget::the_message); @@ -336,6 +336,7 @@ TEST_CASE("Restart the interpreter") { REQUIRE(py_widget.attr("the_message").cast() == "Hello after restart"); } +#if defined(PYBIND11_SUBINTERPRETER_SUPPORT) TEST_CASE("Subinterpreter") { py::module_::import("external_module"); // in the main interpreter @@ -347,6 +348,10 @@ TEST_CASE("Subinterpreter") { REQUIRE(m.attr("add")(1, 2).cast() == 3); } + + auto main_int + = py::module_::import("external_module").attr("internals_at")().cast(); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); @@ -359,7 +364,6 @@ TEST_CASE("Subinterpreter") { // Subinterpreters get their own copy of builtins. REQUIRE_FALSE(has_state_dict_internals_obj()); -#if defined(PYBIND11_SUBINTERPRETER_SUPPORT) && PY_VERSION_HEX >= 0x030C0000 // internals hasn't been populated yet, but will be different for the subinterpreter REQUIRE_FALSE(has_pybind11_internals_static()); @@ -369,14 +373,12 @@ TEST_CASE("Subinterpreter") { py::detail::get_internals(); REQUIRE(has_pybind11_internals_static()); REQUIRE(get_details_as_uintptr() == ext_int); -#else - // This static is still defined - REQUIRE(has_pybind11_internals_static()); -#endif + REQUIRE(main_int != ext_int); // Modules tags should be gone. REQUIRE_FALSE(py::hasattr(py::module_::import("__main__"), "tag")); { + REQUIRE_NOTHROW(py::module_::import("widget_module")); auto m = py::module_::import("widget_module"); REQUIRE_FALSE(py::hasattr(m, "extension_module_tag")); @@ -397,7 +399,6 @@ TEST_CASE("Subinterpreter") { REQUIRE(has_state_dict_internals_obj()); } -#if defined(PYBIND11_SUBINTERPRETER_SUPPORT) TEST_CASE("Multiple Subinterpreters") { // Make sure the module is in the main interpreter and save its pointer auto *main_ext = py::module_::import("external_module").ptr(); @@ -512,10 +513,11 @@ TEST_CASE("Per-Subinterpreter GIL") { // we have switched to the new interpreter and released the main gil - // widget_module did not provide the mod_per_interpreter_gil tag, so it cannot be imported + // trampoline_module did not provide the per_interpreter_gil tag, so it cannot be + // imported bool caught = false; try { - py::module_::import("widget_module"); + py::module_::import("trampoline_module"); } catch (pybind11::error_already_set &pe) { T_REQUIRE(pe.matches(PyExc_ImportError)); std::string msg(pe.what()); @@ -525,6 +527,9 @@ TEST_CASE("Per-Subinterpreter GIL") { } T_REQUIRE(caught); + // widget_module did provide the per_interpreter_gil tag, so it this does not throw + py::module_::import("widget_module"); + T_REQUIRE(!py::hasattr(py::module_::import("external_module"), "multi_interp")); py::module_::import("external_module").attr("multi_interp") = std::to_string(num); @@ -547,8 +552,8 @@ TEST_CASE("Per-Subinterpreter GIL") { Py_EndInterpreter(sub); - PyThreadState_Swap( - main_tstate); // switch back so the scoped_acquire can release the GIL properly + // switch back so the scoped_acquire can release the GIL properly + PyThreadState_Swap(main_tstate); }; std::thread t1(thread_main, 1); @@ -622,12 +627,26 @@ TEST_CASE("Threads") { { py::gil_scoped_release gil_release{}; +#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX < 0x030E0000 + std::mutex mutex; +#endif auto threads = std::vector(); for (auto i = 0; i < num_threads; ++i) { threads.emplace_back([&]() { py::gil_scoped_acquire gil{}; +#ifdef Py_GIL_DISABLED +# if PY_VERSION_HEX < 0x030E0000 + std::lock_guard lock(mutex); + locals["count"] = locals["count"].cast() + 1; +# else + Py_BEGIN_CRITICAL_SECTION(locals.ptr()); locals["count"] = locals["count"].cast() + 1; + Py_END_CRITICAL_SECTION(); +# endif +#else + locals["count"] = locals["count"].cast() + 1; +#endif }); }