feat: change PYBIND11_EMBEDDED_MODULE to multiphase init

.github/workflows/ci.yml

@@ -34,7 +34,9 @@ jobs:
         python:
         - '3.8'
         - '3.13'
+        - '3.13t'
         - '3.14'
+        - '3.14t'
         - 'pypy-3.10'
         - 'pypy-3.11'
         - 'graalpy-24.2'

docs/advanced/embedding.rst

@@ -212,6 +212,11 @@ naturally:
         assert(locals["message"].cast<std::string>() == "1 + 2 = 3");
     }
 
+``PYBIND11_EMBEDDED_MODULE`` also accepts
+:func:`py::mod_gil_not_used()`,
+:func:`py::multiple_interpreters::per_interpreter_gil()`, and
+:func:`py::multiple_interpreters::shared_gil()` tags just like ``PYBIND11_MODULE``.
+See :ref:`misc_subinterp` and :ref:`misc_free_threading` for more information.
 
 Interpreter lifetime
 ====================

docs/advanced/misc.rst

@@ -155,6 +155,8 @@ following checklist.
   within pybind11 that will throw exceptions on certain GIL handling errors
   (reference counting operations).
 
+.. _misc_free_threading:
+
 Free-threading support
 ==================================================================
 
@@ -178,6 +180,8 @@ your code is thread safe.  Modules must still be built against the Python free-t
 enable free-threading, even if they specify this tag.  Adding this tag does not break
 compatibility with non-free-threaded Python.
 
+.. _misc_subinterp:
+
 Sub-interpreter support
 ==================================================================
 

include/pybind11/embed.h

@@ -38,24 +38,43 @@
             });
         }
  \endrst */
-#define PYBIND11_EMBEDDED_MODULE(name, variable)                                                  \
+PYBIND11_WARNING_PUSH
+PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments")
+#define PYBIND11_EMBEDDED_MODULE(name, variable, ...)                                             \
     static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name);           \
+    static ::pybind11::module_::slots_array PYBIND11_CONCAT(pybind11_module_slots_, name);        \
+    static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *);                                 \
     static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &);                     \
     static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() {                            \
-        auto m = ::pybind11::module_::create_extension_module(                                    \
-            PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name));      \
+        static auto result = []() {                                                               \
+            auto &slots = PYBIND11_CONCAT(pybind11_module_slots_, name);                          \
+            slots[0] = {Py_mod_exec,                                                              \
+                        reinterpret_cast<void *>(&PYBIND11_CONCAT(pybind11_exec_, name))};        \
+            slots[1] = {0, nullptr};                                                              \
+            return ::pybind11::module_::initialize_multiphase_module_def(                         \
+                PYBIND11_TOSTRING(name),                                                          \
+                nullptr,                                                                          \
+                &PYBIND11_CONCAT(pybind11_module_def_, name),                                     \
+                slots,                                                                            \
+                ##__VA_ARGS__);                                                                   \
+        }();                                                                                      \
+        return result.ptr();                                                                      \
+    }                                                                                             \
+    PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                           \
+    ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)(                  \
+        PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name));                     \
+    int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject * pm) {                                    \
         try {                                                                                     \
+            auto m = pybind11::reinterpret_borrow<::pybind11::module_>(pm);                       \
             PYBIND11_CONCAT(pybind11_init_, name)(m);                                             \
-            return m.ptr();                                                                       \
+            return 0;                                                                             \
         }                                                                                         \
         PYBIND11_CATCH_INIT_EXCEPTIONS                                                            \
-        return nullptr;                                                                           \
+        return -1;                                                                                \
     }                                                                                             \
-    PYBIND11_EMBEDDED_MODULE_IMPL(name)                                                           \
-    ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)(                  \
-        PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name));                     \
     void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_                                \
                                                & variable) // NOLINT(bugprone-macro-parentheses)
+PYBIND11_WARNING_POP
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)

tests/test_embed/test_interpreter.cpp

@@ -55,7 +55,7 @@ class test_override_cache_helper_trampoline : public test_override_cache_helper
     int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); }
 };
 
-PYBIND11_EMBEDDED_MODULE(widget_module, m) {
+PYBIND11_EMBEDDED_MODULE(widget_module, m, py::multiple_interpreters::per_interpreter_gil()) {
     py::class_<Widget, PyWidget>(m, "Widget")
         .def(py::init<std::string>())
         .def_property_readonly("the_message", &Widget::the_message);
@@ -512,10 +512,11 @@ TEST_CASE("Per-Subinterpreter GIL") {
 
         // we have switched to the new interpreter and released the main gil
 
-        // widget_module did not provide the mod_per_interpreter_gil tag, so it cannot be imported
+        // trampoline_module did not provide the per_interpreter_gil tag, so it cannot be
+        // imported
         bool caught = false;
         try {
-            py::module_::import("widget_module");
+            py::module_::import("trampoline_module");
         } catch (pybind11::error_already_set &pe) {
             T_REQUIRE(pe.matches(PyExc_ImportError));
             std::string msg(pe.what());
@@ -525,6 +526,15 @@ TEST_CASE("Per-Subinterpreter GIL") {
         }
         T_REQUIRE(caught);
 
+        // widget_module did provide the per_interpreter_gil tag, so it this does not throw
+        try {
+            py::module_::import("widget_module");
+            caught = false;
+        } catch (pybind11::error_already_set &) {
+            caught = true;
+        }
+        T_REQUIRE(!caught);
+
         T_REQUIRE(!py::hasattr(py::module_::import("external_module"), "multi_interp"));
         py::module_::import("external_module").attr("multi_interp") = std::to_string(num);
 
@@ -627,7 +637,13 @@ TEST_CASE("Threads") {
         for (auto i = 0; i < num_threads; ++i) {
             threads.emplace_back([&]() {
                 py::gil_scoped_acquire gil{};
+#ifdef Py_GIL_DISABLED
+                Py_BEGIN_CRITICAL_SECTION(locals.ptr());
+                locals["count"] = locals["count"].cast<int>() + 1;
+                Py_END_CRITICAL_SECTION();
+#else
                 locals["count"] = locals["count"].cast<int>() + 1;
+#endif
             });
         }