Make function record subinterpreter safe (#5771)

b-pass · pre-commit-ci[bot] · rwgk · web-flow · commit c5e8fec920cb · 2025-08-04T05:00:33.000-07:00
* Make function_record type subinterpreter safe * Get rid of static state in implicit conversion * style: pre-commit fixes * Fix lambda * Bump ABI because we added an internals member * Set __module__ on the type instance to get rid of DepricationWarning * Work around internal compiler error in CUDA by not using typedef hopefully * Make clang-tidy happy * Use the same __module__ as pybind11_static_property * style: pre-commit fixes * Oops, find-replace error * style: pre-commit fixes * Move the once initialization to happen more behind the scenes * Oops, need those casts... * Undo implicit conversion change, will do a separate PR * Use local_internals for function_record pointer to avoid ABI bump * style: pre-commit fixes * Get rid of this auto for readability * Change back to using unqualified tp_name, set __module__ attribute, explicitly add Py_TPFLAGS_HEAPTYPE → does not resolve DeprecationWarning :-( * Revert "Change back to using unqualified tp_name, set __module__ attribute, explicitly add Py_TPFLAGS_HEAPTYPE → does not resolve DeprecationWarning :-(" This reverts commit 9ccd6de. * Add Py_TPFLAGS_HEAPTYPE to be explicit (more readable). * Remove obsolete PYBIND11_WARNING_DISABLE_... * Make tp_plainname_impl, tp_qualname_impl more DRY * Change PYBIND11_INTERNAL_MODULE_NAME → PYBIND11_DUMMY_MODULE_NAME * Add a long comment to explain the tp_qualname_impl workaround. * Rename local_internals::function_record → function_record_py_type --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ralf W. Grosse-Kunstleve <rwgkio@gmail.com> Co-authored-by: Ralf W. Grosse-Kunstleve <rgrossekunst@nvidia.com>
diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h
@@ -98,7 +98,7 @@ inline PyTypeObject *make_static_property_type() {
         pybind11_fail("make_static_property_type(): failure in PyType_Ready()!");
     }
 
-    setattr((PyObject *) type, "__module__", str("pybind11_builtins"));
+    setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));
     PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);
 
     return type;
@@ -282,7 +282,7 @@ inline PyTypeObject *make_default_metaclass() {
         pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!");
     }
 
-    setattr((PyObject *) type, "__module__", str("pybind11_builtins"));
+    setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));
     PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);
 
     return type;
@@ -544,7 +544,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) {
         pybind11_fail("PyType_Ready failed in make_object_base_type(): " + error_string());
     }
 
-    setattr((PyObject *) type, "__module__", str("pybind11_builtins"));
+    setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));
     PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);
 
     assert(!PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC));
diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h
@@ -45,96 +45,79 @@ static PyMethodDef tp_methods_impl[]
         nullptr},
        {nullptr, nullptr, 0, nullptr}};
 
+// Python 3.12+ emits a DeprecationWarning for heap types whose tp_name does
+// not contain a dot ('.') and that lack a __module__ attribute. For pybind11's
+// internal function_record type, we do not have an actual module object to
+// attach, so we cannot use PyType_FromModuleAndSpec (introduced in Python 3.9)
+// to set __module__ automatically.
+//
+// As a workaround, we define a "qualified" type name that includes a dummy
+// module name (PYBIND11_DUMMY_MODULE_NAME). This is non‑idiomatic but avoids
+// the deprecation warning, and results in reprs like
+//
+//     <class 'pybind11_builtins.pybind11_detail_function_record_...'>
+//
+// even though no real pybind11_builtins module exists. If pybind11 gains an
+// actual module object in the future, this code should switch to
+// PyType_FromModuleAndSpec for Python 3.9+ and drop the dummy module
+// workaround.
+//
 // Note that this name is versioned.
-constexpr char tp_name_impl[]
-    = "pybind11_detail_function_record_" PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID
-      "_" PYBIND11_PLATFORM_ABI_ID;
+#define PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME                                              \
+    "pybind11_detail_function_record_" PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID                     \
+    "_" PYBIND11_PLATFORM_ABI_ID
+constexpr char tp_plainname_impl[] = PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME;
+constexpr char tp_qualname_impl[]
+    = PYBIND11_DUMMY_MODULE_NAME "." PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME;
 
 PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods)
 
-// Designated initializers are a C++20 feature:
-// https://en.cppreference.com/w/cpp/language/aggregate_initialization#Designated_initializers
-// MSVC rejects them unless /std:c++20 is used (error code C7555).
-PYBIND11_WARNING_PUSH
-PYBIND11_WARNING_DISABLE_CLANG("-Wmissing-field-initializers")
-#if defined(__GNUC__) && __GNUC__ >= 8
-PYBIND11_WARNING_DISABLE_GCC("-Wmissing-field-initializers")
-#endif
-static PyTypeObject function_record_PyTypeObject = {
-    PyVarObject_HEAD_INIT(nullptr, 0)
-    /* const char *tp_name */ function_record_PyTypeObject_methods::tp_name_impl,
-    /* Py_ssize_t tp_basicsize */ sizeof(function_record_PyObject),
-    /* Py_ssize_t tp_itemsize */ 0,
-    /* destructor tp_dealloc */ function_record_PyTypeObject_methods::tp_dealloc_impl,
-    /* Py_ssize_t tp_vectorcall_offset */ 0,
-    /* getattrfunc tp_getattr */ nullptr,
-    /* setattrfunc tp_setattr */ nullptr,
-    /* PyAsyncMethods *tp_as_async */ nullptr,
-    /* reprfunc tp_repr */ nullptr,
-    /* PyNumberMethods *tp_as_number */ nullptr,
-    /* PySequenceMethods *tp_as_sequence */ nullptr,
-    /* PyMappingMethods *tp_as_mapping */ nullptr,
-    /* hashfunc tp_hash */ nullptr,
-    /* ternaryfunc tp_call */ nullptr,
-    /* reprfunc tp_str */ nullptr,
-    /* getattrofunc tp_getattro */ nullptr,
-    /* setattrofunc tp_setattro */ nullptr,
-    /* PyBufferProcs *tp_as_buffer */ nullptr,
-    /* unsigned long tp_flags */ Py_TPFLAGS_DEFAULT,
-    /* const char *tp_doc */ nullptr,
-    /* traverseproc tp_traverse */ nullptr,
-    /* inquiry tp_clear */ nullptr,
-    /* richcmpfunc tp_richcompare */ nullptr,
-    /* Py_ssize_t tp_weaklistoffset */ 0,
-    /* getiterfunc tp_iter */ nullptr,
-    /* iternextfunc tp_iternext */ nullptr,
-    /* struct PyMethodDef *tp_methods */ function_record_PyTypeObject_methods::tp_methods_impl,
-    /* struct PyMemberDef *tp_members */ nullptr,
-    /* struct PyGetSetDef *tp_getset */ nullptr,
-    /* struct _typeobject *tp_base */ nullptr,
-    /* PyObject *tp_dict */ nullptr,
-    /* descrgetfunc tp_descr_get */ nullptr,
-    /* descrsetfunc tp_descr_set */ nullptr,
-    /* Py_ssize_t tp_dictoffset */ 0,
-    /* initproc tp_init */ function_record_PyTypeObject_methods::tp_init_impl,
-    /* allocfunc tp_alloc */ function_record_PyTypeObject_methods::tp_alloc_impl,
-    /* newfunc tp_new */ function_record_PyTypeObject_methods::tp_new_impl,
-    /* freefunc tp_free */ function_record_PyTypeObject_methods::tp_free_impl,
-    /* inquiry tp_is_gc */ nullptr,
-    /* PyObject *tp_bases */ nullptr,
-    /* PyObject *tp_mro */ nullptr,
-    /* PyObject *tp_cache */ nullptr,
-    /* PyObject *tp_subclasses */ nullptr,
-    /* PyObject *tp_weaklist */ nullptr,
-    /* destructor tp_del */ nullptr,
-    /* unsigned int tp_version_tag */ 0,
-    /* destructor tp_finalize */ nullptr,
-    /* vectorcallfunc tp_vectorcall */ nullptr,
-};
-PYBIND11_WARNING_POP
-
-static bool function_record_PyTypeObject_PyType_Ready_first_call = true;
-
-inline void function_record_PyTypeObject_PyType_Ready() {
-    if (function_record_PyTypeObject_PyType_Ready_first_call) {
-        if (PyType_Ready(&function_record_PyTypeObject) < 0) {
+static PyType_Slot function_record_PyType_Slots[] = {
+    {Py_tp_dealloc,
+     reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_dealloc_impl)},
+    {Py_tp_methods,
+     reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_methods_impl)},
+    {Py_tp_init, reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_init_impl)},
+    {Py_tp_alloc, reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_alloc_impl)},
+    {Py_tp_new, reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_new_impl)},
+    {Py_tp_free, reinterpret_cast<void *>(function_record_PyTypeObject_methods::tp_free_impl)},
+    {0, nullptr}};
+
+static PyType_Spec function_record_PyType_Spec
+    = {function_record_PyTypeObject_methods::tp_qualname_impl,
+       sizeof(function_record_PyObject),
+       0,
+       Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE,
+       function_record_PyType_Slots};
+
+inline PyTypeObject *get_function_record_PyTypeObject() {
+    PyTypeObject *&py_type_obj = detail::get_local_internals().function_record_py_type;
+    if (!py_type_obj) {
+        PyObject *py_obj = PyType_FromSpec(&function_record_PyType_Spec);
+        if (py_obj == nullptr) {
             throw error_already_set();
         }
-        function_record_PyTypeObject_PyType_Ready_first_call = false;
+        py_type_obj = reinterpret_cast<PyTypeObject *>(py_obj);
     }
+    return py_type_obj;
 }
 
 inline bool is_function_record_PyObject(PyObject *obj) {
     if (PyType_Check(obj) != 0) {
         return false;
     }
     PyTypeObject *obj_type = Py_TYPE(obj);
+
+    PyTypeObject *frtype = get_function_record_PyTypeObject();
+
     // Fast path (pointer comparison).
-    if (obj_type == &function_record_PyTypeObject) {
+    if (obj_type == frtype) {
         return true;
     }
     // This works across extension modules. Note that tp_name is versioned.
-    if (strcmp(obj_type->tp_name, function_record_PyTypeObject.tp_name) == 0) {
+    if (strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_qualname_impl) == 0
+        || strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_plainname_impl)
+               == 0) {
         return true;
     }
     return false;
@@ -148,7 +131,7 @@ inline function_record *function_record_ptr_from_PyObject(PyObject *obj) {
 }
 
 inline object function_record_PyObject_New() {
-    auto *py_func_rec = PyObject_New(function_record_PyObject, &function_record_PyTypeObject);
+    auto *py_func_rec = PyObject_New(function_record_PyObject, get_function_record_PyTypeObject());
     if (py_func_rec == nullptr) {
         throw error_already_set();
     }
diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h
@@ -132,7 +132,8 @@ class thread_specific_storage {
 
 PYBIND11_NAMESPACE_BEGIN(detail)
 
-constexpr const char *internals_function_record_capsule_name = "pybind11_function_record_capsule";
+// This does NOT actually exist as a module.
+#define PYBIND11_DUMMY_MODULE_NAME "pybind11_builtins"
 
 // Forward declarations
 inline PyTypeObject *make_static_property_type();
@@ -298,6 +299,7 @@ struct internals {
 struct local_internals {
     type_map<type_info *> registered_types_cpp;
     std::forward_list<ExceptionTranslator> registered_exception_translators;
+    PyTypeObject *function_record_py_type = nullptr;
 };
 
 enum class holder_enum_t : uint8_t {
diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h
@@ -631,7 +631,6 @@ class cpp_function : public function {
                 = reinterpret_cast<PyCFunction>(reinterpret_cast<void (*)()>(dispatcher));
             rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS;
 
-            detail::function_record_PyTypeObject_PyType_Ready(); // Call-once initialization.
             object py_func_rec = detail::function_record_PyObject_New();
             ((detail::function_record_PyObject *) py_func_rec.ptr())->cpp_func_rec
                 = unique_rec.release();
diff --git a/tests/test_callbacks.cpp b/tests/test_callbacks.cpp
@@ -286,13 +286,7 @@ TEST_SUBMODULE(callbacks, m) {
         return &def;
     }();
 
-    // rec_capsule with name that has the same value (but not pointer) as our internal one
-    // This capsule should be detected by our code as foreign and not inspected as the pointers
-    // shouldn't match
-    constexpr const char *rec_capsule_name
-        = pybind11::detail::internals_function_record_capsule_name;
     py::capsule rec_capsule(std::malloc(1), [](void *data) { std::free(data); });
-    rec_capsule.set_name(rec_capsule_name);
     m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr()));
 
     // rec_capsule with nullptr name

Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,7 @@ inline PyTypeObject *make_static_property_type() {`
`98`	`98`	`pybind11_fail("make_static_property_type(): failure in PyType_Ready()!");`
`99`	`99`	`}`
`100`	`100`
`101`		`- setattr((PyObject *) type, "__module__", str("pybind11_builtins"));`
	`101`	`+ setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));`
`102`	`102`	`PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);`
`103`	`103`
`104`	`104`	`return type;`
`@@ -282,7 +282,7 @@ inline PyTypeObject *make_default_metaclass() {`
`282`	`282`	`pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!");`
`283`	`283`	`}`
`284`	`284`
`285`		`- setattr((PyObject *) type, "__module__", str("pybind11_builtins"));`
	`285`	`+ setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));`
`286`	`286`	`PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);`
`287`	`287`
`288`	`288`	`return type;`
`@@ -544,7 +544,7 @@ inline PyObject make_object_base_type(PyTypeObject metaclass) {`
`544`	`544`	`pybind11_fail("PyType_Ready failed in make_object_base_type(): " + error_string());`
`545`	`545`	`}`
`546`	`546`
`547`		`- setattr((PyObject *) type, "__module__", str("pybind11_builtins"));`
	`547`	`+ setattr((PyObject *) type, "__module__", str(PYBIND11_DUMMY_MODULE_NAME));`
`548`	`548`	`PYBIND11_SET_OLDPY_QUALNAME(type, name_obj);`
`549`	`549`
`550`	`550`	`assert(!PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC));`