gh-76785: Improved Subinterpreters Compatibility with 3.12 (gh-115424)

For the most part, these changes make is substantially easier to backport subinterpreter-related code to 3.12, especially the related modules (e.g. _xxsubinterpreters). The main motivation is to support releasing a PyPI package with the 3.13 capabilities compiled for 3.12.

A lot of the changes here involve either hiding details behind macros/functions or splitting up some files.

Author: ericsnowcurrently

Include/internal/pycore_code.h

@@ -8,6 +8,15 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+
+// We hide some of the newer PyCodeObject fields behind macros.
+// This helps with backporting certain changes to 3.12.
+#define _PyCode_HAS_EXECUTORS(CODE) \
+    (CODE->co_executors != NULL)
+#define _PyCode_HAS_INSTRUMENTATION(CODE) \
+    (CODE->_co_instrumentation_version > 0)
+
+
 #define CODE_MAX_WATCHERS 8
 
 /* PEP 659

Include/internal/pycore_crossinterp.h

@@ -87,6 +87,11 @@ struct _xid {
 PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void);
 PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data);
 
+#define _PyCrossInterpreterData_DATA(DATA) ((DATA)->data)
+#define _PyCrossInterpreterData_OBJ(DATA) ((DATA)->obj)
+#define _PyCrossInterpreterData_INTERPID(DATA) ((DATA)->interpid)
+// Users should not need getters for "new_object" or "free".
+
 
 /* defining cross-interpreter data */
 
@@ -101,6 +106,25 @@ PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize(
 PyAPI_FUNC(void) _PyCrossInterpreterData_Clear(
         PyInterpreterState *, _PyCrossInterpreterData *);
 
+// Normally the Init* functions are sufficient.  The only time
+// additional initialization might be needed is to set the "free" func,
+// though that should be infrequent.
+#define _PyCrossInterpreterData_SET_FREE(DATA, FUNC) \
+    do { \
+        (DATA)->free = (FUNC); \
+    } while (0)
+// Additionally, some shareable types are essentially light wrappers
+// around other shareable types.  The crossinterpdatafunc of the wrapper
+// can often be implemented by calling the wrapped object's
+// crossinterpdatafunc and then changing the "new_object" function.
+// We have _PyCrossInterpreterData_SET_NEW_OBJECT() here for that,
+// but might be better to have a function like
+// _PyCrossInterpreterData_AdaptToWrapper() instead.
+#define _PyCrossInterpreterData_SET_NEW_OBJECT(DATA, FUNC) \
+    do { \
+        (DATA)->new_object = (FUNC); \
+    } while (0)
+
 
 /* using cross-interpreter data */
 
@@ -170,6 +194,8 @@ extern void _PyXI_Fini(PyInterpreterState *interp);
 extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp);
 extern void _PyXI_FiniTypes(PyInterpreterState *interp);
 
+#define _PyInterpreterState_GetXIState(interp) (&(interp)->xi)
+
 
 /***************************/
 /* short-term data sharing */

Include/internal/pycore_tstate.h

@@ -13,6 +13,13 @@ extern "C" {
 #include "pycore_brc.h"           // struct _brc_thread_state
 
 
+static inline void
+_PyThreadState_SetWhence(PyThreadState *tstate, int whence)
+{
+    tstate->_whence = whence;
+}
+
+
 // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
 // PyThreadState fields are exposed as part of the C API, although most fields
 // are intended to be private. The _PyThreadStateImpl fields not exposed.

Makefile.pre.in

@@ -1671,6 +1671,14 @@ Modules/pwdmodule.o: $(srcdir)/Modules/pwdmodule.c $(srcdir)/Modules/posixmodule
 
 Modules/signalmodule.o: $(srcdir)/Modules/signalmodule.c $(srcdir)/Modules/posixmodule.h
 
+Modules/_xxsubinterpretersmodule.o: $(srcdir)/Modules/_xxsubinterpretersmodule.c $(srcdir)/Modules/_interpreters_common.h
+
+Modules/_xxinterpqueuesmodule.o: $(srcdir)/Modules/_xxinterpqueuesmodule.c $(srcdir)/Modules/_interpreters_common.h
+
+Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $(srcdir)/Modules/_interpreters_common.h
+
+Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h
+
 Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile
 	$(CC) -c $(PY_CORE_CFLAGS) \
 		-DSOABI='"$(SOABI)"' \

Modules/_interpreters_common.h

@@ -0,0 +1,13 @@
+
+#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \
+    PyInit_ ## NAME
+#define RESOLVE_MODINIT_FUNC_NAME(NAME) \
+    _RESOLVE_MODINIT_FUNC_NAME(NAME)
+
+
+static int
+ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata)
+{
+    //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE);
+    return _PyCrossInterpreterData_RegisterClass(cls, getdata);
+}

Modules/_xxinterpchannelsmodule.c

@@ -17,6 +17,8 @@
 #include <sched.h>          // sched_yield()
 #endif
 
+#include "_interpreters_common.h"
+
 
 /*
 This module has the following process-global state:
@@ -80,7 +82,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*()
 API..  The module does not create any objects that are shared globally.
 */
 
-#define MODULE_NAME "_xxinterpchannels"
+#define MODULE_NAME _xxinterpchannels
+#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
+#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
 
 
 #define GLOBAL_MALLOC(TYPE) \
@@ -101,7 +105,7 @@ static int
 register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared,
                    struct xid_class_registry *classes)
 {
-    int res = _PyCrossInterpreterData_RegisterClass(cls, shared);
+    int res = ensure_xid_class(cls, shared);
     if (res == 0) {
         assert(classes->count < MAX_XID_CLASSES);
         // The class has refs elsewhere, so we need to incref here.
@@ -167,7 +171,7 @@ _get_current_interp(void)
 static PyObject *
 _get_current_module(void)
 {
-    PyObject *name = PyUnicode_FromString(MODULE_NAME);
+    PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
     if (name == NULL) {
         return NULL;
     }
@@ -217,7 +221,7 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base)
 }
 
 #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \
-    add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE)
+    add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE)
 
 static PyTypeObject *
 add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared,
@@ -299,7 +303,7 @@ _get_current_module_state(void)
     if (mod == NULL) {
         // XXX import it?
         PyErr_SetString(PyExc_RuntimeError,
-                        MODULE_NAME " module not imported yet");
+                        MODULE_NAME_STR " module not imported yet");
         return NULL;
     }
     module_state *state = get_module_state(mod);
@@ -784,7 +788,7 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid)
     while (next != NULL) {
         _channelitem *item = next;
         next = item->next;
-        if (item->data->interpid == interpid) {
+        if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
             if (prev == NULL) {
                 queue->first = item->next;
             }
@@ -2126,7 +2130,7 @@ static PyStructSequence_Field channel_info_fields[] = {
 };
 
 static PyStructSequence_Desc channel_info_desc = {
-    .name = MODULE_NAME ".ChannelInfo",
+    .name = MODULE_NAME_STR ".ChannelInfo",
     .doc = channel_info_doc,
     .fields = channel_info_fields,
     .n_in_sequence = 8,
@@ -2474,10 +2478,11 @@ struct _channelid_xid {
 static PyObject *
 _channelid_from_xid(_PyCrossInterpreterData *data)
 {
-    struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
+    struct _channelid_xid *xid = \
+                (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
 
     // It might not be imported yet, so we can't use _get_current_module().
-    PyObject *mod = PyImport_ImportModule(MODULE_NAME);
+    PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR);
     if (mod == NULL) {
         return NULL;
     }
@@ -2530,7 +2535,8 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj,
     {
         return -1;
     }
-    struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
+    struct _channelid_xid *xid = \
+                (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
     xid->cid = ((channelid *)obj)->cid;
     xid->end = ((channelid *)obj)->end;
     xid->resolve = ((channelid *)obj)->resolve;
@@ -2601,7 +2607,7 @@ static PyType_Slot channelid_typeslots[] = {
 };
 
 static PyType_Spec channelid_typespec = {
-    .name = MODULE_NAME ".ChannelID",
+    .name = MODULE_NAME_STR ".ChannelID",
     .basicsize = sizeof(channelid),
     .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
               Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
@@ -2680,7 +2686,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj,
     if (res < 0) {
         return -1;
     }
-    data->new_object = _channelend_from_xid;
+    _PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid);
     return 0;
 }
 
@@ -3379,7 +3385,7 @@ module_free(void *mod)
 
 static struct PyModuleDef moduledef = {
     .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = MODULE_NAME,
+    .m_name = MODULE_NAME_STR,
     .m_doc = module_doc,
     .m_size = sizeof(module_state),
     .m_methods = module_functions,
@@ -3390,7 +3396,7 @@ static struct PyModuleDef moduledef = {
 };
 
 PyMODINIT_FUNC
-PyInit__xxinterpchannels(void)
+MODINIT_FUNC_NAME(void)
 {
     return PyModuleDef_Init(&moduledef);
 }

Modules/_xxinterpqueuesmodule.c

@@ -8,8 +8,12 @@
 #include "Python.h"
 #include "pycore_crossinterp.h"   // struct _xid
 
+#include "_interpreters_common.h"
 
-#define MODULE_NAME "_xxinterpqueues"
+
+#define MODULE_NAME _xxinterpqueues
+#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
+#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
 
 
 #define GLOBAL_MALLOC(TYPE) \
@@ -64,7 +68,7 @@ _get_current_interp(void)
 static PyObject *
 _get_current_module(void)
 {
-    PyObject *name = PyUnicode_FromString(MODULE_NAME);
+    PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
     if (name == NULL) {
         return NULL;
     }
@@ -602,7 +606,7 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid)
     while (next != NULL) {
         _queueitem *item = next;
         next = item->next;
-        if (item->data->interpid == interpid) {
+        if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
             if (prev == NULL) {
                 queue->items.first = item->next;
             }
@@ -1062,7 +1066,7 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type)
     }
     state->queue_type = (PyTypeObject *)Py_NewRef(queue_type);
 
-    if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) {
+    if (ensure_xid_class(queue_type, _queueobj_shared) < 0) {
         return -1;
     }
 
@@ -1130,7 +1134,7 @@ _queueid_xid_free(void *data)
 static PyObject *
 _queueobj_from_xid(_PyCrossInterpreterData *data)
 {
-    int64_t qid = *(int64_t *)data->data;
+    int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data);
     PyObject *qidobj = PyLong_FromLongLong(qid);
     if (qidobj == NULL) {
         return NULL;
@@ -1140,7 +1144,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data)
     if (mod == NULL) {
         // XXX import it?
         PyErr_SetString(PyExc_RuntimeError,
-                        MODULE_NAME " module not imported yet");
+                        MODULE_NAME_STR " module not imported yet");
         return NULL;
     }
 
@@ -1181,7 +1185,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj,
     _PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL,
                                  _queueobj_from_xid);
     Py_DECREF(qidobj);
-    data->free = _queueid_xid_free;
+    _PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free);
     return 0;
 }
 
@@ -1670,7 +1674,7 @@ module_free(void *mod)
 
 static struct PyModuleDef moduledef = {
     .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = MODULE_NAME,
+    .m_name = MODULE_NAME_STR,
     .m_doc = module_doc,
     .m_size = sizeof(module_state),
     .m_methods = module_functions,
@@ -1681,7 +1685,7 @@ static struct PyModuleDef moduledef = {
 };
 
 PyMODINIT_FUNC
-PyInit__xxinterpqueues(void)
+MODINIT_FUNC_NAME(void)
 {
     return PyModuleDef_Init(&moduledef);
 }