pythongh-140550: Initial implementation of PEP 793 – PyModExport (pythonGH-140556)

Co-authored-by: Victor Stinner <[email protected]>
Co-authored-by: Kumar Aditya <[email protected]>

Author: 3 people

Doc/data/stable_abi.dat

@@ -9,6 +9,7 @@
                     inside the Python core, they are private to the core.
                     If in an extension module, it may be declared with
                     external linkage depending on the platform.
+  PyMODEXPORT_FUNC: Like PyMODINIT_FUNC, but for a slots array
 
   As a number of platforms support/require "__declspec(dllimport/dllexport)",
   we support a HAVE_DECLSPEC_DLL macro to save duplication.
@@ -62,9 +63,9 @@
         /* module init functions inside the core need no external linkage */
         /* except for Cygwin to handle embedding */
 #                       if defined(__CYGWIN__)
-#                               define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject*
+#                               define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL
 #                       else /* __CYGWIN__ */
-#                               define PyMODINIT_FUNC PyObject*
+#                               define _PyINIT_FUNC_DECLSPEC
 #                       endif /* __CYGWIN__ */
 #               else /* Py_BUILD_CORE */
         /* Building an extension module, or an embedded situation */
@@ -78,9 +79,9 @@
 #                       define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE
         /* module init functions outside the core must be exported */
 #                       if defined(__cplusplus)
-#                               define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject*
+#                               define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL
 #                       else /* __cplusplus */
-#                               define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject*
+#                               define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL
 #                       endif /* __cplusplus */
 #               endif /* Py_BUILD_CORE */
 #       endif /* HAVE_DECLSPEC_DLL */
@@ -93,13 +94,15 @@
 #ifndef PyAPI_DATA
 #       define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE
 #endif
-#ifndef PyMODINIT_FUNC
+#ifndef _PyINIT_FUNC_DECLSPEC
 #       if defined(__cplusplus)
-#               define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject*
+#               define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL
 #       else /* __cplusplus */
-#               define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject*
+#               define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL
 #       endif /* __cplusplus */
 #endif
 
+#define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject*
+#define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot*
 
 #endif /* Py_EXPORTS_H */

Include/exports.h

@@ -28,6 +28,11 @@ typedef enum ext_module_origin {
     _Py_ext_module_origin_DYNAMIC = 3,
 } _Py_ext_module_origin;
 
+struct hook_prefixes {
+    const char *const init_prefix;
+    const char *const export_prefix;
+};
+
 /* Input for loading an extension module. */
 struct _Py_ext_module_loader_info {
     PyObject *filename;
@@ -40,7 +45,7 @@ struct _Py_ext_module_loader_info {
      * depending on if it's builtin or not. */
     PyObject *path;
     _Py_ext_module_origin origin;
-    const char *hook_prefix;
+    const struct hook_prefixes *hook_prefixes;
     const char *newcontext;
 };
 extern void _Py_ext_module_loader_info_clear(
@@ -62,7 +67,9 @@ extern int _Py_ext_module_loader_info_init_from_spec(
     PyObject *spec);
 #endif
 
-/* The result from running an extension module's init function. */
+/* The result from running an extension module's init function.
+ * Not used for modules defined via PyModExport (slots array).
+ */
 struct _Py_ext_module_loader_result {
     PyModuleDef *def;
     PyObject *module;
@@ -89,10 +96,11 @@ extern void _Py_ext_module_loader_result_apply_error(
 
 /* The module init function. */
 typedef PyObject *(*PyModInitFunction)(void);
+typedef PyModuleDef_Slot *(*PyModExportFunction)(void);
 #ifdef HAVE_DYNAMIC_LOADING
-extern PyModInitFunction _PyImport_GetModInitFunc(
+extern int _PyImport_GetModuleExportHooks(
     struct _Py_ext_module_loader_info *info,
-    FILE *fp);
+    FILE *fp, PyModInitFunction *modinit, PyModExportFunction *modexport);
 #endif
 extern int _PyImport_RunModInitFunc(
     PyModInitFunction p0,

Include/internal/pycore_importdl.h

@@ -1,5 +1,8 @@
 #ifndef Py_INTERNAL_MODULEOBJECT_H
 #define Py_INTERNAL_MODULEOBJECT_H
+
+#include <stdbool.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -16,32 +19,49 @@ extern int _PyModule_IsPossiblyShadowing(PyObject *);
 
 extern int _PyModule_IsExtension(PyObject *obj);
 
+typedef int (*_Py_modexecfunc)(PyObject *);
+
 typedef struct {
     PyObject_HEAD
     PyObject *md_dict;
-    PyModuleDef *md_def;
     void *md_state;
     PyObject *md_weaklist;
     // for logging purposes after md_dict is cleared
     PyObject *md_name;
+    bool md_token_is_def;  /* if true, `md_token` is the PyModuleDef */
 #ifdef Py_GIL_DISABLED
     void *md_gil;
 #endif
+    Py_ssize_t md_state_size;
+    traverseproc md_state_traverse;
+    inquiry md_state_clear;
+    freefunc md_state_free;
+    void *md_token;
+    _Py_modexecfunc md_exec;  /* only set if md_token_is_def is true */
 } PyModuleObject;
 
-static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) {
-    assert(PyModule_Check(mod));
-    return ((PyModuleObject *)mod)->md_def;
+#define _PyModule_CAST(op) \
+    (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op)))
+
+static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) {
+    PyModuleObject *mod = _PyModule_CAST(arg);
+    if (mod->md_token_is_def) {
+        return (PyModuleDef *)mod->md_token;
+    }
+    return NULL;
+}
+
+static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) {
+    PyModuleObject *mod = _PyModule_CAST(arg);
+    return mod->md_token;
 }
 
 static inline void* _PyModule_GetState(PyObject* mod) {
-    assert(PyModule_Check(mod));
-    return ((PyModuleObject *)mod)->md_state;
+    return _PyModule_CAST(mod)->md_state;
 }
 
 static inline PyObject* _PyModule_GetDict(PyObject *mod) {
-    assert(PyModule_Check(mod));
-    PyObject *dict = ((PyModuleObject *)mod) -> md_dict;
+    PyObject *dict = _PyModule_CAST(mod)->md_dict;
     // _PyModule_GetDict(mod) must not be used after calling module_clear(mod)
     assert(dict != NULL);
     return dict;  // borrowed reference

Include/internal/pycore_moduleobject.h

@@ -83,11 +83,19 @@ struct PyModuleDef_Slot {
 #endif
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15)
 #  define Py_mod_abi 5
+#  define Py_mod_name 6
+#  define Py_mod_doc 7
+#  define Py_mod_state_size 8
+#  define Py_mod_methods 9
+#  define Py_mod_state_traverse 10
+#  define Py_mod_state_clear 11
+#  define Py_mod_state_free 12
+#  define Py_mod_token 13
 #endif
 
 
 #ifndef Py_LIMITED_API
-#define _Py_mod_LAST_SLOT 5
+#define _Py_mod_LAST_SLOT 13
 #endif
 
 #endif /* New in 3.5 */
@@ -109,6 +117,13 @@ struct PyModuleDef_Slot {
 PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil);
 #endif
 
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15)
+PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *,
+                                                 PyObject *spec);
+PyAPI_FUNC(int) PyModule_Exec(PyObject *mod);
+PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *mod, Py_ssize_t *result);
+PyAPI_FUNC(int) PyModule_GetToken(PyObject *, void **result);
+#endif
 
 #ifndef _Py_OPAQUE_PYOBJECT
 struct PyModuleDef {

Include/moduleobject.h

@@ -839,6 +839,11 @@ PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *);
 PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type);
 #endif
 
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15)
+PyAPI_FUNC(PyObject *) PyType_GetModuleByToken(PyTypeObject *type,
+                                               const void *token);
+#endif
+
 #ifdef __cplusplus
 }
 #endif