From 403c7e21182ada18cfe5ce544eb3e2641981257f Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 20 Sep 2025 23:55:44 -0700 Subject: [PATCH 01/14] gh-116146: Add new C-API to create builtin from spec and initfunc --- Include/cpython/import.h | 9 +++++ Python/import.c | 76 ++++++++++++++++++++++++++++++---------- 2 files changed, 66 insertions(+), 19 deletions(-) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cce2a..5a69f9cb7e179a 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,15 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) +PyImport_CreateBuiltinFromSpecAndInitfunc( + PyObject *spec, + PyObject* (*initfunc)(void) + ); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Python/import.c b/Python/import.c index 6cf4a061ca610f..8437cd8c7d6e41 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2362,8 +2362,27 @@ is_builtin(PyObject *name) return 0; } +static PyModInitFunction +lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info) +{ + struct _inittab *found = NULL; + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(info->name, p->name)) { + found = p; + } + } + if (found == NULL) { + // not found + return NULL; + } + return (PyModInitFunction)found->initfunc; +} + static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +create_builtin_ex( + PyThreadState *tstate, PyObject *name, + PyObject *spec, + PyModInitFunction initfunc) { struct _Py_ext_module_loader_info info; if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) { @@ -2394,25 +2413,15 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) _extensions_cache_delete(info.path, info.name); } - struct _inittab *found = NULL; - for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { - found = p; - break; - } - } - if (found == NULL) { - // not found - mod = Py_NewRef(Py_None); - goto finally; - } - - PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + PyModInitFunction p0 = initfunc; if (p0 == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - assert(is_core_module(tstate->interp, info.name, info.path)); - mod = import_add_module(tstate, info.name); - goto finally; + p0 = lookup_inittab_initfunc(&info); + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; + } } #ifdef Py_GIL_DISABLED @@ -2438,6 +2447,35 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) return mod; } +static PyObject* +create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +{ + return create_builtin_ex(tstate, name, spec, NULL); +} + +PyObject* +PyImport_CreateBuiltinFromSpecAndInitfunc( + PyObject *spec, PyObject* (*initfunc)(void)) +{ + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (name == NULL) { + return NULL; + } + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "name must be string, not %.200s", + Py_TYPE(name)->tp_name); + Py_DECREF(name); + return NULL; + } + + PyObject *mod = create_builtin_ex(tstate, name, spec, initfunc); + Py_DECREF(name); + return mod; +} /*****************************/ /* the builtin modules table */ From 074a51080070e63bed8c0ec4c78c5d22c56bdd7b Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 8 Nov 2025 09:47:54 -0800 Subject: [PATCH 02/14] Apply review feedback and CAPI WG decision - use `PyImport_CreateModuleFromInitfunc` for the new API - no need to introduce `create_builtin_ex` - update `create_builtin` and all callers - update exception message --- Include/cpython/import.h | 2 +- Python/import.c | 28 +++++++++------------------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 5a69f9cb7e179a..d7a5aa76d7c921 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -14,7 +14,7 @@ PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); // extension modules directly from a spec and init function, // without needing to go through inittab PyAPI_FUNC(PyObject *) -PyImport_CreateBuiltinFromSpecAndInitfunc( +PyImport_CreateModuleFromInitfunc( PyObject *spec, PyObject* (*initfunc)(void) ); diff --git a/Python/import.c b/Python/import.c index 8437cd8c7d6e41..8f3c17027e81d0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2368,18 +2368,15 @@ lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info) struct _inittab *found = NULL; for (struct _inittab *p = INITTAB; p->name != NULL; p++) { if (_PyUnicode_EqualToASCIIString(info->name, p->name)) { - found = p; + return (PyModInitFunction)p->initfunc; } } - if (found == NULL) { - // not found - return NULL; - } - return (PyModInitFunction)found->initfunc; + // not found + return NULL; } static PyObject* -create_builtin_ex( +create_builtin( PyThreadState *tstate, PyObject *name, PyObject *spec, PyModInitFunction initfunc) @@ -2447,14 +2444,8 @@ create_builtin_ex( return mod; } -static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) -{ - return create_builtin_ex(tstate, name, spec, NULL); -} - PyObject* -PyImport_CreateBuiltinFromSpecAndInitfunc( +PyImport_CreateModuleFromInitfunc( PyObject *spec, PyObject* (*initfunc)(void)) { PyThreadState *tstate = _PyThreadState_GET(); @@ -2466,13 +2457,12 @@ PyImport_CreateBuiltinFromSpecAndInitfunc( if (!PyUnicode_Check(name)) { PyErr_Format(PyExc_TypeError, - "name must be string, not %.200s", - Py_TYPE(name)->tp_name); + "spec name must be string, not %T", name); Py_DECREF(name); return NULL; } - PyObject *mod = create_builtin_ex(tstate, name, spec, initfunc); + PyObject *mod = create_builtin(tstate, name, spec, initfunc); Py_DECREF(name); return mod; } @@ -3245,7 +3235,7 @@ bootstrap_imp(PyThreadState *tstate) } // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_CLEAR(name); Py_DECREF(spec); if (mod == NULL) { @@ -4405,7 +4395,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_DECREF(name); return mod; } From d38c0e4a4bf99238d11889789836fbf1f1394da0 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 8 Nov 2025 10:28:38 -0800 Subject: [PATCH 03/14] Add tests for the PyImport_CreateModuleFromInitfunc API --- Lib/test/test_embed.py | 5 ++++ Programs/_testembed.c | 62 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1933f691a78be5..055b3afefa9db4 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -239,6 +239,11 @@ def test_repeated_init_and_inittab(self): lines = "\n".join(lines) + "\n" self.assertEqual(out, lines) + def test_create_module_from_initfunc(self): + out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") + self.assertEqual(err, "") + self.assertEqual(out, "\n") + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d3600fecbe2775..597bcb51a2cca1 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2213,6 +2213,67 @@ static int test_repeated_init_and_inittab(void) return 0; } +static PyObject* create_module(PyObject* self, PyObject* spec) { + return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); +} + +static PyMethodDef create_static_module_methods[] = { + {"create_module", create_module, METH_O, NULL}, + {} +}; + +static struct PyModuleDef create_static_module_def = { + PyModuleDef_HEAD_INIT, + .m_name = "create_static_module", + .m_size = 0, + .m_methods = create_static_module_methods, + .m_slots = extension_slots, +}; + +PyMODINIT_FUNC PyInit_create_static_module(void) { + return PyModuleDef_Init(&create_static_module_def); +} + +static int test_create_module_from_initfunc(void) +{ + wchar_t* argv[] = {PROGRAM_NAME, L"-c", L"import embedded_ext; print(embedded_ext)"}; + PyConfig config; + if (PyImport_AppendInittab("create_static_module", + &PyInit_create_static_module) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + int result = PyRun_SimpleString( + "import sys\n" + "from importlib._bootstrap import spec_from_loader, _call_with_frames_removed\n" + "import _imp\n" + "import create_static_module\n" + "class StaticExtensionImporter:\n" + " _ORIGIN = \"static-extension\"\n" + " @classmethod\n" + " def find_spec(cls, fullname, path, target=None):\n" + " if fullname == \"embedded_ext\":\n" + " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" + " return None\n" + " @staticmethod\n" + " def create_module(spec):\n" + " return _call_with_frames_removed(create_static_module.create_module, spec)\n" + " @staticmethod\n" + " def exec_module(module):\n" + " _call_with_frames_removed(_imp.exec_builtin, module)\n" + "sys.meta_path.append(StaticExtensionImporter)\n" + ); + if (result < 0) { + fprintf(stderr, "PyRun_SimpleString() failed\n"); + return 1; + } + return Py_RunMain(); +} + static void wrap_allocator(PyMemAllocatorEx *allocator); static void unwrap_allocator(PyMemAllocatorEx *allocator); @@ -2396,6 +2457,7 @@ static struct TestCase TestCases[] = { #endif {"test_get_incomplete_frame", test_get_incomplete_frame}, {"test_gilstate_after_finalization", test_gilstate_after_finalization}, + {"test_create_module_from_initfunc", test_create_module_from_initfunc}, {NULL, NULL} }; From d5be6e593aabec25b7eab3c80cbc40c7d72636ed Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 8 Nov 2025 10:44:52 -0800 Subject: [PATCH 04/14] Document the new `PyImport_CreateModuleFromInitfunc` C-API --- Doc/c-api/import.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8eabc0406b11ce..ffaa7dbe65f780 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -333,3 +333,15 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement custom + static extension importers (e.g. of statically-linked extensions). + The function creates and returns a module object given a *spec* and an *initfunc*. + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object + + *initfunc* is the same as in :c:func:`PyImport_ExtendInittab` + + .. versionadded:: 3.15 From 781a7305418fee5e1e7c75f34aa1fdeeb152a279 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 8 Nov 2025 10:52:10 -0800 Subject: [PATCH 05/14] Add news entry --- .../next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst new file mode 100644 index 00000000000000..3363877af35257 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst @@ -0,0 +1,2 @@ +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a **spec** and **initfunc**. Patch by Itamar Oren. From 7af34eb116d21f8711548b47ee1e0681b0f98440 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 8 Nov 2025 10:54:44 -0800 Subject: [PATCH 06/14] Fix lint --- Doc/c-api/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index ffaa7dbe65f780..51258e3902106d 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -341,7 +341,7 @@ Importing Modules The function creates and returns a module object given a *spec* and an *initfunc*. *spec* must be a :class:`~importlib.machinery.ModuleSpec` object - + *initfunc* is the same as in :c:func:`PyImport_ExtendInittab` .. versionadded:: 3.15 From 35e9e786a4920a67bf151125ff3ad256177e00db Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sat, 8 Nov 2025 16:31:56 -0800 Subject: [PATCH 07/14] fix CI - remove unused `found` variable - use `my_test_extension` instead of `embedded_ext` (the former is free-threading-ready, the latter prints a warning) --- Lib/test/test_embed.py | 2 +- Programs/_testembed.c | 6 +++--- Python/import.c | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 055b3afefa9db4..df89a2fe52eda3 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -242,7 +242,7 @@ def test_repeated_init_and_inittab(self): def test_create_module_from_initfunc(self): out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") self.assertEqual(err, "") - self.assertEqual(out, "\n") + self.assertEqual(out, "\n") def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 597bcb51a2cca1..2c9ea8a803af47 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2214,7 +2214,7 @@ static int test_repeated_init_and_inittab(void) } static PyObject* create_module(PyObject* self, PyObject* spec) { - return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); + return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension); } static PyMethodDef create_static_module_methods[] = { @@ -2236,7 +2236,7 @@ PyMODINIT_FUNC PyInit_create_static_module(void) { static int test_create_module_from_initfunc(void) { - wchar_t* argv[] = {PROGRAM_NAME, L"-c", L"import embedded_ext; print(embedded_ext)"}; + wchar_t* argv[] = {PROGRAM_NAME, L"-c", L"import my_test_extension; print(my_test_extension)"}; PyConfig config; if (PyImport_AppendInittab("create_static_module", &PyInit_create_static_module) != 0) { @@ -2256,7 +2256,7 @@ static int test_create_module_from_initfunc(void) " _ORIGIN = \"static-extension\"\n" " @classmethod\n" " def find_spec(cls, fullname, path, target=None):\n" - " if fullname == \"embedded_ext\":\n" + " if fullname == \"my_test_extension\":\n" " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" " return None\n" " @staticmethod\n" diff --git a/Python/import.c b/Python/import.c index 8f3c17027e81d0..41e57852b36add 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2365,7 +2365,6 @@ is_builtin(PyObject *name) static PyModInitFunction lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info) { - struct _inittab *found = NULL; for (struct _inittab *p = INITTAB; p->name != NULL; p++) { if (_PyUnicode_EqualToASCIIString(info->name, p->name)) { return (PyModInitFunction)p->initfunc; From 6aa2fb2cd6014f59daaafb0677433e827b23b5ff Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Sun, 9 Nov 2025 09:25:57 -0800 Subject: [PATCH 08/14] Apply Kumar's review suggestion Co-authored-by: Kumar Aditya --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 41e57852b36add..bf4b06bd3fea64 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2449,7 +2449,7 @@ PyImport_CreateModuleFromInitfunc( { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *name = PyObject_GetAttrString(spec, "name"); + PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name)); if (name == NULL) { return NULL; } From f65239982dc8f64fc466fb31c42235fbb04eb5f4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 12 Nov 2025 06:02:41 +0100 Subject: [PATCH 09/14] PyImport_CreateModuleFromInitfunc: Doc & test update (#27) * Test single-phase init as well; don't use private APIs in test * Doc update --------- Co-authored-by: Itamar Oren --- Doc/c-api/import.rst | 17 ++++++++++--- Lib/test/test_embed.py | 8 +++++- Programs/_testembed.c | 58 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 51258e3902106d..84749b51ebc609 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -336,12 +336,21 @@ Importing Modules .. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) - This function is a building block that enables embedders to implement custom + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom static extension importers (e.g. of statically-linked extensions). - The function creates and returns a module object given a *spec* and an *initfunc*. - *spec* must be a :class:`~importlib.machinery.ModuleSpec` object + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. - *initfunc* is the same as in :c:func:`PyImport_ExtendInittab` + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. .. versionadded:: 3.15 diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index df89a2fe52eda3..e9729f8a47fe58 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -242,7 +242,13 @@ def test_repeated_init_and_inittab(self): def test_create_module_from_initfunc(self): out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") self.assertEqual(err, "") - self.assertEqual(out, "\n") + self.assertEqual(out, + "\n" + "my_test_extension.executed='yes'\n" + "my_test_extension.exec_slot_ran='yes'\n" + "\n" + "embedded_ext.executed='yes'\n" + ) def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 2c9ea8a803af47..7427c832d25a46 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -166,6 +166,8 @@ static PyModuleDef embedded_ext = { static PyObject* PyInit_embedded_ext(void) { + // keep this a single-phase initialization module; + // see test_create_module_from_initfunc return PyModule_Create(&embedded_ext); } @@ -1894,8 +1896,16 @@ static int test_initconfig_exit(void) } +int +extension_module_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "exec_slot_ran", "yes"); +} + + static PyModuleDef_Slot extension_slots[] = { {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, extension_module_exec}, {0, NULL} }; @@ -2214,11 +2224,33 @@ static int test_repeated_init_and_inittab(void) } static PyObject* create_module(PyObject* self, PyObject* spec) { - return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension); + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (!name) { + return NULL; + } + if (PyUnicode_EqualToUTF8(name, "my_test_extension")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension); + } + if (PyUnicode_EqualToUTF8(name, "embedded_ext")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); + } + PyErr_Format(PyExc_LookupError, "static module %R not found", name); + Py_DECREF(name); + return NULL; +} + +static PyObject* exec_module(PyObject* self, PyObject* mod) { + if (PyModule_Exec(mod) < 0) { + return NULL; + } + Py_RETURN_NONE; } static PyMethodDef create_static_module_methods[] = { {"create_module", create_module, METH_O, NULL}, + {"exec_module", exec_module, METH_O, NULL}, {} }; @@ -2236,7 +2268,19 @@ PyMODINIT_FUNC PyInit_create_static_module(void) { static int test_create_module_from_initfunc(void) { - wchar_t* argv[] = {PROGRAM_NAME, L"-c", L"import my_test_extension; print(my_test_extension)"}; + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + // Multi-phase initialization + L"import my_test_extension;" + L"print(my_test_extension);" + L"print(f'{my_test_extension.executed=}');" + L"print(f'{my_test_extension.exec_slot_ran=}');" + // Single-phase initialization + L"import embedded_ext;" + L"print(embedded_ext);" + L"print(f'{embedded_ext.executed=}');" + }; PyConfig config; if (PyImport_AppendInittab("create_static_module", &PyInit_create_static_module) != 0) { @@ -2249,22 +2293,22 @@ static int test_create_module_from_initfunc(void) init_from_config_clear(&config); int result = PyRun_SimpleString( "import sys\n" - "from importlib._bootstrap import spec_from_loader, _call_with_frames_removed\n" - "import _imp\n" + "from importlib.util import spec_from_loader\n" "import create_static_module\n" "class StaticExtensionImporter:\n" " _ORIGIN = \"static-extension\"\n" " @classmethod\n" " def find_spec(cls, fullname, path, target=None):\n" - " if fullname == \"my_test_extension\":\n" + " if fullname in {'my_test_extension', 'embedded_ext'}:\n" " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" " return None\n" " @staticmethod\n" " def create_module(spec):\n" - " return _call_with_frames_removed(create_static_module.create_module, spec)\n" + " return create_static_module.create_module(spec)\n" " @staticmethod\n" " def exec_module(module):\n" - " _call_with_frames_removed(_imp.exec_builtin, module)\n" + " create_static_module.exec_module(module)\n" + " module.executed = 'yes'\n" "sys.meta_path.append(StaticExtensionImporter)\n" ); if (result < 0) { From c4f1cc8c039e92ae015fbb971a22e434f446f7a7 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Wed, 12 Nov 2025 07:28:14 -0800 Subject: [PATCH 10/14] Apply Victor's review suggestions Co-authored-by: Victor Stinner --- Doc/c-api/import.rst | 2 +- Include/cpython/import.h | 6 ++---- Programs/_testembed.c | 11 ++++++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 84749b51ebc609..dcf2f6bd351a30 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -353,4 +353,4 @@ Importing Modules On error, return NULL with an exception set. - .. versionadded:: 3.15 + .. versionadded:: next diff --git a/Include/cpython/import.h b/Include/cpython/import.h index d7a5aa76d7c921..9b015583dbc256 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -13,11 +13,9 @@ PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); // Custom importers may use this API to initialize statically linked // extension modules directly from a spec and init function, // without needing to go through inittab -PyAPI_FUNC(PyObject *) -PyImport_CreateModuleFromInitfunc( +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( PyObject *spec, - PyObject* (*initfunc)(void) - ); + PyObject* (*initfunc)(void)); struct _frozen { const char *name; /* ASCII encoded string */ diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 7427c832d25a46..0335a261145e2e 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2223,7 +2223,9 @@ static int test_repeated_init_and_inittab(void) return 0; } -static PyObject* create_module(PyObject* self, PyObject* spec) { +static PyObject* +create_module(PyObject* self, PyObject* spec) +{ PyObject *name = PyObject_GetAttrString(spec, "name"); if (!name) { return NULL; @@ -2241,7 +2243,9 @@ static PyObject* create_module(PyObject* self, PyObject* spec) { return NULL; } -static PyObject* exec_module(PyObject* self, PyObject* mod) { +static PyObject* +exec_module(PyObject* self, PyObject* mod) +{ if (PyModule_Exec(mod) < 0) { return NULL; } @@ -2266,7 +2270,8 @@ PyMODINIT_FUNC PyInit_create_static_module(void) { return PyModuleDef_Init(&create_static_module_def); } -static int test_create_module_from_initfunc(void) +static int +test_create_module_from_initfunc(void) { wchar_t* argv[] = { PROGRAM_NAME, From 29b1b68f6cc403c6526b726e4a02513ac60f1c29 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Wed, 12 Nov 2025 07:35:20 -0800 Subject: [PATCH 11/14] Set error if initfunc is NULL --- Python/import.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Python/import.c b/Python/import.c index bf4b06bd3fea64..65d65ed0a49218 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2447,6 +2447,11 @@ PyObject* PyImport_CreateModuleFromInitfunc( PyObject *spec, PyObject* (*initfunc)(void)) { + if (initfunc == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + PyThreadState *tstate = _PyThreadState_GET(); PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name)); From 116595016dcf7f7c14f7005cf7fb6e29ebb9931e Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Wed, 12 Nov 2025 08:15:05 -0800 Subject: [PATCH 12/14] Account for the RuntimeWarning when import a singlephase init extension under the free-threaded build --- Lib/test/test_embed.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e9729f8a47fe58..1078796eae84e2 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -241,7 +241,21 @@ def test_repeated_init_and_inittab(self): def test_create_module_from_initfunc(self): out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") - self.assertEqual(err, "") + if support.Py_GIL_DISABLED: + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + " has been enabled to load module 'embedded_ext'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + self.assertEqual(filtered_err_lines, []) + else: + self.assertEqual(err, "") self.assertEqual(out, "\n" "my_test_extension.executed='yes'\n" From 79e5d0eb4e582e58ed682b7ff4679ada77176b23 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Thu, 13 Nov 2025 07:27:47 -0800 Subject: [PATCH 13/14] Apply suggestions from code review Co-authored-by: Petr Viktorin Co-authored-by: Kumar Aditya Co-authored-by: Victor Stinner --- Doc/c-api/import.rst | 2 +- Include/cpython/import.h | 2 +- .../next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst | 2 +- Programs/_testembed.c | 2 +- Python/import.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index dcf2f6bd351a30..24e673d3d1394f 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -346,7 +346,7 @@ Importing Modules the same as for :c:func:`PyImport_AppendInittab`. On success, create and return a module object. - This module will not be initialized; call :c:func:`PyModule_Exec` + This module will not be initialized; call :c:func:`!PyModule_Exec` to initialize it. (Custom importers should do this in their :py:meth:`~importlib.abc.Loader.exec_module` method.) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 9b015583dbc256..149a20af8b9cbb 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -15,7 +15,7 @@ PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); // without needing to go through inittab PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( PyObject *spec, - PyObject* (*initfunc)(void)); + PyObject *(*initfunc)(void)); struct _frozen { const char *name; /* ASCII encoded string */ diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst index 3363877af35257..be8043e26ddda8 100644 --- a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst +++ b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst @@ -1,2 +1,2 @@ Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a -module from a **spec** and **initfunc**. Patch by Itamar Oren. +module from a *spec* and *initfunc*. Patch by Itamar Oren. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 0335a261145e2e..27224e508bdd3e 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -166,7 +166,7 @@ static PyModuleDef embedded_ext = { static PyObject* PyInit_embedded_ext(void) { - // keep this a single-phase initialization module; + // keep this as a single-phase initialization module; // see test_create_module_from_initfunc return PyModule_Create(&embedded_ext); } diff --git a/Python/import.c b/Python/import.c index 65d65ed0a49218..ccb01c29f1667b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2445,7 +2445,7 @@ create_builtin( PyObject* PyImport_CreateModuleFromInitfunc( - PyObject *spec, PyObject* (*initfunc)(void)) + PyObject *spec, PyObject *(*initfunc)(void)) { if (initfunc == NULL) { PyErr_BadInternalCall(); From fbfde0a31c7f6c3b0426b0385052cdd40b327c87 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Thu, 13 Nov 2025 07:31:15 -0800 Subject: [PATCH 14/14] Mention new C-API in whatsnew --- Doc/whatsnew/3.15.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 5379ac3abba227..5b329f54b11240 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -950,6 +950,10 @@ New features * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + Changed C APIs --------------