Skip to content

Commit 19e4f75

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-100227: Only Use deepfreeze for the Main Interpreter (gh-103794)
Deep-frozen code objects are cannot be shared (currently) by interpreters, due to how adaptive specialization can modify the bytecodes. We work around this by only using the deep-frozen objects in the main interpreter. This does incur a performance penalty for subinterpreters, which we may be able to resolve later.
1 parent ae25855 commit 19e4f75

File tree

3 files changed

+58
-34
lines changed

3 files changed

+58
-34
lines changed

‎Makefile.pre.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1194,7 +1194,7 @@ Tools/build/freeze_modules.py: $(FREEZE_MODULE)
11941194

11951195
.PHONY: regen-frozen
11961196
regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN)
1197-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py
1197+
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py --frozen-modules
11981198
@echo "The Makefile was updated, you may need to re-run make."
11991199

12001200
############################################################################

‎Python/frozen.c‎

Lines changed: 52 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,29 @@
4141
#include <stdbool.h>
4242

4343
/* Includes for frozen modules: */
44+
#include "frozen_modules/importlib._bootstrap.h"
45+
#include "frozen_modules/importlib._bootstrap_external.h"
46+
#include "frozen_modules/zipimport.h"
47+
#include "frozen_modules/abc.h"
48+
#include "frozen_modules/codecs.h"
49+
#include "frozen_modules/io.h"
50+
#include "frozen_modules/_collections_abc.h"
51+
#include "frozen_modules/_sitebuiltins.h"
52+
#include "frozen_modules/genericpath.h"
53+
#include "frozen_modules/ntpath.h"
54+
#include "frozen_modules/posixpath.h"
55+
#include "frozen_modules/os.h"
56+
#include "frozen_modules/site.h"
57+
#include "frozen_modules/stat.h"
58+
#include "frozen_modules/importlib.util.h"
59+
#include "frozen_modules/importlib.machinery.h"
60+
#include "frozen_modules/runpy.h"
61+
#include "frozen_modules/__hello__.h"
62+
#include "frozen_modules/__phello__.h"
63+
#include "frozen_modules/__phello__.ham.h"
64+
#include "frozen_modules/__phello__.ham.eggs.h"
65+
#include "frozen_modules/__phello__.spam.h"
66+
#include "frozen_modules/frozen_only.h"
4467
/* End includes */
4568

4669
#define GET_CODE(name) _Py_get_##name##_toplevel
@@ -78,46 +101,46 @@ extern PyObject *_Py_get_frozen_only_toplevel(void);
78101
/* End extern declarations */
79102

80103
static const struct _frozen bootstrap_modules[] = {
81-
{"_frozen_importlib", NULL, 0, false, GET_CODE(importlib__bootstrap)},
82-
{"_frozen_importlib_external", NULL, 0, false, GET_CODE(importlib__bootstrap_external)},
83-
{"zipimport", NULL, 0, false, GET_CODE(zipimport)},
104+
{"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false, GET_CODE(importlib__bootstrap)},
105+
{"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false, GET_CODE(importlib__bootstrap_external)},
106+
{"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), false, GET_CODE(zipimport)},
84107
{0, 0, 0} /* bootstrap sentinel */
85108
};
86109
static const struct _frozen stdlib_modules[] = {
87110
/* stdlib - startup, without site (python -S) */
88-
{"abc", NULL, 0, false, GET_CODE(abc)},
89-
{"codecs", NULL, 0, false, GET_CODE(codecs)},
90-
{"io", NULL, 0, false, GET_CODE(io)},
111+
{"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false, GET_CODE(abc)},
112+
{"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false, GET_CODE(codecs)},
113+
{"io", _Py_M__io, (int)sizeof(_Py_M__io), false, GET_CODE(io)},
91114

92115
/* stdlib - startup, with site */
93-
{"_collections_abc", NULL, 0, false, GET_CODE(_collections_abc)},
94-
{"_sitebuiltins", NULL, 0, false, GET_CODE(_sitebuiltins)},
95-
{"genericpath", NULL, 0, false, GET_CODE(genericpath)},
96-
{"ntpath", NULL, 0, false, GET_CODE(ntpath)},
97-
{"posixpath", NULL, 0, false, GET_CODE(posixpath)},
98-
{"os.path", NULL, 0, false, GET_CODE(posixpath)},
99-
{"os", NULL, 0, false, GET_CODE(os)},
100-
{"site", NULL, 0, false, GET_CODE(site)},
101-
{"stat", NULL, 0, false, GET_CODE(stat)},
116+
{"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc), false, GET_CODE(_collections_abc)},
117+
{"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins), false, GET_CODE(_sitebuiltins)},
118+
{"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), false, GET_CODE(genericpath)},
119+
{"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), false, GET_CODE(ntpath)},
120+
{"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false, GET_CODE(posixpath)},
121+
{"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), false, GET_CODE(posixpath)},
122+
{"os", _Py_M__os, (int)sizeof(_Py_M__os), false, GET_CODE(os)},
123+
{"site", _Py_M__site, (int)sizeof(_Py_M__site), false, GET_CODE(site)},
124+
{"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), false, GET_CODE(stat)},
102125

103126
/* runpy - run module with -m */
104-
{"importlib.util", NULL, 0, false, GET_CODE(importlib_util)},
105-
{"importlib.machinery", NULL, 0, false, GET_CODE(importlib_machinery)},
106-
{"runpy", NULL, 0, false, GET_CODE(runpy)},
127+
{"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), false, GET_CODE(importlib_util)},
128+
{"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), false, GET_CODE(importlib_machinery)},
129+
{"runpy", _Py_M__runpy, (int)sizeof(_Py_M__runpy), false, GET_CODE(runpy)},
107130
{0, 0, 0} /* stdlib sentinel */
108131
};
109132
static const struct _frozen test_modules[] = {
110-
{"__hello__", NULL, 0, false, GET_CODE(__hello__)},
111-
{"__hello_alias__", NULL, 0, false, GET_CODE(__hello__)},
112-
{"__phello_alias__", NULL, 0, true, GET_CODE(__hello__)},
113-
{"__phello_alias__.spam", NULL, 0, false, GET_CODE(__hello__)},
114-
{"__phello__", NULL, 0, true, GET_CODE(__phello__)},
115-
{"__phello__.__init__", NULL, 0, false, GET_CODE(__phello__)},
116-
{"__phello__.ham", NULL, 0, true, GET_CODE(__phello___ham)},
117-
{"__phello__.ham.__init__", NULL, 0, false, GET_CODE(__phello___ham)},
118-
{"__phello__.ham.eggs", NULL, 0, false, GET_CODE(__phello___ham_eggs)},
119-
{"__phello__.spam", NULL, 0, false, GET_CODE(__phello___spam)},
120-
{"__hello_only__", NULL, 0, false, GET_CODE(frozen_only)},
133+
{"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)},
134+
{"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)},
135+
{"__phello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), true, GET_CODE(__hello__)},
136+
{"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__), false, GET_CODE(__hello__)},
137+
{"__phello__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), true, GET_CODE(__phello__)},
138+
{"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), false, GET_CODE(__phello__)},
139+
{"__phello__.ham", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), true, GET_CODE(__phello___ham)},
140+
{"__phello__.ham.__init__", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), false, GET_CODE(__phello___ham)},
141+
{"__phello__.ham.eggs", _Py_M____phello___ham_eggs, (int)sizeof(_Py_M____phello___ham_eggs), false, GET_CODE(__phello___ham_eggs)},
142+
{"__phello__.spam", _Py_M____phello___spam, (int)sizeof(_Py_M____phello___spam), false, GET_CODE(__phello___spam)},
143+
{"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only), false, GET_CODE(frozen_only)},
121144
{0, 0, 0} /* test sentinel */
122145
};
123146
const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;

‎Python/import.c‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2021,9 +2021,9 @@ find_frozen(PyObject *nameobj, struct frozen_info *info)
20212021
}
20222022

20232023
static PyObject *
2024-
unmarshal_frozen_code(struct frozen_info *info)
2024+
unmarshal_frozen_code(PyInterpreterState *interp, struct frozen_info *info)
20252025
{
2026-
if (info->get_code) {
2026+
if (info->get_code && _Py_IsMainInterpreter(interp)) {
20272027
PyObject *code = info->get_code();
20282028
assert(code != NULL);
20292029
return code;
@@ -2070,7 +2070,7 @@ PyImport_ImportFrozenModuleObject(PyObject *name)
20702070
set_frozen_error(status, name);
20712071
return -1;
20722072
}
2073-
co = unmarshal_frozen_code(&info);
2073+
co = unmarshal_frozen_code(tstate->interp, &info);
20742074
if (co == NULL) {
20752075
return -1;
20762076
}
@@ -3528,7 +3528,8 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name,
35283528
return NULL;
35293529
}
35303530

3531-
PyObject *codeobj = unmarshal_frozen_code(&info);
3531+
PyInterpreterState *interp = _PyInterpreterState_GET();
3532+
PyObject *codeobj = unmarshal_frozen_code(interp, &info);
35323533
if (dataobj != Py_None) {
35333534
PyBuffer_Release(&buf);
35343535
}

0 commit comments

Comments
 (0)