Skip to content

Commit 8c02cff

Browse files
timfeltimfel
committed
Merge branch 'tim/freeze_importlib' into thh/freeze_modules
2 parents b5c1b1a + 4705979 commit 8c02cff

File tree

14 files changed

+162
-596
lines changed

14 files changed

+162
-596
lines changed

‎graalpython/com.oracle.graal.python.frozen/freeze_modules.py

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,17 @@
2323
FROZEN = [
2424
# See parse_frozen_spec() for the format.
2525
# In cases where the frozenid is duplicated, the first one is re-used.
26-
# TODO: Enable freezing import system.
27-
# ('import system', [
28-
# # These frozen modules are necessary for bootstrapping
29-
# # the import system.
30-
# 'importlib._bootstrap : _frozen_importlib',
31-
# 'importlib._bootstrap_external : _frozen_importlib_external',
32-
# # This module is important because some Python builds rely
33-
# # on a builtin zip file instead of a filesystem.
34-
# 'zipimport',
35-
# ]),
26+
(
27+
'import system',
28+
[
29+
# These frozen modules are necessary for bootstrapping the import
30+
# system.
31+
'importlib._bootstrap : _frozen_importlib',
32+
'importlib._bootstrap_external : _frozen_importlib_external',
33+
# CPython freezes zipimport, but we have it entirely intrinsified
34+
# 'zipimport',
35+
]
36+
),
3637
(
3738
"stdlib - startup, without site (python -S)",
3839
[
@@ -67,6 +68,8 @@
6768
"genericpath",
6869
"ntpath",
6970
"posixpath",
71+
# We must explicitly mark os.path as a frozen module
72+
("ntpath" if os.name == "nt" else "posixpath") + " : os.path",
7073
"os",
7174
"site",
7275
"stat",
@@ -524,7 +527,10 @@ def freeze_module(src):
524527
def write_frozen_modules_map(out_file, modules):
525528
out_file.write(" private static final class Map {\n")
526529
for module in modules:
527-
if not module.isalias or not module.orig:
530+
if (not module.isalias or
531+
not module.orig or
532+
not any(module.orig == m.orig for m in modules if m != module)
533+
):
528534
ispkg = "true" if module.ispkg else "false"
529535
out_file.write(
530536
f' private static final PythonFrozenModule {module.symbol} = new PythonFrozenModule("{module.symbol}", "{module.frozenid}", {ispkg});\n'

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -737,7 +737,6 @@ protected void initializeMultipleContexts() {
737737
}
738738

739739
private final ConcurrentHashMap<String, CallTarget> cachedCode = new ConcurrentHashMap<>();
740-
private final ConcurrentHashMap<String, String[]> cachedCodeModulePath = new ConcurrentHashMap<>();
741740

742741
@TruffleBoundary
743742
public CallTarget cacheCode(String filename, Supplier<CallTarget> createCode) {
@@ -747,23 +746,6 @@ public CallTarget cacheCode(String filename, Supplier<CallTarget> createCode) {
747746
});
748747
}
749748

750-
@TruffleBoundary
751-
public String[] cachedCodeModulePath(String name) {
752-
return cachedCodeModulePath.get(name);
753-
}
754-
755-
@TruffleBoundary
756-
public boolean hasCachedCode(String name) {
757-
return cachedCode.get(name) != null;
758-
}
759-
760-
@TruffleBoundary
761-
public CallTarget cacheCode(String filename, Supplier<CallTarget> createCode, String[] modulepath) {
762-
CallTarget ct = cacheCode(filename, createCode);
763-
cachedCodeModulePath.computeIfAbsent(filename, t -> modulepath);
764-
return ct;
765-
}
766-
767749
@Override
768750
protected boolean isThreadAccessAllowed(Thread thread, boolean singleThreaded) {
769751
if (singleThreaded) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 54 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,13 @@
2626
package com.oracle.graal.python.builtins;
2727

2828
import static com.oracle.graal.python.builtins.PythonBuiltinClassType.IndentationError;
29+
import static com.oracle.graal.python.builtins.PythonBuiltinClassType.SyntaxError;
2930
import static com.oracle.graal.python.builtins.PythonBuiltinClassType.TabError;
3031
import static com.oracle.graal.python.builtins.objects.exception.SyntaxErrorBuiltins.SYNTAX_ERROR_ATTR_FACTORY;
3132
import static com.oracle.graal.python.nodes.BuiltinNames.MODULES;
3233
import static com.oracle.graal.python.nodes.BuiltinNames.PRINT;
3334
import static com.oracle.graal.python.nodes.SpecialAttributeNames.__PACKAGE__;
34-
import static com.oracle.graal.python.runtime.exception.PythonErrorType.SyntaxError;
35+
import static com.oracle.graal.python.nodes.SpecialMethodNames.__REPR__;
3536

3637
import java.io.IOException;
3738
import java.math.BigInteger;
@@ -225,6 +226,7 @@
225226
import com.oracle.graal.python.builtins.objects.function.BuiltinFunctionBuiltins;
226227
import com.oracle.graal.python.builtins.objects.function.FunctionBuiltins;
227228
import com.oracle.graal.python.builtins.objects.function.PArguments;
229+
import com.oracle.graal.python.builtins.objects.function.PFunction;
228230
import com.oracle.graal.python.builtins.objects.function.PBuiltinFunction;
229231
import com.oracle.graal.python.builtins.objects.generator.GeneratorBuiltins;
230232
import com.oracle.graal.python.builtins.objects.getsetdescriptor.DescriptorBuiltins;
@@ -268,7 +270,6 @@
268270
import com.oracle.graal.python.builtins.objects.method.DecoratedMethodBuiltins;
269271
import com.oracle.graal.python.builtins.objects.method.InstancemethodBuiltins;
270272
import com.oracle.graal.python.builtins.objects.method.MethodBuiltins;
271-
import com.oracle.graal.python.builtins.objects.method.PMethod;
272273
import com.oracle.graal.python.builtins.objects.method.StaticmethodBuiltins;
273274
import com.oracle.graal.python.builtins.objects.mmap.MMapBuiltins;
274275
import com.oracle.graal.python.builtins.objects.module.ModuleBuiltins;
@@ -309,8 +310,11 @@
309310
import com.oracle.graal.python.builtins.objects.type.TypeBuiltins;
310311
import com.oracle.graal.python.builtins.objects.type.TypeNodes.GetNameNode;
311312
import com.oracle.graal.python.builtins.objects.zipimporter.ZipImporterBuiltins;
313+
import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs;
312314
import com.oracle.graal.python.lib.PyObjectLookupAttr;
313315
import com.oracle.graal.python.nodes.BuiltinNames;
316+
import com.oracle.graal.python.nodes.attributes.ReadAttributeFromDynamicObjectNode;
317+
import com.oracle.graal.python.nodes.attributes.WriteAttributeToDynamicObjectNode;
314318
import com.oracle.graal.python.nodes.call.GenericInvokeNode;
315319
import com.oracle.graal.python.runtime.PythonCodeSerializer;
316320
import com.oracle.graal.python.runtime.PythonContext;
@@ -354,9 +358,6 @@ private static String[] initializeCoreFiles() {
354358
// Order matters!
355359
List<String> coreFiles = new ArrayList<>(Arrays.asList(
356360
"type",
357-
"_imp",
358-
"function",
359-
"_frozen_importlib",
360361
"__graalpython__",
361362
"_weakref",
362363
"faulthandler",
@@ -383,8 +384,6 @@ private static String[] initializeCoreFiles() {
383384
}
384385
}
385386
}
386-
// must be last
387-
coreFiles.add("final_patches");
388387
return coreFiles.toArray(new String[coreFiles.size()]);
389388
}
390389

@@ -716,7 +715,7 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
716715
@CompilationFinal private PythonModule builtinsModule;
717716
@CompilationFinal private PythonModule sysModule;
718717
@CompilationFinal private PDict sysModules;
719-
@CompilationFinal private PMethod importFunc;
718+
@CompilationFinal private PFunction importFunc;
720719
@CompilationFinal private PythonModule importlib;
721720

722721
@CompilationFinal private PInt pyTrue;
@@ -811,6 +810,7 @@ public final boolean isCoreInitialized() {
811810
public final void initialize(PythonContext context) {
812811
objectFactory = new PythonObjectSlowPathFactory(context.getAllocationReporter(), context.getLanguage());
813812
initializeJavaCore();
813+
initializeImportlib();
814814
initializePython3Core(context.getCoreHomeOrFail());
815815
assert SpecialMethodSlot.checkSlotOverrides(this);
816816
initialized = true;
@@ -824,6 +824,42 @@ private void initializeJavaCore() {
824824
builtinsModule = builtinModules.get(BuiltinNames.BUILTINS);
825825
}
826826

827+
private void initializeImportlib() {
828+
PythonModule bootstrap = (PythonModule) ImpModuleBuiltins.importFrozenModuleObject(this, "_frozen_importlib", false);
829+
830+
PyObjectCallMethodObjArgs callNode = PyObjectCallMethodObjArgs.getUncached();
831+
WriteAttributeToDynamicObjectNode writeNode = WriteAttributeToDynamicObjectNode.getUncached();
832+
ReadAttributeFromDynamicObjectNode readNode = ReadAttributeFromDynamicObjectNode.getUncached();
833+
834+
if (bootstrap == null) {
835+
// true when the frozen module is not available
836+
PythonModule bootstrapExternal = createModule("importlib._bootstrap_external");
837+
writeNode.execute(bootstrapExternal, __PACKAGE__, "importlib");
838+
addBuiltinModule("_frozen_importlib_external", bootstrapExternal);
839+
bootstrap = createModule("importlib._bootstrap");
840+
writeNode.execute(bootstrap, __PACKAGE__, "importlib");
841+
addBuiltinModule("_frozen_importlib", bootstrap);
842+
loadFile("importlib/_bootstrap_external", getContext().getStdlibHome(), bootstrapExternal);
843+
loadFile("importlib/_bootstrap", getContext().getStdlibHome(), bootstrap);
844+
}
845+
846+
callNode.execute(null, bootstrap, "_install", getSysModule(), lookupBuiltinModule("_imp"));
847+
writeNode.execute(getBuiltins(), "__import__", readNode.execute(bootstrap, "__import__"));
848+
callNode.execute(null, bootstrap, "_install_external_importers");
849+
importFunc = (PFunction) readNode.execute(bootstrap, "__import__");
850+
importlib = bootstrap;
851+
852+
PythonBuiltinClass moduleType = lookupType(PythonBuiltinClassType.PythonModule);
853+
writeNode.execute(moduleType, __REPR__, readNode.execute(bootstrap, "_module_repr"));
854+
SpecialMethodSlot.reinitializeSpecialMethodSlots(moduleType, getLanguage());
855+
856+
// __package__ needs to be set and doesn't get set by _bootstrap setup
857+
writeNode.execute(bootstrap, __PACKAGE__, "importlib");
858+
859+
PythonModule bootstrapExternal = (PythonModule) getSysModules().getItem("_frozen_importlib_external");
860+
writeNode.execute(bootstrapExternal, __PACKAGE__, "importlib");
861+
}
862+
827863
private void initializePython3Core(String coreHome) {
828864
loadFile(BuiltinNames.BUILTINS, coreHome);
829865
for (String s : coreFiles) {
@@ -890,24 +926,10 @@ public final PythonModule getImportlib() {
890926
return importlib;
891927
}
892928

893-
public final void registerImportlib(PythonModule mod) {
894-
if (importlib != null) {
895-
throw new IllegalStateException("importlib cannot be registered more than once");
896-
}
897-
importlib = mod;
898-
}
899-
900-
public final PMethod getImportFunc() {
929+
public final PFunction getImportFunc() {
901930
return importFunc;
902931
}
903932

904-
public final void registerImportFunc(PMethod func) {
905-
if (importFunc != null) {
906-
throw new IllegalStateException("__import__ func cannot be registered more than once");
907-
}
908-
importFunc = func;
909-
}
910-
911933
@Override
912934
@TruffleBoundary
913935
public final void warn(PythonBuiltinClassType type, String format, Object... args) {
@@ -1005,13 +1027,6 @@ private void populateBuiltins() {
10051027
// core machinery
10061028
sysModule = builtinModules.get("sys");
10071029
sysModules = (PDict) sysModule.getAttribute(MODULES);
1008-
1009-
PythonModule bootstrapExternal = createModule("importlib._bootstrap_external");
1010-
bootstrapExternal.setAttribute(__PACKAGE__, "importlib");
1011-
addBuiltinModule("_frozen_importlib_external", bootstrapExternal);
1012-
PythonModule bootstrap = createModule("importlib._bootstrap");
1013-
bootstrap.setAttribute(__PACKAGE__, "importlib");
1014-
addBuiltinModule("_frozen_importlib", bootstrap);
10151030
}
10161031

10171032
public PythonModule createModule(String name) {
@@ -1087,16 +1102,20 @@ private Source getInternalSource(String basename, String prefix) {
10871102
}
10881103

10891104
private void loadFile(String s, String prefix) {
1090-
Supplier<CallTarget> getCode = () -> {
1091-
Source source = getInternalSource(s, prefix);
1092-
return PythonUtils.getOrCreateCallTarget((RootNode) getParser().parse(ParserMode.File, 0, this, source, null, null));
1093-
};
1094-
RootCallTarget callTarget = (RootCallTarget) getLanguage().cacheCode(s, getCode);
10951105
PythonModule mod = lookupBuiltinModule(s);
10961106
if (mod == null) {
10971107
// use an anonymous module for the side-effects
10981108
mod = factory().createPythonModule("__anonymous__");
10991109
}
1110+
loadFile(s, prefix, mod);
1111+
}
1112+
1113+
private void loadFile(String s, String prefix, PythonModule mod) {
1114+
Supplier<CallTarget> getCode = () -> {
1115+
Source source = getInternalSource(s, prefix);
1116+
return PythonUtils.getOrCreateCallTarget((RootNode) getParser().parse(ParserMode.File, 0, this, source, null, null));
1117+
};
1118+
RootCallTarget callTarget = (RootCallTarget) getLanguage().cacheCode(s, getCode);
11001119
GenericInvokeNode.getUncached().execute(callTarget, PArguments.withGlobals(mod));
11011120
}
11021121

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import static com.oracle.graal.python.nodes.BuiltinNames.EXEC;
4747
import static com.oracle.graal.python.nodes.BuiltinNames.FORMAT;
4848
import static com.oracle.graal.python.nodes.BuiltinNames.GETATTR;
49+
import static com.oracle.graal.python.nodes.BuiltinNames.HASATTR;
4950
import static com.oracle.graal.python.nodes.BuiltinNames.HASH;
5051
import static com.oracle.graal.python.nodes.BuiltinNames.HEX;
5152
import static com.oracle.graal.python.nodes.BuiltinNames.ID;
@@ -1955,6 +1956,17 @@ Object setAttr(VirtualFrame frame, Object object, Object key, Object value,
19551956
}
19561957
}
19571958

1959+
// hasattr(object, name)
1960+
@Builtin(name = HASATTR, minNumOfPositionalArgs = 2)
1961+
@GenerateNodeFactory
1962+
public abstract static class HasAttrNode extends PythonBinaryBuiltinNode {
1963+
@Specialization
1964+
boolean hasAttr(VirtualFrame frame, Object object, Object key,
1965+
@Cached PyObjectLookupAttr pyObjectLookupAttr) {
1966+
return pyObjectLookupAttr.execute(frame, object, key) != PNone.NO_VALUE;
1967+
}
1968+
}
1969+
19581970
// sorted(iterable, key, reverse)
19591971
@Builtin(name = SORTED, minNumOfPositionalArgs = 1, parameterNames = {"$self"}, keywordOnlyNames = {"key", "reverse"})
19601972
@ArgumentClinic(name = "reverse", conversion = ArgumentClinic.ClinicConversion.IntToBoolean, defaultValue = "false")

0 commit comments

Comments
 (0)