Skip to content

Commit 7a9cb61

Browse files
timfeltimfel
committed
[GR-15802] [GR-15835] Fix caching of core sources in a native image with a preinitialized context
PullRequest: graalpython/517
2 parents 0c20d4d + 7aacb5b commit 7a9cb61

File tree

18 files changed

+423
-65
lines changed

18 files changed

+423
-65
lines changed

‎graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_exceptionobject.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,4 +92,3 @@ def compile_module(self, name):
9292
argspec="OO",
9393
arguments=["PyObject* exc", "PyObject* tb"],
9494
)
95-

‎graalpython/com.oracle.graal.python.test/src/tests/test_imports.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,3 +134,16 @@ def test_recursive_import_from():
134134
if sys.version_info.minor >= 6:
135135
import package.recpkg
136136
assert package.recpkg.context is package.recpkg.reduction.context
137+
138+
139+
if sys.implementation.name == "graalpython":
140+
def test_imp_cached_imports():
141+
import _imp
142+
143+
finder = _imp.CachedImportFinder
144+
145+
spec = finder.find_spec("encodings", None)
146+
assert spec.submodule_search_locations
147+
148+
spec = finder.find_spec("encodings.utf_8", None)
149+
assert not spec.submodule_search_locations

‎graalpython/com.oracle.graal.python.test/src/tests/test_set.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 51 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.util.ArrayList;
3131
import java.util.concurrent.ConcurrentHashMap;
3232
import java.util.function.Supplier;
33+
import java.util.logging.Level;
3334

3435
import com.oracle.graal.python.builtins.Python3Core;
3536
import com.oracle.graal.python.builtins.objects.PEllipsis;
@@ -93,6 +94,7 @@
9394
import com.oracle.truffle.api.source.SourceSection;
9495

9596
import org.graalvm.options.OptionDescriptors;
97+
import org.graalvm.options.OptionValues;
9698

9799
@TruffleLanguage.Registration(id = PythonLanguage.ID, //
98100
name = PythonLanguage.NAME, //
@@ -153,8 +155,29 @@ protected void finalizeContext(PythonContext context) {
153155
super.finalizeContext(context);
154156
}
155157

158+
@Override
159+
protected boolean areOptionsCompatible(OptionValues firstOptions, OptionValues newOptions) {
160+
// internal sources were marked during context initialization
161+
return (firstOptions.get(PythonOptions.ExposeInternalSources).equals(newOptions.get(PythonOptions.ExposeInternalSources)) &&
162+
// we cache CatchAllExceptions hard on TryExceptNode
163+
firstOptions.get(PythonOptions.CatchAllExceptions).equals(newOptions.get(PythonOptions.CatchAllExceptions)));
164+
}
165+
166+
private boolean areOptionsCompatibleWithPreinitializedContext(OptionValues firstOptions, OptionValues newOptions) {
167+
return (areOptionsCompatible(firstOptions, newOptions) &&
168+
// we cache WithThread in SysConfigModuleBuiltins
169+
firstOptions.get(PythonOptions.WithThread).equals(newOptions.get(PythonOptions.WithThread)) &&
170+
// disabling TRegex has an effect on the _sre Python functions that are
171+
// dynamically created
172+
firstOptions.get(PythonOptions.WithTRegex).equals(newOptions.get(PythonOptions.WithTRegex)));
173+
}
174+
156175
@Override
157176
protected boolean patchContext(PythonContext context, Env newEnv) {
177+
if (!areOptionsCompatibleWithPreinitializedContext(context.getEnv().getOptions(), newEnv.getOptions())) {
178+
PythonCore.writeInfo("Cannot use preinitialized context.");
179+
return false;
180+
}
158181
ensureHomeInOptions(newEnv);
159182
PythonCore.writeInfo("Using preinitialized context.");
160183
context.patch(newEnv);
@@ -500,6 +523,7 @@ protected String toString(PythonContext context, Object value) {
500523
}
501524
}
502525

526+
@TruffleBoundary
503527
public static TruffleLogger getLogger() {
504528
return TruffleLogger.getLogger(ID);
505529
}
@@ -534,20 +558,8 @@ public static Source newSource(PythonContext ctxt, String src, String name, bool
534558
}
535559
}
536560

537-
private final ConcurrentHashMap<Object, Source> cachedSources = new ConcurrentHashMap<>();
538-
539-
public Source newSource(PythonContext ctxt, TruffleFile src, String name) throws IOException {
540-
try {
541-
return cachedSources.computeIfAbsent(src, t -> {
542-
try {
543-
return newSource(ctxt, Source.newBuilder(ID, src).name(name));
544-
} catch (IOException e) {
545-
throw new RuntimeException(e);
546-
}
547-
});
548-
} catch (RuntimeException e) {
549-
throw (IOException) e.getCause();
550-
}
561+
public static Source newSource(PythonContext ctxt, TruffleFile src, String name) throws IOException {
562+
return newSource(ctxt, Source.newBuilder(ID, src).name(name));
551563
}
552564

553565
private static Source newSource(PythonContext ctxt, SourceBuilder srcBuilder) throws IOException {
@@ -565,10 +577,32 @@ protected void initializeMultipleContexts() {
565577
singleContextAssumption.invalidate();
566578
}
567579

568-
private final ConcurrentHashMap<String, PCode> cachedCode = new ConcurrentHashMap<>();
580+
private final ConcurrentHashMap<String, CallTarget> cachedCode = new ConcurrentHashMap<>();
581+
private final ConcurrentHashMap<String, String[]> cachedCodeModulePath = new ConcurrentHashMap<>();
582+
583+
@TruffleBoundary
584+
public CallTarget cacheCode(String filename, Supplier<CallTarget> createCode) {
585+
return cachedCode.computeIfAbsent(filename, f -> {
586+
PythonLanguage.getLogger().log(Level.FINEST, () -> "Caching CallTarget for " + filename);
587+
return createCode.get();
588+
});
589+
}
569590

570-
public PCode cacheCode(String filename, Supplier<PCode> createCode) {
571-
return cachedCode.computeIfAbsent(filename, f -> createCode.get());
591+
@TruffleBoundary
592+
public String[] cachedCodeModulePath(String name) {
593+
return cachedCodeModulePath.get(name);
594+
}
595+
596+
@TruffleBoundary
597+
public boolean hasCachedCode(String name) {
598+
return cachedCode.get(name) != null;
599+
}
600+
601+
@TruffleBoundary
602+
public CallTarget cacheCode(String filename, Supplier<CallTarget> createCode, String[] modulepath) {
603+
CallTarget ct = cacheCode(filename, createCode);
604+
cachedCodeModulePath.computeIfAbsent(filename, t -> modulepath);
605+
return ct;
572606
}
573607

574608
public static Shape freshShape() {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
*/
2626
package com.oracle.graal.python.builtins;
2727

28-
import static com.oracle.graal.python.nodes.BuiltinNames.__BUILTINS_PATCHES__;
2928
import static com.oracle.graal.python.nodes.SpecialAttributeNames.__PACKAGE__;
3029
import static com.oracle.graal.python.runtime.exception.PythonErrorType.SyntaxError;
3130

@@ -97,7 +96,6 @@
9796
import com.oracle.graal.python.builtins.objects.bytes.BytesBuiltins;
9897
import com.oracle.graal.python.builtins.objects.cell.CellBuiltins;
9998
import com.oracle.graal.python.builtins.objects.code.CodeBuiltins;
100-
import com.oracle.graal.python.builtins.objects.code.PCode;
10199
import com.oracle.graal.python.builtins.objects.complex.ComplexBuiltins;
102100
import com.oracle.graal.python.builtins.objects.dict.DictBuiltins;
103101
import com.oracle.graal.python.builtins.objects.dict.DictItemsIteratorBuiltins;
@@ -168,6 +166,7 @@
168166
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
169167
import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
170168
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
169+
import com.oracle.truffle.api.CallTarget;
171170
import com.oracle.truffle.api.RootCallTarget;
172171
import com.oracle.truffle.api.Truffle;
173172
import com.oracle.truffle.api.TruffleFile;
@@ -235,7 +234,8 @@ private static final String[] initializeCoreFiles() {
235234
"pyio_patches",
236235
"pwd",
237236
"_contextvars"));
238-
237+
// must be last
238+
coreFiles.add("final_patches");
239239
return coreFiles.toArray(new String[coreFiles.size()]);
240240
}
241241

@@ -437,8 +437,6 @@ public void postInitialize() {
437437
builtin.postInitialize(this);
438438
}
439439

440-
loadFile(__BUILTINS_PATCHES__, PythonCore.getCoreHomeOrFail());
441-
442440
initialized = true;
443441
}
444442
}
@@ -593,21 +591,26 @@ private Source getSource(String basename, String prefix) {
593591
Env env = ctxt.getEnv();
594592
String suffix = env.getFileNameSeparator() + basename + ".py";
595593
TruffleFile file = env.getTruffleFile(prefix + suffix);
594+
String errorMessage;
596595
try {
597-
if (file.exists()) {
598-
return getLanguage().newSource(ctxt, file, basename);
599-
}
600-
} catch (SecurityException | IOException t) {
601-
// fall through;
596+
return PythonLanguage.newSource(ctxt, file, basename);
597+
} catch (IOException e) {
598+
errorMessage = "Startup failed, could not read core library from " + file + ". Maybe you need to set python.CoreHome and python.StdLibHome.";
599+
} catch (SecurityException e) {
600+
errorMessage = "Startup failed, a security exception occurred while reading from " + file + ". Maybe you need to set python.CoreHome and python.StdLibHome.";
602601
}
603-
PythonLanguage.getLogger().log(Level.SEVERE, "Startup failed, could not read core library from " + file + ". Maybe you need to set python.CoreHome and python.StdLibHome.");
604-
throw new RuntimeException();
602+
PythonLanguage.getLogger().log(Level.SEVERE, errorMessage);
603+
PException e = new PException(null, null);
604+
e.setMessage(errorMessage);
605+
throw e;
605606
}
606607

607608
private void loadFile(String s, String prefix) {
608-
Source source = getSource(s, prefix);
609-
Supplier<PCode> getCode = () -> objectFactory.createCode(Truffle.getRuntime().createCallTarget((RootNode) getParser().parse(ParserMode.File, this, source, null)));
610-
RootCallTarget callTarget = getLanguage().cacheCode(source.getName(), getCode).getRootCallTarget();
609+
Supplier<CallTarget> getCode = () -> {
610+
Source source = getSource(s, prefix);
611+
return Truffle.getRuntime().createCallTarget((RootNode) getParser().parse(ParserMode.File, this, source, null));
612+
};
613+
RootCallTarget callTarget = (RootCallTarget) getLanguage().cacheCode(s, getCode);
611614
PythonModule mod = lookupBuiltinModule(s);
612615
if (mod == null) {
613616
// use an anonymous module for the side-effects

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@
169169
import com.oracle.graal.python.runtime.exception.PythonErrorType;
170170
import com.oracle.graal.python.runtime.sequence.PSequence;
171171
import com.oracle.truffle.api.Assumption;
172+
import com.oracle.truffle.api.CallTarget;
172173
import com.oracle.truffle.api.CompilerDirectives;
173174
import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
174175
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
@@ -822,7 +823,6 @@ PCode compile(VirtualFrame frame, PBytes source, String filename, String mode, O
822823
@TruffleBoundary
823824
PCode compile(String expression, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize) {
824825
PythonContext context = getContext();
825-
Source source = PythonLanguage.newSource(context, expression, filename, mayBeFromFile);
826826
ParserMode pm;
827827
if (mode.equals("exec")) {
828828
pm = ParserMode.File;
@@ -833,12 +833,17 @@ PCode compile(String expression, String filename, String mode, Object kwFlags, O
833833
} else {
834834
throw raise(ValueError, "compile() mode must be 'exec', 'eval' or 'single'");
835835
}
836-
Supplier<PCode> createCode = () -> factory().createCode(Truffle.getRuntime().createCallTarget((RootNode) getCore().getParser().parse(pm, getCore(), source, null)));
836+
Supplier<CallTarget> createCode = () -> {
837+
Source source = PythonLanguage.newSource(context, expression, filename, mayBeFromFile);
838+
return Truffle.getRuntime().createCallTarget((RootNode) getCore().getParser().parse(pm, getCore(), source, null));
839+
};
840+
RootCallTarget ct;
837841
if (getCore().isInitialized()) {
838-
return createCode.get();
842+
ct = (RootCallTarget) createCode.get();
839843
} else {
840-
return getCore().getLanguage().cacheCode(filename, createCode);
844+
ct = (RootCallTarget) getCore().getLanguage().cacheCode(filename, createCode);
841845
}
846+
return factory().createCode(ct);
842847
}
843848

844849
@SuppressWarnings("unused")

0 commit comments

Comments
 (0)