Skip to content

Commit d185928

Browse files
timfeltimfel
committed
[hotfix] Make sources loaded during initialization internal and add option to expose them
PullRequest: graalpython/132
2 parents d9b432b + 6c0d9a6 commit d185928

File tree

8 files changed

+62
-42
lines changed

8 files changed

+62
-42
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
package com.oracle.graal.python;
2727

2828
import java.io.IOException;
29+
import java.net.URL;
2930
import java.text.MessageFormat;
3031
import java.util.ArrayList;
3132

@@ -60,6 +61,7 @@
6061
import com.oracle.truffle.api.TruffleFile;
6162
import com.oracle.truffle.api.TruffleLanguage;
6263
import com.oracle.truffle.api.TruffleLogger;
64+
import com.oracle.truffle.api.TruffleOptions;
6365
import com.oracle.truffle.api.debug.DebuggerTags;
6466
import com.oracle.truffle.api.frame.Frame;
6567
import com.oracle.truffle.api.frame.MaterializedFrame;
@@ -70,6 +72,7 @@
7072
import com.oracle.truffle.api.nodes.Node;
7173
import com.oracle.truffle.api.nodes.RootNode;
7274
import com.oracle.truffle.api.source.Source;
75+
import com.oracle.truffle.api.source.Source.Builder;
7376

7477
@TruffleLanguage.Registration(id = PythonLanguage.ID, name = PythonLanguage.NAME, version = PythonLanguage.VERSION, mimeType = PythonLanguage.MIME_TYPE, interactive = true, internal = false)
7578
@ProvidedTags({StandardTags.CallTag.class, StandardTags.StatementTag.class, StandardTags.RootTag.class, StandardTags.TryBlockTag.class, DebuggerTags.AlwaysHalt.class})
@@ -84,6 +87,7 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
8487
public static final String MIME_TYPE = "text/x-python";
8588
public static final String EXTENSION = ".py";
8689

90+
@CompilationFinal private boolean nativeBuildTime = TruffleOptions.AOT;
8791
@CompilationFinal private PythonCore sharedCore;
8892
private final NodeFactory nodeFactory;
8993

@@ -103,6 +107,7 @@ protected void finalizeContext(PythonContext context) {
103107

104108
@Override
105109
protected boolean patchContext(PythonContext context, Env newEnv) {
110+
nativeBuildTime = false; // now we're running
106111
ensureSysExecutable(context);
107112
ensureHomeInOptions(newEnv);
108113
if (!optionsAllowPreInitializedContext(context, newEnv)) {
@@ -113,6 +118,7 @@ protected boolean patchContext(PythonContext context, Env newEnv) {
113118
context.setOut(newEnv.out());
114119
context.setErr(newEnv.err());
115120
context.initialize();
121+
context.getCore().postInitialize();
116122
return true;
117123
}
118124

@@ -395,4 +401,30 @@ public static PythonCore getCore() {
395401
public static TruffleLogger getLogger() {
396402
return TruffleLogger.getLogger(ID, PythonLanguage.class);
397403
}
404+
405+
public static Source newSource(PythonContext ctxt, String src, String name) {
406+
return newSource(ctxt, Source.newBuilder(src), name);
407+
}
408+
409+
public static Source newSource(PythonContext ctxt, TruffleFile src, String name) throws IOException {
410+
return newSource(ctxt, ctxt.getEnv().newSourceBuilder(src), name);
411+
}
412+
413+
public static Source newSource(PythonContext ctxt, URL url, String name) throws IOException {
414+
return newSource(ctxt, Source.newBuilder(url), name);
415+
}
416+
417+
private static <E1 extends Exception, E2 extends Exception, E3 extends Exception> Source newSource(PythonContext ctxt, Builder<E1, E2, E3> srcBuilder,
418+
String name) throws E1 {
419+
Builder<E1, RuntimeException, RuntimeException> newBuilder = srcBuilder.name(name).mimeType(MIME_TYPE);
420+
boolean internal = !ctxt.getCore().isInitialized() && !PythonOptions.getOption(ctxt, PythonOptions.ExposeInternalSources) && !PythonOptions.getOption(ctxt, PythonOptions.LazyInit);
421+
if (internal) {
422+
srcBuilder.internal();
423+
}
424+
return newBuilder.build();
425+
}
426+
427+
public boolean isNativeBuildTime() {
428+
return nativeBuildTime;
429+
}
398430
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -195,11 +195,6 @@ public final class Python3Core implements PythonCore {
195195
"_locale",
196196
};
197197

198-
private static final Map<String, Object> BUILTIN_CONSTANTS = new HashMap<>();
199-
static {
200-
BUILTIN_CONSTANTS.put("NotImplemented", PNotImplemented.NOT_IMPLEMENTED);
201-
}
202-
203198
private final PythonBuiltins[] BUILTINS = new PythonBuiltins[]{
204199
new BuiltinConstructors(),
205200
new BuiltinFunctions(),
@@ -293,7 +288,6 @@ public final class Python3Core implements PythonCore {
293288
private final PythonParser parser;
294289

295290
@CompilationFinal private boolean initialized;
296-
@CompilationFinal private boolean builtinsPatchesLoaded;
297291

298292
// used in case PythonOptions.SharedCore is false
299293
@CompilationFinal private PythonContext singletonContext;
@@ -345,14 +339,14 @@ public void bootstrap() {
345339
publishBuiltinModules();
346340

347341
builtinsModule = builtinModules.get("builtins");
348-
setBuiltinsConstants();
349342
}
350343

351344
@Override
352345
public boolean isInitialized() {
353346
return initialized;
354347
}
355348

349+
@Override
356350
public void initialize() {
357351
String coreHome = PythonCore.getCoreHomeOrFail();
358352
loadFile("builtins", coreHome);
@@ -362,15 +356,16 @@ public void initialize() {
362356
}
363357
exportCInterface(getContext());
364358
currentException = null;
359+
postInitialize();
365360
initialized = true;
366361
}
367362

368363
@Override
369-
public void loadBuiltinsPatches() {
370-
if (initialized && !builtinsPatchesLoaded) {
371-
builtinsPatchesLoaded = true;
372-
String coreHome = PythonCore.getCoreHomeOrFail();
373-
loadFile(__BUILTINS_PATCHES__, coreHome);
364+
public void postInitialize() {
365+
if (!getLanguage().isNativeBuildTime()) {
366+
initialized = false;
367+
loadFile(__BUILTINS_PATCHES__, PythonCore.getCoreHomeOrFail());
368+
initialized = true;
374369
}
375370
}
376371

@@ -600,12 +595,6 @@ private void publishBuiltinModules() {
600595
}
601596
}
602597

603-
private void setBuiltinsConstants() {
604-
for (Entry<String, Object> entry : BUILTIN_CONSTANTS.entrySet()) {
605-
builtinsModule.setAttribute(entry.getKey(), entry.getValue());
606-
}
607-
}
608-
609598
public void exportCInterface(PythonContext context) {
610599
Env env = context.getEnv();
611600
if (env != null) {
@@ -768,18 +757,19 @@ private static Source getSource(String basename, String prefix) {
768757
// pass
769758
}
770759
String suffix = FILE_SEPARATOR + basename + ".py";
760+
PythonContext ctxt = PythonLanguage.getContext();
771761
if (url != null) {
772762
// This path is hit when we load the core library e.g. from a Jar file
773763
try {
774-
return Source.newBuilder(new URL(url + suffix)).name(basename).mimeType(PythonLanguage.MIME_TYPE).build();
764+
return PythonLanguage.newSource(ctxt, new URL(url + suffix), basename);
775765
} catch (IOException e) {
776766
throw new RuntimeException("Could not read core library from " + url);
777767
}
778768
} else {
779-
Env env = PythonLanguage.getContext().getEnv();
769+
Env env = ctxt.getEnv();
780770
TruffleFile file = env.getTruffleFile(prefix + suffix);
781771
try {
782-
return env.newSourceBuilder(file).name(basename).mimeType(PythonLanguage.MIME_TYPE).build();
772+
return PythonLanguage.newSource(ctxt, file, basename);
783773
} catch (SecurityException | IOException t) {
784774
throw new RuntimeException("Could not read core library from " + file);
785775
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@
145145
import com.oracle.graal.python.nodes.subscript.SliceLiteralNode;
146146
import com.oracle.graal.python.nodes.truffle.PythonArithmeticTypes;
147147
import com.oracle.graal.python.runtime.JavaTypeConversions;
148+
import com.oracle.graal.python.runtime.PythonCore;
148149
import com.oracle.graal.python.runtime.exception.PException;
149150
import com.oracle.graal.python.runtime.exception.PythonErrorType;
150151
import com.oracle.graal.python.runtime.sequence.PSequence;
@@ -170,6 +171,12 @@ protected List<com.oracle.truffle.api.dsl.NodeFactory<? extends PythonBuiltinBas
170171
return BuiltinConstructorsFactory.getFactories();
171172
}
172173

174+
@Override
175+
public void initialize(PythonCore core) {
176+
super.initialize(core);
177+
builtinConstants.put("NotImplemented", PNotImplemented.NOT_IMPLEMENTED);
178+
}
179+
173180
// bytes([source[, encoding[, errors]]])
174181
@Builtin(name = BYTES, minNumOfArguments = 1, maxNumOfArguments = 4, constructsClass = PBytes.class)
175182
@GenerateNodeFactory

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -499,7 +499,7 @@ private Object evalExpression(String expression, PythonObject globals, PythonObj
499499
}
500500
}
501501
PythonParser parser = getCore().getParser();
502-
Source source = Source.newBuilder(expression).name(name).mimeType(PythonLanguage.MIME_TYPE).build();
502+
Source source = PythonLanguage.newSource(getContext(), expression, name);
503503
RootNode parsed = (RootNode) parser.parse(ParserMode.Eval, getCore(), source, callerFrame);
504504
return evalNode(parsed, globals, locals, closure);
505505
}
@@ -532,7 +532,7 @@ Object compile(PBytes source, String filename, String mode, Object kwFlags, Obje
532532
@Specialization
533533
@TruffleBoundary
534534
Object compile(String expression, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize) {
535-
Source source = Source.newBuilder(expression).name(filename).mimeType(PythonLanguage.MIME_TYPE).build();
535+
Source source = PythonLanguage.newSource(getContext(), expression, filename);
536536
ParserMode pm;
537537
if (mode.equals("exec")) {
538538
pm = ParserMode.File;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@
4646
import static com.oracle.graal.python.runtime.exception.PythonErrorType.NotImplementedError;
4747

4848
import java.io.IOException;
49-
import java.net.URI;
5049
import java.util.List;
5150
import java.util.concurrent.locks.ReentrantLock;
5251
import java.util.regex.Pattern;
@@ -67,6 +66,7 @@
6766
import com.oracle.graal.python.nodes.call.special.CallUnaryMethodNode;
6867
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
6968
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
69+
import com.oracle.graal.python.runtime.PythonContext;
7070
import com.oracle.graal.python.runtime.PythonCore;
7171
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
7272
import com.oracle.graal.python.runtime.exception.PException;
@@ -340,29 +340,21 @@ public Object run(String path, String modulename) {
340340
@Specialization
341341
@TruffleBoundary
342342
public Object run(String path, PythonModule mod) {
343-
Env env = getContext().getEnv();
343+
PythonContext ctxt = getContext();
344+
Env env = ctxt.getEnv();
344345
try {
345346
String[] pathParts = path.split(Pattern.quote(PythonCore.FILE_SEPARATOR));
346347
String fileName = pathParts[pathParts.length - 1];
347348
TruffleFile file = env.getTruffleFile(path);
348-
Source src = null;
349-
try {
350-
if (file.exists()) {
351-
src = env.newSourceBuilder(file).mimeType(PythonLanguage.MIME_TYPE).build();
352-
}
353-
} catch (SecurityException e) {
354-
}
355-
if (src == null) {
356-
src = Source.newBuilder("").uri(URI.create(path)).mimeType(PythonLanguage.MIME_TYPE).name(fileName).build();
357-
}
349+
Source src = PythonLanguage.newSource(ctxt, file, fileName);
358350
RootNode parsedModule = (RootNode) getCore().getParser().parse(ParserMode.File, getCore(), src, null);
359351
if (parsedModule != null) {
360352
CallTarget callTarget = Truffle.getRuntime().createCallTarget(parsedModule);
361353
callTarget.call(PArguments.withGlobals(mod));
362354
}
363355
} catch (PException e) {
364356
throw e;
365-
} catch (IOException e) {
357+
} catch (IOException | SecurityException e) {
366358
throw raise(ImportError, e.getMessage());
367359
}
368360
return PNone.NONE;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonParserImpl.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,6 @@ public Node parse(ParserMode mode, PythonCore core, Source source, Frame current
9292
throw handleParserError(core, source, e);
9393
}
9494
}
95-
// ensure builtins patches are loaded before parsing
96-
core.loadBuiltinsPatches();
9795
TranslationEnvironment environment = new TranslationEnvironment(core.getLanguage());
9896
Node result;
9997
Consumer<TranslationEnvironment> environmentConsumer = (env) -> env.setFreeVarsInRootScope(currentFrame);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonCore.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ public interface PythonCore {
6767
*/
6868
public void initialize();
6969

70+
public void postInitialize();
71+
7072
public boolean isInitialized();
7173

7274
public PythonModule lookupBuiltinModule(String name);
@@ -121,10 +123,6 @@ public interface PythonCore {
121123

122124
public PythonContext getContext();
123125

124-
default void loadBuiltinsPatches() {
125-
126-
}
127-
128126
static void writeWarning(TruffleLanguage.Env env, String warning) {
129127
if (!LIBPOLYGLOT || env.getOptions().get(PythonOptions.VerboseFlag)) {
130128
write(env, "WARNING: " + warning);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ private PythonOptions() {
4343
@Option(category = OptionCategory.DEBUG, help = "Defer loading the core lib until after language initialization, so it can be debugged.") //
4444
public static final OptionKey<Boolean> LazyInit = new OptionKey<>(false);
4545

46+
@Option(category = OptionCategory.DEBUG, help = "Expose internal sources as normal sources, so they will show up in the debugger and stacks") //
47+
public static final OptionKey<Boolean> ExposeInternalSources = new OptionKey<>(false);
48+
4649
@Option(category = OptionCategory.DEBUG, help = "Print the java stacktrace if enabled") //
4750
public static final OptionKey<Boolean> WithJavaStacktrace = new OptionKey<>(false);
4851

0 commit comments

Comments
 (0)