make sources loaded during initialization internal and add option to expose them

timfel · timfel · commit 262c2c79f28b · 2018-07-26T10:18:51.000+02:00
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java
@@ -26,6 +26,7 @@
 package com.oracle.graal.python;
 
 import java.io.IOException;
+import java.net.URL;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 
@@ -70,6 +71,7 @@
 import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.nodes.RootNode;
 import com.oracle.truffle.api.source.Source;
+import com.oracle.truffle.api.source.Source.Builder;
 
 @TruffleLanguage.Registration(id = PythonLanguage.ID, name = PythonLanguage.NAME, version = PythonLanguage.VERSION, mimeType = PythonLanguage.MIME_TYPE, interactive = true, internal = false)
 @ProvidedTags({StandardTags.CallTag.class, StandardTags.StatementTag.class, StandardTags.RootTag.class, StandardTags.TryBlockTag.class, DebuggerTags.AlwaysHalt.class})
@@ -395,4 +397,26 @@ public static PythonCore getCore() {
     public static TruffleLogger getLogger() {
         return TruffleLogger.getLogger(ID, PythonLanguage.class);
     }
+
+    public static Source newSource(PythonContext ctxt, String src, String name) {
+        return newSource(ctxt, Source.newBuilder(src), name);
+    }
+
+    public static Source newSource(PythonContext ctxt, TruffleFile src, String name) throws IOException {
+        return newSource(ctxt, ctxt.getEnv().newSourceBuilder(src), name);
+    }
+
+    public static Source newSource(PythonContext ctxt, URL url, String name) throws IOException {
+        return newSource(ctxt, Source.newBuilder(url), name);
+    }
+
+    private static <E1 extends Exception, E2 extends Exception, E3 extends Exception> Source newSource(PythonContext ctxt, Builder<E1, E2, E3> srcBuilder,
+                    String name) throws E1 {
+        Builder<E1, RuntimeException, RuntimeException> newBuilder = srcBuilder.name(name).mimeType(MIME_TYPE);
+        boolean internal = !ctxt.getCore().isInitialized() && !PythonOptions.getOption(ctxt, PythonOptions.ExposeInternalSources) && !PythonOptions.getOption(ctxt, PythonOptions.LazyInit);
+        if (internal) {
+            srcBuilder.internal();
+        }
+        return newBuilder.build();
+    }
 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java
@@ -747,18 +747,19 @@ private static Source getSource(String basename, String prefix) {
             // pass
         }
         String suffix = FILE_SEPARATOR + basename + ".py";
+        PythonContext ctxt = PythonLanguage.getContext();
         if (url != null) {
             // This path is hit when we load the core library e.g. from a Jar file
             try {
-                return Source.newBuilder(new URL(url + suffix)).name(basename).mimeType(PythonLanguage.MIME_TYPE).build();
+                return PythonLanguage.newSource(ctxt, new URL(url + suffix), basename);
             } catch (IOException e) {
                 throw new RuntimeException("Could not read core library from " + url);
             }
         } else {
-            Env env = PythonLanguage.getContext().getEnv();
+            Env env = ctxt.getEnv();
             TruffleFile file = env.getTruffleFile(prefix + suffix);
             try {
-                return env.newSourceBuilder(file).name(basename).mimeType(PythonLanguage.MIME_TYPE).build();
+                return PythonLanguage.newSource(ctxt, file, basename);
             } catch (SecurityException | IOException t) {
                 throw new RuntimeException("Could not read core library from " + file);
             }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java
@@ -500,7 +500,7 @@ private Object evalExpression(String expression, PythonObject globals, PythonObj
                 }
             }
             PythonParser parser = getCore().getParser();
-            Source source = Source.newBuilder(expression).name(name).mimeType(PythonLanguage.MIME_TYPE).build();
+            Source source = PythonLanguage.newSource(getContext(), expression, name);
             RootNode parsed = (RootNode) parser.parse(ParserMode.Eval, getCore(), source, callerFrame);
             return evalNode(parsed, globals, locals, closure);
         }
@@ -533,7 +533,7 @@ Object compile(PBytes source, String filename, String mode, Object kwFlags, Obje
         @Specialization
         @TruffleBoundary
         Object compile(String expression, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize) {
-            Source source = Source.newBuilder(expression).name(filename).mimeType(PythonLanguage.MIME_TYPE).build();
+            Source source = PythonLanguage.newSource(getContext(), expression, filename);
             ParserMode pm;
             if (mode.equals("exec")) {
                 pm = ParserMode.File;
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java
@@ -46,7 +46,6 @@
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.NotImplementedError;
 
 import java.io.IOException;
-import java.net.URI;
 import java.util.List;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.regex.Pattern;
@@ -67,6 +66,7 @@
 import com.oracle.graal.python.nodes.call.special.CallUnaryMethodNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
+import com.oracle.graal.python.runtime.PythonContext;
 import com.oracle.graal.python.runtime.PythonCore;
 import com.oracle.graal.python.runtime.PythonParser.ParserMode;
 import com.oracle.graal.python.runtime.exception.PException;
@@ -340,29 +340,21 @@ public Object run(String path, String modulename) {
         @Specialization
         @TruffleBoundary
         public Object run(String path, PythonModule mod) {
-            Env env = getContext().getEnv();
+            PythonContext ctxt = getContext();
+            Env env = ctxt.getEnv();
             try {
                 String[] pathParts = path.split(Pattern.quote(PythonCore.FILE_SEPARATOR));
                 String fileName = pathParts[pathParts.length - 1];
                 TruffleFile file = env.getTruffleFile(path);
-                Source src = null;
-                try {
-                    if (file.exists()) {
-                        src = env.newSourceBuilder(file).mimeType(PythonLanguage.MIME_TYPE).build();
-                    }
-                } catch (SecurityException e) {
-                }
-                if (src == null) {
-                    src = Source.newBuilder("").uri(URI.create(path)).mimeType(PythonLanguage.MIME_TYPE).name(fileName).build();
-                }
+                Source src = PythonLanguage.newSource(ctxt, file, fileName);
                 RootNode parsedModule = (RootNode) getCore().getParser().parse(ParserMode.File, getCore(), src, null);
                 if (parsedModule != null) {
                     CallTarget callTarget = Truffle.getRuntime().createCallTarget(parsedModule);
                     callTarget.call(PArguments.withGlobals(mod));
                 }
             } catch (PException e) {
                 throw e;
-            } catch (IOException e) {
+            } catch (IOException | SecurityException e) {
                 throw raise(ImportError, e.getMessage());
             }
             return PNone.NONE;
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java
@@ -43,6 +43,9 @@ private PythonOptions() {
     @Option(category = OptionCategory.DEBUG, help = "Defer loading the core lib until after language initialization, so it can be debugged.") //
     public static final OptionKey<Boolean> LazyInit = new OptionKey<>(false);
 
+    @Option(category = OptionCategory.DEBUG, help = "Expose internal sources as normal sources, so they will show up in the debugger and stacks") //
+    public static final OptionKey<Boolean> ExposeInternalSources = new OptionKey<>(false);
+
     @Option(category = OptionCategory.DEBUG, help = "Print the java stacktrace if enabled") //
     public static final OptionKey<Boolean> WithJavaStacktrace = new OptionKey<>(false);
 

Original file line number	Diff line number	Diff line change
`@@ -500,7 +500,7 @@ private Object evalExpression(String expression, PythonObject globals, PythonObj`
`500`	`500`	`}`
`501`	`501`	`}`
`502`	`502`	`PythonParser parser = getCore().getParser();`
`503`		`- Source source = Source.newBuilder(expression).name(name).mimeType(PythonLanguage.MIME_TYPE).build();`
	`503`	`+ Source source = PythonLanguage.newSource(getContext(), expression, name);`
`504`	`504`	`RootNode parsed = (RootNode) parser.parse(ParserMode.Eval, getCore(), source, callerFrame);`
`505`	`505`	`return evalNode(parsed, globals, locals, closure);`
`506`	`506`	`}`
`@@ -533,7 +533,7 @@ Object compile(PBytes source, String filename, String mode, Object kwFlags, Obje`
`533`	`533`	`@Specialization`
`534`	`534`	`@TruffleBoundary`
`535`	`535`	`Object compile(String expression, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize) {`
`536`		`- Source source = Source.newBuilder(expression).name(filename).mimeType(PythonLanguage.MIME_TYPE).build();`
	`536`	`+ Source source = PythonLanguage.newSource(getContext(), expression, filename);`
`537`	`537`	`ParserMode pm;`
`538`	`538`	`if (mode.equals("exec")) {`
`539`	`539`	`pm = ParserMode.File;`