let sys.executable be defined by the launcher, same for the executable_list

Author: timfel

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -39,6 +39,7 @@
 import java.util.Set;
 
 import org.graalvm.launcher.AbstractLanguageLauncher;
+import org.graalvm.nativeimage.ImageInfo;
 import org.graalvm.nativeimage.ProcessProperties;
 import org.graalvm.options.OptionCategory;
 import org.graalvm.polyglot.Context;
@@ -72,9 +73,11 @@ public static void main(String[] args) {
     private boolean runLLI = false;
     private VersionAction versionAction = VersionAction.None;
     private String sulongLibraryPath = null;
+    private List<String> givenArguments;
 
     @Override
-    protected List<String> preprocessArguments(List<String> givenArguments, Map<String, String> polyglotOptions) {
+    protected List<String> preprocessArguments(List<String> givenArgs, Map<String, String> polyglotOptions) {
+        givenArguments = new ArrayList<>(givenArgs);
         ArrayList<String> unrecognized = new ArrayList<>();
         ArrayList<String> inputArgs = new ArrayList<>(getDefaultEnvironmentArgs());
         inputArgs.addAll(givenArguments);
@@ -266,6 +269,43 @@ private static void print(String string) {
         System.out.println(string);
     }
 
+    private static String[] getExecutableList() {
+        if (ImageInfo.inImageCode()) {
+            return new String[]{getExecutable()};
+        } else {
+            StringBuilder sb = new StringBuilder();
+            ArrayList<String> exec_list = new ArrayList<>();
+            sb.append(System.getProperty("java.home")).append(File.separator).append("bin").append(File.separator).append("java");
+            exec_list.add(sb.toString());
+            for (String arg : ManagementFactory.getRuntimeMXBean().getInputArguments()) {
+                if (arg.matches("-Xrunjdwp:transport=dt_socket,server=y,address=\\d+,suspend=y")) {
+                    arg = arg.replace("suspend=y", "suspend=n");
+                }
+                exec_list.add(arg);
+            }
+            exec_list.add("-classpath");
+            exec_list.add(System.getProperty("java.class.path"));
+            exec_list.add(GraalPythonMain.class.getName());
+            return exec_list.toArray(new String[exec_list.size()]);
+        }
+    }
+
+    private static String getExecutable() {
+        if (ImageInfo.inImageRuntimeCode()) {
+            return ProcessProperties.getExecutableName();
+        } else if (ImageInfo.inImageBuildtimeCode()) {
+            return "";
+        } else {
+            // we quote all arguments here, because using sys.executable directly will only work on
+            // a shell, anyways.
+            String[] executableList = getExecutableList();
+            for (int i = 0; i < executableList.length; i++) {
+                executableList[i] = executableList[i].replace("'", "\\'");
+            }
+            return "'" + String.join("' '", executableList) + "'";
+        }
+    }
+
     @Override
     protected void launch(Builder contextBuilder) {
         if (runLLI) {
@@ -289,7 +329,11 @@ protected void launch(Builder contextBuilder) {
             noUserSite = noUserSite || System.getenv("PYTHONNOUSERSITE") != null;
             verboseFlag = verboseFlag || System.getenv("PYTHONVERBOSE") != null;
         }
-        sulongLibraryPath = System.getenv("SULONG_LIBRARY_PATH");
+
+        // The unlikely separator is used because options need to be strings. See
+        // PythonOptions.getExecutableList()
+        contextBuilder.option("python.ExecutableList", String.join("🏆", getExecutableList()));
+        setContextOptionIfUnset(contextBuilder, "python.Executable", getExecutable());
 
         // setting this to make sure our TopLevelExceptionHandler calls the excepthook
         // to print Python exceptions
@@ -302,6 +346,8 @@ protected void launch(Builder contextBuilder) {
         contextBuilder.option("python.QuietFlag", Boolean.toString(quietFlag));
         contextBuilder.option("python.NoUserSiteFlag", Boolean.toString(noUserSite));
         contextBuilder.option("python.NoSiteFlag", Boolean.toString(noSite));
+
+        sulongLibraryPath = System.getenv("SULONG_LIBRARY_PATH");
         if (sulongLibraryPath != null) {
             contextBuilder.option("llvm.libraryPath", sulongLibraryPath);
         }
@@ -358,6 +404,12 @@ protected void launch(Builder contextBuilder) {
         System.exit(rc);
     }
 
+    private void setContextOptionIfUnset(Builder contextBuilder, String key, String value) {
+        if (!givenArguments.contains("--" + key) && System.getProperty("polyglot." + key) == null) {
+            contextBuilder.option(key, value);
+        }
+    }
+
     private static void printFileNotFoundException(NoSuchFileException e) {
         String reason = e.getReason();
         if (reason == null) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java

@@ -51,6 +51,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.graalvm.nativeimage.ImageInfo;
+
 import com.oracle.graal.python.builtins.Builtin;
 import com.oracle.graal.python.builtins.CoreFunctions;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
@@ -59,9 +61,7 @@
 import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
 import com.oracle.graal.python.builtins.objects.bytes.PBytes;
 import com.oracle.graal.python.builtins.objects.list.PList;
-import com.oracle.graal.python.builtins.objects.module.PythonModule;
 import com.oracle.graal.python.builtins.objects.str.PString;
-import com.oracle.graal.python.nodes.SpecialAttributeNames;
 import com.oracle.graal.python.nodes.expression.CastToBooleanNode;
 import com.oracle.graal.python.nodes.expression.CastToListNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
@@ -70,8 +70,8 @@
 import com.oracle.graal.python.nodes.util.CastToStringNode;
 import com.oracle.graal.python.runtime.PosixResources;
 import com.oracle.graal.python.runtime.PythonContext;
+import com.oracle.graal.python.runtime.PythonOptions;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
-import com.oracle.truffle.api.TruffleOptions;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
@@ -119,13 +119,13 @@ synchronized int forkExec(PList args, @SuppressWarnings("unused") PList execList
             // TODO: fix this better? sys.executable is often used in subprocess tests, but on Java
             // that actually gives you a whole cmdline, which we need to split up for process
             // builder
-            PythonModule sysModule = getCore().lookupBuiltinModule("sys");
-            if (!TruffleOptions.AOT && !argStrings.isEmpty() && argStrings.get(0).equals(sysModule.getAttribute("executable"))) {
-                PList exec_list = (PList) sysModule.getAttribute(SpecialAttributeNames.GRAAL_PYTHON_EXECUTABLE_LIST);
-                Object[] internalArray = exec_list.getSequenceStorage().getCopyOfInternalArray();
-                argStrings.remove(0);
-                for (int i = internalArray.length - 1; i >= 0; i--) {
-                    argStrings.add(0, (String) internalArray[i]);
+            if (!ImageInfo.inImageCode() && !argStrings.isEmpty()) {
+                if (argStrings.get(0).equals(PythonOptions.getOption(context, PythonOptions.Executable))) {
+                    String[] executableList = PythonOptions.getExecutableList();
+                    argStrings.remove(0);
+                    for (int i = executableList.length - 1; i >= 0; i--) {
+                        argStrings.add(0, executableList[i]);
+                    }
                 }
             }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java

@@ -45,10 +45,8 @@
 import static com.oracle.graal.python.nodes.SpecialMethodNames.__SIZEOF__;
 
 import java.io.IOException;
-import java.lang.management.ManagementFactory;
 import java.nio.ByteOrder;
 import java.nio.charset.Charset;
-import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
@@ -60,7 +58,6 @@
 import com.oracle.graal.python.builtins.objects.exception.PBaseException;
 import com.oracle.graal.python.builtins.objects.ints.PInt;
 import com.oracle.graal.python.builtins.objects.str.PString;
-import com.oracle.graal.python.nodes.SpecialAttributeNames;
 import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
 import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode.NoAttributeHandler;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
@@ -113,43 +110,6 @@ public void initialize(PythonCore core) {
         builtinConstants.put("byteorder", ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? "little" : "big");
         builtinConstants.put("copyright", LICENSE);
         builtinConstants.put("dont_write_bytecode", true);
-
-        String executable = PythonOptions.getOption(core.getContext(), PythonOptions.ExecutablePath);
-        if (TruffleOptions.AOT || !core.getContext().isExecutableAccessAllowed()) {
-            // cannot set the path at this time since the binary is not yet known; will be patched
-            // in the context
-            builtinConstants.put("executable", PNone.NONE);
-            builtinConstants.put(SpecialAttributeNames.GRAAL_PYTHON_EXECUTABLE_LIST, PNone.NONE);
-        } else {
-            StringBuilder sb = new StringBuilder();
-            ArrayList<String> exec_list = new ArrayList<>();
-            sb.append(System.getProperty("java.home")).append(PythonCore.FILE_SEPARATOR).append("bin").append(PythonCore.FILE_SEPARATOR).append("java");
-            exec_list.add(sb.toString());
-            sb.append(' ');
-            for (String arg : ManagementFactory.getRuntimeMXBean().getInputArguments()) {
-                if (arg.matches("-Xrunjdwp:transport=dt_socket,server=y,address=\\d+,suspend=y")) {
-                    arg = arg.replace("suspend=y", "suspend=n");
-                }
-                sb.append(arg).append(' ');
-                exec_list.add(arg);
-            }
-            sb.append("-classpath ");
-            exec_list.add("-classpath");
-            sb.append(System.getProperty("java.class.path")).append(' ');
-            exec_list.add(System.getProperty("java.class.path"));
-            // we really don't care what the main class or its arguments were - this should
-            // always help us launch Graal.Python
-            sb.append("com.oracle.graal.python.shell.GraalPythonMain");
-            exec_list.add("com.oracle.graal.python.shell.GraalPythonMain");
-            builtinConstants.put("executable", sb.toString());
-            builtinConstants.put(SpecialAttributeNames.GRAAL_PYTHON_EXECUTABLE_LIST, core.factory().createList(exec_list.toArray()));
-        }
-
-        String executable = PythonOptions.getOption(core.getContext(), PythonOptions.ExecutablePath);
-        if (!executable.isEmpty() && core.getContext().isExecutableAccessAllowed()) {
-            builtinConstants.put("executable", executable);
-        }
-
         builtinConstants.put("modules", core.factory().createDict());
         builtinConstants.put("path", core.factory().createList());
         builtinConstants.put("builtin_module_names", core.factory().createTuple(core.builtinModuleNames()));

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/SpecialAttributeNames.java

@@ -72,7 +72,4 @@ public abstract class SpecialAttributeNames {
     public static final String __DICTOFFSET__ = "__dictoffset__";
     public static final String __ITEMSIZE__ = "__itemsize__";
     public static final String __WEAKREF__ = "__weakref__";
-
-    // GraalPython specific
-    public static final String GRAAL_PYTHON_EXECUTABLE_LIST = "graal_python_executable_list";
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

@@ -36,18 +36,14 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.graalvm.nativeimage.ImageInfo;
-import org.graalvm.nativeimage.ProcessProperties;
 import org.graalvm.options.OptionValues;
 
 import com.oracle.graal.python.PythonLanguage;
-import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.bytes.OpaqueBytes;
 import com.oracle.graal.python.builtins.objects.cext.NativeWrappers.PThreadState;
 import com.oracle.graal.python.builtins.objects.common.HashingStorage;
 import com.oracle.graal.python.builtins.objects.dict.PDict;
 import com.oracle.graal.python.builtins.objects.module.PythonModule;
-import com.oracle.graal.python.nodes.SpecialAttributeNames;
 import com.oracle.graal.python.runtime.exception.PException;
 import com.oracle.truffle.api.Assumption;
 import com.oracle.truffle.api.CallTarget;
@@ -238,12 +234,7 @@ public void patch(Env newEnv) {
 
     private void setupRuntimeInformation() {
         PythonModule sysModule = core.initializeSysModule();
-        if (ImageInfo.inImageRuntimeCode() && isExecutableAccessAllowed()) {
-            if (sysModule.getAttribute("executable") == PNone.NONE) {
-                sysModule.setAttribute("executable", ProcessProperties.getExecutableName());
-            }
-            sysModule.setAttribute(SpecialAttributeNames.GRAAL_PYTHON_EXECUTABLE_LIST, core.factory().createList(new Object[]{ProcessProperties.getExecutableName()}));
-        }
+        sysModule.setAttribute("executable", PythonOptions.getOption(core.getContext(), PythonOptions.Executable));
         sysModules = (PDict) sysModule.getAttribute("modules");
         builtinsModule = (PythonModule) sysModules.getItem("builtins");
         mainModule = core.factory().createPythonModule(__MAIN__);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

@@ -35,6 +35,7 @@
 
 @Option.Group(PythonLanguage.ID)
 public final class PythonOptions {
+    private static final String EXECUTABLE_LIST_SEPARATOR = "🏆";
 
     private PythonOptions() {
         // no instances
@@ -128,8 +129,11 @@ private PythonOptions() {
     @Option(category = OptionCategory.EXPERT, help = "Set by the launcher to the terminal height.") //
     public static final OptionKey<Integer> TerminalHeight = new OptionKey<>(25);
 
-    @Option(category = OptionCategory.EXPERT, help = "The sys.executable path") //
-    public static final OptionKey<String> ExecutablePath = new OptionKey<>("");
+    @Option(category = OptionCategory.EXPERT, help = "The sys.executable path. Set by the launcher, but can may need to be overridden in certain special situations.") //
+    public static final OptionKey<String> Executable = new OptionKey<>("");
+
+    @Option(category = OptionCategory.EXPERT, help = "The executed command list as string joined by the executable list separator char. This must always correspond to the real, valid command list used to run GraalPython.") //
+    public static final OptionKey<String> ExecutableList = new OptionKey<>("");
 
     public static OptionDescriptors createDescriptors() {
         return new PythonOptionsOptionDescriptors();
@@ -190,4 +194,9 @@ public static int getTerminalHeight() {
     public static int getTerminalWidth() {
         return getOption(PythonLanguage.getContextRef().get(), TerminalWidth);
     }
+
+    @TruffleBoundary
+    public static String[] getExecutableList() {
+        return getOption(PythonLanguage.getContextRef().get(), ExecutableList).split(EXECUTABLE_LIST_SEPARATOR);
+    }
 }

graalpython/lib-python/3/venv/__init__.py

@@ -48,7 +48,7 @@ def __init__(self, system_site_packages=False, clear=False,
         self.with_pip = with_pip
         # Truffle change
         if with_pip:
-            logger.warning("Not install pip on GraalPython")
+            logger.warning("We cannot install pip on Graal Python, yet")
         self.with_pip = False
         # End Truffle change
         self.prompt = prompt
@@ -119,45 +119,37 @@ def create_if_needed(d):
         context.python_dir = dirname
         context.python_exe = exename
 
-        # Truffle change: when our executable is not just a file (e.g. we're
-        # running through java), we need to provide a script for launching
-        if hasattr(sys, "executable_list"):
-            # our 'sys.executable' is really a full commandline, so we need a
-            # script to launch it
-            import atexit, tempfile
-            tempdir = tempfile.mkdtemp()
-            script = os.path.join(tempdir, "graalpython")
-            if sys.platform == 'win32':
-                script += ".bat"
-
-            with open(script, "w") as f:
-                if sys.platform != "win32":
-                    f.write("#!/bin/sh\n")
-                for s in sys.executable_list:
-                    if " " in s:
-                        f.write('"')
-                        f.write(s.replace('"', '\\"'))
-                        f.write('" ')
-                    else:
-                        f.write(s)
-                        f.write(" ")
-                if sys.platform == "win32":
-                    f.write("%*")
-                else:
-                    f.write("$@")
+        # Truffle change: our executable may not just be a file (e.g. we're
+        # running through java), we always provide a script for launching in
+        # venv
+        import atexit, tempfile
+        tempdir = tempfile.mkdtemp()
+        script = os.path.join(tempdir, "graalpython")
+        if sys.platform == 'win32':
+            script += ".bat"
 
+        with open(script, "w") as f:
             if sys.platform != "win32":
-                os.chmod(script, 0o777)
+                f.write("#!/bin/sh\n")
+            f.write(sys.executable)
+            if sys.platform == "win32":
+                f.write(" %*")
+            else:
+                f.write(" $@")
+
+        if sys.platform != "win32":
+            os.chmod(script, 0o777)
 
-            atexit.register(lambda: os.unlink(script))
-            atexit.register(lambda: os.rmdir(tempdir))
+        atexit.register(lambda: os.unlink(script))
+        atexit.register(lambda: os.rmdir(tempdir))
 
-            dirname, exename = context.python_dir, context.python_exe = sys.graal_python_home, "graalpython"
-            context.executable = script
+        dirname = context.python_dir = sys.graal_python_home
+        exename = context.python_exe = "graalpython"
+        context.executable = script
 
-            if self.symlinks:
-                logger.warning("We're not using symlinks on GraalPython")
-            self.symlinks = False
+        if self.symlinks:
+            logger.warning("We're not using symlinks in a Graal Python venv")
+        self.symlinks = False
         # End of Truffle change
 
         if sys.platform == 'win32':
@@ -335,7 +327,7 @@ def replace_variables(self, text, context):
         # Truffle change: we need to set some extra options for the launcher to work
         text = text.replace(
             '__VENV_GRAAL_PYTHON_OPTIONS__',
-            "--python.CoreHome='%s' --python.StdLibHome='%s' --python.ExecutablePath='%s'" % (
+            "--python.CoreHome='%s' --python.StdLibHome='%s' --python.Executable='%s'" % (
                 sys.graal_python_core_home,
                 sys.graal_python_stdlib_home,
                 context.env_exe,