Skip to content

Commit 3fd37d7

Browse files
msimacekmsimacek
committed
[GR-56001][GR-56002][GR-56003][GR-56694] Fixes for virtualenv
PullRequest: graalpython/3421
2 parents d7ad467 + ee186b0 commit 3fd37d7

File tree

9 files changed

+130
-106
lines changed

9 files changed

+130
-106
lines changed

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

Lines changed: 48 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
import java.io.File;
3131
import java.io.FileDescriptor;
3232
import java.io.FileOutputStream;
33-
import java.io.IOError;
3433
import java.io.IOException;
3534
import java.io.InputStream;
3635
import java.io.OutputStream;
@@ -717,8 +716,7 @@ protected void launch(Builder contextBuilder) {
717716
// the user did not explicitly pass some options that would be otherwise loaded from
718717
// pyvenv.cfg. Notable usage of this feature is GraalPython venvs which generate a
719718
// launcher script that passes those options explicitly without relying on pyvenv.cfg
720-
boolean tryVenvCfg = !hasContextOptionSetViaCommandLine("SysPrefix") &&
721-
!hasContextOptionSetViaCommandLine("PythonHome") &&
719+
boolean tryVenvCfg = !hasContextOptionSetViaCommandLine("PythonHome") &&
722720
getEnv("GRAAL_PYTHONHOME") == null;
723721
if (tryVenvCfg) {
724722
findAndApplyVenvCfg(contextBuilder, executable);
@@ -872,14 +870,16 @@ private static String toAbsolutePath(String executable) {
872870
return Paths.get(executable).toAbsolutePath().toString();
873871
}
874872

873+
// Rough equivalent of CPython's pyvenv.cfg logic in Modules/getpath.py
875874
private void findAndApplyVenvCfg(Builder contextBuilder, String executable) {
876-
Path binDir;
875+
Path executablePath;
877876
try {
878-
binDir = Paths.get(executable).getParent();
877+
executablePath = Paths.get(executable);
879878
} catch (InvalidPathException e) {
880-
log("cannot determine the parent directory of the executable");
879+
log("cannot determine path of the executable");
881880
return;
882881
}
882+
Path binDir = executablePath.getParent();
883883
if (binDir == null) {
884884
log("parent directory of the executable does not exist");
885885
return;
@@ -907,36 +907,52 @@ private void findAndApplyVenvCfg(Builder contextBuilder, String executable) {
907907
}
908908
String name = parts[0].trim();
909909
if (name.equals("home")) {
910-
Path homeProperty = Paths.get(parts[1].trim());
911-
/*
912-
* (tfel): According to PEP 405, the home key is the directory of the Python
913-
* executable from which this virtual environment was created, that is, it
914-
* usually ends with "/bin" on a Unix system. On Windows, the base Python should
915-
* be in the top-level directory or under "\Scripts". To support running from
916-
* Maven artifacts where we don't have a working executable, we patched our
917-
* shipped venv module to set the home path without a "/bin" or "\\Scripts"
918-
* suffix, so we explicitly check for those two subfolder cases and otherwise
919-
* assume the home key is directly pointing to the Python home.
920-
*/
921-
if (homeProperty.endsWith("bin") || homeProperty.endsWith("Scripts")) {
922-
homeProperty = homeProperty.getParent();
923-
}
924910
try {
925-
contextBuilder.option("python.PythonHome", homeProperty.toString());
926-
} catch (NullPointerException ex) {
911+
Path homeProperty = Paths.get(parts[1].trim());
912+
Path graalpyHome = homeProperty;
913+
/*
914+
* (tfel): According to PEP 405, the home key is the directory of the Python
915+
* executable from which this virtual environment was created, that is, it
916+
* usually ends with "/bin" on a Unix system. On Windows, the base Python
917+
* should be in the top-level directory or under "\Scripts". To support
918+
* running from Maven artifacts where we don't have a working executable, we
919+
* patched our shipped venv module to set the home path without a "/bin" or
920+
* "\\Scripts" suffix, so we explicitly check for those two subfolder cases
921+
* and otherwise assume the home key is directly pointing to the Python
922+
* home.
923+
*/
924+
if (graalpyHome.endsWith("bin") || graalpyHome.endsWith("Scripts")) {
925+
graalpyHome = graalpyHome.getParent();
926+
}
927+
contextBuilder.option("python.PythonHome", graalpyHome.toString());
928+
/*
929+
* First try to resolve symlinked executables, since that may be more
930+
* accurate than assuming the executable in 'home'.
931+
*/
932+
Path baseExecutable = null;
933+
try {
934+
Path realPath = executablePath.toRealPath();
935+
if (!realPath.equals(executablePath.toAbsolutePath())) {
936+
baseExecutable = realPath;
937+
}
938+
} catch (IOException ex) {
939+
// Ignore
940+
}
941+
if (baseExecutable == null) {
942+
baseExecutable = homeProperty.resolve(executablePath.getFileName());
943+
}
944+
if (Files.exists(baseExecutable)) {
945+
contextBuilder.option("python.BaseExecutable", baseExecutable.toString());
946+
/*
947+
* This is needed to support the legacy GraalVM layout where the
948+
* executable is a symlink into the 'languages' directory.
949+
*/
950+
contextBuilder.option("python.PythonHome", baseExecutable.getParent().getParent().toString());
951+
}
952+
} catch (NullPointerException | InvalidPathException ex) {
927953
// NullPointerException covers the possible null result of getParent()
928954
warn("Could not set PYTHONHOME according to the pyvenv.cfg file.");
929955
}
930-
String sysPrefix = null;
931-
try {
932-
sysPrefix = venvCfg.getParent().toAbsolutePath().toString();
933-
} catch (IOError | NullPointerException ex) {
934-
// NullPointerException covers the possible null result of getParent()
935-
warn("Could not set the sys.prefix according to the pyvenv.cfg file.");
936-
}
937-
if (sysPrefix != null) {
938-
contextBuilder.option("python.SysPrefix", sysPrefix);
939-
}
940956
break;
941957
}
942958
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@
5454
import static com.oracle.graal.python.builtins.PythonOS.PLATFORM_WIN32;
5555
import static com.oracle.graal.python.builtins.PythonOS.getPythonOS;
5656
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_BUFFER;
57-
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_CLOSE;
5857
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_ENCODING;
5958
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_MODE;
6059
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_R;
@@ -656,8 +655,10 @@ public void postInitialize0(Python3Core core) {
656655
TruffleString capiHome = context.getCAPIHome();
657656

658657
if (!ImageInfo.inImageBuildtimeCode()) {
659-
sys.setAttribute(tsLiteral("executable"), context.getOption(PythonOptions.Executable));
660-
sys.setAttribute(tsLiteral("_base_executable"), context.getOption(PythonOptions.Executable));
658+
TruffleString executable = context.getOption(PythonOptions.Executable);
659+
TruffleString baseExecutable = context.getOption(PythonOptions.BaseExecutable);
660+
sys.setAttribute(tsLiteral("executable"), executable);
661+
sys.setAttribute(tsLiteral("_base_executable"), baseExecutable.isEmpty() ? executable : baseExecutable);
661662
}
662663
sys.setAttribute(tsLiteral("dont_write_bytecode"), context.getOption(PythonOptions.DontWriteBytecodeFlag));
663664
TruffleString pycachePrefix = context.getOption(PythonOptions.PyCachePrefix);
@@ -790,18 +791,12 @@ public void initStd(Python3Core core) {
790791
PBuffered writer = factory.createBufferedWriter(PythonBuiltinClassType.PBufferedWriter);
791792
BufferedWriterBuiltins.BufferedWriterInit.internalInit(writer, (PFileIO) getBuiltinConstant(T_STDOUT), BufferedReaderBuiltins.DEFAULT_BUFFER_SIZE, factory, posixSupport,
792793
posixLib);
793-
PTextIO stdout = setWrapper(T_STDOUT, T___STDOUT__, T_W, stdioEncoding, stdioError, writer, sysModule, factory);
794+
setWrapper(T_STDOUT, T___STDOUT__, T_W, stdioEncoding, stdioError, writer, sysModule, factory);
794795

795796
writer = factory.createBufferedWriter(PythonBuiltinClassType.PBufferedWriter);
796797
BufferedWriterBuiltins.BufferedWriterInit.internalInit(writer, (PFileIO) getBuiltinConstant(T_STDERR), BufferedReaderBuiltins.DEFAULT_BUFFER_SIZE, factory, posixSupport,
797798
posixLib);
798-
PTextIO stderr = setWrapper(T_STDERR, T___STDERR__, T_W, stdioEncoding, T_BACKSLASHREPLACE, writer, sysModule, factory);
799-
800-
// register atexit close std out/err
801-
core.getContext().registerAtexitHook((ctx) -> {
802-
callClose(stdout);
803-
callClose(stderr);
804-
});
799+
setWrapper(T_STDERR, T___STDERR__, T_W, stdioEncoding, T_BACKSLASHREPLACE, writer, sysModule, factory);
805800
}
806801

807802
private static PTextIO setWrapper(TruffleString name, TruffleString specialName, TruffleString mode, TruffleString encoding, TruffleString error, PBuffered buffered, PythonModule sysModule,
@@ -820,13 +815,6 @@ private static void setAttribute(PythonObject obj, TruffleString key, Object val
820815
obj.setAttribute(key, value);
821816
}
822817

823-
private static void callClose(Object obj) {
824-
try {
825-
PyObjectCallMethodObjArgs.executeUncached(obj, T_CLOSE);
826-
} catch (PException e) {
827-
}
828-
}
829-
830818
public PDict getModules() {
831819
return (PDict) getBuiltinConstant(T_MODULES);
832820
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TimeModuleBuiltins.java

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,11 @@ public final class TimeModuleBuiltins extends PythonBuiltins {
149149
"offset from UTC in seconds"
150150
});
151151

152+
public static final TruffleString T_TZNAME = tsLiteral("tzname");
153+
public static final TruffleString T_DAYLIGHT = tsLiteral("daylight");
154+
public static final TruffleString T_TIMEZONE = tsLiteral("timezone");
155+
public static final TruffleString T_ALTZONE = tsLiteral("altzone");
156+
152157
@Override
153158
protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
154159
return TimeModuleBuiltinsFactory.getFactories();
@@ -157,30 +162,36 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
157162
@Override
158163
public void initialize(Python3Core core) {
159164
super.initialize(core);
165+
StructSequence.initType(core, STRUCT_TIME_DESC);
166+
addBuiltinConstant("_STRUCT_TM_ITEMS", 11);
167+
}
168+
169+
@Override
170+
public void postInitialize(Python3Core core) {
171+
super.postInitialize(core);
160172
// Should we read TZ env variable?
161173
ZoneId defaultZoneId = core.getContext().getEnv().getTimeZone();
162174
ModuleState moduleState = new ModuleState();
163175
moduleState.currentZoneId = defaultZoneId;
164176
moduleState.timeSlept = 0;
165-
core.lookupBuiltinModule(T_TIME).setModuleState(moduleState);
177+
PythonModule timeModule = core.lookupBuiltinModule(T_TIME);
178+
timeModule.setModuleState(moduleState);
166179

167180
TimeZone defaultTimeZone = TimeZone.getTimeZone(defaultZoneId);
168181
TruffleString noDaylightSavingZone = toTruffleStringUncached(defaultTimeZone.getDisplayName(false, TimeZone.SHORT));
169182
TruffleString daylightSavingZone = toTruffleStringUncached(defaultTimeZone.getDisplayName(true, TimeZone.SHORT));
170183

171184
boolean hasDaylightSaving = !noDaylightSavingZone.equalsUncached(daylightSavingZone, TS_ENCODING);
172185
if (hasDaylightSaving) {
173-
addBuiltinConstant("tzname", core.factory().createTuple(new Object[]{noDaylightSavingZone, daylightSavingZone}));
186+
timeModule.setAttribute(T_TZNAME, core.factory().createTuple(new Object[]{noDaylightSavingZone, daylightSavingZone}));
174187
} else {
175-
addBuiltinConstant("tzname", core.factory().createTuple(new Object[]{noDaylightSavingZone}));
188+
timeModule.setAttribute(T_TZNAME, core.factory().createTuple(new Object[]{noDaylightSavingZone}));
176189
}
177190

178-
addBuiltinConstant("daylight", PInt.intValue(hasDaylightSaving));
191+
timeModule.setAttribute(T_DAYLIGHT, PInt.intValue(hasDaylightSaving));
179192
int rawOffsetSeconds = defaultTimeZone.getRawOffset() / -1000;
180-
addBuiltinConstant("timezone", rawOffsetSeconds);
181-
addBuiltinConstant("altzone", rawOffsetSeconds - 3600);
182-
addBuiltinConstant("_STRUCT_TM_ITEMS", 11);
183-
StructSequence.initType(core, STRUCT_TIME_DESC);
193+
timeModule.setAttribute(T_TIMEZONE, rawOffsetSeconds);
194+
timeModule.setAttribute(T_ALTZONE, rawOffsetSeconds - 3600);
184195
}
185196

186197
@TruffleBoundary

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,13 @@
3030
import static com.oracle.graal.python.builtins.PythonOS.getPythonOS;
3131
import static com.oracle.graal.python.builtins.modules.SysModuleBuiltins.T_CACHE_TAG;
3232
import static com.oracle.graal.python.builtins.modules.SysModuleBuiltins.T__MULTIARCH;
33+
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_CLOSED;
34+
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_FLUSH;
3335
import static com.oracle.graal.python.builtins.objects.str.StringUtils.cat;
3436
import static com.oracle.graal.python.builtins.objects.thread.PThread.GRAALPYTHON_THREADS;
3537
import static com.oracle.graal.python.nodes.BuiltinNames.T_SHA3;
38+
import static com.oracle.graal.python.nodes.BuiltinNames.T_STDERR;
39+
import static com.oracle.graal.python.nodes.BuiltinNames.T_STDOUT;
3640
import static com.oracle.graal.python.nodes.BuiltinNames.T_SYS;
3741
import static com.oracle.graal.python.nodes.BuiltinNames.T_THREADING;
3842
import static com.oracle.graal.python.nodes.BuiltinNames.T___BUILTINS__;
@@ -145,10 +149,12 @@
145149
import com.oracle.graal.python.compiler.CodeUnit;
146150
import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs;
147151
import com.oracle.graal.python.lib.PyObjectGetAttr;
152+
import com.oracle.graal.python.lib.PyObjectIsTrueNode;
148153
import com.oracle.graal.python.nodes.ErrorMessages;
149154
import com.oracle.graal.python.nodes.PRaiseNode;
150155
import com.oracle.graal.python.nodes.SpecialAttributeNames;
151156
import com.oracle.graal.python.nodes.SpecialMethodNames;
157+
import com.oracle.graal.python.nodes.WriteUnraisableNode;
152158
import com.oracle.graal.python.nodes.attributes.ReadAttributeFromObjectNode;
153159
import com.oracle.graal.python.nodes.call.CallNode;
154160
import com.oracle.graal.python.nodes.object.SetDictNode;
@@ -1526,19 +1532,24 @@ public void patch(Env newEnv) {
15261532
}
15271533

15281534
private void importSiteIfForced() {
1529-
if (getOption(PythonOptions.ForceImportSite)) {
1530-
AbstractImportNode.importModule(T_SITE);
1531-
}
1532-
if (!getOption(PythonOptions.WarnOptions).isEmpty()) {
1533-
// we must force an import of the warnings module here if warnings were passed
1534-
AbstractImportNode.importModule(T_WARNINGS);
1535-
}
1536-
if (getOption(PythonOptions.InputFilePath).isEmpty()) {
1537-
// When InputFilePath is set, this is handled by __graalpython__.run_path
1538-
addSysPath0();
1539-
}
1540-
if (getOption(PythonOptions.SetupLLVMLibraryPaths)) {
1541-
ImpModuleBuiltins.importFrozenModuleObject(this, toTruffleStringUncached("graalpy.sulong_support"), false);
1535+
try {
1536+
if (getOption(PythonOptions.ForceImportSite)) {
1537+
AbstractImportNode.importModule(T_SITE);
1538+
}
1539+
if (!getOption(PythonOptions.WarnOptions).isEmpty()) {
1540+
// we must force an import of the warnings module here if warnings were passed
1541+
AbstractImportNode.importModule(T_WARNINGS);
1542+
}
1543+
if (getOption(PythonOptions.InputFilePath).isEmpty()) {
1544+
// When InputFilePath is set, this is handled by __graalpython__.run_path
1545+
addSysPath0();
1546+
}
1547+
if (getOption(PythonOptions.SetupLLVMLibraryPaths)) {
1548+
ImpModuleBuiltins.importFrozenModuleObject(this, toTruffleStringUncached("graalpy.sulong_support"), false);
1549+
}
1550+
} catch (PException e) {
1551+
flushStdFiles();
1552+
throw e;
15421553
}
15431554
}
15441555

@@ -2013,6 +2024,7 @@ public void finalizeContext() {
20132024
finalizing = true;
20142025
// interrupt and join or kill python threads
20152026
joinThreads();
2027+
flushStdFiles();
20162028
if (cApiContext != null) {
20172029
cApiContext.finalizeCApi();
20182030
}
@@ -2029,6 +2041,34 @@ public void finalizeContext() {
20292041
mainThread = null;
20302042
}
20312043

2044+
// Equivalent of CPython's flush_std_files
2045+
@TruffleBoundary
2046+
public void flushStdFiles() {
2047+
PythonModule sysModule = getSysModule();
2048+
flushFile(sysModule.getAttribute(T_STDOUT), true);
2049+
flushFile(sysModule.getAttribute(T_STDERR), false);
2050+
}
2051+
2052+
private static void flushFile(Object file, boolean useWriteUnraisable) {
2053+
if (!(file instanceof PNone)) {
2054+
boolean closed = false;
2055+
try {
2056+
closed = PyObjectIsTrueNode.executeUncached(PyObjectGetAttr.executeUncached(file, T_CLOSED));
2057+
} catch (PException e) {
2058+
// Ignore
2059+
}
2060+
if (!closed) {
2061+
try {
2062+
PyObjectCallMethodObjArgs.executeUncached(file, T_FLUSH);
2063+
} catch (PException e) {
2064+
if (useWriteUnraisable) {
2065+
WriteUnraisableNode.getUncached().execute(e.getEscapedException(), null, null);
2066+
}
2067+
}
2068+
}
2069+
}
2070+
}
2071+
20322072
@TruffleBoundary
20332073
public int getAtexitHookCount() {
20342074
return atExitHooks.size();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,9 +315,12 @@ private PythonOptions() {
315315
@Option(category = OptionCategory.EXPERT, usageSyntax = "<height>", help = "Set by the launcher to the terminal height.") //
316316
public static final OptionKey<Integer> TerminalHeight = new OptionKey<>(25);
317317

318-
@Option(category = OptionCategory.EXPERT, usageSyntax = "<path>", help = "The sys.executable path. Set by the launcher, but can may need to be overridden in certain special situations.", stability = OptionStability.STABLE) //
318+
@Option(category = OptionCategory.EXPERT, usageSyntax = "<path>", help = "The sys.executable path. Set by the launcher, but may need to be overridden in certain special situations.", stability = OptionStability.STABLE) //
319319
public static final OptionKey<TruffleString> Executable = new OptionKey<>(T_EMPTY_STRING, TS_OPTION_TYPE);
320320

321+
@Option(category = OptionCategory.EXPERT, usageSyntax = "<path>", help = "The sys._base_executable path. Set by the launcher, but may need to be overridden in certain special situations.", stability = OptionStability.STABLE) //
322+
public static final OptionKey<TruffleString> BaseExecutable = new OptionKey<>(T_EMPTY_STRING, TS_OPTION_TYPE);
323+
321324
@Option(category = OptionCategory.EXPERT, usageSyntax = "<cmdPart>[" + J_STRING_LIST_SEPARATOR +
322325
"<cmdPart>]", help = "The executed command list as string joined by the executable list separator char. This must always correspond to the real, valid command list used to run GraalPy.") //
323326
public static final OptionKey<TruffleString> ExecutableList = new OptionKey<>(T_EMPTY_STRING, TS_OPTION_TYPE);

0 commit comments

Comments
 (0)