Skip to content

Commit 07d524a

Browse files
OracleLabsAutomationOracleLabsAutomation
committed
[GR-64176] Backport to 24.2: Fixes for windows issues in cibuildwheels.
PullRequest: graalpython/3774
2 parents eda145b + 25e0fc6 commit 07d524a

File tree

16 files changed

+403
-135
lines changed

16 files changed

+403
-135
lines changed

‎graalpython/com.oracle.graal.python.frozen/freeze_modules.py

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2021, 2024, Oracle and/or its affiliates.
1+
# Copyright (c) 2021, 2025, Oracle and/or its affiliates.
22
# Copyright (C) 1996-2020 Python Software Foundation
33
#
44
# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -105,20 +105,23 @@ def add_graalpython_core():
105105
l.append("polyglot.arrow : polyglot.arrow = " + os.path.join(lib_graalpython, "modules/_polyglot_arrow.py"))
106106
for name in [
107107
"modules/_sysconfigdata",
108+
"modules/_polyglot",
109+
"modules/_polyglot_datetime",
110+
"modules/_polyglot_time",
108111
]:
109112
modname = os.path.basename(name)
110113
modpath = os.path.join(lib_graalpython, f"{name}.py")
111114
l.append(f"{modname} : {modname} = {modpath}")
112115
for name in [
113116
"__graalpython__",
114-
"_polyglot",
115117
"_sre",
116118
"_sysconfig",
117119
"_weakref",
118120
"builtins",
119121
"java",
120122
"pip_hook",
121123
"unicodedata",
124+
"_nt",
122125
]:
123126
modname = f"graalpy.{os.path.basename(name)}"
124127
modpath = os.path.join(lib_graalpython, f"{name}.py")
@@ -495,7 +498,7 @@ def lower_camel_case(str):
495498
# write frozen files
496499

497500
FROZEN_MODULES_HEADER = """/*
498-
* Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
501+
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
499502
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
500503
*
501504
* The Universal Permissive License (UPL), Version 1.0
@@ -588,19 +591,27 @@ def write_frozen_module_file(file, modules):
588591
if os.path.exists(file):
589592
with open(file, "r", encoding="utf-8", newline=os.linesep) as f:
590593
content = f.read()
594+
if os.linesep != "\n":
595+
if content.replace(os.linesep, "\n") == content:
596+
# Windows file has Unix line endings
597+
linesep = "\n"
598+
else:
599+
linesep = os.linesep
600+
else:
601+
linesep = "\n"
591602
stat_result = os.stat(file)
592603
atime, mtime = stat_result.st_atime, stat_result.st_mtime
593604
else:
594605
content = None
595606
os.makedirs(os.path.dirname(file), exist_ok=True)
596-
with open(file, "w", encoding="utf-8", newline=os.linesep) as out_file:
607+
with open(file, "w", encoding="utf-8", newline=linesep) as out_file:
597608
out_file.write(FROZEN_MODULES_HEADER)
598609
out_file.write("\n\n")
599610
write_frozen_modules_map(out_file, modules)
600611
out_file.write("\n")
601612
write_frozen_lookup(out_file, modules)
602613
out_file.write("}\n")
603-
with open(file, "r", encoding="utf-8", newline=os.linesep) as f:
614+
with open(file, "r", encoding="utf-8", newline=linesep) as f:
604615
new_content = f.read()
605616
if new_content == content:
606617
# set mtime to the old one, if we didn't change anything

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

Lines changed: 59 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -778,9 +778,6 @@ protected void launch(Builder contextBuilder) {
778778
contextBuilder.option("python.IntMaxStrDigits", Integer.toString(intMaxStrDigits));
779779
}
780780
contextBuilder.option("python.DontWriteBytecodeFlag", Boolean.toString(dontWriteBytecode));
781-
if (verboseFlag) {
782-
contextBuilder.option("log.python.level", "INFO");
783-
}
784781
contextBuilder.option("python.QuietFlag", Boolean.toString(quietFlag));
785782
contextBuilder.option("python.NoUserSiteFlag", Boolean.toString(noUserSite));
786783
contextBuilder.option("python.NoSiteFlag", Boolean.toString(noSite));
@@ -931,54 +928,70 @@ private void findAndApplyVenvCfg(Builder contextBuilder, String executable) {
931928
continue;
932929
}
933930
String name = parts[0].trim();
934-
if (name.equals("home")) {
935-
try {
936-
Path homeProperty = Paths.get(parts[1].trim());
937-
Path graalpyHome = homeProperty;
938-
/*
939-
* (tfel): According to PEP 405, the home key is the directory of the Python
940-
* executable from which this virtual environment was created, that is, it
941-
* usually ends with "/bin" on a Unix system. On Windows, the base Python
942-
* should be in the top-level directory or under "\Scripts". To support
943-
* running from Maven artifacts where we don't have a working executable, we
944-
* patched our shipped venv module to set the home path without a "/bin" or
945-
* "\\Scripts" suffix, so we explicitly check for those two subfolder cases
946-
* and otherwise assume the home key is directly pointing to the Python
947-
* home.
948-
*/
949-
if (graalpyHome.endsWith("bin") || graalpyHome.endsWith("Scripts")) {
950-
graalpyHome = graalpyHome.getParent();
951-
}
952-
contextBuilder.option("python.PythonHome", graalpyHome.toString());
953-
/*
954-
* First try to resolve symlinked executables, since that may be more
955-
* accurate than assuming the executable in 'home'.
956-
*/
957-
Path baseExecutable = null;
931+
switch (name) {
932+
case "home":
958933
try {
959-
Path realPath = executablePath.toRealPath();
960-
if (!realPath.equals(executablePath.toAbsolutePath())) {
961-
baseExecutable = realPath;
934+
Path homeProperty = Paths.get(parts[1].trim());
935+
Path graalpyHome = homeProperty;
936+
/*
937+
* (tfel): According to PEP 405, the home key is the directory of the
938+
* Python executable from which this virtual environment was created,
939+
* that is, it usually ends with "/bin" on a Unix system. On Windows,
940+
* the base Python should be in the top-level directory or under
941+
* "\Scripts". To support running from Maven artifacts where we don't
942+
* have a working executable, we patched our shipped venv module to set
943+
* the home path without a "/bin" or "\\Scripts" suffix, so we
944+
* explicitly check for those two subfolder cases and otherwise assume
945+
* the home key is directly pointing to the Python home.
946+
*/
947+
if (graalpyHome.endsWith("bin") || graalpyHome.endsWith("Scripts")) {
948+
graalpyHome = graalpyHome.getParent();
962949
}
963-
} catch (IOException ex) {
964-
// Ignore
965-
}
966-
if (baseExecutable == null) {
967-
baseExecutable = homeProperty.resolve(executablePath.getFileName());
968-
}
969-
if (Files.exists(baseExecutable)) {
970-
contextBuilder.option("python.BaseExecutable", baseExecutable.toString());
950+
contextBuilder.option("python.PythonHome", graalpyHome.toString());
971951
/*
972-
* This is needed to support the legacy GraalVM layout where the
973-
* executable is a symlink into the 'languages' directory.
952+
* First try to resolve symlinked executables, since that may be more
953+
* accurate than assuming the executable in 'home'.
974954
*/
975-
contextBuilder.option("python.PythonHome", baseExecutable.getParent().getParent().toString());
955+
Path baseExecutable = null;
956+
try {
957+
Path realPath = executablePath.toRealPath();
958+
if (!realPath.equals(executablePath.toAbsolutePath())) {
959+
baseExecutable = realPath;
960+
}
961+
} catch (IOException ex) {
962+
// Ignore
963+
}
964+
if (baseExecutable == null) {
965+
baseExecutable = homeProperty.resolve(executablePath.getFileName());
966+
}
967+
if (Files.exists(baseExecutable)) {
968+
contextBuilder.option("python.BaseExecutable", baseExecutable.toString());
969+
/*
970+
* This is needed to support the legacy GraalVM layout where the
971+
* executable is a symlink into the 'languages' directory.
972+
*/
973+
contextBuilder.option("python.PythonHome", baseExecutable.getParent().getParent().toString());
974+
}
975+
} catch (NullPointerException | InvalidPathException ex) {
976+
// NullPointerException covers the possible null result of getParent()
977+
warn("Could not set PYTHONHOME according to the pyvenv.cfg file.");
976978
}
977-
} catch (NullPointerException | InvalidPathException ex) {
978-
// NullPointerException covers the possible null result of getParent()
979-
warn("Could not set PYTHONHOME according to the pyvenv.cfg file.");
980-
}
981-
break;
979+
break;
980+
case "venvlauncher_command":
981+
if (!hasContextOptionSetViaCommandLine("VenvlauncherCommand")) {
982+
contextBuilder.option("python.VenvlauncherCommand", parts[1].trim());
983+
}
984+
break;
985+
case "base-prefix":
986+
if (!hasContextOptionSetViaCommandLine("SysBasePrefix")) {
987+
contextBuilder.option("python.SysBasePrefix", parts[1].trim());
988+
}
989+
break;
990+
case "base-executable":
991+
if (!hasContextOptionSetViaCommandLine("BaseExecutable")) {
992+
contextBuilder.option("python.BaseExecutable", parts[1].trim());
993+
}
994+
break;
982995
}
983996
}
984997
} catch (IOException ex) {

‎graalpython/com.oracle.graal.python.test/src/tests/test_startup.py

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
2+
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3+
#
4+
# The Universal Permissive License (UPL), Version 1.0
5+
#
6+
# Subject to the condition set forth below, permission is hereby granted to any
7+
# person obtaining a copy of this software, associated documentation and/or
8+
# data (collectively the "Software"), free of charge and under any and all
9+
# copyright rights in the Software, and any and all patent rights owned or
10+
# freely licensable by each licensor hereunder covering either (i) the
11+
# unmodified Software as contributed to or provided by such licensor, or (ii)
12+
# the Larger Works (as defined below), to deal in both
13+
#
14+
# (a) the Software, and
15+
#
16+
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
17+
# one is included with the Software each a "Larger Work" to which the Software
18+
# is contributed by such licensors),
19+
#
20+
# without restriction, including without limitation the rights to copy, create
21+
# derivative works of, display, perform, and distribute the Software and make,
22+
# use, sell, offer for sale, import, export, have made, and have sold the
23+
# Software and the Larger Work(s), and to sublicense the foregoing rights on
24+
# either these or other terms.
25+
#
26+
# This license is subject to the following condition:
27+
#
28+
# The above copyright notice and either this complete permission notice or at a
29+
# minimum a reference to the UPL must be included in all copies or substantial
30+
# portions of the Software.
31+
#
32+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38+
# SOFTWARE.
39+
40+
import unittest
41+
import sys
42+
import re
43+
import subprocess
44+
import platform
45+
46+
# Both lists should remain as small as possible to avoid adding overhead to startup
47+
expected_nosite_startup_modules = [
48+
'_frozen_importlib',
49+
'_frozen_importlib_external',
50+
'builtins',
51+
'__graalpython__',
52+
'_weakref',
53+
'unicodedata',
54+
'_sre',
55+
'_sysconfig',
56+
'java',
57+
'pip_hook',
58+
] + (['_nt'] if platform.system() == 'Windows' else [])
59+
60+
expected_full_startup_modules = expected_nosite_startup_modules + [
61+
'_abc',
62+
'types',
63+
'_weakrefset',
64+
'_py_abc',
65+
'abc',
66+
'stat',
67+
'_collections_abc',
68+
'genericpath',
69+
*(['_winapi', 'ntpath'] if platform.system() == 'Windows' else ['posixpath']),
70+
'os',
71+
'_sitebuiltins',
72+
'_io',
73+
'io',
74+
'site',
75+
]
76+
77+
class StartupTests(unittest.TestCase):
78+
@unittest.skipUnless(sys.implementation.name == 'graalpy', "GraalPy-specific test")
79+
def test_startup_nosite(self):
80+
result = subprocess.check_output([sys.executable, '--log.level=FINE', '-S', '-v', '-c', 'print("Hello")'], stderr=subprocess.STDOUT, text=True)
81+
assert 'Hello' in result
82+
imports = re.findall("import '(\S+)'", result)
83+
self.assertEqual(expected_nosite_startup_modules, imports)
84+
85+
@unittest.skipUnless(sys.implementation.name == 'graalpy', "GraalPy-specific test")
86+
def test_startup_full(self):
87+
result = subprocess.check_output([sys.executable, '--log.level=FINE', '-s', '-v', '-c', 'print("Hello")'], stderr=subprocess.STDOUT, text=True)
88+
assert 'Hello' in result
89+
imports = re.findall("import '(\S+)'", result)
90+
self.assertEqual(expected_full_startup_modules, imports)

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,11 @@ private static TruffleString[] initializeCoreFiles() {
424424
toTruffleStringUncached("_sysconfig"),
425425
toTruffleStringUncached("java"),
426426
toTruffleStringUncached("pip_hook")));
427+
if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
428+
coreFiles = new ArrayList<>(coreFiles);
429+
coreFiles.add(toTruffleStringUncached("_nt"));
430+
}
431+
427432
// add service loader defined python file extensions
428433
if (!ImageInfo.inImageRuntimeCode()) {
429434
ServiceLoader<PythonBuiltins> providers = ServiceLoader.load(PythonBuiltins.class, Python3Core.class.getClassLoader());
@@ -1066,9 +1071,6 @@ public void run() {
10661071
}
10671072
}
10681073

1069-
// import polyglot decorators and special interop predefined behavior
1070-
loadFile(toTruffleStringUncached("_polyglot"), getContext().getCoreHomeOrFail());
1071-
10721074
initialized = true;
10731075
}
10741076
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,7 @@ public enum PythonBuiltinClassType implements TruffleObject {
336336
PStringIO("StringIO", "_io", Flags.PUBLIC_BASE_WDICT),
337337
PBytesIO("BytesIO", "_io", Flags.PUBLIC_BASE_WDICT),
338338
PBytesIOBuf("_BytesIOBuffer", "_io", Flags.PRIVATE_BASE_WODICT),
339+
PWindowsConsoleIO("_WindowsConsoleIO", "_io", Flags.PRIVATE_BASE_WODICT),
339340

340341
PStatResult("stat_result", "os", Flags.PUBLIC_DERIVED_WODICT, TUPLE_M_FLAGS),
341342
PStatvfsResult("statvfs_result", "os", Flags.PUBLIC_DERIVED_WODICT, TUPLE_M_FLAGS),

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PolyglotModuleBuiltins.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -113,6 +113,7 @@
113113
import com.oracle.graal.python.nodes.interop.InteropBehaviorMethod;
114114
import com.oracle.graal.python.nodes.interop.PForeignToPTypeNode;
115115
import com.oracle.graal.python.nodes.object.GetForeignObjectClassNode;
116+
import com.oracle.graal.python.nodes.statement.AbstractImportNode;
116117
import com.oracle.graal.python.nodes.truffle.PythonArithmeticTypes;
117118
import com.oracle.graal.python.nodes.util.CannotCastException;
118119
import com.oracle.graal.python.nodes.util.CastToJavaStringNode;
@@ -161,6 +162,7 @@ public final class PolyglotModuleBuiltins extends PythonBuiltins {
161162
private static final TruffleString T_MODIFIABLE = tsLiteral("modifiable");
162163
private static final TruffleString T_INVOKABLE = tsLiteral("invokable");
163164
private static final TruffleString T_INTERNAL = tsLiteral("internal");
165+
private static final TruffleString T_INTERNAL_POLYGLOT_MODULE = tsLiteral("_polyglot");
164166

165167
@Override
166168
protected List<com.oracle.truffle.api.dsl.NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
@@ -189,6 +191,9 @@ public void postInitialize(Python3Core core) {
189191
super.postInitialize(core);
190192

191193
GetForeignObjectClassNode.getUncached().defineSingleTraitClasses();
194+
195+
// import polyglot decorators which are defined in Python code
196+
AbstractImportNode.importModule(T_INTERNAL_POLYGLOT_MODULE);
192197
}
193198

194199
@Builtin(name = "import_value", minNumOfPositionalArgs = 1, parameterNames = {"name"})

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2018, 2024, Oracle and/or its affiliates.
2+
* Copyright (c) 2018, 2025, Oracle and/or its affiliates.
33
* Copyright (c) 2014, Regents of the University of California
44
*
55
* All rights reserved.
@@ -366,15 +366,6 @@ public void postInitialize(Python3Core core) {
366366
if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
367367
// XXX: Until we fix pip
368368
environ.setItem(toTruffleStringUncached("PIP_NO_CACHE_DIR"), toTruffleStringUncached("0"));
369-
// XXX: Until we have working winapi and winreg modules for MSVC discovery
370-
environ.setItem(toTruffleStringUncached("DISTUTILS_USE_SDK"), toTruffleStringUncached("1"));
371-
if (getenv.get("MSSdk") == null) {
372-
String sdkdir = getenv.get("WindowsSdkDir");
373-
if (sdkdir == null) {
374-
sdkdir = "unset";
375-
}
376-
environ.setItem(toTruffleStringUncached("MSSdk"), toTruffleStringUncached(sdkdir));
377-
}
378369
}
379370
PythonModule posix;
380371
if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {

0 commit comments

Comments
 (0)