Skip to content

Commit 7330e39

Browse files
timfeltimfel
committed
Get ctypes to mostly load on Windows
1 parent 7b3f01d commit 7330e39

File tree

10 files changed

+138
-13
lines changed

10 files changed

+138
-13
lines changed

‎graalpython/com.oracle.graal.python.cext/src/capi.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1032,7 +1032,7 @@ PyAPI_FUNC(PyMemoryViewObject*) PyTruffle_AllocateMemoryView(PyMemoryViewObject*
10321032
}
10331033

10341034
#define PRIMITIVE_ARRAY_TO_NATIVE(__jtype__, __ctype__, __polyglot_type__, __element_cast__) \
1035-
void* PyTruffle_##__jtype__##ArrayToNative(const void* jarray, int64_t len) { \
1035+
PyAPI_FUNC(void*) PyTruffle_##__jtype__##ArrayToNative(const void* jarray, int64_t len) { \
10361036
int64_t i; \
10371037
int64_t size = len + 1; \
10381038
__ctype__* carr = (__ctype__*) malloc(size * sizeof(__ctype__)); \
@@ -1042,7 +1042,7 @@ PyAPI_FUNC(PyMemoryViewObject*) PyTruffle_AllocateMemoryView(PyMemoryViewObject*
10421042
} \
10431043
return polyglot_from_##__polyglot_type__##_array(carr, len); \
10441044
} \
1045-
void* PyTruffle_##__jtype__##ArrayRealloc(const void* array, int64_t len) { \
1045+
PyAPI_FUNC(void*) PyTruffle_##__jtype__##ArrayRealloc(const void* array, int64_t len) { \
10461046
int64_t size = len + 1; \
10471047
__ctype__* carr = (__ctype__*) realloc(array, size * sizeof(__ctype__)); \
10481048
carr[len] = (__ctype__)0; \
@@ -1055,22 +1055,22 @@ PRIMITIVE_ARRAY_TO_NATIVE(Long, int64_t, i64, polyglot_as_i64);
10551055
PRIMITIVE_ARRAY_TO_NATIVE(Double, double, double, polyglot_as_double);
10561056
PRIMITIVE_ARRAY_TO_NATIVE(Object, PyObjectPtr, PyObjectPtr, (PyObjectPtr));
10571057

1058-
void PyTruffle_PrimitiveArrayFree(void* array) {
1058+
PyAPI_FUNC(void) PyTruffle_PrimitiveArrayFree(void* array) {
10591059
free(array);
10601060
}
10611061

1062-
void PyTruffle_ObjectArrayFree(PyObject** array, int32_t size) {
1062+
PyAPI_FUNC(void) PyTruffle_ObjectArrayFree(PyObject** array, int32_t size) {
10631063
for (int i = 0; i < size; i++) {
10641064
Py_DECREF(array[i]);
10651065
}
10661066
free(array);
10671067
}
10681068

1069-
void PyTruffle_SetStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
1069+
PyAPI_FUNC(void) PyTruffle_SetStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
10701070
Py_XSETREF(ptr[index], newitem);
10711071
}
10721072

1073-
void PyTruffle_InitializeStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
1073+
PyAPI_FUNC(void) PyTruffle_InitializeStorageItem(PyObject** ptr, int32_t index, PyObject* newitem) {
10741074
ptr[index] = newitem;
10751075
}
10761076

‎graalpython/com.oracle.graal.python.cext/src/dictobject.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ int PyDict_DelItemString(PyObject *d, const char *key) {
110110
CALL_WITH_STRING(key, int, -1, GraalPyDict_DelItem, d, string);
111111
}
112112

113-
PyObject* _PyObject_GenericGetDict(PyObject* obj) {
113+
PyAPI_FUNC(PyObject*) _PyObject_GenericGetDict(PyObject* obj) {
114114
PyObject** dictptr = _PyObject_GetDictPtr(obj);
115115
if (dictptr == NULL) {
116116
return NULL;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/NtModuleBuiltins.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,12 +77,32 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
7777
public void initialize(Python3Core core) {
7878
super.initialize(core);
7979
if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
80+
addBuiltinConstant("_LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR", 0x100);
81+
addBuiltinConstant("_LOAD_LIBRARY_SEARCH_DEFAULT_DIRS", 0x1000);
8082
core.removeBuiltinModule(T_POSIX);
8183
} else {
8284
core.removeBuiltinModule(T_NT);
8385
}
8486
}
8587

88+
@Builtin(name = "_getfullpathname", minNumOfPositionalArgs = 1, parameterNames = {"path"})
89+
@ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"})
90+
@GenerateNodeFactory
91+
abstract static class GetfullpathnameNode extends PythonUnaryClinicBuiltinNode {
92+
@Specialization
93+
@TruffleBoundary
94+
Object getfullpathname(PosixPath path,
95+
@CachedLibrary("getPosixSupport()") PosixSupportLibrary posixLib) {
96+
// TODO should call win api
97+
return posixLib.getPathAsString(getPosixSupport(), path.value);
98+
}
99+
100+
@Override
101+
protected ArgumentClinicProvider getArgumentClinic() {
102+
return NtModuleBuiltinsClinicProviders.GetfullpathnameNodeClinicProviderGen.INSTANCE;
103+
}
104+
}
105+
86106
@Builtin(name = "_path_splitroot", minNumOfPositionalArgs = 1, parameterNames = {"path"})
87107
@ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"})
88108
@GenerateNodeFactory

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ctypes/CtypesModuleBuiltins.java

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@
9999
import com.oracle.graal.python.builtins.Python3Core;
100100
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
101101
import com.oracle.graal.python.builtins.PythonBuiltins;
102+
import com.oracle.graal.python.builtins.PythonOS;
102103
import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins.FsConverterNode;
103104
import com.oracle.graal.python.builtins.modules.SysModuleBuiltins.AuditNode;
104105
import com.oracle.graal.python.builtins.modules.ctypes.CFieldBuiltins.GetFuncNode;
@@ -122,6 +123,7 @@
122123
import com.oracle.graal.python.builtins.objects.cext.common.CArrayWrappers.CByteArrayWrapper;
123124
import com.oracle.graal.python.builtins.objects.cext.common.LoadCExtException.ApiInitException;
124125
import com.oracle.graal.python.builtins.objects.cext.common.LoadCExtException.ImportException;
126+
import com.oracle.graal.python.builtins.objects.cext.hpy.jni.GraalHPyJNIContext;
125127
import com.oracle.graal.python.builtins.objects.common.EconomicMapStorage;
126128
import com.oracle.graal.python.builtins.objects.common.HashingStorage;
127129
import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes.HashingStorageGetItem;
@@ -151,6 +153,9 @@
151153
import com.oracle.graal.python.nodes.SpecialMethodNames;
152154
import com.oracle.graal.python.nodes.StringLiterals;
153155
import com.oracle.graal.python.nodes.attributes.GetAttributeNode;
156+
import com.oracle.graal.python.nodes.attributes.ReadAttributeFromObjectNode;
157+
import com.oracle.graal.python.nodes.attributes.WriteAttributeToDynamicObjectNode;
158+
import com.oracle.graal.python.nodes.attributes.WriteAttributeToObjectNode;
154159
import com.oracle.graal.python.nodes.call.CallNode;
155160
import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
156161
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
@@ -212,6 +217,10 @@ public final class CtypesModuleBuiltins extends PythonBuiltins {
212217
private static final TruffleString T_DL_ERROR = tsLiteral("dlerror");
213218
private static final TruffleString T_DL_OPEN_ERROR = tsLiteral("dlopen() error");
214219

220+
private static final TruffleString T_WINDOWS_ERROR = tsLiteral("Windows Error");
221+
222+
private static final String J_DEFAULT_LIBRARY = PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? "msvcrt.dll" : J_EMPTY_STRING;
223+
215224
@Override
216225
protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
217226
return CtypesModuleBuiltinsFactory.getFactories();
@@ -238,6 +247,9 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
238247
public void initialize(Python3Core core) {
239248
super.initialize(core);
240249
addBuiltinConstant("_pointer_type_cache", core.factory().createDict());
250+
if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
251+
addBuiltinConstant("FUNCFLAG_STDCALL", FUNCFLAG_STDCALL);
252+
}
241253
addBuiltinConstant("FUNCFLAG_CDECL", FUNCFLAG_CDECL);
242254
addBuiltinConstant("FUNCFLAG_USE_ERRNO", FUNCFLAG_USE_ERRNO);
243255
addBuiltinConstant("FUNCFLAG_USE_LASTERROR", FUNCFLAG_USE_LASTERROR);
@@ -263,8 +275,15 @@ public void postInitialize(Python3Core core) {
263275

264276
DLHandler handle;
265277
if (context.getEnv().isNativeAccessAllowed()) {
266-
handle = DlOpenNode.loadNFILibrary(context, NFIBackend.NATIVE, J_EMPTY_STRING, rtldLocal);
278+
handle = DlOpenNode.loadNFILibrary(context, NFIBackend.NATIVE, J_DEFAULT_LIBRARY, rtldLocal);
267279
setCtypeNFIHelpers(this, context, handle);
280+
281+
if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
282+
PythonModule sysModule = context.getSysModule();
283+
Object loadLibraryMethod = ReadAttributeFromObjectNode.getUncached().execute(ctypesModule, toTruffleStringUncached("LoadLibrary"));
284+
Object pythonLib = CallNode.getUncached().execute(loadLibraryMethod, toTruffleStringUncached(GraalHPyJNIContext.getJNILibrary()), 0);
285+
WriteAttributeToDynamicObjectNode.getUncached().execute(sysModule, toTruffleStringUncached("dllhandle"), pythonLib);
286+
}
268287
} else {
269288
try {
270289
CApiContext cApiContext = CApiContext.ensureCapiWasLoaded(null, context, T_EMPTY_STRING, T_EMPTY_STRING);
@@ -456,6 +475,7 @@ static CtypesThreadState get(PythonContext context, PythonLanguage language) {
456475
}
457476
}
458477

478+
@Builtin(name = "get_last_error", maxNumOfPositionalArgs = 1, declaresExplicitSelf = true, os = PythonOS.PLATFORM_WIN32)
459479
@Builtin(name = "get_errno", maxNumOfPositionalArgs = 1, declaresExplicitSelf = true)
460480
@GenerateNodeFactory
461481
protected abstract static class GetErrnoNode extends PythonUnaryBuiltinNode {
@@ -473,6 +493,7 @@ static Object get_errno(PythonContext context, PythonLanguage language) {
473493
}
474494
}
475495

496+
@Builtin(name = "set_last_error", minNumOfPositionalArgs = 1, parameterNames = {"errno"}, os = PythonOS.PLATFORM_WIN32)
476497
@Builtin(name = "set_errno", minNumOfPositionalArgs = 1, parameterNames = {"errno"})
477498
@ArgumentClinic(name = "errno", conversion = ClinicConversion.Int)
478499
@GenerateNodeFactory
@@ -629,6 +650,7 @@ Object resize(CDataObject obj, int size,
629650
}
630651
}
631652

653+
@Builtin(name = "LoadLibrary", minNumOfPositionalArgs = 1, parameterNames = {"$self", "name", "mode"}, declaresExplicitSelf = true, os = PythonOS.PLATFORM_WIN32)
632654
@Builtin(name = "dlopen", minNumOfPositionalArgs = 1, parameterNames = {"$self", "name", "mode"}, declaresExplicitSelf = true)
633655
// TODO: 'name' might need to be processed using FSConverter.
634656
@ArgumentClinic(name = "name", conversion = ClinicConversion.TString, defaultValue = "T_EMPTY_STRING", useDefaultForNone = true)
@@ -1054,6 +1076,35 @@ Object call_function(VirtualFrame frame, Object pointerObj, PTuple arguments,
10541076
}
10551077
}
10561078

1079+
@Builtin(name = "FormatError", minNumOfPositionalArgs = 1, os = PythonOS.PLATFORM_WIN32)
1080+
@GenerateNodeFactory
1081+
protected abstract static class FormatErrorNode extends PythonUnaryBuiltinNode {
1082+
@Specialization
1083+
Object doit(Object errorCode) {
1084+
throw raise(NotImplementedError);
1085+
}
1086+
}
1087+
1088+
@Builtin(name = "_check_HRESULT", minNumOfPositionalArgs = 1, parameterNames = {"hresult"}, os = PythonOS.PLATFORM_WIN32)
1089+
@ArgumentClinic(name = "hresult", conversion = ClinicConversion.Int)
1090+
@GenerateNodeFactory
1091+
protected abstract static class CheckHresultNode extends PythonUnaryClinicBuiltinNode {
1092+
1093+
@Override
1094+
protected ArgumentClinicProvider getArgumentClinic() {
1095+
return CtypesModuleBuiltinsClinicProviders.CheckHresultNodeClinicProviderGen.INSTANCE;
1096+
}
1097+
1098+
@Specialization
1099+
Object check(VirtualFrame frame, int hresult) {
1100+
if (hresult >= 0) {
1101+
return hresult;
1102+
} else {
1103+
throw raiseOSError(frame, hresult, T_WINDOWS_ERROR);
1104+
}
1105+
}
1106+
}
1107+
10571108
protected static final class argument {
10581109
FFIType ffi_type;
10591110
StgDictObject stgDict;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ctypes/FFIType.java

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING;
4747
import static com.oracle.graal.python.util.PythonUtils.tsLiteral;
4848

49+
import com.oracle.graal.python.builtins.PythonOS;
4950
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
5051
import com.oracle.truffle.api.strings.TruffleString;
5152
import com.oracle.truffle.api.strings.TruffleStringBuilder;
@@ -308,8 +309,18 @@ enum FieldDesc {
308309
H('H', FieldSet.H_set, FieldGet.H_get, ffi_type_ushort, FieldSet.H_set_sw, FieldGet.H_get_sw), // unsigned short
309310
i('i', FieldSet.i_set, FieldGet.i_get, ffi_type_sint, FieldSet.i_set_sw, FieldGet.i_get_sw), // int
310311
I('I', FieldSet.I_set, FieldGet.I_get, ffi_type_uint, FieldSet.I_set_sw, FieldGet.I_get_sw), // unsigned int
311-
l('l', FieldSet.l_set, FieldGet.l_get, ffi_type_sint64, FieldSet.l_set_sw, FieldGet.l_get_sw), // long
312-
L('L', FieldSet.L_set, FieldGet.L_get, ffi_type_uint64, FieldSet.L_set_sw, FieldGet.L_get_sw), // unsigned long
312+
l('l',
313+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? FieldSet.i_set : FieldSet.l_set,
314+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? FieldGet.i_get : FieldGet.l_get,
315+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? ffi_type_sint32 : ffi_type_sint64,
316+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? FieldSet.i_set_sw : FieldSet.l_set_sw,
317+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? FieldGet.i_get_sw : FieldGet.l_get_sw), // long
318+
L('L',
319+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? FieldSet.I_set : FieldSet.L_set,
320+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? FieldGet.I_get : FieldGet.L_get,
321+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? ffi_type_uint32 : ffi_type_uint64,
322+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? FieldSet.I_set_sw : FieldSet.L_set_sw,
323+
PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 ? FieldGet.I_get_sw : FieldGet.L_get_sw), // unsigned long
313324
q('q', FieldSet.q_set, FieldGet.q_get, ffi_type_sint64, FieldSet.q_set_sw, FieldGet.q_get_sw), // long long
314325
Q('Q', FieldSet.Q_set, FieldGet.Q_get, ffi_type_uint64, FieldSet.Q_set_sw, FieldGet.Q_get_sw), // long long
315326
P('P', FieldSet.P_set, FieldGet.P_get, ffi_type_pointer), // Pointer

‎graalpython/lib-python/3/subprocess.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1713,13 +1713,28 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
17131713
if executable.startswith('"') and executable.endswith('"'):
17141714
executable = executable[1:-1]
17151715
if (len(args) == 1 and executable != args[0]) or shell:
1716+
if not shell:
1717+
warnings.warn(f"Running\n\t{args[0]!r} in a cmd shell", RuntimeWarning)
17161718
shell = False
17171719
comspec = os.environ.get("COMSPEC", "cmd.exe")
17181720
executable = comspec
17191721
if len(args) == 1:
17201722
args = [comspec, "/u", "/c", *args]
17211723
else:
17221724
args = [comspec, "/u", "/c", list2cmdline(args)]
1725+
for idx, arg in enumerate(args):
1726+
modified = False
1727+
if '\n' in args[idx]:
1728+
# newlines are not passed correctly. the common case
1729+
# where we have arguments like this is python code, so
1730+
# assume that and hope for the best
1731+
args[idx] = args[idx].strip().replace('\n', ';')
1732+
modified = True
1733+
if '"' in args[idx]:
1734+
args[idx] = list2cmdline(args[idx:idx + 1])
1735+
modified = True
1736+
if modified:
1737+
warnings.warn(f"Replacing\n\t{arg!r}\nwith\n\t{args[idx]!r}", RuntimeWarning)
17231738
# End Truffle change
17241739

17251740
if shell:

‎graalpython/lib-python/3/venv/__init__.py

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ def create_if_needed(d):
134134
# symlinks) is the Python home. It is not the case for us since $GRAALVM_HOME/bin/graalpy
135135
# is just a symlink to $GRAALVM_HOME/languages/python/bin/graalpy.
136136
context.python_dir = __graalpython__.home
137-
if not self.symlinks:
137+
if not self.symlinks and os.name != 'nt':
138138
# We cannot copy the launcher to another location, because it is either bash launcher
139139
# that locates the java executable relative to it, or it is a native application that
140140
# loads a shared library also using a relative path
@@ -180,6 +180,8 @@ def create_if_needed(d):
180180
' Actual location: "%s"',
181181
context.env_exe, real_env_exe)
182182
context.env_exec_cmd = real_env_exe
183+
# Truffle change: we install a .cmd script
184+
context.env_exec_cmd = os.path.splitext(context.env_exec_cmd)[0] + ".cmd"
183185
return context
184186

185187

@@ -271,6 +273,8 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
271273
"""
272274
Try symlinking a file, and if that fails, fall back to copying.
273275
"""
276+
# Truffle change: keep src argument around
277+
src_argument = src
274278
bad_src = os.path.lexists(src) and not os.path.exists(src)
275279
if self.symlinks and not bad_src and not os.path.islink(dst):
276280
try:
@@ -309,6 +313,13 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
309313
return
310314

311315
shutil.copyfile(src, dst)
316+
# Truffle change: setup our a launcher script
317+
if src == srcfn:
318+
with open(dst) as f:
319+
contents = f.read()
320+
with open(dst, "w") as f:
321+
f.write(contents.replace("<target>", src_argument))
322+
os.rename(dst, os.path.join(os.path.dirname(dst), basename + ".cmd"))
312323

313324
def setup_python(self, context):
314325
"""
@@ -350,12 +361,14 @@ def setup_python(self, context):
350361
os.path.normcase(f).startswith(('python', 'vcruntime'))
351362
]
352363
else:
353-
suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'}
364+
# Truffle change: we add 'graalpy' to the list
365+
suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe', 'graalpy.exe'}
354366
base_exe = os.path.basename(context.env_exe)
355367
suffixes.add(base_exe)
356368

357369
for suffix in suffixes:
358-
src = os.path.join(dirname, suffix)
370+
# Truffle change: we look in 'bin'
371+
src = os.path.join(dirname, 'bin', suffix)
359372
if os.path.lexists(src):
360373
copier(src, os.path.join(binpath, suffix))
361374

@@ -379,6 +392,9 @@ def _setup_pip(self, context):
379392
# intended for the global Python environment
380393
cmd = [context.env_exec_cmd, '-Im', 'ensurepip', '--upgrade',
381394
'--default-pip']
395+
# Truffle change for windows bug on graalpy.
396+
if sys.platform == 'win32':
397+
os.environ['_PIP_STANDALONE_CERT'] = "."
382398
try:
383399
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
384400
except subprocess.CalledProcessError as err:

‎graalpython/lib-python/3/venv/scripts/nt/graalpy.exe

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
@echo off
2+
3+
"<target>" "--python.Executable=%~0" %*
4+
exit /b %errorlevel%

‎graalpython/lib-python/3/venv/scripts/nt/python.exe

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
@echo off
2+
3+
"<target>" "--python.Executable=%~0" %*
4+
exit /b %errorlevel%

‎graalpython/lib-python/3/venv/scripts/nt/python3.exe

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
@echo off
2+
3+
"<target>" "--python.Executable=%~0" %*
4+
exit /b %errorlevel%

0 commit comments

Comments
 (0)