fix running C extensions with the msvc native toolchain

Author: timfel

graalpython/com.oracle.graal.python.cext/include/pyconfig.h

@@ -51,7 +51,11 @@
 #  define MS_WINDOWS
 #  define Py_ENABLE_SHARED
 #  define HAVE_DECLSPEC_DLL
-#  pragma comment(lib, "python-native.lib")
+// This pragma is only understood by MSVC, not our LLVM toolchain, so it's only
+// relevant for code that is compiled without bitcode and will run only
+// natively. Since the pythonjni library contains all the trampolines to call
+// into the python-native.dll in this case, we must only depend on that.
+#  pragma comment(lib, "pythonjni.lib")
 # endif
 #endif
 

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -41,11 +41,6 @@
 #include "capi.h"
 #include <stdio.h>
 #include <time.h>
-#ifdef MS_WINDOWS
-#include "libloaderapi.h"
-#include "pathcch.h"
-#pragma comment(lib, "Pathcch.lib")
-#endif
 
 #define ASSERTIONS
 
@@ -1453,63 +1448,6 @@ PyAPI_FUNC(void) initialize_graal_capi(ptr_cache_t _pythonToNative, void_ptr_cac
     initialize_hashes();
     initialize_bufferprocs();
 
-#ifdef MS_WINDOWS
-    // when initializing the C API, the appropriate libraries (like
-    // python-native.dll or graalvm-llvm.dll) are loaded with their full paths.
-    // However, they are not automatically on the search path when any
-    // extension modules are loaded later, and Windows wants to resolve them
-    // again. So we get their runtime paths here and add those to the dll
-    // search path.
-    LPSTR lpMsgBuf;
-    wchar_t path[MAX_PATH];
-    char pathA[MAX_PATH];
-    HMODULE hm = NULL;
-    if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
-                           GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-                           (LPCWSTR) &initialize_graal_capi, &hm) == 0) {
-        int ret = GetLastError();
-        FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
-                       NULL, ret, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &lpMsgBuf, 0, NULL);
-        PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "finding python-native.dll handle failed, error = %s\n", lpMsgBuf);
-        LocalFree(lpMsgBuf);
-    } else {
-        if (GetModuleFileNameW(hm, (LPWSTR)path, sizeof(path)) == 0) {
-            int ret = GetLastError();
-            FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
-                           NULL, ret, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &lpMsgBuf, 0, NULL);
-            PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "finding python-native.dll path failed, error = %s\n", lpMsgBuf);
-            LocalFree(lpMsgBuf);
-        } else {
-            wcstombs(pathA, path, sizeof(pathA));
-            PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "Adding python-native.dll path '%s' to search path.\n", pathA);
-            PathCchRemoveFileSpec((PWSTR)path, sizeof(path));
-            AddDllDirectory((LPWSTR)path);
-        }
-    }
-    if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
-                           GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
-                           (LPCWSTR) &polyglot_from_string, &hm) == 0) {
-        int ret = GetLastError();
-        FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
-                       NULL, ret, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &lpMsgBuf, 0, NULL);
-        PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "finding graalvm-llvm.dll handle failed, error = %s\n", lpMsgBuf);
-        LocalFree(lpMsgBuf);
-    } else {
-        if (GetModuleFileNameW(hm, (LPWSTR)path, sizeof(path)) == 0) {
-            int ret = GetLastError();
-            FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
-                           NULL, ret, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &lpMsgBuf, 0, NULL);
-            PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "finding graalvm-llvm.dll path failed, error = %s\n", lpMsgBuf);
-            LocalFree(lpMsgBuf);
-        } else {
-            wcstombs(pathA, path, sizeof(pathA));
-            PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "Adding graalvm-llvm.dll path '%s' to search path.\n", pathA);
-            PathCchRemoveFileSpec((LPWSTR)path, sizeof(path));
-            AddDllDirectory((LPWSTR)path);
-        }
-    }
-#endif
-
     // TODO: initialize during cext initialization doesn't work at the moment
     // This is hardcoded the same way in capi_native.c
     Py_FileSystemDefaultEncoding = "utf-8"; // strdup(PyUnicode_AsUTF8(GraalPyTruffle_FileSystemDefaultEncoding()));

mx.graalpython/mx_graalpython.py

@@ -116,7 +116,21 @@ def wants_debug_build(flags=os.environ.get("CFLAGS", "")):
 
 
 if wants_debug_build():
-    mx_native.DefaultNativeProject.cflags = property(lambda self: self._cflags + ["-fPIC", "-ggdb3"])
+    mx_native.DefaultNativeProject._original_cflags = mx_native.DefaultNativeProject.cflags
+    mx_native.DefaultNativeProject.cflags = property(
+        lambda self: self._original_cflags + (["/Z7"] if WIN32 else ["-fPIC", "-ggdb3"])
+    )
+
+
+if WIN32:
+    # we need the .lib for pythonjni
+    original_DefaultNativeProject_getArchivableResults = mx_native.DefaultNativeProject.getArchivableResults
+    def getArchivableResultsWithLib(self, *args, **kwargs):
+        for result in original_DefaultNativeProject_getArchivableResults(self, *args, **kwargs):
+            if any(r.endswith("pythonjni.dll") for r in result):
+                yield tuple(r.replace(".dll", ".lib") for r in result)
+            yield result
+    mx_native.DefaultNativeProject.getArchivableResults = getArchivableResultsWithLib
 
 
 def _sibling(filename):
@@ -1147,6 +1161,9 @@ def graalpython_gate_runner(args, tasks):
     report = lambda: (not is_collecting_coverage()) and task
     nonZeroIsFatal = not is_collecting_coverage()
     if WIN32:
+        # Windows support is still experimental. we exclude a number of our
+        # unittests on Windows for now. If you add unittests and cannot get
+        # them to work on Windows, yet, add their files here.
         excluded_tests = [
             "test_zipimport.py", # sys.getwindowsversion
             "test_thread", # sys.getwindowsversion
@@ -1158,8 +1175,27 @@ def graalpython_gate_runner(args, tasks):
             "test_imports", # import posix
             "test_ctypes_callbacks.py", # ctypes error
             "test_code.py", # forward slash in path problem
-            "test_csv.py", # module 'os' has no attribute 'O_TEMPORARY'
-            "*/cpyext/*",
+            "*/cpyext/test_wiki.py",
+            "*/cpyext/test_unicode.py",
+            "*/cpyext/test_thread.py",
+            "*/cpyext/test_simple.py",
+            "*/cpyext/test_object.py",
+            "*/cpyext/test_modsupport.py",
+            "*/cpyext/test_mmap.py",
+            "*/cpyext/test_mixed_inheritance.py",
+            "*/cpyext/test_method.py",
+            "*/cpyext/test_memoryview.py",
+            "*/cpyext/test_member.py",
+            "*/cpyext/test_long.py",
+            "*/cpyext/test_functions.py",
+            "*/cpyext/test_float.py",
+            "*/cpyext/test_exceptionobject.py",
+            "*/cpyext/test_err.py",
+            "*/cpyext/test_descr.py",
+            "*/cpyext/test_datetime.py",
+            "*/cpyext/test_bytes.py",
+            "*/cpyext/test_bool.py",
+            "*/cpyext/test_abstract.py",
         ]
     else:
         excluded_tests = []

mx.graalpython/suite.py

@@ -980,15 +980,15 @@
                                 "extracted-dependency:graalpython:GRAALPYTHON_PYTHON_LIB",
                             ],
                             "./libs/": [
-                                "extracted-dependency:GRAALPYTHON_NATIVE_LIBS/python-native.lib",
+                                "extracted-dependency:GRAALPYTHON_NATIVE_LIBS/pythonjni.lib",
                             ],
                             "./lib-graalpython/": [
                                 "file:graalpython/lib-graalpython/*",
                                 {
                                     "source_type": "extracted-dependency",
                                     "dependency": "GRAALPYTHON_NATIVE_LIBS",
                                     "path": "*",
-                                    "exclude": ["python-native.lib"],
+                                    "exclude": ["pythonjni.lib"],
                                 },
                                 "file:graalpython/com.oracle.graal.python.cext/CEXT-WINDOWS-README.md",
                             ],