fix multiple issues around building C extension modules on windows

timfel · timfel · commit 2ce4932e3eb3 · 2023-07-07T11:44:04.000+02:00
diff --git a/graalpython/com.oracle.graal.python.cext/include/Python.h b/graalpython/com.oracle.graal.python.cext/include/Python.h
@@ -42,6 +42,8 @@
 #include <stdlib.h>
 #ifndef MS_WINDOWS
 #include <unistd.h>
+#else
+#include <winsock.h>
 #endif
 
 /* For size_t? */
diff --git a/graalpython/com.oracle.graal.python.cext/include/pyconfig.h b/graalpython/com.oracle.graal.python.cext/include/pyconfig.h
@@ -43,6 +43,18 @@
 
 #define GRAALVM_PYTHON 1
 
+// The graalpy build always sets MS_WINDOWS, so when this is not set, we are
+// dealing with an extension build. In that case, if we're on Windows, we need
+// to set the appropriate flags to link against our python C API dll.
+#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) && !defined(MS_WINDOWS)
+# ifdef _MSC_VER
+#  define MS_WINDOWS
+#  define Py_ENABLE_SHARED
+#  define HAVE_DECLSPEC_DLL
+#  pragma comment(lib, "python-native.lib")
+# endif
+#endif
+
 /* If Cython is involved, avoid accesses to internal structures. While we are
  * supporting this in many cases, it still involves overhead. */
 #define CYTHON_USE_TYPE_SLOTS 0
diff --git a/graalpython/com.oracle.graal.python.cext/src/capi.c b/graalpython/com.oracle.graal.python.cext/src/capi.c
@@ -41,6 +41,11 @@
 #include "capi.h"
 #include <stdio.h>
 #include <time.h>
+#ifdef MS_WINDOWS
+#include "libloaderapi.h"
+#include "pathcch.h"
+#pragma comment(lib, "Pathcch.lib")
+#endif
 
 #define ASSERTIONS
 
@@ -708,15 +713,15 @@ PyAPI_FUNC(Py_ssize_t) PyTruffle_ADDREF(intptr_t ptr, Py_ssize_t value) {
 #ifdef ASSERTIONS
 	if (obj->ob_refcnt & 0xFFFFFFFF00000000L) {
 		char buf[1024];
-		sprintf(buf, "suspicious refcnt value during managed adjustment for %p (%li %p + %li)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
+		sprintf(buf, "suspicious refcnt value during managed adjustment for %p (%lli %p + %lli)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
 		Py_FatalError(buf);
 	}
 	if ((obj->ob_refcnt + value) <= 0) {
 		char buf[1024];
-		sprintf(buf, "refcnt reached zero during managed adjustment for %p (%li %p + %li)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
+		sprintf(buf, "refcnt reached zero during managed adjustment for %p (%lli %p + %lli)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
 		Py_FatalError(buf);
 	}
-//	printf("refcnt value during managed adjustment for %p (%li %p + %li)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
+//	printf("refcnt value during managed adjustment for %p (%lli %p + %lli)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
 #endif // ASSERTIONS
 
 	return (obj->ob_refcnt += value);
@@ -728,15 +733,15 @@ PyAPI_FUNC(Py_ssize_t) PyTruffle_SUBREF(intptr_t ptr, Py_ssize_t value) {
 #ifdef ASSERTIONS
 	if (obj->ob_refcnt & 0xFFFFFFFF00000000L) {
 		char buf[1024];
-		sprintf(buf, "suspicious refcnt value during managed adjustment for %p (%li %p - %li)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
+		sprintf(buf, "suspicious refcnt value during managed adjustment for %p (%lli %p - %lli)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
 		Py_FatalError(buf);
 	}
 	if ((obj->ob_refcnt - value) < 0) {
 		char buf[1024];
-		sprintf(buf, "refcnt below zero during managed adjustment for %p (%li %p - %li)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
+		sprintf(buf, "refcnt below zero during managed adjustment for %p (%lli %p - %lli)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
 		Py_FatalError(buf);
 	}
-//	printf("refcnt value during managed adjustment for %p (%li %p - %li)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
+//	printf("refcnt value during managed adjustment for %p (%lli %p - %lli)\n", obj, obj->ob_refcnt, (void*) obj->ob_refcnt, value);
 #endif // ASSERTIONS
 
     Py_ssize_t new_value = ((obj->ob_refcnt) -= value);
@@ -1448,6 +1453,63 @@ PyAPI_FUNC(void) initialize_graal_capi(ptr_cache_t _pythonToNative, void_ptr_cac
     initialize_hashes();
     initialize_bufferprocs();
 
+#ifdef MS_WINDOWS
+    // when initializing the C API, the appropriate libraries (like
+    // python-native.dll or graalvm-llvm.dll) are loaded with their full paths.
+    // However, they are not automatically on the search path when any
+    // extension modules are loaded later, and Windows wants to resolve them
+    // again. So we get their runtime paths here and add those to the dll
+    // search path.
+    LPSTR lpMsgBuf;
+    wchar_t path[MAX_PATH];
+    char pathA[MAX_PATH];
+    HMODULE hm = NULL;
+    if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+                           GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                           (LPCWSTR) &initialize_graal_capi, &hm) == 0) {
+        int ret = GetLastError();
+        FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+                       NULL, ret, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &lpMsgBuf, 0, NULL);
+        PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "finding python-native.dll handle failed, error = %s\n", lpMsgBuf);
+        LocalFree(lpMsgBuf);
+    } else {
+        if (GetModuleFileNameW(hm, (LPWSTR)path, sizeof(path)) == 0) {
+            int ret = GetLastError();
+            FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+                           NULL, ret, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &lpMsgBuf, 0, NULL);
+            PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "finding python-native.dll path failed, error = %s\n", lpMsgBuf);
+            LocalFree(lpMsgBuf);
+        } else {
+            wcstombs(pathA, path, sizeof(pathA));
+            PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "Adding python-native.dll path '%s' to search path.\n", pathA);
+            PathCchRemoveFileSpec((PWSTR)path, sizeof(path));
+            AddDllDirectory((LPWSTR)path);
+        }
+    }
+    if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
+                           GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+                           (LPCWSTR) &polyglot_from_string, &hm) == 0) {
+        int ret = GetLastError();
+        FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+                       NULL, ret, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &lpMsgBuf, 0, NULL);
+        PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "finding graalvm-llvm.dll handle failed, error = %s\n", lpMsgBuf);
+        LocalFree(lpMsgBuf);
+    } else {
+        if (GetModuleFileNameW(hm, (LPWSTR)path, sizeof(path)) == 0) {
+            int ret = GetLastError();
+            FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+                           NULL, ret, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &lpMsgBuf, 0, NULL);
+            PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "finding graalvm-llvm.dll path failed, error = %s\n", lpMsgBuf);
+            LocalFree(lpMsgBuf);
+        } else {
+            wcstombs(pathA, path, sizeof(pathA));
+            PyTruffle_Log(PY_TRUFFLE_LOG_FINE, "Adding graalvm-llvm.dll path '%s' to search path.\n", pathA);
+            PathCchRemoveFileSpec((LPWSTR)path, sizeof(path));
+            AddDllDirectory((LPWSTR)path);
+        }
+    }
+#endif
+
     // TODO: initialize during cext initialization doesn't work at the moment
     // This is hardcoded the same way in capi_native.c
     Py_FileSystemDefaultEncoding = "utf-8"; // strdup(PyUnicode_AsUTF8(GraalPyTruffle_FileSystemDefaultEncoding()));
diff --git a/graalpython/com.oracle.graal.python.cext/src/capi.h b/graalpython/com.oracle.graal.python.cext/src/capi.h
@@ -944,7 +944,7 @@ PyAPI_DATA(ptr_cache_t) ptr_cache;
 PyAPI_DATA(cache_query_t) points_to_py_handle_space;
 PyAPI_FUNC(void) initialize_type_structure(PyTypeObject* structure, PyTypeObject* ptype, polyglot_typeid tid);
 
-void register_native_slots(PyTypeObject* managed_class, PyGetSetDef* getsets, PyMemberDef* members);
+PyAPI_FUNC(void) register_native_slots(PyTypeObject* managed_class, PyGetSetDef* getsets, PyMemberDef* members);
 
 extern ptr_cache_t pythonToNative;
 extern void_ptr_cache_t javaStringToTruffleString;
diff --git a/graalpython/com.oracle.graal.python.cext/src/modsupport_shared.c b/graalpython/com.oracle.graal.python.cext/src/modsupport_shared.c
@@ -42,6 +42,8 @@
 
 #include <stdio.h>
 
+// export the more functions, because we use them dynamically in contrast to cpython on windows
+PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...);
 
 #define FLAG_SIZE_T 1
 typedef double va_double;
diff --git a/graalpython/lib-graalpython/_sysconfig.py b/graalpython/lib-graalpython/_sysconfig.py
@@ -63,11 +63,14 @@ def _get_posix_vars():
     toolchain_cxx = get_toolchain('CXX')
     have_cxx = toolchain_cxx is not None
 
-    python_inc = os.path.join(
-        os.path.normpath(sys.base_prefix),
-        'include',
-        f'python{sys.version_info[0]}.{sys.version_info[1]}{sys.abiflags}'
-    )
+    if win32_native:
+        python_inc = os.path.join(os.path.normpath(sys.base_prefix), 'Include')
+    else:
+        python_inc = os.path.join(
+            os.path.normpath(sys.base_prefix),
+            'include',
+            f'python{sys.version_info[0]}.{sys.version_info[1]}{sys.abiflags}'
+        )
 
     fpic = "" if win32_native else "-fPIC"
 
@@ -110,7 +113,7 @@ def _get_posix_vars():
     g['RANLIB'] = get_toolchain('RANLIB')
     g['ARFLAGS'] = "rc"
     g['LD'] = get_toolchain('LD')
-    g['EXE'] = ""
+    g['EXE'] = ".exe" if win32_native else ""
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['VERSION'] = ".".join(sys.version.split(".")[:2])
     g['Py_HASH_ALGORITHM'] = 0 # does not give any specific info about the hashing algorithm
@@ -120,6 +123,7 @@ def _get_posix_vars():
     g['Py_DEBUG'] = 0
     g['Py_ENABLE_SHARED'] = 0
     g['LIBDIR'] = __graalpython__.capi_home
+    g['LIBDEST'] = __graalpython__.capi_home
     g['LDLIBRARY'] = 'libpython.' + so_abi + so_ext
     g['WITH_THREAD'] = 1
     return g
diff --git a/graalpython/lib-python/3/distutils/ccompiler.py b/graalpython/lib-python/3/distutils/ccompiler.py
@@ -930,8 +930,7 @@ def mkpath (self, name, mode=0o777):
 
     # OS name mappings
     ('posix', 'unix'),
-    # Truffle change ('nt', 'msvc'),
-    ('nt', 'unix'),
+    ('nt', 'msvc'),
     )
 
 def get_default_compiler(osname=None, platform=None):
diff --git a/graalpython/lib-python/3/distutils/command/build_ext.py b/graalpython/lib-python/3/distutils/command/build_ext.py
@@ -712,7 +712,7 @@ def get_libraries(self, ext):
         # pyconfig.h that MSVC groks.  The other Windows compilers all seem
         # to need it mentioned explicitly, though, so that's what we do.
         # Append '_d' to the python import library on debug builds.
-        if sys.platform == "win32" and False: # Truffle change: never go this path
+        if sys.platform == "win32":
             from distutils._msvccompiler import MSVCCompiler
             if not isinstance(self.compiler, MSVCCompiler):
                 template = "python%d%d"
diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py
@@ -555,6 +555,7 @@
                         },
                         "results" : [
                             "bin/<lib:python-native>",
+                            "bin/python-native.lib",
                             "bin/modules/_mmap<graalpy_ext:native>",
                             "bin/modules/_cpython_sre<graalpy_ext:native>",
                             "bin/modules/_cpython_unicodedata<graalpy_ext:native>",
@@ -978,9 +979,17 @@
                             "./Lib/": [
                                 "extracted-dependency:graalpython:GRAALPYTHON_PYTHON_LIB",
                             ],
+                            "./libs/": [
+                                "extracted-dependency:GRAALPYTHON_NATIVE_LIBS/python-native.lib",
+                            ],
                             "./lib-graalpython/": [
                                 "file:graalpython/lib-graalpython/*",
-                                "extracted-dependency:GRAALPYTHON_NATIVE_LIBS/*",
+                                {
+                                    "source_type": "extracted-dependency",
+                                    "dependency": "GRAALPYTHON_NATIVE_LIBS",
+                                    "path": "*",
+                                    "exclude": ["python-native.lib"],
+                                },
                                 "file:graalpython/com.oracle.graal.python.cext/CEXT-WINDOWS-README.md",
                             ],
                             "./lib-graalpython/modules/graalpy_virtualenv": [

Original file line number	Diff line number	Diff line change
`@@ -930,8 +930,7 @@ def mkpath (self, name, mode=0o777):`
`930`	`930`
`931`	`931`	`# OS name mappings`
`932`	`932`	`('posix', 'unix'),`
`933`		`- # Truffle change ('nt', 'msvc'),`
`934`		`- ('nt', 'unix'),`
	`933`	`+ ('nt', 'msvc'),`
`935`	`934`	`)`
`936`	`935`
`937`	`936`	`def get_default_compiler(osname=None, platform=None):`